From 8aa47eb6210a43355941bd1ccae39610cc958d81 Mon Sep 17 00:00:00 2001 From: Ketor Date: Wed, 23 Sep 2026 18:15:35 +0800 Subject: [PATCH 1/4] Fix charset-aware MySQL expressions and result encoding --- .../src/main/codegen/templates/Parser.jj | 2 +- .../io/dingodb/calcite/DingoSqlValidator.java | 2 + .../calcite/fun/DingoConcatFunction.java | 27 ++++-- .../calcite/fun/DingoOperatorTable.java | 62 +++++++++++-- .../calcite/meta/DingoColumnMetaData.java | 5 +- .../calcite/type/DingoSqlTypeFactory.java | 8 ++ .../dingodb/calcite/visitor/RexConverter.java | 12 +++ .../common/mysql/constant/ColumnStatus.java | 2 +- .../io/dingodb/driver/DingoDriverParser.java | 4 +- .../java/io/dingodb/driver/DingoFactory.java | 3 +- .../driver/DingoResultSetMetaData.java | 44 ++++++++++ .../mysql/command/MysqlResponseHandler.java | 34 +++++--- .../driver/mysql/netty/AsyncStreamReader.java | 55 ++++++++---- .../mysql/packet/MysqlPacketFactory.java | 87 ++++++++++++++++--- .../packet/PrepareResultSetRowPacket.java | 18 ++-- .../mysql/packet/ResultSetRowPacket.java | 36 +++----- .../packet/PrepareResultSetRowPacketTest.java | 36 ++++++++ .../io/dingodb/exec/fun/DingoFunFactory.java | 2 + .../java/io/dingodb/exec/fun/LengthFun.java | 30 ++++++- .../dingodb/exec/fun/mysql/CharBinaryFun.java | 3 +- .../exec/fun/mysql/CharCharsetFun.java | 20 +++-- .../io/dingodb/exec/fun/mysql/CharFun.java | 34 ++++++-- .../exec/fun/mysql/ConvertCharsetFun.java | 25 +++++- .../io/dingodb/exec/fun/mysql/HexFun.java | 31 ++++++- .../io/dingodb/test/dsl/MySqlRunningIT.java | 25 ++++++ .../test/QuerySimpleExpressionTest.java | 72 +++++++++++++++ 26 files changed, 566 insertions(+), 113 deletions(-) create mode 100644 dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSetMetaData.java diff --git a/dingo-calcite/src/main/codegen/templates/Parser.jj b/dingo-calcite/src/main/codegen/templates/Parser.jj index 6af1500eb9..80477c784f 100644 --- a/dingo-calcite/src/main/codegen/templates/Parser.jj +++ b/dingo-calcite/src/main/codegen/templates/Parser.jj @@ -6861,7 +6861,7 @@ SqlNode BuiltinFunctionCall() : ] { SqlIdentifier charFunName = new SqlIdentifier( - charUsingCharset ? (charUsingBinary ? "char_binary" : "char_charset") : "char", s.end(this)); + charUsingCharset ? (charUsingBinary ? "char_binary" : "char_charset") : "char_default", s.end(this)); SqlNode sqlNode = createCall(charFunName, s.end(this), SqlFunctionCategory.STRING, null, args); sqlNode.putAlias("aliasName", alias); return sqlNode; diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java index fe9d13e564..867af3e509 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java @@ -117,6 +117,8 @@ public static TypeCoercion createTypeCoercion(RelDataTypeFactory typeFactory, ) { super( SqlOperatorTables.chain( + // Override only CONCAT: Calcite's version drops its operand charset. + SqlOperatorTables.of(DingoOperatorTable.instance().concatFunction()), SqlStdOperatorTable.instance(), SqlLibraryOperatorTableFactory.INSTANCE .getOperatorTable(SqlLibrary.MYSQL), diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java b/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java index bccc8b8ebd..d5a4916e4b 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java @@ -19,15 +19,13 @@ import lombok.EqualsAndHashCode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.SqlFunctionCategory; -import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorBinding; import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlOperandTypeInference; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.checkerframework.checker.nullness.qual.NonNull; @@ -44,7 +42,7 @@ public DingoConcatFunction( ) { super( name, - returnTypeInference, + returnTypeInference == null ? null : binding -> inferCharsetReturnType(binding, returnTypeInference), operandTypeInference, operandTypeChecker, category @@ -63,9 +61,24 @@ public void validateCall( super.validateCall(call, validator, scope, operandScope); } - - public RelDataType inferReturnType(SqlOperatorBinding opBinding) { - return opBinding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + private static RelDataType inferCharsetReturnType( + SqlOperatorBinding binding, SqlReturnTypeInference returnTypeInference + ) { + RelDataType result = returnTypeInference.inferReturnType(binding); + RelDataType chosen = null; + for (RelDataType operand : binding.collectOperandTypes()) { + if (operand.getCharset() == null || operand.getCollation() == null) { + continue; + } + if (chosen == null || SqlCollation.getCoercibilityDyadicOperator( + chosen.getCollation(), operand.getCollation() + ) == operand.getCollation()) { + chosen = operand; + } + } + return chosen == null ? result : binding.getTypeFactory().createTypeWithCharsetAndCollation( + result, chosen.getCharset(), chosen.getCollation() + ); } } diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoOperatorTable.java b/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoOperatorTable.java index 64a824f7cd..b811ac8827 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoOperatorTable.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoOperatorTable.java @@ -80,6 +80,7 @@ import io.dingodb.expr.runtime.op.time.UnixTimestamp1FunFactory; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlCollation; import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlIdentifier; @@ -101,9 +102,12 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Locale; import static org.apache.calcite.sql.type.OperandTypes.family; import static org.apache.calcite.sql.type.SqlAppointReturnTypeInference.FLOAT; @@ -505,7 +509,7 @@ private void init() { LengthFun.NAME, ReturnTypes.INTEGER, InferTypes.VARCHAR_1024, - OperandTypes.STRING, + OperandTypes.or(OperandTypes.STRING, OperandTypes.BINARY), SqlFunctionCategory.NUMERIC ); registerFunction( @@ -552,7 +556,18 @@ private void init() { ); registerFunction( CharCharsetFun.NAME, - ReturnTypes.explicit(SqlTypeName.VARCHAR), + binding -> { + String name = binding.getOperandLiteralValue(binding.getOperandCount() - 1, String.class); + if (name == null) { + throw new IllegalArgumentException("CHAR USING requires a literal character set"); + } + Charset charset = CharCharsetFun.charset(name); + return binding.getTypeFactory().createTypeWithCharsetAndCollation( + binding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + charset, + charsetCollation(charset) + ); + }, InferTypes.ANY_NULLABLE, OperandTypes.VARIADIC, SqlFunctionCategory.STRING @@ -566,9 +581,29 @@ private void init() { ); registerFunction( ConvertCharsetFun.NAME, - ReturnTypes.explicit(SqlTypeName.VARCHAR), - DingoInferTypes.VARCHAR, - family(SqlTypeFamily.STRING, SqlTypeFamily.STRING), + binding -> { + String name = binding.getOperandLiteralValue(1, String.class); + if (name == null) { + throw new IllegalArgumentException("CONVERT USING requires a literal character set"); + } + Charset charset = CharCharsetFun.charset(name); + RelDataType inputType = binding.getOperandType(0); + SqlCollation inputCollation = inputType.getCollation(); + SqlCollation collation = + charset.equals(inputType.getCharset()) && inputCollation != null + && inputCollation.getCoercibility() != SqlCollation.Coercibility.COERCIBLE + ? inputCollation + : charsetCollation(charset); + RelDataType result = binding.getTypeFactory().createTypeWithCharsetAndCollation( + binding.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), charset, collation + ); + return binding.getTypeFactory().createTypeWithNullability(result, inputType.isNullable()); + }, + null, + OperandTypes.or( + family(SqlTypeFamily.STRING, SqlTypeFamily.STRING), + family(SqlTypeFamily.BINARY, SqlTypeFamily.STRING) + ), SqlFunctionCategory.STRING ); registerFunction( @@ -590,7 +625,9 @@ private void init() { ); registerFunction( ConvertTzFun.NAME, - ReturnTypes.TIMESTAMP, + binding -> binding.getTypeFactory().createTypeWithNullability( + binding.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP), true + ), null, OperandTypes.or( family(SqlTypeFamily.TIMESTAMP, SqlTypeFamily.STRING, SqlTypeFamily.STRING), @@ -615,6 +652,15 @@ private void init() { ); } + private static SqlCollation charsetCollation(Charset charset) { + // Calcite's SqlCollation whitelists SQL charset names. Keep the actual + // MySQL codec on RelDataType; use a supported collation for latin1/ASCII. + Charset collationCharset = charset.equals(StandardCharsets.US_ASCII) + || charset.equals(Charset.forName("windows-1252")) + ? StandardCharsets.ISO_8859_1 : charset; + return new SqlCollation(SqlCollation.Coercibility.IMPLICIT, Locale.ROOT, collationCharset, "primary"); + } + public void registerFunction( @NonNull String name, @Nullable SqlReturnTypeInference returnTypeInference, @@ -647,6 +693,10 @@ public void registerConcatFunction( )); } + public SqlFunction concatFunction() { + return funMap.get(ConcatFun.NAME.toUpperCase(Locale.ROOT)).iterator().next(); + } + @Override public void lookupOperatorOverloads( SqlIdentifier opName, diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/meta/DingoColumnMetaData.java b/dingo-calcite/src/main/java/io/dingodb/calcite/meta/DingoColumnMetaData.java index d7491a31fa..667c74ea2a 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/meta/DingoColumnMetaData.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/meta/DingoColumnMetaData.java @@ -20,6 +20,7 @@ public class DingoColumnMetaData extends ColumnMetaData { public final boolean hidden; + public final String charsetName; public DingoColumnMetaData( int ordinal, @@ -32,11 +33,13 @@ public DingoColumnMetaData( int displaySize, String label, String columnName, String schemaName, int precision, int scale, String tableName, String catalogName, AvaticaType type, boolean readOnly, boolean writable, boolean definitelyWritable, String columnClassName, - boolean hidden + boolean hidden, + String charsetName ) { super(ordinal, autoIncrement, caseSensitive, searchable, currency, nullable, signed, displaySize, label, columnName, schemaName, precision, scale, tableName, catalogName, type, readOnly, writable, definitelyWritable, columnClassName); this.hidden = hidden; + this.charsetName = charsetName; } } diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java b/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java index 54c222f473..8bb198b501 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java @@ -393,6 +393,14 @@ else if (resultType.getSqlTypeName().getName().equalsIgnoreCase("CHAR") collation = collation2; } } + if (collation0 != null && charset1 != null && charset2 != null + && !charset1.equals(charset2)) { + if (collation0.equals(collation1)) { + charset = charset1; + } else if (collation0.equals(collation2)) { + charset = charset2; + } + } if (charset != null) { resultType = createTypeWithCharsetAndCollation( diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/RexConverter.java b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/RexConverter.java index c89eeb5c82..0f11b8486f 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/RexConverter.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/visitor/RexConverter.java @@ -21,6 +21,8 @@ import io.dingodb.common.util.Optional; import io.dingodb.exec.expr.SqlExprCompileContext; import io.dingodb.exec.fun.DingoFunFactory; +import io.dingodb.exec.fun.LengthFun; +import io.dingodb.exec.fun.mysql.HexFun; import io.dingodb.expr.runtime.ExprConfig; import io.dingodb.expr.runtime.compiler.CastingFactory; import io.dingodb.expr.runtime.expr.Expr; @@ -58,6 +60,8 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import java.nio.charset.Charset; + public final class RexConverter implements RexVisitor<@NonNull Expr> { private static final RexConverter INSTANCE = new RexConverter(); @@ -512,6 +516,14 @@ private RexConverter() { } break; case 1: + RexNode operand = call.getOperands().get(0); + Charset charset = operand.getType().getCharset(); + if (charset != null && (funName.equalsIgnoreCase(LengthFun.NAME) + || funName.equalsIgnoreCase(HexFun.NAME))) { + BinaryOp charsetOp = funName.equalsIgnoreCase(LengthFun.NAME) + ? LengthFun.CHARSET_INSTANCE : HexFun.CHARSET_INSTANCE; + return Exprs.op(charsetOp, operand.accept(this), Exprs.val(charset.name())); + } UnaryOp unaryOp = funFactory.getUnaryFun(funName); if (unaryOp != null) { return Exprs.op( diff --git a/dingo-common/src/main/java/io/dingodb/common/mysql/constant/ColumnStatus.java b/dingo-common/src/main/java/io/dingodb/common/mysql/constant/ColumnStatus.java index 28b4c3fbb2..9fd76e1ee0 100644 --- a/dingo-common/src/main/java/io/dingodb/common/mysql/constant/ColumnStatus.java +++ b/dingo-common/src/main/java/io/dingodb/common/mysql/constant/ColumnStatus.java @@ -17,7 +17,7 @@ package io.dingodb.common.mysql.constant; public class ColumnStatus { - public static final int COLUMN_NULLABLE = 0X0001; + public static final int COLUMN_NOT_NULL = 0X0001; public static final int COLUMN_PRIMARY = 0X0002; diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java index ad92492737..6c9af06336 100644 --- a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java +++ b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoDriverParser.java @@ -282,7 +282,9 @@ private static ColumnMetaData metaData( false, false, avaticaType.id == SqlType.FLOAT.id ? "java.lang.Float" : avaticaType.columnClassName(), - hidden + hidden, + (type.getSqlTypeName() == SqlTypeName.CHAR || type.getSqlTypeName() == SqlTypeName.VARCHAR) + && type.getCharset() != null ? type.getCharset().name() : null ); } diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoFactory.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoFactory.java index 136b65464b..5da94d1ff7 100644 --- a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoFactory.java +++ b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoFactory.java @@ -22,7 +22,6 @@ import org.apache.calcite.avatica.AvaticaDatabaseMetaData; import org.apache.calcite.avatica.AvaticaFactory; import org.apache.calcite.avatica.AvaticaResultSet; -import org.apache.calcite.avatica.AvaticaResultSetMetaData; import org.apache.calcite.avatica.AvaticaSpecificDatabaseMetaData; import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.Meta; @@ -127,7 +126,7 @@ public ResultSetMetaData newResultSetMetaData( AvaticaStatement statement, Meta.Signature signature ) { - return new AvaticaResultSetMetaData(statement, null, signature); + return new DingoResultSetMetaData(statement, signature); } // Must inherit, the constructor of the base class is protected. diff --git a/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSetMetaData.java b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSetMetaData.java new file mode 100644 index 0000000000..07edf6c425 --- /dev/null +++ b/dingo-driver/host/src/main/java/io/dingodb/driver/DingoResultSetMetaData.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 DataCanvas + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.dingodb.driver; + +import io.dingodb.calcite.meta.DingoColumnMetaData; +import org.apache.calcite.avatica.AvaticaResultSetMetaData; +import org.apache.calcite.avatica.AvaticaStatement; +import org.apache.calcite.avatica.ColumnMetaData; +import org.apache.calcite.avatica.Meta; + +import java.sql.SQLException; +import java.util.List; + +public class DingoResultSetMetaData extends AvaticaResultSetMetaData { + private final List columns; + + public DingoResultSetMetaData(AvaticaStatement statement, Meta.Signature signature) { + super(statement, null, signature); + this.columns = signature.columns; + } + + /** Returns null when the column has no expression-specific charset. */ + public String getColumnCharsetName(int column) throws SQLException { + if (column < 1 || column > columns.size()) { + throw new SQLException("Invalid column index: " + column); + } + ColumnMetaData metadata = columns.get(column - 1); + return metadata instanceof DingoColumnMetaData ? ((DingoColumnMetaData) metadata).charsetName : null; + } +} diff --git a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/command/MysqlResponseHandler.java b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/command/MysqlResponseHandler.java index c8e800e611..d906aacd18 100644 --- a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/command/MysqlResponseHandler.java +++ b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/command/MysqlResponseHandler.java @@ -45,6 +45,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.lang.reflect.Array; +import java.nio.charset.CharsetEncoder; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -54,6 +55,7 @@ import java.util.concurrent.atomic.AtomicLong; import static io.dingodb.calcite.executor.SetOptionExecutor.CONNECTION_CHARSET; +import static io.dingodb.calcite.executor.SetOptionExecutor.RESULTS_CHARSET; import static io.dingodb.common.mysql.constant.ServerStatus.SERVER_MORE_RESULTS_EXISTS; import static io.dingodb.common.mysql.error.ErrorCode.ErrRecursiveCteErr; import static io.dingodb.common.util.Utils.getCharacterSet; @@ -68,6 +70,14 @@ public final class MysqlResponseHandler { private MysqlResponseHandler() { } + private static String resultCharset(MysqlConnection mysqlConnection) throws SQLException { + String charset = mysqlConnection.getConnection().getClientInfo(RESULTS_CHARSET); + if (charset == null || charset.isEmpty()) { + charset = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); + } + return getCharacterSet(charset == null || charset.isEmpty() ? "utf8mb4" : charset); + } + public static void responseShowField(ResultSet resultSet, AtomicLong packetId, MysqlConnection mysqlConnection) { @@ -75,7 +85,7 @@ public static void responseShowField(ResultSet resultSet, // 2. ok packet String connCharSet = null; try { - connCharSet = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); + connCharSet = resultCharset(mysqlConnection); List columnPackets = factory.getColumnPackets(packetId, resultSet, true, connCharSet); ByteBuf buffer = ByteBufAllocator.DEFAULT.buffer(); for (ColumnPacket columnPacket : columnPackets) { @@ -110,7 +120,7 @@ public static boolean responseResultSet(ResultSet resultSet, String connCharSet = null; boolean stream = false; try { - connCharSet = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); + connCharSet = resultCharset(mysqlConnection); ByteBuf buffer = ByteBufAllocator.DEFAULT.buffer(); ResultSetMetaData metaData = resultSet.getMetaData(); ColumnsNumberPacket columnsNumberPacket = new ColumnsNumberPacket(); @@ -178,6 +188,8 @@ private static Boolean handlerRowPacket( ) throws SQLException { ResultSetMetaData metaData = resultSet.getMetaData(); int columnCount = metaData.getColumnCount(); + CharsetEncoder[] columnEncoders = MysqlPacketFactory.getColumnEncoders( + metaData, resultCharset(mysqlConnection)); String typeName; boolean stream = false; AtomicLong cnt = new AtomicLong(0); @@ -185,9 +197,6 @@ private static Boolean handlerRowPacket( ResultSetRowPacket resultSetRowPacket = new ResultSetRowPacket(); long nextId = packetId.getAndIncrement(); resultSetRowPacket.packetId = (byte) nextId; - String characterSet = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); - characterSet = getCharacterSet(characterSet); - resultSetRowPacket.setCharacterSet(characterSet); for (int i = 1; i <= columnCount; i ++) { Object val = resultSet.getObject(i); typeName = metaData.getColumnTypeName(i); @@ -207,7 +216,7 @@ private static Boolean handlerRowPacket( } } - resultSetRowPacket.addColumnValue(val); + resultSetRowPacket.addColumnValue(val, columnEncoders[i - 1]); } cnt.incrementAndGet(); resultSetRowPacket.write(buffer); @@ -216,7 +225,7 @@ private static Boolean handlerRowPacket( + " cnt:{}, packetId:{}", cnt.get(), packetId.get()); AsyncStreamReader streamReader = new AsyncStreamReader( - resultSet, packetId, mysqlConnection, serverStatus, statement + resultSet, packetId, mysqlConnection, serverStatus, statement, columnEncoders ); Executors.submit("streamReader", streamReader); mysqlConnection.writeAndFlushByStream(buffer); @@ -254,13 +263,14 @@ private static void handlerPrepareRowPacket(ResultSet resultSet, MysqlConnection mysqlConnection, ByteBuf buffer, int columnCount) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + CharsetEncoder[] columnEncoders = MysqlPacketFactory.getColumnEncoders( + metaData, resultCharset(mysqlConnection)); while (resultSet.next()) { PrepareResultSetRowPacket resultSetRowPacket = new PrepareResultSetRowPacket(); - String characterSet = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); - characterSet = getCharacterSet(characterSet); - resultSetRowPacket.setCharacterSet(characterSet); resultSetRowPacket.packetId = (byte) packetId.getAndIncrement(); - resultSetRowPacket.setMetaData(resultSet.getMetaData()); + resultSetRowPacket.setMetaData(metaData); + resultSetRowPacket.setColumnEncoders(columnEncoders); for (int i = 1; i <= columnCount; i ++) { resultSetRowPacket.addColumnValue(resultSet.getObject(i), mysqlConnection); } @@ -445,7 +455,7 @@ public static void responsePrepareExecute(ResultSet resultSet, & ExtendedClientCapabilities.CLIENT_DEPRECATE_EOF) != 0; String connCharSet = null; try { - connCharSet = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); + connCharSet = resultCharset(mysqlConnection); ByteBuf buffer = ByteBufAllocator.DEFAULT.buffer(); ResultSetMetaData metaData = resultSet.getMetaData(); ColumnsNumberPacket columnsNumberPacket = new ColumnsNumberPacket(); diff --git a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/netty/AsyncStreamReader.java b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/netty/AsyncStreamReader.java index 2aa1eb0a15..7df56e7f5e 100644 --- a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/netty/AsyncStreamReader.java +++ b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/netty/AsyncStreamReader.java @@ -18,24 +18,27 @@ import io.dingodb.common.log.LogUtils; import io.dingodb.common.mysql.ExtendedClientCapabilities; +import io.dingodb.common.mysql.MysqlServer; import io.dingodb.common.mysql.scope.ScopeVariables; import io.dingodb.driver.mysql.MysqlConnection; import io.dingodb.driver.mysql.packet.EOFPacket; +import io.dingodb.driver.mysql.packet.ERRPacket; import io.dingodb.driver.mysql.packet.MysqlPacketFactory; import io.dingodb.driver.mysql.packet.OKPacket; import io.dingodb.driver.mysql.packet.ResultSetRowPacket; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.CharsetEncoder; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.concurrent.atomic.AtomicLong; -import static io.dingodb.calcite.executor.SetOptionExecutor.CONNECTION_CHARSET; -import static io.dingodb.common.util.Utils.getCharacterSet; import static io.dingodb.driver.mysql.command.MysqlResponseHandler.getArrayObject; @Slf4j @@ -45,25 +48,29 @@ public class AsyncStreamReader implements Runnable { MysqlConnection mysqlConnection; int initServerStatus; Statement statement; + private final CharsetEncoder[] columnEncoders; public AsyncStreamReader( ResultSet resultSet, AtomicLong packetId, MysqlConnection mysqlConnection, int initServerStatus, - Statement statement + Statement statement, + CharsetEncoder[] columnEncoders ) { this.resultSet = resultSet; this.packetId = packetId; this.mysqlConnection = mysqlConnection; this.initServerStatus = initServerStatus; this.statement = statement; + this.columnEncoders = columnEncoders; } @Override public void run() { int columnCount; ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + SQLException failure = null; try { ResultSetMetaData metaData = resultSet.getMetaData(); columnCount = metaData.getColumnCount(); @@ -73,9 +80,6 @@ public void run() { ResultSetRowPacket resultSetRowPacket = new ResultSetRowPacket(); long nextId = packetId.getAndIncrement(); resultSetRowPacket.packetId = (byte) nextId; - String characterSet = mysqlConnection.getConnection().getClientInfo(CONNECTION_CHARSET); - characterSet = getCharacterSet(characterSet); - resultSetRowPacket.setCharacterSet(characterSet); for (int i = 1; i <= columnCount; i++) { Object val = resultSet.getObject(i); typeName = metaData.getColumnTypeName(i); @@ -94,7 +98,7 @@ public void run() { val = "0x" + Long.toHexString((long) val); } } - resultSetRowPacket.addColumnValue(val); + resultSetRowPacket.addColumnValue(val, columnEncoders[i - 1]); } cnt.incrementAndGet(); resultSetRowPacket.write(outputStream); @@ -113,17 +117,36 @@ public void run() { } } catch (SQLException e) { LogUtils.error(log, "stream reader failed, reason:{}", e.getMessage(), e); + failure = e; } finally { - boolean deprecateEof = (mysqlConnection.authPacket.extendClientFlags - & ExtendedClientCapabilities.CLIENT_DEPRECATE_EOF) != 0; - if (deprecateEof) { - OKPacket okEofPacket = MysqlPacketFactory.getInstance().getOkEofPacket( - 0, packetId, initServerStatus - ); - okEofPacket.write(outputStream); + if (failure != null) { + ERRPacket errorPacket = new ERRPacket(); + errorPacket.packetId = (byte) packetId.getAndIncrement(); + errorPacket.capabilities = MysqlServer.getServerCapabilities(); + errorPacket.errorCode = failure.getErrorCode(); + errorPacket.sqlState = failure.getSQLState(); + errorPacket.errorMessage = failure.getMessage(); + ByteBuf errorBuffer = ByteBufAllocator.DEFAULT.buffer(); + try { + errorPacket.write(errorBuffer); + byte[] bytes = new byte[errorBuffer.readableBytes()]; + errorBuffer.readBytes(bytes); + outputStream.writeBytes(bytes); + } finally { + errorBuffer.release(); + } } else { - EOFPacket eofPacket = MysqlPacketFactory.getEofPacket(packetId, initServerStatus); - eofPacket.write(outputStream); + boolean deprecateEof = (mysqlConnection.authPacket.extendClientFlags + & ExtendedClientCapabilities.CLIENT_DEPRECATE_EOF) != 0; + if (deprecateEof) { + OKPacket okEofPacket = MysqlPacketFactory.getInstance().getOkEofPacket( + 0, packetId, initServerStatus + ); + okEofPacket.write(outputStream); + } else { + EOFPacket eofPacket = MysqlPacketFactory.getEofPacket(packetId, initServerStatus); + eofPacket.write(outputStream); + } } mysqlConnection.writeAndFlush(outputStream.toByteArray()); try { diff --git a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/MysqlPacketFactory.java b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/MysqlPacketFactory.java index d60b04a608..1493d6899d 100644 --- a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/MysqlPacketFactory.java +++ b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/MysqlPacketFactory.java @@ -20,15 +20,22 @@ import io.dingodb.common.mysql.constant.ColumnStatus; import io.dingodb.common.mysql.constant.ColumnType; import io.dingodb.driver.mysql.NativeConstants; +import io.dingodb.driver.mysql.MysqlType; import org.checkerframework.checker.nullness.qual.NonNull; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.SQLWarning; import java.util.ArrayList; -import io.dingodb.driver.mysql.MysqlType; import java.util.List; import java.util.concurrent.atomic.AtomicLong; @@ -36,6 +43,9 @@ public class MysqlPacketFactory { private static final short BINARY_CHARSET = 63; + private static final Charset LATIN1_CHARSET = Charset.forName("windows-1252"); + private static final short LATIN1_COLLATION = 8; + private static final short ASCII_COLLATION = 11; private static MysqlPacketFactory instance = null; public static MysqlPacketFactory getInstance() { @@ -108,11 +118,8 @@ private OKPacket newOkPacket(int affected, public short getColumnFlags(ResultSetMetaData metaData, int column) { try { - int columnFlags = 0; - // 0 not null 1 nullable - int isNullable = metaData.isNullable(column); - columnFlags |= isNullable; - + int columnFlags = metaData.isNullable(column) == ResultSetMetaData.columnNoNulls + ? ColumnStatus.COLUMN_NOT_NULL : 0; String columnTypeName = metaData.getColumnTypeName(column); return (short) combineColumnFlags(columnFlags, columnTypeName); } catch (Exception e) { @@ -122,10 +129,8 @@ public short getColumnFlags(ResultSetMetaData metaData, int column) { public static short getColumnFlags(ResultSet resultSet) { try { - int columnFlags = 0; - // 0 not null 1 nullable - int isNullable = resultSet.getInt("NULLABLE"); - columnFlags |= isNullable; + int columnFlags = resultSet.getInt("NULLABLE") == ResultSetMetaData.columnNoNulls + ? ColumnStatus.COLUMN_NOT_NULL : 0; String columnTypeName = resultSet.getString("TYPE_NAME"); return (short) combineColumnFlags(columnFlags, columnTypeName); @@ -197,6 +202,7 @@ public List getColumnPackets(AtomicLong packetId, List columns = new ArrayList<>(); String catalog = "def"; if (showFields) { + Charset resultsCharset = textCharset(io.dingodb.common.util.Utils.getCharacterSet(columnNmCharset)); while (resultSet.next()) { String dataType = resultSet.getString("DATA_TYPE"); String tableName = resultSet.getString("TABLE_NAME"); @@ -208,13 +214,13 @@ public List getColumnPackets(AtomicLong packetId, tableName, columnName, columnName, - "VARBINARY".equals(dataType) ? BINARY_CHARSET : MysqlPacket.charsetNumber, + getColumnCharsetNumber(dataType, resultsCharset), resultSet.getInt("COLUMN_SIZE"), getColumnType(dataType), getColumnFlags(resultSet), MysqlPacket.decimals, (byte) packetId.getAndIncrement(), - columnNmCharset + resultsCharset.name() ); columns.add(columnPacket); } @@ -240,6 +246,7 @@ public void addColumnPacketFromMeta(AtomicLong packetId, ResultSetMetaData metaD String schema = metaData.getSchemaName(1); table = table != null ? table : ""; schema = schema != null ? schema : ""; + Charset resultCharset = textCharset(io.dingodb.common.util.Utils.getCharacterSet(columnNmCharset)); for (int i = 1; i <= columnCount; i++) { String columnLabel = metaData.getColumnLabel(i); @@ -260,16 +267,68 @@ public void addColumnPacketFromMeta(AtomicLong packetId, ResultSetMetaData metaD table, table, columnLabel, columnName, - "VARBINARY".equals(columnTypeName) ? BINARY_CHARSET : MysqlPacket.charsetNumber, + getColumnCharsetNumber(columnTypeName, resultCharset), metaData.getColumnDisplaySize(i), columnType, getColumnFlags(metaData, i), MysqlPacket.decimals, - (byte) packetId.getAndIncrement(), columnNmCharset); + (byte) packetId.getAndIncrement(), resultCharset.name()); columns.add(columnPacket); } } + private static Charset textCharset(String name) throws SQLException { + if ("UTF-8".equalsIgnoreCase(name) || "UTF8".equalsIgnoreCase(name) + || "utf8mb4".equalsIgnoreCase(name) || "utf8mb3".equalsIgnoreCase(name)) { + return StandardCharsets.UTF_8; + } + if ("windows-1252".equalsIgnoreCase(name) || "Cp1252".equalsIgnoreCase(name) + || "latin1".equalsIgnoreCase(name)) { + return LATIN1_CHARSET; + } + if ("US-ASCII".equalsIgnoreCase(name) || "ASCII".equalsIgnoreCase(name)) { + return StandardCharsets.US_ASCII; + } + throw new SQLException("No MySQL collation for result charset: " + name); + } + + private static short getColumnCharsetNumber(String typeName, Charset resultsCharset) { + if ("VARBINARY".equals(typeName)) { + return BINARY_CHARSET; + } + if ("VARCHAR".equals(typeName) || "CHAR".equals(typeName)) { + if (resultsCharset == LATIN1_CHARSET) { + return LATIN1_COLLATION; + } + return resultsCharset == StandardCharsets.US_ASCII ? ASCII_COLLATION : MysqlPacket.charsetNumber; + } + return MysqlPacket.charsetNumber; + } + + /** Result text uses the connection's character_set_results, regardless of expression charset. */ + public static CharsetEncoder[] getColumnEncoders(ResultSetMetaData metaData, String connectionCharset) + throws SQLException { + Charset charset = textCharset(io.dingodb.common.util.Utils.getCharacterSet(connectionCharset)); + CharsetEncoder[] encoders = new CharsetEncoder[metaData.getColumnCount()]; + for (int i = 0; i < encoders.length; i++) { + encoders[i] = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + } + return encoders; + } + + public static byte[] encodeText(String text, CharsetEncoder encoder) throws SQLException { + try { + ByteBuffer encoded = encoder.encode(CharBuffer.wrap(text)); + byte[] bytes = new byte[encoded.remaining()]; + encoded.get(bytes); + return bytes; + } catch (CharacterCodingException e) { + throw new SQLException("Text cannot be encoded as " + encoder.charset().name(), e); + } + } + public ColumnPacket getParamColumnPacket(AtomicLong packetId, String columnNmCharset) { return getColumnPacket("def", "", "", "", "?", "", diff --git a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacket.java b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacket.java index 70d321e559..6e34be33c6 100644 --- a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacket.java +++ b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacket.java @@ -21,7 +21,7 @@ import io.netty.buffer.ByteBuf; import lombok.Setter; -import java.io.UnsupportedEncodingException; +import java.nio.charset.CharsetEncoder; import java.sql.Array; import java.sql.Date; import java.sql.ResultSetMetaData; @@ -37,7 +37,7 @@ public class PrepareResultSetRowPacket extends MysqlPacket { List values = new ArrayList<>(); @Setter - private String characterSet; + private CharsetEncoder[] columnEncoders; @Setter private ResultSetMetaData metaData; @@ -84,13 +84,7 @@ public int calcPacketSize() { case "CHAR": case "ARRAY": case "MULTISET": - byte[] v; - try { - v = val.toString().getBytes(characterSet); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - values.set(i - 1, v); + byte[] v = (byte[]) val; totalSize += BufferUtil.getLength(v); break; case "VARBINARY": @@ -189,6 +183,12 @@ public void addColumnValue(Object val, MysqlConnection connection) throws SQLExc if (val instanceof Array) { val = getArrayObject(connection, val); } + int column = values.size() + 1; + String typeName = metaData.getColumnTypeName(column); + if (val != null && ("VARCHAR".equals(typeName) || "CHAR".equals(typeName) + || "ARRAY".equals(typeName) || "MULTISET".equals(typeName))) { + val = MysqlPacketFactory.encodeText(val.toString(), columnEncoders[column - 1]); + } values.add(val); } diff --git a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/ResultSetRowPacket.java b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/ResultSetRowPacket.java index 1cb1933233..1b5ac8cf23 100644 --- a/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/ResultSetRowPacket.java +++ b/dingo-driver/mysql-service/src/main/java/io/dingodb/driver/mysql/packet/ResultSetRowPacket.java @@ -20,21 +20,20 @@ import io.dingodb.driver.mysql.util.BufferUtil; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import lombok.Setter; import java.io.ByteArrayOutputStream; -import java.io.UnsupportedEncodingException; import java.math.BigDecimal; +import java.nio.charset.CharsetEncoder; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; public class ResultSetRowPacket extends MysqlPacket { - @Setter - private String characterSet; - public List values = new ArrayList<>(); private static final byte NULL_MARK = (byte) 251; + private static final byte[] ZERO = {'0'}; + private static final byte[] ONE = {'1'}; public long columnCount; @@ -44,11 +43,7 @@ public void read(byte[] data) { //packetLength = message.readUB3(); packetId = message.read(); for (int i = 0; i < columnCount; i++) { - try { - values.add(message.readStringWithLength().getBytes(characterSet)); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } + values.add(message.readBytesWithLength()); } } @@ -141,7 +136,7 @@ public String toString() { + "}\n"; } - public void addColumnValue(Object val) { + public void addColumnValue(Object val, CharsetEncoder encoder) throws SQLException { if (val == null) { values.add(null); } else { @@ -149,18 +144,13 @@ public void addColumnValue(Object val) { values.add((byte[]) val); return; } - try { - if (val instanceof Boolean) { - // MySQL wire protocol never uses "true"/"false" text for - // boolean expressions; it always sends 1/0 like TINYINT. - values.add(((Boolean) val) ? "1".getBytes(characterSet) : "0".getBytes(characterSet)); - } else if (val instanceof BigDecimal) { - values.add(((BigDecimal) val).toPlainString().getBytes(characterSet)); - } else { - values.add(val.toString().getBytes(characterSet)); - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); + if (val instanceof Boolean) { + // MySQL wire protocol sends boolean expressions as integer 0/1. + values.add((Boolean) val ? ONE : ZERO); + } else if (val instanceof BigDecimal) { + values.add(MysqlPacketFactory.encodeText(((BigDecimal) val).toPlainString(), encoder)); + } else { + values.add(MysqlPacketFactory.encodeText(val.toString(), encoder)); } } } diff --git a/dingo-driver/mysql-service/src/test/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacketTest.java b/dingo-driver/mysql-service/src/test/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacketTest.java index 835570b2b3..98f96e105f 100644 --- a/dingo-driver/mysql-service/src/test/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacketTest.java +++ b/dingo-driver/mysql-service/src/test/java/io/dingodb/driver/mysql/packet/PrepareResultSetRowPacketTest.java @@ -16,6 +16,7 @@ package io.dingodb.driver.mysql.packet; +import io.dingodb.common.mysql.constant.ColumnStatus; import io.dingodb.driver.mysql.MysqlType; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; @@ -23,6 +24,7 @@ import javax.sql.rowset.RowSetMetaDataImpl; import java.sql.SQLException; +import java.sql.ResultSetMetaData; import java.sql.Types; import java.util.ArrayList; import java.util.List; @@ -40,11 +42,13 @@ void computedBooleanUsesAdvertisedBigintWidthWithoutShiftingNextColumn() throws metaData.setColumnName(1, "comparison"); metaData.setColumnLabel(1, "comparison"); metaData.setTableName(1, ""); + metaData.setNullable(1, ResultSetMetaData.columnNullable); metaData.setColumnTypeName(2, "BOOLEAN"); metaData.setColumnType(2, Types.BOOLEAN); metaData.setColumnName(2, "declared_boolean"); metaData.setColumnLabel(2, "declared_boolean"); metaData.setTableName(2, "sample"); + metaData.setNullable(2, ResultSetMetaData.columnNoNulls); metaData.setColumnTypeName(3, "INTEGER"); metaData.setColumnType(3, Types.INTEGER); metaData.setColumnName(3, "following"); @@ -57,6 +61,9 @@ void computedBooleanUsesAdvertisedBigintWidthWithoutShiftingNextColumn() throws ); assertThat(columns.get(0).type).isEqualTo((byte) MysqlType.FIELD_TYPE_LONGLONG); assertThat(columns.get(1).type).isEqualTo((byte) MysqlType.FIELD_TYPE_TINY); + assertThat(columns.get(0).flags & ColumnStatus.COLUMN_NOT_NULL).isZero(); + assertThat(columns.get(1).flags & ColumnStatus.COLUMN_NOT_NULL) + .isEqualTo(ColumnStatus.COLUMN_NOT_NULL); PrepareResultSetRowPacket packet = new PrepareResultSetRowPacket(); packet.setMetaData(metaData); @@ -78,4 +85,33 @@ void computedBooleanUsesAdvertisedBigintWidthWithoutShiftingNextColumn() throws bytes.release(); } } + + @Test + void preparedTextRowUsesNegotiatedUtf8Encoding() throws SQLException { + RowSetMetaDataImpl metaData = new RowSetMetaDataImpl(); + metaData.setColumnCount(1); + metaData.setColumnTypeName(1, "VARCHAR"); + metaData.setColumnType(1, Types.VARCHAR); + + PrepareResultSetRowPacket packet = new PrepareResultSetRowPacket(); + packet.setMetaData(metaData); + packet.setColumnEncoders(MysqlPacketFactory.getColumnEncoders(metaData, "UTF-8")); + packet.addColumnValue("€", null); + + ByteBuf bytes = Unpooled.buffer(); + try { + packet.write(bytes); + assertThat(bytes.readUnsignedMediumLE()).isEqualTo(6); + bytes.readUnsignedByte(); // packet sequence ID + bytes.readUnsignedByte(); // binary row marker + bytes.readUnsignedByte(); // null bitmap + assertThat(bytes.readUnsignedByte()).isEqualTo((short) 3); + assertThat(bytes.readUnsignedByte()).isEqualTo((short) 0xE2); + assertThat(bytes.readUnsignedByte()).isEqualTo((short) 0x82); + assertThat(bytes.readUnsignedByte()).isEqualTo((short) 0xAC); + assertThat(bytes.isReadable()).isFalse(); + } finally { + bytes.release(); + } + } } diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/DingoFunFactory.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/DingoFunFactory.java index bdd8ac4d87..5be61cdc72 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/DingoFunFactory.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/DingoFunFactory.java @@ -88,6 +88,7 @@ private DingoFunFactory() { registerUnaryFun(LastValFun.NAME, LastValFun.INSTANCE); registerBinaryFun(SetValFun.NAME, SetValFun.INSTANCE); registerUnaryFun(LengthFun.NAME, LengthFun.INSTANCE); + registerBinaryFun(LengthFun.CHARSET_NAME, LengthFun.CHARSET_INSTANCE); registerTertiaryFun(IfFun.NAME, IfFun.INSTANCE); registerBinaryFun(DateAddFun.NAME, DateAddFun.INSTANCE); registerBinaryFun(DateSubFun.NAME, DateSubFun.INSTANCE); @@ -95,6 +96,7 @@ private DingoFunFactory() { registerUnaryFun(ValuesFun.NAME, ValuesFun.INSTANCE); registerUnaryFun(UnHexFun.NAME, UnHexFun.INSTANCE); registerUnaryFun(HexFun.NAME, HexFun.INSTANCE); + registerBinaryFun(HexFun.CHARSET_NAME, HexFun.CHARSET_INSTANCE); registerBinaryFun(DaySubFun.NAME, DaySubFun.INSTANCE); registerUnaryFun(GetDateFun.NAME, GetDateFun.INSTANCE); registerVariadicFun(ConcatWsFun.NAME, ConcatWsFun.INSTANCE); diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/LengthFun.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/LengthFun.java index f99e146091..cc6bf6dc7d 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/LengthFun.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/LengthFun.java @@ -17,13 +17,18 @@ package io.dingodb.exec.fun; import io.dingodb.expr.runtime.ExprConfig; +import io.dingodb.expr.runtime.op.BinaryOp; import io.dingodb.expr.runtime.op.UnaryOp; import org.checkerframework.checker.nullness.qual.NonNull; import java.io.Serial; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; public class LengthFun extends UnaryOp { public static final LengthFun INSTANCE = new LengthFun(); + public static final String CHARSET_NAME = "length_charset"; + public static final BinaryOp CHARSET_INSTANCE = new CharsetLengthFun(); public static final String NAME = "length"; @Serial @@ -31,10 +36,31 @@ public class LengthFun extends UnaryOp { @Override public Object evalValue(Object value, ExprConfig config) { + return length(value, StandardCharsets.UTF_8); + } + + private static Object length(Object value, Charset charset) { if (value == null) { return null; - } else { - return value.toString().getBytes().length; + } + if (value instanceof byte[]) { + return ((byte[]) value).length; + } + return value.toString().getBytes(charset).length; + } + + private static final class CharsetLengthFun extends BinaryOp { + @Serial + private static final long serialVersionUID = 6817736234195783327L; + + @Override + public Object evalValue(Object value, Object charsetName, ExprConfig config) { + return value == null ? null : length(value, Charset.forName(charsetName.toString())); + } + + @Override + public @NonNull String getName() { + return CHARSET_NAME; } } diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharBinaryFun.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharBinaryFun.java index fafa1745ff..d9ff0d086d 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharBinaryFun.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharBinaryFun.java @@ -20,7 +20,6 @@ import io.dingodb.expr.common.type.Types; import io.dingodb.expr.runtime.ExprConfig; import io.dingodb.expr.runtime.op.OpKey; -import io.dingodb.expr.runtime.op.OpKeys; import io.dingodb.expr.runtime.op.VariadicOp; import org.checkerframework.checker.nullness.qual.NonNull; @@ -37,7 +36,7 @@ public Object evalValue(Object @NonNull [] values, ExprConfig config) { @Override public OpKey keyOf(@NonNull Type @NonNull ... types) { - return OpKeys.ALL_STRING.keyOf(types); + return Types.ANY; } @Override diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharCharsetFun.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharCharsetFun.java index 739589e74a..3f5d8ed5fd 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharCharsetFun.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharCharsetFun.java @@ -20,11 +20,13 @@ import io.dingodb.expr.common.type.Types; import io.dingodb.expr.runtime.ExprConfig; import io.dingodb.expr.runtime.op.OpKey; -import io.dingodb.expr.runtime.op.OpKeys; import io.dingodb.expr.runtime.op.VariadicOp; import org.checkerframework.checker.nullness.qual.NonNull; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; /** @@ -46,19 +48,27 @@ public Object evalValue(Object @NonNull [] values, ExprConfig config) { } String charsetName = values[values.length - 1].toString(); byte[] bytes = CharFun.buildBytes(values, values.length - 1); - return new String(bytes, charset(charsetName)); + try { + return charset(charsetName).newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Cannot decode CHAR using charset " + charsetName, e); + } } /** Map MySQL charset names to an available Java codec; reject unknown names. */ - static Charset charset(@NonNull String name) { + public static Charset charset(@NonNull String name) { String normalized = name.trim().toLowerCase(java.util.Locale.ROOT); switch (normalized) { case "utf8": case "utf8mb4": return StandardCharsets.UTF_8; case "binary": - case "latin1": return StandardCharsets.ISO_8859_1; + case "latin1": + return Charset.forName("windows-1252"); case "ascii": return StandardCharsets.US_ASCII; default: @@ -68,7 +78,7 @@ static Charset charset(@NonNull String name) { @Override public OpKey keyOf(@NonNull Type @NonNull ... types) { - return OpKeys.ALL_STRING.keyOf(types); + return Types.ANY; } @Override diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharFun.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharFun.java index d46ca5e884..c533ba4723 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharFun.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/CharFun.java @@ -20,13 +20,15 @@ import io.dingodb.expr.common.type.Types; import io.dingodb.expr.runtime.ExprConfig; import io.dingodb.expr.runtime.op.OpKey; -import io.dingodb.expr.runtime.op.OpKeys; import io.dingodb.expr.runtime.op.VariadicOp; import org.checkerframework.checker.nullness.qual.NonNull; -/** MySQL CHAR() constructs a binary string from integer byte values. */ +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** MySQL CHAR() constructs a binary string from numeric byte values. */ public class CharFun extends VariadicOp { - public static final String NAME = "char"; + public static final String NAME = "char_default"; @SuppressWarnings("serial") private static final long serialVersionUID = -5487904391104558091L; @@ -37,7 +39,7 @@ public Object evalValue(Object @NonNull [] values, ExprConfig config) { return buildBytes(values, values.length); } - /** Each integer contributes its nonzero leading bytes, most significant first. */ + /** Each numeric argument contributes its nonzero leading bytes, most significant first. */ static byte[] buildBytes(Object @NonNull [] values, int count) { byte[] bytes = new byte[Math.max(count, 4)]; int length = 0; @@ -46,7 +48,7 @@ static byte[] buildBytes(Object @NonNull [] values, int count) { byte[] valueBytes; if (value instanceof Number || value instanceof String) { long number = value instanceof Number - ? ((Number) value).longValue() : Long.parseLong(((String) value).trim()); + ? numericValue((Number) value) : numericStringValue((String) value); int width = 1; for (long remaining = number >>> 8; remaining != 0; remaining >>>= 8) { width++; @@ -85,9 +87,29 @@ static byte[] buildBytes(Object @NonNull [] values, int count) { return exact; } + private static long numericValue(Number value) { + if (value instanceof BigDecimal) { + return ((BigDecimal) value).setScale(0, RoundingMode.HALF_UP).longValue(); + } + if (value instanceof Double || value instanceof Float) { + return new BigDecimal(value.toString()).setScale(0, RoundingMode.HALF_UP).longValue(); + } + return value.longValue(); + } + + private static long numericStringValue(String value) { + String text = value.trim(); + if (text.indexOf('.') >= 0 || text.indexOf('e') >= 0 || text.indexOf('E') >= 0) { + // MySQL truncates fractional numeric strings rather than rounding numeric values. + // Reject malformed strings; the expression evaluator has no SQL warning channel. + return new BigDecimal(text).setScale(0, RoundingMode.DOWN).longValueExact(); + } + return Long.parseLong(text); + } + @Override public OpKey keyOf(@NonNull Type @NonNull ... types) { - return OpKeys.ALL_STRING.keyOf(types); + return Types.ANY; } @Override diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/ConvertCharsetFun.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/ConvertCharsetFun.java index e78ff2d7af..0edaed783a 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/ConvertCharsetFun.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/ConvertCharsetFun.java @@ -22,7 +22,11 @@ import io.dingodb.expr.runtime.op.BinaryOp; import org.checkerframework.checker.nullness.qual.NonNull; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.Charset; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; /** * MySQL CONVERT(expr USING charset) for character sets. The binary charset is @@ -41,11 +45,24 @@ public Object evalValue(Object value0, Object value1, ExprConfig config) { return null; } Charset charset = CharCharsetFun.charset(value1.toString()); - if (value0 instanceof byte[]) { - return new String((byte[]) value0, charset); + try { + if (value0 instanceof byte[]) { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap((byte[]) value0)).toString(); + } + ByteBuffer bytes = charset.newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .encode(CharBuffer.wrap(value0.toString())); + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(bytes).toString(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("Cannot convert value using charset " + charset.name(), e); } - String text = value0.toString(); - return new String(text.getBytes(charset), charset); } @Override diff --git a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/HexFun.java b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/HexFun.java index b467a3418a..a97961ee29 100644 --- a/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/HexFun.java +++ b/dingo-exec/src/main/java/io/dingodb/exec/fun/mysql/HexFun.java @@ -19,14 +19,19 @@ import io.dingodb.expr.common.type.Type; import io.dingodb.expr.common.type.Types; import io.dingodb.expr.runtime.ExprConfig; +import io.dingodb.expr.runtime.op.BinaryOp; import io.dingodb.expr.runtime.op.UnaryOp; import org.checkerframework.checker.nullness.qual.NonNull; import java.io.Serial; import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; public class HexFun extends UnaryOp { public static final String NAME = "hex"; + public static final String CHARSET_NAME = "hex_charset"; + public static final BinaryOp CHARSET_INSTANCE = new CharsetHexFun(); @Serial private static final long serialVersionUID = -2489040936115125799L; private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray(); @@ -35,6 +40,10 @@ public class HexFun extends UnaryOp { @Override protected Object evalNonNullValue(@NonNull Object value, ExprConfig config) { + return hex(value, StandardCharsets.UTF_8); + } + + private static Object hex(@NonNull Object value, Charset charset) { if (value instanceof byte[]) { return toHex((byte[]) value); } @@ -51,7 +60,27 @@ protected Object evalNonNullValue(@NonNull Object value, ExprConfig config) { } else if (value instanceof Long) { return Long.toHexString((Long) value).toUpperCase(); } else { - return toHex(value.toString().getBytes()).toUpperCase(); + return toHex(value.toString().getBytes(charset)); + } + } + + private static final class CharsetHexFun extends BinaryOp { + @Serial + private static final long serialVersionUID = 4669681289107118463L; + + @Override + public Object evalValue(Object value, Object charsetName, ExprConfig config) { + return value == null ? null : hex(value, Charset.forName(charsetName.toString())); + } + + @Override + public @NonNull String getName() { + return CHARSET_NAME; + } + + @Override + public Type getType() { + return Types.STRING; } } diff --git a/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java b/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java index 38e1c2aeac..874594d674 100644 --- a/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java +++ b/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java @@ -19,13 +19,20 @@ import io.dingodb.test.dsl.builder.SqlTestCaseYamlBuilder; import io.dingodb.test.dsl.run.SqlTestRunner; import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Types; import java.util.Properties; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + // Before run this, you must set up your cluster. public class MySqlRunningIT extends SqlTestRunner { public Connection getConnection() throws Exception { @@ -36,6 +43,24 @@ public Connection getConnection() throws Exception { return DriverManager.getConnection(url, properties); } + @Test + public void convertLatin1PreservesTextAndByteSemantics() throws Exception { + try (Connection connection = getConnection(); + Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery( + "select convert('é' using latin1), " + + "length(convert('é' using latin1)), hex(convert('é' using latin1)), " + + "convert('€' using latin1)" + )) { + assertEquals(Types.VARCHAR, result.getMetaData().getColumnType(1)); + assertTrue(result.next()); + assertEquals("é", result.getString(1)); + assertEquals(1, result.getInt(2)); + assertEquals("E9", result.getString(3)); + assertEquals("€", result.getString(4)); + } + } + @TestFactory public Stream testAggregation() { return getTests(SqlTestCaseYamlBuilder.of("cases/aggregation.yml")); diff --git a/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java b/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java index d106f0e05e..983e364cdc 100644 --- a/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java +++ b/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java @@ -16,6 +16,7 @@ package io.dingodb.test; +import io.dingodb.driver.DingoResultSetMetaData; import io.dingodb.test.asserts.Assert; import io.dingodb.test.cases.RexCasesJUnit5; import io.dingodb.test.dsl.run.exec.SqlExecContext; @@ -29,9 +30,14 @@ import org.junit.jupiter.params.provider.ArgumentsSource; import java.sql.SQLException; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; import java.sql.Timestamp; +import java.sql.Types; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class QuerySimpleExpressionTest { @@ -74,6 +80,60 @@ public void mysqlBinaryAndQuoteKeepLiteralBytes() throws SQLException { assertThat(context.querySingleValue("select hex(quote(null))")).isEqualTo("4E554C4C"); } + @Test + public void mysqlCharCoercesDecimalArguments() throws SQLException { + assertThat(context.querySingleValue("select hex(char(77.6))")).isEqualTo("4E"); + assertThat(context.querySingleValue("select hex(char('77.3'))")).isEqualTo("4D"); + assertThat(context.querySingleValue("select hex(char(77.6 using binary))")).isEqualTo("4E"); + assertThat(context.querySingleValue("select char(77.6 using utf8mb4)")).isEqualTo("N"); + } + + @Test + public void convertCharsetRetainsEncodedByteSemantics() throws SQLException { + try (Statement statement = context.getConnection().createStatement(); + ResultSet result = statement.executeQuery("select length(convert('é' using latin1))")) { + assertThat(result.next()).isTrue(); + assertThat(result.getInt(1)).isEqualTo(1); + } + assertThat(context.querySingleValue("select hex(convert('€' using latin1))")).isEqualTo("80"); + assertThat(context.querySingleValue("select char(128 using latin1)")).isEqualTo("€"); + assertThat(context.querySingleValue("select length(char(128 using latin1))")).isEqualTo(1); + assertThat(context.querySingleValue("select hex(char(128 using latin1))")).isEqualTo("80"); + assertThat(context.querySingleValue("select hex(convert('é' using latin1))")).isEqualTo("E9"); + try (Statement statement = context.getConnection().createStatement(); + ResultSet result = statement.executeQuery("select concat(convert('é' using latin1), 'x')")) { + assertThat(((DingoResultSetMetaData) result.getMetaData()).getColumnCharsetName(1)) + .isEqualTo("windows-1252"); + assertThat(result.next()).isTrue(); + assertThat(result.getString(1)).isEqualTo("éx"); + } + assertThat(context.querySingleValue( + "select length(concat(convert('é' using latin1), 'x'))" + )).isEqualTo(2); + assertThat(context.querySingleValue( + "select hex(concat(convert('é' using latin1), 'x'))" + )).isEqualTo("E978"); + try (Statement statement = context.getConnection().createStatement(); + ResultSet result = statement.executeQuery("select convert('é' using latin1)")) { + assertThat(result.getMetaData().getColumnType(1)).isEqualTo(Types.VARCHAR); + assertThat(result.getMetaData()).isInstanceOf(DingoResultSetMetaData.class); + assertThat(((DingoResultSetMetaData) result.getMetaData()).getColumnCharsetName(1)) + .isEqualTo("windows-1252"); + assertThat(result.next()).isTrue(); + assertThat(result.getString(1)).isEqualTo("é"); + } + } + + @Test + public void convertCharsetRejectsUnrepresentableText() { + assertThatThrownBy(() -> { + try (Statement statement = context.getConnection().createStatement(); + ResultSet result = statement.executeQuery("select convert('é' using ascii)")) { + result.next(); + } + }).hasMessageContaining("Cannot convert value"); + } + @Test public void convertTzHandlesFractionalSecondsAndInvalidZones() throws SQLException { assertThat(context.querySingleValue( @@ -84,6 +144,18 @@ public void convertTzHandlesFractionalSecondsAndInvalidZones() throws SQLExcepti )).isNull(); } + @Test + public void convertTzInvalidZoneAdvertisesNullableResult() throws SQLException { + try (Statement statement = context.getConnection().createStatement(); + ResultSet result = statement.executeQuery( + "select convert_tz('2024-01-01 00:00:00', 'invalid-zone', 'UTC')" + )) { + assertThat(result.getMetaData().isNullable(1)).isEqualTo(ResultSetMetaData.columnNullable); + assertThat(result.next()).isTrue(); + assertThat(result.getObject(1)).isNull(); + } + } + @Test public void setExpressionUsesGlobalScopeAndRejectsUnsupportedCalls() throws SQLException { String globalMode = (String) context.querySingleValue("select @@GLOBAL.sql_mode"); From 2657cca1d357619a2dbd87aa5206c0d78fa97882 Mon Sep 17 00:00:00 2001 From: Ketor Date: Wed, 23 Sep 2026 18:17:40 +0800 Subject: [PATCH 2/4] Cover latin1 and utf8mb4 negotiated result charsets --- .../io/dingodb/test/dsl/MySqlRunningIT.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java b/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java index 874594d674..88da719292 100644 --- a/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java +++ b/dingo-test/src/intTest/java/io/dingodb/test/dsl/MySqlRunningIT.java @@ -61,6 +61,26 @@ public void convertLatin1PreservesTextAndByteSemantics() throws Exception { } } + @Test + public void resultCharsetNegotiationPreservesLatin1Text() throws Exception { + try (Connection connection = getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("SET NAMES latin1"); + for (String resultCharset : new String[]{"latin1", "utf8mb4"}) { + statement.execute("SET character_set_results=" + resultCharset); + try (ResultSet result = statement.executeQuery( + "SELECT CHAR(128 USING latin1), LENGTH(CHAR(128 USING latin1)), HEX(CHAR(128 USING latin1))" + )) { + assertTrue(result.next()); + assertEquals(Types.VARCHAR, result.getMetaData().getColumnType(1)); + assertEquals("€", result.getString(1)); + assertEquals(1, result.getInt(2)); + assertEquals("80", result.getString(3)); + } + } + } + } + @TestFactory public Stream testAggregation() { return getTests(SqlTestCaseYamlBuilder.of("cases/aggregation.yml")); From d2eb1b30e2225b1a1b9584b107052ad79d0568f5 Mon Sep 17 00:00:00 2001 From: Ketor Date: Wed, 23 Sep 2026 18:39:19 +0800 Subject: [PATCH 3/4] Preserve mixed-charset CONCAT operands without data loss --- .../calcite/fun/DingoConcatFunction.java | 49 ++++++++++++++++++- .../test/QuerySimpleExpressionTest.java | 22 +++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java b/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java index d5a4916e4b..c8dc76eef7 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/fun/DingoConcatFunction.java @@ -31,6 +31,8 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import java.nio.charset.Charset; + @EqualsAndHashCode(callSuper = true, onlyExplicitlyIncluded = true) public class DingoConcatFunction extends DingoSqlFunction { public DingoConcatFunction( @@ -66,14 +68,55 @@ private static RelDataType inferCharsetReturnType( ) { RelDataType result = returnTypeInference.inferReturnType(binding); RelDataType chosen = null; - for (RelDataType operand : binding.collectOperandTypes()) { + int chosenIndex = -1; + for (int i = 0; i < binding.getOperandCount(); i++) { + RelDataType operand = binding.getOperandType(i); if (operand.getCharset() == null || operand.getCollation() == null) { continue; } + if (chosen != null + && !chosen.getCharset().equals(operand.getCharset()) + && chosen.getCollation().getCoercibility() == operand.getCollation().getCoercibility()) { + Charset left = chosen.getCharset(); + Charset right = operand.getCharset(); + boolean leftLiteral = binding.isOperandLiteral(chosenIndex, false); + boolean rightLiteral = binding.isOperandLiteral(i, false); + if (leftLiteral != rightLiteral) { + String literal = binding.getOperandLiteralValue( + leftLiteral ? chosenIndex : i, String.class + ); + Charset target = leftLiteral ? right : left; + if (literal != null && target.newEncoder().canEncode(literal)) { + // Literals are weaker than character expressions when conversion is lossless. + if (leftLiteral) { + chosen = operand; + chosenIndex = i; + } + continue; + } + } + if (isUnicode(left) != isUnicode(right)) { + // At equal coercibility MySQL promotes non-Unicode text to Unicode. + if (isUnicode(right)) { + chosen = operand; + chosenIndex = i; + } + } else if (!left.contains(right)) { + if (right.contains(left)) { + chosen = operand; + chosenIndex = i; + } else { + throw new IllegalArgumentException("Incompatible CONCAT character sets: " + + left.name() + " and " + right.name()); + } + } + continue; + } if (chosen == null || SqlCollation.getCoercibilityDyadicOperator( chosen.getCollation(), operand.getCollation() ) == operand.getCollation()) { chosen = operand; + chosenIndex = i; } } return chosen == null ? result : binding.getTypeFactory().createTypeWithCharsetAndCollation( @@ -81,4 +124,8 @@ private static RelDataType inferCharsetReturnType( ); } + private static boolean isUnicode(Charset charset) { + return charset.name().startsWith("UTF-"); + } + } diff --git a/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java b/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java index 983e364cdc..41a23933f5 100644 --- a/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java +++ b/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java @@ -124,6 +124,28 @@ public void convertCharsetRetainsEncodedByteSemantics() throws SQLException { } } + @Test + public void concatPromotesMixedCharsetsWithoutLosingCharacters() throws SQLException { + String latin1 = "convert(char(128 using latin1) using latin1)"; + String unicode = "convert(char(240,159,153,130 using utf8mb4) using utf8mb4)"; + assertThat(context.querySingleValue("select hex(concat(" + latin1 + ", " + unicode + "))")) + .isEqualTo("E282ACF09F9982"); + assertThat(context.querySingleValue("select hex(concat(" + unicode + ", " + latin1 + "))")) + .isEqualTo("F09F9982E282AC"); + assertThat(context.querySingleValue("select length(concat(" + latin1 + ", " + unicode + "))")) + .isEqualTo(7); + } + + @Test + public void concatKeepsRepresentableNonUnicodeCharset() throws SQLException { + String latin1 = "convert(char(128 using latin1) using latin1)"; + String ascii = "convert(char(65 using ascii) using ascii)"; + assertThat(context.querySingleValue("select hex(concat(" + latin1 + ", " + ascii + "))")) + .isEqualTo("8041"); + assertThat(context.querySingleValue("select hex(concat(" + ascii + ", " + latin1 + "))")) + .isEqualTo("4180"); + } + @Test public void convertCharsetRejectsUnrepresentableText() { assertThatThrownBy(() -> { From bd854cab7bd1f17936232cc415fe5f47600d0723 Mon Sep 17 00:00:00 2001 From: Ketor Date: Wed, 23 Sep 2026 20:18:54 +0800 Subject: [PATCH 4/4] Fix mixed-charset CASE result typing --- .../calcite/DingoSqlToRelConverter.java | 42 ++++++++++ .../io/dingodb/calcite/DingoSqlValidator.java | 83 +++++++++++++++++++ .../calcite/type/DingoSqlTypeFactory.java | 21 +++-- .../test/QuerySimpleExpressionTest.java | 22 +++++ 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java index 4a21b62096..4af3009bbf 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlToRelConverter.java @@ -76,10 +76,12 @@ import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlSelectKeyword; import org.apache.calcite.sql.SqlUtil; +import org.apache.calcite.sql.fun.SqlCase; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserUtil; import org.apache.calcite.sql.type.BasicSqlType; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorUtil; @@ -141,6 +143,46 @@ public DingoSqlToRelConverter( @Override protected @Nullable RexNode convertExtendedExpression(@NonNull SqlNode node, Blackboard bb) { + if (node.getKind() == SqlKind.CASE) { + SqlCase caseCall = (SqlCase) node; + RelDataType resultType = bb.getValidator().getValidatedNodeType(caseCall); + if (resultType.getCharset() != null) { + boolean mixed = false; + for (SqlNode branch : caseCall.getThenOperands()) { + if (!SqlUtil.isNullLiteral(branch, false)) { + RelDataType branchType = bb.getValidator().getValidatedNodeType(branch); + mixed |= SqlTypeUtil.isCharacter(branchType) + && !resultType.getCharset().equals(branchType.getCharset()); + } + } + SqlNode otherwise = caseCall.getElseOperand(); + if (otherwise != null && !SqlUtil.isNullLiteral(otherwise, false)) { + RelDataType branchType = bb.getValidator().getValidatedNodeType(otherwise); + mixed |= SqlTypeUtil.isCharacter(branchType) + && !resultType.getCharset().equals(branchType.getCharset()); + } + if (mixed) { + // Calcite's convertCase repeats the SQL type-name-only fast path and + // casts every Rex branch to the first branch's charset. + RexBuilder rb = bb.getRexBuilder(); + List operands = new ArrayList<>(caseCall.getWhenOperands().size() * 2 + 1); + for (int i = 0; i < caseCall.getWhenOperands().size(); i++) { + SqlNode when = caseCall.getWhenOperands().get(i); + operands.add(SqlUtil.isNullLiteral(when, false) + ? rb.makeNullLiteral(rb.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN)) + : bb.convertExpression(when)); + SqlNode branch = caseCall.getThenOperands().get(i); + operands.add(SqlUtil.isNullLiteral(branch, false) + ? rb.makeNullLiteral(resultType) + : rb.ensureType(resultType, bb.convertExpression(branch), false)); + } + operands.add(otherwise == null || SqlUtil.isNullLiteral(otherwise, false) + ? rb.makeNullLiteral(resultType) + : rb.ensureType(resultType, bb.convertExpression(otherwise), false)); + return rb.makeCall(resultType, SqlStdOperatorTable.CASE, operands); + } + } + } // MySQL dialect if (node.getKind() == SqlKind.OTHER_FUNCTION) { SqlOperator operator = ((SqlCall) node).getOperator(); diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java index 867af3e509..48ff973310 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/DingoSqlValidator.java @@ -74,7 +74,9 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.util.AbstractList; +import java.util.ArrayList; import java.util.Calendar; import java.util.EnumSet; import java.util.List; @@ -139,6 +141,87 @@ public static TypeCoercion createTypeCoercion(RelDataTypeFactory typeFactory, @Override public void validateCall(SqlCall call, SqlValidatorScope scope) { super.validateCall(call, scope); + if (call.getKind() != SqlKind.CASE) { + return; + } + // Calcite's CASE fast path compares only SQL type names, so a mixed-charset + // VARCHAR result inherits the first branch's charset without merging them. + org.apache.calcite.sql.fun.SqlCase caseCall = (org.apache.calcite.sql.fun.SqlCase) call; + List branches = new ArrayList<>(); + boolean nullable = false; + for (SqlNode branch : caseCall.getThenOperands()) { + if (SqlUtil.isNullLiteral(branch, false)) { + nullable = true; + } else { + RelDataType branchType = deriveType(scope, branch); + branches.add(branchType); + nullable |= branchType.isNullable(); + } + } + SqlNode otherwise = caseCall.getElseOperand(); + nullable |= otherwise == null || SqlUtil.isNullLiteral(otherwise, false); + if (otherwise != null && !SqlUtil.isNullLiteral(otherwise, false)) { + RelDataType branchType = deriveType(scope, otherwise); + branches.add(branchType); + nullable |= branchType.isNullable(); + } + if (branches.size() < 2) { + return; + } + Charset charset = branches.get(0).getCharset(); + if (charset == null) { + return; + } + boolean mixed = false; + for (RelDataType branch : branches) { + if (!SqlTypeUtil.isCharacter(branch) || branch.getCharset() == null) { + return; + } + mixed |= !charset.equals(branch.getCharset()); + } + if (mixed) { + RelDataType merged = typeFactory.leastRestrictive(branches); + if (merged != null) { + setValidatedNodeType(call, typeFactory.createTypeWithNullability(merged, nullable)); + } + } + } + + @Override + protected void validateSelect(SqlSelect select, RelDataType targetRowType, boolean skipMeasure) { + super.validateSelect(select, targetRowType, skipMeasure); + SqlValidatorNamespace namespace = getNamespace(select); + RelDataType rowType = namespace.getRowType(); + SqlNodeList selectList = select.getSelectList(); + if (selectList.size() != rowType.getFieldCount()) { + return; + } + RelDataTypeFactory.Builder corrected = null; + for (int i = 0; i < selectList.size(); i++) { + SqlNode expression = selectList.get(i); + if (expression.getKind() == SqlKind.AS) { + expression = ((SqlCall) expression).operand(0); + } + RelDataTypeField field = rowType.getFieldList().get(i); + RelDataType validated = expression.getKind() == SqlKind.CASE + ? getValidatedNodeTypeIfKnown(expression) : null; + if (validated != null && !validated.equals(field.getType())) { + if (corrected == null) { + corrected = typeFactory.builder(); + for (int j = 0; j < i; j++) { + corrected.add(rowType.getFieldList().get(j)); + } + } + corrected.add(field.getName(), validated); + } else if (corrected != null) { + corrected.add(field); + } + } + if (corrected != null) { + RelDataType result = corrected.build(); + namespace.setType(result); + setValidatedNodeType(select, result); + } } @Override diff --git a/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java b/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java index 8bb198b501..4b8f97190b 100644 --- a/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java +++ b/dingo-calcite/src/main/java/io/dingodb/calcite/type/DingoSqlTypeFactory.java @@ -385,6 +385,9 @@ else if (resultType.getSqlTypeName().getName().equalsIgnoreCase("CHAR") } else if (charset1.equals(charset2)) { charset = charset1; collation = collation1; + } else if (charset1.name().startsWith("UTF-") != charset2.name().startsWith("UTF-")) { + charset = charset1.name().startsWith("UTF-") ? charset1 : charset2; + collation = charset == charset1 ? collation1 : collation2; } else if (charset1.contains(charset2)) { charset = charset1; collation = collation1; @@ -393,20 +396,24 @@ else if (resultType.getSqlTypeName().getName().equalsIgnoreCase("CHAR") collation = collation2; } } - if (collation0 != null && charset1 != null && charset2 != null - && !charset1.equals(charset2)) { + if (collation0 != null && collation1 != null && collation2 != null + && charset1 != null && charset2 != null && !charset1.equals(charset2) + && collation1.getCoercibility() != collation2.getCoercibility()) { if (collation0.equals(collation1)) { charset = charset1; + collation = collation1; } else if (collation0.equals(collation2)) { charset = charset2; + collation = collation2; } } if (charset != null) { - resultType = - createTypeWithCharsetAndCollation( - resultType, - charset, - collation0 != null ? collation0 : requireNonNull(collation, "collation")); + SqlCollation selectedCollation = charset1 != null && charset2 != null + && !charset1.equals(charset2) ? collation : collation0; + resultType = createTypeWithCharsetAndCollation( + resultType, charset, + requireNonNull(selectedCollation != null ? selectedCollation : collation, "collation") + ); } } else if (SqlTypeUtil.isExactNumeric(type)) { if (SqlTypeUtil.isExactNumeric(resultType)) { diff --git a/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java b/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java index 41a23933f5..c707b88ee1 100644 --- a/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java +++ b/dingo-test/src/test/java/io/dingodb/test/QuerySimpleExpressionTest.java @@ -146,6 +146,28 @@ public void concatKeepsRepresentableNonUnicodeCharset() throws SQLException { .isEqualTo("4180"); } + @Test + public void casePromotesMixedCharsetsIndependentOfBranchOrder() throws SQLException { + String latin1 = "convert(char(128 using latin1) using latin1)"; + String unicode = "convert(char(240,159,153,130 using utf8mb4) using utf8mb4)"; + String unicodeValue = "case when 1=0 then " + latin1 + " else " + unicode + " end"; + String latin1Value = "case when 1=0 then " + unicode + " else " + latin1 + " end"; + assertThat(context.querySingleValue("select hex(" + unicodeValue + ")")).isEqualTo("F09F9982"); + assertThat(context.querySingleValue("select length(" + unicodeValue + ")")).isEqualTo(4); + assertThat(context.querySingleValue("select hex(" + latin1Value + ")")).isEqualTo("E282AC"); + assertThat(context.querySingleValue("select length(" + latin1Value + ")")).isEqualTo(3); + String nullableValue = "case when 1=0 then " + latin1 + " when 1=1 then " + unicode + + " else null end"; + assertThat(context.querySingleValue("select hex(" + nullableValue + ")")).isEqualTo("F09F9982"); + String nullValue = "case when 1=0 then " + latin1 + " when 1=0 then " + unicode + + " else null end"; + assertThat(context.querySingleValue("select " + nullValue)).isNull(); + try (Statement statement = context.getConnection().createStatement(); + ResultSet result = statement.executeQuery("select " + nullableValue)) { + assertThat(result.getMetaData().isNullable(1)).isEqualTo(ResultSetMetaData.columnNullable); + } + } + @Test public void convertCharsetRejectsUnrepresentableText() { assertThatThrownBy(() -> {