From 498f6d131bc3293d14a343dd150ad6573888a278 Mon Sep 17 00:00:00 2001 From: kyle-hawk Date: Wed, 1 Feb 2023 10:41:36 +0800 Subject: [PATCH] close #251 [BitSail][Features] Support list and map in current FileMappingTypeInfoConverter --- .../common/type/BitSailTypeParser.java | 20 ++++++- .../FileMappingTypeInfoConverter.java | 54 +++++++++++++++++- .../FileMappingTypeInfoReader.java | 27 +++++++++ .../type/SimpleTypeInfoConverterTest.java | 55 +++++++++++++++++-- .../test/resources/fake-type-converter.yaml | 37 ++++++++++++- 5 files changed, 181 insertions(+), 12 deletions(-) diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/BitSailTypeParser.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/BitSailTypeParser.java index 0e38dd7f9..82dba9384 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/BitSailTypeParser.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/BitSailTypeParser.java @@ -41,7 +41,25 @@ public class BitSailTypeParser { private static final String SPLIT_TOKEN = ","; public static String fromTypeInfo(TypeInfo typeInfo) { - return TypeInfoBridge.bridgeTypes(typeInfo); + Preconditions.checkNotNull(typeInfo, "typeInfo can not be null."); + + if (typeInfo instanceof ListTypeInfo || typeInfo instanceof MapTypeInfo) { + + if (typeInfo instanceof MapTypeInfo) { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + return String.format("%s<%s, %s>", Types.MAP.name(), fromTypeInfo(mapTypeInfo.getKeyTypeInfo()), fromTypeInfo(mapTypeInfo.getValueTypeInfo())); + } else { + + ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; + return String.format("%s<%s>", Types.LIST.name(), fromTypeInfo(listTypeInfo.getElementTypeInfo())); + } + } + String typeString = TypeInfoBridge.bridgeTypes(typeInfo); + if (StringUtils.isEmpty(typeString)) { + throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR, + String.format("Not support type typeInfo %s.", typeInfo)); + } + return typeString; } public static List fromTypePropertyString(String typePropertyString) { diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoConverter.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoConverter.java index 34f56f376..f3c531d7b 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoConverter.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoConverter.java @@ -16,12 +16,20 @@ package com.bytedance.bitsail.common.type.filemapping; +import com.bytedance.bitsail.common.BitSailException; +import com.bytedance.bitsail.common.exception.CommonErrorCode; +import com.bytedance.bitsail.common.type.BitSailTypeParser; import com.bytedance.bitsail.common.type.TypeInfoConverter; +import com.bytedance.bitsail.common.typeinfo.ListTypeInfo; +import com.bytedance.bitsail.common.typeinfo.MapTypeInfo; import com.bytedance.bitsail.common.typeinfo.TypeInfo; +import com.bytedance.bitsail.common.typeinfo.Types; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import java.util.Objects; + public class FileMappingTypeInfoConverter implements TypeInfoConverter { @Getter protected FileMappingTypeInfoReader reader; @@ -52,11 +60,53 @@ protected static String trim(String typeName) { @Override public TypeInfo fromTypeString(String typeString) { - return reader.getToTypeInformation().get(typeString); + TypeInfo typeInfo = reader.getToTypeInformation().get(typeString); + + if (!Objects.isNull(typeInfo)) { + return typeInfo; + } + + String engineMapStr = reader.customToEngineTypeStringMap.get(Types.MAP.name().toLowerCase()); + String engineListStr = reader.customToEngineTypeStringMap.get(Types.LIST.name().toLowerCase()); + if (StringUtils.isNotBlank(engineMapStr) && typeString.contains(engineMapStr) + || StringUtils.isNotBlank(engineListStr) && typeString.contains(engineListStr)) { + String customTypeString = this.toCustomTypeString(typeString); + return BitSailTypeParser.fromTypeString(customTypeString); + } + + throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR, + String.format("Not support type string %s.", typeString)); } @Override public String fromTypeInfo(TypeInfo typeInfo) { - return reader.getFromTypeInformation().get(typeInfo); + String typeStr = reader.getFromTypeInformation().get(typeInfo); + + if (StringUtils.isNotBlank(typeStr)) { + return typeStr; + } + + if (typeInfo instanceof ListTypeInfo || typeInfo instanceof MapTypeInfo) { + String customTypeString = BitSailTypeParser.fromTypeInfo(typeInfo); + return this.toEngineTypeString(customTypeString); + } + + throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR, + String.format("Not support typeInfo %s.", typeInfo)); + } + + private String toEngineTypeString(String customTypeString) { + customTypeString = customTypeString.toLowerCase(); + for (String key : reader.customToEngineTypeStringMap.keySet()) { + customTypeString = customTypeString.replace(key, reader.customToEngineTypeStringMap.get(key)); + } + return customTypeString; + } + + private String toCustomTypeString(String engineTypeString) { + for (String key : reader.engineToCustomTypeStringMap.keySet()) { + engineTypeString = engineTypeString.replace(key, reader.engineToCustomTypeStringMap.get(key)); + } + return engineTypeString.toLowerCase(); } } diff --git a/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoReader.java b/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoReader.java index 39d17807b..8305e2ddd 100644 --- a/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoReader.java +++ b/bitsail-common/src/main/java/com/bytedance/bitsail/common/type/filemapping/FileMappingTypeInfoReader.java @@ -16,6 +16,8 @@ package com.bytedance.bitsail.common.type.filemapping; +import com.bytedance.bitsail.common.BitSailException; +import com.bytedance.bitsail.common.exception.CommonErrorCode; import com.bytedance.bitsail.common.type.BitSailTypeParser; import com.bytedance.bitsail.common.typeinfo.TypeInfo; @@ -31,6 +33,7 @@ import java.io.Serializable; import java.net.URL; import java.text.MessageFormat; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -45,12 +48,19 @@ public class FileMappingTypeInfoReader implements Serializable { private static final String ENGINE_TO_CUSTOM_KEY = "engine.type.to.bitsail.type.converter"; private static final String CUSTOM_TO_ENGINE_KEY = "bitsail.type.to.engine.type.converter"; + private static final String ENGINE_TO_CUSTOM_COMPLEX_KEY = "engine.type.to.bitsail.complex.converter"; + private static final String CUSTOM_TO_ENGINE_COMPLEX_KEY = "bitsail.type.to.engine.complex.converter"; + private final String converterFileName; @Getter protected Map> toTypeInformation = Maps.newHashMap(); @Getter protected Map, String> fromTypeInformation = Maps.newHashMap(); + @Getter + protected Map engineToCustomTypeStringMap = Maps.newHashMap(); + @Getter + protected Map customToEngineTypeStringMap = Maps.newHashMap(); public FileMappingTypeInfoReader(String engine) { LOG.info("File mapping reader from engine = {}.", engine); @@ -99,6 +109,13 @@ private void read() throws IOException { readerOption(converterConf, ENGINE_TO_CUSTOM_KEY, tmpToTypeInformation); readerOption(converterConf, CUSTOM_TO_ENGINE_KEY, tmpFromTypeInformation); + readerOption(converterConf, ENGINE_TO_CUSTOM_COMPLEX_KEY, engineToCustomTypeStringMap); + readerOption(converterConf, CUSTOM_TO_ENGINE_COMPLEX_KEY, customToEngineTypeStringMap); + this.customTypeStringCaseCheck(engineToCustomTypeStringMap.values()); + this.customTypeStringCaseCheck(customToEngineTypeStringMap.keySet()); + engineToCustomTypeStringMap.putAll(tmpToTypeInformation); + customToEngineTypeStringMap.putAll(tmpFromTypeInformation); + handleEngineTypeToCustom(tmpToTypeInformation); handleCustomToEngineType(tmpFromTypeInformation); } @@ -128,4 +145,14 @@ protected void handleCustomToEngineType(Map tmpFromTypeInformati fromTypeInformation.put(typeInfo, entry.getValue()); } } + + private void customTypeStringCaseCheck(Collection customTypeStrings) { + for (String typeStr : customTypeStrings) { + if (!typeStr.equals(typeStr.toLowerCase())) { + throw BitSailException.asBitSailException(CommonErrorCode.INTERNAL_ERROR, + String.format("custom type string must be lowercase:%s", typeStr)); + } + } + } + } diff --git a/bitsail-common/src/test/java/com/bytedance/bitsail/common/type/SimpleTypeInfoConverterTest.java b/bitsail-common/src/test/java/com/bytedance/bitsail/common/type/SimpleTypeInfoConverterTest.java index 7ed11f39e..db01e359d 100644 --- a/bitsail-common/src/test/java/com/bytedance/bitsail/common/type/SimpleTypeInfoConverterTest.java +++ b/bitsail-common/src/test/java/com/bytedance/bitsail/common/type/SimpleTypeInfoConverterTest.java @@ -21,8 +21,10 @@ import com.bytedance.bitsail.common.typeinfo.TypeInfo; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import java.util.List; import java.util.Map; /** @@ -30,15 +32,56 @@ */ public class SimpleTypeInfoConverterTest { - @Test - public void testSimpleTypeInfoConverter() { - FileMappingTypeInfoConverter fileMappingTypeInfoConverter = new FileMappingTypeInfoConverter("fake"); + private FileMappingTypeInfoConverter fileMappingTypeInfoConverter; + private Map> toTypeInformation; + private Map, String> fromTypeInformation; + + @Before + public void before() { + fileMappingTypeInfoConverter = new FileMappingTypeInfoConverter("fake"); FileMappingTypeInfoReader reader = fileMappingTypeInfoConverter.getReader(); - Map> toTypeInformation = reader.getToTypeInformation(); - Map, String> fromTypeInformation = reader.getFromTypeInformation(); + toTypeInformation = reader.getToTypeInformation(); + fromTypeInformation = reader.getFromTypeInformation(); + } + @Test + public void testSimpleTypeInfoConverter() { Assert.assertEquals(toTypeInformation.size(), 8); - Assert.assertEquals(fromTypeInformation.size(), 6); + Assert.assertEquals(fromTypeInformation.size(), 7); } + @Test + public void testFromTypeString_ListType() { + TypeInfo typeInfo01 = fileMappingTypeInfoConverter.fromTypeString("Array(double)"); + Assert.assertEquals(typeInfo01.getTypeClass(), List.class); + + TypeInfo typeInfo02 = fileMappingTypeInfoConverter.fromTypeString("Array(Array(double))"); + Assert.assertEquals(typeInfo02.getTypeClass(), List.class); + } + + @Test + public void testFromTypeString_MapType() { + TypeInfo typeInfo01 = fileMappingTypeInfoConverter.fromTypeString("Map(string,double)"); + Assert.assertEquals(typeInfo01.getTypeClass(), Map.class); + + TypeInfo typeInfo02 = fileMappingTypeInfoConverter.fromTypeString("Map(string, Map(string, double))"); + Assert.assertEquals(typeInfo02.getTypeClass(), Map.class); + } + + @Test + public void testFromTypeInfo_ListType() { + TypeInfo typeInfo = fileMappingTypeInfoConverter.fromTypeString("Array(Array(Int32))"); + Assert.assertEquals(typeInfo.getTypeClass(), List.class); + + String typeString = fileMappingTypeInfoConverter.fromTypeInfo(typeInfo); + Assert.assertEquals(typeString, "Array(Array(Int32))"); + } + + @Test + public void testFromTypeInfo_MapType() { + TypeInfo typeInfo = fileMappingTypeInfoConverter.fromTypeString("Map(string, Map(string, double))"); + + String typeString = fileMappingTypeInfoConverter.fromTypeInfo(typeInfo); + Assert.assertEquals(typeString, "Map(string, Map(string, double))"); + } } \ No newline at end of file diff --git a/bitsail-common/src/test/resources/fake-type-converter.yaml b/bitsail-common/src/test/resources/fake-type-converter.yaml index f0fc43d9c..66c8500cc 100644 --- a/bitsail-common/src/test/resources/fake-type-converter.yaml +++ b/bitsail-common/src/test/resources/fake-type-converter.yaml @@ -17,8 +17,8 @@ engine.type.to.bitsail.type.converter: - source.type: double target.type: double - - source.type: int - target.type: long + - source.type: Int32 + target.type: int - source.type: bigint target.type: long @@ -26,7 +26,7 @@ engine.type.to.bitsail.type.converter: bitsail.type.to.engine.type.converter: - source.type: long - target.type: long + target.type: bigint - source.type: double target.type: double @@ -42,3 +42,34 @@ bitsail.type.to.engine.type.converter: - source.type: bytes target.type: bytes + + - source.type: int + target.type: Int32 + + +engine.type.to.bitsail.complex.converter: + - source.type: Array + target.type: list + + - source.type: Map + target.type: map + + - source.type: ( + target.type: < + + - source.type: ) + target.type: '>' + +bitsail.type.to.engine.complex.converter: + - source.type: list + target.type: Array + + - source.type: map + target.type: Map + + - source.type: < + target.type: ( + + - source.type: '>' + target.type: ) +