diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala index 6843929ef..88e20216a 100644 --- a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala @@ -92,10 +92,14 @@ object BCDNumberDecoders { var sign = "" + // Since when scaleFactor < 0 an additional zero is always added as '0.' we need to scale the original + // value left by 1 digit (Fixes: https://github.com/AbsaOSS/cobrix/issues/837) + val scaleDotPosition = if (scaleFactor < 0 && scale > 0) scale - 1 else scale + val intendedDecimalPosition = if (mandatorySignNibble) - bytes.length * 2 - (scale + 1) + bytes.length * 2 - (scaleDotPosition + 1) else - bytes.length * 2 - scale + bytes.length * 2 - scaleDotPosition val additionalZeros = if (intendedDecimalPosition <= 0) { -intendedDecimalPosition + 1 diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala index 2028a571d..1ea45e95f 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala @@ -19,7 +19,8 @@ package za.co.absa.cobrix.cobol.parser.copybooks import org.scalatest.funsuite.AnyFunSuite import org.slf4j.{Logger, LoggerFactory} import za.co.absa.cobrix.cobol.parser.CopybookParser -import za.co.absa.cobrix.cobol.parser.ast.Group +import za.co.absa.cobrix.cobol.parser.ast.datatype.Decimal +import za.co.absa.cobrix.cobol.parser.ast.{Group, Primitive} import za.co.absa.cobrix.cobol.parser.exceptions.SyntaxErrorException import za.co.absa.cobrix.cobol.parser.policies.FillerNamingPolicy import za.co.absa.cobrix.cobol.testutils.SimpleComparisonBase @@ -312,4 +313,26 @@ class ParseCopybookFeaturesSpec extends AnyFunSuite with SimpleComparisonBase { assertEqualsMultiline(layout, expectedLayout) } + + test("Test parsing copybooks with scaled decimals") { + val copybookStr = " 10 N PIC SVPP9(5) COMP-3." + + val copybook = CopybookParser.parseSimple(copybookStr) + val layout = copybook.generateRecordLayoutPositions() + + val field = copybook.getFieldByName("N").asInstanceOf[Primitive] + val dataType = field.dataType.asInstanceOf[Decimal] + + assert(dataType.scale == 5) + assert(dataType.scaleFactor == -2) + + val expectedLayout = + """-------- FIELD LEVEL/NAME --------- --ATTRIBS-- FLD START END LENGTH + | + |10 N 1 1 3 3 + |""" + .stripMargin.replace("\r\n", "\n") + + assertEqualsMultiline(layout, expectedLayout) + } } diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala index 06028b1ab..0b52629e4 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala @@ -226,6 +226,26 @@ class BinaryDecoderSpec extends AnyFunSuite { assert (v2.contains("92233720368547757.98")) } + test("Test COMP-3 decimal with scale factor cases") { + val v1 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = 0, mandatorySignNibble = true) + assert (v1 == "06547") + + val v2 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = -1, mandatorySignNibble = true) + assert (v2 == "0.006547") + + val v3 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = -2, mandatorySignNibble = true) + assert (v3 == "0.0006547") + + val v4 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -1, mandatorySignNibble = true) + assert (v4 == "0.006547") + + val v5 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -2, mandatorySignNibble = true) + assert (v5 == "0.0006547") + + val v6 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x16.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -2, mandatorySignNibble = true) + assert (v6 == "0.0016547") + } + test("Test COMP-3U decimal cases") { // A simple decimal number val v1 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x15.toByte, 0x88.toByte, 0x40.toByte), scale = 2, scaleFactor = 0, mandatorySignNibble = false) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala index a7a2e8519..68ff680f3 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala @@ -281,6 +281,46 @@ class Test17NumericConversions extends AnyWordSpec with SparkTestBase with Binar } } } + + "negative scale factor for DISPLAY AND COMP-3 numbers should be decoded correctly" when { + val copybook = + """ 10 N1 PIC SVPP9(5) COMP-3. + | 10 N2 PIC SVPP9(5). + |""".stripMargin + + withTempBinFile("scelad_commp3", ".dat", Array(0x06, 0x54, 0x7C, 0xF0, 0xF6, 0xF5, 0xF4, 0xF7).map(_.toByte)) { tmpFileName => + val df = spark + .read + .format("cobol") + .option("copybook_contents", copybook) + .option("record_format", "F") + .option("pedantic", "true") + .load(tmpFileName) + + val actualSchema = df.schema.treeString + val row = df.collect().head + val actualData1 = row.getDecimal(0).toString + val actualData2 = row.getDecimal(1).toString + + "schema should match" in { + val expectedSchema = + """root + | |-- N1: decimal(7,7) (nullable = true) + | |-- N2: decimal(7,7) (nullable = true) + |""".stripMargin + + assertEqualsMultiline(actualSchema, expectedSchema) + } + + "data should match" in { + val expectedData = "0.0006547" + + assertEqualsMultiline(actualData1, expectedData) + assertEqualsMultiline(actualData2, expectedData) + } + } + } + } }