From cc9e480aa07374f540e422b7378722f50f34c56c Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Tue, 14 Apr 2026 10:24:41 +0200 Subject: [PATCH 1/2] #837 Fix "PIC SVPP9(5) COMP-3" when the scale factor is negative. --- .../parser/decoders/BCDNumberDecoders.scala | 8 +++- .../copybooks/ParseCopybookFeaturesSpec.scala | 25 +++++++++++- .../parser/decoders/BinaryDecoderSpec.scala | 20 ++++++++++ .../regression/Test17NumericConversions.scala | 39 +++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala index 6843929ef..88e20216a 100644 --- a/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala +++ b/cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/parser/decoders/BCDNumberDecoders.scala @@ -92,10 +92,14 @@ object BCDNumberDecoders { var sign = "" + // Since when scaleFactor < 0 an additional zero is always added as '0.' we need to scale the original + // value left by 1 digit (Fixes: https://github.com/AbsaOSS/cobrix/issues/837) + val scaleDotPosition = if (scaleFactor < 0 && scale > 0) scale - 1 else scale + val intendedDecimalPosition = if (mandatorySignNibble) - bytes.length * 2 - (scale + 1) + bytes.length * 2 - (scaleDotPosition + 1) else - bytes.length * 2 - scale + bytes.length * 2 - scaleDotPosition val additionalZeros = if (intendedDecimalPosition <= 0) { -intendedDecimalPosition + 1 diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala index 2028a571d..1ea45e95f 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/copybooks/ParseCopybookFeaturesSpec.scala @@ -19,7 +19,8 @@ package za.co.absa.cobrix.cobol.parser.copybooks import org.scalatest.funsuite.AnyFunSuite import org.slf4j.{Logger, LoggerFactory} import za.co.absa.cobrix.cobol.parser.CopybookParser -import za.co.absa.cobrix.cobol.parser.ast.Group +import za.co.absa.cobrix.cobol.parser.ast.datatype.Decimal +import za.co.absa.cobrix.cobol.parser.ast.{Group, Primitive} import za.co.absa.cobrix.cobol.parser.exceptions.SyntaxErrorException import za.co.absa.cobrix.cobol.parser.policies.FillerNamingPolicy import za.co.absa.cobrix.cobol.testutils.SimpleComparisonBase @@ -312,4 +313,26 @@ class ParseCopybookFeaturesSpec extends AnyFunSuite with SimpleComparisonBase { assertEqualsMultiline(layout, expectedLayout) } + + test("Test parsing copybooks with scaled decimals") { + val copybookStr = " 10 N PIC SVPP9(5) COMP-3." + + val copybook = CopybookParser.parseSimple(copybookStr) + val layout = copybook.generateRecordLayoutPositions() + + val field = copybook.getFieldByName("N").asInstanceOf[Primitive] + val dataType = field.dataType.asInstanceOf[Decimal] + + assert(dataType.scale == 5) + assert(dataType.scaleFactor == -2) + + val expectedLayout = + """-------- FIELD LEVEL/NAME --------- --ATTRIBS-- FLD START END LENGTH + | + |10 N 1 1 3 3 + |""" + .stripMargin.replace("\r\n", "\n") + + assertEqualsMultiline(layout, expectedLayout) + } } diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala index 06028b1ab..8d07e986c 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala @@ -226,6 +226,26 @@ class BinaryDecoderSpec extends AnyFunSuite { assert (v2.contains("92233720368547757.98")) } + test("Test COMP-3 decimal with scale factor cases") { + val v1 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = 0, mandatorySignNibble = true) + assert (v1.contains("06547")) + + val v2 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = -1, mandatorySignNibble = true) + assert (v2.contains("0.006547")) + + val v3 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = -2, mandatorySignNibble = true) + assert (v3.contains("0.0006547")) + + val v4 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -1, mandatorySignNibble = true) + assert (v4.contains("0.006547")) + + val v5 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -2, mandatorySignNibble = true) + assert (v5.contains("0.0006547")) + + val v6 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x16.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -2, mandatorySignNibble = true) + assert (v6.contains("0.0016547")) + } + test("Test COMP-3U decimal cases") { // A simple decimal number val v1 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x15.toByte, 0x88.toByte, 0x40.toByte), scale = 2, scaleFactor = 0, mandatorySignNibble = false) diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala index a7a2e8519..9c62635df 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala @@ -281,6 +281,45 @@ class Test17NumericConversions extends AnyWordSpec with SparkTestBase with Binar } } } + + "negative scale factor for DISPLAY AND COMP-3 numbers should be decoded correctly" when { + val copybook = + """ 10 N1 PIC SVPP9(5) COMP-3. + | 10 N2 PIC SVPP9(5). + |""".stripMargin + + withTempBinFile("scelad_commp3", ".dat", Array(0x06, 0x54, 0x7C, 0xF0, 0xF6, 0xF5, 0xF4, 0xF7).map(_.toByte)) { tmpFileName => + val df = spark + .read + .format("cobol") + .option("copybook_contents", copybook) + .option("record_format", "F") + .option("pedantic", "true") + .load(tmpFileName) + + val actualSchema = df.schema.treeString + val actualData1 = df.collect()(0).getDecimal(0).toString + val actualData2 = df.collect()(0).getDecimal(1).toString + + "schema should match" in { + val expectedSchema = + """root + | |-- N1: decimal(7,7) (nullable = true) + | |-- N2: decimal(7,7) (nullable = true) + |""".stripMargin + + assertEqualsMultiline(actualSchema, expectedSchema) + } + + "data should match" in { + val expectedData = "0.0006547" + + assertEqualsMultiline(actualData1, expectedData) + assertEqualsMultiline(actualData2, expectedData) + } + } + } + } } From 940a1576495038c2bbd68cc8c97c39ab34d5d564 Mon Sep 17 00:00:00 2001 From: Ruslan Iushchenko Date: Tue, 14 Apr 2026 11:41:03 +0200 Subject: [PATCH 2/2] #837 Make COMP-3 with scale factor tests stricter. --- .../cobol/parser/decoders/BinaryDecoderSpec.scala | 12 ++++++------ .../source/regression/Test17NumericConversions.scala | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala index 8d07e986c..0b52629e4 100644 --- a/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala +++ b/cobol-parser/src/test/scala/za/co/absa/cobrix/cobol/parser/decoders/BinaryDecoderSpec.scala @@ -228,22 +228,22 @@ class BinaryDecoderSpec extends AnyFunSuite { test("Test COMP-3 decimal with scale factor cases") { val v1 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = 0, mandatorySignNibble = true) - assert (v1.contains("06547")) + assert (v1 == "06547") val v2 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = -1, mandatorySignNibble = true) - assert (v2.contains("0.006547")) + assert (v2 == "0.006547") val v3 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 0, scaleFactor = -2, mandatorySignNibble = true) - assert (v3.contains("0.0006547")) + assert (v3 == "0.0006547") val v4 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -1, mandatorySignNibble = true) - assert (v4.contains("0.006547")) + assert (v4 == "0.006547") val v5 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x06.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -2, mandatorySignNibble = true) - assert (v5.contains("0.0006547")) + assert (v5 == "0.0006547") val v6 = BCDNumberDecoders.decodeBigBCDNumber(Array[Byte](0x16.toByte,0x54.toByte,0x7C.toByte), scale = 5, scaleFactor = -2, mandatorySignNibble = true) - assert (v6.contains("0.0016547")) + assert (v6 == "0.0016547") } test("Test COMP-3U decimal cases") { diff --git a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala index 9c62635df..68ff680f3 100644 --- a/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala +++ b/spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/regression/Test17NumericConversions.scala @@ -298,8 +298,9 @@ class Test17NumericConversions extends AnyWordSpec with SparkTestBase with Binar .load(tmpFileName) val actualSchema = df.schema.treeString - val actualData1 = df.collect()(0).getDecimal(0).toString - val actualData2 = df.collect()(0).getDecimal(1).toString + val row = df.collect().head + val actualData1 = row.getDecimal(0).toString + val actualData2 = row.getDecimal(1).toString "schema should match" in { val expectedSchema =