Skip to content

Reproduce UTF8 substring issue #1065

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sodre wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Reproduce UTF8 substring issue #1065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
34fc68d
Reproduce UTF8 substring issue
sodre Apr 12, 2021
a4adb21
Fix UTF8String substring issue
sodre Apr 12, 2021
7d5b69d
Fix size/length issue for UTF-8 Strings
sodre Apr 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion jolt-core/src/main/java/com/bazaarvoice/jolt/modifier/function/Objects.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ else if(args[0] instanceof List ) {
return Optional.of(((List) args[0]).size());
}
else if(args[0] instanceof String) {
return Optional.of( ((String) args[0]).length() );
return Optional.of(((String) args[0]).codePointCount(0, ((String) args[0]).length()));
}
else if(args[0] instanceof Map) {
return Optional.of( ((Map) args[0]).size() );
Expand Down
4 changes: 2 additions & 2 deletions jolt-core/src/main/java/com/bazaarvoice/jolt/modifier/function/Strings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ protected Optional<Object> applyList(List<Object> argList) {

// If we get here, then all these casts should work.
String tuna = (String) argList.get(0);
int start = (Integer) argList.get(1);
int end = (Integer) argList.get(2);
int start = tuna.offsetByCodePoints(0, (Integer) argList.get(1));
int end = tuna.offsetByCodePoints(0, (Integer) argList.get(2));

// do start and end make sense?
if ( start >= end || start < 0 || end < 1 || end > tuna.length() ) {
Expand Down
6 changes: 5 additions & 1 deletion jolt-core/src/test/resources/json/modifier/functions/sizeTests.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"legitList" : [ 1, "foo" ],
"legitMap" : { "a":"b", "d":"e"},
"legitString" : "foo",
"legitUtf8String": "\uD83D\uDC47",
"legitNumber" : 3.1415
},

Expand All @@ -20,6 +21,7 @@
"legitListSize": "=size(@(1,legitList))",
"legitMapSize": "=size(@(1,legitMap))",
"legitStringSize": "=size(@(1,legitString))",
"legitUtf8StringSize": "=size(@(1,legitUtf8String))",
"legitNumberSize": "=size(@(1,legitNumber))"
},
"context": {},
Expand All @@ -39,11 +41,13 @@
"legitList" : [ 1, "foo" ],
"legitMap" : { "a":"b", "d":"e"},
"legitString" : "foo",
"legitUtf8String": "\uD83D\uDC47",
"legitNumber" : 3.1415,

"legitListSize": 2,
"legitMapSize": 2,
"legitStringSize": 3
"legitStringSize": 3,
"legitUtf8StringSize": 1
// legitNumberSize does not get created because there can be no value for it
}
}
10 changes: 7 additions & 3 deletions jolt-core/src/test/resources/json/modifier/functions/stringsTests.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
"string": "the QuIcK brOwn fox",
"zeroIndex" : 0,
"threeIndex" : 3,
"trimMe" : " tuna "
"trimMe" : " tuna ",
"utf8String": "\uD83D\uDC47\uD83D\uDD34 UTF-8 String"
},

"spec": {
Expand Down Expand Up @@ -44,7 +45,8 @@
"custom2": "=substring('the quick brown fox', 16, 19)",
//
// verify that we can actually lookup start and end indices
"advancedLookupRanges" : "=substring(@(2,string), @(2,zeroIndex), @(2,threeIndex))"
"advancedLookupRanges" : "=substring(@(2,string), @(2,zeroIndex), @(2,threeIndex))",
"utf8Substring": "=substring(@(2,utf8String), 3, 15)"
},
"trim" :{
"trimed" : "=trim(@(2,trimMe))"
Expand All @@ -61,6 +63,7 @@
"string" : "the QuIcK brOwn fox",
"zeroIndex" : 0,
"threeIndex" : 3,
"utf8String": "\uD83D\uDC47\uD83D\uDD34 UTF-8 String",
//
// from the input, but overwritten by modify
"trimMe" : "tuna",
Expand Down Expand Up @@ -88,7 +91,8 @@
"basic2": "QuIcK",
"custom1": "the quick brown",
"custom2": "fox",
"advancedLookupRanges": "the"
"advancedLookupRanges": "the",
"utf8Substring": "UTF-8 String"
},
"trim" :{
"trimed" : "tuna"
Expand Down