diff --git a/libutils/json.c b/libutils/json.c index e46e0716..513dbdda 100644 --- a/libutils/json.c +++ b/libutils/json.c @@ -2061,6 +2061,7 @@ void JsonWriteCompact(Writer *const w, const JsonElement *const element) static JsonParseError JsonParseAsObject( void *lookup_context, JsonLookup *lookup_function, + bool reject_duplicates, const char **data, JsonElement **json_out); @@ -2176,7 +2177,9 @@ const char *JsonParseErrorToString(const JsonParseError error) "CFEngine was not built with libyaml support", [JSON_PARSE_ERROR_LIBYAML_FAILURE] = "libyaml internal failure", [JSON_PARSE_ERROR_NO_SUCH_FILE] = "No such file or directory", - [JSON_PARSE_ERROR_NO_DATA] = "No data"}; + [JSON_PARSE_ERROR_NO_DATA] = "No data", + [JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY] = + "Unwilling to parse json data with a duplicate object member"}; return parse_errors[error]; } @@ -2440,6 +2443,7 @@ static JsonParseError JsonParseAsPrimitive( static JsonParseError JsonParseAsArray( void *const lookup_context, JsonLookup *const lookup_function, + const bool reject_duplicates, const char **const data, JsonElement **const json_out) { @@ -2490,7 +2494,11 @@ static JsonParseError JsonParseAsArray( } JsonElement *child_array = NULL; JsonParseError err = JsonParseAsArray( - lookup_context, lookup_function, data, &child_array); + lookup_context, + lookup_function, + reject_duplicates, + data, + &child_array); if (err != JSON_PARSE_OK) { JsonDestroy(array); @@ -2511,7 +2519,11 @@ static JsonParseError JsonParseAsArray( } JsonElement *child_object = NULL; JsonParseError err = JsonParseAsObject( - lookup_context, lookup_function, data, &child_object); + lookup_context, + lookup_function, + reject_duplicates, + data, + &child_object); if (err != JSON_PARSE_OK) { JsonDestroy(array); @@ -2618,9 +2630,22 @@ static const char *unquoted_key_with_colon(const char *data) return data + i; } +/** + * Does the object under construction already hold a member called key? + * JsonObjectAppendElement() would silently replace it. + */ +static bool JsonParseIsDuplicateKey( + const bool reject_duplicates, + const JsonElement *const object, + const char *const key) +{ + return reject_duplicates && JsonObjectGet(object, key) != NULL; +} + static JsonParseError JsonParseAsObject( void *const lookup_context, JsonLookup *const lookup_function, + const bool reject_duplicates, const char **const data, JsonElement **const json_out) { @@ -2685,6 +2710,13 @@ static JsonParseError JsonParseAsObject( return err; } assert(property_name); + if (JsonParseIsDuplicateKey( + reject_duplicates, object, property_name)) + { + free(property_name); + JsonDestroy(object); + return JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY; + } } break; @@ -2712,7 +2744,11 @@ static JsonParseError JsonParseAsObject( { JsonElement *child_array = NULL; JsonParseError err = JsonParseAsArray( - lookup_context, lookup_function, data, &child_array); + lookup_context, + lookup_function, + reject_duplicates, + data, + &child_array); if (err != JSON_PARSE_OK) { free(property_name); @@ -2737,7 +2773,11 @@ static JsonParseError JsonParseAsObject( { JsonElement *child_object = NULL; JsonParseError err = JsonParseAsObject( - lookup_context, lookup_function, data, &child_object); + lookup_context, + lookup_function, + reject_duplicates, + data, + &child_object); if (err != JSON_PARSE_OK) { free(property_name); @@ -2792,6 +2832,13 @@ static JsonParseError JsonParseAsObject( property_name = xstrndup(*data, ws - *data); *data = colon; + if (JsonParseIsDuplicateKey( + reject_duplicates, object, property_name)) + { + free(property_name); + JsonDestroy(object); + return JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY; + } break; } else if (property_name != NULL) @@ -2888,9 +2935,10 @@ JsonParseError JsonParseAll(const char **const data, JsonElement **const json_ou return error; } -JsonParseError JsonParseWithLookup( +static JsonParseError JsonParseDocument( void *const lookup_context, JsonLookup *const lookup_function, + const bool reject_duplicates, const char **const data, JsonElement **const json_out) { @@ -2906,12 +2954,20 @@ JsonParseError JsonParseWithLookup( if (**data == '{') { return JsonParseAsObject( - lookup_context, lookup_function, data, json_out); + lookup_context, + lookup_function, + reject_duplicates, + data, + json_out); } else if (**data == '[') { return JsonParseAsArray( - lookup_context, lookup_function, data, json_out); + lookup_context, + lookup_function, + reject_duplicates, + data, + json_out); } else if (IsWhitespace(**data)) { @@ -2926,6 +2982,48 @@ JsonParseError JsonParseWithLookup( return JSON_PARSE_ERROR_NO_DATA; } +JsonParseError JsonParseWithLookup( + void *const lookup_context, + JsonLookup *const lookup_function, + const char **const data, + JsonElement **const json_out) +{ + return JsonParseDocument( + lookup_context, lookup_function, false, data, json_out); +} + +JsonParseError JsonParseStrict( + const char **const data, JsonElement **const json_out) +{ + assert(json_out != NULL); + *json_out = NULL; + + const JsonParseError error = + JsonParseDocument(NULL, NULL, true, data, json_out); + if (error != JSON_PARSE_OK) + { + return error; + } + + /* Like JsonParseAll(), the parser has left data on the last byte it + * used, so only whitespace may follow until the terminating null. */ + const char *ch = *data; + if (*ch != '\0') + { + ch++; + } + for (; *ch != '\0'; ch++) + { + if (!IsWhitespace(*ch)) + { + JsonDestroy(*json_out); + *json_out = NULL; + return JSON_PARSE_ERROR_INVALID_END; + } + } + return JSON_PARSE_OK; +} + JsonParseError JsonParseAnyFile( const char *const path, const size_t size_max, @@ -2970,6 +3068,54 @@ JsonParseError JsonParseFile( return JsonParseAnyFile(path, size_max, json_out, false); } +JsonParseError JsonParseFileStrict( + const char *const path, + const size_t size_max, + JsonElement **const json_out) +{ + assert(json_out != NULL); + *json_out = NULL; + + const int fd = safe_open(path, O_RDONLY); + if (fd == -1) + { + return JSON_PARSE_ERROR_NO_SUCH_FILE; + } + + bool truncated = false; + Writer *const contents = FileReadFromFd(fd, size_max, &truncated); + struct stat sb; + const bool have_size = (fstat(fd, &sb) == 0); + close(fd); + if (contents == NULL) + { + return JSON_PARSE_ERROR_NO_SUCH_FILE; + } + if (truncated) + { + WriterClose(contents); + return JSON_PARSE_ERROR_TRUNCATED; + } + + const char *data = StringWriterData(contents); + JsonParseError error; + if (!have_size || (size_t) sb.st_size != StringWriterLength(contents)) + { + /* The Writer is a C string: a null byte ends it early and hides + * the rest of the file from the parser, so fewer bytes arrive than + * the file holds. A file that changed while it was read is refused + * too. */ + error = JSON_PARSE_ERROR_INVALID_END; + } + else + { + error = JsonParseStrict(&data, json_out); + } + + WriterClose(contents); + return error; +} + bool JsonWalk(JsonElement *element, JsonElementVisitor object_visitor, JsonElementVisitor array_visitor, diff --git a/libutils/json.h b/libutils/json.h index fd0faa8c..ad0c0fdf 100644 --- a/libutils/json.h +++ b/libutils/json.h @@ -118,6 +118,7 @@ typedef enum JSON_PARSE_ERROR_NO_SUCH_FILE, JSON_PARSE_ERROR_NO_DATA, JSON_PARSE_ERROR_TRUNCATED, + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY, JSON_PARSE_ERROR_MAX } JsonParseError; @@ -604,6 +605,22 @@ JsonParseError JsonParse(const char **data, JsonElement **json_out); */ JsonParseError JsonParseAll(const char **data, JsonElement **json_out); +/** + @brief Parse the whole string, refusing input a lenient parse would repair + @param data [in] Pointer to the string to parse + @param json_out Resulting JSON object, NULL on error + @note In contrast to JsonParse() and JsonParseAll(), a document with two + members of the same name in one object, at any depth, is refused + with JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY before the earlier + member is replaced, rather than silently keeping the last one. + As in JsonParseAll(), trailing non-whitespace characters are refused + with JSON_PARSE_ERROR_INVALID_END. Names are compared as the parser + decodes them, which is the comparison JsonObjectGet() makes, so no + member is ever replaced. + @returns See JsonParseError and JsonParseErrorToString + */ +JsonParseError JsonParseStrict(const char **data, JsonElement **json_out); + /** @brief Parse a string to create a JsonElement @param lookup_data [in] Evaluation context for variable lookups @@ -645,6 +662,19 @@ JsonParseError JsonParseAnyFile( JsonParseError JsonParseFile( const char *path, size_t size_max, JsonElement **json_out); +/** + * @brief Parse JSON from a file as JsonParseStrict() parses a string. + * @param path Path to the file + * @param size_max Maximum size to read in memory; a larger file is refused + * with JSON_PARSE_ERROR_TRUNCATED, as in JsonParseFile() + * @param json_out Resulting JSON object, NULL on error + * @note A null byte in the file is refused with JSON_PARSE_ERROR_INVALID_END, + * as it would otherwise end the text early and hide what follows. + * @return See JsonParseError and JsonParseErrorToString + */ +JsonParseError JsonParseFileStrict( + const char *path, size_t size_max, JsonElement **json_out); + const char *JsonParseErrorToString(JsonParseError error); diff --git a/tests/unit/json_test.c b/tests/unit/json_test.c index 200478a8..762d6015 100644 --- a/tests/unit/json_test.c +++ b/tests/unit/json_test.c @@ -1662,6 +1662,288 @@ static void test_remove_key_from_object(void) JsonDestroy(object); } +/* Strict parsing: JsonParseStrict() and JsonParseFileStrict() refuse what the + * lenient functions repair, and the lenient functions stay as they were. */ + +static void assert_strict_refused( + const char *const text, const JsonParseError expected) +{ + const char *data = text; + JsonElement *json = NULL; + assert_int_equal(expected, JsonParseStrict(&data, &json)); + assert_true(json == NULL); +} + +static void test_parse_strict_valid(void) +{ + const char *data = + "{\"vars\": {\"a\": \"1\", \"b\": null}," + " \"list\": [{\"k\": 1}, {\"k\": 2}, [{\"k\": 3}]]," + " \"other\": {\"a\": \"2\"}, plain: true}"; + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); + assert_true(json != NULL); + assert_string_equal("1", JsonObjectGetAsString( + JsonObjectGetAsObject(json, "vars"), "a")); + assert_string_equal("2", JsonObjectGetAsString( + JsonObjectGetAsObject(json, "other"), "a")); + assert_int_equal(3, JsonLength(JsonObjectGet(json, "list"))); + JsonDestroy(json); + + /* Whitespace is the only thing allowed after the document. */ + data = "{} \n\t\r "; + json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); + assert_true(json != NULL); + JsonDestroy(json); + + data = "[]\n"; + json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); + JsonDestroy(json); + + data = " 12 "; + json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); + JsonDestroy(json); + + data = "\"s\""; + json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseStrict(&data, &json)); + JsonDestroy(json); +} + +static void test_parse_strict_duplicate_member(void) +{ + /* Top level, nested, in an object inside an array, unquoted names, and + * a quoted name against an unquoted one. */ + assert_strict_refused( + "{\"a\": \"1\", \"a\": \"2\"}", + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_strict_refused( + "{\"vars\": {\"x\": \"1\", \"y\": \"2\", \"x\": \"3\"}}", + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_strict_refused( + "{\"l\": [{\"k\": 1}, {\"k\": 1, \"k\": 2}]}", + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_strict_refused( + "[[{\"a\": {\"b\": [], \"b\": []}}]]", + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_strict_refused( + "{a: 1, a: 2}", JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_strict_refused( + "{a: 1, \"a\": 2}", JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + + /* Whatever the value types, however far apart the two are. */ + assert_strict_refused( + "{\"a\": {}, \"b\": 1, \"a\": []}", + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_strict_refused( + "{\"a\": true, \"a\": true}", JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); +} + +static void test_parse_strict_invalid_earlier_duplicate(void) +{ + /* The lenient parser drops the null before anything can look at it. */ + const char *const text = "{\"vars\": {\"good\": null, \"good\": \"ok\"}}"; + + assert_strict_refused(text, JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + + const char *data = text; + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); + JsonElement *vars = JsonObjectGetAsObject(json, "vars"); + assert_int_equal(1, JsonLength(vars)); + assert_string_equal("ok", JsonObjectGetAsString(vars, "good")); + JsonDestroy(json); +} + +static void test_parse_strict_repeated_section(void) +{ + const char *const text = + "{\"vars\": {\"a\": \"1\"}, \"classes\": {}," + " \"vars\": {\"b\": \"2\"}}"; + + assert_strict_refused(text, JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + + /* The lenient parser keeps only the last section. */ + const char *data = text; + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); + JsonElement *vars = JsonObjectGetAsObject(json, "vars"); + assert_true(JsonObjectGet(vars, "a") == NULL); + assert_string_equal("2", JsonObjectGetAsString(vars, "b")); + JsonDestroy(json); +} + +static void test_parse_strict_trailing_characters(void) +{ + assert_strict_refused( + "{\"vars\": {}} not-json", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("{} {}", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("{}}", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("{}\n\n x", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("[] ,", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("12 x", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("true false", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("null x", JSON_PARSE_ERROR_INVALID_END); + assert_strict_refused("\"s\" \"t\"", JSON_PARSE_ERROR_INVALID_END); + + /* An ordinary syntax error is still its own error. */ + assert_strict_refused("{\"a\": ", JSON_PARSE_ERROR_OBJECT_END); + assert_strict_refused("", JSON_PARSE_ERROR_NO_DATA); + assert_strict_refused(" \n", JSON_PARSE_ERROR_NO_DATA); + + /* The lenient functions accept the valid prefix, as before. */ + const char *data = "{\"vars\": {}} not-json"; + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); + assert_true(json != NULL); + JsonDestroy(json); + + data = "{\"vars\": {}} not-json"; + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_INVALID_END, JsonParseAll(&data, &json)); + assert_true(json == NULL); +} + +static void test_parse_lenient_duplicate_last_wins(void) +{ + const char *const text = + "{\"a\": \"1\", \"b\": {\"x\": 1, \"x\": 2}, \"a\": \"3\"," + " c: 1, c: 4}"; + + const char *data = text; + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParse(&data, &json)); + assert_int_equal(3, JsonLength(json)); + assert_string_equal("3", JsonObjectGetAsString(json, "a")); + assert_int_equal(1, JsonLength(JsonObjectGetAsObject(json, "b"))); + JsonDestroy(json); + + data = text; + json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseAll(&data, &json)); + assert_string_equal("3", JsonObjectGetAsString(json, "a")); + JsonDestroy(json); + + data = text; + json = NULL; + assert_int_equal( + JSON_PARSE_OK, JsonParseWithLookup(NULL, NULL, &data, &json)); + assert_string_equal("3", JsonObjectGetAsString(json, "a")); + JsonDestroy(json); +} + +static void test_parse_strict_error_string(void) +{ + const char *const message = + JsonParseErrorToString(JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY); + assert_true(message != NULL); + assert_true(strstr(message, "duplicate") != NULL); +} + +static void write_strict_test_file( + char *const filename, const char *const content, const size_t length) +{ + const int fd = mkstemp(filename); + assert_true(fd >= 0); + assert_int_equal(length, FullWrite(fd, content, length)); + close(fd); +} + +static JsonParseError strict_file( + const char *const content, + const size_t length, + const size_t size_max, + JsonElement **const json_out) +{ + char filename[] = "/tmp/json_test_strict_XXXXXX"; + write_strict_test_file(filename, content, length); + const JsonParseError error = + JsonParseFileStrict(filename, size_max, json_out); + unlink(filename); + return error; +} + +#define STRICT_FILE(text, size_max, json_out) \ + strict_file(text, sizeof(text) - 1, size_max, json_out) + +static void test_parse_file_strict(void) +{ + JsonElement *json = NULL; + assert_int_equal( + JSON_PARSE_OK, + STRICT_FILE("{\"vars\": {\"a\": \"1\"}}\n", 1024, &json)); + assert_string_equal( + "1", JsonObjectGetAsString(JsonObjectGetAsObject(json, "vars"), "a")); + JsonDestroy(json); + + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY, + STRICT_FILE("{\"vars\": {\"a\": null, \"a\": \"1\"}}", 1024, &json)); + assert_true(json == NULL); + + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_OBJECT_DUPLICATE_KEY, + STRICT_FILE("{\"vars\": {}, \"vars\": {}}", 1024, &json)); + assert_true(json == NULL); + + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_INVALID_END, + STRICT_FILE("{\"vars\": {}} not-json\n", 1024, &json)); + assert_true(json == NULL); + + /* A null byte must not hide what follows it from the parser. */ + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_INVALID_END, + STRICT_FILE("{\"vars\": {}}\0not-json", 1024, &json)); + assert_true(json == NULL); + + /* The size cap is kept: a larger file is refused, not cut short. */ + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_TRUNCATED, + STRICT_FILE("{\"vars\": {\"a\": \"1\"}}", 8, &json)); + assert_true(json == NULL); + + json = NULL; + assert_int_equal( + JSON_PARSE_ERROR_NO_SUCH_FILE, + JsonParseFileStrict("/nonexistent/json_test_strict", 1024, &json)); + assert_true(json == NULL); + + json = NULL; + assert_int_equal(JSON_PARSE_ERROR_NO_DATA, STRICT_FILE("", 1024, &json)); + assert_true(json == NULL); +} + +static void test_parse_file_lenient_unchanged(void) +{ + /* JsonParseFile() still repairs both, as before. */ + char filename[] = "/tmp/json_test_lenient_XXXXXX"; + const char content[] = "{\"a\": \"1\", \"a\": \"2\"} not-json\n"; + write_strict_test_file(filename, content, sizeof(content) - 1); + + JsonElement *json = NULL; + assert_int_equal(JSON_PARSE_OK, JsonParseFile(filename, 1024, &json)); + assert_string_equal("2", JsonObjectGetAsString(json, "a")); + JsonDestroy(json); + + json = NULL; + assert_int_equal( + JSON_PARSE_OK, JsonParseAnyFile(filename, 1024, &json, false)); + assert_string_equal("2", JsonObjectGetAsString(json, "a")); + JsonDestroy(json); + + unlink(filename); +} + static void test_detach_key_from_object(void) { JsonElement *object = JsonObjectCreate(3); @@ -2491,6 +2773,15 @@ int main() { PRINT_TEST_BANNER(); const UnitTest tests[] = { + unit_test(test_parse_strict_valid), + unit_test(test_parse_strict_duplicate_member), + unit_test(test_parse_strict_invalid_earlier_duplicate), + unit_test(test_parse_strict_repeated_section), + unit_test(test_parse_strict_trailing_characters), + unit_test(test_parse_lenient_duplicate_last_wins), + unit_test(test_parse_strict_error_string), + unit_test(test_parse_file_strict), + unit_test(test_parse_file_lenient_unchanged), unit_test(test_array_get_string), unit_test(test_array_iterator), unit_test(test_array_remove_range),