Skip to content

feat!: Make schema parser reuse the JSON allocations #656

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Kriskras99 merged 5 commits into from
Sep 7, 2026
Merged

feat!: Make schema parser reuse the JSON allocations #656

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f6d4c0b
feat!: Make schema parser reuse the JSON allocations
Kriskras99 Sep 4, 2026
4a8f9bd
Merge branch 'main' into feat/improve_schema_parser
martin-g Sep 5, 2026
b4187d1
fix build
martin-g Sep 5, 2026
935953c
fix: Wrong error message and unnecessary clone
Kriskras99 Sep 6, 2026
2a938ae
fix: Small fixes and extra tests (from @JosephLenton)
Kriskras99 Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions avro/src/error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,9 @@ pub enum Details {
#[error("No `name` field")]
GetNameField,

#[error("Expected a string for the `namespace` field, got a {0}")]
GetNamespaceFieldWrongType(&'static str),

#[error("No `name` in record field")]
GetNameFieldFromRecord,

Expand Down Expand Up @@ -431,9 +434,21 @@ pub enum Details {
#[error("No `fields` in record")]
GetRecordFieldsJson,

#[error("Expected an object in the array of the `fields` field, got a {0}")]
GetRecordFieldsArrayInvalidType(&'static str),

#[error("Expected an array of objects for the `fields` field, got a {0}")]
GetRecordFieldsInvalidType(&'static str),

#[error("No `symbols` field in enum")]
GetEnumSymbolsField,

#[error("Expected an array of strings for the `symbols` field, got a {0}")]
GetEnumSymbolsFieldInvalidType(&'static str),

#[error("Expected a string in the array of the `symbols` field, got a {0}")]
GetEnumSymbolsFieldArrayInvalidType(&'static str),

#[error("Unable to parse `symbols` in enum")]
GetEnumSymbols,

Expand Down Expand Up @@ -487,6 +502,18 @@ pub enum Details {
#[error("Fixed schema has no `size`")]
GetFixedSizeField,

#[error("Expected an unsigned integer for the `size` field, got a {0}")]
GetFixedSizeFieldInvalidType(&'static str),

#[error("Expected an array of strings for the `aliases` field, got a {0}")]
GetAliasesFieldInvalidType(&'static str),

#[error("Expected a string in the array for the `aliases` field, got a {0}")]
GetAliasesFieldArrayInvalidType(&'static str),

#[error("Expected a string for the `{0}` field, got a {1}")]
GetStringInvalidType(&'static str, &'static str),

#[deprecated(since = "0.22.0", note = "This error variant is not generated anymore")]
#[error("Fixed schema's default value length ({0}) does not match its size ({1})")]
FixedDefaultLenSizeMismatch(usize, u64),
Expand Down
4 changes: 2 additions & 2 deletions avro/src/reader/block.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,9 +288,9 @@ impl<'r, R: Read> Block<'r, R> {
&HashMap::new(),
)?;
self.names_refs = names.into_iter().map(|(n, s)| (n, s.clone())).collect();
self.writer_schema = Schema::parse_with_names(&json, self.names_refs.clone())?;
self.writer_schema = Schema::parse_with_names(json, self.names_refs.clone())?;
} else {
self.writer_schema = Schema::parse(&json)?;
self.writer_schema = Schema::parse(json)?;
let mut names = HashMap::new();
resolve_names(&self.writer_schema, &mut names, None, &HashMap::new())?;
self.names_refs = names.into_iter().map(|(n, s)| (n, s.clone())).collect();
Expand Down
205 changes: 187 additions & 18 deletions avro/src/schema/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,16 @@ impl Schema {
let json = json.as_ref();
let schema: JsonValue = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?;
if let JsonValue::Object(inner) = &schema {
let name = Name::parse(inner, None)?;
// Only clone the values needed for the name parsing, can be a significant time/memory
// save on large schemas
let mut name_json = Map::with_capacity(2);
if let Some(v) = inner.get("name") {
name_json.insert("name".into(), v.clone());
}
if let Some(v) = inner.get("namespace") {
name_json.insert("namespace".into(), v.clone());
}
let name = Name::parse(&mut name_json, None)?;
let previous_value = input_schemas.insert(name.clone(), schema);
if previous_value.is_some() {
return Err(Details::NameCollision(name.fullname(None)).into());
Expand Down Expand Up @@ -588,7 +597,16 @@ impl Schema {
let json = json.as_ref();
let schema: JsonValue = serde_json::from_str(json).map_err(Details::ParseSchemaJson)?;
if let JsonValue::Object(inner) = &schema {
let name = Name::parse(inner, None)?;
// Only clone the values needed for the name parsing, can be a significant time/memory
// save on large schemas
let mut name_json = Map::with_capacity(2);
if let Some(v) = inner.get("name") {
name_json.insert("name".into(), v.clone());
}
if let Some(v) = inner.get("namespace") {
name_json.insert("namespace".into(), v.clone());
}
let name = Name::parse(&mut name_json, None)?;
if let Some(_previous) = input_schemas.insert(name.clone(), schema) {
return Err(Details::NameCollision(name.fullname(None)).into());
}
Expand All @@ -605,7 +623,7 @@ impl Schema {
parser.parse_input_schemas()?;

let value = serde_json::from_str(schema).map_err(Details::ParseSchemaJson)?;
let schema = parser.parse(&value, None)?;
let schema = parser.parse(value, None)?;
let schemata = parser.parse_list()?;
Ok((schema, schemata))
}
Expand All @@ -620,14 +638,14 @@ impl Schema {
}

/// Parses an Avro schema from JSON.
pub fn parse(value: &JsonValue) -> AvroResult<Schema> {
pub fn parse(value: JsonValue) -> AvroResult<Schema> {
let mut parser = Parser::default();
parser.parse(value, None)
}

/// Parses an Avro schema from JSON.
/// Any `Schema::Ref`s must be known in the `names` map.
pub(crate) fn parse_with_names(value: &JsonValue, names: Names) -> AvroResult<Schema> {
pub(crate) fn parse_with_names(value: JsonValue, names: Names) -> AvroResult<Schema> {
let mut parser = Parser::new(HashMap::with_capacity(1), Vec::with_capacity(1), names);
parser.parse(value, None)
}
Expand Down Expand Up @@ -3094,7 +3112,7 @@ mod tests {
]
});

let parse_result = Schema::parse(&schema);
let parse_result = Schema::parse(schema);
assert!(
parse_result.is_ok(),
"parse result must be ok, got: {parse_result:?}"
Expand Down Expand Up @@ -3341,7 +3359,7 @@ mod tests {
}
);

let parse_result = Schema::parse(&schema);
let parse_result = Schema::parse(schema);
assert!(
parse_result.is_ok(),
"parse result must be ok, got: {parse_result:?}"
Expand Down Expand Up @@ -3631,7 +3649,7 @@ mod tests {
};

// Serialize using the writer schema.
let writer_schema = Schema::parse(&writer_schema)?;
let writer_schema = Schema::parse(writer_schema)?;
let avro_value = crate::to_value(s)?;
assert!(
avro_value.validate(&writer_schema),
Expand All @@ -3642,7 +3660,7 @@ mod tests {
.write_value_to_vec(avro_value)?;

// Now, attempt to deserialize using the reader schema.
let reader_schema = Schema::parse(&reader_schema)?;
let reader_schema = Schema::parse(reader_schema)?;
let mut x = &datum[..];

// Deserialization should succeed and we should be able to resolve the schema.
Expand Down Expand Up @@ -4328,7 +4346,7 @@ mod tests {
"precision": 9,
"scale": 2
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert!(matches!(
parse_result,
Schema::Decimal(DecimalSchema {
Expand All @@ -4345,7 +4363,7 @@ mod tests {
"name": "LongDecimal",
"logicalType": "decimal"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
// assert!(matches!(parse_result, Schema::Long));
assert_eq!(parse_result, Schema::Long);

Expand All @@ -4361,7 +4379,7 @@ mod tests {
"name": "StringUUID",
"logicalType": "uuid"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert_eq!(parse_result, Schema::Uuid(UuidSchema::String));

Ok(())
Expand All @@ -4376,7 +4394,7 @@ mod tests {
"size": 16,
"logicalType": "uuid"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert_eq!(
parse_result,
Schema::Uuid(UuidSchema::Fixed(FixedSchema {
Expand All @@ -4402,7 +4420,7 @@ mod tests {
"name": "BytesUUID",
"logicalType": "uuid"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert_eq!(parse_result, Schema::Uuid(UuidSchema::Bytes));

Ok(())
Expand All @@ -4417,7 +4435,7 @@ mod tests {
"size": 6,
"logicalType": "uuid"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;

assert_eq!(
parse_result,
Expand Down Expand Up @@ -4445,7 +4463,7 @@ mod tests {
"name": "LongTimestampMillis",
"logicalType": "timestamp-millis"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert_eq!(parse_result, Schema::TimestampMillis);

// int timestamp-millis, represents as native complex type.
Expand All @@ -4455,7 +4473,7 @@ mod tests {
"name": "IntTimestampMillis",
"logicalType": "timestamp-millis"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert_eq!(parse_result, Schema::Int);

Ok(())
Expand All @@ -4470,7 +4488,7 @@ mod tests {
"name": "BytesLog",
"logicalType": "custom"
});
let parse_result = Schema::parse(&schema)?;
let parse_result = Schema::parse(schema)?;
assert_eq!(parse_result, Schema::Bytes);
assert_eq!(parse_result.custom_attributes(), None);

Expand Down Expand Up @@ -5234,4 +5252,155 @@ mod tests {

Ok(())
}

#[test]
fn avro_rs_654_unknown_logical_type_must_survive_roundtrip() -> TestResult {
let schema_str = r#"{
"type": "array",
"logicalType": "map",
"items": {
"type": "record",
"name": "k12_v13",
"fields": [
{
"name": "key",
"type": "int",
"field-id": 12
},
{
"name": "value",
"type": "string",
"field-id": 13
}
]
}
}"#;
let Schema::Array(schema) = Schema::parse_str(schema_str)? else {
panic!("This must be an array schema")
};
assert_eq!(
schema.attributes.get("logicalType").unwrap(),
&serde_json::Value::String("map".into())
);

let schema_str_2 = serde_json::to_string(&Schema::Array(schema))?;

let Schema::Array(schema) = Schema::parse_str(&schema_str_2)? else {
panic!("This must be an array schema")
};
assert_eq!(
schema.attributes.get("logicalType").unwrap(),
&serde_json::Value::String("map".into())
);

Ok(())
}

#[test]
fn avro_rs_654_preserve_unknown_logical_type_on_outer_item() -> TestResult {
let raw_schema = r#"{
"type": "array",
"logicalType": "blub",
"items": {
"type": "record",
"name": "k12_v13",
"fields": [
{
"name": "key",
"type": "int",
"field-id": 12
},
{
"name": "value",
"type": "string",
"field-id": 13
}
]
}
}"#;

let schema = Schema::parse_str(raw_schema)?;

let output = serde_json::to_string_pretty(&schema).unwrap();
pretty_assertions::assert_eq!(
r#"{
"type": "array",
"items": {
"type": "record",
"name": "k12_v13",
"fields": [
{
"name": "key",
"type": "int",
"field-id": 12
},
{
"name": "value",
"type": "string",
"field-id": 13
}
]
},
"logicalType": "blub"
}"#,
output
);

let logical_type = schema.custom_attributes().unwrap().get("logicalType");
assert_eq!(
logical_type,
Some(&serde_json::Value::String("blub".to_string()))
);

Ok(())
}

#[test]
fn avro_rs_654_preserve_unknown_logical_type_on_inner_item() -> TestResult {
let raw_schema = r#"{
"type": "record",
"name": "test_record",
"fields": [
{
"name": "example_map",
"type": {
"type": "array",
"logicalType": "fish",
"items": {
"type": "record",
"name": "k12_v13",
"fields": [
{
"name": "key",
"type": "int",
"field-id": 12
},
{
"name": "value",
"type": "string",
"field-id": 13
}
]
}
}
}
]
}"#;

let schema = Schema::parse_str(raw_schema)?;
let Schema::Record(record) = &schema else {
panic!("Expected a record schema");
};
let example_map_schema = &record.fields[0].schema;
let logical_type = example_map_schema
.custom_attributes()
.unwrap()
.get("logicalType");
assert_eq!(
logical_type,
Some(&serde_json::Value::String("fish".to_string()))
);

Ok(())
}
}
Loading
Loading