Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/CatDb/Data/Comparer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ public static Expression CreateComparerBody(List<Expression>? expressions, List<
var exitPoint = Expression.Label(typeof(int));
var list = new List<Expression>();
var variables = new List<ParameterExpression>();
var haveCmp = false;

if (expressions != null)
foreach (var expression in expressions)
Expand All @@ -70,14 +69,15 @@ public static Expression CreateComparerBody(List<Expression>? expressions, List<
{
var i = 0;
var cmp = Expression.Variable(typeof(int));
// The `cmp` temp is assigned by every non-last member compare (numeric, Guid, DateTime,
// char, byte[], decimal, string, …), so it must always be declared in the block scope.
// The old `haveCmp` gate only registered it for char/byte[]/decimal/string members, so a
// composite whose non-last members are e.g. Guid/int/DateTime (a non-unique index key over
// a Guid-keyed table builds Slots<field, Guid>) referenced an undeclared variable and the
// expression tree failed to compile.
variables.Add(cmp);
foreach (var field in DataTypeUtils.GetPublicMembers(x.Type, membersOrder))
{
if (!haveCmp && (field.GetPropertyOrFieldType() == typeof(char) || field.GetPropertyOrFieldType() == typeof(byte[]) || field.GetPropertyOrFieldType() == typeof(Decimal) || field.GetPropertyOrFieldType() == typeof(String)))
{
haveCmp = true;
variables.Add(cmp);
}

foreach (var command in GetCompareCommands(Expression.PropertyOrField(x, field.Name), Expression.PropertyOrField(y, field.Name), cmp, exitPoint, field.GetPropertyOrFieldType(), compareOptions[i++], i == DataTypeUtils.GetPublicMembers(x.Type, membersOrder).Count()))
list.Add(command);
}
Expand Down
25 changes: 25 additions & 0 deletions src/CatDb/Data/DataTypeUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,31 @@ public static bool IsAllPrimitive(Type type, Func<Type, MemberInfo, int>? member
return true;
}

/// <summary>
/// Like <see cref="IsAllPrimitive"/> but also treats <see cref="Guid"/> as a comparable scalar.
/// Used to decide whether a type (e.g. a composite Slots index key) can get a compiled
/// <c>DataComparer</c>/<c>DataEqualityComparer</c>. A non-unique index over a Guid-keyed table
/// builds a composite key <c>Slots(field, Guid)</c>; that Guid slot is not "primitive", so
/// <see cref="IsAllPrimitive"/> returns false and the key would otherwise get no comparer.
/// </summary>
public static bool IsAllComparable(Type type, Func<Type, MemberInfo, int>? membersOrder = null)
{
if (DataType.IsPrimitiveType(type) || type == typeof(Guid))
return true;

if (type.IsArray || type.IsList() || type.IsDictionary() || type.IsKeyValuePair() || type.IsNullable())
return false;

foreach (var member in GetPublicMembers(type, membersOrder))
{
var memberType = member.GetPropertyOrFieldType();
if (!DataType.IsPrimitiveType(memberType) && memberType != typeof(Guid))
return false;
}

return true;
}

private static bool InternalIsAnonymousType(Type type, Func<Type, MemberInfo, int>? membersOrder = null)
{
if (DataType.IsPrimitiveType(type))
Expand Down
70 changes: 69 additions & 1 deletion src/CatDb/Database/Indexing/SlotAccessor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,16 @@ internal static Type NormalizeStorageType(Type t)
if (t.IsEnum)
return t.GetEnumUnderlyingType();
var underlying = Nullable.GetUnderlyingType(t);
return underlying is not null ? NormalizeStorageType(underlying) : t;
if (underlying is not null)
return NormalizeStorageType(underlying);
// Remote/portable tables transport a Nullable<T> as a single-slot Slots<T> (a CLR type the
// Nullable check above can't see). Flatten it to the inner storage type so the index key stays
// a flat scalar/composite — otherwise the key type is a NESTED Slots(Slots<T>, pk) whose
// comparer can't be built (null KeyComparer → NRE on insert → silently dropped rows). See
// RemoteInsertReproTests.
if (TryGetSingleSlotField(t, out var slot0))
return NormalizeStorageType(slot0.FieldType);
return t;
}

/// <summary>Wraps a value expression so it yields the normalized storage value (see <see cref="NormalizeStorageType"/>).</summary>
Expand All @@ -294,16 +303,75 @@ private static Expression NormalizeStorageValue(Expression access)
return underlying.IsEnum ? Expression.Convert(value, underlying.GetEnumUnderlyingType()) : value;
}

// Portable Nullable<T> representation (single-slot Slots<T>): read the inner slot, then
// normalize it. Matches NormalizeStorageType's flattening above. The Slots<T> wrapper is a
// reference type, so a null Nullable<T> arrives as a NULL wrapper — guard it to default(inner)
// (same null → default(T) contract as the CLR Nullable path above), else extraction NREs and
// the whole write batch is dropped.
if (TryGetSingleSlotField(t, out var slot0))
{
var inner = NormalizeStorageValue(Expression.Field(access, slot0));
if (!t.IsValueType)
return Expression.Condition(
Expression.ReferenceEqual(access, Expression.Constant(null, t)),
Expression.Default(inner.Type),
inner);
return inner;
}

return access;
}

/// <summary>
/// True when <paramref name="t"/> is the portable representation of a <see cref="Nullable{T}"/>
/// (or any single-field composite): a one-slot <c>ISlots</c>. Multi-slot composites (real composite
/// index keys) and non-Slots types return false so only the nullable wrapper is unwrapped.
/// </summary>
private static bool TryGetSingleSlotField(Type t, out FieldInfo slot0)
{
slot0 = null!;
if (!typeof(ISlots).IsAssignableFrom(t))
return false;
if (t.GetField("Slot1") != null) // 2+ slots → genuine composite, leave intact
return false;
var f = t.GetField("Slot0");
if (f == null)
return false;
slot0 = f;
return true;
}

/// <summary>
/// If <paramref name="t"/> is the portable representation of a <see cref="Nullable{T}"/> over a
/// value type (a single-slot <c>Slots&lt;T&gt;</c>), returns <c>Nullable&lt;T&gt;</c> — the CLR type
/// a remote client serialized the field value with (typeof(TField)). Lets the server keep the
/// RemoteFieldCodec symmetric instead of calling <c>.PrimitiveType</c> on the non-primitive wrapper.
/// </summary>
internal static bool TryGetPortableNullableType(Type t, out Type nullableType)
{
nullableType = null!;
if (!TryGetSingleSlotField(t, out var slot0))
return false;
var inner = slot0.FieldType;
if (!inner.IsValueType || Nullable.GetUnderlyingType(inner) != null)
return false;
nullableType = typeof(Nullable<>).MakeGenericType(inner);
return true;
}

/// <summary>
/// Builds a boxed-value normalizer for query inputs: a user-supplied field value of the declared
/// member type (e.g. an enum literal, or a Nullable&lt;DateTime&gt;) is converted to the index's
/// normalized storage representation. Identity (no allocation) when no normalization is needed.
/// </summary>
internal static Func<IData, IData> BuildValueNormalizer(Type rawType)
{
// Portable Nullable<T> raw type (single-slot Slots<T>): remote query values arrive already
// flattened to the inner T (the client boxes/serializes the underlying value, never the
// wrapper), so normalize as the inner type. Casting the incoming T to Slots<T> would throw.
if (TryGetSingleSlotField(rawType, out var slot0))
return BuildValueNormalizer(slot0.FieldType);

if (NormalizeStorageType(rawType) == rawType)
return v => v;

Expand Down
11 changes: 10 additions & 1 deletion src/CatDb/Database/Indexing/TableIndexManager.Query.cs
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@
return ScanPrefixKeys(entry, prefixValue, prefixFieldCount, backward);
}

bool IQueryEngineContext.TryFetch(IData pk, out IData record) => _table.TryGet(pk, out record);

Check warning on line 105 in src/CatDb/Database/Indexing/TableIndexManager.Query.cs

View workflow job for this annotation

GitHub Actions / build-and-test

Possible null reference assignment.

IEnumerable<KeyValuePair<IData, IData>> IQueryEngineContext.ScanRows(
IData? from, bool hasFrom, IData? to, bool hasTo, bool backward)
Expand Down Expand Up @@ -137,7 +137,16 @@
{
var slot = ResolveSlotIndices([member])[0];
var dt = _locator.RecordDataType;
return dt.IsPrimitive ? dt.PrimitiveType : dt[slot].PrimitiveType;
var slotDt = dt.IsPrimitive ? dt : dt[slot];
if (slotDt.IsPrimitive)
return slotDt.PrimitiveType;

// Non-primitive slot: on a portable/remote table a Nullable<T> field is carried as a
// single-slot Slots<T>. Reconstruct Nullable<T> so RemoteFieldCodec decodes the value with the
// same CLR type the client encoded it with (typeof(TField)); otherwise `.PrimitiveType` throws
// "The type (T) is not primitive" on the wrapper.
var clr = DataTypeUtils.BuildType(slotDt);
return SlotAccessor.TryGetPortableNullableType(clr, out var nullable) ? nullable : clr;
}

/// <summary>CLR type of the primary key (for remote key-range decoding).</summary>
Expand Down
6 changes: 4 additions & 2 deletions src/CatDb/WaterfallTree/TypeEngine.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ private static TypeEngine Create(Type type)
Persist = new DataPersist(type, null, AllowNull.OnlyMembers)
};

if (DataTypeUtils.IsAllPrimitive(type) || type == typeof(Guid))
if (DataTypeUtils.IsAllComparable(type))
{
descriptor.Comparer = new DataComparer(type);
descriptor.EqualityComparer = new DataEqualityComparer(type);

if (type != typeof(Guid))
// IndexerPersist only for pure primitives — Guid (bare or as a composite slot,
// e.g. a non-unique index key over a Guid-keyed table) has no DataIndexerPersist.
if (DataTypeUtils.IsAllPrimitive(type) && type != typeof(Guid))
descriptor.IndexerPersist = new DataIndexerPersist(type);
}

Expand Down
47 changes: 47 additions & 0 deletions tests/CatDb.Tests/Database/GuidPkNonUniqueReproTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Repro for reported: NonUnique index + Guid primary key -> NRE in OrderedSet.FindIndexes
using CatDb.Database;
using CatDb.Database.Indexing;
using CatDb.Extensions;
using FluentAssertions;

namespace CatDb.Tests.Database;

public class GuidPkNonUniqueReproTests : IDisposable
{
private readonly IStorageEngine _engine = CatDb.Database.CatDb.FromMemory();
public void Dispose() => _engine.Dispose();

public class Entity
{
public Guid AppId { get; set; }
public string City { get; set; } = "";
public int Age { get; set; }
}

[Fact]
public void GuidPk_NonUnique_StringField()
{
var t = _engine.OpenXTable<Guid, Entity>("g_str");
t.CreateIndex("City", e => e.City, IndexType.NonUnique);
for (int i = 0; i < 50; i++)
t.Replace(Guid.NewGuid(), new Entity { AppId = Guid.NewGuid(), City = i % 2 == 0 ? "NYC" : "LA", Age = i });
_engine.Commit();

var res = t.Query(x => x.City).Equal("NYC").ToList();
res.Should().HaveCount(25);
}

[Fact]
public void GuidPk_NonUnique_GuidField()
{
var t = _engine.OpenXTable<Guid, Entity>("g_guid");
t.CreateIndex("AppId", e => e.AppId, IndexType.NonUnique);
var app = Guid.NewGuid();
for (int i = 0; i < 20; i++)
t.Replace(Guid.NewGuid(), new Entity { AppId = i < 10 ? app : Guid.NewGuid(), City = "X", Age = i });
_engine.Commit();

var res = t.Query(x => x.AppId).Equal(app).ToList();
res.Should().HaveCount(10);
}
}
79 changes: 79 additions & 0 deletions tests/CatDb.Tests/Database/RemoteInsertReproTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// PRE-EXISTING BUG (documented, not yet fixed) — remote/portable + Nullable<T> secondary index.
//
// A remote table's records reach the server in its PORTABLE form (StorageEngineServer opens
// OpenXTablePortable(name, KeyDataType, RecordDataType) — record type built from the wire schema,
// NOT the CLR record type). A Nullable<T> field transports as a nested Slots(T). The index layer's
// NormalizeStorageType only strips a CLR Nullable<T> (Nullable.GetUnderlyingType), NOT the portable
// Slots<T>, so a non-unique index over a nullable field builds a malformed composite key
// Slots(Slots<T>, pk). Inserting through that index then SILENTLY drops most main-table records
// (no exception): after 100 remote Replace + Commit only ~2 rows survive server-side.
//
// Confirmed scope: non-nullable DateTime index works (100). Enum index works (normalizes to a flat
// integral). ONLY Nullable<T> index fields lose records, independent of the values (null / non-null /
// distinct all fail). Embedded (typed) tables are unaffected.
//
// These tests are Skip-marked so the suite stays green; un-skip when fixing the portable-nullable path.
using System.Net;
using System.Net.Sockets;
using CatDb.Database;
using CatDb.Database.Indexing;
using CatDb.Remote;
using CatDb.General.Communication;
using FluentAssertions;

namespace CatDb.Tests.Database;

public class RemoteInsertReproTests
{
private const string BugSkip =
"Pre-existing: remote/portable Nullable<T> secondary index silently drops main-table records.";

public enum St { A = 0, B = 1, C = 2 }
public class Rec { public Guid Id { get; set; } public St Status { get; set; } public DateTime? ExpiresAt { get; set; } }
public class Rec2 { public Guid Id { get; set; } public DateTime When { get; set; } }

private static int FreePort()
{
var l = new TcpListener(IPAddress.Loopback, 0); l.Start();
var p = ((IPEndPoint)l.LocalEndpoint).Port; l.Stop(); return p;
}

private static async Task<long> RemoteInsertCount<TRec>(Func<long, TRec> make,
Action<ITable<long, TRec>>? createIndexes = null)
{
var port = FreePort();
using var serverEngine = CatDb.Database.CatDb.FromMemory();
await using var tcp = new TcpServer(port);
var server = new StorageEngineServer(serverEngine, tcp, accessPolicy: null);
await server.StartAsync();
try
{
using var client = CatDb.Database.CatDb.FromNetwork("localhost", port, "default", "u", "p");
var t = client.OpenXTable<long, TRec>("t");
createIndexes?.Invoke(t);
for (long i = 0; i < 100; i++) t.Replace(i, make(i));
client.Commit();
return t.Count();
}
finally { await server.StopAsync(); }
}

[Fact]
public async Task Remote_Insert_NoIndex_GuidEnumNullable()
=> (await RemoteInsertCount(i => new Rec { Id = Guid.NewGuid(), Status = (St)(i % 3), ExpiresAt = i % 4 == 0 ? null : DateTime.UtcNow })).Should().Be(100);

[Fact]
public async Task Remote_Insert_EnumIndex()
=> (await RemoteInsertCount(i => new Rec { Id = Guid.NewGuid(), Status = (St)(i % 3), ExpiresAt = DateTime.UtcNow },
t => t.CreateIndex("Status", r => r.Status, IndexType.NonUnique))).Should().Be(100);

[Fact]
public async Task Remote_Insert_NonNullableDateTimeIndex()
=> (await RemoteInsertCount(i => new Rec2 { Id = Guid.NewGuid(), When = new DateTime(2026, 1, 1).AddDays(i) },
t => t.CreateIndex("When", r => r.When, IndexType.NonUnique))).Should().Be(100);

[Fact]
public async Task Remote_Insert_NullableIndex()
=> (await RemoteInsertCount(i => new Rec { Id = Guid.NewGuid(), Status = St.A, ExpiresAt = new DateTime(2026, 1, 1).AddDays(i) },
t => t.CreateIndex("ExpiresAt", r => r.ExpiresAt, IndexType.NonUnique))).Should().Be(100);
}
55 changes: 55 additions & 0 deletions tests/CatDb.Tests/Database/RemoteNullableQueryTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
using System.Net;
using System.Net.Sockets;
using CatDb.Database;
using CatDb.Database.Indexing;
using CatDb.Extensions;
using CatDb.Remote;
using CatDb.General.Communication;
using FluentAssertions;

namespace CatDb.Tests.Database;

public class RemoteNullableQueryTests
{
public class Item { public Guid Id { get; set; } public DateTime? ExpiresAt { get; set; } }

private static int FreePort()
{
var l = new TcpListener(IPAddress.Loopback, 0); l.Start();
var p = ((IPEndPoint)l.LocalEndpoint).Port; l.Stop(); return p;
}

[Fact]
public async Task Remote_NullableIndex_DirectApi_And_Query()
{
var port = FreePort();
using var serverEngine = CatDb.Database.CatDb.FromMemory();
await using var tcp = new TcpServer(port);
var server = new StorageEngineServer(serverEngine, tcp, accessPolicy: null);
await server.StartAsync();
try
{
using var client = CatDb.Database.CatDb.FromNetwork("localhost", port, "default", "u", "p");
var t = client.OpenXTable<long, Item>("items");
t.CreateIndex("ExpiresAt", r => r.ExpiresAt, IndexType.NonUnique);

var baseTime = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
for (long i = 0; i < 100; i++)
t.Replace(i, new Item { Id = Guid.NewGuid(), ExpiresAt = i % 4 == 0 ? null : baseTime.AddDays(i) });
client.Commit();

t.Count().Should().Be(100);

// direct index API
var d5 = baseTime.AddDays(5);
t.Indexes.FindByIndex("ExpiresAt", d5).Count().Should().Be(1);
t.Indexes.CountByIndex("ExpiresAt", d5).Should().Be(1);

// general query API (the path the stress test uses)
t.Query(x => x.ExpiresAt).Equal(d5).Count().Should().Be(1);
t.Query(x => x.ExpiresAt).AtLeast(baseTime.AddDays(10)).AtMost(baseTime.AddDays(50)).Count()
.Should().Be(Enumerable.Range(0, 100).Count(i => i % 4 != 0 && baseTime.AddDays(i) >= baseTime.AddDays(10) && baseTime.AddDays(i) <= baseTime.AddDays(50)));
}
finally { await server.StopAsync(); }
}
}
Loading