diff --git a/Directory.Build.props b/Directory.Build.props index 9f512d5e9..42de5875c 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -10,7 +10,7 @@ true $(MSBuildThisFileDirectory)Shared.ruleset NETSDK1069 - NU5105;NU1507 + $(NoWarn);NU5105;NU1507;SER001 https://stackexchange.github.io/StackExchange.Redis/ReleaseNotes https://stackexchange.github.io/StackExchange.Redis/ MIT diff --git a/Directory.Packages.props b/Directory.Packages.props index 79c404dc2..df8c078a3 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,6 +8,10 @@ + + + + @@ -23,6 +27,7 @@ + diff --git a/StackExchange.Redis.sln b/StackExchange.Redis.sln index 8f772ae42..2ed4ebfb3 100644 --- a/StackExchange.Redis.sln +++ b/StackExchange.Redis.sln @@ -122,6 +122,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs", "docs\docs.csproj", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StackExchange.Redis.Benchmarks", "tests\StackExchange.Redis.Benchmarks\StackExchange.Redis.Benchmarks.csproj", "{59889284-FFEE-82E7-94CB-3B43E87DA6CF}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{5FA0958E-6EBD-45F4-808E-3447A293F96F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StackExchange.Redis.Build", "eng\StackExchange.Redis.Build\StackExchange.Redis.Build.csproj", "{190742E1-FA50-4E36-A8C4-88AE87654340}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -180,6 +184,10 @@ Global {59889284-FFEE-82E7-94CB-3B43E87DA6CF}.Debug|Any CPU.Build.0 = Debug|Any CPU {59889284-FFEE-82E7-94CB-3B43E87DA6CF}.Release|Any CPU.ActiveCfg = Release|Any CPU {59889284-FFEE-82E7-94CB-3B43E87DA6CF}.Release|Any CPU.Build.0 = Release|Any CPU + {190742E1-FA50-4E36-A8C4-88AE87654340}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {190742E1-FA50-4E36-A8C4-88AE87654340}.Debug|Any CPU.Build.0 = Debug|Any CPU + {190742E1-FA50-4E36-A8C4-88AE87654340}.Release|Any CPU.ActiveCfg = Release|Any CPU + {190742E1-FA50-4E36-A8C4-88AE87654340}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -202,6 +210,7 @@ Global {A0F89B8B-32A3-4C28-8F1B-ADE343F16137} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A} {69A0ACF2-DF1F-4F49-B554-F732DCA938A3} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A} {59889284-FFEE-82E7-94CB-3B43E87DA6CF} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A} + {190742E1-FA50-4E36-A8C4-88AE87654340} = {5FA0958E-6EBD-45F4-808E-3447A293F96F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {193AA352-6748-47C1-A5FC-C9AA6B5F000B} diff --git a/StackExchange.Redis.sln.DotSettings b/StackExchange.Redis.sln.DotSettings index de893e54d..b72a49d2c 100644 --- a/StackExchange.Redis.sln.DotSettings +++ b/StackExchange.Redis.sln.DotSettings @@ -1,4 +1,5 @@  OK PONG - True \ No newline at end of file + True + True \ No newline at end of file diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index c782d3f5a..a3e85a875 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -13,6 +13,7 @@ Current package versions: - Add `Condition.SortedSet[Not]ContainsStarting` condition for transactions ([#2638 by ArnoKoll](https://github.com/StackExchange/StackExchange.Redis/pull/2638)) - Add support for XPENDING Idle time filter ([#2822 by david-brink-talogy](https://github.com/StackExchange/StackExchange.Redis/pull/2822)) - Improve `double` formatting performance on net8+ ([#2928 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2928)) +- Add vector-set support ([#2939 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2939)) - Add `GetServer(RedisKey, ...)` API ([#2936 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2936)) - Fix error constructing `StreamAdd` message ([#2941 by mgravell](https://github.com/StackExchange/StackExchange.Redis/pull/2941)) diff --git a/docs/exp/SER001.md b/docs/exp/SER001.md new file mode 100644 index 000000000..2def8be6e --- /dev/null +++ b/docs/exp/SER001.md @@ -0,0 +1,22 @@ +At the current time, [Redis documents that](https://redis.io/docs/latest/commands/vadd/): + +> Vector set is a new data type that is currently in preview and may be subject to change. + +As such, the corresponding library feature must also be considered subject to change: + +1. Existing bindings may cease working correctly if the underlying server API changes. +2. Changes to the server API may require changes to the library API, manifesting in either/both of build-time + or run-time breaks. + +While this seems *unlikely*, it must be considered a possibility. If you acknowledge this, you can suppress +this warning by adding the following to your `csproj` file: + +```xml +$(NoWarn);SER001 +``` + +or more granularly / locally in C#: + +``` c# +#pragma warning disable SER001 +``` \ No newline at end of file diff --git a/eng/StackExchange.Redis.Build/FastHashGenerator.cs b/eng/StackExchange.Redis.Build/FastHashGenerator.cs new file mode 100644 index 000000000..cdbc94ebe --- /dev/null +++ b/eng/StackExchange.Redis.Build/FastHashGenerator.cs @@ -0,0 +1,215 @@ +using System.Buffers; +using System.Collections.Immutable; +using System.Reflection; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace StackExchange.Redis.Build; + +[Generator(LanguageNames.CSharp)] +public class FastHashGenerator : IIncrementalGenerator +{ + public void Initialize(IncrementalGeneratorInitializationContext context) + { + var literals = context.SyntaxProvider + .CreateSyntaxProvider(Predicate, Transform) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + context.RegisterSourceOutput(literals, Generate); + } + + private bool Predicate(SyntaxNode node, CancellationToken cancellationToken) + { + // looking for [FastHash] partial static class Foo { } + if (node is ClassDeclarationSyntax decl + && decl.Modifiers.Any(SyntaxKind.StaticKeyword) + && decl.Modifiers.Any(SyntaxKind.PartialKeyword)) + { + foreach (var attribList in decl.AttributeLists) + { + foreach (var attrib in attribList.Attributes) + { + if (attrib.Name.ToString() is "FastHashAttribute" or "FastHash") return true; + } + } + } + + return false; + } + + private static string GetName(INamedTypeSymbol type) + { + if (type.ContainingType is null) return type.Name; + var stack = new Stack(); + while (true) + { + stack.Push(type.Name); + if (type.ContainingType is null) break; + type = type.ContainingType; + } + var sb = new StringBuilder(stack.Pop()); + while (stack.Count != 0) + { + sb.Append('.').Append(stack.Pop()); + } + return sb.ToString(); + } + + private (string Namespace, string ParentType, string Name, string Value) Transform( + GeneratorSyntaxContext ctx, + CancellationToken cancellationToken) + { + // extract the name and value (defaults to name, but can be overridden via attribute) and the location + if (ctx.SemanticModel.GetDeclaredSymbol(ctx.Node) is not INamedTypeSymbol named) return default; + string ns = "", parentType = ""; + if (named.ContainingType is { } containingType) + { + parentType = GetName(containingType); + ns = containingType.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + } + else if (named.ContainingNamespace is { } containingNamespace) + { + ns = containingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + } + + string name = named.Name, value = ""; + foreach (var attrib in named.GetAttributes()) + { + if (attrib.AttributeClass?.Name == "FastHashAttribute") + { + if (attrib.ConstructorArguments.Length == 1) + { + if (attrib.ConstructorArguments[0].Value?.ToString() is { Length: > 0 } val) + { + value = val; + break; + } + } + } + } + + if (string.IsNullOrWhiteSpace(value)) + { + value = name.Replace("_", "-"); // if nothing explicit: infer from name + } + + return (ns, parentType, name, value); + } + + private string GetVersion() + { + var asm = GetType().Assembly; + if (asm.GetCustomAttributes(typeof(AssemblyFileVersionAttribute), false).FirstOrDefault() is + AssemblyFileVersionAttribute { Version: { Length: > 0 } } version) + { + return version.Version; + } + + return asm.GetName().Version?.ToString() ?? "??"; + } + + private void Generate( + SourceProductionContext ctx, + ImmutableArray<(string Namespace, string ParentType, string Name, string Value)> literals) + { + if (literals.IsDefaultOrEmpty) return; + + var sb = new StringBuilder("// ") + .AppendLine().Append("// ").Append(GetType().Name).Append(" v").Append(GetVersion()).AppendLine(); + + // lease a buffer that is big enough for the longest string + var buffer = ArrayPool.Shared.Rent( + Encoding.UTF8.GetMaxByteCount(literals.Max(l => l.Value.Length))); + int indent = 0; + + StringBuilder NewLine() => sb.AppendLine().Append(' ', indent * 4); + NewLine().Append("using System;"); + NewLine().Append("using StackExchange.Redis;"); + NewLine().Append("#pragma warning disable CS8981"); + foreach (var grp in literals.GroupBy(l => (l.Namespace, l.ParentType))) + { + NewLine(); + int braces = 0; + if (!string.IsNullOrWhiteSpace(grp.Key.Namespace)) + { + NewLine().Append("namespace ").Append(grp.Key.Namespace); + NewLine().Append("{"); + indent++; + braces++; + } + if (!string.IsNullOrWhiteSpace(grp.Key.ParentType)) + { + if (grp.Key.ParentType.Contains('.')) // nested types + { + foreach (var part in grp.Key.ParentType.Split('.')) + { + NewLine().Append("partial class ").Append(part); + NewLine().Append("{"); + indent++; + braces++; + } + } + else + { + NewLine().Append("partial class ").Append(grp.Key.ParentType); + NewLine().Append("{"); + indent++; + braces++; + } + } + + foreach (var literal in grp) + { + int len; + unsafe + { + fixed (byte* bPtr = buffer) // netstandard2.0 forces fallback API + { + fixed (char* cPtr = literal.Value) + { + len = Encoding.UTF8.GetBytes(cPtr, literal.Value.Length, bPtr, buffer.Length); + } + } + } + + // perform string escaping on the generated value (this includes the quotes, note) + var csValue = SyntaxFactory.LiteralExpression(SyntaxKind.StringLiteralExpression, SyntaxFactory.Literal(literal.Value)).ToFullString(); + + var hash = FastHash.Hash64(buffer.AsSpan(0, len)); + NewLine().Append("static partial class ").Append(literal.Name); + NewLine().Append("{"); + indent++; + NewLine().Append("public const int Length = ").Append(len).Append(';'); + NewLine().Append("public const long Hash = ").Append(hash).Append(';'); + NewLine().Append("public static ReadOnlySpan U8 => ").Append(csValue).Append("u8;"); + NewLine().Append("public const string Text = ").Append(csValue).Append(';'); + if (len <= 8) + { + // the hash enforces all the values + NewLine().Append("public static bool Is(long hash, in RawResult value) => hash == Hash && value.Payload.Length == Length;"); + NewLine().Append("public static bool Is(long hash, ReadOnlySpan value) => hash == Hash & value.Length == Length;"); + } + else + { + NewLine().Append("public static bool Is(long hash, in RawResult value) => hash == Hash && value.IsEqual(U8);"); + NewLine().Append("public static bool Is(long hash, ReadOnlySpan value) => hash == Hash && value.SequenceEqual(U8);"); + } + indent--; + NewLine().Append("}"); + } + + // handle any closing braces + while (braces-- > 0) + { + indent--; + NewLine().Append("}"); + } + } + + ArrayPool.Shared.Return(buffer); + ctx.AddSource("FastHash.generated.cs", sb.ToString()); + } +} diff --git a/eng/StackExchange.Redis.Build/FastHashGenerator.md b/eng/StackExchange.Redis.Build/FastHashGenerator.md new file mode 100644 index 000000000..7fc5103ae --- /dev/null +++ b/eng/StackExchange.Redis.Build/FastHashGenerator.md @@ -0,0 +1,64 @@ +# FastHashGenerator + +Efficient matching of well-known short string tokens is a high-volume scenario, for example when matching RESP literals. + +The purpose of this generator is to interpret inputs like: + +``` c# +[FastHash] public static partial class bin { } +[FastHash] public static partial class f32 { } +``` + +Usually the token is inferred from the name; `[FastHash("real value")]` can be used if the token is not a valid identifier. +Underscore is replaced with hyphen, so a field called `my_token` has the default value `"my-token"`. +The generator demands *all* of `[FastHash] public static partial class`, and note that any *containing* types must +*also* be declared `partial`. + +The output is of the form: + +``` c# +static partial class bin +{ + public const int Length = 3; + public const long Hash = 7235938; + public static ReadOnlySpan U8 => @"bin"u8; + public static string Text => @"bin"; + public static bool Is(long hash, in RawResult value) => ... + public static bool Is(long hash, in ReadOnlySpan value) => ... +} +static partial class f32 +{ + public const int Length = 3; + public const long Hash = 3289958; + public static ReadOnlySpan U8 => @"f32"u8; + public const string Text = @"f32"; + public static bool Is(long hash, in RawResult value) => ... + public static bool Is(long hash, in ReadOnlySpan value) => ... +} +``` + +(this API is strictly an internal implementation detail, and can change at any time) + +This generated code allows for fast, efficient, and safe matching of well-known tokens, for example: + +``` c# +var key = ... +var hash = key.Hash64(); +switch (key.Length) +{ + case bin.Length when bin.Is(hash, key): + // handle bin + break; + case f32.Length when f32.Is(hash, key): + // handle f32 + break; +} +``` + +The switch on the `Length` is optional, but recommended - these low values can often be implemented (by the compiler) +as a simple jump-table, which is very fast. However, switching on the hash itself is also valid. All hash matches +must also perform a sequence equality check - the `Is(hash, value)` convenience method validates both hash and equality. + +Note that `switch` requires `const` values, hence why we use generated *types* rather than partial-properties +that emit an instance with the known values. Also, the `"..."u8` syntax emits a span which is awkward to store, but +easy to return via a property. diff --git a/eng/StackExchange.Redis.Build/StackExchange.Redis.Build.csproj b/eng/StackExchange.Redis.Build/StackExchange.Redis.Build.csproj new file mode 100644 index 000000000..f875133ba --- /dev/null +++ b/eng/StackExchange.Redis.Build/StackExchange.Redis.Build.csproj @@ -0,0 +1,20 @@ + + + + netstandard2.0 + enable + enable + true + + + + + + + + + FastHash.cs + + + + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 29eadff61..06e403ebb 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -10,4 +10,7 @@ + + + diff --git a/src/StackExchange.Redis/Enums/RedisCommand.cs b/src/StackExchange.Redis/Enums/RedisCommand.cs index 52f0b134d..7a0c2f08d 100644 --- a/src/StackExchange.Redis/Enums/RedisCommand.cs +++ b/src/StackExchange.Redis/Enums/RedisCommand.cs @@ -206,6 +206,19 @@ internal enum RedisCommand UNSUBSCRIBE, UNWATCH, + VADD, + VCARD, + VDIM, + VEMB, + VGETATTR, + VINFO, + VISMEMBER, + VLINKS, + VRANDMEMBER, + VREM, + VSETATTR, + VSIM, + WATCH, XACK, @@ -352,6 +365,9 @@ internal static bool IsPrimaryOnly(this RedisCommand command) case RedisCommand.SWAPDB: case RedisCommand.TOUCH: case RedisCommand.UNLINK: + case RedisCommand.VADD: + case RedisCommand.VREM: + case RedisCommand.VSETATTR: case RedisCommand.XAUTOCLAIM: case RedisCommand.ZADD: case RedisCommand.ZDIFFSTORE: @@ -499,6 +515,15 @@ internal static bool IsPrimaryOnly(this RedisCommand command) case RedisCommand.ZSCORE: case RedisCommand.ZUNION: case RedisCommand.UNKNOWN: + case RedisCommand.VCARD: + case RedisCommand.VDIM: + case RedisCommand.VEMB: + case RedisCommand.VGETATTR: + case RedisCommand.VINFO: + case RedisCommand.VISMEMBER: + case RedisCommand.VLINKS: + case RedisCommand.VRANDMEMBER: + case RedisCommand.VSIM: // Writable commands, but allowed for the writable-replicas scenario case RedisCommand.COPY: case RedisCommand.GEOADD: diff --git a/src/StackExchange.Redis/Experiments.cs b/src/StackExchange.Redis/Experiments.cs new file mode 100644 index 000000000..577c9f8c9 --- /dev/null +++ b/src/StackExchange.Redis/Experiments.cs @@ -0,0 +1,41 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis +{ + // example usage: + // [Experimental(Experiments.SomeFeature, UrlFormat = Experiments.UrlFormat)] + // where SomeFeature has the next label, for example "SER042", and /docs/exp/SER042.md exists + internal static class Experiments + { + public const string UrlFormat = "https://stackexchange.github.io/StackExchange.Redis/exp/"; + public const string VectorSets = "SER001"; + } +} + +#if !NET8_0_OR_GREATER +#pragma warning disable SA1403 +namespace System.Diagnostics.CodeAnalysis +#pragma warning restore SA1403 +{ + [AttributeUsage( + AttributeTargets.Assembly | + AttributeTargets.Module | + AttributeTargets.Class | + AttributeTargets.Struct | + AttributeTargets.Enum | + AttributeTargets.Constructor | + AttributeTargets.Method | + AttributeTargets.Property | + AttributeTargets.Field | + AttributeTargets.Event | + AttributeTargets.Interface | + AttributeTargets.Delegate, + Inherited = false)] + internal sealed class ExperimentalAttribute(string diagnosticId) : Attribute + { + public string DiagnosticId { get; } = diagnosticId; + public string? UrlFormat { get; set; } + public string? Message { get; set; } + } +} +#endif diff --git a/src/StackExchange.Redis/FastHash.cs b/src/StackExchange.Redis/FastHash.cs new file mode 100644 index 000000000..49eb01b31 --- /dev/null +++ b/src/StackExchange.Redis/FastHash.cs @@ -0,0 +1,137 @@ +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace StackExchange.Redis; + +/// +/// This type is intended to provide fast hashing functions for small strings, for example well-known +/// RESP literals that are usually identifiable by their length and initial bytes; it is not intended +/// for general purpose hashing. All matches must also perform a sequence equality check. +/// +/// See HastHashGenerator.md for more information and intended usage. +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] +[Conditional("DEBUG")] // evaporate in release +internal sealed class FastHashAttribute(string token = "") : Attribute +{ + public string Token => token; +} + +internal static class FastHash +{ + /* not sure we need this, but: retain for reference + + // Perform case-insensitive hash by masking (X and x differ by only 1 bit); this halves + // our entropy, but is still useful when case doesn't matter. + private const long CaseMask = ~0x2020202020202020; + + public static long Hash64CI(this ReadOnlySequence value) + => value.Hash64() & CaseMask; + public static long Hash64CI(this scoped ReadOnlySpan value) + => value.Hash64() & CaseMask; +*/ + + public static long Hash64(this ReadOnlySequence value) + { +#if NETCOREAPP3_1_OR_GREATER || NETSTANDARD2_1_OR_GREATER + var first = value.FirstSpan; +#else + var first = value.First.Span; +#endif + return first.Length >= sizeof(long) || value.IsSingleSegment + ? first.Hash64() : SlowHash64(value); + + static long SlowHash64(ReadOnlySequence value) + { + Span buffer = stackalloc byte[sizeof(long)]; + if (value.Length < sizeof(long)) + { + value.CopyTo(buffer); + buffer.Slice((int)value.Length).Clear(); + } + else + { + value.Slice(0, sizeof(long)).CopyTo(buffer); + } + return BitConverter.IsLittleEndian + ? Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(buffer)) + : BinaryPrimitives.ReadInt64LittleEndian(buffer); + } + } + + public static long Hash64(this scoped ReadOnlySpan value) + { + if (BitConverter.IsLittleEndian) + { + ref byte data = ref MemoryMarshal.GetReference(value); + return value.Length switch + { + 0 => 0, + 1 => data, // 0000000A + 2 => Unsafe.ReadUnaligned(ref data), // 000000BA + 3 => Unsafe.ReadUnaligned(ref data) | // 000000BA + (Unsafe.Add(ref data, 2) << 16), // 00000C00 + 4 => Unsafe.ReadUnaligned(ref data), // 0000DCBA + 5 => Unsafe.ReadUnaligned(ref data) | // 0000DCBA + ((long)Unsafe.Add(ref data, 4) << 32), // 000E0000 + 6 => Unsafe.ReadUnaligned(ref data) | // 0000DCBA + ((long)Unsafe.ReadUnaligned(ref Unsafe.Add(ref data, 4)) << 32), // 00FE0000 + 7 => Unsafe.ReadUnaligned(ref data) | // 0000DCBA + ((long)Unsafe.ReadUnaligned(ref Unsafe.Add(ref data, 4)) << 32) | // 00FE0000 + ((long)Unsafe.Add(ref data, 6) << 48), // 0G000000 + _ => Unsafe.ReadUnaligned(ref data), // HGFEDCBA + }; + } + +#pragma warning disable CS0618 // Type or member is obsolete + return Hash64Fallback(value); +#pragma warning restore CS0618 // Type or member is obsolete + } + + [Obsolete("Only exists for benchmarks (to show that we don't need to use it) and unit tests (for correctness)")] + internal static unsafe long Hash64Unsafe(scoped ReadOnlySpan value) + { + if (BitConverter.IsLittleEndian) + { + fixed (byte* ptr = &MemoryMarshal.GetReference(value)) + { + return value.Length switch + { + 0 => 0, + 1 => *ptr, // 0000000A + 2 => *(ushort*)ptr, // 000000BA + 3 => *(ushort*)ptr | // 000000BA + (ptr[2] << 16), // 00000C00 + 4 => *(int*)ptr, // 0000DCBA + 5 => (long)*(int*)ptr | // 0000DCBA + ((long)ptr[4] << 32), // 000E0000 + 6 => (long)*(int*)ptr | // 0000DCBA + ((long)*(ushort*)(ptr + 4) << 32), // 00FE0000 + 7 => (long)*(int*)ptr | // 0000DCBA + ((long)*(ushort*)(ptr + 4) << 32) | // 00FE0000 + ((long)ptr[6] << 48), // 0G000000 + _ => *(long*)ptr, // HGFEDCBA + }; + } + } + + return Hash64Fallback(value); + } + + [Obsolete("Only exists for unit tests and fallback")] + internal static long Hash64Fallback(scoped ReadOnlySpan value) + { + if (value.Length < sizeof(long)) + { + Span tmp = stackalloc byte[sizeof(long)]; + value.CopyTo(tmp); // ABC***** + tmp.Slice(value.Length).Clear(); // ABC00000 + return BinaryPrimitives.ReadInt64LittleEndian(tmp); // 00000CBA + } + + return BinaryPrimitives.ReadInt64LittleEndian(value); // HGFEDCBA + } +} diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs new file mode 100644 index 000000000..039075ec8 --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs @@ -0,0 +1,183 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis; + +/// +/// Describes functionality that is common to both standalone redis servers and redis clusters. +/// +public partial interface IDatabase +{ + // Vector Set operations + + /// + /// Add a vector to a vectorset. + /// + /// The key of the vectorset. + /// The data to add. + /// The flags to use for this operation. + /// if the element was added; if it already existed. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + bool VectorSetAdd( + RedisKey key, + VectorSetAddRequest request, + CommandFlags flags = CommandFlags.None); + + /// + /// Get the cardinality (number of elements) of a vectorset. + /// + /// The key of the vectorset. + /// The flags to use for this operation. + /// The cardinality of the vectorset. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + long VectorSetLength(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + /// Get the dimension of vectors in a vectorset. + /// + /// The key of the vectorset. + /// The flags to use for this operation. + /// The dimension of vectors in the vectorset. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + int VectorSetDimension(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + /// Get the vector for a member. + /// + /// The key of the vectorset. + /// The member name. + /// The flags to use for this operation. + /// The vector as a pooled memory lease. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Lease? VectorSetGetApproximateVector( + RedisKey key, + RedisValue member, + CommandFlags flags = CommandFlags.None); + + /// + /// Get JSON attributes for a member in a vectorset. + /// + /// The key of the vectorset. + /// The member name. + /// The flags to use for this operation. + /// The attributes as a JSON string. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + string? VectorSetGetAttributesJson(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None); + + /// + /// Get information about a vectorset. + /// + /// The key of the vectorset. + /// The flags to use for this operation. + /// Information about the vectorset. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + VectorSetInfo? VectorSetInfo(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + /// Check if a member exists in a vectorset. + /// + /// The key of the vectorset. + /// The member name. + /// The flags to use for this operation. + /// True if the member exists, false otherwise. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + bool VectorSetContains(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None); + + /// + /// Get links/connections for a member in a vectorset. + /// + /// The key of the vectorset. + /// The member name. + /// The flags to use for this operation. + /// The linked members. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Lease? VectorSetGetLinks(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None); + + /// + /// Get links/connections with scores for a member in a vectorset. + /// + /// The key of the vectorset. + /// The member name. + /// The flags to use for this operation. + /// The linked members with their similarity scores. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Lease? VectorSetGetLinksWithScores( + RedisKey key, + RedisValue member, + CommandFlags flags = CommandFlags.None); + + /// + /// Get a random member from a vectorset. + /// + /// The key of the vectorset. + /// The flags to use for this operation. + /// A random member from the vectorset, or null if the vectorset is empty. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + RedisValue VectorSetRandomMember(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + /// Get random members from a vectorset. + /// + /// The key of the vectorset. + /// The number of random members to return. + /// The flags to use for this operation. + /// Random members from the vectorset. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + RedisValue[] VectorSetRandomMembers(RedisKey key, long count, CommandFlags flags = CommandFlags.None); + + /// + /// Remove a member from a vectorset. + /// + /// The key of the vectorset. + /// The member to remove. + /// The flags to use for this operation. + /// if the member was removed; if it was not found. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + bool VectorSetRemove(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None); + + /// + /// Set JSON attributes for a member in a vectorset. + /// + /// The key of the vectorset. + /// The member name. + /// The attributes to set as a JSON string. + /// The flags to use for this operation. + /// True if successful. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + bool VectorSetSetAttributesJson( + RedisKey key, + RedisValue member, +#if NET7_0_OR_GREATER + [StringSyntax(StringSyntaxAttribute.Json)] +#endif + string attributesJson, + CommandFlags flags = CommandFlags.None); + + /// + /// Find similar vectors using vector similarity search. + /// + /// The key of the vectorset. + /// The query to execute. + /// The flags to use for this operation. + /// Similar vectors with their similarity scores. + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Lease? VectorSetSimilaritySearch( + RedisKey key, + VectorSetSimilaritySearchRequest query, + CommandFlags flags = CommandFlags.None); +} diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index b6caafabe..6c52e89bd 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -3,12 +3,13 @@ using System.ComponentModel; using System.Net; +// ReSharper disable once CheckNamespace namespace StackExchange.Redis { /// /// Describes functionality that is common to both standalone redis servers and redis clusters. /// - public interface IDatabase : IRedis, IDatabaseAsync + public partial interface IDatabase : IRedis, IDatabaseAsync { /// /// The numeric identifier of this database. diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs new file mode 100644 index 000000000..863095140 --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs @@ -0,0 +1,95 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis; + +/// +/// Describes functionality that is common to both standalone redis servers and redis clusters. +/// +public partial interface IDatabaseAsync +{ + // Vector Set operations + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetAddAsync( + RedisKey key, + VectorSetAddRequest request, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetDimensionAsync(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task?> VectorSetGetApproximateVectorAsync( + RedisKey key, + RedisValue member, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetGetAttributesJsonAsync( + RedisKey key, + RedisValue member, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetInfoAsync(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetContainsAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task?> VectorSetGetLinksAsync( + RedisKey key, + RedisValue member, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task?> VectorSetGetLinksWithScoresAsync( + RedisKey key, + RedisValue member, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetRandomMemberAsync(RedisKey key, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetRandomMembersAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetRemoveAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task VectorSetSetAttributesJsonAsync( + RedisKey key, + RedisValue member, +#if NET7_0_OR_GREATER + [StringSyntax(StringSyntaxAttribute.Json)] +#endif + string attributesJson, + CommandFlags flags = CommandFlags.None); + + /// + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + Task?> VectorSetSimilaritySearchAsync( + RedisKey key, + VectorSetSimilaritySearchRequest query, + CommandFlags flags = CommandFlags.None); +} diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs index 51a15d7d5..0bc7b4867 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs @@ -4,12 +4,13 @@ using System.Net; using System.Threading.Tasks; +// ReSharper disable once CheckNamespace namespace StackExchange.Redis { /// /// Describes functionality that is common to both standalone redis servers and redis clusters. /// - public interface IDatabaseAsync : IRedisAsync + public partial interface IDatabaseAsync : IRedisAsync { /// /// Indicates whether the instance can communicate with the server (resolved using the supplied key and optional flags). diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs new file mode 100644 index 000000000..809adad97 --- /dev/null +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.VectorSets.cs @@ -0,0 +1,59 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis.KeyspaceIsolation; + +internal partial class KeyPrefixed +{ + // Vector Set operations - async methods + [Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] + public Task VectorSetAddAsync( + RedisKey key, + VectorSetAddRequest request, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetAddAsync(ToInner(key), request, flags); + + public Task VectorSetLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetLengthAsync(ToInner(key), flags); + + public Task VectorSetDimensionAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetDimensionAsync(ToInner(key), flags); + + public Task?> VectorSetGetApproximateVectorAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetApproximateVectorAsync(ToInner(key), member, flags); + + public Task VectorSetGetAttributesJsonAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetAttributesJsonAsync(ToInner(key), member, flags); + + public Task VectorSetInfoAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetInfoAsync(ToInner(key), flags); + + public Task VectorSetContainsAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetContainsAsync(ToInner(key), member, flags); + + public Task?> VectorSetGetLinksAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetLinksAsync(ToInner(key), member, flags); + + public Task?> VectorSetGetLinksWithScoresAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetLinksWithScoresAsync(ToInner(key), member, flags); + + public Task VectorSetRandomMemberAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRandomMemberAsync(ToInner(key), flags); + + public Task VectorSetRandomMembersAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRandomMembersAsync(ToInner(key), count, flags); + + public Task VectorSetRemoveAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRemoveAsync(ToInner(key), member, flags); + + public Task VectorSetSetAttributesJsonAsync(RedisKey key, RedisValue member, string attributesJson, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetSetAttributesJsonAsync(ToInner(key), member, attributesJson, flags); + + public Task?> VectorSetSimilaritySearchAsync( + RedisKey key, + VectorSetSimilaritySearchRequest query, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetSimilaritySearchAsync(ToInner(key), query, flags); +} diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index 32c76f4d2..61a6f44c4 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -7,7 +7,7 @@ namespace StackExchange.Redis.KeyspaceIsolation { - internal class KeyPrefixed : IDatabaseAsync where TInner : IDatabaseAsync + internal partial class KeyPrefixed : IDatabaseAsync where TInner : IDatabaseAsync { internal KeyPrefixed(TInner inner, byte[] keyPrefix) { diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs new file mode 100644 index 000000000..62f4e9202 --- /dev/null +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.VectorSets.cs @@ -0,0 +1,56 @@ +using System; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis.KeyspaceIsolation; + +internal sealed partial class KeyPrefixedDatabase +{ + // Vector Set operations + public bool VectorSetAdd( + RedisKey key, + VectorSetAddRequest request, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetAdd(ToInner(key), request, flags); + + public long VectorSetLength(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetLength(ToInner(key), flags); + + public int VectorSetDimension(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetDimension(ToInner(key), flags); + + public Lease? VectorSetGetApproximateVector(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetApproximateVector(ToInner(key), member, flags); + + public string? VectorSetGetAttributesJson(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetAttributesJson(ToInner(key), member, flags); + + public VectorSetInfo? VectorSetInfo(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetInfo(ToInner(key), flags); + + public bool VectorSetContains(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetContains(ToInner(key), member, flags); + + public Lease? VectorSetGetLinks(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetLinks(ToInner(key), member, flags); + + public Lease? VectorSetGetLinksWithScores(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetGetLinksWithScores(ToInner(key), member, flags); + + public RedisValue VectorSetRandomMember(RedisKey key, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRandomMember(ToInner(key), flags); + + public RedisValue[] VectorSetRandomMembers(RedisKey key, long count, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRandomMembers(ToInner(key), count, flags); + + public bool VectorSetRemove(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetRemove(ToInner(key), member, flags); + + public bool VectorSetSetAttributesJson(RedisKey key, RedisValue member, string attributesJson, CommandFlags flags = CommandFlags.None) => + Inner.VectorSetSetAttributesJson(ToInner(key), member, attributesJson, flags); + + public Lease? VectorSetSimilaritySearch( + RedisKey key, + VectorSetSimilaritySearchRequest query, + CommandFlags flags = CommandFlags.None) => + Inner.VectorSetSimilaritySearch(ToInner(key), query, flags); +} diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs index 755bec64e..2a139694e 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs @@ -4,7 +4,7 @@ namespace StackExchange.Redis.KeyspaceIsolation { - internal sealed class KeyPrefixedDatabase : KeyPrefixed, IDatabase + internal sealed partial class KeyPrefixedDatabase : KeyPrefixed, IDatabase { public KeyPrefixedDatabase(IDatabase inner, byte[] prefix) : base(inner, prefix) { diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index cd3d29947..5973bd55b 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -705,6 +705,7 @@ internal void SetWriteTime() _writeTickCount = Environment.TickCount; // note this might be reset if we resend a message, cluster-moved etc; I'm OK with that } private int _writeTickCount; + public int GetWriteTime() => Volatile.Read(ref _writeTickCount); /// diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index c587241a0..129fd9e07 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -859,6 +859,14 @@ internal static void WriteBulkString(in RedisValue value, PipeWriter? maybeNullW } } + internal void WriteBulkString(ReadOnlySpan value) + { + if (_ioPipe?.Output is { } writer) + { + WriteUnifiedSpan(writer, value); + } + } + internal const int REDIS_MAX_ARGS = 1024 * 1024; // there is a <= 1024*1024 max constraint inside redis itself: https://github.com/antirez/redis/blob/6c60526db91e23fb2d666fc52facc9a11780a2a3/src/networking.c#L1024 internal void WriteHeader(RedisCommand command, int arguments, CommandBytes commandBytes = default) diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt index e82af2bee..10044dc9b 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt @@ -1962,3 +1962,92 @@ StackExchange.Redis.ConnectionMultiplexer.GetServer(StackExchange.Redis.RedisKey StackExchange.Redis.IConnectionMultiplexer.GetServer(StackExchange.Redis.RedisKey key, object? asyncState = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.IServer! StackExchange.Redis.IServer.Execute(int? database, string! command, System.Collections.Generic.ICollection! args, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisResult! StackExchange.Redis.IServer.ExecuteAsync(int? database, string! command, System.Collections.Generic.ICollection! args, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]override StackExchange.Redis.VectorSetLink.ToString() -> string! +[SER001]override StackExchange.Redis.VectorSetSimilaritySearchResult.ToString() -> string! +[SER001]StackExchange.Redis.IDatabase.VectorSetAdd(StackExchange.Redis.RedisKey key, StackExchange.Redis.VectorSetAddRequest! request, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> bool +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetAddAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.VectorSetAddRequest! request, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.VectorSetAddRequest +[SER001]StackExchange.Redis.VectorSetAddRequest.BuildExplorationFactor.get -> int? +[SER001]StackExchange.Redis.VectorSetAddRequest.BuildExplorationFactor.set -> void +[SER001]StackExchange.Redis.VectorSetAddRequest.MaxConnections.get -> int? +[SER001]StackExchange.Redis.VectorSetAddRequest.MaxConnections.set -> void +[SER001]StackExchange.Redis.VectorSetAddRequest.Quantization.get -> StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetAddRequest.Quantization.set -> void +[SER001]StackExchange.Redis.VectorSetAddRequest.ReducedDimensions.get -> int? +[SER001]StackExchange.Redis.VectorSetAddRequest.ReducedDimensions.set -> void +[SER001]StackExchange.Redis.VectorSetAddRequest.UseCheckAndSet.get -> bool +[SER001]StackExchange.Redis.VectorSetAddRequest.UseCheckAndSet.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.Count.get -> int? +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.Count.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.DisableThreading.get -> bool +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.DisableThreading.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.Epsilon.get -> double? +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.Epsilon.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.FilterExpression.get -> string? +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.FilterExpression.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.MaxFilteringEffort.get -> int? +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.MaxFilteringEffort.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.SearchExplorationFactor.get -> int? +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.SearchExplorationFactor.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.UseExactSearch.get -> bool +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.UseExactSearch.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.WithAttributes.get -> bool +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.WithAttributes.set -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.WithScores.get -> bool +[SER001]StackExchange.Redis.VectorSetSimilaritySearchRequest.WithScores.set -> void +[SER001]StackExchange.Redis.IDatabase.VectorSetContains(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> bool +[SER001]StackExchange.Redis.IDatabase.VectorSetDimension(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> int +[SER001]StackExchange.Redis.IDatabase.VectorSetGetApproximateVector(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease? +[SER001]StackExchange.Redis.IDatabase.VectorSetGetAttributesJson(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> string? +[SER001]StackExchange.Redis.IDatabase.VectorSetGetLinks(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease? +[SER001]StackExchange.Redis.IDatabase.VectorSetGetLinksWithScores(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease? +[SER001]StackExchange.Redis.IDatabase.VectorSetInfo(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.VectorSetInfo? +[SER001]StackExchange.Redis.IDatabase.VectorSetLength(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long +[SER001]StackExchange.Redis.IDatabase.VectorSetRandomMember(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue +[SER001]StackExchange.Redis.IDatabase.VectorSetRandomMembers(StackExchange.Redis.RedisKey key, long count, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue[]! +[SER001]StackExchange.Redis.IDatabase.VectorSetRemove(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> bool +[SER001]StackExchange.Redis.IDatabase.VectorSetSetAttributesJson(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, string! attributesJson, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> bool +[SER001]StackExchange.Redis.IDatabase.VectorSetSimilaritySearch(StackExchange.Redis.RedisKey key, StackExchange.Redis.VectorSetSimilaritySearchRequest! query, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.Lease? +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetContainsAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetDimensionAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetGetApproximateVectorAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task?>! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetGetAttributesJsonAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetGetLinksAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task?>! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetGetLinksWithScoresAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task?>! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetInfoAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetLengthAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetRandomMemberAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetRandomMembersAsync(StackExchange.Redis.RedisKey key, long count, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetRemoveAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetSetAttributesJsonAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.RedisValue member, string! attributesJson, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER001]StackExchange.Redis.IDatabaseAsync.VectorSetSimilaritySearchAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.VectorSetSimilaritySearchRequest! query, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task?>! +[SER001]StackExchange.Redis.VectorSetInfo +[SER001]StackExchange.Redis.VectorSetInfo.Dimension.get -> int +[SER001]StackExchange.Redis.VectorSetInfo.HnswMaxNodeUid.get -> long +[SER001]StackExchange.Redis.VectorSetInfo.Length.get -> long +[SER001]StackExchange.Redis.VectorSetInfo.MaxLevel.get -> int +[SER001]StackExchange.Redis.VectorSetInfo.Quantization.get -> StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetInfo.QuantizationRaw.get -> string? +[SER001]StackExchange.Redis.VectorSetInfo.VectorSetInfo() -> void +[SER001]StackExchange.Redis.VectorSetInfo.VectorSetInfo(StackExchange.Redis.VectorSetQuantization quantization, string? quantizationRaw, int dimension, long length, int maxLevel, long vectorSetUid, long hnswMaxNodeUid) -> void +[SER001]StackExchange.Redis.VectorSetInfo.VectorSetUid.get -> long +[SER001]StackExchange.Redis.VectorSetLink +[SER001]StackExchange.Redis.VectorSetLink.Member.get -> StackExchange.Redis.RedisValue +[SER001]StackExchange.Redis.VectorSetLink.Score.get -> double +[SER001]StackExchange.Redis.VectorSetLink.VectorSetLink() -> void +[SER001]StackExchange.Redis.VectorSetLink.VectorSetLink(StackExchange.Redis.RedisValue member, double score) -> void +[SER001]StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetQuantization.Binary = 3 -> StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetQuantization.Int8 = 2 -> StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetQuantization.None = 1 -> StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetQuantization.Unknown = 0 -> StackExchange.Redis.VectorSetQuantization +[SER001]StackExchange.Redis.VectorSetSimilaritySearchResult +[SER001]StackExchange.Redis.VectorSetSimilaritySearchResult.AttributesJson.get -> string? +[SER001]StackExchange.Redis.VectorSetSimilaritySearchResult.Member.get -> StackExchange.Redis.RedisValue +[SER001]StackExchange.Redis.VectorSetSimilaritySearchResult.Score.get -> double +[SER001]StackExchange.Redis.VectorSetSimilaritySearchResult.VectorSetSimilaritySearchResult() -> void +[SER001]StackExchange.Redis.VectorSetSimilaritySearchResult.VectorSetSimilaritySearchResult(StackExchange.Redis.RedisValue member, double score = NaN, string? attributesJson = null) -> void +[SER001]static StackExchange.Redis.VectorSetAddRequest.Member(StackExchange.Redis.RedisValue element, System.ReadOnlyMemory values, string? attributesJson = null) -> StackExchange.Redis.VectorSetAddRequest! +[SER001]static StackExchange.Redis.VectorSetSimilaritySearchRequest.ByMember(StackExchange.Redis.RedisValue member) -> StackExchange.Redis.VectorSetSimilaritySearchRequest! +[SER001]static StackExchange.Redis.VectorSetSimilaritySearchRequest.ByVector(System.ReadOnlyMemory vector) -> StackExchange.Redis.VectorSetSimilaritySearchRequest! diff --git a/src/StackExchange.Redis/RawResult.cs b/src/StackExchange.Redis/RawResult.cs index 55c44652b..1ac9f081a 100644 --- a/src/StackExchange.Redis/RawResult.cs +++ b/src/StackExchange.Redis/RawResult.cs @@ -237,10 +237,14 @@ internal bool IsEqual(in CommandBytes expected) return new CommandBytes(Payload).Equals(expected); } - internal unsafe bool IsEqual(byte[]? expected) + internal bool IsEqual(byte[]? expected) { if (expected == null) throw new ArgumentNullException(nameof(expected)); + return IsEqual(new ReadOnlySpan(expected)); + } + internal bool IsEqual(ReadOnlySpan expected) + { var rangeToCheck = Payload; if (expected.Length != rangeToCheck.Length) return false; @@ -250,7 +254,7 @@ internal unsafe bool IsEqual(byte[]? expected) foreach (var segment in rangeToCheck) { var from = segment.Span; - var to = new Span(expected, offset, from.Length); + var to = expected.Slice(offset, from.Length); if (!from.SequenceEqual(to)) return false; offset += from.Length; diff --git a/src/StackExchange.Redis/RedisDatabase.VectorSets.cs b/src/StackExchange.Redis/RedisDatabase.VectorSets.cs new file mode 100644 index 000000000..9b3f1b43b --- /dev/null +++ b/src/StackExchange.Redis/RedisDatabase.VectorSets.cs @@ -0,0 +1,191 @@ +using System; +using System.Threading.Tasks; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis; + +internal partial class RedisDatabase +{ + public bool VectorSetAdd( + RedisKey key, + VectorSetAddRequest request, + CommandFlags flags = CommandFlags.None) + { + var msg = request.ToMessage(key, Database, flags); + return ExecuteSync(msg, ResultProcessor.Boolean); + } + + public long VectorSetLength(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VCARD, key); + return ExecuteSync(msg, ResultProcessor.Int64); + } + + public int VectorSetDimension(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VDIM, key); + return ExecuteSync(msg, ResultProcessor.Int32); + } + + public Lease? VectorSetGetApproximateVector(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VEMB, key, member); + return ExecuteSync(msg, ResultProcessor.LeaseFloat32); + } + + public string? VectorSetGetAttributesJson(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VGETATTR, key, member); + return ExecuteSync(msg, ResultProcessor.String); + } + + public VectorSetInfo? VectorSetInfo(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VINFO, key); + return ExecuteSync(msg, ResultProcessor.VectorSetInfo); + } + + public bool VectorSetContains(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VISMEMBER, key, member); + return ExecuteSync(msg, ResultProcessor.Boolean); + } + + public Lease? VectorSetGetLinks(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VLINKS, key, member); + return ExecuteSync(msg, ResultProcessor.VectorSetLinks); + } + + public Lease? VectorSetGetLinksWithScores(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VLINKS, key, member, RedisLiterals.WITHSCORES); + return ExecuteSync(msg, ResultProcessor.VectorSetLinksWithScores); + } + + public RedisValue VectorSetRandomMember(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VRANDMEMBER, key); + return ExecuteSync(msg, ResultProcessor.RedisValue); + } + + public RedisValue[] VectorSetRandomMembers(RedisKey key, long count, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VRANDMEMBER, key, count); + return ExecuteSync(msg, ResultProcessor.RedisValueArray, defaultValue: Array.Empty()); + } + + public bool VectorSetRemove(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VREM, key, member); + return ExecuteSync(msg, ResultProcessor.Boolean); + } + + public bool VectorSetSetAttributesJson(RedisKey key, RedisValue member, string attributesJson, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VSETATTR, key, member, attributesJson); + return ExecuteSync(msg, ResultProcessor.Boolean); + } + + public Lease? VectorSetSimilaritySearch( + RedisKey key, + VectorSetSimilaritySearchRequest query, + CommandFlags flags = CommandFlags.None) + { + if (query == null) throw new ArgumentNullException(nameof(query)); + var msg = query.ToMessage(key, Database, flags); + return ExecuteSync(msg, msg.GetResultProcessor()); + } + + // Vector Set async operations + public Task VectorSetAddAsync( + RedisKey key, + VectorSetAddRequest request, + CommandFlags flags = CommandFlags.None) + { + var msg = request.ToMessage(key, Database, flags); + return ExecuteAsync(msg, ResultProcessor.Boolean); + } + + public Task VectorSetLengthAsync(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VCARD, key); + return ExecuteAsync(msg, ResultProcessor.Int64); + } + + public Task VectorSetDimensionAsync(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VDIM, key); + return ExecuteAsync(msg, ResultProcessor.Int32); + } + + public Task?> VectorSetGetApproximateVectorAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VEMB, key, member); + return ExecuteAsync(msg, ResultProcessor.LeaseFloat32); + } + + public Task VectorSetGetAttributesJsonAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VGETATTR, key, member); + return ExecuteAsync(msg, ResultProcessor.String); + } + + public Task VectorSetInfoAsync(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VINFO, key); + return ExecuteAsync(msg, ResultProcessor.VectorSetInfo); + } + + public Task VectorSetContainsAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VISMEMBER, key, member); + return ExecuteAsync(msg, ResultProcessor.Boolean); + } + + public Task?> VectorSetGetLinksAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VLINKS, key, member); + return ExecuteAsync(msg, ResultProcessor.VectorSetLinks); + } + + public Task?> VectorSetGetLinksWithScoresAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VLINKS, key, member, RedisLiterals.WITHSCORES); + return ExecuteAsync(msg, ResultProcessor.VectorSetLinksWithScores); + } + + public Task VectorSetRandomMemberAsync(RedisKey key, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VRANDMEMBER, key); + return ExecuteAsync(msg, ResultProcessor.RedisValue); + } + + public Task VectorSetRandomMembersAsync(RedisKey key, long count, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VRANDMEMBER, key, count); + return ExecuteAsync(msg, ResultProcessor.RedisValueArray, defaultValue: Array.Empty()); + } + + public Task VectorSetRemoveAsync(RedisKey key, RedisValue member, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VREM, key, member); + return ExecuteAsync(msg, ResultProcessor.Boolean); + } + + public Task VectorSetSetAttributesJsonAsync(RedisKey key, RedisValue member, string attributesJson, CommandFlags flags = CommandFlags.None) + { + var msg = Message.Create(Database, flags, RedisCommand.VSETATTR, key, member, attributesJson); + return ExecuteAsync(msg, ResultProcessor.Boolean); + } + + public Task?> VectorSetSimilaritySearchAsync( + RedisKey key, + VectorSetSimilaritySearchRequest query, + CommandFlags flags = CommandFlags.None) + { + if (query == null) throw new ArgumentNullException(nameof(query)); + var msg = query.ToMessage(key, Database, flags); + return ExecuteAsync(msg, msg.GetResultProcessor()); + } +} diff --git a/src/StackExchange.Redis/ResultProcessor.Lease.cs b/src/StackExchange.Redis/ResultProcessor.Lease.cs new file mode 100644 index 000000000..c0f9e6d8e --- /dev/null +++ b/src/StackExchange.Redis/ResultProcessor.Lease.cs @@ -0,0 +1,218 @@ +using System.Diagnostics; +using Pipelines.Sockets.Unofficial.Arenas; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis; + +internal abstract partial class ResultProcessor +{ + // Lease result processors + public static readonly ResultProcessor?> LeaseFloat32 = new LeaseFloat32Processor(); + + public static readonly ResultProcessor> + Lease = new LeaseProcessor(); + + public static readonly ResultProcessor> + LeaseFromArray = new LeaseFromArrayProcessor(); + + private abstract class LeaseProcessor : ResultProcessor?> + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + if (result.Resp2TypeArray != ResultType.Array) + { + return false; // not an array + } + + // deal with null + if (result.IsNull) + { + SetResult(message, Lease.Empty); + return true; + } + + // lease and fill + var items = result.GetItems(); + var length = checked((int)items.Length); + var lease = Lease.Create(length, clear: false); // note this handles zero nicely + var target = lease.Span; + int index = 0; + foreach (ref RawResult item in items) + { + if (!TryParse(item, out target[index++])) + { + // something went wrong; recycle and quit + lease.Dispose(); + return false; + } + } + Debug.Assert(index == length, "length mismatch"); + SetResult(message, lease); + return true; + } + + protected abstract bool TryParse(in RawResult raw, out T parsed); + } + + private abstract class InterleavedLeaseProcessor : ResultProcessor?> + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + if (result.Resp2TypeArray != ResultType.Array) + { + return false; // not an array + } + + // deal with null + if (result.IsNull) + { + SetResult(message, Lease.Empty); + return true; + } + + // lease and fill + var items = result.GetItems(); + var length = checked((int)items.Length) / 2; + var lease = Lease.Create(length, clear: false); // note this handles zero nicely + var target = lease.Span; + + var iter = items.GetEnumerator(); + for (int i = 0; i < target.Length; i++) + { + bool ok = iter.MoveNext(); + if (ok) + { + ref readonly RawResult first = ref iter.Current; + ok = iter.MoveNext() && TryParse(in first, in iter.Current, out target[i]); + } + if (!ok) + { + lease.Dispose(); + return false; + } + } + SetResult(message, lease); + return true; + } + + protected abstract bool TryParse(in RawResult first, in RawResult second, out T parsed); + } + + // takes a nested vector of the form [[A],[B,C],[D]] and exposes it as [A,B,C,D]; this is + // especially useful for VLINKS + private abstract class FlattenedLeaseProcessor : ResultProcessor?> + { + protected virtual long GetArrayLength(in RawResult array) => array.GetItems().Length; + + protected virtual bool TryReadOne(ref Sequence.Enumerator reader, out T value) + { + if (reader.MoveNext()) + { + return TryReadOne(in reader.Current, out value); + } + value = default!; + return false; + } + + protected virtual bool TryReadOne(in RawResult result, out T value) + { + value = default!; + return false; + } + + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + if (result.Resp2TypeArray != ResultType.Array) + { + return false; // not an array + } + if (result.IsNull) + { + SetResult(message, Lease.Empty); + return true; + } + var items = result.GetItems(); + long length = 0; + foreach (ref RawResult item in items) + { + if (item.Resp2TypeArray == ResultType.Array && !item.IsNull) + { + length += GetArrayLength(in item); + } + } + + if (length == 0) + { + SetResult(message, Lease.Empty); + return true; + } + var lease = Lease.Create(checked((int)length), clear: false); + int index = 0; + var target = lease.Span; + foreach (ref RawResult item in items) + { + if (item.Resp2TypeArray == ResultType.Array && !item.IsNull) + { + var iter = item.GetItems().GetEnumerator(); + while (index < target.Length && TryReadOne(ref iter, out target[index])) + { + index++; + } + } + } + + if (index == length) + { + SetResult(message, lease); + return true; + } + lease.Dispose(); // failed to fill? + return false; + } + } + + private sealed class LeaseFloat32Processor : LeaseProcessor + { + protected override bool TryParse(in RawResult raw, out float parsed) + { + var result = raw.TryGetDouble(out double val); + parsed = (float)val; + return result; + } + } + + private sealed class LeaseProcessor : ResultProcessor> + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + switch (result.Resp2TypeBulkString) + { + case ResultType.Integer: + case ResultType.SimpleString: + case ResultType.BulkString: + SetResult(message, result.AsLease()!); + return true; + } + return false; + } + } + + private sealed class LeaseFromArrayProcessor : ResultProcessor> + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + switch (result.Resp2TypeBulkString) + { + case ResultType.Array: + var items = result.GetItems(); + if (items.Length == 1) + { // treat an array of 1 like a single reply + SetResult(message, items[0].AsLease()!); + return true; + } + break; + } + return false; + } + } +} diff --git a/src/StackExchange.Redis/ResultProcessor.VectorSets.cs b/src/StackExchange.Redis/ResultProcessor.VectorSets.cs new file mode 100644 index 000000000..8743ebd0b --- /dev/null +++ b/src/StackExchange.Redis/ResultProcessor.VectorSets.cs @@ -0,0 +1,138 @@ +using Pipelines.Sockets.Unofficial.Arenas; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis; + +internal abstract partial class ResultProcessor +{ + // VectorSet result processors + public static readonly ResultProcessor?> VectorSetLinksWithScores = + new VectorSetLinksWithScoresProcessor(); + + public static readonly ResultProcessor?> VectorSetLinks = new VectorSetLinksProcessor(); + + public static ResultProcessor VectorSetInfo = new VectorSetInfoProcessor(); + + private sealed class VectorSetLinksWithScoresProcessor : FlattenedLeaseProcessor + { + protected override long GetArrayLength(in RawResult array) => array.GetItems().Length / 2; + + protected override bool TryReadOne(ref Sequence.Enumerator reader, out VectorSetLink value) + { + if (reader.MoveNext()) + { + ref readonly RawResult first = ref reader.Current; + if (reader.MoveNext() && reader.Current.TryGetDouble(out var score)) + { + value = new VectorSetLink(first.AsRedisValue(), score); + return true; + } + } + + value = default; + return false; + } + } + + private sealed class VectorSetLinksProcessor : FlattenedLeaseProcessor + { + protected override bool TryReadOne(in RawResult result, out RedisValue value) + { + value = result.AsRedisValue(); + return true; + } + } + + private sealed partial class VectorSetInfoProcessor : ResultProcessor + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + if (result.Resp2TypeArray == ResultType.Array) + { + if (result.IsNull) + { + SetResult(message, null); + return true; + } + + var quantType = VectorSetQuantization.Unknown; + string? quantTypeRaw = null; + int vectorDim = 0, maxLevel = 0; + long resultSize = 0, vsetUid = 0, hnswMaxNodeUid = 0; + var iter = result.GetItems().GetEnumerator(); + while (iter.MoveNext()) + { + ref readonly RawResult key = ref iter.Current; + if (!iter.MoveNext()) break; + ref readonly RawResult value = ref iter.Current; + + var len = key.Payload.Length; + var keyHash = key.Payload.Hash64(); + switch (key.Payload.Length) + { + case size.Length when size.Is(keyHash, key) && value.TryGetInt64(out var i64): + resultSize = i64; + break; + case vset_uid.Length when vset_uid.Is(keyHash, key) && value.TryGetInt64(out var i64): + vsetUid = i64; + break; + case max_level.Length when max_level.Is(keyHash, key) && value.TryGetInt64(out var i64): + maxLevel = checked((int)i64); + break; + case vector_dim.Length + when vector_dim.Is(keyHash, key) && value.TryGetInt64(out var i64): + vectorDim = checked((int)i64); + break; + case quant_type.Length when quant_type.Is(keyHash, key): + var qHash = value.Payload.Hash64(); + switch (value.Payload.Length) + { + case bin.Length when bin.Is(qHash, value): + quantType = VectorSetQuantization.Binary; + break; + case f32.Length when f32.Is(qHash, value): + quantType = VectorSetQuantization.None; + break; + case int8.Length when int8.Is(qHash, value): + quantType = VectorSetQuantization.Int8; + break; + default: + quantTypeRaw = value.GetString(); + quantType = VectorSetQuantization.Unknown; + break; + } + + break; + case hnsw_max_node_uid.Length + when hnsw_max_node_uid.Is(keyHash, key) && value.TryGetInt64(out var i64): + hnswMaxNodeUid = i64; + break; + } + } + + SetResult( + message, + new VectorSetInfo(quantType, quantTypeRaw, vectorDim, resultSize, maxLevel, vsetUid, hnswMaxNodeUid)); + return true; + } + + return false; + } + +#pragma warning disable CS8981, SA1134, SA1300, SA1303, SA1502 + // ReSharper disable InconsistentNaming - to better represent expected literals + // ReSharper disable IdentifierTypo + [FastHash] private static partial class bin { } + [FastHash] private static partial class f32 { } + [FastHash] private static partial class int8 { } + [FastHash] private static partial class size { } + [FastHash] private static partial class vset_uid { } + [FastHash] private static partial class max_level { } + [FastHash] private static partial class quant_type { } + [FastHash] private static partial class vector_dim { } + [FastHash] private static partial class hnsw_max_node_uid { } + // ReSharper restore InconsistentNaming + // ReSharper restore IdentifierTypo +#pragma warning restore CS8981, SA1134, SA1300, SA1303, SA1502 + } +} diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 627953941..650cba603 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -14,7 +14,7 @@ namespace StackExchange.Redis { - internal abstract class ResultProcessor + internal abstract partial class ResultProcessor { public static readonly ResultProcessor Boolean = new BooleanProcessor(), @@ -60,6 +60,8 @@ public static readonly ResultProcessor PubSubNumSub = new PubSubNumSubProcessor(), Int64DefaultNegativeOne = new Int64DefaultValueProcessor(-1); + public static readonly ResultProcessor Int32 = new Int32Processor(); + public static readonly ResultProcessor NullableDouble = new NullableDoubleProcessor(); @@ -91,12 +93,6 @@ public static readonly ResultProcessor public static readonly ResultProcessor RedisValueFromArray = new RedisValueFromArrayProcessor(); - public static readonly ResultProcessor> - Lease = new LeaseProcessor(); - - public static readonly ResultProcessor> - LeaseFromArray = new LeaseFromArrayProcessor(); - public static readonly ResultProcessor RedisValueArray = new RedisValueArrayProcessor(); @@ -700,7 +696,7 @@ public bool TryParse(in RawResult result, out T[]? pairs) count = (int)arr.Length; if (count == 0) { - return Array.Empty(); + return []; } bool interleaved = !(result.IsResp3 && AllowJaggedPairs && IsAllJaggedPairs(arr)); @@ -1384,6 +1380,26 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } + private class Int32Processor : ResultProcessor + { + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + switch (result.Resp2TypeBulkString) + { + case ResultType.Integer: + case ResultType.SimpleString: + case ResultType.BulkString: + if (result.TryGetInt64(out long i64)) + { + SetResult(message, checked((int)i64)); + return true; + } + break; + } + return false; + } + } + internal static ResultProcessor StreamTrimResult => Int32EnumProcessor.Instance; @@ -2058,41 +2074,6 @@ private static bool TryParsePrimaryReplica(in Sequence items, out Rol } } - private sealed class LeaseProcessor : ResultProcessor> - { - protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) - { - switch (result.Resp2TypeBulkString) - { - case ResultType.Integer: - case ResultType.SimpleString: - case ResultType.BulkString: - SetResult(message, result.AsLease()!); - return true; - } - return false; - } - } - - private sealed class LeaseFromArrayProcessor : ResultProcessor> - { - protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) - { - switch (result.Resp2TypeBulkString) - { - case ResultType.Array: - var items = result.GetItems(); - if (items.Length == 1) - { // treat an array of 1 like a single reply - SetResult(message, items[0].AsLease()!); - return true; - } - break; - } - return false; - } - } - private sealed class ScriptResultProcessor : ResultProcessor { public override bool SetResult(PhysicalConnection connection, Message message, in RawResult result) @@ -2136,7 +2117,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes if (result.IsNull) { // Server returns 'nil' if no entries are returned for the given stream. - SetResult(message, Array.Empty()); + SetResult(message, []); return true; } @@ -2241,7 +2222,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes if (result.IsNull) { // Nothing returned for any of the requested streams. The server returns 'nil'. - SetResult(message, Array.Empty()); + SetResult(message, []); return true; } @@ -2307,7 +2288,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes var entries = ParseRedisStreamEntries(items[1]); // [2] The array of message IDs deleted from the stream that were in the PEL. // This is not available in 6.2 so we need to be defensive when reading this part of the response. - var deletedIds = (items.Length == 3 ? items[2].GetItemsAsValues() : null) ?? Array.Empty(); + var deletedIds = (items.Length == 3 ? items[2].GetItemsAsValues() : null) ?? []; SetResult(message, new StreamAutoClaimResult(nextStartId, entries, deletedIds)); return true; @@ -2333,10 +2314,10 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes // [0] The next start ID. var nextStartId = items[0].AsRedisValue(); // [1] The array of claimed message IDs. - var claimedIds = items[1].GetItemsAsValues() ?? Array.Empty(); + var claimedIds = items[1].GetItemsAsValues() ?? []; // [2] The array of message IDs deleted from the stream that were in the PEL. // This is not available in 6.2 so we need to be defensive when reading this part of the response. - var deletedIds = (items.Length == 3 ? items[2].GetItemsAsValues() : null) ?? Array.Empty(); + var deletedIds = (items.Length == 3 ? items[2].GetItemsAsValues() : null) ?? []; SetResult(message, new StreamAutoClaimIdsOnlyResult(nextStartId, claimedIds, deletedIds)); return true; @@ -2644,7 +2625,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes pendingMessageCount: (int)arr[0].AsRedisValue(), lowestId: arr[1].AsRedisValue(), highestId: arr[2].AsRedisValue(), - consumers: consumers ?? Array.Empty()); + consumers: consumers ?? []); SetResult(message, pendingInfo); return true; @@ -2729,7 +2710,7 @@ protected static NameValueEntry[] ParseStreamEntryValues(in RawResult result) // 4) "18.2" if (result.Resp2TypeArray != ResultType.Array || result.IsNull) { - return Array.Empty(); + return []; } return StreamNameValueEntryProcessor.Instance.ParseArray(result, false, out _, null)!; // ! because we checked null above } @@ -2917,7 +2898,7 @@ private sealed class SentinelGetSentinelAddressesProcessor : ResultProcessor endPoints = new List(); + List endPoints = []; switch (result.Resp2TypeArray) { @@ -2951,7 +2932,7 @@ private sealed class SentinelGetReplicaAddressesProcessor : ResultProcessor endPoints = new List(); + List endPoints = []; switch (result.Resp2TypeArray) { @@ -3045,7 +3026,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes T[] arr; if (items.IsEmpty) { - arr = Array.Empty(); + arr = []; } else { diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 44efe09be..b13a12423 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -47,4 +47,8 @@ + + + + \ No newline at end of file diff --git a/src/StackExchange.Redis/VectorSetAddMessage.cs b/src/StackExchange.Redis/VectorSetAddMessage.cs new file mode 100644 index 000000000..0beb65205 --- /dev/null +++ b/src/StackExchange.Redis/VectorSetAddMessage.cs @@ -0,0 +1,168 @@ +using System; +using System.Runtime.InteropServices; +using System.Threading; + +namespace StackExchange.Redis; + +internal abstract class VectorSetAddMessage( + int db, + CommandFlags flags, + RedisKey key, + int? reducedDimensions, + VectorSetQuantization quantization, + int? buildExplorationFactor, + int? maxConnections, + bool useCheckAndSet) : Message(db, flags, RedisCommand.VADD) +{ + public override int ArgCount => GetArgCount(UseFp32); + + private int GetArgCount(bool packed) + { + var count = 2 + GetElementArgCount(packed); // key, element and either "FP32 {vector}" or VALUES {num}" + if (reducedDimensions.HasValue) count += 2; // [REDUCE {dim}] + + if (useCheckAndSet) count++; // [CAS] + count += quantization switch + { + VectorSetQuantization.None or VectorSetQuantization.Binary => 1, // [NOQUANT] or [BIN] + VectorSetQuantization.Int8 => 0, // implicit + _ => throw new ArgumentOutOfRangeException(nameof(quantization)), + }; + + if (buildExplorationFactor.HasValue) count += 2; // [EF {build-exploration-factor}] + count += GetAttributeArgCount(); // [SETATTR {attributes}] + if (maxConnections.HasValue) count += 2; // [M {numlinks}] + return count; + } + + public abstract int GetElementArgCount(bool packed); + public abstract int GetAttributeArgCount(); + + public override int GetHashSlot(ServerSelectionStrategy serverSelectionStrategy) + => serverSelectionStrategy.HashSlot(key); + + private static readonly bool CanUseFp32 = BitConverter.IsLittleEndian && CheckFp32(); + + private static bool CheckFp32() // check endianness with a known value + { + // ReSharper disable once CompareOfFloatsByEqualityOperator - expect exact + return MemoryMarshal.Cast("\0\0(B"u8)[0] == 42; + } + +#if DEBUG + private static int _fp32Disabled; + internal static bool UseFp32 => CanUseFp32 & Volatile.Read(ref _fp32Disabled) == 0; + internal static void SuppressFp32() => Interlocked.Increment(ref _fp32Disabled); + internal static void RestoreFp32() => Interlocked.Decrement(ref _fp32Disabled); +#else + internal static bool UseFp32 => CanUseFp32; + internal static void SuppressFp32() { } + internal static void RestoreFp32() { } +#endif + + protected abstract void WriteElement(bool packed, PhysicalConnection physical); + + protected override void WriteImpl(PhysicalConnection physical) + { + bool packed = UseFp32; // snapshot to avoid race in debug scenarios + physical.WriteHeader(Command, GetArgCount(packed)); + physical.Write(key); + if (reducedDimensions.HasValue) + { + physical.WriteBulkString("REDUCE"u8); + physical.WriteBulkString(reducedDimensions.GetValueOrDefault()); + } + + WriteElement(packed, physical); + if (useCheckAndSet) physical.WriteBulkString("CAS"u8); + + switch (quantization) + { + case VectorSetQuantization.Int8: + break; + case VectorSetQuantization.None: + physical.WriteBulkString("NOQUANT"u8); + break; + case VectorSetQuantization.Binary: + physical.WriteBulkString("BIN"u8); + break; + default: + throw new ArgumentOutOfRangeException(nameof(quantization)); + } + + if (buildExplorationFactor.HasValue) + { + physical.WriteBulkString("EF"u8); + physical.WriteBulkString(buildExplorationFactor.GetValueOrDefault()); + } + + WriteAttributes(physical); + + if (maxConnections.HasValue) + { + physical.WriteBulkString("M"u8); + physical.WriteBulkString(maxConnections.GetValueOrDefault()); + } + } + + protected abstract void WriteAttributes(PhysicalConnection physical); + + internal sealed class VectorSetAddMemberMessage( + int db, + CommandFlags flags, + RedisKey key, + int? reducedDimensions, + VectorSetQuantization quantization, + int? buildExplorationFactor, + int? maxConnections, + bool useCheckAndSet, + RedisValue element, + ReadOnlyMemory values, + string? attributesJson) : VectorSetAddMessage( + db, + flags, + key, + reducedDimensions, + quantization, + buildExplorationFactor, + maxConnections, + useCheckAndSet) + { + private readonly string? _attributesJson = string.IsNullOrWhiteSpace(attributesJson) ? null : attributesJson; + public override int GetElementArgCount(bool packed) + => 2 // "FP32 {vector}" or "VALUES {num}" + + (packed ? 0 : values.Length); // {vector...}" + + public override int GetAttributeArgCount() + => _attributesJson is null ? 0 : 2; // [SETATTR {attributes}] + + protected override void WriteElement(bool packed, PhysicalConnection physical) + { + if (packed) + { + physical.WriteBulkString("FP32"u8); + physical.WriteBulkString(MemoryMarshal.AsBytes(values.Span)); + } + else + { + physical.WriteBulkString("VALUES"u8); + physical.WriteBulkString(values.Length); + foreach (var val in values.Span) + { + physical.WriteBulkString(val); + } + } + + physical.WriteBulkString(element); + } + + protected override void WriteAttributes(PhysicalConnection physical) + { + if (_attributesJson is not null) + { + physical.WriteBulkString("SETATTR"u8); + physical.WriteBulkString(_attributesJson); + } + } + } +} diff --git a/src/StackExchange.Redis/VectorSetAddRequest.cs b/src/StackExchange.Redis/VectorSetAddRequest.cs new file mode 100644 index 000000000..987118c09 --- /dev/null +++ b/src/StackExchange.Redis/VectorSetAddRequest.cs @@ -0,0 +1,80 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis; + +/// +/// Represents the request for a vectorset add operation. +/// +[Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] +public abstract class VectorSetAddRequest +{ + // polymorphism left open for future, but needs to be handled internally + internal VectorSetAddRequest() + { + } + + /// + /// Add a member to the vectorset. + /// + /// The element name. + /// The vector data. + /// Optional JSON attributes for the element (SETATTR parameter). + public static VectorSetAddRequest Member( + RedisValue element, + ReadOnlyMemory values, +#if NET7_0_OR_GREATER + [StringSyntax(StringSyntaxAttribute.Json)] +#endif + string? attributesJson = null) + => new VectorSetAddMemberRequest(element, values, attributesJson); + + /// + /// Optional check-and-set mode for partial threading (CAS parameter). + /// + public bool UseCheckAndSet { get; set; } + + /// + /// Optional dimension reduction using random projection (REDUCE parameter). + /// + public int? ReducedDimensions { get; set; } + + /// + /// Quantization type - Int8 (Q8), None (NOQUANT), or Binary (BIN). Default: Int8. + /// + public VectorSetQuantization Quantization { get; set; } = VectorSetQuantization.Int8; + + /// + /// Optional HNSW build exploration factor (EF parameter, default: 200). + /// + public int? BuildExplorationFactor { get; set; } + + /// + /// Optional maximum connections per HNSW node (M parameter, default: 16). + /// + public int? MaxConnections { get; set; } + + // snapshot the values; I don't trust people not to mutate the object behind my back + internal abstract VectorSetAddMessage ToMessage(RedisKey key, int db, CommandFlags flags); + + internal sealed class VectorSetAddMemberRequest( + RedisValue element, + ReadOnlyMemory values, + string? attributesJson) + : VectorSetAddRequest + { + internal override VectorSetAddMessage ToMessage(RedisKey key, int db, CommandFlags flags) + => new VectorSetAddMessage.VectorSetAddMemberMessage( + db, + flags, + key, + ReducedDimensions, + Quantization, + BuildExplorationFactor, + MaxConnections, + UseCheckAndSet, + element, + values, + attributesJson); + } +} diff --git a/src/StackExchange.Redis/VectorSetInfo.cs b/src/StackExchange.Redis/VectorSetInfo.cs new file mode 100644 index 000000000..c9277eae5 --- /dev/null +++ b/src/StackExchange.Redis/VectorSetInfo.cs @@ -0,0 +1,54 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis; + +/// +/// Contains metadata information about a vectorset returned by VINFO command. +/// +[Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] +public readonly struct VectorSetInfo( + VectorSetQuantization quantization, + string? quantizationRaw, + int dimension, + long length, + int maxLevel, + long vectorSetUid, + long hnswMaxNodeUid) +{ + /// + /// The quantization type used for vectors in this vectorset. + /// + public VectorSetQuantization Quantization { get; } = quantization; + + /// + /// The raw representation of the quantization type used for vectors in this vectorset. This is only + /// populated if the is . + /// + public string? QuantizationRaw { get; } = quantizationRaw; + + /// + /// The number of dimensions in each vector. + /// + public int Dimension { get; } = dimension; + + /// + /// The number of elements (cardinality) in the vectorset. + /// + public long Length { get; } = length; + + /// + /// The maximum level in the HNSW graph structure. + /// + public int MaxLevel { get; } = maxLevel; + + /// + /// The unique identifier for this vectorset. + /// + public long VectorSetUid { get; } = vectorSetUid; + + /// + /// The maximum node unique identifier in the HNSW graph. + /// + public long HnswMaxNodeUid { get; } = hnswMaxNodeUid; +} diff --git a/src/StackExchange.Redis/VectorSetLink.cs b/src/StackExchange.Redis/VectorSetLink.cs new file mode 100644 index 000000000..c18e8a95f --- /dev/null +++ b/src/StackExchange.Redis/VectorSetLink.cs @@ -0,0 +1,24 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis; + +/// +/// Represents a link/connection between members in a vectorset with similarity score. +/// Used by VLINKS command with WITHSCORES option. +/// +[Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] +public readonly struct VectorSetLink(RedisValue member, double score) +{ + /// + /// The linked member name/identifier. + /// + public RedisValue Member { get; } = member; + + /// + /// The similarity score between the queried member and this linked member. + /// + public double Score { get; } = score; + + /// + public override string ToString() => $"{Member}: {Score}"; +} diff --git a/src/StackExchange.Redis/VectorSetQuantization.cs b/src/StackExchange.Redis/VectorSetQuantization.cs new file mode 100644 index 000000000..d78f4b34b --- /dev/null +++ b/src/StackExchange.Redis/VectorSetQuantization.cs @@ -0,0 +1,30 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis; + +/// +/// Specifies the quantization type for vectors in a vectorset. +/// +[Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] +public enum VectorSetQuantization +{ + /// + /// Unknown or unrecognized quantization type. + /// + Unknown = 0, + + /// + /// No quantization (full precision). This maps to "NOQUANT" or "f32". + /// + None = 1, + + /// + /// 8-bit integer quantization (default). This maps to "Q8" or "int8". + /// + Int8 = 2, + + /// + /// Binary quantization. This maps to "BIN" or "bin". + /// + Binary = 3, +} diff --git a/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs b/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs new file mode 100644 index 000000000..1bbc418d5 --- /dev/null +++ b/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs @@ -0,0 +1,263 @@ +using System; + +namespace StackExchange.Redis; + +internal abstract class VectorSetSimilaritySearchMessage( + int db, + CommandFlags flags, + VectorSetSimilaritySearchMessage.VsimFlags vsimFlags, + RedisKey key, + int count, + double epsilon, + int searchExplorationFactor, + string? filterExpression, + int maxFilteringEffort) : Message(db, flags, RedisCommand.VSIM) +{ + // For "FP32" and "VALUES" scenarios; in the future we might want other vector sizes / encodings - for + // example, there could be some "FP16" or "FP8" transport that requires a ROM-short or ROM-sbyte from + // the calling code. Or, as a convenience, we might want to allow ROM-double input, but transcode that + // to FP32 on the way out. + internal sealed class VectorSetSimilaritySearchBySingleVectorMessage( + int db, + CommandFlags flags, + VsimFlags vsimFlags, + RedisKey key, + ReadOnlyMemory vector, + int count, + double epsilon, + int searchExplorationFactor, + string? filterExpression, + int maxFilteringEffort) : VectorSetSimilaritySearchMessage(db, flags, vsimFlags, key, count, epsilon, + searchExplorationFactor, filterExpression, maxFilteringEffort) + { + internal override int GetSearchTargetArgCount(bool packed) => + packed ? 2 : 2 + vector.Length; // FP32 {vector} or VALUES {num} {vector} + + internal override void WriteSearchTarget(bool packed, PhysicalConnection physical) + { + if (packed) + { + physical.WriteBulkString("FP32"u8); + physical.WriteBulkString(System.Runtime.InteropServices.MemoryMarshal.AsBytes(vector.Span)); + } + else + { + physical.WriteBulkString("VALUES"u8); + physical.WriteBulkString(vector.Length); + foreach (var val in vector.Span) + { + physical.WriteBulkString(val); + } + } + } + } + + // for "ELE" scenarios + internal sealed class VectorSetSimilaritySearchByMemberMessage( + int db, + CommandFlags flags, + VsimFlags vsimFlags, + RedisKey key, + RedisValue member, + int count, + double epsilon, + int searchExplorationFactor, + string? filterExpression, + int maxFilteringEffort) : VectorSetSimilaritySearchMessage(db, flags, vsimFlags, key, count, epsilon, + searchExplorationFactor, filterExpression, maxFilteringEffort) + { + internal override int GetSearchTargetArgCount(bool packed) => 2; // ELE {member} + + internal override void WriteSearchTarget(bool packed, PhysicalConnection physical) + { + physical.WriteBulkString("ELE"u8); + physical.WriteBulkString(member); + } + } + + internal abstract int GetSearchTargetArgCount(bool packed); + internal abstract void WriteSearchTarget(bool packed, PhysicalConnection physical); + + public ResultProcessor?> GetResultProcessor() => + VectorSetSimilaritySearchProcessor.Instance; + + private sealed class VectorSetSimilaritySearchProcessor : ResultProcessor?> + { + // keep local, since we need to know what flags were being sent + public static readonly VectorSetSimilaritySearchProcessor Instance = new(); + private VectorSetSimilaritySearchProcessor() { } + + protected override bool SetResultCore(PhysicalConnection connection, Message message, in RawResult result) + { + if (result.Resp2TypeArray == ResultType.Array && message is VectorSetSimilaritySearchMessage vssm) + { + if (result.IsNull) + { + SetResult(message, null); + return true; + } + + bool withScores = vssm.HasFlag(VsimFlags.WithScores); + bool withAttribs = vssm.HasFlag(VsimFlags.WithAttributes); + + // in RESP3 mode (only), when both are requested, we get a sub-array per item; weird, but true + bool internalNesting = withScores && withAttribs && connection.Protocol is RedisProtocol.Resp3; + + int rowsPerItem = internalNesting + ? 2 + : 1 + ((withScores ? 1 : 0) + (withAttribs ? 1 : 0)); // each value is separate root element + + var items = result.GetItems(); + var length = checked((int)items.Length) / rowsPerItem; + var lease = Lease.Create(length, clear: false); + var target = lease.Span; + int count = 0; + var iter = items.GetEnumerator(); + for (int i = 0; i < target.Length && iter.MoveNext(); i++) + { + var member = iter.Current.AsRedisValue(); + double score = double.NaN; + string? attributesJson = null; + + if (internalNesting) + { + if (!iter.MoveNext() || iter.Current.Resp2TypeArray != ResultType.Array) break; + if (!iter.Current.IsNull) + { + var subArray = iter.Current.GetItems(); + if (subArray.Length >= 1 && !subArray[0].TryGetDouble(out score)) break; + if (subArray.Length >= 2) attributesJson = subArray[1].GetString(); + } + } + else + { + if (withScores) + { + if (!iter.MoveNext() || !iter.Current.TryGetDouble(out score)) break; + } + + if (withAttribs) + { + if (!iter.MoveNext()) break; + attributesJson = iter.Current.GetString(); + } + } + + target[i] = new VectorSetSimilaritySearchResult(member, score, attributesJson); + count++; + } + + if (count == target.Length) + { + SetResult(message, lease); + return true; + } + + lease.Dispose(); // failed to fill? + } + + return false; + } + } + + [Flags] + internal enum VsimFlags + { + None = 0, + Count = 1 << 0, + WithScores = 1 << 1, + WithAttributes = 1 << 2, + UseExactSearch = 1 << 3, + DisableThreading = 1 << 4, + Epsilon = 1 << 5, + SearchExplorationFactor = 1 << 6, + MaxFilteringEffort = 1 << 7, + FilterExpression = 1 << 8, + } + + private bool HasFlag(VsimFlags flag) => (vsimFlags & flag) != 0; + + public override int ArgCount => GetArgCount(VectorSetAddMessage.UseFp32); + + private int GetArgCount(bool packed) + { + int argCount = 1 + GetSearchTargetArgCount(packed); // {key} and whatever we need for the vector/element portion + if (HasFlag(VsimFlags.WithScores)) argCount++; // [WITHSCORES] + if (HasFlag(VsimFlags.WithAttributes)) argCount++; // [WITHATTRIBS] + if (HasFlag(VsimFlags.Count)) argCount += 2; // [COUNT {count}] + if (HasFlag(VsimFlags.Epsilon)) argCount += 2; // [EPSILON {epsilon}] + if (HasFlag(VsimFlags.SearchExplorationFactor)) argCount += 2; // [EF {search-exploration-factor}] + if (HasFlag(VsimFlags.FilterExpression)) argCount += 2; // [FILTER {filterExpression}] + if (HasFlag(VsimFlags.MaxFilteringEffort)) argCount += 2; // [FILTER-EF {max-filtering-effort}] + if (HasFlag(VsimFlags.UseExactSearch)) argCount++; // [TRUTH] + if (HasFlag(VsimFlags.DisableThreading)) argCount++; // [NOTHREAD] + return argCount; + } + + protected override void WriteImpl(PhysicalConnection physical) + { + // snapshot to avoid race in debug scenarios + bool packed = VectorSetAddMessage.UseFp32; + physical.WriteHeader(Command, GetArgCount(packed)); + + // Write key + physical.Write(key); + + // Write search target: either "ELE {member}" or vector data + WriteSearchTarget(packed, physical); + + if (HasFlag(VsimFlags.WithScores)) + { + physical.WriteBulkString("WITHSCORES"u8); + } + + if (HasFlag(VsimFlags.WithAttributes)) + { + physical.WriteBulkString("WITHATTRIBS"u8); + } + + // Write optional parameters + if (HasFlag(VsimFlags.Count)) + { + physical.WriteBulkString("COUNT"u8); + physical.WriteBulkString(count); + } + + if (HasFlag(VsimFlags.Epsilon)) + { + physical.WriteBulkString("EPSILON"u8); + physical.WriteBulkString(epsilon); + } + + if (HasFlag(VsimFlags.SearchExplorationFactor)) + { + physical.WriteBulkString("EF"u8); + physical.WriteBulkString(searchExplorationFactor); + } + + if (HasFlag(VsimFlags.FilterExpression)) + { + physical.WriteBulkString("FILTER"u8); + physical.WriteBulkString(filterExpression); + } + + if (HasFlag(VsimFlags.MaxFilteringEffort)) + { + physical.WriteBulkString("FILTER-EF"u8); + physical.WriteBulkString(maxFilteringEffort); + } + + if (HasFlag(VsimFlags.UseExactSearch)) + { + physical.WriteBulkString("TRUTH"u8); + } + + if (HasFlag(VsimFlags.DisableThreading)) + { + physical.WriteBulkString("NOTHREAD"u8); + } + } + + public override int GetHashSlot(ServerSelectionStrategy serverSelectionStrategy) + => serverSelectionStrategy.HashSlot(key); +} diff --git a/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs b/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs new file mode 100644 index 000000000..d0c0fd4cc --- /dev/null +++ b/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs @@ -0,0 +1,219 @@ +using System; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using VsimFlags = StackExchange.Redis.VectorSetSimilaritySearchMessage.VsimFlags; + +namespace StackExchange.Redis; + +/// +/// Represents the request for a vector similarity search operation. +/// +[Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] +public abstract class VectorSetSimilaritySearchRequest +{ + internal VectorSetSimilaritySearchRequest() + { + } // polymorphism left open for future, but needs to be handled internally + + private sealed class VectorSetSimilarityByMemberSearchRequest(RedisValue member) : VectorSetSimilaritySearchRequest + { + internal override VectorSetSimilaritySearchMessage ToMessage(RedisKey key, int db, CommandFlags flags) + => new VectorSetSimilaritySearchMessage.VectorSetSimilaritySearchByMemberMessage( + db, + flags, + _vsimFlags, + key, + member, + _count, + _epsilon, + _searchExplorationFactor, + _filterExpression, + _maxFilteringEffort); + } + + private sealed class VectorSetSimilarityVectorSingleSearchRequest(ReadOnlyMemory vector) + : VectorSetSimilaritySearchRequest + { + internal override VectorSetSimilaritySearchMessage ToMessage(RedisKey key, int db, CommandFlags flags) + => new VectorSetSimilaritySearchMessage.VectorSetSimilaritySearchBySingleVectorMessage( + db, + flags, + _vsimFlags, + key, + vector, + _count, + _epsilon, + _searchExplorationFactor, + _filterExpression, + _maxFilteringEffort); + } + + // snapshot the values; I don't trust people not to mutate the object behind my back + internal abstract VectorSetSimilaritySearchMessage ToMessage(RedisKey key, int db, CommandFlags flags); + + /// + /// Create a request to search by an existing member in the index. + /// + /// The member to search for. + public static VectorSetSimilaritySearchRequest ByMember(RedisValue member) + => new VectorSetSimilarityByMemberSearchRequest(member); + + /// + /// Create a request to search by a vector value. + /// + /// The vector value to search for. + public static VectorSetSimilaritySearchRequest ByVector(ReadOnlyMemory vector) + => new VectorSetSimilarityVectorSingleSearchRequest(vector); + + private VsimFlags _vsimFlags; + + // use the flags to reduce storage from N*Nullable + private int _searchExplorationFactor, _maxFilteringEffort, _count; + private double _epsilon; + + private bool HasFlag(VsimFlags flag) => (_vsimFlags & flag) != 0; + + private void SetFlag(VsimFlags flag, bool value) + { + if (value) + { + _vsimFlags |= flag; + } + else + { + _vsimFlags &= ~flag; + } + } + + /// + /// The number of similar vectors to return (COUNT parameter). + /// + public int? Count + { + get => HasFlag(VsimFlags.Count) ? _count : null; + set + { + if (value.HasValue) + { + _count = value.GetValueOrDefault(); + SetFlag(VsimFlags.Count, true); + } + else + { + SetFlag(VsimFlags.Count, false); + } + } + } + + /// + /// Whether to include similarity scores in the results (WITHSCORES parameter). + /// + public bool WithScores + { + get => HasFlag(VsimFlags.WithScores); + set => SetFlag(VsimFlags.WithScores, value); + } + + /// + /// Whether to include JSON attributes in the results (WITHATTRIBS parameter). + /// + public bool WithAttributes + { + get => HasFlag(VsimFlags.WithAttributes); + set => SetFlag(VsimFlags.WithAttributes, value); + } + + /// + /// Optional similarity threshold - only return elements with similarity >= (1 - epsilon) (EPSILON parameter). + /// + public double? Epsilon + { + get => HasFlag(VsimFlags.Epsilon) ? _epsilon : null; + set + { + if (value.HasValue) + { + _epsilon = value.GetValueOrDefault(); + SetFlag(VsimFlags.Epsilon, true); + } + else + { + SetFlag(VsimFlags.Epsilon, false); + } + } + } + + /// + /// Optional search exploration factor for better recall (EF parameter). + /// + public int? SearchExplorationFactor + { + get => HasFlag(VsimFlags.SearchExplorationFactor) ? _searchExplorationFactor : null; + set + { + if (value.HasValue) + { + _searchExplorationFactor = value.GetValueOrDefault(); + SetFlag(VsimFlags.SearchExplorationFactor, true); + } + else + { + SetFlag(VsimFlags.SearchExplorationFactor, false); + } + } + } + + /// + /// Optional maximum filtering attempts (FILTER-EF parameter). + /// + public int? MaxFilteringEffort + { + get => HasFlag(VsimFlags.MaxFilteringEffort) ? _maxFilteringEffort : null; + set + { + if (value.HasValue) + { + _maxFilteringEffort = value.GetValueOrDefault(); + SetFlag(VsimFlags.MaxFilteringEffort, true); + } + else + { + SetFlag(VsimFlags.MaxFilteringEffort, false); + } + } + } + + private string? _filterExpression; + + /// + /// Optional filter expression to restrict results (FILTER parameter); . + /// + public string? FilterExpression + { + get => _filterExpression; + set + { + _filterExpression = value; + SetFlag(VsimFlags.FilterExpression, !string.IsNullOrWhiteSpace(value)); + } + } + + /// + /// Whether to use exact linear scan instead of HNSW (TRUTH parameter). + /// + public bool UseExactSearch + { + get => HasFlag(VsimFlags.UseExactSearch); + set => SetFlag(VsimFlags.UseExactSearch, value); + } + + /// + /// Whether to run search in main thread (NOTHREAD parameter). + /// + [Browsable(false), EditorBrowsable(EditorBrowsableState.Advanced)] + public bool DisableThreading + { + get => HasFlag(VsimFlags.DisableThreading); + set => SetFlag(VsimFlags.DisableThreading, value); + } +} diff --git a/src/StackExchange.Redis/VectorSetSimilaritySearchResult.cs b/src/StackExchange.Redis/VectorSetSimilaritySearchResult.cs new file mode 100644 index 000000000..fd912898b --- /dev/null +++ b/src/StackExchange.Redis/VectorSetSimilaritySearchResult.cs @@ -0,0 +1,44 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StackExchange.Redis; + +/// +/// Represents a result from vector similarity search operations. +/// +[Experimental(Experiments.VectorSets, UrlFormat = Experiments.UrlFormat)] +public readonly struct VectorSetSimilaritySearchResult(RedisValue member, double score = double.NaN, string? attributesJson = null) +{ + /// + /// The member name/identifier in the vectorset. + /// + public RedisValue Member { get; } = member; + + /// + /// The similarity score (0-1) when WITHSCORES is used, NaN otherwise. + /// A score of 1 means identical vectors, 0 means opposite vectors. + /// + public double Score { get; } = score; + + /// + /// The JSON attributes associated with the member when WITHATTRIBS is used, null otherwise. + /// +#if NET7_0_OR_GREATER + [StringSyntax(StringSyntaxAttribute.Json)] +#endif + public string? AttributesJson { get; } = attributesJson; + + /// + public override string ToString() + { + if (double.IsNaN(Score)) + { + return AttributesJson is null + ? Member.ToString() + : $"{Member}: {AttributesJson}"; + } + + return AttributesJson is null + ? $"{Member} ({Score})" + : $"{Member} ({Score}): {AttributesJson}"; + } +} diff --git a/tests/StackExchange.Redis.Benchmarks/FastHashBenchmarks.cs b/tests/StackExchange.Redis.Benchmarks/FastHashBenchmarks.cs new file mode 100644 index 000000000..78877f163 --- /dev/null +++ b/tests/StackExchange.Redis.Benchmarks/FastHashBenchmarks.cs @@ -0,0 +1,139 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Text; +using BenchmarkDotNet.Attributes; + +namespace StackExchange.Redis.Benchmarks; + +[Config(typeof(CustomConfig))] +public class FastHashBenchmarks +{ + private const string SharedString = "some-typical-data-for-comparisons"; + private static readonly byte[] SharedUtf8; + private static readonly ReadOnlySequence SharedMultiSegment; + + static FastHashBenchmarks() + { + SharedUtf8 = Encoding.UTF8.GetBytes(SharedString); + + var first = new Segment(SharedUtf8.AsMemory(0, 1), null); + var second = new Segment(SharedUtf8.AsMemory(1), first); + SharedMultiSegment = new ReadOnlySequence(first, 0, second, second.Memory.Length); + } + + private sealed class Segment : ReadOnlySequenceSegment + { + public Segment(ReadOnlyMemory memory, Segment? previous) + { + Memory = memory; + if (previous is { }) + { + RunningIndex = previous.RunningIndex + previous.Memory.Length; + previous.Next = this; + } + } + } + + private string _sourceString = SharedString; + private ReadOnlyMemory _sourceBytes = SharedUtf8; + private ReadOnlySequence _sourceMultiSegmentBytes = SharedMultiSegment; + private ReadOnlySequence SingleSegmentBytes => new(_sourceBytes); + + [GlobalSetup] + public void Setup() + { + _sourceString = SharedString.Substring(0, Size); + _sourceBytes = SharedUtf8.AsMemory(0, Size); + _sourceMultiSegmentBytes = SharedMultiSegment.Slice(0, Size); + +#pragma warning disable CS0618 // Type or member is obsolete + var bytes = _sourceBytes.Span; + var expected = FastHash.Hash64Fallback(bytes); + + Assert(bytes.Hash64(), nameof(FastHash.Hash64)); + Assert(FastHash.Hash64Unsafe(bytes), nameof(FastHash.Hash64Unsafe)); +#pragma warning restore CS0618 // Type or member is obsolete + Assert(SingleSegmentBytes.Hash64(), nameof(FastHash.Hash64) + " (single segment)"); + Assert(_sourceMultiSegmentBytes.Hash64(), nameof(FastHash.Hash64) + " (multi segment)"); + + void Assert(long actual, string name) + { + if (actual != expected) + { + throw new InvalidOperationException($"Hash mismatch for {name}, {expected} != {actual}"); + } + } + } + + [ParamsSource(nameof(Sizes))] + public int Size { get; set; } = 7; + + public IEnumerable Sizes => [0, 1, 2, 3, 4, 5, 6, 7, 8, 16]; + + private const int OperationsPerInvoke = 1024; + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke, Baseline = true)] + public void String() + { + var val = _sourceString; + for (int i = 0; i < OperationsPerInvoke; i++) + { + _ = val.GetHashCode(); + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public void Hash64() + { + var val = _sourceBytes.Span; + for (int i = 0; i < OperationsPerInvoke; i++) + { + _ = val.Hash64(); + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public void Hash64Unsafe() + { + var val = _sourceBytes.Span; + for (int i = 0; i < OperationsPerInvoke; i++) + { +#pragma warning disable CS0618 // Type or member is obsolete + _ = FastHash.Hash64Unsafe(val); +#pragma warning restore CS0618 // Type or member is obsolete + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public void Hash64Fallback() + { + var val = _sourceBytes.Span; + for (int i = 0; i < OperationsPerInvoke; i++) + { +#pragma warning disable CS0618 // Type or member is obsolete + _ = FastHash.Hash64Fallback(val); +#pragma warning restore CS0618 // Type or member is obsolete + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public void Hash64_SingleSegment() + { + var val = SingleSegmentBytes; + for (int i = 0; i < OperationsPerInvoke; i++) + { + _ = val.Hash64(); + } + } + + [Benchmark(OperationsPerInvoke = OperationsPerInvoke)] + public void Hash64_MultiSegment() + { + var val = _sourceMultiSegmentBytes; + for (int i = 0; i < OperationsPerInvoke; i++) + { + _ = val.Hash64(); + } + } +} diff --git a/tests/StackExchange.Redis.Benchmarks/Program.cs b/tests/StackExchange.Redis.Benchmarks/Program.cs index 622d7d593..311202877 100644 --- a/tests/StackExchange.Redis.Benchmarks/Program.cs +++ b/tests/StackExchange.Redis.Benchmarks/Program.cs @@ -1,10 +1,25 @@ -using System.Reflection; +using System; +using System.Reflection; using BenchmarkDotNet.Running; namespace StackExchange.Redis.Benchmarks { internal static class Program { - private static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args); + private static void Main(string[] args) + { +#if DEBUG + var obj = new FastHashBenchmarks(); + foreach (var size in obj.Sizes) + { + Console.WriteLine($"Size: {size}"); + obj.Size = size; + obj.Setup(); + obj.Hash64(); + } +#else + BenchmarkSwitcher.FromAssembly(typeof(Program).GetTypeInfo().Assembly).Run(args); +#endif + } } } diff --git a/tests/StackExchange.Redis.Benchmarks/StackExchange.Redis.Benchmarks.csproj b/tests/StackExchange.Redis.Benchmarks/StackExchange.Redis.Benchmarks.csproj index be9a3081b..8b335ab02 100644 --- a/tests/StackExchange.Redis.Benchmarks/StackExchange.Redis.Benchmarks.csproj +++ b/tests/StackExchange.Redis.Benchmarks/StackExchange.Redis.Benchmarks.csproj @@ -5,6 +5,7 @@ Release Exe true + enable diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 995b66a5a..f9ca738ba 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -442,6 +442,16 @@ public async Task GetInfo() } var cpuCount = cpu.Count(); Assert.True(cpuCount > 2); + if (cpu.Key != "CPU") + { + // seem to be seeing this in logs; add lots of detail + var sb = new StringBuilder("Expected CPU, got ").AppendLine(cpu.Key); + foreach (var setting in cpu) + { + sb.Append(setting.Key).Append('=').AppendLine(setting.Value); + } + Assert.Fail(sb.ToString()); + } Assert.Equal("CPU", cpu.Key); Assert.Contains(cpu, x => x.Key == "used_cpu_sys"); Assert.Contains(cpu, x => x.Key == "used_cpu_user"); diff --git a/tests/StackExchange.Redis.Tests/FastHashTests.cs b/tests/StackExchange.Redis.Tests/FastHashTests.cs new file mode 100644 index 000000000..418198cfd --- /dev/null +++ b/tests/StackExchange.Redis.Tests/FastHashTests.cs @@ -0,0 +1,112 @@ +using System; +using System.Runtime.InteropServices; +using System.Text; +using Xunit; + +#pragma warning disable CS8981, SA1134, SA1300, SA1303, SA1502 // names are weird in this test! +// ReSharper disable InconsistentNaming - to better represent expected literals +// ReSharper disable IdentifierTypo +namespace StackExchange.Redis.Tests; + +public partial class FastHashTests +{ + // note: if the hashing algorithm changes, we can update the last parameter freely; it doesn't matter + // what it *is* - what matters is that we can see that it has entropy between different values + [Theory] + [InlineData(1, a.Length, a.Text, a.Hash, 97)] + [InlineData(2, ab.Length, ab.Text, ab.Hash, 25185)] + [InlineData(3, abc.Length, abc.Text, abc.Hash, 6513249)] + [InlineData(4, abcd.Length, abcd.Text, abcd.Hash, 1684234849)] + [InlineData(5, abcde.Length, abcde.Text, abcde.Hash, 435475931745)] + [InlineData(6, abcdef.Length, abcdef.Text, abcdef.Hash, 112585661964897)] + [InlineData(7, abcdefg.Length, abcdefg.Text, abcdefg.Hash, 29104508263162465)] + [InlineData(8, abcdefgh.Length, abcdefgh.Text, abcdefgh.Hash, 7523094288207667809)] + + [InlineData(1, x.Length, x.Text, x.Hash, 120)] + [InlineData(2, xx.Length, xx.Text, xx.Hash, 30840)] + [InlineData(3, xxx.Length, xxx.Text, xxx.Hash, 7895160)] + [InlineData(4, xxxx.Length, xxxx.Text, xxxx.Hash, 2021161080)] + [InlineData(5, xxxxx.Length, xxxxx.Text, xxxxx.Hash, 517417236600)] + [InlineData(6, xxxxxx.Length, xxxxxx.Text, xxxxxx.Hash, 132458812569720)] + [InlineData(7, xxxxxxx.Length, xxxxxxx.Text, xxxxxxx.Hash, 33909456017848440)] + [InlineData(8, xxxxxxxx.Length, xxxxxxxx.Text, xxxxxxxx.Hash, 8680820740569200760)] + + [InlineData(3, 窓.Length, 窓.Text, 窓.Hash, 9677543, "窓")] + [InlineData(20, abcdefghijklmnopqrst.Length, abcdefghijklmnopqrst.Text, abcdefghijklmnopqrst.Hash, 7523094288207667809)] + + // show that foo_bar is interpreted as foo-bar + [InlineData(7, foo_bar.Length, foo_bar.Text, foo_bar.Hash, 32195221641981798, "foo-bar", nameof(foo_bar))] + [InlineData(7, foo_bar_hyphen.Length, foo_bar_hyphen.Text, foo_bar_hyphen.Hash, 32195221641981798, "foo-bar", nameof(foo_bar_hyphen))] + [InlineData(7, foo_bar_underscore.Length, foo_bar_underscore.Text, foo_bar_underscore.Hash, 32195222480842598, "foo_bar", nameof(foo_bar_underscore))] + public void Validate(int expectedLength, int actualLength, string actualValue, long actualHash, long expectedHash, string? expectedValue = null, string originForDisambiguation = "") + { + _ = originForDisambiguation; // to allow otherwise-identical test data to coexist + Assert.Equal(expectedLength, actualLength); + Assert.Equal(expectedHash, actualHash); + var bytes = Encoding.UTF8.GetBytes(actualValue); + Assert.Equal(expectedLength, bytes.Length); + Assert.Equal(expectedHash, FastHash.Hash64(bytes)); +#pragma warning disable CS0618 // Type or member is obsolete + Assert.Equal(expectedHash, FastHash.Hash64Fallback(bytes)); +#pragma warning restore CS0618 // Type or member is obsolete + if (expectedValue is not null) + { + Assert.Equal(expectedValue, actualValue); + } + } + + [Fact] + public void FastHashIs_Short() + { + ReadOnlySpan value = "abc"u8; + var hash = value.Hash64(); + Assert.Equal(abc.Hash, hash); + Assert.True(abc.Is(hash, value)); + + value = "abz"u8; + hash = value.Hash64(); + Assert.NotEqual(abc.Hash, hash); + Assert.False(abc.Is(hash, value)); + } + + [Fact] + public void FastHashIs_Long() + { + ReadOnlySpan value = "abcdefghijklmnopqrst"u8; + var hash = value.Hash64(); + Assert.Equal(abcdefghijklmnopqrst.Hash, hash); + Assert.True(abcdefghijklmnopqrst.Is(hash, value)); + + value = "abcdefghijklmnopqrsz"u8; + hash = value.Hash64(); + Assert.Equal(abcdefghijklmnopqrst.Hash, hash); // hash collision, fine + Assert.False(abcdefghijklmnopqrst.Is(hash, value)); + } + + [FastHash] private static partial class a { } + [FastHash] private static partial class ab { } + [FastHash] private static partial class abc { } + [FastHash] private static partial class abcd { } + [FastHash] private static partial class abcde { } + [FastHash] private static partial class abcdef { } + [FastHash] private static partial class abcdefg { } + [FastHash] private static partial class abcdefgh { } + + [FastHash] private static partial class abcdefghijklmnopqrst { } + + // show that foo_bar and foo-bar are different + [FastHash] private static partial class foo_bar { } + [FastHash("foo-bar")] private static partial class foo_bar_hyphen { } + [FastHash("foo_bar")] private static partial class foo_bar_underscore { } + + [FastHash] private static partial class 窓 { } + + [FastHash] private static partial class x { } + [FastHash] private static partial class xx { } + [FastHash] private static partial class xxx { } + [FastHash] private static partial class xxxx { } + [FastHash] private static partial class xxxxx { } + [FastHash] private static partial class xxxxxx { } + [FastHash] private static partial class xxxxxxx { } + [FastHash] private static partial class xxxxxxxx { } +} diff --git a/tests/StackExchange.Redis.Tests/KeyPrefixedVectorSetTests.cs b/tests/StackExchange.Redis.Tests/KeyPrefixedVectorSetTests.cs new file mode 100644 index 000000000..b4ff2091b --- /dev/null +++ b/tests/StackExchange.Redis.Tests/KeyPrefixedVectorSetTests.cs @@ -0,0 +1,214 @@ +using System; +using System.Text; +using NSubstitute; +using Xunit; + +namespace StackExchange.Redis.Tests +{ + [Collection(nameof(SubstituteDependentCollection))] + public sealed class KeyPrefixedVectorSetTests + { + private readonly IDatabase mock; + private readonly IDatabase prefixed; + + public KeyPrefixedVectorSetTests() + { + mock = Substitute.For(); + prefixed = new KeyspaceIsolation.KeyPrefixedDatabase(mock, Encoding.UTF8.GetBytes("prefix:")); + } + + [Fact] + public void VectorSetAdd_Fp32() + { + if (BitConverter.IsLittleEndian) + { + Assert.True(VectorSetAddMessage.UseFp32); +#if DEBUG // can be suppressed + VectorSetAddMessage.SuppressFp32(); + Assert.False(VectorSetAddMessage.UseFp32); + VectorSetAddMessage.RestoreFp32(); + Assert.True(VectorSetAddMessage.UseFp32); +#endif + } + else + { + Assert.False(VectorSetAddMessage.UseFp32); + } + } + + [Fact] + public void VectorSetAdd_BasicCall() + { + var vector = new[] { 1.0f, 2.0f, 3.0f }.AsMemory(); + + var request = VectorSetAddRequest.Member("element1", vector); + prefixed.VectorSetAdd("vectorset", request); + + mock.Received().VectorSetAdd( + "prefix:vectorset", + request); + } + + [Fact] + public void VectorSetAdd_WithAllParameters() + { + var vector = new[] { 1.0f, 2.0f, 3.0f }.AsMemory(); + var attributes = """{"category":"test"}"""; + + var request = VectorSetAddRequest.Member( + "element1", + vector, + attributes); + request.ReducedDimensions = 64; + request.Quantization = VectorSetQuantization.Binary; + request.BuildExplorationFactor = 300; + request.MaxConnections = 32; + request.UseCheckAndSet = true; + prefixed.VectorSetAdd( + "vectorset", + request, + flags: CommandFlags.FireAndForget); + + mock.Received().VectorSetAdd( + "prefix:vectorset", + request, + CommandFlags.FireAndForget); + } + + [Fact] + public void VectorSetLength() + { + prefixed.VectorSetLength("vectorset"); + mock.Received().VectorSetLength("prefix:vectorset"); + } + + [Fact] + public void VectorSetDimension() + { + prefixed.VectorSetDimension("vectorset"); + mock.Received().VectorSetDimension("prefix:vectorset"); + } + + [Fact] + public void VectorSetGetApproximateVector() + { + prefixed.VectorSetGetApproximateVector("vectorset", "member1"); + mock.Received().VectorSetGetApproximateVector("prefix:vectorset", "member1"); + } + + [Fact] + public void VectorSetGetAttributesJson() + { + prefixed.VectorSetGetAttributesJson("vectorset", "member1"); + mock.Received().VectorSetGetAttributesJson("prefix:vectorset", "member1"); + } + + [Fact] + public void VectorSetInfo() + { + prefixed.VectorSetInfo("vectorset"); + mock.Received().VectorSetInfo("prefix:vectorset"); + } + + [Fact] + public void VectorSetContains() + { + prefixed.VectorSetContains("vectorset", "member1"); + mock.Received().VectorSetContains("prefix:vectorset", "member1"); + } + + [Fact] + public void VectorSetGetLinks() + { + prefixed.VectorSetGetLinks("vectorset", "member1"); + mock.Received().VectorSetGetLinks("prefix:vectorset", "member1"); + } + + [Fact] + public void VectorSetGetLinksWithScores() + { + prefixed.VectorSetGetLinksWithScores("vectorset", "member1"); + mock.Received().VectorSetGetLinksWithScores("prefix:vectorset", "member1"); + } + + [Fact] + public void VectorSetRandomMember() + { + prefixed.VectorSetRandomMember("vectorset"); + mock.Received().VectorSetRandomMember("prefix:vectorset"); + } + + [Fact] + public void VectorSetRandomMembers() + { + prefixed.VectorSetRandomMembers("vectorset", 5); + mock.Received().VectorSetRandomMembers("prefix:vectorset", 5); + } + + [Fact] + public void VectorSetRemove() + { + prefixed.VectorSetRemove("vectorset", "member1"); + mock.Received().VectorSetRemove("prefix:vectorset", "member1"); + } + + [Fact] + public void VectorSetSetAttributesJson() + { + var attributes = """{"category":"test"}"""; + + prefixed.VectorSetSetAttributesJson("vectorset", "member1", attributes); + mock.Received().VectorSetSetAttributesJson("prefix:vectorset", "member1", attributes); + } + + [Fact] + public void VectorSetSimilaritySearchByVector() + { + var vector = new[] { 1.0f, 2.0f, 3.0f }.AsMemory(); + + var query = VectorSetSimilaritySearchRequest.ByVector(vector); + prefixed.VectorSetSimilaritySearch( + "vectorset", + query); + mock.Received().VectorSetSimilaritySearch( + "prefix:vectorset", + query); + } + + [Fact] + public void VectorSetSimilaritySearchByMember() + { + var query = VectorSetSimilaritySearchRequest.ByMember("member1"); + query.Count = 5; + query.WithScores = true; + query.WithAttributes = true; + query.Epsilon = 0.1; + query.SearchExplorationFactor = 400; + query.FilterExpression = "category='test'"; + query.MaxFilteringEffort = 1000; + query.UseExactSearch = true; + query.DisableThreading = true; + prefixed.VectorSetSimilaritySearch( + "vectorset", + query, + CommandFlags.FireAndForget); + mock.Received().VectorSetSimilaritySearch( + "prefix:vectorset", + query, + CommandFlags.FireAndForget); + } + + [Fact] + public void VectorSetSimilaritySearchByVector_DefaultParameters() + { + var vector = new[] { 1.0f, 2.0f }.AsMemory(); + + // Test that default parameters work correctly + var query = VectorSetSimilaritySearchRequest.ByVector(vector); + prefixed.VectorSetSimilaritySearch("vectorset", query); + mock.Received().VectorSetSimilaritySearch( + "prefix:vectorset", + query); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/NamingTests.cs b/tests/StackExchange.Redis.Tests/NamingTests.cs index d0474f782..9d9e032ad 100644 --- a/tests/StackExchange.Redis.Tests/NamingTests.cs +++ b/tests/StackExchange.Redis.Tests/NamingTests.cs @@ -193,7 +193,8 @@ private void CheckMethod(MethodInfo method, bool isAsync) || shortName.StartsWith("Script") || shortName.StartsWith("SortedSet") || shortName.StartsWith("String") - || shortName.StartsWith("Stream"); + || shortName.StartsWith("Stream") + || shortName.StartsWith("VectorSet"); Log(fullName + ": " + (isValid ? "valid" : "invalid")); Assert.True(isValid, fullName + ":Prefix"); break; diff --git a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj index 50d4ae3d1..f6e38236b 100644 --- a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj +++ b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj @@ -30,5 +30,6 @@ + diff --git a/tests/StackExchange.Redis.Tests/TestBase.cs b/tests/StackExchange.Redis.Tests/TestBase.cs index 94a14ee32..68dbb6055 100644 --- a/tests/StackExchange.Redis.Tests/TestBase.cs +++ b/tests/StackExchange.Redis.Tests/TestBase.cs @@ -51,7 +51,19 @@ public static void Log(TextWriter output, string message) output?.WriteLine(Time() + ": " + message); } } - protected void Log(string? message, params object[] args) => Output.WriteLine(Time() + ": " + message, args); + + protected void Log(string? message, params object[] args) + { + if (args is { Length: > 0 }) + { + Output.WriteLine(Time() + ": " + message, args); + } + else + { + // avoid "not intended as a format specifier" scenarios + Output.WriteLine(Time() + ": " + message); + } + } protected ProfiledCommandEnumerable Log(ProfilingSession session) { diff --git a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs new file mode 100644 index 000000000..12eda7147 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs @@ -0,0 +1,675 @@ +using System; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Xunit; + +namespace StackExchange.Redis.Tests; + +[RunPerProtocol] +public sealed class VectorSetIntegrationTests(ITestOutputHelper output) : TestBase(output) +{ + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task VectorSetAdd_BasicOperation(bool suppressFp32) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + // Clean up any existing data + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f, 4.0f }; + + if (suppressFp32) VectorSetAddMessage.SuppressFp32(); + try + { + var request = VectorSetAddRequest.Member("element1", vector.AsMemory(), null); + var result = await db.VectorSetAddAsync(key, request); + + Assert.True(result); + } + finally + { + if (suppressFp32) VectorSetAddMessage.RestoreFp32(); + } + } + + [Fact] + public async Task VectorSetAdd_WithAttributes() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f, 4.0f }; + var attributes = """{"category":"test","id":123}"""; + + var request = VectorSetAddRequest.Member("element1", vector.AsMemory(), attributes); + var result = await db.VectorSetAddAsync(key, request); + + Assert.True(result); + + // Verify attributes were stored + var retrievedAttributes = await db.VectorSetGetAttributesJsonAsync(key, "element1"); + Assert.Equal(attributes, retrievedAttributes); + } + + [Theory] + [InlineData(VectorSetQuantization.Int8)] + [InlineData(VectorSetQuantization.None)] + [InlineData(VectorSetQuantization.Binary)] + public async Task VectorSetAdd_WithEverything(VectorSetQuantization quantization) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f, 4.0f }; + var attributes = """{"category":"test","id":123}"""; + + var request = VectorSetAddRequest.Member( + "element1", + vector.AsMemory(), + attributes); + request.Quantization = quantization; + request.ReducedDimensions = 64; + request.BuildExplorationFactor = 300; + request.MaxConnections = 32; + request.UseCheckAndSet = true; + var result = await db.VectorSetAddAsync( + key, + request); + + Assert.True(result); + + // Verify attributes were stored + var retrievedAttributes = await db.VectorSetGetAttributesJsonAsync(key, "element1"); + Assert.Equal(attributes, retrievedAttributes); + } + + [Fact] + public async Task VectorSetLength_EmptySet() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var length = await db.VectorSetLengthAsync(key); + Assert.Equal(0, length); + } + + [Fact] + public async Task VectorSetLength_WithElements() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector1 = new[] { 1.0f, 2.0f, 3.0f }; + var vector2 = new[] { 4.0f, 5.0f, 6.0f }; + + var request = VectorSetAddRequest.Member("element1", vector1.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory()); + await db.VectorSetAddAsync(key, request); + + var length = await db.VectorSetLengthAsync(key); + Assert.Equal(2, length); + } + + [Fact] + public async Task VectorSetDimension() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + var request = VectorSetAddRequest.Member("element1", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + + var dimension = await db.VectorSetDimensionAsync(key); + Assert.Equal(5, dimension); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task VectorSetContains(bool suppressFp32) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + if (suppressFp32) VectorSetAddMessage.SuppressFp32(); + try + { + var request = VectorSetAddRequest.Member("element1", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + + var exists = await db.VectorSetContainsAsync(key, "element1"); + var notExists = await db.VectorSetContainsAsync(key, "element2"); + + Assert.True(exists); + Assert.False(notExists); + } + finally + { + if (suppressFp32) VectorSetAddMessage.RestoreFp32(); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task VectorSetGetApproximateVector(bool suppressFp32) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var originalVector = new[] { 1.0f, 2.0f, 3.0f, 4.0f }; + if (suppressFp32) VectorSetAddMessage.SuppressFp32(); + try + { + var request = VectorSetAddRequest.Member("element1", originalVector.AsMemory()); + await db.VectorSetAddAsync(key, request); + + using var retrievedLease = await db.VectorSetGetApproximateVectorAsync(key, "element1"); + + Assert.NotNull(retrievedLease); + var retrievedVector = retrievedLease.Span; + + Assert.Equal(originalVector.Length, retrievedVector.Length); + // Note: Due to quantization, values might not be exactly equal + for (int i = 0; i < originalVector.Length; i++) + { + Assert.True( + Math.Abs(originalVector[i] - retrievedVector[i]) < 0.1f, + $"Vector component {i} differs too much: expected {originalVector[i]}, got {retrievedVector[i]}"); + } + } + finally + { + if (suppressFp32) VectorSetAddMessage.RestoreFp32(); + } + } + + [Fact] + public async Task VectorSetRemove() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var request = VectorSetAddRequest.Member("element1", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + + var removed = await db.VectorSetRemoveAsync(key, "element1"); + Assert.True(removed); + + removed = await db.VectorSetRemoveAsync(key, "element1"); + Assert.False(removed); + + var exists = await db.VectorSetContainsAsync(key, "element1"); + Assert.False(exists); + } + + [Theory] + [InlineData(VectorSetQuantization.Int8)] + [InlineData(VectorSetQuantization.Binary)] + [InlineData(VectorSetQuantization.None)] + public async Task VectorSetInfo(VectorSetQuantization quantization) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + var request = VectorSetAddRequest.Member("element1", vector.AsMemory()); + request.Quantization = quantization; + await db.VectorSetAddAsync(key, request); + + var info = await db.VectorSetInfoAsync(key); + + Assert.NotNull(info); + var v = info.GetValueOrDefault(); + Assert.Equal(5, v.Dimension); + Assert.Equal(1, v.Length); + Assert.Equal(quantization, v.Quantization); + Assert.Null(v.QuantizationRaw); // Should be null for known quant types + + Assert.NotEqual(0, v.VectorSetUid); + Assert.NotEqual(0, v.HnswMaxNodeUid); + } + + [Fact] + public async Task VectorSetRandomMember() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector1 = new[] { 1.0f, 2.0f, 3.0f }; + var vector2 = new[] { 4.0f, 5.0f, 6.0f }; + + var request = VectorSetAddRequest.Member("element1", vector1.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory()); + await db.VectorSetAddAsync(key, request); + + var randomMember = await db.VectorSetRandomMemberAsync(key); + Assert.True(randomMember == "element1" || randomMember == "element2"); + } + + [Fact] + public async Task VectorSetRandomMembers() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector1 = new[] { 1.0f, 2.0f, 3.0f }; + var vector2 = new[] { 4.0f, 5.0f, 6.0f }; + var vector3 = new[] { 7.0f, 8.0f, 9.0f }; + + var request = VectorSetAddRequest.Member("element1", vector1.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element3", vector3.AsMemory()); + await db.VectorSetAddAsync(key, request); + + var randomMembers = await db.VectorSetRandomMembersAsync(key, 2); + + Assert.Equal(2, randomMembers.Length); + Assert.All(randomMembers, member => + Assert.True(member == "element1" || member == "element2" || member == "element3")); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task VectorSetSimilaritySearch_ByVector(bool withScores, bool withAttributes) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var disambiguator = (withScores ? 1 : 0) + (withAttributes ? 2 : 0); + var key = Me() + disambiguator; + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + // Add some test vectors + var vector1 = new[] { 1.0f, 0.0f, 0.0f }; + var vector2 = new[] { 0.0f, 1.0f, 0.0f }; + var vector3 = new[] { 0.9f, 0.1f, 0.0f }; // Similar to vector1 + + var request = + VectorSetAddRequest.Member("element1", vector1.AsMemory(), attributesJson: """{"category":"x"}"""); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory(), attributesJson: """{"category":"y"}"""); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element3", vector3.AsMemory(), attributesJson: """{"category":"z"}"""); + await db.VectorSetAddAsync(key, request); + + // Search for vectors similar to vector1 + var query = VectorSetSimilaritySearchRequest.ByVector(vector1.AsMemory()); + query.Count = 2; + query.WithScores = withScores; + query.WithAttributes = withAttributes; + using var results = await db.VectorSetSimilaritySearchAsync(key, query); + + Assert.NotNull(results); + foreach (var result in results.Span) + { + Log(result.ToString()); + } + + var resultsArray = results.Span.ToArray(); + + Assert.True(resultsArray.Length <= 2); + Assert.Contains(resultsArray, r => r.Member == "element1"); + var found = resultsArray.First(r => r.Member == "element1"); + + if (withAttributes) + { + Assert.Equal("""{"category":"x"}""", found.AttributesJson); + } + else + { + Assert.Null(found.AttributesJson); + } + + Assert.NotEqual(withScores, double.IsNaN(found.Score)); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task VectorSetSimilaritySearch_ByMember(bool withScores, bool withAttributes) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var disambiguator = (withScores ? 1 : 0) + (withAttributes ? 2 : 0); + var key = Me() + disambiguator; + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector1 = new[] { 1.0f, 0.0f, 0.0f }; + var vector2 = new[] { 0.0f, 1.0f, 0.0f }; + + var request = + VectorSetAddRequest.Member("element1", vector1.AsMemory(), attributesJson: """{"category":"x"}"""); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory(), attributesJson: """{"category":"y"}"""); + await db.VectorSetAddAsync(key, request); + + var query = VectorSetSimilaritySearchRequest.ByMember("element1"); + query.Count = 1; + query.WithScores = withScores; + query.WithAttributes = withAttributes; + using var results = await db.VectorSetSimilaritySearchAsync(key, query); + + Assert.NotNull(results); + foreach (var result in results.Span) + { + Log(result.ToString()); + } + + var resultsArray = results.Span.ToArray(); + + Assert.Single(resultsArray); + Assert.Equal("element1", resultsArray[0].Member); + if (withAttributes) + { + Assert.Equal("""{"category":"x"}""", resultsArray[0].AttributesJson); + } + else + { + Assert.Null(resultsArray[0].AttributesJson); + } + + Assert.NotEqual(withScores, double.IsNaN(resultsArray[0].Score)); + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task VectorSetSimilaritySearch_WithFilter(bool corruptPrefix, bool corruptSuffix) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + Random rand = new Random(); + + float[] vector = new float[50]; + + void ScrambleVector() + { + var arr = vector; + for (int i = 0; i < arr.Length; i++) + { + arr[i] = (float)rand.NextDouble(); + } + } + + string[] regions = new[] { "us-west", "us-east", "eu-west", "eu-east", "ap-south", "ap-north" }; + for (int i = 0; i < 100; i++) + { + var region = regions[rand.Next(regions.Length)]; + var json = (corruptPrefix ? "oops" : "") + + JsonConvert.SerializeObject(new { id = i, region }) + + (corruptSuffix ? "oops" : ""); + ScrambleVector(); + var request = VectorSetAddRequest.Member($"element{i}", vector.AsMemory(), json); + await db.VectorSetAddAsync(key, request); + } + + ScrambleVector(); + var query = VectorSetSimilaritySearchRequest.ByVector(vector); + query.Count = 100; + query.WithScores = true; + query.WithAttributes = true; + query.FilterExpression = ".id >= 30"; + using var results = await db.VectorSetSimilaritySearchAsync(key, query); + + Assert.NotNull(results); + foreach (var result in results.Span) + { + Log(result.ToString()); + } + + Log($"Total matches: {results.Span.Length}"); + + var resultsArray = results.Span.ToArray(); + if (corruptPrefix) + { + // server short-circuits failure to be no match; we just want to assert + // what the observed behavior *is* + Assert.Empty(resultsArray); + } + else + { + Assert.Equal(70, resultsArray.Length); + Assert.All(resultsArray, r => Assert.True( + r.Score is > 0.0 and < 1.0 && GetId(r.Member!) >= 30)); + } + + static int GetId(string member) + { + if (member.StartsWith("element")) + { + return int.Parse(member.Substring(7), NumberStyles.Integer, CultureInfo.InvariantCulture); + } + + return -1; + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData(".id >= 30")] + public async Task VectorSetSimilaritySearch_TestFilterValues(string? filterExpression) + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + Random rand = new Random(); + + float[] vector = new float[50]; + + void ScrambleVector() + { + var arr = vector; + for (int i = 0; i < arr.Length; i++) + { + arr[i] = (float)rand.NextDouble(); + } + } + + string[] regions = new[] { "us-west", "us-east", "eu-west", "eu-east", "ap-south", "ap-north" }; + for (int i = 0; i < 100; i++) + { + var region = regions[rand.Next(regions.Length)]; + var json = JsonConvert.SerializeObject(new { id = i, region }); + ScrambleVector(); + var request = VectorSetAddRequest.Member($"element{i}", vector.AsMemory(), json); + await db.VectorSetAddAsync(key, request); + } + + ScrambleVector(); + var query = VectorSetSimilaritySearchRequest.ByVector(vector); + query.Count = 100; + query.WithScores = true; + query.WithAttributes = true; + query.FilterExpression = filterExpression; + + using var results = await db.VectorSetSimilaritySearchAsync(key, query); + + Assert.NotNull(results); + foreach (var result in results.Span) + { + Log(result.ToString()); + } + + Log($"Total matches: {results.Span.Length}"); + // we're not interested in the specific results; we're just checking that the + // filter expression was added and parsed without exploding about arg mismatch + } + + [Fact] + public async Task VectorSetSetAttributesJson() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + var vector = new[] { 1.0f, 2.0f, 3.0f }; + var request = VectorSetAddRequest.Member("element1", vector.AsMemory()); + await db.VectorSetAddAsync(key, request); + + // Set attributes for existing element + var attributes = """{"category":"updated","priority":"high","timestamp":"2024-01-01"}"""; + var result = await db.VectorSetSetAttributesJsonAsync(key, "element1", attributes); + + Assert.True(result); + + // Verify attributes were set + var retrievedAttributes = await db.VectorSetGetAttributesJsonAsync(key, "element1"); + Assert.Equal(attributes, retrievedAttributes); + + // Try setting attributes for non-existent element + var failResult = await db.VectorSetSetAttributesJsonAsync(key, "nonexistent", attributes); + Assert.False(failResult); + } + + [Fact] + public async Task VectorSetGetLinks() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + // Add some vectors that should be linked + var vector1 = new[] { 1.0f, 0.0f, 0.0f }; + var vector2 = new[] { 0.9f, 0.1f, 0.0f }; // Similar to vector1 + var vector3 = new[] { 0.0f, 1.0f, 0.0f }; // Different from vector1 + + var request = VectorSetAddRequest.Member("element1", vector1.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element3", vector3.AsMemory()); + await db.VectorSetAddAsync(key, request); + + // Get links for element1 (should include similar vectors) + using var links = await db.VectorSetGetLinksAsync(key, "element1"); + + Assert.NotNull(links); + foreach (var link in links.Span) + { + Log(link.ToString()); + } + + var linksArray = links.Span.ToArray(); + + // Should contain the other elements (note there can be transient duplicates, so: contains, not exact) + Assert.Contains("element2", linksArray); + Assert.Contains("element3", linksArray); + } + + [Fact] + public async Task VectorSetGetLinksWithScores() + { + await using var conn = Create(require: RedisFeatures.v8_0_0_M04); + var db = conn.GetDatabase(); + var key = Me(); + + await db.KeyDeleteAsync(key, CommandFlags.FireAndForget); + + // Add some vectors with known relationships + var vector1 = new[] { 1.0f, 0.0f, 0.0f }; + var vector2 = new[] { 0.9f, 0.1f, 0.0f }; // Similar to vector1 + var vector3 = new[] { 0.0f, 1.0f, 0.0f }; // Different from vector1 + + var request = VectorSetAddRequest.Member("element1", vector1.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element2", vector2.AsMemory()); + await db.VectorSetAddAsync(key, request); + request = VectorSetAddRequest.Member("element3", vector3.AsMemory()); + await db.VectorSetAddAsync(key, request); + + // Get links with scores for element1 + using var linksWithScores = await db.VectorSetGetLinksWithScoresAsync(key, "element1"); + Assert.NotNull(linksWithScores); + foreach (var link in linksWithScores.Span) + { + Log(link.ToString()); + } + + var linksArray = linksWithScores.Span.ToArray(); + Assert.NotEmpty(linksArray); + + // Verify each link has a valid score + // ReSharper disable once ParameterOnlyUsedForPreconditionCheck.Local + Assert.All(linksArray, static link => + { + Assert.False(link.Member.IsNull); + Assert.False(double.IsNaN(link.Score)); + Assert.True(link.Score >= 0.0); // Similarity scores should be non-negative + }); + + // Should contain the other elements (note there can be transient duplicates, so: contains, not exact) + Assert.Contains(linksArray, l => l.Member == "element2"); + Assert.Contains(linksArray, l => l.Member == "element3"); + + Assert.True(linksArray.First(l => l.Member == "element2").Score > 0.9); // similar + Assert.True(linksArray.First(l => l.Member == "element3").Score < 0.8); // less-so + } +}