Adding basic version of DUMP and RESTORE commands

libs/common/Crc64.cs

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Port of redis crc64 from https://github.com/redis/redis/blob/7.2/src/crc64.c
+/// </summary>
+public static class Crc64
+{
+    /// <summary>
+    /// Polynomial (same as redis)
+    /// </summary>
+    private const ulong POLY = 0xad93d23594c935a9UL;
+
+    /// <summary>
+    /// Reverse all bits in a 64-bit value (bit reflection).
+    /// Only used for data_len == 64 in this code.
+    /// </summary>
+    private static ulong Reflect64(ulong data)
+    {
+        // swap odd/even bits
+        data = ((data >> 1) & 0x5555555555555555UL) | ((data & 0x5555555555555555UL) << 1);
+        // swap consecutive pairs
+        data = ((data >> 2) & 0x3333333333333333UL) | ((data & 0x3333333333333333UL) << 2);
+        // swap nibbles
+        data = ((data >> 4) & 0x0F0F0F0F0F0F0F0FUL) | ((data & 0x0F0F0F0F0F0F0F0FUL) << 4);
+        // swap bytes, then 2-byte pairs, then 4-byte pairs
+        data = System.Buffers.Binary.BinaryPrimitives.ReverseEndianness(data);
+        return data;
+    }
+
+    /// <summary>
+    /// A direct bit-by-bit CRC64 calculation (like _crc64 in C).
+    /// </summary>
+    private static ulong Crc64Bitwise(ReadOnlySpan<byte> data)
+    {
+        ulong crc = 0;
+
+        foreach (var c in data)
+        {
+            for (byte i = 1; i != 0; i <<= 1)
+            {
+                // interpret the top bit of 'crc' and current bit of 'c'
+                var bitSet = (crc & 0x8000000000000000UL) != 0;
+                var cbit = (c & i) != 0;
+
+                // if cbit flips the sense, invert bitSet
+                if (cbit)
+                    bitSet = !bitSet;
+
+                // shift
+                crc <<= 1;
+
+                // apply polynomial if needed
+                if (bitSet)
+                    crc ^= POLY;
+            }
+
+            // ensure it stays in 64 bits
+            crc &= 0xffffffffffffffffUL;
+        }
+
+        // reflect and XOR, per standard
+        crc &= 0xffffffffffffffffUL;
+        crc = Reflect64(crc) ^ 0x0000000000000000UL;
+        return crc;
+    }
+
+    /// <summary>
+    /// Computes crc64
+    /// </summary>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    public static byte[] Hash(ReadOnlySpan<byte> data)
+    {
+        var bitwiseCrc = Crc64Bitwise(data);
+        return BitConverter.GetBytes(bitwiseCrc);
+    }
+}

libs/common/RespLengthEncodingUtils.cs

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Linq;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Utils for working with RESP length encoding
+/// </summary>
+public static class RespLengthEncodingUtils
+{
+    /// <summary>
+    /// Decodes the RESP-encoded length and returns payload start
+    /// </summary>
+    /// <param name="buff"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentException"></exception>
+    public static (long length, byte payloadStart) DecodeLength(ref ReadOnlySpan<byte> buff)
+    {
+        // remove the value type byte
+        var encoded = buff.Slice(1);
+
+        if (encoded.Length == 0)
+        {
+            throw new ArgumentException("Encoded length cannot be empty.", nameof(encoded));
+        }
+
+        var firstByte = encoded[0];
+        return (firstByte >> 6) switch
+        {
+            // 6-bit encoding
+            0 => (firstByte & 0x3F, 1),
+            // 14-bit encoding
+            1 when encoded.Length < 2 => throw new ArgumentException("Not enough bytes for 14-bit encoding."),
+            1 => (((firstByte & 0x3F) << 8) | encoded[1], 2),
+            // 32-bit encoding
+            2 when encoded.Length < 5 => throw new ArgumentException("Not enough bytes for 32-bit encoding."),
+            2 => ((long)((encoded[1] << 24) | (encoded[2] << 16) | (encoded[3] << 8) | encoded[4]), 5),
+            _ => throw new ArgumentException("Invalid encoding type.", nameof(encoded))
+        };
+    }
+
+    /// <summary>
+    /// Encoded payload length to RESP-encoded payload length
+    /// </summary>
+    /// <param name="length"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentOutOfRangeException"></exception>
+    public static byte[] EncodeLength(long length)
+    {
+        switch (length)
+        {
+            // 6-bit encoding (length ≤ 63)
+            case < 1 << 6:
+                return [(byte)(length & 0x3F)]; // 00xxxxxx
+            // 14-bit encoding (64 ≤ length ≤ 16,383)
+            case < 1 << 14:
+                {
+                    var firstByte = (byte)(((length >> 8) & 0x3F) | (1 << 6)); // 01xxxxxx
+                    var secondByte = (byte)(length & 0xFF);
+                    return [firstByte, secondByte];
+                }
+            // 32-bit encoding (length ≤ 4,294,967,295)
+            case <= 0xFFFFFFFF:
+                {
+                    var firstByte = (byte)(2 << 6); // 10xxxxxx
+                    var lengthBytes = BitConverter.GetBytes((uint)length); // Ensure unsigned
+                    if (BitConverter.IsLittleEndian)
+                    {
+                        Array.Reverse(lengthBytes); // Convert to big-endian
+                    }
+
+                    return [firstByte, .. lengthBytes];
+                }
+            default:
+                throw new ArgumentOutOfRangeException(
+                    nameof(length), "Length exceeds maximum allowed for Redis encoding (4,294,967,295).");
+        }
+    }
+}

libs/host/Configuration/Options.cs

@@ -509,6 +509,10 @@ internal sealed class Options
         [Option("fail-on-recovery-error", Default = false, Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")]
         public bool? FailOnRecoveryError { get; set; }
 
+        [OptionValidation]
+        [Option("skip-checksum-validation", Default = false, Required = false, HelpText = "Skip checksum validation")]
+        public bool? SkipChecksumValidation { get; set; }
+
         [Option("lua-memory-management-mode", Default = LuaMemoryManagementMode.Native, Required = false, HelpText = "Memory management mode for Lua scripts, must be set to LimittedNative or Managed to impose script limits")]
         public LuaMemoryManagementMode LuaMemoryManagementMode { get; set; }
 
@@ -728,6 +732,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 IndexResizeThreshold = IndexResizeThreshold,
                 LoadModuleCS = LoadModuleCS,
                 FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(),
+                SkipChecksumValidation = SkipChecksumValidation.GetValueOrDefault(),
                 LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, logger) : null,
             };
         }

libs/host/defaults.conf

@@ -344,6 +344,9 @@
 	/* Fails if encounters error during AOF replay or checkpointing */
 	"FailOnRecoveryError": false,
 
+	/* Skips crc64 validation in restore command */
+	"SkipChecksumValidation": false,
+
 	/* Lua uses the default, unmanaged and untracked, allocator */
 	"LuaMemoryManagementMode": "Native",
 

libs/resources/RespCommandsDocs.json

@@ -748,7 +748,6 @@
     "Summary": "Pops an element from a list, pushes it to another list and returns it. Block until an element is available otherwise. Deletes the list if the last element was popped.",
     "Group": "List",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060BLMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments",
     "Arguments": [
       {
@@ -1427,7 +1426,6 @@
         "Summary": "Returns the mapping of cluster slots to nodes.",
         "Group": "Cluster",
         "Complexity": "O(N) where N is the total number of Cluster nodes",
-        "DocFlags": "Deprecated",
         "ReplacedBy": "\u0060CLUSTER SHARDS\u0060"
       }
     ]
@@ -1654,6 +1652,22 @@
     "Group": "Transactions",
     "Complexity": "O(N), when N is the number of queued commands"
   },
+  {
+    "Command": "DUMP",
+    "Name": "DUMP",
+    "Summary": "Returns a serialized representation of the value stored at a key.",
+    "Group": "Generic",
+    "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1).",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      }
+    ]
+  },
   {
     "Command": "ECHO",
     "Name": "ECHO",
@@ -2821,7 +2835,6 @@
     "Summary": "Returns the previous string value of a key after setting it to a new value.",
     "Group": "String",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060SET\u0060 with the \u0060!GET\u0060 argument",
     "Arguments": [
       {
@@ -3107,7 +3120,6 @@
     "Summary": "Sets the values of multiple fields.",
     "Group": "Hash",
     "Complexity": "O(N) where N is the number of fields being set.",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060HSET\u0060 with multiple field-value pairs",
     "Arguments": [
       {
@@ -4452,7 +4464,6 @@
     "Summary": "Sets both string value and expiration time in milliseconds of a key. The key is created if it doesn\u0027t exist.",
     "Group": "String",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060SET\u0060 with the \u0060PX\u0060 argument",
     "Arguments": [
       {
@@ -4635,7 +4646,6 @@
     "Summary": "Closes the connection.",
     "Group": "Connection",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "just closing the connection"
   },
   {
@@ -4878,6 +4888,34 @@
       }
     ]
   },
+  {
+    "Command": "RESTORE",
+    "Name": "RESTORE",
+    "Summary": "Creates a key from the serialized representation of a value.",
+    "Group": "Generic",
+    "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).",
+    "Arguments": [
+      {
+        "TypeDiscriminator": "RespCommandKeyArgument",
+        "Name": "KEY",
+        "DisplayText": "key",
+        "Type": "Key",
+        "KeySpecIndex": 0
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "TTL",
+        "DisplayText": "ttl",
+        "Type": "Integer"
+      },
+      {
+        "TypeDiscriminator": "RespCommandBasicArgument",
+        "Name": "SERIALIZEDVALUE",
+        "DisplayText": "serialized-value",
+        "Type": "String"
+      }
+    ]
+  },
   {
     "Command": "RPOP",
     "Name": "RPOP",
@@ -4907,7 +4945,6 @@
     "Summary": "Returns the last element of a list after removing and pushing it to another list. Deletes the list if the last element was popped.",
     "Group": "List",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060LMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments",
     "Arguments": [
       {
@@ -5400,7 +5437,6 @@
     "Summary": "Sets the string value and expiration time of a key. Creates the key if it doesn\u0027t exist.",
     "Group": "String",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060SET\u0060 with the \u0060EX\u0060 argument",
     "Arguments": [
       {
@@ -5480,7 +5516,6 @@
     "Summary": "Set the string value of a key only when the key doesn\u0027t exist.",
     "Group": "String",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060SET\u0060 with the \u0060NX\u0060 argument",
     "Arguments": [
       {
@@ -5626,7 +5661,6 @@
     "Summary": "Sets a Redis server as a replica of another, or promotes it to being a master.",
     "Group": "Server",
     "Complexity": "O(1)",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060REPLICAOF\u0060",
     "Arguments": [
       {
@@ -5891,7 +5925,6 @@
     "Summary": "Returns a substring from a string value.",
     "Group": "String",
     "Complexity": "O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060GETRANGE\u0060",
     "Arguments": [
       {
@@ -6763,7 +6796,6 @@
     "Summary": "Returns members in a sorted set within a range of scores.",
     "Group": "SortedSet",
     "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060BYSCORE\u0060 argument",
     "Arguments": [
       {
@@ -7045,7 +7077,6 @@
     "Summary": "Returns members in a sorted set within a range of indexes in reverse order.",
     "Group": "SortedSet",
     "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements returned.",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 argument",
     "Arguments": [
       {
@@ -7083,7 +7114,6 @@
     "Summary": "Returns members in a sorted set within a lexicographical range in reverse order.",
     "Group": "SortedSet",
     "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYLEX\u0060 arguments",
     "Arguments": [
       {
@@ -7134,7 +7164,6 @@
     "Summary": "Returns members in a sorted set within a range of scores in reverse order.",
     "Group": "SortedSet",
     "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).",
-    "DocFlags": "Deprecated",
     "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYSCORE\u0060 arguments",
     "Arguments": [
       {