Adding basic version of DUMP and RESTORE commands

libs/common/Crc64.cs

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Port of redis crc64 from https://github.com/redis/redis/blob/7.2/src/crc64.c
+/// </summary>
+public static class Crc64
+{
+    /// <summary>
+    /// Polynomial (same as redis)
+    /// </summary>
+    private const ulong POLY = 0xad93d23594c935a9UL;
+
+    /// <summary>
+    /// Reverse all bits in a 64-bit value (bit reflection).
+    /// Only used for data_len == 64 in this code.
+    /// </summary>
+    private static ulong Reflect64(ulong data)
+    {
+        // swap odd/even bits
+        data = ((data >> 1) & 0x5555555555555555UL) | ((data & 0x5555555555555555UL) << 1);
+        // swap consecutive pairs
+        data = ((data >> 2) & 0x3333333333333333UL) | ((data & 0x3333333333333333UL) << 2);
+        // swap nibbles
+        data = ((data >> 4) & 0x0F0F0F0F0F0F0F0FUL) | ((data & 0x0F0F0F0F0F0F0F0FUL) << 4);
+        // swap bytes, then 2-byte pairs, then 4-byte pairs
+        data = System.Buffers.Binary.BinaryPrimitives.ReverseEndianness(data);
+        return data;
+    }
+
+    /// <summary>
+    /// A direct bit-by-bit CRC64 calculation (like _crc64 in C).
+    /// </summary>
+    private static ulong Crc64Bitwise(ReadOnlySpan<byte> data)
+    {
+        ulong crc = 0;
+
+        foreach (var c in data)
+        {
+            for (byte i = 1; i != 0; i <<= 1)
+            {
+                // interpret the top bit of 'crc' and current bit of 'c'
+                var bitSet = (crc & 0x8000000000000000UL) != 0;
+                var cbit = (c & i) != 0;
+
+                // if cbit flips the sense, invert bitSet
+                if (cbit)
+                    bitSet = !bitSet;
+
+                // shift
+                crc <<= 1;
+
+                // apply polynomial if needed
+                if (bitSet)
+                    crc ^= POLY;
+            }
+
+            // ensure it stays in 64 bits
+            crc &= 0xffffffffffffffffUL;
+        }
+
+        // reflect and XOR, per standard
+        crc &= 0xffffffffffffffffUL;
+        crc = Reflect64(crc) ^ 0x0000000000000000UL;
+        return crc;
+    }
+
+    /// <summary>
+    /// Computes crc64
+    /// </summary>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    public static byte[] Hash(ReadOnlySpan<byte> data)
+    {
+        var bitwiseCrc = Crc64Bitwise(data);
+        return BitConverter.GetBytes(bitwiseCrc);
+    }
+}

libs/common/RespLengthEncodingUtils.cs

@@ -0,0 +1,111 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Buffers.Binary;
+
+namespace Garnet.common;
+
+/// <summary>
+/// Utils for working with RESP length encoding
+/// </summary>
+public static class RespLengthEncodingUtils
+{
+    /// <summary>
+    /// Maximum length that can be encoded
+    /// </summary>
+    private const int MaxLength = 0xFFFFFF;
+
+    /// <summary>
+    /// Try read RESP-encoded length
+    /// </summary>
+    /// <param name="input"></param>
+    /// <param name="length"></param>
+    /// <param name="bytesRead"></param>
+    /// <returns></returns>
+    public static bool TryReadLength(ReadOnlySpan<byte> input, out int length, out int bytesRead)
+    {
+        length = 0;
+        bytesRead = 0;
+        if (input.Length < 1)
+        {
+            return false;
+        }
+
+        var firstByte = input[0];
+        switch (firstByte >> 6)
+        {
+            case 0:
+                bytesRead = 1;
+                length = firstByte & 0x3F;
+                return true;
+            case 1 when input.Length > 1:
+                bytesRead = 2;
+                length = ((firstByte & 0x3F) << 8) | input[1];
+                return true;
+            case 2:
+                bytesRead = 5;
+                return BinaryPrimitives.TryReadInt32BigEndian(input, out length);
+            default:
+                return false;
+        }
+    }
+
+    /// <summary>
+    /// Try to write RESP-encoded length
+    /// </summary>
+    /// <param name="length"></param>
+    /// <param name="output"></param>
+    /// <param name="bytesWritten"></param>
+    /// <returns></returns>
+    public static bool TryWriteLength(int length, Span<byte> output, out int bytesWritten)
+    {
+        bytesWritten = 0;
+
+        if (length > MaxLength)
+        {
+            return false;
+        }
+
+        // 6-bit encoding (length ≤ 63)
+        if (length < 1 << 6)
+        {
+            if (output.Length < 1)
+            {
+                return false;
+            }
+
+            output[0] = (byte)(length & 0x3F);
+
+            bytesWritten = 1;
+            return true;
+        }
+
+        // 14-bit encoding (64 ≤ length ≤ 16,383)
+        if (length < 1 << 14)
+        {
+            if (output.Length < 2)
+            {
+                return false;
+            }
+
+            output[0] = (byte)(((length >> 8) & 0x3F) | (1 << 6));
+            output[1] = (byte)(length & 0xFF);
+
+            bytesWritten = 2;
+            return true;
+        }
+
+        // 32-bit encoding (length ≤ 4,294,967,295)
+        if (output.Length < 5)
+        {
+            return false;
+        }
+
+        output[0] = 2 << 6;
+        BinaryPrimitives.WriteUInt32BigEndian(output.Slice(1), (uint)length);
+
+        bytesWritten = 5;
+        return true;
+    }
+}

libs/host/Configuration/Options.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation.
+// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
 using System;
@@ -509,6 +509,10 @@ internal sealed class Options
         [Option("fail-on-recovery-error", Default = false, Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")]
         public bool? FailOnRecoveryError { get; set; }
 
+        [OptionValidation]
+        [Option("skip-rdb-restore-checksum-validation", Default = false, Required = false, HelpText = "Skip RDB restore checksum validation")]
+        public bool? SkipRDBRestoreChecksumValidation { get; set; }
+
         [Option("lua-memory-management-mode", Default = LuaMemoryManagementMode.Native, Required = false, HelpText = "Memory management mode for Lua scripts, must be set to LimittedNative or Managed to impose script limits")]
         public LuaMemoryManagementMode LuaMemoryManagementMode { get; set; }
 
@@ -728,6 +732,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 IndexResizeThreshold = IndexResizeThreshold,
                 LoadModuleCS = LoadModuleCS,
                 FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(),
+                SkipRDBRestoreChecksumValidation = SkipRDBRestoreChecksumValidation.GetValueOrDefault(),
                 LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, logger) : null,
             };
         }

libs/host/defaults.conf

@@ -344,6 +344,9 @@
 	/* Fails if encounters error during AOF replay or checkpointing */
 	"FailOnRecoveryError": false,
 
+	/* Skips crc64 validation in restore command */
+	"SkipRDBRestoreChecksumValidation": false,
+
 	/* Lua uses the default, unmanaged and untracked, allocator */
 	"LuaMemoryManagementMode": "Native",