Normalize text new line to Unix-style (#998)

Make text more consistent across Unix and Windows environments, avoiding the `\r` character in favor of the Unix new line ending. See #975
microsoft · Feb 7, 2025 · e49783a · e49783a
1 parent a490102
commit e49783a
Show file tree

Hide file tree

Showing 19 changed files with 244 additions and 41 deletions.
diff --git a/applications/evaluation/Evaluators/Faithfulness/FaithfulnessEvaluator.cs b/applications/evaluation/Evaluators/Faithfulness/FaithfulnessEvaluator.cs
@@ -59,7 +59,7 @@ internal async Task<float> Evaluate(MemoryAnswer answer, Dictionary<string, obje
         {
             var evaluation = await this.FaithfulnessEvaluation.InvokeAsync(this._kernel, new KernelArguments
             {
-                { "context", string.Join(Environment.NewLine, answer.RelevantSources.SelectMany(c => c.Partitions.Select(p => p.Text))) },
+                { "context", string.Join('\n', answer.RelevantSources.SelectMany(c => c.Partitions.Select(p => p.Text))) },
                 { "answer", answer.Result },
                 { "statements", JsonSerializer.Serialize(extraction) }
             }).ConfigureAwait(false);

diff --git a/applications/evaluation/Evaluators/Relevance/RelevanceEvaluator.cs b/applications/evaluation/Evaluators/Relevance/RelevanceEvaluator.cs
@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Numerics.Tensors;
@@ -68,7 +67,7 @@ private async IAsyncEnumerable<RelevanceEvaluation> GetEvaluations(MemoryAnswer
             {
                 var extraction = await this.ExtractQuestion.InvokeAsync(this._kernel, new KernelArguments
                 {
-                    { "context", string.Join(Environment.NewLine, answer.RelevantSources.SelectMany(c => c.Partitions.Select(p => p.Text))) },
+                    { "context", string.Join('\n', answer.RelevantSources.SelectMany(c => c.Partitions.Select(p => p.Text))) },
                     { "answer", answer.Result }
                 }).ConfigureAwait(false);
 

diff --git a/extensions/Chunkers/Chunkers/MarkDownChunker.cs b/extensions/Chunkers/Chunkers/MarkDownChunker.cs
@@ -12,6 +12,7 @@
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Chunkers.internals;
 using Microsoft.KernelMemory.DataFormats;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.Chunkers;
 
@@ -152,10 +153,7 @@ public List<string> Split(string text, MarkDownChunkerOptions options)
         ArgumentNullException.ThrowIfNull(options);
 
         // Clean up text. Note: LLMs don't use \r char
-        text = text
-            .Replace("\r\n", "\n", StringComparison.OrdinalIgnoreCase)
-            .Replace("\r", "\n", StringComparison.OrdinalIgnoreCase)
-            .Trim();
+        text = text.NormalizeNewlines(true);
 
         // Calculate chunk size leaving room for the optional chunk header
         int maxChunk1Size = options.MaxTokensPerChunk - this.TokenCount(options.ChunkHeader);

diff --git a/extensions/Chunkers/Chunkers/PlainTextChunker.cs b/extensions/Chunkers/Chunkers/PlainTextChunker.cs
@@ -12,6 +12,7 @@
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Chunkers.internals;
 using Microsoft.KernelMemory.DataFormats;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.Chunkers;
 
@@ -127,10 +128,7 @@ public List<string> Split(string text, PlainTextChunkerOptions options)
         ArgumentNullException.ThrowIfNull(options);
 
         // Clean up text. Note: LLMs don't use \r char
-        text = text
-            .Replace("\r\n", "\n", StringComparison.OrdinalIgnoreCase)
-            .Replace("\r", "\n", StringComparison.OrdinalIgnoreCase)
-            .Trim();
+        text = text.NormalizeNewlines(true);
 
         // Calculate chunk size leaving room for the optional chunk header
         int maxChunk1Size = options.MaxTokensPerChunk - this.TokenCount(options.ChunkHeader);

diff --git a/extensions/Postgres/Postgres/PostgresMemory.cs b/extensions/Postgres/Postgres/PostgresMemory.cs
@@ -13,6 +13,7 @@
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.MemoryStorage;
+using Microsoft.KernelMemory.Text;
 using Pgvector;
 
 namespace Microsoft.KernelMemory.Postgres;
@@ -272,7 +273,7 @@ private static string NormalizeTableNamePrefix(string? name)
         foreach (MemoryFilter filter in filters.Where(f => !f.IsEmpty()))
         {
             var andSql = new StringBuilder();
-            andSql.AppendLine("(");
+            andSql.AppendLineNix("(");
 
             if (filter is PostgresMemoryFilter)
             {
@@ -298,10 +299,10 @@ private static string NormalizeTableNamePrefix(string? name)
                 //  $"{PostgresSchema.PlaceholdersTags} @> " + safeSqlPlaceholder
                 //  $"{PostgresSchema.PlaceholdersTags} @> " + safeSqlPlaceholder + "::text[]"
                 //  $"{PostgresSchema.PlaceholdersTags} @> ARRAY[" + safeSqlPlaceholder + "]::text[]"
-                andSql.AppendLine($"{PostgresSchema.PlaceholdersTags} @> " + safeSqlPlaceholder);
+                andSql.AppendLineNix($"{PostgresSchema.PlaceholdersTags} @> " + safeSqlPlaceholder);
             }
 
-            andSql.AppendLine(")");
+            andSql.AppendLineNix(")");
             orConditions.Add(andSql.ToString());
         }
 

diff --git a/service/Abstractions/HTTP/SSE.cs b/service/Abstractions/HTTP/SSE.cs
@@ -8,6 +8,7 @@
 using System.Text;
 using System.Text.Json;
 using System.Threading;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.HTTP;
 
@@ -39,7 +40,7 @@ public async static IAsyncEnumerable<T> ParseStreamAsync<T>(
             }
             else
             {
-                buffer.AppendLine(line);
+                buffer.AppendLineNix(line);
             }
         }
 

diff --git a/service/Abstractions/Models/MemoryAnswer.cs b/service/Abstractions/Models/MemoryAnswer.cs
@@ -6,6 +6,7 @@
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory;
 
@@ -92,7 +93,7 @@ public string ToJson(bool optimizeForStream)
     public override string ToString()
     {
         var result = new StringBuilder();
-        result.AppendLine(this.Result);
+        result.AppendLineNix(this.Result);
 
         if (!this.NoResult && this.RelevantSources is { Count: > 0 })
         {
@@ -103,8 +104,8 @@ public override string ToString()
                 sources[x.Index + x.Link] = $"  - {x.SourceName} [{date}]";
             }
 
-            result.AppendLine("- Sources:");
-            result.AppendLine(string.Join("\n", sources.Values));
+            result.AppendLineNix("- Sources:");
+            result.AppendLineNix(string.Join("\n", sources.Values));
         }
 
         return result.ToString();

diff --git a/service/Abstractions/Text/StringBuilderExtensions.cs b/service/Abstractions/Text/StringBuilderExtensions.cs
@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text;
+
+namespace Microsoft.KernelMemory.Text;
+
+public static class StringBuilderExtensions
+{
+    /// <summary>
+    /// Append line using Unix line ending "\n"
+    /// </summary>
+    public static void AppendLineNix(this StringBuilder sb)
+    {
+        sb.Append('\n');
+    }
+
+    /// <summary>
+    /// Append line using Unix line ending "\n"
+    /// </summary>
+    public static void AppendLineNix(this StringBuilder sb, string value)
+    {
+        sb.Append(value);
+        sb.Append('\n');
+    }
+
+    /// <summary>
+    /// Append line using Unix line ending "\n"
+    /// </summary>
+    public static void AppendLineNix(this StringBuilder sb, char value)
+    {
+        sb.Append(value);
+        sb.Append('\n');
+    }
+
+    /// <summary>
+    /// Append line using Unix line ending "\n"
+    /// </summary>
+    public static void AppendLineNix(this StringBuilder sb, StringBuilder value)
+    {
+        sb.Append(value);
+        sb.Append('\n');
+    }
+}
diff --git a/service/Abstractions/Text/StringExtensions.cs b/service/Abstractions/Text/StringExtensions.cs
@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.KernelMemory.Text;
+
+public static class StringExtensions
+{
+    public static string NormalizeNewlines(this string text, bool trim = false)
+    {
+        if (string.IsNullOrEmpty(text))
+        {
+            return text;
+        }
+
+        // We won't need more than the original length
+        char[] buffer = new char[text.Length];
+        int bufferPos = 0;
+
+        // Skip leading whitespace if trimming
+        int i = 0;
+        if (trim)
+        {
+            while (i < text.Length && char.IsWhiteSpace(text[i])) { i++; }
+        }
+
+        // Tracks the last non-whitespace position written into buffer
+        int lastNonWhitespacePos = -1;
+
+        // Single pass: replace \r\n or \r with \n, record last non-whitespace
+        for (; i < text.Length; i++)
+        {
+            char c = text[i];
+
+            if (c == '\r')
+            {
+                // If \r\n then skip the \n
+                if (i + 1 < text.Length && text[i + 1] == '\n') { i++; }
+
+                // Write a single \n
+                buffer[bufferPos] = '\n';
+            }
+            else
+            {
+                buffer[bufferPos] = c;
+            }
+
+            // If trimming, update lastNonWhitespacePos only when char isn't whitespace
+            // If not trimming, always update because we keep everything
+            if (!trim || !char.IsWhiteSpace(buffer[bufferPos]))
+            {
+                lastNonWhitespacePos = bufferPos;
+            }
+
+            bufferPos++;
+        }
+
+        // Cut off trailing whitespace if trimming
+        // If every char was whitespace, lastNonWhitespacePos stays -1 and the result is an empty string
+        int finalLength = (trim && lastNonWhitespacePos >= 0)
+            ? lastNonWhitespacePos + 1
+            : bufferPos;
+
+        // Safety check if everything was trimmed away
+        if (finalLength < 0) { finalLength = 0; }
+
+        return new string(buffer, 0, finalLength);
+    }
+}
diff --git a/service/Core/DataFormats/Image/ImageDecoder.cs b/service/Core/DataFormats/Image/ImageDecoder.cs
@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Pipeline;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.DataFormats.Image;
 
@@ -64,7 +65,7 @@ public async Task<FileContent> DecodeAsync(Stream data, CancellationToken cancel
 
         var result = new FileContent(MimeTypes.PlainText);
         var content = await this.ImageToTextAsync(data, cancellationToken).ConfigureAwait(false);
-        result.Sections.Add(new(content.Trim(), 1, Chunk.Meta(sentencesAreComplete: true)));
+        result.Sections.Add(new(content, 1, Chunk.Meta(sentencesAreComplete: true)));
 
         return result;
     }
@@ -87,10 +88,14 @@ private async Task<string> ImageToTextAsync(BinaryData data, CancellationToken c
         }
     }
 
-    private Task<string> ImageToTextAsync(Stream data, CancellationToken cancellationToken = default)
+    private async Task<string> ImageToTextAsync(Stream data, CancellationToken cancellationToken = default)
     {
-        return this._ocrEngine is null
-            ? throw new NotSupportedException($"Image extraction not configured")
-            : this._ocrEngine.ExtractTextFromImageAsync(data, cancellationToken);
+        if (this._ocrEngine is null)
+        {
+            throw new NotSupportedException($"Image extraction not configured");
+        }
+
+        string text = await this._ocrEngine.ExtractTextFromImageAsync(data, cancellationToken).ConfigureAwait(false);
+        return text.NormalizeNewlines(true);
     }
 }
diff --git a/service/Core/DataFormats/Office/MsExcelDecoder.cs b/service/Core/DataFormats/Office/MsExcelDecoder.cs
@@ -11,6 +11,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Pipeline;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.DataFormats.Office;
 
@@ -61,7 +62,7 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
             worksheetNumber++;
             if (this._config.WithWorksheetNumber)
             {
-                sb.AppendLine(this._config.WorksheetNumberTemplate.Replace("{number}", $"{worksheetNumber}", StringComparison.OrdinalIgnoreCase));
+                sb.AppendLineNix(this._config.WorksheetNumberTemplate.Replace("{number}", $"{worksheetNumber}", StringComparison.OrdinalIgnoreCase));
             }
 
             var rowsUsed = worksheet.RangeUsed()?.RowsUsed();
@@ -142,15 +143,15 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
                     }
                 }
 
-                sb.AppendLine(this._config.RowSuffix);
+                sb.AppendLineNix(this._config.RowSuffix);
             }
 
             if (this._config.WithEndOfWorksheetMarker)
             {
-                sb.AppendLine(this._config.EndOfWorksheetMarkerTemplate.Replace("{number}", $"{worksheetNumber}", StringComparison.OrdinalIgnoreCase));
+                sb.AppendLineNix(this._config.EndOfWorksheetMarkerTemplate.Replace("{number}", $"{worksheetNumber}", StringComparison.OrdinalIgnoreCase));
             }
 
-            string worksheetContent = sb.ToString().Trim();
+            string worksheetContent = sb.ToString().NormalizeNewlines(true);
             sb.Clear();
             result.Sections.Add(new Chunk(worksheetContent, worksheetNumber, Chunk.Meta(sentencesAreComplete: true)));
         }

diff --git a/service/Core/DataFormats/Office/MsPowerPointDecoder.cs b/service/Core/DataFormats/Office/MsPowerPointDecoder.cs
@@ -13,6 +13,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Pipeline;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.DataFormats.Office;
 
@@ -99,20 +100,20 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
                     // Prepend slide number before the slide text
                     if (this._config.WithSlideNumber)
                     {
-                        sb.AppendLine(this._config.SlideNumberTemplate.Replace("{number}", $"{slideNumber}", StringComparison.OrdinalIgnoreCase));
+                        sb.AppendLineNix(this._config.SlideNumberTemplate.Replace("{number}", $"{slideNumber}", StringComparison.OrdinalIgnoreCase));
                     }
 
                     sb.Append(currentSlideContent);
-                    sb.AppendLine();
+                    sb.AppendLineNix();
 
                     // Append the end of slide marker
                     if (this._config.WithEndOfSlideMarker)
                     {
-                        sb.AppendLine(this._config.EndOfSlideMarkerTemplate.Replace("{number}", $"{slideNumber}", StringComparison.OrdinalIgnoreCase));
+                        sb.AppendLineNix(this._config.EndOfSlideMarkerTemplate.Replace("{number}", $"{slideNumber}", StringComparison.OrdinalIgnoreCase));
                     }
                 }
 
-                string slideContent = sb.ToString().Trim();
+                string slideContent = sb.ToString().NormalizeNewlines(true);
                 sb.Clear();
                 result.Sections.Add(new Chunk(slideContent, slideNumber, Chunk.Meta(sentencesAreComplete: true)));
             }

diff --git a/service/Core/DataFormats/Office/MsWordDecoder.cs b/service/Core/DataFormats/Office/MsWordDecoder.cs
@@ -12,6 +12,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.Diagnostics;
 using Microsoft.KernelMemory.Pipeline;
+using Microsoft.KernelMemory.Text;
 
 namespace Microsoft.KernelMemory.DataFormats.Office;
 
@@ -79,17 +80,19 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
                     var lastRenderedPageBreak = p.GetFirstChild<Run>()?.GetFirstChild<LastRenderedPageBreak>();
                     if (lastRenderedPageBreak != null)
                     {
-                        string pageContent = sb.ToString().Trim();
+                        // Note: no trimming, use original spacing when working with pages
+                        string pageContent = sb.ToString().NormalizeNewlines(false);
                         sb.Clear();
                         result.Sections.Add(new Chunk(pageContent, pageNumber, Chunk.Meta(sentencesAreComplete: true)));
                         pageNumber++;
                     }
 
-                    sb.AppendLine(p.InnerText);
+                    sb.AppendLineNix(p.InnerText);
                 }
             }
 
-            var lastPageContent = sb.ToString().Trim();
+            // Note: no trimming, use original spacing when working with pages
+            string lastPageContent = sb.ToString().NormalizeNewlines(false);
             result.Sections.Add(new Chunk(lastPageContent, pageNumber, Chunk.Meta(sentencesAreComplete: true)));
 
             return Task.FromResult(result);