Improve performance, avoid redundant seeks by tracking unmatched records during the search process.

src/ZoneTree.FullTextSearch.Playground/SearchEngineApp.cs

@@ -20,6 +20,8 @@ public sealed class SearchEngineApp : IDisposable
 
     readonly bool UseDiacriticNormalizer = false;
 
+    readonly bool IndexInBackground = false;
+
     readonly HashedSearchEngine<long> SearchEngine;
 
     readonly RecordTable<long, string> RecordTable;
@@ -66,7 +68,10 @@ void MainMenu()
                 {
                     case "1":
                         var o = ConfigureIndex();
-                        CreateIndex(o.indexPath, o.pattern, true);
+                        if (IndexInBackground)
+                            Task.Run(() => CreateIndex(o.indexPath, o.pattern, false));
+                        else
+                            CreateIndex(o.indexPath, o.pattern, true);
                         break;
                     case "2":
                         Search();

src/ZoneTree.FullTextSearch/Directory.Build.props

@@ -5,8 +5,8 @@
     <Authors>Ahmed Yasin Koculu</Authors>
     <PackageId>ZoneTree.FullTextSearch</PackageId>
     <Title>ZoneTree.FullTextSearch</Title>
-    <ProductVersion>1.0.3.0</ProductVersion>
-    <Version>1.0.3.0</Version>
+    <ProductVersion>1.0.4.0</ProductVersion>
+    <Version>1.0.4.0</Version>
     <Authors>Ahmed Yasin Koculu</Authors>
     <AssemblyTitle>ZoneTree.FullTextSearch</AssemblyTitle>
     <Description>ZoneTree.FullTextSearch is an open-source library that extends ZoneTree to provide efficient full-text search capabilities. It offers a fast, embedded search engine suitable for applications that require high performance and do not rely on external databases.</Description>

src/ZoneTree.FullTextSearch/Index/IndexOfTokenRecordPreviousToken.cs

@@ -20,6 +20,18 @@
     where TRecord : unmanaged
     where TToken : unmanaged
 {
+    readonly bool useSecondaryIndex;
+
+    bool isDropped;
+
+    bool isDisposed;
+
+    readonly SearchOnIndexOfTokenRecordPreviousToken<TRecord, TToken>
+        searchAlgorithm;
+
+    readonly AdvancedSearchOnIndexOfTokenRecordPreviousToken<TRecord, TToken>
+        advancedSearchAlgorithm;
+
     /// <summary>
     /// Gets the primary zone tree used to store and retrieve records by token and previous token.
     /// </summary>
@@ -70,15 +82,10 @@
         }
     }
 
-    readonly bool useSecondaryIndex;
-
-    bool isDropped = false;
-
-    readonly SearchOnIndexOfTokenRecordPreviousToken<TRecord, TToken>
-        searchAlgorithm;
-
-    readonly AdvancedSearchOnIndexOfTokenRecordPreviousToken<TRecord, TToken>
-        advancedSearchAlgorithm;
+    /// <summary>
+    /// Returns true if the index is dropped, otherwise false.
+    /// </summary>
+    public bool IsIndexDropped { get => isDropped; }
 
     /// <summary>
     /// Initializes a new instance of the <see cref="IndexOfTokenRecordPreviousToken{TRecord, TToken}"/> class,
@@ -112,7 +119,8 @@
             tokenComparer = ComponentsForKnownTypes.GetComparer<TToken>();
         var factory1 = new ZoneTreeFactory<CompositeKeyOfTokenRecordPrevious<TRecord, TToken>, byte>()
             .SetDataDirectory($"{dataPath}/index1")
-            .SetIsValueDeletedDelegate((in byte x) => x == 1)
+            .SetIsDeletedDelegate(
+                (in CompositeKeyOfTokenRecordPrevious<TRecord, TToken> key, in byte value) => value == 1)
             .SetMarkValueDeletedDelegate((ref byte x) => x = 1)
             .SetKeySerializer(new StructSerializer<CompositeKeyOfTokenRecordPrevious<TRecord, TToken>>())
             .SetComparer(
@@ -135,7 +143,8 @@
         {
             var factory2 = new ZoneTreeFactory<CompositeKeyOfRecordToken<TRecord, TToken>, byte>()
                 .SetDataDirectory($"{dataPath}/index2")
-                .SetIsValueDeletedDelegate((in byte x) => x == 1)
+                .SetIsDeletedDelegate(
+                    (in CompositeKeyOfRecordToken<TRecord, TToken> key, in byte value) => value == 1)
                 .SetMarkValueDeletedDelegate((ref byte x) => x = 1)
                 .SetKeySerializer(new StructSerializer<CompositeKeyOfRecordToken<TRecord, TToken>>())
                 .SetComparer(
@@ -161,15 +170,10 @@
     /// </summary>
     public void ThrowIfIndexIsDropped()
     {
         if (isDropped) throw new Exception($"{nameof(
             IndexOfTokenRecordPreviousToken<TRecord, TToken>)} is dropped.");
     }
 
-    /// <summary>
-    /// Returns true if the index is dropped, otherwise false.
-    /// </summary>
-    public bool IsIndexDropped { get => isDropped; }
-
     /// <summary>
     /// Evicts data from memory to disk in both primary and secondary zone trees.
     /// </summary>
@@ -237,7 +241,7 @@
             Record = record,
             Token = token,
         };
-        ZoneTree2.TryAdd(key, new());
+        ZoneTree2.TryAdd(key, new(), out _);
     }
 
     /// <summary>
@@ -401,8 +405,6 @@
         return advancedSearchAlgorithm.Search(query, cancellationToken);
     }
 
-    bool isDisposed = false;
-
     /// <summary>
     /// Disposes the resources used by the index.
     /// </summary>

src/ZoneTree.FullTextSearch/Model/CompositeKeyOfRecordToken.cs

@@ -10,7 +10,9 @@ namespace ZoneTree.FullTextSearch.Model;
 /// <typeparam name="TRecord">The type of the record component of the key. Must be an unmanaged type.</typeparam>
 /// <typeparam name="TToken">The type of the token component of the key. Must be an unmanaged type.</typeparam>
 [StructLayout(LayoutKind.Sequential)]
-public struct CompositeKeyOfRecordToken<TRecord, TToken> : IEquatable<CompositeKeyOfRecordToken<TRecord, TToken>> where TRecord : unmanaged
+public struct CompositeKeyOfRecordToken<TRecord, TToken>
+    : IEquatable<CompositeKeyOfRecordToken<TRecord, TToken>>
+    where TRecord : unmanaged
     where TToken : unmanaged
 {
     /// <summary>

src/ZoneTree.FullTextSearch/Model/CompositeKeyOfTokenRecordPrevious.cs

@@ -10,7 +10,9 @@ namespace ZoneTree.FullTextSearch;
 /// <typeparam name="TRecord">The type of the record component of the key. Must be an unmanaged type.</typeparam>
 /// <typeparam name="TToken">The type of the token components of the key. Must be an unmanaged type.</typeparam>
 [StructLayout(LayoutKind.Sequential)]
-public struct CompositeKeyOfTokenRecordPrevious<TRecord, TToken> : IEquatable<CompositeKeyOfTokenRecordPrevious<TRecord, TToken>> where TRecord : unmanaged
+public struct CompositeKeyOfTokenRecordPrevious<TRecord, TToken>
+    : IEquatable<CompositeKeyOfTokenRecordPrevious<TRecord, TToken>>
+    where TRecord : unmanaged
     where TToken : unmanaged
 {
     /// <summary>

src/ZoneTree.FullTextSearch/Search/AdvancedSearchOnIndexOfTokenRecordPreviousToken.cs

@@ -284,7 +284,10 @@ HashSet<TRecord> ProcessAllTokens(
                     if (records.Contains(record)) continue;
 
                     if (!DoesRecordMatchesTheQuery(query.QueryNode, record))
+                    {
+                        skipRecords.Add(record);
                         continue;
+                    }
 
                     if (off >= skip)
                     {
@@ -325,7 +328,10 @@ HashSet<TRecord> ProcessEntireIndex(
                 if (records.Contains(record)) continue;
 
                 if (!DoesRecordMatchesTheQuery(query.QueryNode, record))
+                {
+                    skipRecords.Add(record);
                     continue;
+                }
 
                 if (off >= skip)
                 {

src/ZoneTree.FullTextSearch/Search/SearchOnIndexOfTokenRecordPreviousToken.cs

@@ -198,10 +198,8 @@ HashSet<TRecord> FindRecordsMatchingAllTokens(
                 // different previous token.
                 if (records.Contains(record)) continue;
 
-                if (!DoesRecordContainAllTokens(tokens, record))
-                    continue;
-
-                if (!DoesRecordContainAnyOfTheFacets(facets, record))
+                if (!DoesRecordContainAllTokens(tokens, record) ||
+                    !DoesRecordContainAnyOfTheFacets(facets, record))
                     continue;
 
                 if (off >= skip)

src/ZoneTree.FullTextSearch/SearchEngines/HashedSearchEngine.cs

@@ -413,7 +413,7 @@ public void Drop()
         Index.Drop();
     }
 
-    bool isDisposed = false;
+    bool isDisposed;
 
     /// <summary>
     /// Disposes the resources used by the search engine.

src/ZoneTree.FullTextSearch/ZoneTree.FullTextSearch.csproj

@@ -38,7 +38,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="ZoneTree" Version="1.8.0" />
+    <PackageReference Include="ZoneTree" Version="1.8.1" />
   </ItemGroup>
 
   <ItemGroup>