From 7f50900207379bc20532af0f2803d1aa39f4dac2 Mon Sep 17 00:00:00 2001 From: Allen Luo Date: Tue, 5 Nov 2024 19:52:05 +0800 Subject: [PATCH 1/3] Update the lru cache in both client and server side to be size-based instead of entry-based --- packages/core/src/util/cache.js | 49 +++++++++++++++++++-------------- packages/duckdb/src/Cache.js | 47 +++++++++++++++++-------------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/packages/core/src/util/cache.js b/packages/core/src/util/cache.js index fcce1bad..fd12a29d 100644 --- a/packages/core/src/util/cache.js +++ b/packages/core/src/util/cache.js @@ -9,35 +9,41 @@ export const voidCache = () => ({ }); export function lruCache({ - max = 1000, // max entries + max = 1 * 1024 * 1024, // 1 MB cache size as default ttl = 3 * 60 * 60 * 1000 // time-to-live, default 3 hours } = {}) { let cache = new Map; + let curr_size = 0; function evict() { const expire = performance.now() - ttl; - let lruKey = null; - let lruLast = Infinity; - for (const [key, value] of cache) { - const { last } = value; + while (curr_size > max) { + let lruKey = null; + let lruLast = Infinity; - // least recently used entry seen so far - if (last < lruLast) { - lruKey = key; - lruLast = last; + for (const [key, value] of cache) { + const { last } = value; + + // least recently used entry seen so far + if (last < lruLast) { + lruKey = key; + lruLast = last; + } + + // remove if time since last access exceeds ttl + if (expire > last) { + cache.delete(key); + curr_size -= new Blob([value]).size; + } } - - // remove if time since last access exceeds ttl - if (expire > last) { - cache.delete(key); + + // remove lru entry + if (cache.has(lruKey) && curr_size > max) { + curr_size -= new Blob([cache.get(lruKey)]).size; + cache.delete(lruKey); } } - - // remove lru entry - if (lruKey) { - cache.delete(lruKey); - } } return { @@ -49,8 +55,11 @@ export function lruCache({ } }, set(key, value) { - cache.set(key, { last: performance.now(), value }); - if (cache.size > max) requestIdle(evict); + let set_value = { last: performance.now(), value }; + cache.set(key, set_value); + curr_size += new Blob([value]).size; + + if (curr_size > max) requestIdle(evict); return value; }, clear() { cache = new Map; } diff --git a/packages/duckdb/src/Cache.js b/packages/duckdb/src/Cache.js index 3867efe3..96a0c8ee 100644 --- a/packages/duckdb/src/Cache.js +++ b/packages/duckdb/src/Cache.js @@ -22,7 +22,7 @@ class CacheEntry { export class Cache { constructor({ - max = 10000, // max entries + max = 10 * 1024 * 1024, // 10 MB cache size as default dir = DEFAULT_CACHE_DIR, ttl = DEFAULT_TTL }) { @@ -30,6 +30,7 @@ export class Cache { this.max = max; this.dir = dir; this.ttl = ttl; + this.curr_size = 0; readEntries(dir, this.cache); } @@ -52,40 +53,46 @@ export class Cache { set(key, data, { persist = false, ttl = this.ttl } = {}) { const entry = new CacheEntry(data, persist ? Infinity : ttl); this.cache.set(key, entry); + this.curr_size += new Blob([entry]).size; if (persist) writeEntry(this.dir, key, entry); if (this.shouldEvict()) setTimeout(() => this.evict()); return this; } shouldEvict() { - return this.cache.size > this.max; + return this.curr_size > this.max; } evict() { const expire = performance.now(); - let lruKey = null; - let lruLast = Infinity; - for (const [key, entry] of this.cache) { - const { last } = entry; - if (last === Infinity) continue; - - // least recently used entry seen so far - if (last < lruLast) { - lruKey = key; - lruLast = last; + while (this.shouldEvict()) { + let lruKey = null; + let lruLast = Infinity; + + for (const [key, entry] of this.cache) { + const { last } = entry; + if (last === Infinity) continue; + + // least recently used entry seen so far + if (last < lruLast) { + lruKey = key; + lruLast = last; + } + + // remove if time since last access exceeds ttl + if (expire > last) { + this.cache.delete(key); + this.curr_size -= new Blob([entry]).size; + } } - // remove if time since last access exceeds ttl - if (expire > last) { - this.cache.delete(key); + // remove lru entry + if (this.shouldEvict() && this.cache.has(lruKey)) { + this.curr_size -= new Blob([this.cache.get(lruKey)]).size; + this.cache.delete(lruKey); } } - - // remove lru entry - if (this.cache.size > this.max && lruKey) { - this.cache.delete(lruKey); - } } } From 31ad0e59bd929da8061799e3d92ce56f3deb190b Mon Sep 17 00:00:00 2001 From: Allen Luo Date: Tue, 19 Nov 2024 22:19:22 +0800 Subject: [PATCH 2/3] Optimize the size-based memory to avoid redundant calculation of entry size --- packages/core/src/util/cache.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/core/src/util/cache.js b/packages/core/src/util/cache.js index fd12a29d..dae2e843 100644 --- a/packages/core/src/util/cache.js +++ b/packages/core/src/util/cache.js @@ -13,34 +13,36 @@ export function lruCache({ ttl = 3 * 60 * 60 * 1000 // time-to-live, default 3 hours } = {}) { let cache = new Map; - let curr_size = 0; + let currSize = 0; function evict() { const expire = performance.now() - ttl; - while (curr_size > max) { + while (currSize > max) { let lruKey = null; let lruLast = Infinity; + let lruSize = null; for (const [key, value] of cache) { - const { last } = value; + const { last, size } = value; // least recently used entry seen so far if (last < lruLast) { lruKey = key; lruLast = last; + lruSize = size; } // remove if time since last access exceeds ttl if (expire > last) { cache.delete(key); - curr_size -= new Blob([value]).size; + currSize -= lruSize; } } // remove lru entry - if (cache.has(lruKey) && curr_size > max) { - curr_size -= new Blob([cache.get(lruKey)]).size; + if (cache.has(lruKey)) { + currSize -= cache.get(lruKey).size; cache.delete(lruKey); } } @@ -55,11 +57,15 @@ export function lruCache({ } }, set(key, value) { - let set_value = { last: performance.now(), value }; - cache.set(key, set_value); - curr_size += new Blob([value]).size; + let setValue = { + last: performance.now(), + size: new Blob([value]).size, + value + }; + cache.set(key, setValue); + currSize += setValue.size; - if (curr_size > max) requestIdle(evict); + if (currSize > max) requestIdle(evict); return value; }, clear() { cache = new Map; } From ced98de286b9bad668d7e2a1baf03cd55810a05c Mon Sep 17 00:00:00 2001 From: Allen Luo Date: Tue, 19 Nov 2024 22:29:52 +0800 Subject: [PATCH 3/3] Fix the bug of stucking after reaching cache limit --- packages/core/src/util/cache.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/core/src/util/cache.js b/packages/core/src/util/cache.js index dae2e843..ee3bba0f 100644 --- a/packages/core/src/util/cache.js +++ b/packages/core/src/util/cache.js @@ -36,13 +36,13 @@ export function lruCache({ // remove if time since last access exceeds ttl if (expire > last) { cache.delete(key); - currSize -= lruSize; + currSize -= size; } } // remove lru entry if (cache.has(lruKey)) { - currSize -= cache.get(lruKey).size; + currSize -= lruSize; cache.delete(lruKey); } } @@ -51,6 +51,7 @@ export function lruCache({ return { get(key) { const entry = cache.get(key); + console.log("Size", currSize, cache.size) if (entry) { entry.last = performance.now(); return entry.value; @@ -62,6 +63,10 @@ export function lruCache({ size: new Blob([value]).size, value }; + + if (cache.has(key)) { + currSize -= cache.get(key).size; + } cache.set(key, setValue); currSize += setValue.size;