diff --git a/src/builder/CacheBuilder.ts b/src/builder/CacheBuilder.ts index 61e700c..501f2e1 100644 --- a/src/builder/CacheBuilder.ts +++ b/src/builder/CacheBuilder.ts @@ -1,3 +1,4 @@ +import { NoopMetrics, StandardMetrics } from '../cache'; import { BoundedCache } from '../cache/bounded/BoundedCache'; import { BoundlessCache } from '../cache/boundless/BoundlessCache'; import { Cache } from '../cache/Cache'; @@ -5,10 +6,8 @@ import { Expirable } from '../cache/expiration/Expirable'; import { ExpirationCache } from '../cache/expiration/ExpirationCache'; import { MaxAgeDecider } from '../cache/expiration/MaxAgeDecider'; import { KeyType } from '../cache/KeyType'; -import { DefaultLoadingCache } from '../cache/loading/DefaultLoadingCache'; import { Loader } from '../cache/loading/Loader'; import { LoadingCache } from '../cache/loading/LoadingCache'; -import { MetricsCache } from '../cache/metrics/MetricsCache'; import { RemovalListener } from '../cache/RemovalListener'; import { Weigher } from '../cache/Weigher'; @@ -17,24 +16,26 @@ export interface CacheBuilder { * Set a listener that will be called every time something is removed * from the cache. */ - withRemovalListener(listener: RemovalListener): this; + withRemovalListener(listener: RemovalListener): CacheBuilder; /** * Set the maximum number of items to keep in the cache before evicting * something. */ - maxSize(size: number): this; + maxSize(size: number): CacheBuilder; /** * Set a function to use to determine the size of a cached object. */ - withWeigher(weigher: Weigher): this; + withWeigher(weigher: Weigher): CacheBuilder; /** * Change to a cache where get can also resolve values if provided with * a function as the second argument. + * + * @deprecated - all caches support loading with a second parameter */ - loading(): LoadingCacheBuilder; + loading(): CacheBuilder; /** * Change to a loading cache, where the get-method will return instances @@ -46,18 +47,18 @@ export interface CacheBuilder { * Set that the cache should expire items some time after they have been * written to the cache. */ - expireAfterWrite(time: number | MaxAgeDecider): this; + expireAfterWrite(time: number | MaxAgeDecider): CacheBuilder; /** * Set that the cache should expire items some time after they have been * read from the cache. */ - expireAfterRead(time: number | MaxAgeDecider): this; + expireAfterRead(time: number | MaxAgeDecider): CacheBuilder; /** * Activate tracking of metrics for this cache. */ - metrics(): this; + metrics(): CacheBuilder; /** * Build the cache. @@ -82,6 +83,7 @@ export class CacheBuilderImpl implements CacheBuilder; private optMaxNoReadAge?: MaxAgeDecider; private optMetrics: boolean = false; + private optLoader?: Loader; /** * Set a listener that will be called every time something is removed @@ -129,9 +131,11 @@ export class CacheBuilderImpl implements CacheBuilder { - return new LoadingCacheBuilderImpl(this, null); + public loading(): CacheBuilder { + return this; } /** @@ -146,7 +150,8 @@ export class CacheBuilderImpl implements CacheBuilder; } /** @@ -208,6 +213,10 @@ export class CacheBuilderImpl implements CacheBuilder; + + const metrics = this.optMetrics ? new StandardMetrics() : NoopMetrics; + const loader = this.optLoader; + if(typeof this.optMaxWriteAge !== 'undefined' || typeof this.optMaxNoReadAge !== 'undefined') { /* * Requested expiration - wrap the base cache a bit as it needs @@ -230,90 +239,32 @@ export class CacheBuilderImpl implements CacheBuilder implements LoadingCacheBuilder { - private parent: CacheBuilder; - private loader: Loader | null; - - public constructor(parent: CacheBuilder, loader: Loader | null) { - this.parent = parent; - this.loader = loader; - } - - public withRemovalListener(listener: RemovalListener): this { - this.parent.withRemovalListener(listener); - return this; - } - - public maxSize(size: number): this { - this.parent.maxSize(size); - return this; - } - - public withWeigher(weigher: Weigher): this { - this.parent.withWeigher(weigher); - return this; - } - - public loading(): LoadingCacheBuilder { - throw new Error('Already building a loading cache'); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - public withLoader(loader: Loader): LoadingCacheBuilder { - throw new Error('Already building a loading cache'); - } - - public expireAfterWrite(time: number | MaxAgeDecider): this { - this.parent.expireAfterWrite(time); - return this; - } - - public expireAfterRead(time: number | MaxAgeDecider): this { - this.parent.expireAfterRead(time); - return this; - } - - public metrics(): this { - this.parent.metrics(); - return this; - } - - public build(): LoadingCache { - return new DefaultLoadingCache({ - loader: this.loader, - - parent: this.parent.build() - }); - } -} - /** * Helper function to create a weigher that uses an Expirable object. * diff --git a/src/cache/AbstractCache.ts b/src/cache/AbstractCache.ts index 0a324f8..b507946 100644 --- a/src/cache/AbstractCache.ts +++ b/src/cache/AbstractCache.ts @@ -1,5 +1,7 @@ import { Cache } from './Cache'; import { KeyType } from './KeyType'; +import { Loader } from './loading'; +import { LoaderResult } from './loading/LoaderManager'; import { Metrics } from './metrics/Metrics'; /** @@ -12,6 +14,7 @@ export abstract class AbstractCache implements Cache public abstract weightedSize: number; public abstract set(key: K, value: V): V | null; + public abstract get(key: K, loader?: Loader | undefined): Promise> ; public abstract getIfPresent(key: K): V | null; public abstract peek(key: K): V | null; public abstract has(key: K): boolean; diff --git a/src/cache/Cache.ts b/src/cache/Cache.ts index 16ff72c..39a7a94 100644 --- a/src/cache/Cache.ts +++ b/src/cache/Cache.ts @@ -1,11 +1,12 @@ import { KeyType } from './KeyType'; +import { Loader } from './loading/Loader'; +import { LoaderResult } from './loading/LoaderManager'; import { Metrics } from './metrics/Metrics'; /** * Cache for a mapping between keys and values. */ export interface Cache { - /** * The maximum size the cache can be. Will be -1 if the cache is unbounded. */ @@ -46,6 +47,19 @@ export interface Cache { */ getIfPresent(key: K): V | null; + /** + * Get cached value or load it if not currently cached. Updates the usage + * of the key. + * + * @param key - + * key to get + * @param loader - + * optional loader to use for loading the object + * @returns + * promise that resolves to the loaded value + */ + get(key: K, loader: Loader): Promise>; + /** * Peek to see if a key is present without updating the usage of the * key. Returns the value associated with the key or `null` if the key diff --git a/src/cache/bounded/BoundedCache.ts b/src/cache/bounded/BoundedCache.ts index afc0960..20338fe 100644 --- a/src/cache/bounded/BoundedCache.ts +++ b/src/cache/bounded/BoundedCache.ts @@ -3,7 +3,11 @@ import { Cache } from '../Cache'; import { CacheNode } from '../CacheNode'; import { CacheSPI } from '../CacheSPI'; import { KeyType } from '../KeyType'; +import { Loader, resolveLoader } from '../loading'; +import { LoaderManager, LoaderResult } from '../loading/LoaderManager'; import { Metrics } from '../metrics/Metrics'; +import { MetricsRecorder } from '../metrics/MetricsRecorder'; +import { NoopMetrics } from '../metrics/NoopMetrics'; import { RemovalListener } from '../RemovalListener'; import { RemovalReason } from '../RemovalReason'; import { ON_REMOVE, ON_MAINTENANCE, TRIGGER_REMOVE, MAINTENANCE } from '../symbols'; @@ -11,7 +15,6 @@ import { Weigher } from '../Weigher'; import { CountMinSketch } from './CountMinSketch'; - const percentInMain = 0.99; const percentProtected = 0.8; const percentOverflow = 0.01; @@ -44,27 +47,32 @@ export interface BoundedCacheOptions { * Listener to call whenever something is removed from the cache. */ removalListener?: RemovalListener | null; + + /** + * The default loader for this cache. + */ + loader?: Loader | undefined | null; + + /** + * Metrics recorder for this cache. + */ + metrics?: MetricsRecorder; } /** * Data as used by the bounded cache. */ -interface BoundedCacheData { +interface BoundedCacheData extends BoundedCacheOptions { /** * Values within the cache. */ values: Map>; /** - * The maximum size of the cache or -1 if the cache uses weighing. + * Manages all loader promises. */ - maxSize: number; + promises: LoaderManager; - /** - * Weigher being used for this cache. Invoked to determine the weight of - * an item being cached. - */ - weigher: Weigher | null; /** * Maximum size of the cache as a weight. */ @@ -74,11 +82,6 @@ interface BoundedCacheData { */ weightedSize: number; - /** - * Listener to invoke when removals occur. - */ - removalListener: RemovalListener | null; - /** * Sketch used to keep track of the frequency of which items are used. */ @@ -122,6 +125,11 @@ interface BoundedCacheData { * SLRU probation segment, 20% * (100% - windowSize) of the total cache */ probation: ProbationSection; + + /** + * Metrics recorder for this cache. + */ + metrics: MetricsRecorder; } /** @@ -283,7 +291,15 @@ export class BoundedCache extends AbstractCache impl maintenanceTimeout: null, forceEvictionLimit: options.maxSize + Math.max(Math.floor(options.maxSize * percentOverflow), 5), - maintenanceInterval: 5000 + maintenanceInterval: 5000, + + metrics: options.metrics ?? NoopMetrics, + + loader: options.loader, + + promises: new LoaderManager((key, value) => { + this.set(key, value); + }), }; } @@ -404,53 +420,29 @@ export class BoundedCache extends AbstractCache impl * current value or `null` */ public getIfPresent(key: K) { - const data = this[DATA]; - - const node = data.values.get(key); - if(! node) { - // This value does not exist in the cache - data.adaptiveData.misses++; - return null; - } - - // Keep track of the hit - data.adaptiveData.hits++; - - // Register access to the key - data.sketch.update(node.hashCode); - - switch(node.location) { - case Location.WINDOW: - // In window cache, mark as most recently used - node.moveToTail(data.window.head); - break; - case Location.PROBATION: - // In SLRU probation segment, move to protected - node.location = Location.PROTECTED; - node.moveToTail(data.protected.head); - - // Plenty of room, keep track of the size - data.protected.size += node.weight; + const node = this.getNodeIfPresent(key); + return node?.value ?? null; + } - while(data.protected.size > data.protected.maxSize) { - /* - * There is now too many nodes in the protected segment - * so demote the least recently used. - */ - const lru = data.protected.head.next; - lru.location = Location.PROBATION; - lru.moveToTail(data.probation.head); - data.protected.size -= lru.weight; - } + /** + * Get cached value or load it if not currently cached. Updates the usage + * of the key. + * + * @param key - + * key to get + * @param loader - + * optional loader to use for loading the object + * @returns + * promise that resolves to the loaded value + */ + public get(key: K, loader?: Loader): Promise> { + const existingNode = this.getNodeIfPresent(key); - break; - case Location.PROTECTED: - // SLRU protected segment, mark as most recently used - node.moveToTail(data.protected.head); - break; + if(existingNode && existingNode.value !== null) { + return Promise.resolve(existingNode.value as LoaderResult); } - return node.value; + return this[DATA].promises.get(key, resolveLoader(this[DATA].loader, loader)) as Promise>; } /** @@ -588,9 +580,74 @@ export class BoundedCache extends AbstractCache impl * Get metrics for this cache. Returns an object with the keys `hits`, * `misses` and `hitRate`. For caches that do not have metrics enabled * trying to access metrics will throw an error. + * + * @returns + * the metrics for this cache */ public get metrics(): Metrics { - throw new Error('Metrics are not supported by this cache'); + return this[DATA].metrics; + } + + /** + * Get the cached value for the specified key if it exists. Will return + * the value or `null` if no cached value exist. Updates the usage of the + * key. + * + * @param key - + * key to get + * @returns + * current value or `null` + */ + protected getNodeIfPresent(key: K): BoundedNode | null { + const data = this[DATA]; + + const node = data.values.get(key); + if(! node) { + // This value does not exist in the cache + data.metrics.miss(); + data.adaptiveData.misses += 1; + return null; + } + + // Keep track of the hit + data.metrics.hit(); + data.adaptiveData.hits += 1; + + // Register access to the key + data.sketch.update(node.hashCode); + + switch(node.location) { + case Location.WINDOW: + // In window cache, mark as most recently used + node.moveToTail(data.window.head); + break; + case Location.PROBATION: + // In SLRU probation segment, move to protected + node.location = Location.PROTECTED; + node.moveToTail(data.protected.head); + + // Plenty of room, keep track of the size + data.protected.size += node.weight; + + while(data.protected.size > data.protected.maxSize) { + /* + * There is now too many nodes in the protected segment + * so demote the least recently used. + */ + const lru = data.protected.head.next; + lru.location = Location.PROBATION; + lru.moveToTail(data.probation.head); + data.protected.size -= lru.weight; + } + + break; + case Location.PROTECTED: + // SLRU protected segment, mark as most recently used + node.moveToTail(data.protected.head); + break; + } + + return node; } private [TRIGGER_REMOVE](key: K, value: any, cause: RemovalReason) { diff --git a/src/cache/boundless/BoundlessCache.ts b/src/cache/boundless/BoundlessCache.ts index f2c427e..097b18d 100644 --- a/src/cache/boundless/BoundlessCache.ts +++ b/src/cache/boundless/BoundlessCache.ts @@ -2,7 +2,12 @@ import { AbstractCache } from '../AbstractCache'; import { Cache } from '../Cache'; import { CacheSPI } from '../CacheSPI'; import { KeyType } from '../KeyType'; +import { Loader } from '../loading/Loader'; +import { LoaderManager, LoaderResult } from '../loading/LoaderManager'; +import { resolveLoader } from '../loading/LoadingCache'; import { Metrics } from '../metrics/Metrics'; +import { MetricsRecorder } from '../metrics/MetricsRecorder'; +import { NoopMetrics } from '../metrics/NoopMetrics'; import { RemovalListener } from '../RemovalListener'; import { RemovalReason } from '../RemovalReason'; import { ON_REMOVE, ON_MAINTENANCE, TRIGGER_REMOVE, MAINTENANCE } from '../symbols'; @@ -19,17 +24,35 @@ export interface BoundlessCacheOptions { * Listener that triggers when a cached value is removed. */ removalListener?: RemovalListener | undefined | null; + + /** + * The default loader for this cache. + */ + loader?: Loader | undefined | null; + + /** + * Metrics recorder for this cache. + */ + metrics?: MetricsRecorder; } /** * Data as used by the boundless cache. */ -interface BoundlessCacheData { +interface BoundlessCacheData extends BoundlessCacheOptions { values: Map; - removalListener: RemovalListener | null; + /** + * Manages all loader promises. + */ + promises: LoaderManager; evictionTimeout: any; + + /** + * Metrics recorder for this cache. + */ + metrics: MetricsRecorder; } /** @@ -49,7 +72,15 @@ export class BoundlessCache extends AbstractCache im removalListener: options.removalListener || null, - evictionTimeout: null + evictionTimeout: null, + + metrics: options.metrics ?? NoopMetrics, + + loader: options.loader, + + promises: new LoaderManager((key, value) => { + this.set(key, value); + }), }; } @@ -129,7 +160,34 @@ export class BoundlessCache extends AbstractCache im public getIfPresent(key: K): V | null { const data = this[DATA]; const value = data.values.get(key); - return value === undefined ? null : value; + if(value === undefined || value === null) { + data.metrics.miss(); + return null; + } + data.metrics.hit(); + return value; + } + + /** + * Get cached value or load it if not currently cached. Updates the usage + * of the key. + * + * @param key - + * key to get + * @param loader - + * optional loader to use for loading the object + * @returns + * promise that resolves to the loaded value + */ + public get(key: K, loader?: Loader): Promise> { + const data = this[DATA]; + const value = data.values.get(key); + if(value !== null && value !== undefined) { + data.metrics.hit(); + return Promise.resolve(value as LoaderResult); + } + data.metrics.miss(); + return this[DATA].promises.get(key, resolveLoader(this[DATA].loader, loader)) as Promise>; } /** @@ -241,9 +299,12 @@ export class BoundlessCache extends AbstractCache im * Get metrics for this cache. Returns an object with the keys `hits`, * `misses` and `hitRate`. For caches that do not have metrics enabled * trying to access metrics will throw an error. + * + * @returns + * the metrics for this cache */ public get metrics(): Metrics { - throw new Error('Metrics are not supported by this cache'); + return this[DATA].metrics; } private [TRIGGER_REMOVE](key: K, value: any, reason: RemovalReason) { diff --git a/src/cache/expiration/ExpirationCache.ts b/src/cache/expiration/ExpirationCache.ts index 2fe30cb..6e16839 100644 --- a/src/cache/expiration/ExpirationCache.ts +++ b/src/cache/expiration/ExpirationCache.ts @@ -1,10 +1,16 @@ +import { isPromise } from 'util/types'; + import { AbstractCache } from '../AbstractCache'; import { Cache } from '../Cache'; import { CacheSPI } from '../CacheSPI'; import { CommonCacheOptions } from '../CommonCacheOptions'; import { KeyType } from '../KeyType'; +import { Loader } from '../loading/Loader'; +import { LoaderResult } from '../loading/LoaderManager'; +import { NoopMetrics } from '../metrics'; import { Metrics } from '../metrics/Metrics'; +import { MetricsRecorder } from '../metrics/MetricsRecorder'; import { RemovalListener } from '../RemovalListener'; import { RemovalReason } from '../RemovalReason'; import { PARENT, ON_REMOVE, TRIGGER_REMOVE, ON_MAINTENANCE, MAINTENANCE } from '../symbols'; @@ -24,6 +30,16 @@ export interface ExpirationCacheOptions extends CommonCach maxNoReadAge?: MaxAgeDecider; parent: Cache>; + + /** + * The default loader for this cache. + */ + loader?: Loader | undefined | null; + + /** + * Metrics recorder for this cache. + */ + metrics?: MetricsRecorder; } interface ExpirationCacheData { @@ -33,6 +49,16 @@ interface ExpirationCacheData { maxWriteAge?: MaxAgeDecider; maxNoReadAge?: MaxAgeDecider; + + /** + * The default loader for this cache. + */ + loader?: Loader> | undefined | null; + + /** + * Metrics recorder for this cache. + */ + metrics: MetricsRecorder; } /** @@ -52,17 +78,23 @@ export class ExpirationCache extends AbstractCache i this[PARENT] = options.parent; + const timerWheel = new TimerWheel(keys => { + for(const key of keys) { + this.delete(key); + } + }); + this[DATA] = { maxWriteAge: options.maxWriteAge, maxNoReadAge: options.maxNoReadAge, removalListener: options.removalListener || null, - timerWheel: new TimerWheel(keys => { - for(const key of keys) { - this.delete(key); - } - }) + timerWheel, + + metrics: options.metrics ?? NoopMetrics, + + loader: options.loader ? wrapLoader(timerWheel, options.loader) : undefined, }; // Custom onRemove handler for the parent cache @@ -154,27 +186,68 @@ export class ExpirationCache extends AbstractCache i * current value or `null` */ public getIfPresent(key: K): V | null { + const data = this[DATA]; const node = this[PARENT].getIfPresent(key); - if(node) { - if(node.isExpired()) { - // Check if the node is expired and return null if so - return null; - } + const value = this.unwrapNode(key, node); + data.metrics.record(value !== null); + return value; + } - // Reschedule if we have a maximum age between reads - const data = this[DATA]; - if(data.maxNoReadAge) { - const age = data.maxNoReadAge(key, node.value as V); - if(! data.timerWheel.schedule(node as TimerNode, age)) { - // Age was not accepted by wheel, expire it directly - this.delete(key); - } - } + /** + * Get cached value or load it if not currently cached. Updates the usage + * of the key. + * + * @param key - + * key to get + * @param loader - + * optional loader to use for loading the object + * @returns + * promise that resolves to the loaded value + */ + public get(key: K, loader?: Loader): Promise> { + const data = this[DATA]; + const nodePromise = this[PARENT].get(key, resolveWrappedLoader(data.timerWheel, data.loader, loader)); + return nodePromise.then(node => { + const value = this.unwrapNode(key, node); + data.metrics.record(value !== null); + return value as LoaderResult; + }); + } + + /** + * Unwraps an expirable node as a value, handling expiration logic. + * + * @param key - + * the key associated to the node + * + * @param node - + * the node to unwrap + * + * @returns + * the unwrapped value or null + */ + protected unwrapNode(key: K, node: Expirable | null): V | null { + const data = this[DATA]; + + if(! node) { + return null; + } - return node.value; + if(node.isExpired()) { + // Check if the node is expired and return null if so + return null; } - return null; + // Reschedule if we have a maximum age between reads + if(data.maxNoReadAge) { + const age = data.maxNoReadAge(key, node.value as V); + if(! data.timerWheel.schedule(node as TimerNode, age)) { + // Age was not accepted by wheel, expire it directly + this.delete(key); + } + } + + return node.value; } /** @@ -263,7 +336,7 @@ export class ExpirationCache extends AbstractCache i * metrics if available via the parent cache */ public get metrics(): Metrics { - return this[PARENT].metrics; + return this[DATA].metrics; } private [MAINTENANCE]() { @@ -289,3 +362,65 @@ export class ExpirationCache extends AbstractCache i } } } + +/** + * Wraps a loader so it returns expirable nodes. + * + * @param timerWheel - + * the timerWheel + * + * @param loader - + * the loader to wrap + * + * @returns + * a wrapped loader that returns Expirable nodes + */ +function wrapLoader(timerWheel: TimerWheel, loader: Loader): Loader> { + return key => { + const maybePromise = loader(key); + if(maybePromise === null || maybePromise === undefined) { + return null; + } + if(! isPromise(maybePromise)) { + return timerWheel.node(key, maybePromise); + } + return maybePromise.then(value => { + if(value === null || value === undefined) { + return null; + } + return timerWheel.node(key, value); + }); + }; +} + +/** + * Given two loaders, returns the most relevant one, or throws an exception if the provded loaders are invalid. + * + * @param timerWheel - + * the timerWheel + * + * @param defaultLoader - + * the primary loader + * + * @param loader - + * the specific loader that can override the default one + * + * @returns + * resolved loader, or throws an exception + */ +export function resolveWrappedLoader( + timerWheel: TimerWheel, + defaultLoader: Loader> | undefined | null, + loader: Loader | undefined | null, +) : Loader> { + if(defaultLoader !== undefined && defaultLoader !== null) { + return defaultLoader; + } + if(loader !== undefined && loader !== null) { + if(typeof loader !== 'function') { + throw new Error('If loader is used it must be a function that returns a value or a Promise'); + } + return wrapLoader(timerWheel, loader); + } + throw new Error('No loader is provided'); +} diff --git a/src/cache/loading/DefaultLoadingCache.ts b/src/cache/loading/DefaultLoadingCache.ts deleted file mode 100644 index 359ab35..0000000 --- a/src/cache/loading/DefaultLoadingCache.ts +++ /dev/null @@ -1,94 +0,0 @@ - -import { Cache } from '../Cache'; -import { CommonCacheOptions } from '../CommonCacheOptions'; -import { KeyType } from '../KeyType'; -import { WrappedCache } from '../WrappedCache'; - -import { Loader } from './Loader'; -import { LoadingCache } from './LoadingCache'; - -const DATA = Symbol('loadingData'); - -/** - * Options available for a loading cache. - */ -export interface LoadingCacheOptions extends CommonCacheOptions { - loader?: Loader | undefined | null; - - parent: Cache; -} - -interface LoadingCacheData { - promises: Map>; - - loader: Loader | null; -} - -/** - * Extension to another cache that will load items if they are not cached. - */ -export class DefaultLoadingCache extends WrappedCache implements LoadingCache { - private [DATA]: LoadingCacheData; - - public constructor(options: LoadingCacheOptions) { - super(options.parent, options.removalListener || null); - - this[DATA] = { - promises: new Map(), - loader: options.loader || null - }; - } - - /** - * Get cached value or load it if not currently cached. Updates the usage - * of the key. - * - * @param key - - * key to get - * @param loader - - * optional loader to use for loading the object - * @returns - * promise that resolves to the loaded value - */ - public get(key: K, loader?: Loader): Promise { - const currentValue = this.getIfPresent(key); - if(currentValue !== null) { - return Promise.resolve(currentValue); - } - - const data = this[DATA]; - - // First check if we are already loading this value - let promise = data.promises.get(key); - if(promise) return promise; - - // Create the initial promise if we are not already loading - if(typeof loader !== 'undefined') { - if(typeof loader !== 'function') { - throw new Error('If loader is used it must be a function that returns a value or a Promise'); - } - promise = Promise.resolve(loader(key)); - } else if(data.loader) { - promise = Promise.resolve(data.loader(key)); - } - - if(! promise) { - throw new Error('No way to load data for key: ' + key); - } - - // Enhance with handler that will remove promise and set value if success - const resolve = () => data.promises.delete(key); - promise = promise.then(result => { - this.set(key, result); - resolve(); - return result; - }).catch(err => { - resolve(); - throw err; - }); - - data.promises.set(key, promise); - - return promise; - } -} diff --git a/src/cache/loading/Loader.ts b/src/cache/loading/Loader.ts index 8ed1d38..0e3f367 100644 --- a/src/cache/loading/Loader.ts +++ b/src/cache/loading/Loader.ts @@ -4,4 +4,4 @@ import { KeyType } from '../KeyType'; * Function used to load a value in the cache. Can return a promise or a * value directly. */ -export type Loader = (key: K) => Promise | V; +export type Loader = (key: K) => Promise | V | null | undefined; diff --git a/src/cache/loading/LoaderManager.ts b/src/cache/loading/LoaderManager.ts new file mode 100644 index 0000000..bf0bbd6 --- /dev/null +++ b/src/cache/loading/LoaderManager.ts @@ -0,0 +1,84 @@ +import { isPromise } from 'util/types'; + +import { KeyType } from '../KeyType'; + +import { Loader } from './Loader'; + +export type LoaderResult = Exclude; + +/** + * Manages multiple concurrent loading get requests to a cache. + * Makes sure that only 1 promise is created which concurrent requests ultimately use. + * This is useful in case many concurrent database requests for the same piece of data + * come in, so only 1 database request will be made. + * + * @example + * ```ts + * // all of the requests are for the same key, and the cache will only allow 1 fetchMetadata to run concurrently. + * const requests = new Array(100).fill('key') + * .map(key => cache.get(key, async () => fetchMetadata(key))); + * + * const results = Promise.all(requests); + * ``` + */ +export class LoaderManager { + private readonly promises = new Map>(); + private readonly onResolved: (key: K, value: V) => void; + + public constructor(onResolved: (key: K, value: V) => void) { + this.onResolved = onResolved; + } + + public get( + key: K, + loader: Loader + ): Promise> { + // See if the promise for a key is already present + const existingPromise = this.promises.get(key); + if(existingPromise !== undefined) { + // existing loader already working, just use its result + return existingPromise as Promise>; + } + + // create a loader + const loaderResult = loader(key); + + // if it just returned nil, simply return null + if(loaderResult === null || loaderResult === undefined) { + return Promise.resolve(null) as Promise>; + } + + // if it's not a promise, there is nothing to manage, just return the result + if(! isPromise(loaderResult)) { + this.onResolved(key, loaderResult); + return Promise.resolve(loaderResult as LoaderResult); + } + + // create a new promise that waits for the loader to finish + // this promise will be reused for any concurrent loads on the same key + const newPromise = new Promise>((resolve, reject) => { + loaderResult.then( + r => { + // remove self from map + this.promises.delete(key); + if(r !== null && r !== undefined) { + // callback for when loader is resolved + this.onResolved(key, r); + } + // resolve the promise + resolve((r ?? null) as LoaderResult); + }, + err => { + // remove self from map + this.promises.delete(key); + reject(err); + } + ); + }); + + // add promise to map so others can share it + this.promises.set(key, newPromise as Promise>); + + return newPromise; + } +} diff --git a/src/cache/loading/LoadingCache.ts b/src/cache/loading/LoadingCache.ts index 19c7611..32e7052 100644 --- a/src/cache/loading/LoadingCache.ts +++ b/src/cache/loading/LoadingCache.ts @@ -2,6 +2,7 @@ import { Cache } from '../Cache'; import { KeyType } from '../KeyType'; import { Loader } from './Loader'; +import { LoaderResult } from './LoaderManager'; /** * Cache that also supports loading of data if it's not in the cache. @@ -18,5 +19,28 @@ export interface LoadingCache extends Cache { * @returns * promise that resolves to the loaded value */ - get(key: K, loader?: Loader): Promise; + get(key: K, loader?: Loader): Promise>; +} + +/** + * Given two loaders, returns the most relevant one, or throws an exception if the provded loaders are invalid. + * + * @param defaultLoader - + * the primary loader + * @param loader - + * the specific loader that can override the default one + * @returns + * resolved loader, or throws an exception + */ +export function resolveLoader(defaultLoader: Loader | undefined | null, loader: Loader | undefined | null) : Loader { + if(defaultLoader !== undefined && defaultLoader !== null) { + return defaultLoader; + } + if(loader !== undefined && loader !== null) { + if(typeof loader !== 'function') { + throw new Error('If loader is used it must be a function that returns a value or a Promise'); + } + return loader; + } + throw new Error('No loader is provided'); } diff --git a/src/cache/loading/index.ts b/src/cache/loading/index.ts index d545cbb..d4dc351 100644 --- a/src/cache/loading/index.ts +++ b/src/cache/loading/index.ts @@ -1,3 +1,2 @@ -export * from './DefaultLoadingCache'; export * from './LoadingCache'; export * from './Loader'; diff --git a/src/cache/metrics/MetricsCache.ts b/src/cache/metrics/MetricsCache.ts deleted file mode 100644 index 45060d7..0000000 --- a/src/cache/metrics/MetricsCache.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Cache } from '../Cache'; -import { CommonCacheOptions } from '../CommonCacheOptions'; -import { KeyType } from '../KeyType'; -import { WrappedCache } from '../WrappedCache'; - -import { Metrics } from './Metrics'; - - -const METRICS = Symbol('metrics'); - -/** - * Options available for a metrics cache. - */ -export interface MetricsCacheOptions extends CommonCacheOptions { - parent: Cache; -} - -/** - * Extension to a cache that tracks metrics about the size and hit rate of - * a cache. - */ -export class MetricsCache extends WrappedCache { - private [METRICS]: Metrics; - - public constructor(options: MetricsCacheOptions) { - super(options.parent, options.removalListener || null); - - this[METRICS] = { - hits: 0, - misses: 0, - - get hitRate() { - const total = this.hits + this.misses; - if(total === 0) return 1.0; - - return this.hits / total; - } - }; - } - - /** - * Get metrics for this cache. Returns an object with the keys `hits`, - * `misses` and `hitRate`. For caches that do not have metrics enabled - * trying to access metrics will throw an error. - * - * @returns - * metrics of cache - */ - public get metrics(): Metrics { - return this[METRICS]; - } - - /** - * Get the cached value for the specified key if it exists. Will return - * the value or `null` if no cached value exist. Updates the usage of the - * key. - * - * @param key - - * key to get - * @returns - * current value or `null` - */ - public getIfPresent(key: K): V | null { - const result = super.getIfPresent(key); - - if(result === null) { - this[METRICS].misses++; - } else { - this[METRICS].hits++; - } - return result; - } -} diff --git a/src/cache/metrics/MetricsRecorder.ts b/src/cache/metrics/MetricsRecorder.ts new file mode 100644 index 0000000..fbdb7bb --- /dev/null +++ b/src/cache/metrics/MetricsRecorder.ts @@ -0,0 +1,44 @@ +import { Metrics } from './Metrics'; + +/** A mutable version of metrics, for updating cache statistics. */ +export interface MetricsRecorder extends Metrics { + /** + * Increment number of hits to the metrics (default 1). + * + * @param count - + * amount to increment the hits by + */ + hit(count?: number): void; + + /** + * Increment number of hits to the cache (default 1). + * + * @param count - + * amount to increment the misses by + */ + miss(count?: number): void; + + /** + * Increment hits or misses by 1 (true = hit, false = miss). + * + * @param isHit - + * increment hits or misses + */ + record(isHit: boolean): void; + + /** + * Increment number of hits and misses to the metrics. + * + * @param hits - + * amount to increment the hits by + * + * @param misses - + * amount to increment the misses by + */ + count(hits: number, misses: number): void; + + /** + * Reset all metrics. + */ + reset(): void; +} diff --git a/src/cache/metrics/NoopMetrics.ts b/src/cache/metrics/NoopMetrics.ts new file mode 100644 index 0000000..fdeb09e --- /dev/null +++ b/src/cache/metrics/NoopMetrics.ts @@ -0,0 +1,14 @@ +import { MetricsRecorder } from './MetricsRecorder'; + +const noop: () => void = () => undefined; + +export const NoopMetrics: MetricsRecorder = Object.freeze({ + hitRate: 1.0, + hits: 0, + misses: 0, + record: noop, + reset: noop, + count: noop, + hit: noop, + miss: noop, +} satisfies MetricsRecorder); diff --git a/src/cache/metrics/StandardMetrics.ts b/src/cache/metrics/StandardMetrics.ts new file mode 100644 index 0000000..0604d41 --- /dev/null +++ b/src/cache/metrics/StandardMetrics.ts @@ -0,0 +1,51 @@ +import { MetricsRecorder } from './MetricsRecorder'; + +export class StandardMetrics implements MetricsRecorder { + private _hits: number = 0; + public get hits(): number { + return this._hits; + } + + private _misses: number = 0; + public get misses(): number { + return this._misses; + } + + public get hitRate(): number { + const total = this.hits + this.misses; + return total === 0 ? 1.0 : this.hits / total; + } + + public hit(count?: number) { + this._hits += count ?? 1; + } + + public miss(count?: number) { + this._misses += count ?? 1; + } + + public record(isHit: boolean) { + if(isHit) { + this._hits += 1; + } else { + this._misses += 1; + } + } + + public count(hits: number, misses: number) { + this._hits += hits; + this._misses += misses; + } + + public reset(): void { + this._hits = 0; + this._misses = 0; + } + + public toJSON() { + return { + hits: this.hits, + misses: this.misses, + }; + } +} diff --git a/src/cache/metrics/index.ts b/src/cache/metrics/index.ts index 7db4702..ce12924 100644 --- a/src/cache/metrics/index.ts +++ b/src/cache/metrics/index.ts @@ -1,2 +1,4 @@ export * from './Metrics'; -export * from './MetricsCache'; +export * from './MetricsRecorder'; +export * from './NoopMetrics'; +export * from './StandardMetrics'; diff --git a/test/loading.test.ts b/test/loading.test.ts index 566e1c8..f2613f2 100644 --- a/test/loading.test.ts +++ b/test/loading.test.ts @@ -1,7 +1,7 @@ - +import { newCache as newCacheImpl } from '../src/builder'; import { BoundlessCache } from '../src/cache/boundless'; import { KeyType } from '../src/cache/KeyType'; -import { DefaultLoadingCache, Loader } from '../src/cache/loading'; +import { Loader } from '../src/cache/loading'; import { RemovalReason } from '../src/cache/RemovalReason'; import { RemovalHelper } from './removal-helper'; @@ -11,10 +11,11 @@ import { RemovalHelper } from './removal-helper'; * @param loader */ function newCache(loader?: Loader) { - return new DefaultLoadingCache({ - loader: loader, - parent: new BoundlessCache({}) - }); + const builder = newCacheImpl(); + if(loader) { + return builder.withLoader(loader).build(); + } + return builder.build(); } describe('LoadingCache', function() { @@ -87,6 +88,48 @@ describe('LoadingCache', function() { .catch(() => null); }); + it('Caches loader promises for concurrent gets', async function() { + const resolves: Array<() => void> = []; + const cache = newCache(k => { + return new Promise(resolve => { + resolves.push(() => { + resolve((k * 2).toString()); + }); + }); + }); + + const promise1 = cache.get(100); + const promise2 = cache.get(100); + const promise3 = cache.get(100, () => 'whatever'); + const promise4 = cache.get(200); + + expect(promise1).toBe(promise2); + expect(promise2).toBe(promise3); + expect(promise1).not.toBe(promise4); + + resolves.forEach(r => r()); + + expect(await promise1).toBe('200'); + expect(await promise2).toBe('200'); + expect(await promise3).toBe('200'); + expect(await promise4).toBe('400'); + }); + + it.each([ + [ 'Standard', newCacheImpl().metrics().build() ], + [ 'Bounded', newCacheImpl().metrics().maxSize(100).build() ], + [ 'Expiring', newCacheImpl().metrics().expireAfterRead(1000).build() ], + ])('%s cache does not load null values', async (_type, cache) => { + const result1 = await cache.get(100, () => null); // misses +1 + expect(result1).toBe(null); + expect(cache.metrics).toMatchObject({ hits: 0, misses: 1 }); + expect(cache.size).toBe(0); + expect(cache.has(100)).toBe(false); + expect(cache.peek(100)).toBe(null); + expect(cache.getIfPresent(100)).toBe(null); // misses +1 + expect(cache.metrics).toMatchObject({ hits: 0, misses: 2 }); + }); + describe('Removal listeners', function() { it('Triggers on delete', function() { const removal = new RemovalHelper(); diff --git a/test/metrics.test.ts b/test/metrics.test.ts index 7251506..01419a1 100644 --- a/test/metrics.test.ts +++ b/test/metrics.test.ts @@ -1,6 +1,7 @@ +import { newCache as newCacheImpl } from '../src/builder'; +import { StandardMetrics } from '../src/cache'; import { BoundlessCache } from '../src/cache/boundless'; import { KeyType } from '../src/cache/KeyType'; -import { MetricsCache } from '../src/cache/metrics/index'; import { RemovalReason } from '../src/cache/RemovalReason'; import { RemovalHelper } from './removal-helper'; @@ -9,9 +10,7 @@ import { RemovalHelper } from './removal-helper'; * */ function newCache() { - return new MetricsCache({ - parent: new BoundlessCache({}) - }); + return newCacheImpl().metrics().build(); } describe('MetricsCache', function() { @@ -61,6 +60,25 @@ describe('MetricsCache', function() { expect(cache.metrics.hitRate).toEqual(1); }); + it('Records stats properly', function() { + const metrics = new StandardMetrics(); + + metrics.count(1, 1); + expect(metrics).toMatchObject({ hits: 1, misses: 1 }); + + metrics.hit(); + expect(metrics).toMatchObject({ hits: 2, misses: 1 }); + + metrics.hit(2); + expect(metrics).toMatchObject({ hits: 4, misses: 1 }); + + metrics.record(false); + expect(metrics).toMatchObject({ hits: 4, misses: 2 }); + + metrics.miss(-1); + expect(metrics).toMatchObject({ hits: 4, misses: 1 }); + }); + describe('Removal listeners', function() { it('Triggers on delete', function() { const removal = new RemovalHelper();