548 lines
No EOL
16 KiB
TypeScript
548 lines
No EOL
16 KiB
TypeScript
// =============================================================================
|
|
// CACHE MANAGEMENT FOR THREAT SCORING (TypeScript)
|
|
// =============================================================================
|
|
|
|
import { CACHE_CONFIG } from './constants.js';
|
|
import { parseDuration } from '../time.js';
|
|
|
|
// Pre-computed durations for hot path cache operations
|
|
const REQUEST_HISTORY_TTL = parseDuration('30m');
|
|
|
|
// =============================================================================
|
|
// TYPE DEFINITIONS
|
|
// =============================================================================
|
|
|
|
interface CachedEntry<T> {
|
|
readonly data: T;
|
|
readonly timestamp: number;
|
|
readonly ttl?: number;
|
|
}
|
|
|
|
interface RequestHistoryEntry {
|
|
readonly timestamp: number;
|
|
readonly method?: string;
|
|
readonly path?: string;
|
|
readonly userAgent?: string;
|
|
readonly score?: number;
|
|
readonly responseTime?: number;
|
|
readonly statusCode?: number;
|
|
}
|
|
|
|
interface CachedRequestHistory {
|
|
readonly history: readonly RequestHistoryEntry[];
|
|
readonly timestamp: number;
|
|
}
|
|
|
|
interface IPScoreEntry {
|
|
readonly score: number;
|
|
readonly confidence: number;
|
|
readonly lastCalculated: number;
|
|
readonly components: Record<string, number>;
|
|
}
|
|
|
|
interface SessionEntry {
|
|
readonly sessionId: string;
|
|
readonly startTime: number;
|
|
readonly lastActivity: number;
|
|
readonly requestCount: number;
|
|
readonly behaviorScore: number;
|
|
readonly flags: readonly string[];
|
|
}
|
|
|
|
interface BehaviorEntry {
|
|
readonly patterns: Record<string, unknown>;
|
|
readonly anomalies: readonly string[];
|
|
readonly riskScore: number;
|
|
readonly lastUpdated: number;
|
|
readonly requestPattern: Record<string, number>;
|
|
}
|
|
|
|
interface VerifiedBotEntry {
|
|
readonly botName: string;
|
|
readonly verified: boolean;
|
|
readonly verificationMethod: 'dns' | 'user_agent' | 'signature' | 'manual';
|
|
readonly lastVerified: number;
|
|
readonly trustScore: number;
|
|
}
|
|
|
|
interface CacheStats {
|
|
readonly ipScore: number;
|
|
readonly session: number;
|
|
readonly behavior: number;
|
|
readonly verifiedBots: number;
|
|
}
|
|
|
|
interface CacheCleanupResult {
|
|
readonly beforeSize: CacheStats;
|
|
readonly afterSize: CacheStats;
|
|
readonly totalCleaned: number;
|
|
readonly emergencyTriggered: boolean;
|
|
}
|
|
|
|
// Generic cache interface for type safety
|
|
interface TypedCache<T> {
|
|
get(key: string): T | undefined;
|
|
set(key: string, value: T): void;
|
|
delete(key: string): boolean;
|
|
has(key: string): boolean;
|
|
clear(): void;
|
|
readonly size: number;
|
|
[Symbol.iterator](): IterableIterator<[string, T]>;
|
|
}
|
|
|
|
// =============================================================================
|
|
// CACHE MANAGER CLASS
|
|
// =============================================================================
|
|
|
|
export class CacheManager {
|
|
// Type-safe cache instances
|
|
private readonly ipScoreCache: TypedCache<CachedEntry<IPScoreEntry>>;
|
|
private readonly sessionCache: TypedCache<CachedEntry<SessionEntry>>;
|
|
private readonly behaviorCache: TypedCache<CachedEntry<BehaviorEntry | CachedRequestHistory>>;
|
|
private readonly verifiedBotsCache: TypedCache<CachedEntry<VerifiedBotEntry>>;
|
|
|
|
// Cleanup timer reference for proper disposal
|
|
private cleanupTimer: NodeJS.Timeout | null = null;
|
|
|
|
constructor() {
|
|
// Initialize in-memory caches with size limits
|
|
this.ipScoreCache = new Map<string, CachedEntry<IPScoreEntry>>() as TypedCache<CachedEntry<IPScoreEntry>>;
|
|
this.sessionCache = new Map<string, CachedEntry<SessionEntry>>() as TypedCache<CachedEntry<SessionEntry>>;
|
|
this.behaviorCache = new Map<string, CachedEntry<BehaviorEntry | CachedRequestHistory>>() as TypedCache<CachedEntry<BehaviorEntry | CachedRequestHistory>>;
|
|
this.verifiedBotsCache = new Map<string, CachedEntry<VerifiedBotEntry>>() as TypedCache<CachedEntry<VerifiedBotEntry>>;
|
|
|
|
// Start cache cleanup timer
|
|
this.startCacheCleanup();
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// CACHE LIFECYCLE MANAGEMENT
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Starts the cache cleanup timer - CRITICAL for memory stability
|
|
* This prevents memory leaks under high load by periodically cleaning expired entries
|
|
*/
|
|
private startCacheCleanup(): void {
|
|
// CRITICAL: This timer prevents memory leaks under high load
|
|
// If this cleanup stops running, the system will eventually crash due to memory exhaustion
|
|
// The cleanup interval affects both memory usage and performance - too frequent = CPU waste,
|
|
// too infrequent = memory problems
|
|
this.cleanupTimer = setInterval(() => {
|
|
this.cleanupCaches();
|
|
}, CACHE_CONFIG.CACHE_CLEANUP_INTERVAL);
|
|
|
|
// Ensure cleanup timer doesn't keep process alive
|
|
if (this.cleanupTimer.unref) {
|
|
this.cleanupTimer.unref();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stops the cache cleanup timer and clears all caches
|
|
* Should be called during application shutdown
|
|
*/
|
|
public destroy(): void {
|
|
if (this.cleanupTimer) {
|
|
clearInterval(this.cleanupTimer);
|
|
this.cleanupTimer = null;
|
|
}
|
|
|
|
// Clear all caches
|
|
this.ipScoreCache.clear();
|
|
this.sessionCache.clear();
|
|
this.behaviorCache.clear();
|
|
this.verifiedBotsCache.clear();
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// CACHE CLEANUP OPERATIONS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Performs comprehensive cache cleanup to prevent memory exhaustion
|
|
* @returns Cleanup statistics
|
|
*/
|
|
public cleanupCaches(): CacheCleanupResult {
|
|
const beforeSize: CacheStats = {
|
|
ipScore: this.ipScoreCache.size,
|
|
session: this.sessionCache.size,
|
|
behavior: this.behaviorCache.size,
|
|
verifiedBots: this.verifiedBotsCache.size
|
|
};
|
|
|
|
// Clean each cache using the optimized cleanup method
|
|
this.cleanupCache(this.ipScoreCache);
|
|
this.cleanupCache(this.sessionCache);
|
|
this.cleanupCache(this.behaviorCache);
|
|
this.cleanupCache(this.verifiedBotsCache);
|
|
|
|
const afterSize: CacheStats = {
|
|
ipScore: this.ipScoreCache.size,
|
|
session: this.sessionCache.size,
|
|
behavior: this.behaviorCache.size,
|
|
verifiedBots: this.verifiedBotsCache.size
|
|
};
|
|
|
|
const totalCleaned = Object.keys(beforeSize).reduce((total, key) => {
|
|
const beforeCount = beforeSize[key as keyof CacheStats];
|
|
const afterCount = afterSize[key as keyof CacheStats];
|
|
return total + (beforeCount - afterCount);
|
|
}, 0);
|
|
|
|
let emergencyTriggered = false;
|
|
|
|
if (totalCleaned > 0) {
|
|
console.log(`Threat scorer: cleaned ${totalCleaned} expired cache entries`);
|
|
}
|
|
|
|
// Emergency cleanup if caches are still too large
|
|
// This prevents memory exhaustion under extreme load
|
|
if (this.ipScoreCache.size > CACHE_CONFIG.MAX_CACHE_SIZE * CACHE_CONFIG.EMERGENCY_CLEANUP_THRESHOLD) {
|
|
console.warn('Threat scorer: Emergency cleanup triggered - system under high load');
|
|
this.emergencyCleanup();
|
|
emergencyTriggered = true;
|
|
}
|
|
|
|
return {
|
|
beforeSize,
|
|
afterSize,
|
|
totalCleaned,
|
|
emergencyTriggered
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Optimized cache cleanup - removes oldest entries when cache exceeds size limit
|
|
* Maps maintain insertion order, so we can efficiently remove oldest entries
|
|
*/
|
|
private cleanupCache<T>(cache: TypedCache<T>): number {
|
|
if (cache.size <= CACHE_CONFIG.MAX_CACHE_SIZE) {
|
|
return 0;
|
|
}
|
|
|
|
const excess = cache.size - CACHE_CONFIG.MAX_CACHE_SIZE;
|
|
let removed = 0;
|
|
|
|
// Remove oldest entries (Maps maintain insertion order)
|
|
const cacheAsMap = cache as unknown as Map<string, T>;
|
|
for (const [key] of Array.from(cacheAsMap.entries())) {
|
|
if (removed >= excess) {
|
|
break;
|
|
}
|
|
cache.delete(key);
|
|
removed++;
|
|
}
|
|
|
|
return removed;
|
|
}
|
|
|
|
/**
|
|
* Emergency cleanup for extreme memory pressure
|
|
* Aggressively reduces cache sizes to prevent system crashes
|
|
*/
|
|
private emergencyCleanup(): void {
|
|
// Aggressively reduce cache sizes to 25% of max
|
|
const targetSize = Math.floor(CACHE_CONFIG.MAX_CACHE_SIZE * CACHE_CONFIG.EMERGENCY_CLEANUP_TARGET);
|
|
|
|
// Clean each cache individually to avoid type issues
|
|
this.emergencyCleanupCache(this.ipScoreCache, targetSize);
|
|
this.emergencyCleanupCache(this.sessionCache, targetSize);
|
|
this.emergencyCleanupCache(this.behaviorCache, targetSize);
|
|
this.emergencyCleanupCache(this.verifiedBotsCache, targetSize);
|
|
}
|
|
|
|
/**
|
|
* Helper method for emergency cleanup of individual cache
|
|
*/
|
|
private emergencyCleanupCache<T>(cache: TypedCache<T>, targetSize: number): void {
|
|
if (cache.size <= targetSize) {
|
|
return;
|
|
}
|
|
|
|
const toRemove = cache.size - targetSize;
|
|
let removed = 0;
|
|
|
|
// Clear the cache if we need to remove too many entries (emergency scenario)
|
|
if (toRemove > cache.size * 0.8) {
|
|
cache.clear();
|
|
return;
|
|
}
|
|
|
|
// Otherwise, remove oldest entries using the Map's iteration order
|
|
const cacheAsMap = cache as unknown as Map<string, T>;
|
|
const keysToDelete: string[] = [];
|
|
|
|
for (const [key] of Array.from(cacheAsMap.entries())) {
|
|
if (keysToDelete.length >= toRemove) {
|
|
break;
|
|
}
|
|
keysToDelete.push(key);
|
|
}
|
|
|
|
for (const key of keysToDelete) {
|
|
cache.delete(key);
|
|
removed++;
|
|
}
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// IP SCORE CACHE OPERATIONS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Retrieves cached IP score if still valid
|
|
*/
|
|
public getCachedIPScore(ip: string): IPScoreEntry | null {
|
|
if (!ip || typeof ip !== 'string') {
|
|
return null;
|
|
}
|
|
|
|
const cached = this.ipScoreCache.get(ip);
|
|
if (cached && this.isEntryValid(cached)) {
|
|
return cached.data;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Caches IP score with optional TTL
|
|
*/
|
|
public setCachedIPScore(ip: string, scoreData: IPScoreEntry, ttlMs?: number): void {
|
|
if (!ip || typeof ip !== 'string' || !scoreData) {
|
|
return;
|
|
}
|
|
|
|
const entry: CachedEntry<IPScoreEntry> = {
|
|
data: scoreData,
|
|
timestamp: Date.now(),
|
|
ttl: ttlMs
|
|
};
|
|
|
|
this.ipScoreCache.set(ip, entry);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// SESSION CACHE OPERATIONS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Retrieves cached session data if still valid
|
|
*/
|
|
public getCachedSession(sessionId: string): SessionEntry | null {
|
|
if (!sessionId || typeof sessionId !== 'string') {
|
|
return null;
|
|
}
|
|
|
|
const cached = this.sessionCache.get(sessionId);
|
|
if (cached && this.isEntryValid(cached)) {
|
|
return cached.data;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Caches session data with optional TTL
|
|
*/
|
|
public setCachedSession(sessionId: string, sessionData: SessionEntry, ttlMs?: number): void {
|
|
if (!sessionId || typeof sessionId !== 'string' || !sessionData) {
|
|
return;
|
|
}
|
|
|
|
const entry: CachedEntry<SessionEntry> = {
|
|
data: sessionData,
|
|
timestamp: Date.now(),
|
|
ttl: ttlMs
|
|
};
|
|
|
|
this.sessionCache.set(sessionId, entry);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// BEHAVIOR CACHE OPERATIONS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Retrieves cached behavior data if still valid
|
|
*/
|
|
public getCachedBehavior(key: string): BehaviorEntry | null {
|
|
if (!key || typeof key !== 'string') {
|
|
return null;
|
|
}
|
|
|
|
const cached = this.behaviorCache.get(key);
|
|
if (cached && this.isEntryValid(cached) && this.isBehaviorEntry(cached.data)) {
|
|
return cached.data;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Caches behavior data with optional TTL
|
|
*/
|
|
public setCachedBehavior(key: string, behaviorData: BehaviorEntry, ttlMs?: number): void {
|
|
if (!key || typeof key !== 'string' || !behaviorData) {
|
|
return;
|
|
}
|
|
|
|
const entry: CachedEntry<BehaviorEntry> = {
|
|
data: behaviorData,
|
|
timestamp: Date.now(),
|
|
ttl: ttlMs
|
|
};
|
|
|
|
this.behaviorCache.set(key, entry);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// REQUEST HISTORY CACHE OPERATIONS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Retrieves cached request history if still valid
|
|
*/
|
|
public getCachedRequestHistory(ip: string, cutoff: number): readonly RequestHistoryEntry[] | null {
|
|
if (!ip || typeof ip !== 'string' || typeof cutoff !== 'number') {
|
|
return null;
|
|
}
|
|
|
|
const cacheKey = `history:${ip}`;
|
|
const cached = this.behaviorCache.get(cacheKey);
|
|
|
|
if (cached && cached.timestamp > cutoff && this.isRequestHistoryEntry(cached.data)) {
|
|
return cached.data.history.filter(h => h.timestamp > cutoff);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Caches request history with automatic TTL
|
|
*/
|
|
public setCachedRequestHistory(ip: string, history: readonly RequestHistoryEntry[]): void {
|
|
if (!ip || typeof ip !== 'string' || !Array.isArray(history)) {
|
|
return;
|
|
}
|
|
|
|
const cacheKey = `history:${ip}`;
|
|
const cachedHistory: CachedRequestHistory = {
|
|
history,
|
|
timestamp: Date.now()
|
|
};
|
|
|
|
const entry: CachedEntry<CachedRequestHistory> = {
|
|
data: cachedHistory,
|
|
timestamp: Date.now(),
|
|
ttl: REQUEST_HISTORY_TTL // 30 minutes TTL for request history
|
|
};
|
|
|
|
this.behaviorCache.set(cacheKey, entry);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// VERIFIED BOTS CACHE OPERATIONS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Retrieves cached bot verification if still valid
|
|
*/
|
|
public getCachedBotVerification(userAgent: string): VerifiedBotEntry | null {
|
|
if (!userAgent || typeof userAgent !== 'string') {
|
|
return null;
|
|
}
|
|
|
|
const cached = this.verifiedBotsCache.get(userAgent);
|
|
if (cached && this.isEntryValid(cached)) {
|
|
return cached.data;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/**
|
|
* Caches bot verification with TTL from configuration
|
|
*/
|
|
public setCachedBotVerification(userAgent: string, botData: VerifiedBotEntry, ttlMs: number): void {
|
|
if (!userAgent || typeof userAgent !== 'string' || !botData || typeof ttlMs !== 'number') {
|
|
return;
|
|
}
|
|
|
|
const entry: CachedEntry<VerifiedBotEntry> = {
|
|
data: botData,
|
|
timestamp: Date.now(),
|
|
ttl: ttlMs
|
|
};
|
|
|
|
this.verifiedBotsCache.set(userAgent, entry);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// CACHE STATISTICS AND MONITORING
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Gets current cache statistics for monitoring
|
|
*/
|
|
public getCacheStats(): CacheStats & { totalEntries: number; memoryPressure: boolean } {
|
|
const stats: CacheStats = {
|
|
ipScore: this.ipScoreCache.size,
|
|
session: this.sessionCache.size,
|
|
behavior: this.behaviorCache.size,
|
|
verifiedBots: this.verifiedBotsCache.size
|
|
};
|
|
|
|
const totalEntries = Object.values(stats).reduce((sum, count) => sum + count, 0);
|
|
const memoryPressure = totalEntries > (CACHE_CONFIG.MAX_CACHE_SIZE * 4 * CACHE_CONFIG.EMERGENCY_CLEANUP_THRESHOLD);
|
|
|
|
return {
|
|
...stats,
|
|
totalEntries,
|
|
memoryPressure
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Clears all caches - use with caution
|
|
*/
|
|
public clearAllCaches(): void {
|
|
this.ipScoreCache.clear();
|
|
this.sessionCache.clear();
|
|
this.behaviorCache.clear();
|
|
this.verifiedBotsCache.clear();
|
|
|
|
console.log('Threat scorer: All caches cleared');
|
|
}
|
|
|
|
// -----------------------------------------------------------------------------
|
|
// UTILITY METHODS
|
|
// -----------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Checks if a cached entry is still valid based on TTL
|
|
*/
|
|
private isEntryValid<T>(entry: CachedEntry<T>): boolean {
|
|
if (!entry.ttl) {
|
|
return true; // No TTL means it doesn't expire
|
|
}
|
|
|
|
const now = Date.now();
|
|
return (now - entry.timestamp) < entry.ttl;
|
|
}
|
|
|
|
/**
|
|
* Type guard to check if cached data is BehaviorEntry
|
|
*/
|
|
private isBehaviorEntry(data: BehaviorEntry | CachedRequestHistory): data is BehaviorEntry {
|
|
return 'patterns' in data && 'anomalies' in data && 'riskScore' in data;
|
|
}
|
|
|
|
/**
|
|
* Type guard to check if cached data is CachedRequestHistory
|
|
*/
|
|
private isRequestHistoryEntry(data: BehaviorEntry | CachedRequestHistory): data is CachedRequestHistory {
|
|
return 'history' in data && Array.isArray((data as CachedRequestHistory).history);
|
|
}
|
|
}
|