Initial commit of massive v2 rewrite
This commit is contained in:
parent
1025f3b523
commit
dc120fe78a
55 changed files with 21733 additions and 0 deletions
548
src/utils/threat-scoring/cache-manager.ts
Normal file
548
src/utils/threat-scoring/cache-manager.ts
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
// =============================================================================
|
||||
// CACHE MANAGEMENT FOR THREAT SCORING (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
import { CACHE_CONFIG } from './constants.js';
|
||||
import { parseDuration } from '../time.js';
|
||||
|
||||
// Pre-computed durations for hot path cache operations
|
||||
const REQUEST_HISTORY_TTL = parseDuration('30m');
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface CachedEntry<T> {
|
||||
readonly data: T;
|
||||
readonly timestamp: number;
|
||||
readonly ttl?: number;
|
||||
}
|
||||
|
||||
interface RequestHistoryEntry {
|
||||
readonly timestamp: number;
|
||||
readonly method?: string;
|
||||
readonly path?: string;
|
||||
readonly userAgent?: string;
|
||||
readonly score?: number;
|
||||
readonly responseTime?: number;
|
||||
readonly statusCode?: number;
|
||||
}
|
||||
|
||||
interface CachedRequestHistory {
|
||||
readonly history: readonly RequestHistoryEntry[];
|
||||
readonly timestamp: number;
|
||||
}
|
||||
|
||||
interface IPScoreEntry {
|
||||
readonly score: number;
|
||||
readonly confidence: number;
|
||||
readonly lastCalculated: number;
|
||||
readonly components: Record<string, number>;
|
||||
}
|
||||
|
||||
interface SessionEntry {
|
||||
readonly sessionId: string;
|
||||
readonly startTime: number;
|
||||
readonly lastActivity: number;
|
||||
readonly requestCount: number;
|
||||
readonly behaviorScore: number;
|
||||
readonly flags: readonly string[];
|
||||
}
|
||||
|
||||
interface BehaviorEntry {
|
||||
readonly patterns: Record<string, unknown>;
|
||||
readonly anomalies: readonly string[];
|
||||
readonly riskScore: number;
|
||||
readonly lastUpdated: number;
|
||||
readonly requestPattern: Record<string, number>;
|
||||
}
|
||||
|
||||
interface VerifiedBotEntry {
|
||||
readonly botName: string;
|
||||
readonly verified: boolean;
|
||||
readonly verificationMethod: 'dns' | 'user_agent' | 'signature' | 'manual';
|
||||
readonly lastVerified: number;
|
||||
readonly trustScore: number;
|
||||
}
|
||||
|
||||
interface CacheStats {
|
||||
readonly ipScore: number;
|
||||
readonly session: number;
|
||||
readonly behavior: number;
|
||||
readonly verifiedBots: number;
|
||||
}
|
||||
|
||||
interface CacheCleanupResult {
|
||||
readonly beforeSize: CacheStats;
|
||||
readonly afterSize: CacheStats;
|
||||
readonly totalCleaned: number;
|
||||
readonly emergencyTriggered: boolean;
|
||||
}
|
||||
|
||||
// Generic cache interface for type safety
|
||||
interface TypedCache<T> {
|
||||
get(key: string): T | undefined;
|
||||
set(key: string, value: T): void;
|
||||
delete(key: string): boolean;
|
||||
has(key: string): boolean;
|
||||
clear(): void;
|
||||
readonly size: number;
|
||||
[Symbol.iterator](): IterableIterator<[string, T]>;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CACHE MANAGER CLASS
|
||||
// =============================================================================
|
||||
|
||||
export class CacheManager {
|
||||
// Type-safe cache instances
|
||||
private readonly ipScoreCache: TypedCache<CachedEntry<IPScoreEntry>>;
|
||||
private readonly sessionCache: TypedCache<CachedEntry<SessionEntry>>;
|
||||
private readonly behaviorCache: TypedCache<CachedEntry<BehaviorEntry | CachedRequestHistory>>;
|
||||
private readonly verifiedBotsCache: TypedCache<CachedEntry<VerifiedBotEntry>>;
|
||||
|
||||
// Cleanup timer reference for proper disposal
|
||||
private cleanupTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor() {
|
||||
// Initialize in-memory caches with size limits
|
||||
this.ipScoreCache = new Map<string, CachedEntry<IPScoreEntry>>() as TypedCache<CachedEntry<IPScoreEntry>>;
|
||||
this.sessionCache = new Map<string, CachedEntry<SessionEntry>>() as TypedCache<CachedEntry<SessionEntry>>;
|
||||
this.behaviorCache = new Map<string, CachedEntry<BehaviorEntry | CachedRequestHistory>>() as TypedCache<CachedEntry<BehaviorEntry | CachedRequestHistory>>;
|
||||
this.verifiedBotsCache = new Map<string, CachedEntry<VerifiedBotEntry>>() as TypedCache<CachedEntry<VerifiedBotEntry>>;
|
||||
|
||||
// Start cache cleanup timer
|
||||
this.startCacheCleanup();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// CACHE LIFECYCLE MANAGEMENT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Starts the cache cleanup timer - CRITICAL for memory stability
|
||||
* This prevents memory leaks under high load by periodically cleaning expired entries
|
||||
*/
|
||||
private startCacheCleanup(): void {
|
||||
// CRITICAL: This timer prevents memory leaks under high load
|
||||
// If this cleanup stops running, the system will eventually crash due to memory exhaustion
|
||||
// The cleanup interval affects both memory usage and performance - too frequent = CPU waste,
|
||||
// too infrequent = memory problems
|
||||
this.cleanupTimer = setInterval(() => {
|
||||
this.cleanupCaches();
|
||||
}, CACHE_CONFIG.CACHE_CLEANUP_INTERVAL);
|
||||
|
||||
// Ensure cleanup timer doesn't keep process alive
|
||||
if (this.cleanupTimer.unref) {
|
||||
this.cleanupTimer.unref();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the cache cleanup timer and clears all caches
|
||||
* Should be called during application shutdown
|
||||
*/
|
||||
public destroy(): void {
|
||||
if (this.cleanupTimer) {
|
||||
clearInterval(this.cleanupTimer);
|
||||
this.cleanupTimer = null;
|
||||
}
|
||||
|
||||
// Clear all caches
|
||||
this.ipScoreCache.clear();
|
||||
this.sessionCache.clear();
|
||||
this.behaviorCache.clear();
|
||||
this.verifiedBotsCache.clear();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// CACHE CLEANUP OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Performs comprehensive cache cleanup to prevent memory exhaustion
|
||||
* @returns Cleanup statistics
|
||||
*/
|
||||
public cleanupCaches(): CacheCleanupResult {
|
||||
const beforeSize: CacheStats = {
|
||||
ipScore: this.ipScoreCache.size,
|
||||
session: this.sessionCache.size,
|
||||
behavior: this.behaviorCache.size,
|
||||
verifiedBots: this.verifiedBotsCache.size
|
||||
};
|
||||
|
||||
// Clean each cache using the optimized cleanup method
|
||||
this.cleanupCache(this.ipScoreCache);
|
||||
this.cleanupCache(this.sessionCache);
|
||||
this.cleanupCache(this.behaviorCache);
|
||||
this.cleanupCache(this.verifiedBotsCache);
|
||||
|
||||
const afterSize: CacheStats = {
|
||||
ipScore: this.ipScoreCache.size,
|
||||
session: this.sessionCache.size,
|
||||
behavior: this.behaviorCache.size,
|
||||
verifiedBots: this.verifiedBotsCache.size
|
||||
};
|
||||
|
||||
const totalCleaned = Object.keys(beforeSize).reduce((total, key) => {
|
||||
const beforeCount = beforeSize[key as keyof CacheStats];
|
||||
const afterCount = afterSize[key as keyof CacheStats];
|
||||
return total + (beforeCount - afterCount);
|
||||
}, 0);
|
||||
|
||||
let emergencyTriggered = false;
|
||||
|
||||
if (totalCleaned > 0) {
|
||||
console.log(`Threat scorer: cleaned ${totalCleaned} expired cache entries`);
|
||||
}
|
||||
|
||||
// Emergency cleanup if caches are still too large
|
||||
// This prevents memory exhaustion under extreme load
|
||||
if (this.ipScoreCache.size > CACHE_CONFIG.MAX_CACHE_SIZE * CACHE_CONFIG.EMERGENCY_CLEANUP_THRESHOLD) {
|
||||
console.warn('Threat scorer: Emergency cleanup triggered - system under high load');
|
||||
this.emergencyCleanup();
|
||||
emergencyTriggered = true;
|
||||
}
|
||||
|
||||
return {
|
||||
beforeSize,
|
||||
afterSize,
|
||||
totalCleaned,
|
||||
emergencyTriggered
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized cache cleanup - removes oldest entries when cache exceeds size limit
|
||||
* Maps maintain insertion order, so we can efficiently remove oldest entries
|
||||
*/
|
||||
private cleanupCache<T>(cache: TypedCache<T>): number {
|
||||
if (cache.size <= CACHE_CONFIG.MAX_CACHE_SIZE) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const excess = cache.size - CACHE_CONFIG.MAX_CACHE_SIZE;
|
||||
let removed = 0;
|
||||
|
||||
// Remove oldest entries (Maps maintain insertion order)
|
||||
const cacheAsMap = cache as unknown as Map<string, T>;
|
||||
for (const [key] of Array.from(cacheAsMap.entries())) {
|
||||
if (removed >= excess) {
|
||||
break;
|
||||
}
|
||||
cache.delete(key);
|
||||
removed++;
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Emergency cleanup for extreme memory pressure
|
||||
* Aggressively reduces cache sizes to prevent system crashes
|
||||
*/
|
||||
private emergencyCleanup(): void {
|
||||
// Aggressively reduce cache sizes to 25% of max
|
||||
const targetSize = Math.floor(CACHE_CONFIG.MAX_CACHE_SIZE * CACHE_CONFIG.EMERGENCY_CLEANUP_TARGET);
|
||||
|
||||
// Clean each cache individually to avoid type issues
|
||||
this.emergencyCleanupCache(this.ipScoreCache, targetSize);
|
||||
this.emergencyCleanupCache(this.sessionCache, targetSize);
|
||||
this.emergencyCleanupCache(this.behaviorCache, targetSize);
|
||||
this.emergencyCleanupCache(this.verifiedBotsCache, targetSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method for emergency cleanup of individual cache
|
||||
*/
|
||||
private emergencyCleanupCache<T>(cache: TypedCache<T>, targetSize: number): void {
|
||||
if (cache.size <= targetSize) {
|
||||
return;
|
||||
}
|
||||
|
||||
const toRemove = cache.size - targetSize;
|
||||
let removed = 0;
|
||||
|
||||
// Clear the cache if we need to remove too many entries (emergency scenario)
|
||||
if (toRemove > cache.size * 0.8) {
|
||||
cache.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, remove oldest entries using the Map's iteration order
|
||||
const cacheAsMap = cache as unknown as Map<string, T>;
|
||||
const keysToDelete: string[] = [];
|
||||
|
||||
for (const [key] of Array.from(cacheAsMap.entries())) {
|
||||
if (keysToDelete.length >= toRemove) {
|
||||
break;
|
||||
}
|
||||
keysToDelete.push(key);
|
||||
}
|
||||
|
||||
for (const key of keysToDelete) {
|
||||
cache.delete(key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// IP SCORE CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached IP score if still valid
|
||||
*/
|
||||
public getCachedIPScore(ip: string): IPScoreEntry | null {
|
||||
if (!ip || typeof ip !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.ipScoreCache.get(ip);
|
||||
if (cached && this.isEntryValid(cached)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches IP score with optional TTL
|
||||
*/
|
||||
public setCachedIPScore(ip: string, scoreData: IPScoreEntry, ttlMs?: number): void {
|
||||
if (!ip || typeof ip !== 'string' || !scoreData) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<IPScoreEntry> = {
|
||||
data: scoreData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.ipScoreCache.set(ip, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// SESSION CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached session data if still valid
|
||||
*/
|
||||
public getCachedSession(sessionId: string): SessionEntry | null {
|
||||
if (!sessionId || typeof sessionId !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.sessionCache.get(sessionId);
|
||||
if (cached && this.isEntryValid(cached)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches session data with optional TTL
|
||||
*/
|
||||
public setCachedSession(sessionId: string, sessionData: SessionEntry, ttlMs?: number): void {
|
||||
if (!sessionId || typeof sessionId !== 'string' || !sessionData) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<SessionEntry> = {
|
||||
data: sessionData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.sessionCache.set(sessionId, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// BEHAVIOR CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached behavior data if still valid
|
||||
*/
|
||||
public getCachedBehavior(key: string): BehaviorEntry | null {
|
||||
if (!key || typeof key !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.behaviorCache.get(key);
|
||||
if (cached && this.isEntryValid(cached) && this.isBehaviorEntry(cached.data)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches behavior data with optional TTL
|
||||
*/
|
||||
public setCachedBehavior(key: string, behaviorData: BehaviorEntry, ttlMs?: number): void {
|
||||
if (!key || typeof key !== 'string' || !behaviorData) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<BehaviorEntry> = {
|
||||
data: behaviorData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.behaviorCache.set(key, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// REQUEST HISTORY CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached request history if still valid
|
||||
*/
|
||||
public getCachedRequestHistory(ip: string, cutoff: number): readonly RequestHistoryEntry[] | null {
|
||||
if (!ip || typeof ip !== 'string' || typeof cutoff !== 'number') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cacheKey = `history:${ip}`;
|
||||
const cached = this.behaviorCache.get(cacheKey);
|
||||
|
||||
if (cached && cached.timestamp > cutoff && this.isRequestHistoryEntry(cached.data)) {
|
||||
return cached.data.history.filter(h => h.timestamp > cutoff);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches request history with automatic TTL
|
||||
*/
|
||||
public setCachedRequestHistory(ip: string, history: readonly RequestHistoryEntry[]): void {
|
||||
if (!ip || typeof ip !== 'string' || !Array.isArray(history)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cacheKey = `history:${ip}`;
|
||||
const cachedHistory: CachedRequestHistory = {
|
||||
history,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
|
||||
const entry: CachedEntry<CachedRequestHistory> = {
|
||||
data: cachedHistory,
|
||||
timestamp: Date.now(),
|
||||
ttl: REQUEST_HISTORY_TTL // 30 minutes TTL for request history
|
||||
};
|
||||
|
||||
this.behaviorCache.set(cacheKey, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// VERIFIED BOTS CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached bot verification if still valid
|
||||
*/
|
||||
public getCachedBotVerification(userAgent: string): VerifiedBotEntry | null {
|
||||
if (!userAgent || typeof userAgent !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.verifiedBotsCache.get(userAgent);
|
||||
if (cached && this.isEntryValid(cached)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches bot verification with TTL from configuration
|
||||
*/
|
||||
public setCachedBotVerification(userAgent: string, botData: VerifiedBotEntry, ttlMs: number): void {
|
||||
if (!userAgent || typeof userAgent !== 'string' || !botData || typeof ttlMs !== 'number') {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<VerifiedBotEntry> = {
|
||||
data: botData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.verifiedBotsCache.set(userAgent, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// CACHE STATISTICS AND MONITORING
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Gets current cache statistics for monitoring
|
||||
*/
|
||||
public getCacheStats(): CacheStats & { totalEntries: number; memoryPressure: boolean } {
|
||||
const stats: CacheStats = {
|
||||
ipScore: this.ipScoreCache.size,
|
||||
session: this.sessionCache.size,
|
||||
behavior: this.behaviorCache.size,
|
||||
verifiedBots: this.verifiedBotsCache.size
|
||||
};
|
||||
|
||||
const totalEntries = Object.values(stats).reduce((sum, count) => sum + count, 0);
|
||||
const memoryPressure = totalEntries > (CACHE_CONFIG.MAX_CACHE_SIZE * 4 * CACHE_CONFIG.EMERGENCY_CLEANUP_THRESHOLD);
|
||||
|
||||
return {
|
||||
...stats,
|
||||
totalEntries,
|
||||
memoryPressure
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all caches - use with caution
|
||||
*/
|
||||
public clearAllCaches(): void {
|
||||
this.ipScoreCache.clear();
|
||||
this.sessionCache.clear();
|
||||
this.behaviorCache.clear();
|
||||
this.verifiedBotsCache.clear();
|
||||
|
||||
console.log('Threat scorer: All caches cleared');
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// UTILITY METHODS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Checks if a cached entry is still valid based on TTL
|
||||
*/
|
||||
private isEntryValid<T>(entry: CachedEntry<T>): boolean {
|
||||
if (!entry.ttl) {
|
||||
return true; // No TTL means it doesn't expire
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
return (now - entry.timestamp) < entry.ttl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if cached data is BehaviorEntry
|
||||
*/
|
||||
private isBehaviorEntry(data: BehaviorEntry | CachedRequestHistory): data is BehaviorEntry {
|
||||
return 'patterns' in data && 'anomalies' in data && 'riskScore' in data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if cached data is CachedRequestHistory
|
||||
*/
|
||||
private isRequestHistoryEntry(data: BehaviorEntry | CachedRequestHistory): data is CachedRequestHistory {
|
||||
return 'history' in data && Array.isArray((data as CachedRequestHistory).history);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue