Massive v2 rewrite
This commit is contained in:
parent
1025f3b523
commit
5f1328f626
77 changed files with 28105 additions and 3542 deletions
437
src/utils/threat-scoring/index.ts
Normal file
437
src/utils/threat-scoring/index.ts
Normal file
|
|
@ -0,0 +1,437 @@
|
|||
// =============================================================================
|
||||
// THREAT SCORING ENGINE (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
import { STATIC_WHITELIST, type ThreatThresholds, type SignalWeights } from './constants.js';
|
||||
import { type IncomingHttpHeaders } from 'http';
|
||||
import type { NetworkRequest } from '../network.js';
|
||||
import * as logs from '../logs.js';
|
||||
import { performance } from 'perf_hooks';
|
||||
|
||||
// Simple utility functions
|
||||
function performSecurityChecks(ip: string): string {
|
||||
if (typeof ip !== 'string' || ip.length === 0 || ip.length > 45) {
|
||||
throw new Error('Invalid IP address');
|
||||
}
|
||||
return ip.trim();
|
||||
}
|
||||
|
||||
function normalizeMetricValue(value: number, min: number, max: number): number {
|
||||
if (typeof value !== 'number' || isNaN(value)) return 0;
|
||||
if (max <= min) return value >= max ? 1 : 0;
|
||||
const clampedValue = Math.max(min, Math.min(max, value));
|
||||
return (clampedValue - min) / (max - min);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
export interface ThreatScore {
|
||||
readonly totalScore: number;
|
||||
readonly confidence: number;
|
||||
readonly riskLevel: 'allow' | 'challenge' | 'block';
|
||||
readonly components: {
|
||||
readonly behaviorScore: number;
|
||||
readonly contentScore: number;
|
||||
readonly networkScore: number;
|
||||
readonly anomalyScore: number;
|
||||
};
|
||||
readonly signalsTriggered: readonly string[];
|
||||
readonly normalizedFeatures: Record<string, number>;
|
||||
readonly processingTimeMs: number;
|
||||
}
|
||||
|
||||
export interface ThreatScoringConfig {
|
||||
readonly enabled: boolean;
|
||||
readonly thresholds: ThreatThresholds;
|
||||
readonly signalWeights: SignalWeights;
|
||||
readonly enableBotVerification?: boolean;
|
||||
readonly enableGeoAnalysis?: boolean;
|
||||
readonly enableBehaviorAnalysis?: boolean;
|
||||
readonly enableContentAnalysis?: boolean;
|
||||
readonly logDetailedScores?: boolean;
|
||||
}
|
||||
|
||||
interface RequestMetadata {
|
||||
readonly startTime: number;
|
||||
readonly ip: string;
|
||||
readonly userAgent?: string;
|
||||
readonly method: string;
|
||||
readonly path: string;
|
||||
readonly headers: IncomingHttpHeaders;
|
||||
readonly body?: string;
|
||||
readonly sessionId?: string;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// THREAT SCORING ENGINE
|
||||
// =============================================================================
|
||||
|
||||
export class ThreatScorer {
|
||||
private readonly config: ThreatScoringConfig;
|
||||
|
||||
constructor(config: ThreatScoringConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs comprehensive threat scoring on a request
|
||||
*/
|
||||
public async scoreRequest(request: NetworkRequest): Promise<ThreatScore> {
|
||||
const startTime = performance.now();
|
||||
|
||||
try {
|
||||
// Check if scoring is enabled
|
||||
if (!this.config.enabled) {
|
||||
return this.createAllowScore(startTime);
|
||||
}
|
||||
|
||||
// Extract request metadata
|
||||
const metadata = this.extractRequestMetadata(request, startTime);
|
||||
|
||||
// Validate input and perform security checks
|
||||
performSecurityChecks(metadata.ip);
|
||||
|
||||
// Check static whitelist (quick path for assets)
|
||||
if (this.isWhitelisted(metadata.path)) {
|
||||
return this.createAllowScore(startTime);
|
||||
}
|
||||
|
||||
// Perform threat analysis
|
||||
const score = this.performBasicThreatAnalysis(metadata);
|
||||
|
||||
return score;
|
||||
|
||||
} catch (error) {
|
||||
logs.error('threat-scorer', `Error scoring request: ${error}`);
|
||||
return this.createErrorScore(startTime);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract basic metadata from request
|
||||
*/
|
||||
private extractRequestMetadata(request: NetworkRequest, startTime: number): RequestMetadata {
|
||||
const headers = request.headers || {};
|
||||
const userAgent = this.extractUserAgent(headers);
|
||||
const ip = this.extractClientIP(request);
|
||||
|
||||
return {
|
||||
startTime,
|
||||
ip,
|
||||
userAgent,
|
||||
method: (request as any).method || 'GET',
|
||||
path: this.extractPath(request),
|
||||
headers: headers as IncomingHttpHeaders,
|
||||
body: (request as any).body,
|
||||
sessionId: this.extractSessionId(headers)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract user agent from headers
|
||||
*/
|
||||
private extractUserAgent(headers: any): string {
|
||||
if (headers && typeof headers.get === 'function') {
|
||||
return headers.get('user-agent') || '';
|
||||
}
|
||||
if (headers && typeof headers === 'object') {
|
||||
return headers['user-agent'] || '';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract client IP from request
|
||||
*/
|
||||
private extractClientIP(request: NetworkRequest): string {
|
||||
// Try common IP extraction methods
|
||||
const headers = request.headers;
|
||||
if (headers) {
|
||||
if (typeof headers.get === 'function') {
|
||||
return headers.get('x-forwarded-for') ||
|
||||
headers.get('x-real-ip') ||
|
||||
headers.get('cf-connecting-ip') || '127.0.0.1';
|
||||
}
|
||||
if (typeof headers === 'object') {
|
||||
const h = headers as any;
|
||||
return h['x-forwarded-for'] || h['x-real-ip'] || h['cf-connecting-ip'] || '127.0.0.1';
|
||||
}
|
||||
}
|
||||
return '127.0.0.1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract path from request
|
||||
*/
|
||||
private extractPath(request: NetworkRequest): string {
|
||||
if ((request as any).url) {
|
||||
try {
|
||||
const url = new URL((request as any).url, 'http://localhost');
|
||||
return url.pathname;
|
||||
} catch {
|
||||
return (request as any).url || '/';
|
||||
}
|
||||
}
|
||||
return '/';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract session ID from headers
|
||||
*/
|
||||
private extractSessionId(headers: any): string | undefined {
|
||||
// Basic session ID extraction from cookies
|
||||
if (headers && headers.cookie) {
|
||||
const cookies = headers.cookie.split(';');
|
||||
for (const cookie of cookies) {
|
||||
const [name, value] = cookie.trim().split('=');
|
||||
if (name && name.toLowerCase().includes('session')) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if path is in static whitelist
|
||||
*/
|
||||
private isWhitelisted(path: string): boolean {
|
||||
// Check static file extensions
|
||||
for (const ext of STATIC_WHITELIST.extensions) {
|
||||
if (path.endsWith(ext)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check whitelisted paths
|
||||
for (const whitelistPath of STATIC_WHITELIST.paths) {
|
||||
if (path.startsWith(whitelistPath)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check patterns
|
||||
for (const pattern of STATIC_WHITELIST.patterns) {
|
||||
if (pattern.test(path)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform basic threat analysis (simplified version)
|
||||
*/
|
||||
private performBasicThreatAnalysis(metadata: RequestMetadata): ThreatScore {
|
||||
const startTime = performance.now();
|
||||
const signalsTriggered: string[] = [];
|
||||
let totalScore = 0;
|
||||
|
||||
const components = {
|
||||
networkScore: 0,
|
||||
behaviorScore: 0,
|
||||
contentScore: 0,
|
||||
anomalyScore: 0
|
||||
};
|
||||
|
||||
// Basic checks
|
||||
if (!metadata.userAgent || metadata.userAgent.length === 0) {
|
||||
components.anomalyScore += this.config.signalWeights.MISSING_UA?.weight || 10;
|
||||
signalsTriggered.push('MISSING_UA');
|
||||
}
|
||||
|
||||
// WAF signal integration - use WAF results if available
|
||||
const wafSignals = this.extractWAFSignals(metadata);
|
||||
if (wafSignals) {
|
||||
const wafScore = this.calculateWAFScore(wafSignals, signalsTriggered);
|
||||
components.contentScore += wafScore;
|
||||
}
|
||||
|
||||
totalScore = components.networkScore + components.behaviorScore +
|
||||
components.contentScore + components.anomalyScore;
|
||||
|
||||
// Determine risk level
|
||||
const riskLevel = this.determineRiskLevel(totalScore);
|
||||
|
||||
// Calculate confidence (simplified)
|
||||
const confidence = Math.min(0.8, signalsTriggered.length * 0.2 + 0.3);
|
||||
|
||||
const processingTimeMs = performance.now() - startTime;
|
||||
|
||||
return {
|
||||
totalScore,
|
||||
confidence,
|
||||
riskLevel,
|
||||
components,
|
||||
signalsTriggered,
|
||||
normalizedFeatures: {
|
||||
networkRisk: normalizeMetricValue(components.networkScore, 0, 100),
|
||||
behaviorRisk: normalizeMetricValue(components.behaviorScore, 0, 100),
|
||||
contentRisk: normalizeMetricValue(components.contentScore, 0, 100),
|
||||
anomalyRisk: normalizeMetricValue(components.anomalyScore, 0, 100)
|
||||
},
|
||||
processingTimeMs
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract WAF signals from request metadata
|
||||
*/
|
||||
private extractWAFSignals(metadata: RequestMetadata): Record<string, unknown> | null {
|
||||
// WAF signals are attached to the request object by WAF middleware
|
||||
// Try multiple ways to access them depending on request type
|
||||
const request = metadata as any;
|
||||
|
||||
// Express-style: res.locals.wafSignals (if request has res)
|
||||
if (request.res?.locals?.wafSignals) {
|
||||
return request.res.locals.wafSignals;
|
||||
}
|
||||
|
||||
// Direct attachment: request.wafSignals
|
||||
if (request.wafSignals) {
|
||||
return request.wafSignals;
|
||||
}
|
||||
|
||||
// Headers may contain WAF detection flags
|
||||
if (metadata.headers) {
|
||||
const wafHeader = metadata.headers['x-waf-signals'] || metadata.headers['X-WAF-Signals'];
|
||||
if (wafHeader && typeof wafHeader === 'string') {
|
||||
try {
|
||||
return JSON.parse(wafHeader);
|
||||
} catch {
|
||||
// Invalid JSON, ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate threat score from WAF signals
|
||||
*/
|
||||
private calculateWAFScore(wafSignals: Record<string, unknown>, signalsTriggered: string[]): number {
|
||||
let score = 0;
|
||||
|
||||
// Map WAF detections to configured signal weights
|
||||
if (wafSignals.sqlInjection || wafSignals.sql_injection) {
|
||||
score += this.config.signalWeights.SQL_INJECTION?.weight || 80;
|
||||
signalsTriggered.push('SQL_INJECTION');
|
||||
}
|
||||
|
||||
if (wafSignals.xss || wafSignals.xssAttempt) {
|
||||
score += this.config.signalWeights.XSS_ATTEMPT?.weight || 85;
|
||||
signalsTriggered.push('XSS_ATTEMPT');
|
||||
}
|
||||
|
||||
if (wafSignals.commandInjection || wafSignals.command_injection) {
|
||||
score += this.config.signalWeights.COMMAND_INJECTION?.weight || 95;
|
||||
signalsTriggered.push('COMMAND_INJECTION');
|
||||
}
|
||||
|
||||
if (wafSignals.pathTraversal || wafSignals.path_traversal) {
|
||||
score += this.config.signalWeights.PATH_TRAVERSAL?.weight || 70;
|
||||
signalsTriggered.push('PATH_TRAVERSAL');
|
||||
}
|
||||
|
||||
// Handle unverified bot detection - CRITICAL for fake bots
|
||||
if (wafSignals.unverified_bot) {
|
||||
score += 50; // High penalty for fake bot user agents
|
||||
signalsTriggered.push('UNVERIFIED_BOT');
|
||||
}
|
||||
|
||||
// Handle WAF attack tool detection in user agents
|
||||
const detectedAttacks = wafSignals.detected_attacks;
|
||||
if (Array.isArray(detectedAttacks)) {
|
||||
if (detectedAttacks.includes('attack_tool_user_agent')) {
|
||||
score += this.config.signalWeights.ATTACK_TOOL_UA?.weight || 30;
|
||||
signalsTriggered.push('ATTACK_TOOL_UA');
|
||||
}
|
||||
|
||||
// Additional detection for unverified bots via attack list
|
||||
if (detectedAttacks.includes('unverified_bot')) {
|
||||
score += 50;
|
||||
signalsTriggered.push('UNVERIFIED_BOT');
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines risk level based on score and configured thresholds
|
||||
*/
|
||||
private determineRiskLevel(score: number): 'allow' | 'challenge' | 'block' {
|
||||
if (score <= this.config.thresholds.ALLOW) return 'allow';
|
||||
if (score <= this.config.thresholds.CHALLENGE) return 'challenge';
|
||||
return 'block';
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an allow score for whitelisted or disabled requests
|
||||
*/
|
||||
private createAllowScore(startTime: number): ThreatScore {
|
||||
return {
|
||||
totalScore: 0,
|
||||
confidence: 1.0,
|
||||
riskLevel: 'allow',
|
||||
components: {
|
||||
behaviorScore: 0,
|
||||
contentScore: 0,
|
||||
networkScore: 0,
|
||||
anomalyScore: 0
|
||||
},
|
||||
signalsTriggered: [],
|
||||
normalizedFeatures: {},
|
||||
processingTimeMs: performance.now() - startTime
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an error score when threat analysis fails
|
||||
*/
|
||||
private createErrorScore(startTime: number): ThreatScore {
|
||||
return {
|
||||
totalScore: 0,
|
||||
confidence: 0,
|
||||
riskLevel: 'allow', // Fail open
|
||||
components: {
|
||||
behaviorScore: 0,
|
||||
contentScore: 0,
|
||||
networkScore: 0,
|
||||
anomalyScore: 0
|
||||
},
|
||||
signalsTriggered: ['ERROR'],
|
||||
normalizedFeatures: {},
|
||||
processingTimeMs: performance.now() - startTime
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and configures a threat scorer instance
|
||||
*/
|
||||
export function createThreatScorer(config: ThreatScoringConfig): ThreatScorer {
|
||||
return new ThreatScorer(config);
|
||||
}
|
||||
|
||||
// Default threat scorer for convenience (requires configuration)
|
||||
let defaultScorer: ThreatScorer | null = null;
|
||||
|
||||
export function configureDefaultThreatScorer(config: ThreatScoringConfig): void {
|
||||
defaultScorer = new ThreatScorer(config);
|
||||
}
|
||||
|
||||
export const threatScorer = {
|
||||
scoreRequest: async (request: NetworkRequest): Promise<ThreatScore> => {
|
||||
if (!defaultScorer) {
|
||||
throw new Error('Default threat scorer not configured. Call configureDefaultThreatScorer() first.');
|
||||
}
|
||||
return defaultScorer.scoreRequest(request);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue