Initial commit of massive v2 rewrite
This commit is contained in:
parent
1025f3b523
commit
dc120fe78a
55 changed files with 21733 additions and 0 deletions
449
src/utils/pattern-matching.ts
Normal file
449
src/utils/pattern-matching.ts
Normal file
|
|
@ -0,0 +1,449 @@
|
|||
// =============================================================================
|
||||
// CENTRALIZED PATTERN MATCHING UTILITY
|
||||
// =============================================================================
|
||||
// Consolidates all pattern matching logic to prevent duplication
|
||||
|
||||
// @ts-ignore - string-dsa doesn't have TypeScript definitions
|
||||
import { AhoCorasick } from 'string-dsa';
|
||||
import * as logs from './logs.js';
|
||||
|
||||
export interface PatternMatcher {
|
||||
find(text: string): string[];
|
||||
}
|
||||
|
||||
export interface PatternMatchResult {
|
||||
readonly matched: boolean;
|
||||
readonly matches: readonly string[];
|
||||
readonly matchCount: number;
|
||||
}
|
||||
|
||||
export interface RegexMatchResult {
|
||||
readonly matched: boolean;
|
||||
readonly pattern?: string;
|
||||
readonly match?: string;
|
||||
}
|
||||
|
||||
export interface PatternCollection {
|
||||
readonly name: string;
|
||||
readonly patterns: readonly string[];
|
||||
readonly description?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Centralized Aho-Corasick pattern matcher
|
||||
*/
|
||||
export class AhoCorasickPatternMatcher {
|
||||
private matcher: PatternMatcher | null = null;
|
||||
private readonly patterns: readonly string[];
|
||||
private readonly name: string;
|
||||
|
||||
constructor(name: string, patterns: readonly string[]) {
|
||||
this.name = name;
|
||||
this.patterns = patterns;
|
||||
this.initialize();
|
||||
}
|
||||
|
||||
private initialize(): void {
|
||||
try {
|
||||
if (this.patterns.length === 0) {
|
||||
logs.warn('pattern-matching', `No patterns provided for matcher ${this.name}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.patterns.length > 10000) {
|
||||
logs.warn('pattern-matching', `Too many patterns for ${this.name}: ${this.patterns.length}, truncating to 10000`);
|
||||
this.matcher = new AhoCorasick(this.patterns.slice(0, 10000)) as PatternMatcher;
|
||||
} else {
|
||||
this.matcher = new AhoCorasick(this.patterns) as PatternMatcher;
|
||||
}
|
||||
|
||||
logs.plugin('pattern-matching', `Initialized ${this.name} matcher with ${this.patterns.length} patterns`);
|
||||
} catch (error) {
|
||||
logs.error('pattern-matching', `Failed to initialize ${this.name} matcher: ${error}`);
|
||||
this.matcher = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds pattern matches in text
|
||||
*/
|
||||
find(text: string): PatternMatchResult {
|
||||
if (!this.matcher || !text) {
|
||||
return { matched: false, matches: [], matchCount: 0 };
|
||||
}
|
||||
|
||||
try {
|
||||
const matches = this.matcher.find(text.toLowerCase());
|
||||
return {
|
||||
matched: matches.length > 0,
|
||||
matches: matches.slice(0, 100), // Limit matches to prevent memory issues
|
||||
matchCount: matches.length
|
||||
};
|
||||
} catch (error) {
|
||||
logs.warn('pattern-matching', `Pattern matching failed for ${this.name}: ${error}`);
|
||||
return { matched: false, matches: [], matchCount: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if text contains any patterns
|
||||
*/
|
||||
hasMatch(text: string): boolean {
|
||||
return this.find(text).matched;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets first match found
|
||||
*/
|
||||
getFirstMatch(text: string): string | null {
|
||||
const result = this.find(text);
|
||||
return result.matches.length > 0 ? (result.matches[0] || null) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reinitializes the matcher (useful for pattern updates)
|
||||
*/
|
||||
reinitialize(): void {
|
||||
this.initialize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets pattern count
|
||||
*/
|
||||
getPatternCount(): number {
|
||||
return this.patterns.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if matcher is ready
|
||||
*/
|
||||
isReady(): boolean {
|
||||
return this.matcher !== null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Centralized regex pattern matcher
|
||||
*/
|
||||
export class RegexPatternMatcher {
|
||||
private readonly patterns: Map<string, RegExp> = new Map();
|
||||
private readonly name: string;
|
||||
|
||||
constructor(name: string, patterns: Record<string, string> = {}) {
|
||||
this.name = name;
|
||||
this.compilePatterns(patterns);
|
||||
}
|
||||
|
||||
private compilePatterns(patterns: Record<string, string>): void {
|
||||
let compiled = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const [name, pattern] of Object.entries(patterns)) {
|
||||
try {
|
||||
// Validate pattern length to prevent ReDoS
|
||||
if (pattern.length > 500) {
|
||||
logs.warn('pattern-matching', `Pattern ${name} too long: ${pattern.length} chars, skipping`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
this.patterns.set(name, new RegExp(pattern, 'i'));
|
||||
compiled++;
|
||||
} catch (error) {
|
||||
logs.error('pattern-matching', `Failed to compile regex pattern ${name}: ${error}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
logs.plugin('pattern-matching', `${this.name}: compiled ${compiled} patterns, ${failed} failed`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests text against a specific pattern
|
||||
*/
|
||||
test(patternName: string, text: string): RegexMatchResult {
|
||||
const pattern = this.patterns.get(patternName);
|
||||
if (!pattern) {
|
||||
return { matched: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const match = pattern.exec(text);
|
||||
return {
|
||||
matched: match !== null,
|
||||
pattern: patternName,
|
||||
match: match ? match[0] : undefined
|
||||
};
|
||||
} catch (error) {
|
||||
logs.warn('pattern-matching', `Regex test failed for ${patternName}: ${error}`);
|
||||
return { matched: false };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests text against all patterns
|
||||
*/
|
||||
testAll(text: string): RegexMatchResult[] {
|
||||
const results: RegexMatchResult[] = [];
|
||||
|
||||
for (const patternName of this.patterns.keys()) {
|
||||
const result = this.test(patternName, text);
|
||||
if (result.matched) {
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if any pattern matches
|
||||
*/
|
||||
hasAnyMatch(text: string): boolean {
|
||||
for (const pattern of this.patterns.values()) {
|
||||
try {
|
||||
if (pattern.test(text)) {
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
// Continue with other patterns
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new pattern
|
||||
*/
|
||||
addPattern(name: string, pattern: string): boolean {
|
||||
try {
|
||||
if (pattern.length > 500) {
|
||||
logs.warn('pattern-matching', `Pattern ${name} too long, rejecting`);
|
||||
return false;
|
||||
}
|
||||
|
||||
this.patterns.set(name, new RegExp(pattern, 'i'));
|
||||
return true;
|
||||
} catch (error) {
|
||||
logs.error('pattern-matching', `Failed to add pattern ${name}: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a pattern
|
||||
*/
|
||||
removePattern(name: string): boolean {
|
||||
return this.patterns.delete(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets pattern count
|
||||
*/
|
||||
getPatternCount(): number {
|
||||
return this.patterns.size;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pattern matcher factory for common use cases
|
||||
*/
|
||||
export class PatternMatcherFactory {
|
||||
private static ahoCorasickMatchers: Map<string, AhoCorasickPatternMatcher> = new Map();
|
||||
private static regexMatchers: Map<string, RegexPatternMatcher> = new Map();
|
||||
|
||||
/**
|
||||
* Creates or gets an Aho-Corasick matcher
|
||||
*/
|
||||
static getAhoCorasickMatcher(name: string, patterns: readonly string[]): AhoCorasickPatternMatcher {
|
||||
if (!this.ahoCorasickMatchers.has(name)) {
|
||||
this.ahoCorasickMatchers.set(name, new AhoCorasickPatternMatcher(name, patterns));
|
||||
}
|
||||
return this.ahoCorasickMatchers.get(name)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates or gets a regex matcher
|
||||
*/
|
||||
static getRegexMatcher(name: string, patterns: Record<string, string> = {}): RegexPatternMatcher {
|
||||
if (!this.regexMatchers.has(name)) {
|
||||
this.regexMatchers.set(name, new RegexPatternMatcher(name, patterns));
|
||||
}
|
||||
return this.regexMatchers.get(name)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a matcher
|
||||
*/
|
||||
static removeMatcher(name: string): void {
|
||||
this.ahoCorasickMatchers.delete(name);
|
||||
this.regexMatchers.delete(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all matchers
|
||||
*/
|
||||
static clearAll(): void {
|
||||
this.ahoCorasickMatchers.clear();
|
||||
this.regexMatchers.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all matcher names
|
||||
*/
|
||||
static getMatcherNames(): { ahoCorasick: string[]; regex: string[] } {
|
||||
return {
|
||||
ahoCorasick: Array.from(this.ahoCorasickMatchers.keys()),
|
||||
regex: Array.from(this.regexMatchers.keys())
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Common pattern collections for reuse
|
||||
*/
|
||||
export const CommonPatterns = {
|
||||
// Attack tool patterns
|
||||
ATTACK_TOOLS: [
|
||||
'sqlmap', 'nikto', 'nmap', 'burpsuite', 'w3af', 'acunetix',
|
||||
'nessus', 'openvas', 'gobuster', 'dirbuster', 'wfuzz', 'ffuf',
|
||||
'hydra', 'medusa', 'masscan', 'zmap', 'metasploit', 'burp suite',
|
||||
'scanner', 'exploit', 'payload', 'injection', 'vulnerability'
|
||||
],
|
||||
|
||||
// Suspicious bot patterns
|
||||
SUSPICIOUS_BOTS: [
|
||||
'bot', 'crawler', 'spider', 'scraper', 'scanner', 'harvest',
|
||||
'extract', 'collect', 'gather', 'fetch'
|
||||
],
|
||||
|
||||
// SQL injection patterns
|
||||
SQL_INJECTION: [
|
||||
'union select', 'insert into', 'delete from', 'drop table', 'select * from',
|
||||
"' or '1'='1", "' or 1=1", "admin'--", "' union select", "'; drop table",
|
||||
'union all select', 'group_concat', 'version()', 'database()', 'user()',
|
||||
'information_schema', 'pg_sleep', 'waitfor delay', 'benchmark(',
|
||||
'extractvalue', 'updatexml', 'load_file', 'into outfile',
|
||||
// More aggressive patterns
|
||||
'exec sp_', 'exec xp_', 'execute immediate', 'dbms_',
|
||||
'; shutdown', '; exec', '; execute', '; xp_cmdshell', '; sp_',
|
||||
'cast(', 'convert(', 'concat(', 'substring(', 'ascii(', 'char(',
|
||||
'hex(', 'unhex(', 'md5(', 'sha1(', 'sha2(', 'encode(', 'decode(',
|
||||
'compress(', 'uncompress(', 'aes_encrypt(', 'aes_decrypt(', 'des_encrypt(',
|
||||
'sleep(', 'benchmark(', 'pg_sleep(', 'waitfor delay', 'dbms_lock.sleep',
|
||||
'randomblob(', 'load_extension(', 'sql', 'mysql', 'mssql', 'oracle',
|
||||
'sqlite_', 'pragma ', 'attach database', 'create table', 'alter table',
|
||||
'update set', 'bulk insert', 'openrowset', 'opendatasource', 'openquery',
|
||||
'xtype', 'sysobjects', 'syscolumns', 'sysusers', 'systables',
|
||||
'all_tables', 'user_tables', 'user_tab_columns', 'table_schema',
|
||||
'column_name', 'table_name', 'schema_name', 'database_name',
|
||||
'@@version', '@@datadir', '@@hostname', '@@basedir', 'session_user',
|
||||
'current_user', 'system_user', 'user_name()', 'suser_name()',
|
||||
'is_srvrolemember', 'is_member', 'has_dbaccess', 'has_perms_by_name'
|
||||
],
|
||||
|
||||
// XSS patterns
|
||||
XSS: [
|
||||
'<script>', '</script>', 'javascript:', 'document.cookie', 'document.write',
|
||||
'alert(', 'prompt(', 'confirm(', 'onload=', 'onerror=', 'onclick=',
|
||||
'<iframe', '<object', '<embed', '<svg', 'onmouseover=', 'onfocus=',
|
||||
'eval(', 'unescape(', 'fromcharcode(', 'expression(', 'vbscript:',
|
||||
// ... existing code ...
|
||||
// Add more aggressive XSS patterns
|
||||
'<script', 'script>', 'javascript:', 'data:text/html', 'data:application',
|
||||
'ondblclick=', 'onmouseenter=', 'onmouseleave=', 'onmousemove=', 'onkeydown=',
|
||||
'onkeypress=', 'onkeyup=', 'onsubmit=', 'onreset=', 'onblur=', 'onchange=',
|
||||
'onsearch=', 'onselect=', 'ontoggle=', 'ondrag=', 'ondrop=', 'oninput=',
|
||||
'oninvalid=', 'onpaste=', 'oncopy=', 'oncut=', 'onwheel=', 'ontouchstart=',
|
||||
'ontouchend=', 'ontouchmove=', 'onpointerdown=', 'onpointerup=', 'onpointermove=',
|
||||
'srcdoc=', '<applet', '<base', '<meta', '<link', 'import(', 'constructor.',
|
||||
'prototype.', '__proto__', 'contenteditable', 'designmode', 'javascript://',
|
||||
'vbs:', 'vbscript://', 'data:text/javascript', 'behavior:', 'mhtml:',
|
||||
'-moz-binding', 'xlink:href', 'autofocus', 'onfocusin=', 'onfocusout=',
|
||||
'onhashchange=', 'onmessage=', 'onoffline=', 'ononline=', 'onpagehide=',
|
||||
'onpageshow=', 'onpopstate=', 'onresize=', 'onstorage=', 'onunload=',
|
||||
'onbeforeunload=', 'onanimationstart=', 'onanimationend=', 'onanimationiteration=',
|
||||
'ontransitionend=', '<style', 'style=', '@import'
|
||||
],
|
||||
|
||||
// Command injection patterns
|
||||
COMMAND_INJECTION: [
|
||||
'rm -rf', 'wget http', 'curl http', '| nc', '| netcat', '| sh',
|
||||
'/bin/sh', '/bin/bash', 'cat /etc/passwd', '$(', '`', 'powershell',
|
||||
'cmd.exe', 'system(', 'exec(', 'shell_exec', 'passthru', 'popen',
|
||||
// More dangerous patterns
|
||||
'; ls', '; dir', '; cat', '; type', '; more', '; less', '; head', '; tail',
|
||||
'; ps', '; kill', '; pkill', '; killall', '; timeout', '; sleep',
|
||||
'; uname', '; id', '; whoami', '; groups', '; users', '; w', '; who',
|
||||
'; netstat', '; ss', '; ifconfig', '; ip addr', '; arp', '; route',
|
||||
'; ping', '; traceroute', '; nslookup', '; dig', '; host', '; whois',
|
||||
'; ssh', '; telnet', '; ftp', '; tftp', '; scp', '; rsync', '; rcp',
|
||||
'; chmod', '; chown', '; chgrp', '; umask', '; touch', '; mkdir',
|
||||
'; cp', '; mv', '; ln', '; dd', '; tar', '; zip', '; unzip', '; gzip',
|
||||
'; find', '; locate', '; grep', '; egrep', '; fgrep', '; sed', '; awk',
|
||||
'; perl', '; python', '; ruby', '; php', '; node', '; java', '; gcc',
|
||||
'; make', '; cmake', '; apt', '; yum', '; dnf', '; pacman', '; brew',
|
||||
'; systemctl', '; service', '; init', '; cron', '; at', '; batch',
|
||||
'; mount', '; umount', '; fdisk', '; parted', '; mkfs', '; fsck',
|
||||
'; iptables', '; firewall-cmd', '; ufw', '; fail2ban', '; tcpdump',
|
||||
'; nmap', '; masscan', '; zmap', '; nikto', '; sqlmap', '; metasploit',
|
||||
'& ', '&& ', '|| ', '| ', '; ', '\n', '\r\n', '%0a', '%0d',
|
||||
'eval ', 'assert ', 'preg_replace', 'create_function', 'include ',
|
||||
'require ', 'require_once ', 'include_once ', 'file_get_contents',
|
||||
'file_put_contents', 'fopen', 'fwrite', 'fputs', 'file', 'readfile',
|
||||
'highlight_file', 'show_source', 'proc_open', 'pcntl_exec',
|
||||
'dl(', 'expect ', 'popen(', 'proc_', 'shellexec', 'pcntl_',
|
||||
'posix_', 'getenv', 'putenv', 'setenv', 'mail(', 'mb_send_mail'
|
||||
],
|
||||
|
||||
// Path traversal patterns
|
||||
PATH_TRAVERSAL: [
|
||||
'../../../', '/etc/passwd', '/etc/shadow', '/windows/system32',
|
||||
'..\\..\\..\\', 'boot.ini', '..%2f', '%2e%2e%2f', '..%5c', '%2e%2e%5c'
|
||||
]
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Utility functions for pattern matching
|
||||
*/
|
||||
export const PatternUtils = {
|
||||
/**
|
||||
* Creates a pattern collection
|
||||
*/
|
||||
createCollection(name: string, patterns: readonly string[], description?: string): PatternCollection {
|
||||
return { name, patterns, description };
|
||||
},
|
||||
|
||||
/**
|
||||
* Merges multiple pattern collections
|
||||
*/
|
||||
mergeCollections(...collections: PatternCollection[]): PatternCollection {
|
||||
const allPatterns = collections.flatMap(c => c.patterns);
|
||||
const uniquePatterns = Array.from(new Set(allPatterns));
|
||||
const names = collections.map(c => c.name).join('+');
|
||||
|
||||
return {
|
||||
name: names,
|
||||
patterns: uniquePatterns,
|
||||
description: `Merged collection: ${names}`
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* Validates pattern array
|
||||
*/
|
||||
validatePatterns(patterns: readonly string[]): { valid: readonly string[]; invalid: readonly string[] } {
|
||||
const valid: string[] = [];
|
||||
const invalid: string[] = [];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
if (typeof pattern === 'string' && pattern.length > 0 && pattern.length <= 200) {
|
||||
valid.push(pattern);
|
||||
} else {
|
||||
invalid.push(pattern);
|
||||
}
|
||||
}
|
||||
|
||||
return { valid, invalid };
|
||||
}
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue