Compare commits
No commits in common. "master" and "masterv1" have entirely different histories.
77 changed files with 3538 additions and 28101 deletions
|
|
@ -1,411 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
|
||||
// Mock dependencies
|
||||
jest.unstable_mockModule('../dist/utils/behavioral-detection.js', () => ({
|
||||
behavioralDetection: {
|
||||
config: { enabled: true, Responses: {} },
|
||||
isBlocked: jest.fn(),
|
||||
getRateLimit: jest.fn(),
|
||||
analyzeRequest: jest.fn()
|
||||
}
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('../dist/utils/network.js', () => ({
|
||||
getRealIP: jest.fn()
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('../dist/utils/logs.js', () => ({
|
||||
plugin: jest.fn(),
|
||||
error: jest.fn()
|
||||
}));
|
||||
|
||||
// Import the modules after mocking
|
||||
const BehavioralDetectionMiddleware = (await import('../dist/utils/behavioral-middleware.js')).default;
|
||||
const { behavioralDetection } = await import('../dist/utils/behavioral-detection.js');
|
||||
const { getRealIP } = await import('../dist/utils/network.js');
|
||||
const logs = await import('../dist/utils/logs.js');
|
||||
|
||||
describe('Behavioral Middleware', () => {
|
||||
let mockReq, mockRes, mockNext;
|
||||
let activeTimeouts = [];
|
||||
let activeImmediates = [];
|
||||
|
||||
// Track async operations for cleanup
|
||||
const originalSetTimeout = global.setTimeout;
|
||||
const originalSetImmediate = global.setImmediate;
|
||||
|
||||
global.setTimeout = (fn, delay, ...args) => {
|
||||
const id = originalSetTimeout(fn, delay, ...args);
|
||||
activeTimeouts.push(id);
|
||||
return id;
|
||||
};
|
||||
|
||||
global.setImmediate = (fn, ...args) => {
|
||||
const id = originalSetImmediate(fn, ...args);
|
||||
activeImmediates.push(id);
|
||||
return id;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
activeTimeouts = [];
|
||||
activeImmediates = [];
|
||||
|
||||
// Mock Express request object
|
||||
mockReq = {
|
||||
url: '/api/test',
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'user-agent': 'test-agent',
|
||||
'x-forwarded-for': '192.168.1.1'
|
||||
},
|
||||
ip: '192.168.1.1'
|
||||
};
|
||||
|
||||
// Mock Express response object
|
||||
mockRes = {
|
||||
statusCode: 200,
|
||||
status: jest.fn().mockReturnThis(),
|
||||
setHeader: jest.fn().mockReturnThis(),
|
||||
end: jest.fn(),
|
||||
json: jest.fn(),
|
||||
send: jest.fn(),
|
||||
locals: {}
|
||||
};
|
||||
|
||||
// Mock next function
|
||||
mockNext = jest.fn();
|
||||
|
||||
// Default mock returns
|
||||
getRealIP.mockReturnValue('192.168.1.1');
|
||||
behavioralDetection.isBlocked.mockResolvedValue({ blocked: false });
|
||||
behavioralDetection.getRateLimit.mockResolvedValue(null);
|
||||
behavioralDetection.analyzeRequest.mockResolvedValue({ totalScore: 0, patterns: [] });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
// Clear any pending timeouts and immediates
|
||||
activeTimeouts.forEach(id => clearTimeout(id));
|
||||
activeImmediates.forEach(id => clearImmediate(id));
|
||||
activeTimeouts = [];
|
||||
activeImmediates = [];
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
// Restore original functions
|
||||
global.setTimeout = originalSetTimeout;
|
||||
global.setImmediate = originalSetImmediate;
|
||||
});
|
||||
|
||||
describe('plugin creation', () => {
|
||||
test('should create a behavioral detection middleware plugin', () => {
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
|
||||
expect(plugin.name).toBe('behavioral-detection');
|
||||
expect(plugin.priority).toBe(90);
|
||||
expect(typeof plugin.middleware).toBe('function');
|
||||
});
|
||||
});
|
||||
|
||||
describe('middleware execution', () => {
|
||||
test('should skip processing when behavioral detection is disabled', async () => {
|
||||
behavioralDetection.config.enabled = false;
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
expect(mockNext).toHaveBeenCalled();
|
||||
expect(behavioralDetection.isBlocked).not.toHaveBeenCalled();
|
||||
|
||||
// Restore enabled state
|
||||
behavioralDetection.config.enabled = true;
|
||||
});
|
||||
|
||||
test('should process request when behavioral detection is enabled', async () => {
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
expect(mockNext).toHaveBeenCalled();
|
||||
expect(getRealIP).toHaveBeenCalledWith(mockReq);
|
||||
});
|
||||
|
||||
test('should capture client IP correctly', async () => {
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
getRealIP.mockReturnValue('10.0.0.1');
|
||||
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
expect(getRealIP).toHaveBeenCalledWith(mockReq);
|
||||
});
|
||||
});
|
||||
|
||||
describe('blocking functionality', () => {
|
||||
test('should block requests from blocked IPs', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({
|
||||
blocked: true,
|
||||
reason: 'Malicious activity detected'
|
||||
});
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
// Trigger response capture by calling end and wait for async processing
|
||||
mockRes.end();
|
||||
|
||||
// Wait for setImmediate and async processing to complete
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.status).toHaveBeenCalledWith(403);
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-Behavioral-Block', 'true');
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-Block-Reason', 'Malicious activity detected');
|
||||
expect(logs.plugin).toHaveBeenCalledWith('behavioral', expect.stringContaining('Blocked IP'));
|
||||
});
|
||||
|
||||
test('should use default block message when none configured', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({
|
||||
blocked: true,
|
||||
reason: 'Suspicious activity'
|
||||
});
|
||||
behavioralDetection.config.Responses.BlockMessage = undefined;
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.status).toHaveBeenCalledWith(403);
|
||||
});
|
||||
|
||||
test('should handle blocked IP without reason', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({
|
||||
blocked: true
|
||||
});
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-Block-Reason', 'suspicious activity');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rate limiting functionality', () => {
|
||||
test('should apply rate limiting when limit exceeded', async () => {
|
||||
// Make sure isBlocked returns false so rate limiting is checked
|
||||
behavioralDetection.isBlocked.mockResolvedValue({ blocked: false });
|
||||
behavioralDetection.getRateLimit.mockResolvedValue({
|
||||
exceeded: true,
|
||||
requests: 150,
|
||||
limit: 100,
|
||||
window: 60000,
|
||||
resetTime: Date.now() + 60000
|
||||
});
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.status).toHaveBeenCalledWith(429);
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-RateLimit-Limit', '100');
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-RateLimit-Remaining', '0');
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('Retry-After', '60');
|
||||
expect(logs.plugin).toHaveBeenCalledWith('behavioral', expect.stringContaining('Rate limit exceeded'));
|
||||
});
|
||||
|
||||
test('should set rate limit headers for non-exceeded limits', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({ blocked: false });
|
||||
behavioralDetection.getRateLimit.mockResolvedValue({
|
||||
exceeded: false,
|
||||
requests: 50,
|
||||
limit: 100,
|
||||
resetTime: Date.now() + 30000
|
||||
});
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-RateLimit-Limit', '100');
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-RateLimit-Remaining', '50');
|
||||
expect(mockRes.status).not.toHaveBeenCalledWith(429);
|
||||
});
|
||||
});
|
||||
|
||||
describe('behavioral analysis', () => {
|
||||
test('should analyze request and set behavioral headers', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({ blocked: false });
|
||||
behavioralDetection.getRateLimit.mockResolvedValue(null);
|
||||
behavioralDetection.analyzeRequest.mockResolvedValue({
|
||||
totalScore: 25,
|
||||
patterns: [
|
||||
{ name: 'rapid_requests', score: 15 },
|
||||
{ name: 'suspicious_user_agent', score: 10 }
|
||||
]
|
||||
});
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(behavioralDetection.analyzeRequest).toHaveBeenCalledWith(
|
||||
'192.168.1.1',
|
||||
mockReq,
|
||||
expect.objectContaining({
|
||||
status: 200,
|
||||
responseTime: expect.any(Number)
|
||||
})
|
||||
);
|
||||
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-Behavioral-Score', '25');
|
||||
expect(mockRes.setHeader).toHaveBeenCalledWith('X-Behavioral-Patterns', 'rapid_requests, suspicious_user_agent');
|
||||
});
|
||||
|
||||
test('should store behavioral signals in response locals', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({ blocked: false });
|
||||
behavioralDetection.getRateLimit.mockResolvedValue(null);
|
||||
const analysis = {
|
||||
totalScore: 10,
|
||||
patterns: [{ name: 'test_pattern', score: 10 }]
|
||||
};
|
||||
behavioralDetection.analyzeRequest.mockResolvedValue(analysis);
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.locals.behavioralSignals).toEqual(analysis);
|
||||
});
|
||||
|
||||
test('should not set headers when no patterns detected', async () => {
|
||||
behavioralDetection.isBlocked.mockResolvedValue({ blocked: false });
|
||||
behavioralDetection.getRateLimit.mockResolvedValue(null);
|
||||
behavioralDetection.analyzeRequest.mockResolvedValue({
|
||||
totalScore: 0,
|
||||
patterns: []
|
||||
});
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
expect(mockRes.setHeader).not.toHaveBeenCalledWith('X-Behavioral-Score', expect.anything());
|
||||
expect(mockRes.setHeader).not.toHaveBeenCalledWith('X-Behavioral-Patterns', expect.anything());
|
||||
});
|
||||
});
|
||||
|
||||
describe('response method interception', () => {
|
||||
test('should intercept res.end() calls', async () => {
|
||||
const originalEnd = mockRes.end;
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end('test data');
|
||||
|
||||
// Should have called the original method
|
||||
expect(originalEnd).toHaveBeenCalledWith('test data');
|
||||
});
|
||||
|
||||
test('should intercept res.json() calls', async () => {
|
||||
const originalJson = mockRes.json;
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
const testData = { test: 'data' };
|
||||
mockRes.json(testData);
|
||||
|
||||
expect(originalJson).toHaveBeenCalledWith(testData);
|
||||
});
|
||||
|
||||
test('should intercept res.send() calls', async () => {
|
||||
const originalSend = mockRes.send;
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.send('test response');
|
||||
|
||||
expect(originalSend).toHaveBeenCalledWith('test response');
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
test('should handle errors in behavioral analysis gracefully', async () => {
|
||||
behavioralDetection.analyzeRequest.mockRejectedValue(new Error('Analysis failed'));
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setTimeout(resolve, 10)); // Give error handling time
|
||||
|
||||
expect(logs.error).toHaveBeenCalledWith('behavioral', expect.stringContaining('Error in behavioral analysis'));
|
||||
expect(mockNext).toHaveBeenCalled(); // Should not block request flow
|
||||
});
|
||||
|
||||
test('should handle errors in isBlocked check', async () => {
|
||||
behavioralDetection.isBlocked.mockRejectedValue(new Error('Block check failed'));
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
|
||||
expect(logs.error).toHaveBeenCalledWith('behavioral', expect.stringContaining('Error in behavioral analysis'));
|
||||
});
|
||||
|
||||
test('should fail open for availability on errors', async () => {
|
||||
behavioralDetection.isBlocked.mockRejectedValue(new Error('Service unavailable'));
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
expect(mockNext).toHaveBeenCalled();
|
||||
// Should not block the request even if behavioral detection fails
|
||||
});
|
||||
});
|
||||
|
||||
describe('response locals handling', () => {
|
||||
test('should handle missing response locals gracefully', async () => {
|
||||
delete mockRes.locals;
|
||||
|
||||
const plugin = BehavioralDetectionMiddleware();
|
||||
await plugin.middleware(mockReq, mockRes, mockNext);
|
||||
|
||||
mockRes.end();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
// Should not throw error even without locals
|
||||
expect(mockNext).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,525 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
// Mock the dependencies
|
||||
jest.unstable_mockModule('../dist/index.js', () => ({
|
||||
registerPlugin: jest.fn(),
|
||||
loadConfig: jest.fn().mockResolvedValue({
|
||||
Core: {
|
||||
Enabled: true,
|
||||
CookieName: '__checkpoint',
|
||||
SanitizeURLs: true
|
||||
},
|
||||
ThreatScoring: {
|
||||
Enabled: true,
|
||||
AllowThreshold: 20,
|
||||
ChallengeThreshold: 60,
|
||||
BlockThreshold: 80
|
||||
},
|
||||
ProofOfWork: {
|
||||
Difficulty: 16,
|
||||
SaltLength: 32,
|
||||
ChallengeExpiration: '5m'
|
||||
}
|
||||
}),
|
||||
rootDir: '/test/root'
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('../dist/utils/logs.js', () => ({
|
||||
plugin: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('../dist/utils/threat-scoring.js', () => ({
|
||||
threatScorer: {
|
||||
calculateThreatScore: jest.fn()
|
||||
},
|
||||
THREAT_THRESHOLDS: {
|
||||
ALLOW: 20,
|
||||
CHALLENGE: 60,
|
||||
BLOCK: 80
|
||||
}
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('../dist/utils/proof.js', () => ({
|
||||
challengeStore: new Map(),
|
||||
generateRequestID: jest.fn(() => 'test-request-id'),
|
||||
getChallengeParams: jest.fn(),
|
||||
deleteChallenge: jest.fn(),
|
||||
verifyPoW: jest.fn(),
|
||||
verifyPoS: jest.fn()
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('level', () => ({
|
||||
Level: jest.fn(() => ({
|
||||
open: jest.fn().mockResolvedValue(undefined),
|
||||
put: jest.fn().mockResolvedValue(undefined),
|
||||
get: jest.fn().mockResolvedValue(undefined),
|
||||
del: jest.fn().mockResolvedValue(undefined),
|
||||
close: jest.fn().mockResolvedValue(undefined),
|
||||
iterator: jest.fn(() => [])
|
||||
}))
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('level-ttl', () => ({
|
||||
default: jest.fn((db) => db)
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('fs', () => ({
|
||||
existsSync: jest.fn(() => true),
|
||||
promises: {
|
||||
mkdir: jest.fn().mockResolvedValue(undefined),
|
||||
readFile: jest.fn().mockResolvedValue('<html><body>{{TargetPath}}</body></html>')
|
||||
}
|
||||
}));
|
||||
|
||||
// Import after mocking
|
||||
const checkpoint = await import('../dist/checkpoint.js');
|
||||
|
||||
describe('Checkpoint Security System', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('Utility Functions', () => {
|
||||
describe('sanitizePath', () => {
|
||||
test('should sanitize basic paths', () => {
|
||||
// This function isn't directly exported, so we'll test through integration
|
||||
expect(typeof checkpoint).toBe('object');
|
||||
});
|
||||
|
||||
test('should handle invalid input types gracefully', () => {
|
||||
// Testing integration behaviors since sanitizePath is internal
|
||||
expect(checkpoint).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('LimitedMap', () => {
|
||||
test('should respect size limits', () => {
|
||||
// LimitedMap is internal, testing through checkpoint behaviors
|
||||
expect(checkpoint).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Template System', () => {
|
||||
test('should handle template data replacement', () => {
|
||||
const templateStr = 'Hello {{name}}, your score is {{score}}';
|
||||
const data = { name: 'John', score: 85 };
|
||||
|
||||
const result = templateStr.replace(/\{\{\s*([^{}]+?)\s*\}\}/g, (_, key) => {
|
||||
let value = data;
|
||||
for (const part of key.trim().split('.')) {
|
||||
if (value && typeof value === 'object' && part in value) {
|
||||
value = value[part];
|
||||
} else {
|
||||
value = undefined;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return value != null ? String(value) : '';
|
||||
});
|
||||
|
||||
expect(result).toBe('Hello John, your score is 85');
|
||||
});
|
||||
|
||||
test('should handle nested template data', () => {
|
||||
const templateStr = 'Request {{request.id}} to {{request.path}}';
|
||||
const data = {
|
||||
request: { id: '123', path: '/test' }
|
||||
};
|
||||
|
||||
const result = templateStr.replace(/\{\{\s*([^{}]+?)\s*\}\}/g, (_, key) => {
|
||||
let value = data;
|
||||
for (const part of key.trim().split('.')) {
|
||||
if (value && typeof value === 'object' && part in value) {
|
||||
value = value[part];
|
||||
} else {
|
||||
value = undefined;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return value != null ? String(value) : '';
|
||||
});
|
||||
|
||||
expect(result).toBe('Request 123 to /test');
|
||||
});
|
||||
|
||||
test('should handle missing template data gracefully', () => {
|
||||
const templateStr = 'Hello {{missing.key}}';
|
||||
const data = {};
|
||||
|
||||
const result = templateStr.replace(/\{\{\s*([^{}]+?)\s*\}\}/g, (_, key) => {
|
||||
let value = data;
|
||||
for (const part of key.trim().split('.')) {
|
||||
if (value && typeof value === 'object' && part in value) {
|
||||
value = value[part];
|
||||
} else {
|
||||
value = undefined;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return value != null ? String(value) : '';
|
||||
});
|
||||
|
||||
expect(result).toBe('Hello ');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Response Generation', () => {
|
||||
const mockRequest = {
|
||||
url: '/test',
|
||||
headers: {
|
||||
host: 'example.com',
|
||||
'user-agent': 'Mozilla/5.0 Test Browser'
|
||||
}
|
||||
};
|
||||
|
||||
test('should generate threat level descriptions', () => {
|
||||
const getThreatLevel = (score) => {
|
||||
if (score >= 80) return 'critical';
|
||||
if (score >= 60) return 'high';
|
||||
if (score >= 40) return 'medium';
|
||||
if (score >= 20) return 'low';
|
||||
return 'minimal';
|
||||
};
|
||||
|
||||
expect(getThreatLevel(0)).toBe('minimal');
|
||||
expect(getThreatLevel(15)).toBe('minimal');
|
||||
expect(getThreatLevel(25)).toBe('low');
|
||||
expect(getThreatLevel(45)).toBe('medium');
|
||||
expect(getThreatLevel(65)).toBe('high');
|
||||
expect(getThreatLevel(85)).toBe('critical');
|
||||
});
|
||||
|
||||
test('should format signal names correctly', () => {
|
||||
const formatSignalName = (signal) => {
|
||||
const formatMap = {
|
||||
'sql_injection': 'SQL Injection Attempt',
|
||||
'xss_attempt': 'Cross-Site Scripting',
|
||||
'command_injection': 'Command Injection',
|
||||
'blacklisted_ip': 'Blacklisted IP Address',
|
||||
'tor_exit_node': 'Tor Exit Node',
|
||||
'attack_tool_ua': 'Attack Tool Detected'
|
||||
};
|
||||
return formatMap[signal] || signal.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
|
||||
};
|
||||
|
||||
expect(formatSignalName('sql_injection')).toBe('SQL Injection Attempt');
|
||||
expect(formatSignalName('xss_attempt')).toBe('Cross-Site Scripting');
|
||||
expect(formatSignalName('unknown_signal')).toBe('Unknown Signal');
|
||||
});
|
||||
|
||||
test('should generate appropriate challenge types based on threat score', () => {
|
||||
const getChallengeType = (score) => score > 60 ? 'advanced' : 'standard';
|
||||
const getEstimatedTime = (score) => score > 60 ? '10-15' : '5-10';
|
||||
|
||||
expect(getChallengeType(30)).toBe('standard');
|
||||
expect(getEstimatedTime(30)).toBe('5-10');
|
||||
expect(getChallengeType(70)).toBe('advanced');
|
||||
expect(getEstimatedTime(70)).toBe('10-15');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Client Identification', () => {
|
||||
test('should hash IP addresses consistently', () => {
|
||||
const ip = '192.168.1.100';
|
||||
const hash1 = crypto.createHash('sha256').update(ip).digest().slice(0, 8).toString('hex');
|
||||
const hash2 = crypto.createHash('sha256').update(ip).digest().slice(0, 8).toString('hex');
|
||||
|
||||
expect(hash1).toBe(hash2);
|
||||
expect(hash1).toHaveLength(16); // 8 bytes = 16 hex chars
|
||||
});
|
||||
|
||||
test('should generate different hashes for different IPs', () => {
|
||||
const ip1 = '192.168.1.100';
|
||||
const ip2 = '192.168.1.101';
|
||||
const hash1 = crypto.createHash('sha256').update(ip1).digest().slice(0, 8).toString('hex');
|
||||
const hash2 = crypto.createHash('sha256').update(ip2).digest().slice(0, 8).toString('hex');
|
||||
|
||||
expect(hash1).not.toBe(hash2);
|
||||
});
|
||||
|
||||
test('should hash user agents consistently', () => {
|
||||
const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
|
||||
const hash1 = crypto.createHash('sha256').update(ua).digest().slice(0, 8).toString('hex');
|
||||
const hash2 = crypto.createHash('sha256').update(ua).digest().slice(0, 8).toString('hex');
|
||||
|
||||
expect(hash1).toBe(hash2);
|
||||
expect(hash1).toHaveLength(16);
|
||||
});
|
||||
|
||||
test('should handle empty user agents', () => {
|
||||
const emptyUA = '';
|
||||
const hash = emptyUA ? crypto.createHash('sha256').update(emptyUA).digest().slice(0, 8).toString('hex') : '';
|
||||
|
||||
expect(hash).toBe('');
|
||||
});
|
||||
|
||||
test('should extract browser fingerprint from headers', () => {
|
||||
const headers = {
|
||||
'sec-ch-ua': '"Google Chrome";v="119"',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
'sec-ch-ua-mobile': '?0'
|
||||
};
|
||||
|
||||
const headerNames = [
|
||||
'sec-ch-ua',
|
||||
'sec-ch-ua-platform',
|
||||
'sec-ch-ua-mobile',
|
||||
'sec-ch-ua-platform-version',
|
||||
'sec-ch-ua-arch',
|
||||
'sec-ch-ua-model'
|
||||
];
|
||||
|
||||
const parts = headerNames
|
||||
.map(h => headers[h])
|
||||
.filter(part => typeof part === 'string' && part.length > 0);
|
||||
|
||||
expect(parts).toHaveLength(3);
|
||||
|
||||
if (parts.length > 0) {
|
||||
const fingerprint = crypto.createHash('sha256')
|
||||
.update(Buffer.from(parts.join('|')))
|
||||
.digest()
|
||||
.slice(0, 12)
|
||||
.toString('hex');
|
||||
|
||||
expect(fingerprint).toHaveLength(24); // 12 bytes = 24 hex chars
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle fetch-style headers', () => {
|
||||
const fetchHeaders = {
|
||||
get: jest.fn((name) => {
|
||||
const headers = {
|
||||
'sec-ch-ua': '"Chrome";v="119"',
|
||||
'sec-ch-ua-platform': '"Windows"'
|
||||
};
|
||||
return headers[name] || null;
|
||||
})
|
||||
};
|
||||
|
||||
expect(fetchHeaders.get('sec-ch-ua')).toBe('"Chrome";v="119"');
|
||||
expect(fetchHeaders.get('nonexistent')).toBe(null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Security Configuration', () => {
|
||||
test('should validate threat score thresholds', () => {
|
||||
const thresholds = {
|
||||
ALLOW: 20,
|
||||
CHALLENGE: 60,
|
||||
BLOCK: 80
|
||||
};
|
||||
|
||||
expect(thresholds.ALLOW).toBeLessThan(thresholds.CHALLENGE);
|
||||
expect(thresholds.CHALLENGE).toBeLessThan(thresholds.BLOCK);
|
||||
expect(thresholds.ALLOW).toBeGreaterThanOrEqual(0);
|
||||
expect(thresholds.BLOCK).toBeLessThanOrEqual(100);
|
||||
});
|
||||
|
||||
test('should handle user-defined thresholds', () => {
|
||||
const determineAction = (score, userThresholds = null) => {
|
||||
if (userThresholds) {
|
||||
const allowThreshold = userThresholds.ALLOW || userThresholds.AllowThreshold || 20;
|
||||
const challengeThreshold = userThresholds.CHALLENGE || userThresholds.ChallengeThreshold || 60;
|
||||
|
||||
if (score <= allowThreshold) return 'allow';
|
||||
if (score <= challengeThreshold) return 'challenge';
|
||||
return 'block';
|
||||
}
|
||||
|
||||
if (score <= 20) return 'allow';
|
||||
if (score <= 60) return 'challenge';
|
||||
return 'block';
|
||||
};
|
||||
|
||||
const userThresholds = { AllowThreshold: 15, ChallengeThreshold: 50, BlockThreshold: 80 };
|
||||
|
||||
expect(determineAction(10, userThresholds)).toBe('allow');
|
||||
expect(determineAction(30, userThresholds)).toBe('challenge');
|
||||
expect(determineAction(80, userThresholds)).toBe('block');
|
||||
});
|
||||
|
||||
test('should validate configuration structure', () => {
|
||||
const mockConfig = {
|
||||
Core: {
|
||||
Enabled: true,
|
||||
CookieName: '__checkpoint',
|
||||
SanitizeURLs: true
|
||||
},
|
||||
ThreatScoring: {
|
||||
Enabled: true,
|
||||
AllowThreshold: 20,
|
||||
ChallengeThreshold: 60,
|
||||
BlockThreshold: 80
|
||||
},
|
||||
ProofOfWork: {
|
||||
Difficulty: 16,
|
||||
SaltLength: 32,
|
||||
ChallengeExpiration: '5m'
|
||||
}
|
||||
};
|
||||
|
||||
expect(mockConfig.Core.Enabled).toBe(true);
|
||||
expect(mockConfig.ThreatScoring.AllowThreshold).toBe(20);
|
||||
expect(mockConfig.ProofOfWork.Difficulty).toBe(16);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Token Management', () => {
|
||||
test('should generate consistent token signatures', () => {
|
||||
const secret = 'test-secret';
|
||||
const token = 'test-token';
|
||||
|
||||
const signature1 = crypto.createHmac('sha256', secret).update(token).digest('hex');
|
||||
const signature2 = crypto.createHmac('sha256', secret).update(token).digest('hex');
|
||||
|
||||
expect(signature1).toBe(signature2);
|
||||
expect(signature1).toHaveLength(64); // SHA256 hex = 64 chars
|
||||
});
|
||||
|
||||
test('should generate different signatures for different tokens', () => {
|
||||
const secret = 'test-secret';
|
||||
const token1 = 'test-token-1';
|
||||
const token2 = 'test-token-2';
|
||||
|
||||
const signature1 = crypto.createHmac('sha256', secret).update(token1).digest('hex');
|
||||
const signature2 = crypto.createHmac('sha256', secret).update(token2).digest('hex');
|
||||
|
||||
expect(signature1).not.toBe(signature2);
|
||||
});
|
||||
|
||||
test('should handle token expiration logic', () => {
|
||||
const now = Date.now();
|
||||
const oneHour = 60 * 60 * 1000;
|
||||
const expiration = now + oneHour;
|
||||
|
||||
expect(expiration).toBeGreaterThan(now);
|
||||
expect(expiration - now).toBe(oneHour);
|
||||
|
||||
// Test if token is expired
|
||||
const isExpired = (expirationTime) => Date.now() > expirationTime;
|
||||
expect(isExpired(expiration)).toBe(false);
|
||||
expect(isExpired(now - 1000)).toBe(true);
|
||||
});
|
||||
|
||||
test('should validate nonce uniqueness', () => {
|
||||
const nonce1 = crypto.randomBytes(16).toString('hex');
|
||||
const nonce2 = crypto.randomBytes(16).toString('hex');
|
||||
|
||||
expect(nonce1).not.toBe(nonce2);
|
||||
expect(nonce1).toHaveLength(32); // 16 bytes = 32 hex chars
|
||||
expect(nonce2).toHaveLength(32);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Rate Limiting', () => {
|
||||
test('should track request attempts per IP', () => {
|
||||
const ipAttempts = new Map();
|
||||
const maxAttempts = 10;
|
||||
|
||||
const recordAttempt = (ip) => {
|
||||
const currentAttempts = ipAttempts.get(ip) || 0;
|
||||
const newAttempts = currentAttempts + 1;
|
||||
ipAttempts.set(ip, newAttempts);
|
||||
return newAttempts <= maxAttempts;
|
||||
};
|
||||
|
||||
expect(recordAttempt('192.168.1.100')).toBe(true);
|
||||
expect(ipAttempts.get('192.168.1.100')).toBe(1);
|
||||
|
||||
// Simulate many attempts
|
||||
for (let i = 0; i < 10; i++) {
|
||||
recordAttempt('192.168.1.100');
|
||||
}
|
||||
|
||||
expect(recordAttempt('192.168.1.100')).toBe(false); // Should exceed limit
|
||||
});
|
||||
|
||||
test('should handle time-based rate limiting', () => {
|
||||
const now = Date.now();
|
||||
const oneHour = 60 * 60 * 1000;
|
||||
const windowStart = now - oneHour;
|
||||
|
||||
const isWithinWindow = (timestamp) => timestamp > windowStart;
|
||||
|
||||
expect(isWithinWindow(now)).toBe(true);
|
||||
expect(isWithinWindow(now - oneHour - 1000)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Extension Handling', () => {
|
||||
test('should handle file extension filtering', () => {
|
||||
const path = '/static/style.css';
|
||||
const extension = path.substring(path.lastIndexOf('.')).toLowerCase();
|
||||
|
||||
expect(extension).toBe('.css');
|
||||
});
|
||||
|
||||
test('should identify static file extensions', () => {
|
||||
const staticExtensions = new Set(['.css', '.js', '.png', '.jpg', '.gif', '.ico', '.svg']);
|
||||
|
||||
expect(staticExtensions.has('.css')).toBe(true);
|
||||
expect(staticExtensions.has('.js')).toBe(true);
|
||||
expect(staticExtensions.has('.html')).toBe(false);
|
||||
});
|
||||
|
||||
test('should handle paths without extensions', () => {
|
||||
const pathWithoutExt = '/api/users';
|
||||
const lastDot = pathWithoutExt.lastIndexOf('.');
|
||||
|
||||
expect(lastDot).toBe(-1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Security Validation', () => {
|
||||
test('should sanitize URL paths', () => {
|
||||
const sanitizePath = (inputPath) => {
|
||||
if (typeof inputPath !== 'string') {
|
||||
return '/';
|
||||
}
|
||||
let pathOnly = inputPath.replace(/[\x00-\x1F\x7F]/g, '');
|
||||
pathOnly = pathOnly.replace(/[<>;"'`|]/g, '');
|
||||
const parts = pathOnly.split('/').filter(seg => seg && seg !== '.' && seg !== '..');
|
||||
return '/' + parts.map(seg => encodeURIComponent(seg)).join('/');
|
||||
};
|
||||
|
||||
expect(sanitizePath('/path/../../../etc/passwd')).toBe('/path/etc/passwd'); // .. filtered out
|
||||
expect(sanitizePath('/path<script>alert("xss")</script>')).toBe('/pathscriptalert(xss)/script');
|
||||
expect(sanitizePath('/path\x00\x1F\x7F/file')).toBe('/path/file');
|
||||
expect(sanitizePath(null)).toBe('/');
|
||||
});
|
||||
|
||||
test('should handle extension filtering', () => {
|
||||
const isStaticFile = (path) => {
|
||||
const staticExtensions = new Set(['.css', '.js', '.png', '.jpg', '.gif', '.ico', '.svg']);
|
||||
const ext = path.substring(path.lastIndexOf('.')).toLowerCase();
|
||||
return staticExtensions.has(ext);
|
||||
};
|
||||
|
||||
expect(isStaticFile('/static/style.css')).toBe(true);
|
||||
expect(isStaticFile('/app.js')).toBe(true);
|
||||
expect(isStaticFile('/api/users')).toBe(false);
|
||||
expect(isStaticFile('/index.html')).toBe(false);
|
||||
});
|
||||
|
||||
test('should handle control characters in paths', () => {
|
||||
const pathWithControlChars = '/path\x00\x1F\x7F/file';
|
||||
const sanitized = pathWithControlChars.replace(/[\x00-\x1F\x7F]/g, '');
|
||||
|
||||
expect(sanitized).toBe('/path/file');
|
||||
expect(sanitized).not.toMatch(/[\x00-\x1F\x7F]/);
|
||||
});
|
||||
|
||||
test('should filter dangerous characters', () => {
|
||||
const pathWithDangerousChars = '/path<script>alert("xss")</script>';
|
||||
const sanitized = pathWithDangerousChars.replace(/[<>;"'`|]/g, '');
|
||||
|
||||
expect(sanitized).toBe('/pathscriptalert(xss)/script'); // Correct expectation
|
||||
expect(sanitized).not.toContain('<script>');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,623 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
|
||||
// Mock all external dependencies before importing the module
|
||||
const mockMkdir = jest.fn();
|
||||
const mockReadFile = jest.fn();
|
||||
const mockWriteFileSync = jest.fn();
|
||||
const mockReadFileSync = jest.fn();
|
||||
const mockUnlinkSync = jest.fn();
|
||||
const mockExistsSync = jest.fn();
|
||||
const mockReaddirSync = jest.fn();
|
||||
|
||||
jest.unstable_mockModule('fs/promises', () => ({
|
||||
mkdir: mockMkdir,
|
||||
readFile: mockReadFile
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('fs', () => ({
|
||||
writeFileSync: mockWriteFileSync,
|
||||
readFileSync: mockReadFileSync,
|
||||
unlinkSync: mockUnlinkSync,
|
||||
existsSync: mockExistsSync,
|
||||
readdirSync: mockReaddirSync
|
||||
}));
|
||||
|
||||
const mockJoin = jest.fn();
|
||||
const mockDirname = jest.fn();
|
||||
const mockBasename = jest.fn();
|
||||
|
||||
jest.unstable_mockModule('path', () => ({
|
||||
join: mockJoin,
|
||||
dirname: mockDirname,
|
||||
basename: mockBasename
|
||||
}));
|
||||
|
||||
const mockFileURLToPath = jest.fn();
|
||||
jest.unstable_mockModule('url', () => ({
|
||||
fileURLToPath: mockFileURLToPath
|
||||
}));
|
||||
|
||||
const mockSecureImportModule = jest.fn();
|
||||
jest.unstable_mockModule('../dist/utils/plugins.js', () => ({
|
||||
secureImportModule: mockSecureImportModule
|
||||
}));
|
||||
|
||||
const mockLogs = {
|
||||
section: jest.fn(),
|
||||
init: jest.fn(),
|
||||
config: jest.fn(),
|
||||
error: jest.fn(),
|
||||
msg: jest.fn(),
|
||||
server: jest.fn()
|
||||
};
|
||||
jest.unstable_mockModule('../dist/utils/logs.js', () => mockLogs);
|
||||
|
||||
const mockApp = {
|
||||
set: jest.fn(),
|
||||
use: jest.fn(),
|
||||
listen: jest.fn()
|
||||
};
|
||||
|
||||
const mockExpress = jest.fn(() => mockApp);
|
||||
mockExpress.json = jest.fn(() => (req, res, next) => next());
|
||||
mockExpress.urlencoded = jest.fn(() => (req, res, next) => next());
|
||||
mockExpress.static = jest.fn(() => (req, res, next) => next());
|
||||
mockExpress.Router = jest.fn(() => ({
|
||||
use: jest.fn()
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('express', () => ({
|
||||
default: mockExpress
|
||||
}));
|
||||
|
||||
const mockServer = {
|
||||
on: jest.fn(),
|
||||
close: jest.fn(),
|
||||
listen: jest.fn()
|
||||
};
|
||||
|
||||
const mockCreateServer = jest.fn(() => mockServer);
|
||||
jest.unstable_mockModule('http', () => ({
|
||||
createServer: mockCreateServer
|
||||
}));
|
||||
|
||||
const mockSpawn = jest.fn();
|
||||
jest.unstable_mockModule('child_process', () => ({
|
||||
spawn: mockSpawn
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('dotenv', () => ({
|
||||
config: jest.fn()
|
||||
}));
|
||||
|
||||
// Mock process for command line argument testing
|
||||
const originalArgv = process.argv;
|
||||
const originalEnv = process.env;
|
||||
const originalExit = process.exit;
|
||||
const originalKill = process.kill;
|
||||
const originalOn = process.on;
|
||||
|
||||
// Mock imports properly for ES modules
|
||||
const mockTomlModule = {
|
||||
parse: jest.fn(() => ({}))
|
||||
};
|
||||
|
||||
jest.unstable_mockModule('@iarna/toml', () => ({
|
||||
default: mockTomlModule
|
||||
}));
|
||||
|
||||
// Import the module after all mocking is set up
|
||||
let indexModule;
|
||||
|
||||
describe('Index (Main Application)', () => {
|
||||
let mockProcess;
|
||||
|
||||
beforeAll(async () => {
|
||||
// Import the module once with proper mocking setup
|
||||
indexModule = await import('../dist/index.js');
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset all mocks
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Setup default mock returns
|
||||
mockFileURLToPath.mockReturnValue('/app/src/index.js');
|
||||
mockDirname.mockReturnValue('/app/src');
|
||||
mockJoin.mockImplementation((...parts) => parts.join('/'));
|
||||
mockBasename.mockImplementation((path, ext) => {
|
||||
const name = path.split('/').pop();
|
||||
return ext ? name.replace(ext, '') : name;
|
||||
});
|
||||
mockExistsSync.mockReturnValue(false);
|
||||
mockReaddirSync.mockReturnValue([]);
|
||||
mockReadFile.mockResolvedValue('');
|
||||
mockTomlModule.parse.mockReturnValue({});
|
||||
|
||||
// Setup process mocking
|
||||
mockProcess = {
|
||||
argv: ['node', 'index.js'],
|
||||
env: { ...originalEnv },
|
||||
exit: jest.fn(),
|
||||
kill: jest.fn(),
|
||||
on: jest.fn()
|
||||
};
|
||||
|
||||
// Note: Plugin registry persists across tests in ES modules
|
||||
// This is expected behavior in the test environment
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.argv = originalArgv;
|
||||
process.env = originalEnv;
|
||||
process.exit = originalExit;
|
||||
process.kill = originalKill;
|
||||
process.on = originalOn;
|
||||
});
|
||||
|
||||
describe('Plugin Registration System', () => {
|
||||
test('should register plugin successfully', () => {
|
||||
const pluginName = 'test-plugin';
|
||||
const handler = { middleware: jest.fn() };
|
||||
|
||||
indexModule.registerPlugin(pluginName, handler);
|
||||
|
||||
const registeredNames = indexModule.getRegisteredPluginNames();
|
||||
expect(registeredNames).toContain(pluginName);
|
||||
});
|
||||
|
||||
test('should throw error for invalid plugin name', () => {
|
||||
expect(() => {
|
||||
indexModule.registerPlugin('', { middleware: jest.fn() });
|
||||
}).toThrow('Plugin name must be a non-empty string');
|
||||
|
||||
expect(() => {
|
||||
indexModule.registerPlugin(null, { middleware: jest.fn() });
|
||||
}).toThrow('Plugin name must be a non-empty string');
|
||||
});
|
||||
|
||||
test('should throw error for invalid handler', () => {
|
||||
expect(() => {
|
||||
indexModule.registerPlugin('test', null);
|
||||
}).toThrow('Plugin handler must be an object');
|
||||
|
||||
expect(() => {
|
||||
indexModule.registerPlugin('test', 'invalid');
|
||||
}).toThrow('Plugin handler must be an object');
|
||||
});
|
||||
|
||||
test('should prevent duplicate plugin registration', () => {
|
||||
const pluginName = 'duplicate-plugin';
|
||||
const handler = { middleware: jest.fn() };
|
||||
|
||||
indexModule.registerPlugin(pluginName, handler);
|
||||
|
||||
expect(() => {
|
||||
indexModule.registerPlugin(pluginName, handler);
|
||||
}).toThrow(`Plugin 'duplicate-plugin' is already registered`);
|
||||
});
|
||||
|
||||
test('should load plugins in registration order', () => {
|
||||
const handler1 = { middleware: jest.fn() };
|
||||
const handler2 = { middleware: jest.fn() };
|
||||
|
||||
indexModule.registerPlugin('order-test-1', handler1);
|
||||
indexModule.registerPlugin('order-test-2', handler2);
|
||||
|
||||
const handlers = indexModule.loadPlugins();
|
||||
const registeredNames = indexModule.getRegisteredPluginNames();
|
||||
|
||||
// Check that our test plugins are in the registry
|
||||
expect(registeredNames).toContain('order-test-1');
|
||||
expect(registeredNames).toContain('order-test-2');
|
||||
|
||||
// Check that our handlers are in the handlers array
|
||||
expect(handlers).toContain(handler1);
|
||||
expect(handlers).toContain(handler2);
|
||||
|
||||
// Find the indices of our test handlers
|
||||
const handler1Index = handlers.indexOf(handler1);
|
||||
const handler2Index = handlers.indexOf(handler2);
|
||||
|
||||
// Verify order (handler1 should come before handler2)
|
||||
expect(handler1Index).toBeLessThan(handler2Index);
|
||||
});
|
||||
|
||||
test('should freeze plugin registry', () => {
|
||||
const handler = { middleware: jest.fn() };
|
||||
indexModule.registerPlugin('before-freeze', handler);
|
||||
|
||||
indexModule.freezePlugins();
|
||||
|
||||
expect(mockLogs.msg).toHaveBeenCalledWith('Plugin registration frozen');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Configuration Loading', () => {
|
||||
test('should load config successfully', async () => {
|
||||
const configName = 'test-config';
|
||||
const target = {};
|
||||
const configData = { setting1: 'value1' };
|
||||
|
||||
mockReadFile.mockResolvedValue('setting1 = "value1"');
|
||||
mockJoin.mockReturnValue('/app/src/config/test-config.toml');
|
||||
|
||||
// Mock the TOML parser
|
||||
const mockTomlModule = {
|
||||
default: {
|
||||
parse: jest.fn(() => configData)
|
||||
}
|
||||
};
|
||||
jest.unstable_mockModule('@iarna/toml', () => mockTomlModule);
|
||||
|
||||
await indexModule.loadConfig(configName, target);
|
||||
|
||||
expect(mockReadFile).toHaveBeenCalledWith('/app/src/config/test-config.toml', 'utf8');
|
||||
expect(target).toEqual(configData);
|
||||
});
|
||||
|
||||
test('should throw error for invalid config name', async () => {
|
||||
await expect(indexModule.loadConfig('', {})).rejects.toThrow('Config name must be a non-empty string');
|
||||
await expect(indexModule.loadConfig(null, {})).rejects.toThrow('Config name must be a non-empty string');
|
||||
});
|
||||
|
||||
test('should throw error for invalid target', async () => {
|
||||
await expect(indexModule.loadConfig('test', null)).rejects.toThrow('Config target must be an object');
|
||||
await expect(indexModule.loadConfig('test', 'invalid')).rejects.toThrow('Config target must be an object');
|
||||
});
|
||||
|
||||
test('should handle config loading errors', async () => {
|
||||
const configName = 'failing-config';
|
||||
const target = {};
|
||||
|
||||
mockReadFile.mockRejectedValue(new Error('File not found'));
|
||||
|
||||
await expect(indexModule.loadConfig(configName, target)).rejects.toThrow(
|
||||
"Failed to load config 'failing-config': File not found"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Command Line Argument Handling', () => {
|
||||
test('should validate PID file operations for kill mode', () => {
|
||||
// Test the logic for handling kill mode operations
|
||||
mockExistsSync.mockReturnValue(true);
|
||||
mockReadFileSync.mockReturnValue('1234');
|
||||
mockJoin.mockReturnValue('/app/src/checkpoint.pid');
|
||||
|
||||
const pidContent = mockReadFileSync('checkpoint.pid', 'utf8').trim();
|
||||
const pid = parseInt(pidContent, 10);
|
||||
|
||||
expect(pid).toBe(1234);
|
||||
expect(isNaN(pid)).toBe(false);
|
||||
expect(pid > 0).toBe(true);
|
||||
});
|
||||
|
||||
test('should validate invalid PID handling', () => {
|
||||
mockReadFileSync.mockReturnValue('invalid');
|
||||
|
||||
const pidContent = mockReadFileSync('checkpoint.pid', 'utf8').trim();
|
||||
const pid = parseInt(pidContent, 10);
|
||||
|
||||
expect(isNaN(pid)).toBe(true);
|
||||
});
|
||||
|
||||
test('should validate daemon spawn parameters', () => {
|
||||
const mockChildProcess = {
|
||||
pid: 5678,
|
||||
unref: jest.fn()
|
||||
};
|
||||
mockSpawn.mockReturnValue(mockChildProcess);
|
||||
|
||||
const args = ['index.js'];
|
||||
const nodeExecutable = 'node';
|
||||
|
||||
mockSpawn(nodeExecutable, args, {
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
});
|
||||
|
||||
expect(mockSpawn).toHaveBeenCalledWith(
|
||||
'node',
|
||||
['index.js'],
|
||||
expect.objectContaining({
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Directory Initialization', () => {
|
||||
test('should validate directory creation logic', async () => {
|
||||
mockMkdir.mockResolvedValue(undefined);
|
||||
|
||||
const expectedDirs = [
|
||||
'/app/src/data',
|
||||
'/app/src/db',
|
||||
'/app/src/config'
|
||||
];
|
||||
|
||||
// Test the directory creation logic
|
||||
for (const dirPath of expectedDirs) {
|
||||
mockJoin.mockReturnValueOnce(dirPath);
|
||||
await mockMkdir(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
expect(mockMkdir).toHaveBeenCalledTimes(3);
|
||||
expect(mockMkdir).toHaveBeenCalledWith(expect.any(String), { recursive: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('Plugin Discovery', () => {
|
||||
test('should discover plugins in correct order', () => {
|
||||
mockExistsSync.mockImplementation(path => {
|
||||
return path.includes('plugins');
|
||||
});
|
||||
|
||||
mockReaddirSync.mockReturnValue([
|
||||
'waf.js',
|
||||
'basic-auth.js',
|
||||
'ipfilter.js',
|
||||
'proxy.js',
|
||||
'rate-limit.js'
|
||||
]);
|
||||
|
||||
mockBasename.mockImplementation((file, ext) =>
|
||||
file.replace(ext || '', '')
|
||||
);
|
||||
|
||||
// Plugin load order should be: ipfilter, waf, proxy, then alphabetical
|
||||
const expectedOrder = ['ipfilter', 'waf', 'proxy', 'basic-auth', 'rate-limit'];
|
||||
|
||||
expect(mockReaddirSync).toHaveBeenCalled;
|
||||
});
|
||||
});
|
||||
|
||||
describe('Express Server Setup', () => {
|
||||
test('should validate Express app creation', () => {
|
||||
// Test Express app creation
|
||||
const app = mockExpress();
|
||||
expect(mockExpress).toHaveBeenCalled();
|
||||
expect(app).toBeDefined();
|
||||
expect(app.set).toBeDefined();
|
||||
expect(app.use).toBeDefined();
|
||||
});
|
||||
|
||||
test('should validate middleware configuration', () => {
|
||||
// Test middleware setup logic
|
||||
const app = mockExpress();
|
||||
|
||||
// Test trust proxy setting
|
||||
app.set('trust proxy', true);
|
||||
expect(app.set).toHaveBeenCalledWith('trust proxy', true);
|
||||
|
||||
// Test body parsing middleware
|
||||
const jsonMiddleware = mockExpress.json({ limit: '10mb' });
|
||||
const urlencodedMiddleware = mockExpress.urlencoded({ extended: true, limit: '10mb' });
|
||||
|
||||
expect(mockExpress.json).toHaveBeenCalledWith({ limit: '10mb' });
|
||||
expect(mockExpress.urlencoded).toHaveBeenCalledWith({ extended: true, limit: '10mb' });
|
||||
});
|
||||
|
||||
test('should handle WebSocket upgrade detection logic', () => {
|
||||
const mockReq = {
|
||||
headers: {
|
||||
upgrade: 'websocket',
|
||||
connection: 'Upgrade'
|
||||
}
|
||||
};
|
||||
const mockRes = {};
|
||||
const mockNext = jest.fn();
|
||||
|
||||
// Test WebSocket detection logic
|
||||
const upgradeHeader = mockReq.headers.upgrade;
|
||||
const connectionHeader = mockReq.headers.connection;
|
||||
|
||||
const isWebSocket = upgradeHeader === 'websocket' ||
|
||||
(connectionHeader && connectionHeader.toLowerCase().includes('upgrade'));
|
||||
|
||||
if (isWebSocket) {
|
||||
mockReq.isWebSocketRequest = true;
|
||||
mockNext();
|
||||
}
|
||||
|
||||
expect(mockReq.isWebSocketRequest).toBe(true);
|
||||
expect(mockNext).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Environment Configuration', () => {
|
||||
test('should handle production environment', () => {
|
||||
process.env.NODE_ENV = 'production';
|
||||
|
||||
// In production, console.log should be disabled
|
||||
// This is handled at module load time
|
||||
expect(process.env.NODE_ENV).toBe('production');
|
||||
});
|
||||
|
||||
test('should handle custom port configuration', () => {
|
||||
process.env.PORT = '8080';
|
||||
|
||||
// Port validation would happen during server startup
|
||||
expect(process.env.PORT).toBe('8080');
|
||||
});
|
||||
|
||||
test('should use default port when not specified', () => {
|
||||
delete process.env.PORT;
|
||||
|
||||
// Default port should be 3000
|
||||
const expectedPort = 3000;
|
||||
expect(expectedPort).toBe(3000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
test('should validate plugin loading error handling', async () => {
|
||||
const error = new Error('Plugin load failed');
|
||||
mockSecureImportModule.mockRejectedValue(error);
|
||||
|
||||
// Test error handling logic
|
||||
try {
|
||||
await mockSecureImportModule('test-plugin.js');
|
||||
} catch (e) {
|
||||
expect(e.message).toBe('Plugin load failed');
|
||||
}
|
||||
|
||||
expect(mockSecureImportModule).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should validate config loading error handling', async () => {
|
||||
const error = new Error('Config read failed');
|
||||
mockReadFile.mockRejectedValue(error);
|
||||
|
||||
// Test config loading error handling
|
||||
try {
|
||||
await indexModule.loadConfig('test-config', {});
|
||||
} catch (e) {
|
||||
expect(e.message).toContain('Failed to load config');
|
||||
expect(e.message).toContain('Config read failed');
|
||||
}
|
||||
});
|
||||
|
||||
test('should validate port number range', () => {
|
||||
const invalidPorts = [-1, 0, 65536, 70000];
|
||||
const validPorts = [80, 443, 3000, 8080, 65535];
|
||||
|
||||
invalidPorts.forEach(port => {
|
||||
expect(isNaN(Number(port)) || Number(port) < 1 || Number(port) > 65535).toBe(true);
|
||||
});
|
||||
|
||||
validPorts.forEach(port => {
|
||||
expect(Number(port) >= 1 && Number(port) <= 65535).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Exclusion Rules Processing', () => {
|
||||
test('should compile exclusion patterns correctly', () => {
|
||||
const exclusionRules = [
|
||||
{
|
||||
Path: '/api/health',
|
||||
Hosts: ['localhost', '127.0.0.1'],
|
||||
UserAgents: ['healthcheck.*']
|
||||
}
|
||||
];
|
||||
|
||||
// Test exclusion rule compilation logic
|
||||
const compiledRule = {
|
||||
...exclusionRules[0],
|
||||
pathStartsWith: exclusionRules[0].Path,
|
||||
hostsSet: new Set(exclusionRules[0].Hosts),
|
||||
userAgentPatterns: exclusionRules[0].UserAgents.map(pattern => new RegExp(pattern, 'i'))
|
||||
};
|
||||
|
||||
expect(compiledRule.hostsSet.has('localhost')).toBe(true);
|
||||
expect(compiledRule.userAgentPatterns[0].test('healthcheck-bot')).toBe(true);
|
||||
});
|
||||
|
||||
test('should handle invalid regex patterns in UserAgents', () => {
|
||||
const invalidPattern = '[invalid regex';
|
||||
|
||||
try {
|
||||
new RegExp(invalidPattern, 'i');
|
||||
} catch (e) {
|
||||
// Should handle invalid regex gracefully
|
||||
const fallbackPattern = /(?!)/; // Never matches
|
||||
expect(fallbackPattern.test('anything')).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Static File Middleware', () => {
|
||||
test('should validate static file setup for existing directories', () => {
|
||||
const webfontPath = '/app/src/pages/interstitial/webfont';
|
||||
const jsPath = '/app/src/pages/interstitial/js';
|
||||
|
||||
mockExistsSync.mockImplementation(path => {
|
||||
return path === webfontPath || path === jsPath;
|
||||
});
|
||||
|
||||
const router = mockExpress.Router();
|
||||
|
||||
// Test static directory setup logic
|
||||
if (mockExistsSync(webfontPath)) {
|
||||
router.use('/webfont', mockExpress.static(webfontPath, { maxAge: '7d' }));
|
||||
}
|
||||
|
||||
if (mockExistsSync(jsPath)) {
|
||||
router.use('/js', mockExpress.static(jsPath, { maxAge: '7d' }));
|
||||
}
|
||||
|
||||
expect(mockExistsSync).toHaveBeenCalledWith(webfontPath);
|
||||
expect(mockExistsSync).toHaveBeenCalledWith(jsPath);
|
||||
expect(mockExpress.static).toHaveBeenCalledWith(webfontPath, { maxAge: '7d' });
|
||||
expect(mockExpress.static).toHaveBeenCalledWith(jsPath, { maxAge: '7d' });
|
||||
});
|
||||
|
||||
test('should skip static setup for non-existent directories', () => {
|
||||
mockExistsSync.mockReturnValue(false);
|
||||
|
||||
const webfontPath = '/app/src/pages/interstitial/webfont';
|
||||
const jsPath = '/app/src/pages/interstitial/js';
|
||||
|
||||
// Test logic when directories don't exist
|
||||
const webfontExists = mockExistsSync(webfontPath);
|
||||
const jsExists = mockExistsSync(jsPath);
|
||||
|
||||
expect(webfontExists).toBe(false);
|
||||
expect(jsExists).toBe(false);
|
||||
expect(mockExistsSync).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Graceful Shutdown', () => {
|
||||
test('should validate shutdown handler logic', () => {
|
||||
const mockShutdownHandler = jest.fn();
|
||||
const activeSockets = new Set();
|
||||
const mockSocketInstance = {
|
||||
destroy: jest.fn(),
|
||||
setTimeout: jest.fn(),
|
||||
setKeepAlive: jest.fn(),
|
||||
on: jest.fn()
|
||||
};
|
||||
|
||||
activeSockets.add(mockSocketInstance);
|
||||
|
||||
// Test shutdown logic
|
||||
const isShuttingDown = false;
|
||||
if (!isShuttingDown) {
|
||||
activeSockets.forEach(sock => sock.destroy());
|
||||
mockShutdownHandler();
|
||||
}
|
||||
|
||||
expect(mockSocketInstance.destroy).toHaveBeenCalled();
|
||||
expect(mockShutdownHandler).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should validate socket management', () => {
|
||||
const activeSockets = new Set();
|
||||
const mockSocket = {
|
||||
destroy: jest.fn(),
|
||||
setTimeout: jest.fn(),
|
||||
setKeepAlive: jest.fn(),
|
||||
on: jest.fn()
|
||||
};
|
||||
|
||||
// Test socket lifecycle
|
||||
activeSockets.add(mockSocket);
|
||||
expect(activeSockets.has(mockSocket)).toBe(true);
|
||||
|
||||
// Test socket configuration
|
||||
mockSocket.setTimeout(120000);
|
||||
mockSocket.setKeepAlive(true, 60000);
|
||||
|
||||
expect(mockSocket.setTimeout).toHaveBeenCalledWith(120000);
|
||||
expect(mockSocket.setKeepAlive).toHaveBeenCalledWith(true, 60000);
|
||||
|
||||
// Test cleanup
|
||||
activeSockets.delete(mockSocket);
|
||||
expect(activeSockets.has(mockSocket)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,231 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
import {
|
||||
init,
|
||||
safeAsync,
|
||||
safeSync,
|
||||
plugin,
|
||||
config,
|
||||
db,
|
||||
server,
|
||||
section,
|
||||
warn,
|
||||
error,
|
||||
msg
|
||||
} from '../dist/utils/logs.js';
|
||||
|
||||
describe('Logs utilities', () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(console, 'log').mockImplementation();
|
||||
jest.spyOn(console, 'warn').mockImplementation();
|
||||
jest.spyOn(console, 'error').mockImplementation();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('init', () => {
|
||||
test('should log initialization message', () => {
|
||||
init('Initializing Checkpoint...');
|
||||
expect(console.log).toHaveBeenCalledWith('Initializing Checkpoint...');
|
||||
});
|
||||
});
|
||||
|
||||
describe('safeAsync', () => {
|
||||
test('should return result of successful async operation', async () => {
|
||||
const successfulOperation = async () => 'success';
|
||||
const result = await safeAsync(successfulOperation, 'test', 'Failed to process');
|
||||
expect(result).toBe('success');
|
||||
});
|
||||
|
||||
test('should handle async errors and return fallback', async () => {
|
||||
const failingOperation = async () => {
|
||||
throw new Error('Operation failed');
|
||||
};
|
||||
const result = await safeAsync(failingOperation, 'test', 'Failed to process', 'fallback');
|
||||
expect(result).toBe('fallback');
|
||||
});
|
||||
|
||||
test('should return null by default on error', async () => {
|
||||
const failingOperation = async () => {
|
||||
throw new Error('Operation failed');
|
||||
};
|
||||
const result = await safeAsync(failingOperation, 'test', 'Failed to process');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('should handle non-Error objects thrown', async () => {
|
||||
const failingOperation = async () => {
|
||||
throw 'string error';
|
||||
};
|
||||
const result = await safeAsync(failingOperation, 'test', 'Failed to process', 'fallback');
|
||||
expect(result).toBe('fallback');
|
||||
});
|
||||
|
||||
test('should handle undefined errors', async () => {
|
||||
const failingOperation = async () => {
|
||||
throw undefined;
|
||||
};
|
||||
const result = await safeAsync(failingOperation, 'test', 'Failed to process', 'fallback');
|
||||
expect(result).toBe('fallback');
|
||||
});
|
||||
});
|
||||
|
||||
describe('safeSync', () => {
|
||||
test('should return result of successful sync operation', () => {
|
||||
const successfulOperation = () => 'success';
|
||||
const result = safeSync(successfulOperation, 'test', 'Failed to process');
|
||||
expect(result).toBe('success');
|
||||
});
|
||||
|
||||
test('should handle sync errors and return fallback', () => {
|
||||
const failingOperation = () => {
|
||||
throw new Error('Operation failed');
|
||||
};
|
||||
const result = safeSync(failingOperation, 'test', 'Failed to process', 'fallback');
|
||||
expect(result).toBe('fallback');
|
||||
});
|
||||
|
||||
test('should return null by default on error', () => {
|
||||
const failingOperation = () => {
|
||||
throw new Error('Operation failed');
|
||||
};
|
||||
const result = safeSync(failingOperation, 'test', 'Failed to process');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('should handle non-Error objects thrown', () => {
|
||||
const failingOperation = () => {
|
||||
throw 'string error';
|
||||
};
|
||||
const result = safeSync(failingOperation, 'test', 'Failed to process', 'fallback');
|
||||
expect(result).toBe('fallback');
|
||||
});
|
||||
|
||||
test('should handle number errors', () => {
|
||||
const failingOperation = () => {
|
||||
throw 42;
|
||||
};
|
||||
const result = safeSync(failingOperation, 'test', 'Failed to process', 'fallback');
|
||||
expect(result).toBe('fallback');
|
||||
});
|
||||
});
|
||||
|
||||
describe('plugin', () => {
|
||||
test('should log plugin message', () => {
|
||||
plugin('test-plugin', 'Plugin loaded successfully');
|
||||
expect(console.log).toHaveBeenCalledWith('Plugin loaded successfully');
|
||||
});
|
||||
|
||||
test('should ignore plugin name parameter', () => {
|
||||
plugin('ignored-name', 'Test message');
|
||||
expect(console.log).toHaveBeenCalledWith('Test message');
|
||||
});
|
||||
});
|
||||
|
||||
describe('config', () => {
|
||||
test('should log config message for first occurrence', () => {
|
||||
config('database', 'Database configuration loaded');
|
||||
expect(console.log).toHaveBeenCalledWith('Config Database configuration loaded for database');
|
||||
});
|
||||
|
||||
test('should not log duplicate config messages', () => {
|
||||
config('test-config', 'Test configuration loaded');
|
||||
config('test-config', 'Test configuration loaded');
|
||||
expect(console.log).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('should log different config names', () => {
|
||||
config('database-unique1', 'Database config');
|
||||
config('server-unique1', 'Server config');
|
||||
expect(console.log).toHaveBeenCalledTimes(2);
|
||||
expect(console.log).toHaveBeenCalledWith('Config Database config for database-unique1');
|
||||
expect(console.log).toHaveBeenCalledWith('Config Server config for server-unique1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('db', () => {
|
||||
test('should log database message', () => {
|
||||
db('Database connection established');
|
||||
expect(console.log).toHaveBeenCalledWith('Database connection established');
|
||||
});
|
||||
});
|
||||
|
||||
describe('server', () => {
|
||||
test('should log server message', () => {
|
||||
server('Server started on port 3000');
|
||||
expect(console.log).toHaveBeenCalledWith('Server started on port 3000');
|
||||
});
|
||||
});
|
||||
|
||||
describe('section', () => {
|
||||
test('should log section header with uppercase title', () => {
|
||||
section('initialization');
|
||||
expect(console.log).toHaveBeenCalledWith('\n=== INITIALIZATION ===');
|
||||
});
|
||||
|
||||
test('should handle already uppercase titles', () => {
|
||||
section('ALREADY_UPPER');
|
||||
expect(console.log).toHaveBeenCalledWith('\n=== ALREADY_UPPER ===');
|
||||
});
|
||||
|
||||
test('should handle mixed case titles', () => {
|
||||
section('MiXeD_cAsE');
|
||||
expect(console.log).toHaveBeenCalledWith('\n=== MIXED_CASE ===');
|
||||
});
|
||||
});
|
||||
|
||||
describe('warn', () => {
|
||||
test('should log warning with prefix', () => {
|
||||
warn('security', 'Potential security issue detected');
|
||||
expect(console.warn).toHaveBeenCalledWith('WARNING: Potential security issue detected');
|
||||
});
|
||||
|
||||
test('should ignore category parameter', () => {
|
||||
warn('ignored-category', 'Warning message');
|
||||
expect(console.warn).toHaveBeenCalledWith('WARNING: Warning message');
|
||||
});
|
||||
});
|
||||
|
||||
describe('error', () => {
|
||||
test('should log error with prefix', () => {
|
||||
error('database', 'Failed to connect to database');
|
||||
expect(console.error).toHaveBeenCalledWith('ERROR: Failed to connect to database');
|
||||
});
|
||||
|
||||
test('should ignore category parameter', () => {
|
||||
error('ignored-category', 'Error message');
|
||||
expect(console.error).toHaveBeenCalledWith('ERROR: Error message');
|
||||
});
|
||||
});
|
||||
|
||||
describe('msg', () => {
|
||||
test('should log general message', () => {
|
||||
msg('General information message');
|
||||
expect(console.log).toHaveBeenCalledWith('General information message');
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
test('should handle empty messages', () => {
|
||||
msg('');
|
||||
warn('category', '');
|
||||
error('category', '');
|
||||
|
||||
expect(console.log).toHaveBeenCalledWith('');
|
||||
expect(console.warn).toHaveBeenCalledWith('WARNING: ');
|
||||
expect(console.error).toHaveBeenCalledWith('ERROR: ');
|
||||
});
|
||||
|
||||
test('should handle special characters in messages', () => {
|
||||
msg('Message with émojis 🚀 and spëcial chars!');
|
||||
expect(console.log).toHaveBeenCalledWith('Message with émojis 🚀 and spëcial chars!');
|
||||
});
|
||||
|
||||
test('should handle very long messages', () => {
|
||||
const longMessage = 'a'.repeat(1000);
|
||||
msg(longMessage);
|
||||
expect(console.log).toHaveBeenCalledWith(longMessage);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,426 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
|
||||
// Mock the logs module
|
||||
jest.unstable_mockModule('../dist/utils/logs.js', () => ({
|
||||
warn: jest.fn(),
|
||||
error: jest.fn()
|
||||
}));
|
||||
|
||||
// Import modules after mocking
|
||||
const { getRequestURL, getRealIP } = await import('../dist/utils/network.js');
|
||||
const logs = await import('../dist/utils/logs.js');
|
||||
|
||||
describe('Network utilities', () => {
|
||||
|
||||
describe('getRequestURL', () => {
|
||||
test('should handle http URLs', () => {
|
||||
const request = { url: 'http://example.com/path' };
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result).toBeInstanceOf(URL);
|
||||
expect(result.href).toBe('http://example.com/path');
|
||||
});
|
||||
|
||||
test('should handle https URLs', () => {
|
||||
const request = { url: 'https://example.com/path' };
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result).toBeInstanceOf(URL);
|
||||
expect(result.href).toBe('https://example.com/path');
|
||||
});
|
||||
|
||||
test('should construct URL from path and host header', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: { host: 'example.com' }
|
||||
};
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result).toBeInstanceOf(URL);
|
||||
expect(result.href).toBe('http://example.com/path');
|
||||
});
|
||||
|
||||
test('should use https when secure property is true', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
secure: true,
|
||||
headers: { host: 'example.com' }
|
||||
};
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result.href).toBe('https://example.com/path');
|
||||
});
|
||||
|
||||
test('should use https when x-forwarded-proto is https', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
host: 'example.com',
|
||||
'x-forwarded-proto': 'https'
|
||||
}
|
||||
};
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result.href).toBe('https://example.com/path');
|
||||
});
|
||||
|
||||
test('should handle Fetch-style headers with get() method', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
get: jest.fn((name) => {
|
||||
if (name === 'host') return 'example.com';
|
||||
return null;
|
||||
})
|
||||
}
|
||||
};
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result.href).toBe('http://example.com/path');
|
||||
});
|
||||
|
||||
test('should return null for missing URL', () => {
|
||||
const request = {};
|
||||
const result = getRequestURL(request);
|
||||
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
test('should return null for invalid URL', () => {
|
||||
const request = {
|
||||
url: '/test',
|
||||
headers: { host: 'localhost' }
|
||||
};
|
||||
|
||||
// Mock URL constructor to throw
|
||||
const originalURL = global.URL;
|
||||
global.URL = function() {
|
||||
throw new TypeError('Invalid URL');
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result).toBeNull();
|
||||
expect(logs.warn).toHaveBeenCalled();
|
||||
|
||||
// Restore
|
||||
global.URL = originalURL;
|
||||
});
|
||||
|
||||
test('should handle non-Error objects in catch block', () => {
|
||||
const request = {
|
||||
url: '/test',
|
||||
headers: { host: 'localhost' }
|
||||
};
|
||||
|
||||
// Mock URL constructor to throw non-Error
|
||||
const originalURL = global.URL;
|
||||
global.URL = function() {
|
||||
throw "String error instead of Error object";
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result).toBeNull();
|
||||
|
||||
// Restore
|
||||
global.URL = originalURL;
|
||||
});
|
||||
|
||||
test('should handle array host headers in Express requests', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: { host: ['example.com', 'backup.com'] } // Array host
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://example.com/path');
|
||||
});
|
||||
|
||||
test('should handle empty array host headers', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: { host: [] } // Empty array
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://localhost/path'); // Falls back to localhost
|
||||
});
|
||||
|
||||
test('should handle x-forwarded-host with Fetch-style headers', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
get: jest.fn((name) => {
|
||||
if (name === 'host') return null;
|
||||
if (name === 'x-forwarded-host') return 'forwarded.example.com';
|
||||
return null;
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://forwarded.example.com/path');
|
||||
});
|
||||
|
||||
test('should fallback to localhost for Fetch-style headers when all host headers are missing', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
get: jest.fn(() => null) // All headers return null
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://localhost/path');
|
||||
});
|
||||
|
||||
test('should handle x-forwarded-host in Express headers', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
'x-forwarded-host': 'forwarded.example.com'
|
||||
// No host header
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://forwarded.example.com/path');
|
||||
});
|
||||
|
||||
test('should handle array x-forwarded-host headers', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
'x-forwarded-host': ['forwarded.example.com', 'backup.example.com']
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://forwarded.example.com/path');
|
||||
});
|
||||
|
||||
test('should handle empty array x-forwarded-host headers', () => {
|
||||
const request = {
|
||||
url: '/path',
|
||||
headers: {
|
||||
'x-forwarded-host': []
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRequestURL(request);
|
||||
expect(result.href).toBe('http://localhost/path');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getRealIP', () => {
|
||||
test('should extract IP from x-forwarded-for header', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': '192.168.1.100' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should extract first IP from comma-separated x-forwarded-for', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': '192.168.1.100, 10.0.0.1, 127.0.0.1' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should use x-real-ip when x-forwarded-for is missing', () => {
|
||||
const request = {
|
||||
headers: { 'x-real-ip': '192.168.1.100' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should handle Fetch-style headers using get() method', () => {
|
||||
const request = {
|
||||
headers: {
|
||||
get: jest.fn((name) => {
|
||||
if (name === 'x-forwarded-for') return '192.168.1.100';
|
||||
return null;
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should use server remoteAddress when headers are empty', () => {
|
||||
const request = { headers: {} };
|
||||
const server = { remoteAddress: '192.168.1.100' };
|
||||
|
||||
const result = getRealIP(request, server);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should use connection.remoteAddress for Express requests', () => {
|
||||
const request = {
|
||||
headers: {},
|
||||
connection: { remoteAddress: '192.168.1.100' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should use req.ip property when available', () => {
|
||||
const request = {
|
||||
headers: {},
|
||||
ip: '192.168.1.100'
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should clean IPv6 mapped IPv4 addresses', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': '::ffff:192.168.1.100' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should return 127.0.0.1 as ultimate fallback', () => {
|
||||
const request = { headers: {} };
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('127.0.0.1');
|
||||
});
|
||||
|
||||
test('should prioritize x-forwarded-for over other sources', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': '192.168.1.100' },
|
||||
connection: { remoteAddress: '10.0.0.1' },
|
||||
ip: '172.16.0.1'
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should handle array x-forwarded-for headers', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': ['192.168.1.100', '10.0.0.1'] }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should handle array x-real-ip headers', () => {
|
||||
const request = {
|
||||
headers: { 'x-real-ip': ['192.168.1.100', '10.0.0.1'] }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should use x-real-ip with Fetch-style headers when x-forwarded-for is missing', () => {
|
||||
const request = {
|
||||
headers: {
|
||||
get: jest.fn((name) => {
|
||||
if (name === 'x-forwarded-for') return null;
|
||||
if (name === 'x-real-ip') return '192.168.1.100';
|
||||
return null;
|
||||
})
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should use socket.remoteAddress when connection.remoteAddress is missing', () => {
|
||||
const request = {
|
||||
headers: {},
|
||||
connection: {}, // connection exists but no remoteAddress
|
||||
socket: { remoteAddress: '192.168.1.100' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should handle missing connection but present socket', () => {
|
||||
const request = {
|
||||
headers: {},
|
||||
socket: { remoteAddress: '192.168.1.100' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
// Note: socket.remoteAddress is only checked within the connection block
|
||||
// When no connection exists, it falls back to URL hostname/127.0.0.1
|
||||
expect(result).toBe('127.0.0.1');
|
||||
});
|
||||
|
||||
test('should handle whitespace in comma-separated IPs', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': ' 192.168.1.100 , 10.0.0.1 , 127.0.0.1 ' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('192.168.1.100');
|
||||
});
|
||||
|
||||
test('should fallback to URL hostname when all IP sources fail', () => {
|
||||
const request = {
|
||||
headers: {},
|
||||
url: '/test'
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
// getRequestURL constructs URL with localhost as default host, so hostname is 'localhost'
|
||||
expect(result).toBe('localhost');
|
||||
});
|
||||
|
||||
test('should handle undefined IP values gracefully', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': undefined, 'x-real-ip': undefined }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('127.0.0.1');
|
||||
});
|
||||
|
||||
test('should handle null IP values gracefully', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': null, 'x-real-ip': null }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('127.0.0.1');
|
||||
});
|
||||
|
||||
test('should handle empty string IP values', () => {
|
||||
const request = {
|
||||
headers: { 'x-forwarded-for': '', 'x-real-ip': '' }
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('127.0.0.1');
|
||||
});
|
||||
|
||||
test('should handle empty array header values', () => {
|
||||
const request = {
|
||||
headers: {
|
||||
'x-forwarded-for': [],
|
||||
'x-real-ip': []
|
||||
}
|
||||
};
|
||||
|
||||
const result = getRealIP(request);
|
||||
expect(result).toBe('127.0.0.1');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,719 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
import {
|
||||
LRUCache,
|
||||
RateLimiter,
|
||||
ObjectPool,
|
||||
BatchProcessor,
|
||||
debounce,
|
||||
throttle,
|
||||
memoize,
|
||||
StringMatcher,
|
||||
ConnectionPool
|
||||
} from '../dist/utils/performance.js';
|
||||
|
||||
describe('Performance utilities', () => {
|
||||
describe('LRUCache', () => {
|
||||
test('should store and retrieve values', () => {
|
||||
const cache = new LRUCache(3);
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
cache.set('key2', 'value2');
|
||||
|
||||
expect(cache.get('key1')).toBe('value1');
|
||||
expect(cache.get('key2')).toBe('value2');
|
||||
expect(cache.get('nonexistent')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('should evict least recently used items when at capacity', () => {
|
||||
const cache = new LRUCache(2);
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
cache.set('key2', 'value2');
|
||||
cache.set('key3', 'value3'); // Should evict key1
|
||||
|
||||
expect(cache.get('key1')).toBeUndefined();
|
||||
expect(cache.get('key2')).toBe('value2');
|
||||
expect(cache.get('key3')).toBe('value3');
|
||||
});
|
||||
|
||||
test('should handle TTL expiration', () => {
|
||||
jest.useFakeTimers();
|
||||
const cache = new LRUCache(10, 100); // 100ms TTL
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
expect(cache.get('key1')).toBe('value1');
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
expect(cache.get('key1')).toBeUndefined();
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should delete and clear items', () => {
|
||||
const cache = new LRUCache(5);
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
cache.set('key2', 'value2');
|
||||
|
||||
expect(cache.delete('key1')).toBe(true);
|
||||
expect(cache.get('key1')).toBeUndefined();
|
||||
|
||||
cache.clear();
|
||||
expect(cache.size).toBe(0);
|
||||
});
|
||||
|
||||
test('should clean up expired entries with cleanup method', () => {
|
||||
jest.useFakeTimers();
|
||||
const cache = new LRUCache(10, 100); // 100ms TTL
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
cache.set('key2', 'value2');
|
||||
cache.set('key3', 'value3');
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
|
||||
const cleaned = cache.cleanup();
|
||||
expect(cleaned).toBe(3);
|
||||
expect(cache.size).toBe(0);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should handle has() method with TTL expiration', () => {
|
||||
jest.useFakeTimers();
|
||||
const cache = new LRUCache(10, 100); // 100ms TTL
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
expect(cache.has('key1')).toBe(true);
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
expect(cache.has('key1')).toBe(false);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should cleanup without TTL should return 0', () => {
|
||||
const cache = new LRUCache(10); // No TTL
|
||||
|
||||
cache.set('key1', 'value1');
|
||||
cache.set('key2', 'value2');
|
||||
|
||||
const cleaned = cache.cleanup();
|
||||
expect(cleaned).toBe(0);
|
||||
expect(cache.size).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('RateLimiter', () => {
|
||||
let limiterInstances = [];
|
||||
|
||||
afterEach(() => {
|
||||
// Clean up instances to prevent Jest hanging
|
||||
limiterInstances.forEach(limiter => {
|
||||
if (limiter && typeof limiter.destroy === 'function') {
|
||||
limiter.destroy();
|
||||
}
|
||||
});
|
||||
limiterInstances = [];
|
||||
});
|
||||
|
||||
test('should allow requests within limit', () => {
|
||||
const limiter = new RateLimiter(1000, 2);
|
||||
limiterInstances.push(limiter);
|
||||
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
expect(limiter.isAllowed('user1')).toBe(false);
|
||||
});
|
||||
|
||||
test('should reset after window expires', () => {
|
||||
jest.useFakeTimers();
|
||||
const limiter = new RateLimiter(100, 2);
|
||||
limiterInstances.push(limiter);
|
||||
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
expect(limiter.isAllowed('user1')).toBe(false);
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
});
|
||||
|
||||
test('should track different identifiers separately', () => {
|
||||
const limiter = new RateLimiter(1000, 2);
|
||||
limiterInstances.push(limiter);
|
||||
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
expect(limiter.isAllowed('user1')).toBe(false);
|
||||
|
||||
expect(limiter.isAllowed('user2')).toBe(true);
|
||||
expect(limiter.isAllowed('user2')).toBe(true);
|
||||
expect(limiter.isAllowed('user2')).toBe(false);
|
||||
});
|
||||
|
||||
test('should clean up expired entries manually', () => {
|
||||
jest.useFakeTimers();
|
||||
const limiter = new RateLimiter(100, 2);
|
||||
limiterInstances.push(limiter);
|
||||
|
||||
limiter.isAllowed('user1');
|
||||
limiter.isAllowed('user2');
|
||||
limiter.isAllowed('user3');
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
|
||||
const cleaned = limiter.cleanup();
|
||||
expect(cleaned).toBeGreaterThan(0);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should automatically clean up on interval', () => {
|
||||
jest.useFakeTimers();
|
||||
const limiter = new RateLimiter(100, 2);
|
||||
limiterInstances.push(limiter);
|
||||
|
||||
limiter.isAllowed('user1');
|
||||
limiter.isAllowed('user2');
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
|
||||
// Trigger auto-cleanup (runs every 60 seconds)
|
||||
jest.advanceTimersByTime(60000);
|
||||
|
||||
// Should still work after cleanup
|
||||
expect(limiter.isAllowed('user1')).toBe(true);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should handle cleanup of identifiers with partial expired requests', () => {
|
||||
jest.useFakeTimers();
|
||||
const limiter = new RateLimiter(200, 3);
|
||||
limiterInstances.push(limiter);
|
||||
|
||||
// Make some requests
|
||||
limiter.isAllowed('user1');
|
||||
jest.advanceTimersByTime(100);
|
||||
limiter.isAllowed('user1');
|
||||
jest.advanceTimersByTime(150); // First request now expired, second still valid
|
||||
|
||||
const cleaned = limiter.cleanup();
|
||||
expect(cleaned).toBe(0); // user1 still has valid requests, not removed
|
||||
|
||||
// Advance further to expire all requests
|
||||
jest.advanceTimersByTime(100);
|
||||
const cleaned2 = limiter.cleanup();
|
||||
expect(cleaned2).toBe(1); // user1 removed
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
||||
describe('ObjectPool', () => {
|
||||
test('should create and reuse objects', () => {
|
||||
let created = 0;
|
||||
const factory = () => ({ id: ++created });
|
||||
const reset = (obj) => { obj.used = false; };
|
||||
|
||||
const pool = new ObjectPool(factory, reset);
|
||||
|
||||
const obj1 = pool.acquire();
|
||||
expect(obj1.id).toBe(1);
|
||||
|
||||
pool.release(obj1);
|
||||
|
||||
const obj2 = pool.acquire();
|
||||
expect(obj2.id).toBe(1); // Reused
|
||||
expect(obj2.used).toBe(false); // Reset was called
|
||||
});
|
||||
|
||||
test('should create new objects when pool is empty', () => {
|
||||
let created = 0;
|
||||
const factory = () => ({ id: ++created });
|
||||
const reset = () => {};
|
||||
|
||||
const pool = new ObjectPool(factory, reset);
|
||||
|
||||
const obj1 = pool.acquire();
|
||||
const obj2 = pool.acquire();
|
||||
|
||||
expect(obj1.id).toBe(1);
|
||||
expect(obj2.id).toBe(2);
|
||||
});
|
||||
|
||||
test('should not exceed max size', () => {
|
||||
const factory = () => ({});
|
||||
const reset = () => {};
|
||||
|
||||
const pool = new ObjectPool(factory, reset, 2);
|
||||
|
||||
const obj1 = pool.acquire();
|
||||
const obj2 = pool.acquire();
|
||||
const obj3 = pool.acquire();
|
||||
|
||||
pool.release(obj1);
|
||||
pool.release(obj2);
|
||||
pool.release(obj3);
|
||||
|
||||
const stats = pool.size;
|
||||
expect(stats.available).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
test('should ignore release of objects not in use', () => {
|
||||
const factory = () => ({ id: Math.random() });
|
||||
const reset = () => {};
|
||||
|
||||
const pool = new ObjectPool(factory, reset);
|
||||
const strangerObj = factory();
|
||||
|
||||
// Should not throw or affect pool
|
||||
pool.release(strangerObj);
|
||||
|
||||
expect(pool.size.available).toBe(0);
|
||||
expect(pool.size.inUse).toBe(0);
|
||||
});
|
||||
|
||||
test('should clear all objects from pool', () => {
|
||||
const factory = () => ({ id: Math.random() });
|
||||
const reset = () => {};
|
||||
|
||||
const pool = new ObjectPool(factory, reset);
|
||||
|
||||
const obj1 = pool.acquire();
|
||||
const obj2 = pool.acquire();
|
||||
pool.release(obj1);
|
||||
|
||||
pool.clear();
|
||||
|
||||
const stats = pool.size;
|
||||
expect(stats.available).toBe(0);
|
||||
expect(stats.inUse).toBe(0);
|
||||
expect(stats.total).toBe(0);
|
||||
});
|
||||
|
||||
test('should provide accurate size statistics', () => {
|
||||
const factory = () => ({ id: Math.random() });
|
||||
const reset = () => {};
|
||||
|
||||
const pool = new ObjectPool(factory, reset);
|
||||
|
||||
const obj1 = pool.acquire();
|
||||
const obj2 = pool.acquire();
|
||||
pool.release(obj1);
|
||||
|
||||
const stats = pool.size;
|
||||
expect(stats.available).toBe(1);
|
||||
expect(stats.inUse).toBe(1);
|
||||
expect(stats.total).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('BatchProcessor', () => {
|
||||
let batcherInstances = [];
|
||||
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
// Clean up any batcher instances to prevent memory leaks
|
||||
batcherInstances.forEach(batcher => {
|
||||
if (batcher && typeof batcher.destroy === 'function') {
|
||||
batcher.destroy();
|
||||
}
|
||||
});
|
||||
batcherInstances = [];
|
||||
});
|
||||
|
||||
test('should process batch when size is reached', async () => {
|
||||
const processor = jest.fn();
|
||||
const batcher = new BatchProcessor(processor, { batchSize: 3 });
|
||||
batcherInstances.push(batcher);
|
||||
|
||||
batcher.add('item1');
|
||||
batcher.add('item2');
|
||||
expect(processor).not.toHaveBeenCalled();
|
||||
|
||||
batcher.add('item3');
|
||||
await Promise.resolve(); // Let async processing complete
|
||||
|
||||
expect(processor).toHaveBeenCalledWith(['item1', 'item2', 'item3']);
|
||||
});
|
||||
|
||||
test('should auto-flush on interval', async () => {
|
||||
const processor = jest.fn();
|
||||
const batcher = new BatchProcessor(processor, {
|
||||
batchSize: 10,
|
||||
flushInterval: 100
|
||||
});
|
||||
batcherInstances.push(batcher);
|
||||
|
||||
batcher.add('item1');
|
||||
batcher.add('item2');
|
||||
|
||||
jest.advanceTimersByTime(100);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(processor).toHaveBeenCalledWith(['item1', 'item2']);
|
||||
});
|
||||
|
||||
test('should handle processing errors', async () => {
|
||||
const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation();
|
||||
const processor = jest.fn().mockRejectedValue(new Error('Process error'));
|
||||
const batcher = new BatchProcessor(processor, { batchSize: 1 });
|
||||
batcherInstances.push(batcher);
|
||||
|
||||
batcher.add('item1');
|
||||
await Promise.resolve();
|
||||
|
||||
expect(consoleErrorSpy).toHaveBeenCalledWith('Batch processing error:', expect.any(Error));
|
||||
consoleErrorSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('should not flush when already processing', async () => {
|
||||
let resolveProcessor;
|
||||
const processor = jest.fn(() => new Promise(resolve => {
|
||||
resolveProcessor = resolve;
|
||||
}));
|
||||
|
||||
const batcher = new BatchProcessor(processor, { batchSize: 2 });
|
||||
batcherInstances.push(batcher);
|
||||
|
||||
batcher.add('item1');
|
||||
batcher.add('item2'); // Triggers flush
|
||||
|
||||
// Add more items while first batch is processing
|
||||
batcher.add('item3');
|
||||
batcher.flush(); // Should return early
|
||||
|
||||
expect(processor).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Resolve the first batch
|
||||
resolveProcessor();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(processor).toHaveBeenCalledWith(['item1', 'item2']);
|
||||
});
|
||||
|
||||
test('should not flush empty queue', async () => {
|
||||
const processor = jest.fn();
|
||||
const batcher = new BatchProcessor(processor, { batchSize: 5 });
|
||||
batcherInstances.push(batcher);
|
||||
|
||||
await batcher.flush();
|
||||
|
||||
expect(processor).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('debounce', () => {
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should delay function execution', () => {
|
||||
const func = jest.fn();
|
||||
const debounced = debounce(func, 100);
|
||||
|
||||
debounced('arg1');
|
||||
debounced('arg2');
|
||||
debounced('arg3');
|
||||
|
||||
expect(func).not.toHaveBeenCalled();
|
||||
|
||||
jest.advanceTimersByTime(100);
|
||||
|
||||
expect(func).toHaveBeenCalledTimes(1);
|
||||
expect(func).toHaveBeenCalledWith('arg3');
|
||||
});
|
||||
});
|
||||
|
||||
describe('throttle', () => {
|
||||
beforeEach(() => {
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should limit function execution rate', () => {
|
||||
const func = jest.fn();
|
||||
const throttled = throttle(func, 100);
|
||||
|
||||
throttled('arg1');
|
||||
throttled('arg2');
|
||||
throttled('arg3');
|
||||
|
||||
expect(func).toHaveBeenCalledTimes(1);
|
||||
expect(func).toHaveBeenCalledWith('arg1');
|
||||
|
||||
jest.advanceTimersByTime(100);
|
||||
|
||||
throttled('arg4');
|
||||
expect(func).toHaveBeenCalledTimes(2);
|
||||
expect(func).toHaveBeenCalledWith('arg4');
|
||||
});
|
||||
});
|
||||
|
||||
describe('memoize', () => {
|
||||
test('should cache function results', () => {
|
||||
const func = jest.fn((a, b) => a + b);
|
||||
const memoized = memoize(func);
|
||||
|
||||
expect(memoized(1, 2)).toBe(3);
|
||||
expect(memoized(1, 2)).toBe(3);
|
||||
expect(memoized(1, 2)).toBe(3);
|
||||
|
||||
expect(func).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('should handle different arguments', () => {
|
||||
const func = jest.fn((a, b) => a + b);
|
||||
const memoized = memoize(func);
|
||||
|
||||
expect(memoized(1, 2)).toBe(3);
|
||||
expect(memoized(2, 3)).toBe(5);
|
||||
|
||||
expect(func).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test('should respect TTL option', async () => {
|
||||
jest.useFakeTimers();
|
||||
const func = jest.fn((a) => a * 2);
|
||||
const memoized = memoize(func, { ttl: 100 });
|
||||
|
||||
expect(memoized(5)).toBe(10);
|
||||
expect(func).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(memoized(5)).toBe(10);
|
||||
expect(func).toHaveBeenCalledTimes(1);
|
||||
|
||||
jest.advanceTimersByTime(150);
|
||||
|
||||
expect(memoized(5)).toBe(10);
|
||||
expect(func).toHaveBeenCalledTimes(2);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should handle undefined cached values correctly', () => {
|
||||
const func = jest.fn(() => undefined);
|
||||
const memoized = memoize(func);
|
||||
|
||||
expect(memoized('test')).toBeUndefined();
|
||||
expect(memoized('test')).toBeUndefined();
|
||||
|
||||
// Function should be called twice since undefined is returned
|
||||
expect(func).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test('should handle functions returning falsy values', () => {
|
||||
const func = jest.fn((x) => x === 'zero' ? 0 : x === 'false' ? false : x === 'null' ? null : 'default');
|
||||
const memoized = memoize(func);
|
||||
|
||||
expect(memoized('zero')).toBe(0);
|
||||
expect(memoized('zero')).toBe(0); // Should be cached
|
||||
expect(func).toHaveBeenCalledTimes(1);
|
||||
|
||||
expect(memoized('false')).toBe(false);
|
||||
expect(memoized('false')).toBe(false); // Should be cached
|
||||
expect(func).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(memoized('null')).toBe(null);
|
||||
expect(memoized('null')).toBe(null); // Should be cached
|
||||
expect(func).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StringMatcher', () => {
|
||||
test('should match strings case-insensitively', () => {
|
||||
const matcher = new StringMatcher(['apple', 'BANANA', 'Cherry']);
|
||||
|
||||
expect(matcher.contains('apple')).toBe(true);
|
||||
expect(matcher.contains('APPLE')).toBe(true);
|
||||
expect(matcher.contains('banana')).toBe(true);
|
||||
expect(matcher.contains('cherry')).toBe(true);
|
||||
expect(matcher.contains('grape')).toBe(false);
|
||||
});
|
||||
|
||||
test('should add and remove patterns', () => {
|
||||
const matcher = new StringMatcher(['apple']);
|
||||
|
||||
matcher.add('banana');
|
||||
expect(matcher.contains('banana')).toBe(true);
|
||||
expect(matcher.size).toBe(2);
|
||||
|
||||
expect(matcher.remove('apple')).toBe(true);
|
||||
expect(matcher.contains('apple')).toBe(false);
|
||||
expect(matcher.size).toBe(1);
|
||||
});
|
||||
|
||||
test('should check if any text matches with containsAny', () => {
|
||||
const matcher = new StringMatcher(['apple', 'banana', 'cherry']);
|
||||
|
||||
expect(matcher.containsAny(['grape', 'orange'])).toBe(false);
|
||||
expect(matcher.containsAny(['grape', 'apple'])).toBe(true);
|
||||
expect(matcher.containsAny(['BANANA', 'orange'])).toBe(true);
|
||||
expect(matcher.containsAny([])).toBe(false);
|
||||
expect(matcher.containsAny(['cherry', 'apple', 'banana'])).toBe(true);
|
||||
});
|
||||
|
||||
test('should handle empty patterns', () => {
|
||||
const matcher = new StringMatcher([]);
|
||||
|
||||
expect(matcher.contains('anything')).toBe(false);
|
||||
expect(matcher.containsAny(['test', 'values'])).toBe(false);
|
||||
expect(matcher.size).toBe(0);
|
||||
});
|
||||
|
||||
test('should remove non-existent patterns gracefully', () => {
|
||||
const matcher = new StringMatcher(['apple']);
|
||||
|
||||
expect(matcher.remove('banana')).toBe(false);
|
||||
expect(matcher.size).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ConnectionPool', () => {
|
||||
test('should create and reuse connections', () => {
|
||||
const pool = new ConnectionPool({ maxConnections: 5 });
|
||||
|
||||
const conn1 = pool.getConnection('host1');
|
||||
expect(conn1).not.toBeNull();
|
||||
expect(conn1.host).toBe('host1');
|
||||
|
||||
pool.releaseConnection('host1', conn1);
|
||||
|
||||
const conn2 = pool.getConnection('host1');
|
||||
expect(conn2).toBe(conn1); // Reused
|
||||
});
|
||||
|
||||
test('should respect max connections limit', () => {
|
||||
const pool = new ConnectionPool({ maxConnections: 2 });
|
||||
|
||||
const conn1 = pool.getConnection('host1');
|
||||
const conn2 = pool.getConnection('host1');
|
||||
const conn3 = pool.getConnection('host1');
|
||||
|
||||
expect(conn1).not.toBeNull();
|
||||
expect(conn2).not.toBeNull();
|
||||
expect(conn3).toBeNull(); // Pool exhausted
|
||||
});
|
||||
|
||||
test('should create separate pools for different hosts', () => {
|
||||
const pool = new ConnectionPool({ maxConnections: 2 });
|
||||
|
||||
const conn1 = pool.getConnection('host1');
|
||||
const conn2 = pool.getConnection('host2');
|
||||
|
||||
expect(conn1).not.toBeNull();
|
||||
expect(conn2).not.toBeNull();
|
||||
expect(conn1.host).toBe('host1');
|
||||
expect(conn2.host).toBe('host2');
|
||||
});
|
||||
|
||||
test('should handle release of non-existent connections gracefully', () => {
|
||||
const pool = new ConnectionPool({ maxConnections: 5 });
|
||||
|
||||
const fakeConn = { host: 'fake', created: Date.now() };
|
||||
|
||||
// Should not throw
|
||||
pool.releaseConnection('host1', fakeConn);
|
||||
pool.releaseConnection('nonexistent', fakeConn);
|
||||
});
|
||||
|
||||
test('should close connections when pool is over half capacity', () => {
|
||||
const pool = new ConnectionPool({ maxConnections: 4 });
|
||||
|
||||
// Spy on closeConnection method
|
||||
const closeConnectionSpy = jest.spyOn(pool, 'closeConnection');
|
||||
|
||||
// Fill pool
|
||||
const conn1 = pool.getConnection('host1');
|
||||
const conn2 = pool.getConnection('host1');
|
||||
const conn3 = pool.getConnection('host1');
|
||||
|
||||
// Release connections
|
||||
pool.releaseConnection('host1', conn1); // Should keep (pool size 1)
|
||||
pool.releaseConnection('host1', conn2); // Should keep (pool size 2 = maxConnections/2)
|
||||
pool.releaseConnection('host1', conn3); // Should close (pool would exceed half capacity)
|
||||
|
||||
expect(closeConnectionSpy).toHaveBeenCalledTimes(1);
|
||||
expect(closeConnectionSpy).toHaveBeenCalledWith(conn3);
|
||||
|
||||
closeConnectionSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('should provide access to connectionTimeout property', () => {
|
||||
const pool = new ConnectionPool({ timeout: 5000 });
|
||||
|
||||
expect(pool.connectionTimeout).toBe(5000);
|
||||
});
|
||||
|
||||
test('should destroy all connections when destroyed', () => {
|
||||
const pool = new ConnectionPool({ maxConnections: 3 });
|
||||
|
||||
// Spy on closeConnection method
|
||||
const closeConnectionSpy = jest.spyOn(pool, 'closeConnection');
|
||||
|
||||
// Create some connections
|
||||
const conn1 = pool.getConnection('host1');
|
||||
const conn2 = pool.getConnection('host1');
|
||||
const conn3 = pool.getConnection('host2');
|
||||
|
||||
// Release one connection back to pool
|
||||
pool.releaseConnection('host1', conn1);
|
||||
|
||||
// Destroy pool
|
||||
pool.destroy();
|
||||
|
||||
// Should close all connections (1 in pool + 2 in use)
|
||||
expect(closeConnectionSpy).toHaveBeenCalledTimes(3);
|
||||
expect(closeConnectionSpy).toHaveBeenCalledWith(conn1);
|
||||
expect(closeConnectionSpy).toHaveBeenCalledWith(conn2);
|
||||
expect(closeConnectionSpy).toHaveBeenCalledWith(conn3);
|
||||
|
||||
closeConnectionSpy.mockRestore();
|
||||
});
|
||||
|
||||
test('should create connections with timestamp', () => {
|
||||
const pool = new ConnectionPool();
|
||||
|
||||
const before = Date.now();
|
||||
const conn = pool.getConnection('host1');
|
||||
const after = Date.now();
|
||||
|
||||
expect(conn.created).toBeGreaterThanOrEqual(before);
|
||||
expect(conn.created).toBeLessThanOrEqual(after);
|
||||
});
|
||||
|
||||
test('should use default maxConnections and timeout values', () => {
|
||||
const pool = new ConnectionPool();
|
||||
|
||||
expect(pool.connectionTimeout).toBe(30000); // Default timeout
|
||||
|
||||
// Create connections up to default limit (50)
|
||||
const connections = [];
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const conn = pool.getConnection('host1');
|
||||
if (conn) connections.push(conn);
|
||||
}
|
||||
|
||||
expect(connections).toHaveLength(50);
|
||||
|
||||
// 51st connection should be null
|
||||
const extraConn = pool.getConnection('host1');
|
||||
expect(extraConn).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -1,264 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
|
||||
// Mock the path and url modules for testing
|
||||
jest.unstable_mockModule('path', () => ({
|
||||
resolve: jest.fn(),
|
||||
extname: jest.fn(),
|
||||
sep: '/',
|
||||
isAbsolute: jest.fn(),
|
||||
normalize: jest.fn()
|
||||
}));
|
||||
|
||||
jest.unstable_mockModule('url', () => ({
|
||||
pathToFileURL: jest.fn()
|
||||
}));
|
||||
|
||||
// Mock the root directory
|
||||
jest.unstable_mockModule('../dist/index.js', () => ({
|
||||
rootDir: '/app/root'
|
||||
}));
|
||||
|
||||
// Import after mocking
|
||||
const { secureImportModule, hasExport, getExport } = await import('../dist/utils/plugins.js');
|
||||
const path = await import('path');
|
||||
const url = await import('url');
|
||||
|
||||
describe('Plugins utilities', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Default mock implementations
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
path.resolve.mockImplementation((root, rel) => `${root}/${rel}`);
|
||||
path.extname.mockImplementation((p) => {
|
||||
const parts = p.split('.');
|
||||
return parts.length > 1 ? `.${parts[parts.length - 1]}` : '';
|
||||
});
|
||||
path.isAbsolute.mockImplementation((p) => p.startsWith('/'));
|
||||
url.pathToFileURL.mockImplementation((p) => ({ href: `file://${p}` }));
|
||||
});
|
||||
|
||||
describe('secureImportModule', () => {
|
||||
describe('input validation', () => {
|
||||
test('should reject non-string module paths', async () => {
|
||||
await expect(secureImportModule(null)).rejects.toThrow('Module path must be a string');
|
||||
await expect(secureImportModule(undefined)).rejects.toThrow('Module path must be a string');
|
||||
await expect(secureImportModule(123)).rejects.toThrow('Module path must be a string');
|
||||
});
|
||||
|
||||
test('should reject empty module paths', async () => {
|
||||
await expect(secureImportModule('')).rejects.toThrow('Module path cannot be empty');
|
||||
});
|
||||
|
||||
test('should reject paths that are too long', async () => {
|
||||
const longPath = 'a'.repeat(1025) + '.js';
|
||||
await expect(secureImportModule(longPath)).rejects.toThrow('Module path too long');
|
||||
});
|
||||
});
|
||||
|
||||
describe('security pattern validation', () => {
|
||||
test('should reject directory traversal attempts', async () => {
|
||||
await expect(secureImportModule('../evil.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('../../system.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('folder/../escape.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
});
|
||||
|
||||
test('should reject double slash patterns', async () => {
|
||||
await expect(secureImportModule('folder//file.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('//root.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
});
|
||||
|
||||
test('should reject null byte injections', async () => {
|
||||
await expect(secureImportModule('file\0.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
});
|
||||
|
||||
test('should reject control characters', async () => {
|
||||
await expect(secureImportModule('file\x01.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('file\x1f.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
});
|
||||
|
||||
test('should reject node_modules access', async () => {
|
||||
await expect(secureImportModule('node_modules/evil.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('NODE_MODULES/evil.js')).rejects.toThrow('Module path contains blocked pattern');
|
||||
});
|
||||
|
||||
test('should reject sensitive file access', async () => {
|
||||
await expect(secureImportModule('package.json')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('.env')).rejects.toThrow('Module path contains blocked pattern');
|
||||
await expect(secureImportModule('config/.ENV')).rejects.toThrow('Module path contains blocked pattern');
|
||||
});
|
||||
|
||||
test('should reject paths that are too deep', async () => {
|
||||
const deepPath = 'a/'.repeat(25) + 'file.js';
|
||||
path.normalize.mockReturnValue(deepPath);
|
||||
|
||||
await expect(secureImportModule(deepPath)).rejects.toThrow('Module path too deep');
|
||||
});
|
||||
});
|
||||
|
||||
describe('file extension validation', () => {
|
||||
test('should reject invalid file extensions', async () => {
|
||||
path.extname.mockReturnValue('.txt');
|
||||
|
||||
await expect(secureImportModule('file.txt')).rejects.toThrow('Only .js, .mjs files can be imported');
|
||||
});
|
||||
|
||||
test('should reject files without extensions', async () => {
|
||||
path.extname.mockReturnValue('');
|
||||
|
||||
await expect(secureImportModule('file')).rejects.toThrow('Only .js, .mjs files can be imported');
|
||||
});
|
||||
|
||||
test('should accept valid .js extension validation', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockReturnValue('/app/root/valid.js');
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
|
||||
// This will fail at import but pass extension validation
|
||||
await expect(secureImportModule('valid.js')).rejects.toThrow('Failed to import module');
|
||||
});
|
||||
|
||||
test('should accept valid .mjs extension validation', async () => {
|
||||
path.extname.mockReturnValue('.mjs');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockReturnValue('/app/root/valid.mjs');
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
|
||||
// This will fail at import but pass extension validation
|
||||
await expect(secureImportModule('valid.mjs')).rejects.toThrow('Failed to import module');
|
||||
});
|
||||
});
|
||||
|
||||
describe('path resolution security', () => {
|
||||
test('should reject absolute paths', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(true);
|
||||
|
||||
await expect(secureImportModule('/absolute/path.js')).rejects.toThrow('Absolute paths are not allowed');
|
||||
});
|
||||
|
||||
test('should reject paths outside application root', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockReturnValue('/outside/root/file.js');
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
|
||||
await expect(secureImportModule('file.js')).rejects.toThrow('Module path outside of application root');
|
||||
});
|
||||
|
||||
test('should detect symbolic link traversal', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockReturnValue('/app/root/../outside.js');
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
|
||||
await expect(secureImportModule('file.js')).rejects.toThrow('Path traversal detected');
|
||||
});
|
||||
|
||||
test('should validate paths within application root', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockReturnValue('/app/root/plugins/test.js');
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
|
||||
// This will fail at import but pass path validation
|
||||
await expect(secureImportModule('plugins/test.js')).rejects.toThrow('Failed to import module');
|
||||
});
|
||||
});
|
||||
|
||||
describe('import operation and error handling', () => {
|
||||
test('should handle import failures gracefully', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockReturnValue('/app/root/file.js');
|
||||
path.normalize.mockImplementation((p) => p);
|
||||
|
||||
// Since we can't easily mock import() in ES modules,
|
||||
// this will naturally fail and test our error handling
|
||||
await expect(secureImportModule('file.js')).rejects.toThrow('Failed to import module');
|
||||
});
|
||||
|
||||
test('should handle non-Error exceptions in path validation', async () => {
|
||||
path.extname.mockReturnValue('.js');
|
||||
path.isAbsolute.mockReturnValue(false);
|
||||
path.resolve.mockImplementation(() => {
|
||||
throw "Non-error exception";
|
||||
});
|
||||
|
||||
await expect(secureImportModule('file.js')).rejects.toThrow('Module import failed');
|
||||
});
|
||||
|
||||
test('should handle unknown errors gracefully', async () => {
|
||||
path.extname.mockImplementation(() => {
|
||||
throw { unknown: 'error' };
|
||||
});
|
||||
|
||||
await expect(secureImportModule('file.js')).rejects.toThrow('Module import failed due to unknown error');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('hasExport', () => {
|
||||
test('should return true for existing exports', () => {
|
||||
const module = { testFunction: () => {}, testValue: 'test' };
|
||||
|
||||
expect(hasExport(module, 'testFunction')).toBe(true);
|
||||
expect(hasExport(module, 'testValue')).toBe(true);
|
||||
});
|
||||
|
||||
test('should return false for non-existent exports', () => {
|
||||
const module = { testFunction: () => {} };
|
||||
|
||||
expect(hasExport(module, 'nonExistent')).toBe(false);
|
||||
});
|
||||
|
||||
test('should return false for undefined exports', () => {
|
||||
const module = { testValue: undefined };
|
||||
|
||||
expect(hasExport(module, 'testValue')).toBe(false);
|
||||
});
|
||||
|
||||
test('should handle edge cases', () => {
|
||||
const module = {};
|
||||
|
||||
expect(hasExport(module, 'toString')).toBe(true); // inherited from Object.prototype
|
||||
expect(hasExport(module, 'constructor')).toBe(true); // inherited from Object.prototype
|
||||
expect(hasExport(module, 'nonExistentMethod')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getExport', () => {
|
||||
test('should return export values correctly', () => {
|
||||
const testFunction = () => 'test';
|
||||
const module = {
|
||||
testFunction,
|
||||
testValue: 'hello'
|
||||
};
|
||||
|
||||
expect(getExport(module, 'testFunction')).toBe(testFunction);
|
||||
expect(getExport(module, 'testValue')).toBe('hello');
|
||||
});
|
||||
|
||||
test('should return undefined for non-existent exports', () => {
|
||||
const module = { testFunction: () => {} };
|
||||
|
||||
expect(getExport(module, 'nonExistent')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('should return actual values including falsy ones', () => {
|
||||
const module = {
|
||||
zero: 0,
|
||||
false: false,
|
||||
empty: '',
|
||||
null: null
|
||||
};
|
||||
|
||||
expect(getExport(module, 'zero')).toBe(0);
|
||||
expect(getExport(module, 'false')).toBe(false);
|
||||
expect(getExport(module, 'empty')).toBe('');
|
||||
expect(getExport(module, 'null')).toBe(null);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,533 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
import * as crypto from 'crypto';
|
||||
import {
|
||||
generateChallenge,
|
||||
calculateHash,
|
||||
verifyPoW,
|
||||
checkPoSTimes,
|
||||
generateRequestID,
|
||||
getChallengeParams,
|
||||
deleteChallenge,
|
||||
verifyPoS,
|
||||
challengeStore
|
||||
} from '../dist/utils/proof.js';
|
||||
|
||||
describe('Proof utilities', () => {
|
||||
beforeEach(() => {
|
||||
challengeStore.clear();
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('generateChallenge', () => {
|
||||
test('should generate challenge with valid config', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const result = generateChallenge(config);
|
||||
|
||||
expect(result).toHaveProperty('challenge');
|
||||
expect(result).toHaveProperty('salt');
|
||||
expect(typeof result.challenge).toBe('string');
|
||||
expect(typeof result.salt).toBe('string');
|
||||
expect(result.challenge.length).toBe(32); // 16 bytes as hex
|
||||
expect(result.salt.length).toBe(32); // 16 bytes as hex
|
||||
});
|
||||
|
||||
test('should respect salt length configuration', () => {
|
||||
const config = { SaltLength: 32, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const result = generateChallenge(config);
|
||||
|
||||
expect(result.salt.length).toBe(64); // 32 bytes as hex
|
||||
});
|
||||
|
||||
test('should throw error for invalid config', () => {
|
||||
expect(() => generateChallenge(null)).toThrow('CheckpointConfig must be an object');
|
||||
expect(() => generateChallenge(undefined)).toThrow('CheckpointConfig must be an object');
|
||||
expect(() => generateChallenge({})).toThrow();
|
||||
});
|
||||
|
||||
test('should validate configuration bounds', () => {
|
||||
// Salt length validation
|
||||
expect(() => generateChallenge({ SaltLength: 0, Difficulty: 4, ChallengeExpiration: 300000 }))
|
||||
.toThrow('SaltLength must be between');
|
||||
|
||||
// Difficulty validation
|
||||
expect(() => generateChallenge({ SaltLength: 16, Difficulty: 0, ChallengeExpiration: 300000 }))
|
||||
.toThrow('Difficulty must be between');
|
||||
|
||||
// Expiration validation
|
||||
expect(() => generateChallenge({ SaltLength: 16, Difficulty: 4, ChallengeExpiration: 0 }))
|
||||
.toThrow('ChallengeExpiration must be between');
|
||||
});
|
||||
|
||||
test('should validate configuration maximum bounds', () => {
|
||||
// Test maximum salt length (1024)
|
||||
expect(() => generateChallenge({ SaltLength: 1025, Difficulty: 4, ChallengeExpiration: 300000 }))
|
||||
.toThrow('SaltLength must be between 1 and 1024');
|
||||
|
||||
// Test maximum difficulty (64)
|
||||
expect(() => generateChallenge({ SaltLength: 16, Difficulty: 65, ChallengeExpiration: 300000 }))
|
||||
.toThrow('Difficulty must be between 1 and 64');
|
||||
|
||||
// Test maximum expiration (1 year)
|
||||
const oneYearMs = 365 * 24 * 60 * 60 * 1000;
|
||||
expect(() => generateChallenge({ SaltLength: 16, Difficulty: 4, ChallengeExpiration: oneYearMs + 1 }))
|
||||
.toThrow(`ChallengeExpiration must be between 1000 and ${oneYearMs}`);
|
||||
});
|
||||
|
||||
test('should handle config with optional fields', () => {
|
||||
const config = {
|
||||
SaltLength: 16,
|
||||
Difficulty: 4,
|
||||
ChallengeExpiration: 300000,
|
||||
CheckPoSTimes: true,
|
||||
PoSTimeConsistencyRatio: 3.5
|
||||
};
|
||||
|
||||
const result = generateChallenge(config);
|
||||
expect(result.challenge).toBeDefined();
|
||||
expect(result.salt).toBeDefined();
|
||||
});
|
||||
|
||||
test('should handle config with invalid PoSTimeConsistencyRatio', () => {
|
||||
const config = {
|
||||
SaltLength: 16,
|
||||
Difficulty: 4,
|
||||
ChallengeExpiration: 300000,
|
||||
PoSTimeConsistencyRatio: 0 // Invalid - should use default
|
||||
};
|
||||
|
||||
const result = generateChallenge(config);
|
||||
expect(result.challenge).toBeDefined();
|
||||
expect(result.salt).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('calculateHash', () => {
|
||||
test('should generate consistent SHA-256 hash', () => {
|
||||
const input = 'test input';
|
||||
const hash1 = calculateHash(input);
|
||||
const hash2 = calculateHash(input);
|
||||
|
||||
expect(hash1).toBe(hash2);
|
||||
expect(hash1.length).toBe(64); // SHA-256 hex length
|
||||
expect(/^[0-9a-f]+$/.test(hash1)).toBe(true);
|
||||
});
|
||||
|
||||
test('should handle different inputs', () => {
|
||||
const hash1 = calculateHash('input1');
|
||||
const hash2 = calculateHash('input2');
|
||||
|
||||
expect(hash1).not.toBe(hash2);
|
||||
});
|
||||
|
||||
test('should throw error for invalid inputs', () => {
|
||||
expect(() => calculateHash('')).toThrow('Hash input cannot be empty');
|
||||
expect(() => calculateHash(null)).toThrow('Hash input must be a string');
|
||||
expect(() => calculateHash(undefined)).toThrow('Hash input must be a string');
|
||||
});
|
||||
|
||||
test('should handle maximum input length validation', () => {
|
||||
const maxLength = 100000; // ABSOLUTE_MAX_INPUT_LENGTH
|
||||
const longInput = 'a'.repeat(maxLength + 1);
|
||||
|
||||
expect(() => calculateHash(longInput)).toThrow(`Hash input exceeds maximum length of ${maxLength}`);
|
||||
});
|
||||
|
||||
test('should handle edge case input lengths', () => {
|
||||
const validInput = 'a'.repeat(100000); // Exactly at limit
|
||||
const hash = calculateHash(validInput);
|
||||
|
||||
expect(hash).toBeDefined();
|
||||
expect(hash.length).toBe(64);
|
||||
});
|
||||
});
|
||||
|
||||
describe('verifyPoW', () => {
|
||||
test('should verify valid proof of work', () => {
|
||||
const challenge = 'abc123def456'; // Valid hex string
|
||||
const salt = 'def456abc123'; // Valid hex string
|
||||
const difficulty = 1;
|
||||
|
||||
// For difficulty 1, we need hash starting with '0'
|
||||
// Let's find a working nonce
|
||||
let validNonce = '0';
|
||||
for (let i = 0; i < 10000; i++) {
|
||||
const testNonce = i.toString(16).padStart(4, '0');
|
||||
const hash = calculateHash(challenge + salt + testNonce);
|
||||
if (hash.startsWith('0')) {
|
||||
validNonce = testNonce;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const result = verifyPoW(challenge, salt, validNonce, difficulty);
|
||||
expect(result).toBe(true);
|
||||
});
|
||||
|
||||
test('should reject invalid proof of work', () => {
|
||||
const challenge = 'abc123def456'; // Valid hex string
|
||||
const salt = 'def456abc123'; // Valid hex string
|
||||
const nonce = 'ffff'; // This should not produce required zeros
|
||||
const difficulty = 4;
|
||||
|
||||
const result = verifyPoW(challenge, salt, nonce, difficulty);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
test('should validate input parameters', () => {
|
||||
expect(() => verifyPoW('', 'abcdef', 'abcd', 4)).toThrow('challenge cannot be empty');
|
||||
expect(() => verifyPoW('abcdef', '', 'abcd', 4)).toThrow('salt cannot be empty');
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', '', 4)).toThrow('nonce cannot be empty');
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', 'abcd', 0)).toThrow('difficulty must be between');
|
||||
});
|
||||
|
||||
test('should validate hex strings', () => {
|
||||
expect(() => verifyPoW('invalid_hex!', 'abcdef', 'abcd', 4)).toThrow('must be a valid hexadecimal string');
|
||||
expect(() => verifyPoW('abcdef', 'invalid_hex!', 'abcd', 4)).toThrow('must be a valid hexadecimal string');
|
||||
});
|
||||
|
||||
test('should validate hex string lengths', () => {
|
||||
const maxLength = 100000; // ABSOLUTE_MAX_INPUT_LENGTH
|
||||
const longHex = 'a'.repeat(maxLength + 1);
|
||||
|
||||
expect(() => verifyPoW(longHex, 'abcdef', 'abcd', 4)).toThrow(`challenge exceeds maximum length of ${maxLength}`);
|
||||
expect(() => verifyPoW('abcdef', longHex, 'abcd', 4)).toThrow(`salt exceeds maximum length of ${maxLength}`);
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', longHex, 4)).toThrow(`nonce exceeds maximum length of ${maxLength}`);
|
||||
});
|
||||
|
||||
test('should validate input types for hex validation', () => {
|
||||
expect(() => verifyPoW(123, 'abcdef', 'abcd', 4)).toThrow('challenge must be a string');
|
||||
expect(() => verifyPoW('abcdef', 123, 'abcd', 4)).toThrow('salt must be a string');
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', 123, 4)).toThrow('nonce must be a string');
|
||||
});
|
||||
|
||||
test('should validate difficulty bounds', () => {
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', 'abcd', 65)).toThrow('difficulty must be between 1 and 64');
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', 'abcd', -1)).toThrow('difficulty must be between 1 and 64');
|
||||
});
|
||||
|
||||
test('should validate difficulty type', () => {
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', 'abcd', 'invalid')).toThrow('difficulty must be an integer');
|
||||
expect(() => verifyPoW('abcdef', 'abcdef', 'abcd', 4.5)).toThrow('difficulty must be an integer');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkPoSTimes', () => {
|
||||
test('should pass when check is disabled', () => {
|
||||
const times = [100, 200, 300];
|
||||
expect(() => checkPoSTimes(times, false, 2.0)).not.toThrow();
|
||||
});
|
||||
|
||||
test('should pass for consistent times', () => {
|
||||
const times = [100, 110, 120]; // Close times
|
||||
expect(() => checkPoSTimes(times, true, 2.0)).not.toThrow();
|
||||
});
|
||||
|
||||
test('should fail for inconsistent times', () => {
|
||||
const times = [100, 500, 600]; // 5x difference > 2.0 ratio
|
||||
expect(() => checkPoSTimes(times, true, 2.0)).toThrow('PoS run times inconsistent');
|
||||
});
|
||||
|
||||
test('should fail for zero times', () => {
|
||||
const times = [0, 100, 200];
|
||||
expect(() => checkPoSTimes(times, true, 2.0)).toThrow('PoS run times cannot be zero');
|
||||
});
|
||||
|
||||
test('should validate times array structure', () => {
|
||||
expect(() => checkPoSTimes(null, true, 2.0)).toThrow('times must be an array');
|
||||
expect(() => checkPoSTimes([1, 2], true, 2.0)).toThrow('times must have exactly 3 elements');
|
||||
expect(() => checkPoSTimes([1, 2, 3, 4], true, 2.0)).toThrow('times must have exactly 3 elements');
|
||||
});
|
||||
|
||||
test('should validate individual time values', () => {
|
||||
expect(() => checkPoSTimes(['invalid', 100, 200], true, 2.0)).toThrow('times[0] must be a non-negative finite number');
|
||||
expect(() => checkPoSTimes([100, null, 200], true, 2.0)).toThrow('times[1] must be a non-negative finite number');
|
||||
expect(() => checkPoSTimes([100, 200, undefined], true, 2.0)).toThrow('times[2] must be a non-negative finite number');
|
||||
});
|
||||
|
||||
test('should validate time value bounds', () => {
|
||||
const largeTimes = [10000001, 100, 200]; // Exceeds 10M ms limit
|
||||
expect(() => checkPoSTimes(largeTimes, true, 2.0)).toThrow('times[0] exceeds maximum allowed value');
|
||||
});
|
||||
|
||||
test('should validate negative time values', () => {
|
||||
expect(() => checkPoSTimes([-100, 100, 200], true, 2.0)).toThrow('times[0] must be a non-negative finite number');
|
||||
expect(() => checkPoSTimes([100, -200, 300], true, 2.0)).toThrow('times[1] must be a non-negative finite number');
|
||||
});
|
||||
|
||||
test('should validate infinite and NaN values', () => {
|
||||
expect(() => checkPoSTimes([Infinity, 100, 200], true, 2.0)).toThrow('times[0] must be a non-negative finite number');
|
||||
expect(() => checkPoSTimes([100, NaN, 200], true, 2.0)).toThrow('times[1] must be a non-negative finite number');
|
||||
});
|
||||
|
||||
test('should handle default parameters gracefully', () => {
|
||||
const times = [100, 110, 120];
|
||||
|
||||
// Test with default enableCheck (should be false)
|
||||
expect(() => checkPoSTimes(times)).not.toThrow();
|
||||
|
||||
// Test with default ratio (should be 2.0)
|
||||
expect(() => checkPoSTimes(times, true)).not.toThrow();
|
||||
|
||||
// Test with invalid ratio (should use default 2.0)
|
||||
expect(() => checkPoSTimes(times, true, 0)).not.toThrow();
|
||||
expect(() => checkPoSTimes(times, true, 'invalid')).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateRequestID', () => {
|
||||
beforeEach(() => {
|
||||
challengeStore.clear();
|
||||
});
|
||||
|
||||
test('should generate unique request IDs', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const mockRequest = { headers: { host: 'localhost' }, url: '/test' };
|
||||
|
||||
const requestId1 = generateRequestID(mockRequest, config);
|
||||
const requestId2 = generateRequestID(mockRequest, config);
|
||||
|
||||
expect(requestId1).not.toBe(requestId2);
|
||||
expect(requestId1.length).toBe(32);
|
||||
expect(requestId2.length).toBe(32);
|
||||
});
|
||||
|
||||
test('should store challenge parameters', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const mockRequest = { headers: { host: 'localhost' }, url: '/test' };
|
||||
|
||||
const requestId = generateRequestID(mockRequest, config);
|
||||
const params = challengeStore.get(requestId);
|
||||
|
||||
expect(params).toBeDefined();
|
||||
expect(params.Challenge).toBeDefined();
|
||||
expect(params.Salt).toBeDefined();
|
||||
expect(params.Difficulty).toBe(4);
|
||||
expect(params.ClientIP).toBeDefined();
|
||||
});
|
||||
|
||||
test('should validate request object', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
|
||||
expect(() => generateRequestID(null, config)).toThrow('Request must be an object');
|
||||
expect(() => generateRequestID({}, config)).toThrow('Request must have headers object');
|
||||
});
|
||||
|
||||
test('should store complete challenge parameters', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const mockRequest = { headers: { host: 'localhost' }, url: '/test' };
|
||||
|
||||
const requestId = generateRequestID(mockRequest, config);
|
||||
const params = challengeStore.get(requestId);
|
||||
|
||||
expect(params.Challenge).toBeDefined();
|
||||
expect(params.Salt).toBeDefined();
|
||||
expect(params.Difficulty).toBe(4);
|
||||
expect(params.ExpiresAt).toBeGreaterThan(Date.now());
|
||||
expect(params.CreatedAt).toBeLessThanOrEqual(Date.now());
|
||||
expect(params.PoSSeed).toBeDefined();
|
||||
expect(params.PoSSeed.length).toBe(64); // 32 bytes as hex
|
||||
});
|
||||
});
|
||||
|
||||
describe('getChallengeParams', () => {
|
||||
beforeEach(() => {
|
||||
challengeStore.clear();
|
||||
});
|
||||
|
||||
test('should retrieve stored challenge parameters', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const mockRequest = { headers: { host: 'localhost' }, url: '/test' };
|
||||
|
||||
const requestId = generateRequestID(mockRequest, config);
|
||||
const params = getChallengeParams(requestId);
|
||||
|
||||
expect(params).toBeDefined();
|
||||
expect(params.Difficulty).toBe(4);
|
||||
});
|
||||
|
||||
test('should return undefined for non-existent request ID', () => {
|
||||
const result = getChallengeParams('12345678123456781234567812345678'); // 32 char hex
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
test('should validate request ID format', () => {
|
||||
expect(() => getChallengeParams(null)).toThrow('Request ID must be a string');
|
||||
expect(() => getChallengeParams('short')).toThrow('Invalid request ID format');
|
||||
expect(() => getChallengeParams('1234567890abcdef1234567890abcdeg')).toThrow('Request ID must be hexadecimal');
|
||||
});
|
||||
|
||||
test('should validate request ID length limits', () => {
|
||||
const longRequestId = 'a'.repeat(65); // Exceeds max length
|
||||
expect(() => getChallengeParams(longRequestId)).toThrow('Request ID exceeds maximum length of 64');
|
||||
});
|
||||
|
||||
test('should validate request ID exact length requirement', () => {
|
||||
const shortHex = '1234567890abcdef1234567890abcde'; // 31 chars (too short)
|
||||
const longHex = '1234567890abcdef1234567890abcdef1'; // 33 chars (too long)
|
||||
|
||||
expect(() => getChallengeParams(shortHex)).toThrow('Invalid request ID format');
|
||||
expect(() => getChallengeParams(longHex)).toThrow('Invalid request ID format');
|
||||
});
|
||||
|
||||
test('should validate hex character requirement', () => {
|
||||
const invalidHex = '1234567890abcdef1234567890abcdex'; // Contains 'x' (invalid hex)
|
||||
expect(() => getChallengeParams(invalidHex)).toThrow('Request ID must be hexadecimal');
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteChallenge', () => {
|
||||
beforeEach(() => {
|
||||
challengeStore.clear();
|
||||
});
|
||||
|
||||
test('should delete existing challenge', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
const mockRequest = { headers: { host: 'localhost' }, url: '/test' };
|
||||
|
||||
const requestId = generateRequestID(mockRequest, config);
|
||||
expect(getChallengeParams(requestId)).toBeDefined();
|
||||
|
||||
const deleteResult = deleteChallenge(requestId);
|
||||
expect(deleteResult).toBe(true);
|
||||
expect(getChallengeParams(requestId)).toBeUndefined();
|
||||
});
|
||||
|
||||
test('should return false for non-existent challenge', () => {
|
||||
const result = deleteChallenge('12345678123456781234567812345678');
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
test('should validate request ID type', () => {
|
||||
expect(() => deleteChallenge(123)).toThrow('Request ID must be a string');
|
||||
});
|
||||
});
|
||||
|
||||
describe('verifyPoS', () => {
|
||||
test('should verify valid proof of stake', () => {
|
||||
// Use 64-character hex hashes (SHA-256 length)
|
||||
const validHash = 'a'.repeat(64);
|
||||
const hashes = [validHash, validHash, validHash];
|
||||
const times = [100, 110, 120];
|
||||
const config = {
|
||||
SaltLength: 16,
|
||||
Difficulty: 4,
|
||||
ChallengeExpiration: 300000,
|
||||
CheckPoSTimes: true,
|
||||
PoSTimeConsistencyRatio: 2.0
|
||||
};
|
||||
|
||||
expect(() => verifyPoS(hashes, times, config)).not.toThrow();
|
||||
});
|
||||
|
||||
test('should fail for mismatched hashes', () => {
|
||||
// Use different 64-character hex hashes
|
||||
const hash1 = 'a'.repeat(64);
|
||||
const hash2 = 'b'.repeat(64);
|
||||
const hashes = [hash1, hash2, hash1];
|
||||
const times = [100, 110, 120];
|
||||
const config = {
|
||||
SaltLength: 16,
|
||||
Difficulty: 4,
|
||||
ChallengeExpiration: 300000,
|
||||
CheckPoSTimes: false
|
||||
};
|
||||
|
||||
expect(() => verifyPoS(hashes, times, config)).toThrow('PoS hashes do not match');
|
||||
});
|
||||
|
||||
test('should validate hashes array structure', () => {
|
||||
const times = [100, 110, 120];
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
|
||||
expect(() => verifyPoS(null, times, config)).toThrow('hashes must be an array');
|
||||
expect(() => verifyPoS([1, 2], times, config)).toThrow('hashes must have exactly 3 elements');
|
||||
});
|
||||
|
||||
test('should validate hash format', () => {
|
||||
const invalidHash = 'invalid!';
|
||||
const validHash = 'a'.repeat(64);
|
||||
const hashes = [invalidHash, validHash, validHash];
|
||||
const times = [100, 110, 120];
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
|
||||
expect(() => verifyPoS(hashes, times, config)).toThrow('must be a valid hexadecimal string');
|
||||
});
|
||||
|
||||
test('should validate hash length requirement', () => {
|
||||
const shortHash = 'a'.repeat(63); // Too short
|
||||
const validHash = 'a'.repeat(64);
|
||||
const hashes = [shortHash, validHash, validHash];
|
||||
const times = [100, 110, 120];
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
|
||||
expect(() => verifyPoS(hashes, times, config)).toThrow('hashes[0] must be exactly 64 characters');
|
||||
});
|
||||
|
||||
test('should validate individual hash array elements', () => {
|
||||
const validHash = 'a'.repeat(64);
|
||||
const times = [100, 110, 120];
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 300000 };
|
||||
|
||||
// Test non-string hash
|
||||
expect(() => verifyPoS([123, validHash, validHash], times, config)).toThrow('hashes[0] must be a string');
|
||||
|
||||
// Test empty hash
|
||||
expect(() => verifyPoS(['', validHash, validHash], times, config)).toThrow('hashes[0] cannot be empty');
|
||||
});
|
||||
|
||||
test('should properly call timing validation when enabled', () => {
|
||||
const validHash = 'a'.repeat(64);
|
||||
const hashes = [validHash, validHash, validHash];
|
||||
const inconsistentTimes = [100, 1000, 1100]; // Large ratio
|
||||
const config = {
|
||||
SaltLength: 16,
|
||||
Difficulty: 4,
|
||||
ChallengeExpiration: 300000,
|
||||
CheckPoSTimes: true,
|
||||
PoSTimeConsistencyRatio: 2.0
|
||||
};
|
||||
|
||||
expect(() => verifyPoS(hashes, inconsistentTimes, config)).toThrow('PoS run times inconsistent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('expired challenge cleanup', () => {
|
||||
beforeEach(() => {
|
||||
challengeStore.clear();
|
||||
jest.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
test('should clean up expired challenges automatically', () => {
|
||||
const config = { SaltLength: 16, Difficulty: 4, ChallengeExpiration: 1000 }; // 1 second
|
||||
const mockRequest = { headers: { host: 'localhost' }, url: '/test' };
|
||||
|
||||
// Generate some challenges
|
||||
const requestId1 = generateRequestID(mockRequest, config);
|
||||
const requestId2 = generateRequestID(mockRequest, config);
|
||||
|
||||
expect(challengeStore.size).toBe(2);
|
||||
|
||||
// Advance time to expire challenges
|
||||
jest.advanceTimersByTime(2000); // 2 seconds
|
||||
|
||||
// Trigger cleanup (runs every 5 minutes)
|
||||
jest.advanceTimersByTime(5 * 60 * 1000);
|
||||
|
||||
// Challenges should still be there since cleanup hasn't run based on expiration
|
||||
// This tests the cleanup mechanism exists
|
||||
expect(challengeStore.size).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('should handle empty challenge store during cleanup', () => {
|
||||
// Advance time to trigger cleanup with empty store
|
||||
jest.advanceTimersByTime(5 * 60 * 1000);
|
||||
|
||||
// Should not throw
|
||||
expect(challengeStore.size).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
152
.tests/setup.js
152
.tests/setup.js
|
|
@ -1,152 +0,0 @@
|
|||
// Jest setup file - runs before all tests
|
||||
// Set NODE_ENV to test to prevent main application from starting
|
||||
process.env.NODE_ENV = 'test';
|
||||
|
||||
// Mock fs operations globally to prevent checkpoint.js errors
|
||||
const mockFn = () => {
|
||||
const fn = function(...args) { return fn.mockReturnValue; };
|
||||
fn.mockReturnValue = undefined;
|
||||
fn.mockResolvedValue = (value) => { fn.mockReturnValue = Promise.resolve(value); return fn; };
|
||||
return fn;
|
||||
};
|
||||
|
||||
global.fs = {
|
||||
readFileSync: () => 'mocked-file-content',
|
||||
writeFileSync: () => {},
|
||||
mkdirSync: () => {},
|
||||
existsSync: () => false,
|
||||
readdirSync: () => [],
|
||||
unlinkSync: () => {}
|
||||
};
|
||||
|
||||
// Mock fs/promises
|
||||
global.fsPromises = {
|
||||
readFile: () => Promise.resolve('mocked-content'),
|
||||
writeFile: () => Promise.resolve(undefined),
|
||||
mkdir: () => Promise.resolve(undefined)
|
||||
};
|
||||
|
||||
// Suppress console warnings and errors during tests for cleaner output
|
||||
const originalConsoleWarn = console.warn;
|
||||
const originalConsoleError = console.error;
|
||||
|
||||
console.warn = () => {};
|
||||
console.error = () => {};
|
||||
|
||||
// Track timers and intervals for cleanup
|
||||
const timers = new Set();
|
||||
const immediates = new Set();
|
||||
const originalSetTimeout = global.setTimeout;
|
||||
const originalSetInterval = global.setInterval;
|
||||
const originalSetImmediate = global.setImmediate;
|
||||
const originalClearTimeout = global.clearTimeout;
|
||||
const originalClearInterval = global.clearInterval;
|
||||
const originalClearImmediate = global.clearImmediate;
|
||||
|
||||
global.setTimeout = (fn, delay, ...args) => {
|
||||
const id = originalSetTimeout(fn, delay, ...args);
|
||||
timers.add(id);
|
||||
return id;
|
||||
};
|
||||
|
||||
global.setInterval = (fn, delay, ...args) => {
|
||||
const id = originalSetInterval(fn, delay, ...args);
|
||||
timers.add(id);
|
||||
return id;
|
||||
};
|
||||
|
||||
global.setImmediate = (fn, ...args) => {
|
||||
const id = originalSetImmediate(fn, ...args);
|
||||
immediates.add(id);
|
||||
return id;
|
||||
};
|
||||
|
||||
global.clearTimeout = (id) => {
|
||||
timers.delete(id);
|
||||
return originalClearTimeout(id);
|
||||
};
|
||||
|
||||
global.clearInterval = (id) => {
|
||||
timers.delete(id);
|
||||
return originalClearInterval(id);
|
||||
};
|
||||
|
||||
global.clearImmediate = (id) => {
|
||||
immediates.delete(id);
|
||||
return originalClearImmediate(id);
|
||||
};
|
||||
|
||||
global.pendingImmediates = immediates;
|
||||
|
||||
// Mock @iarna/toml to prevent async import after teardown
|
||||
// Note: This mock will be applied globally for all tests
|
||||
|
||||
// Comprehensive behavioral detection mocking to prevent async operations
|
||||
global.mockBehavioralDetection = {
|
||||
config: { enabled: false },
|
||||
isBlocked: () => Promise.resolve({ blocked: false }),
|
||||
getRateLimit: () => Promise.resolve(null),
|
||||
analyzeRequest: () => Promise.resolve({ totalScore: 0, patterns: [] }),
|
||||
loadRules: () => Promise.resolve(),
|
||||
init: () => Promise.resolve()
|
||||
};
|
||||
|
||||
// Mock the loadConfig function globally to prevent TOML imports
|
||||
global.mockLoadConfig = async (name, target) => {
|
||||
// Return immediately without trying to import TOML
|
||||
return Promise.resolve();
|
||||
};
|
||||
|
||||
// Mock dynamic imports to prevent teardown issues
|
||||
const originalImport = global.import;
|
||||
if (originalImport) {
|
||||
global.import = (modulePath) => {
|
||||
if (modulePath === '@iarna/toml') {
|
||||
return Promise.resolve({
|
||||
default: { parse: () => ({}) },
|
||||
parse: () => ({})
|
||||
});
|
||||
}
|
||||
if (modulePath.includes('behavioral-detection')) {
|
||||
return Promise.resolve(global.mockBehavioralDetection);
|
||||
}
|
||||
return originalImport(modulePath);
|
||||
};
|
||||
}
|
||||
|
||||
// Function to clear all timers
|
||||
global.clearAllTimers = () => {
|
||||
timers.forEach(id => {
|
||||
originalClearTimeout(id);
|
||||
originalClearInterval(id);
|
||||
});
|
||||
timers.clear();
|
||||
|
||||
immediates.forEach(id => {
|
||||
originalClearImmediate(id);
|
||||
});
|
||||
immediates.clear();
|
||||
};
|
||||
|
||||
// Clean up after all tests
|
||||
afterAll(() => {
|
||||
// Clear all remaining timers
|
||||
global.clearAllTimers();
|
||||
|
||||
// Restore original console methods
|
||||
console.warn = originalConsoleWarn;
|
||||
console.error = originalConsoleError;
|
||||
|
||||
// Restore original import if it was mocked
|
||||
if (originalImport) {
|
||||
global.import = originalImport;
|
||||
}
|
||||
|
||||
// Restore original timer functions
|
||||
global.setTimeout = originalSetTimeout;
|
||||
global.setInterval = originalSetInterval;
|
||||
global.setImmediate = originalSetImmediate;
|
||||
global.clearTimeout = originalClearTimeout;
|
||||
global.clearInterval = originalClearInterval;
|
||||
global.clearImmediate = originalClearImmediate;
|
||||
});
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
// Jest teardown file - runs after all tests complete
|
||||
// This helps prevent async operations from continuing after Jest environment teardown
|
||||
|
||||
// Global teardown to ensure clean test environment shutdown
|
||||
module.exports = async () => {
|
||||
// Clear all timers first
|
||||
if (typeof global.clearAllTimers === 'function') {
|
||||
global.clearAllTimers();
|
||||
}
|
||||
|
||||
// Clear any pending setImmediate calls
|
||||
if (global.pendingImmediates) {
|
||||
global.pendingImmediates.forEach(id => clearImmediate(id));
|
||||
global.pendingImmediates.clear();
|
||||
}
|
||||
|
||||
// Force close any remaining handles
|
||||
if (process._getActiveHandles) {
|
||||
const handles = process._getActiveHandles();
|
||||
handles.forEach(handle => {
|
||||
if (handle && typeof handle.close === 'function') {
|
||||
try {
|
||||
handle.close();
|
||||
} catch (e) {
|
||||
// Ignore errors during forced cleanup
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Force garbage collection if available
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
|
||||
// Add a small delay to allow any pending async operations to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
|
||||
// Clear any remaining console overrides
|
||||
if (global.originalConsole) {
|
||||
Object.assign(console, global.originalConsole);
|
||||
}
|
||||
};
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
import { jest } from '@jest/globals';
|
||||
|
||||
// Mock behavioral detection and config loading before any imports
|
||||
jest.unstable_mockModule('../dist/utils/behavioral-detection.js', () => ({
|
||||
behavioralDetection: {
|
||||
config: { enabled: false },
|
||||
isBlocked: () => Promise.resolve({ blocked: false }),
|
||||
getRateLimit: () => Promise.resolve(null),
|
||||
analyzeRequest: () => Promise.resolve({ totalScore: 0, patterns: [] }),
|
||||
loadRules: () => Promise.resolve(),
|
||||
init: () => Promise.resolve()
|
||||
},
|
||||
BehavioralDetectionEngine: class MockBehavioralDetectionEngine {
|
||||
constructor() {
|
||||
this.config = { enabled: false };
|
||||
}
|
||||
async loadRules() { return Promise.resolve(); }
|
||||
async init() { return Promise.resolve(); }
|
||||
async isBlocked() { return Promise.resolve({ blocked: false }); }
|
||||
async getRateLimit() { return Promise.resolve(null); }
|
||||
async analyzeRequest() { return Promise.resolve({ totalScore: 0, patterns: [] }); }
|
||||
}
|
||||
}));
|
||||
|
||||
// Mock the main index loadConfig function to prevent TOML imports
|
||||
jest.unstable_mockModule('../dist/index.js', () => ({
|
||||
loadConfig: () => Promise.resolve(),
|
||||
registerPlugin: () => {},
|
||||
getRegisteredPluginNames: () => [],
|
||||
loadPlugins: () => [],
|
||||
freezePlugins: () => {},
|
||||
rootDir: '/mock/root'
|
||||
}));
|
||||
|
||||
import { threatScorer, configureDefaultThreatScorer, createThreatScorer } from '../dist/utils/threat-scoring.js';
|
||||
|
||||
describe('Threat Scoring (Re-export)', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Configure the default threat scorer with test config
|
||||
const testConfig = {
|
||||
enabled: true,
|
||||
thresholds: {
|
||||
ALLOW: 20,
|
||||
CHALLENGE: 60,
|
||||
BLOCK: 100
|
||||
},
|
||||
signalWeights: {
|
||||
BLACKLISTED_IP: { weight: 50, confidence: 0.95 },
|
||||
RAPID_ENUMERATION: { weight: 35, confidence: 0.80 },
|
||||
BRUTE_FORCE_PATTERN: { weight: 45, confidence: 0.88 },
|
||||
SQL_INJECTION: { weight: 60, confidence: 0.92 },
|
||||
XSS_ATTEMPT: { weight: 50, confidence: 0.88 },
|
||||
COMMAND_INJECTION: { weight: 65, confidence: 0.95 },
|
||||
ATTACK_TOOL_UA: { weight: 30, confidence: 0.75 },
|
||||
MISSING_UA: { weight: 10, confidence: 0.60 },
|
||||
IMPOSSIBLE_TRAVEL: { weight: 30, confidence: 0.80 },
|
||||
HIGH_RISK_COUNTRY: { weight: 15, confidence: 0.60 }
|
||||
},
|
||||
enableBotVerification: true,
|
||||
enableGeoAnalysis: true,
|
||||
enableBehaviorAnalysis: true,
|
||||
enableContentAnalysis: true
|
||||
};
|
||||
|
||||
configureDefaultThreatScorer(testConfig);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Wait for any pending async operations to complete
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
});
|
||||
|
||||
describe('exports', () => {
|
||||
test('should export threatScorer instance', () => {
|
||||
expect(threatScorer).toBeDefined();
|
||||
expect(typeof threatScorer).toBe('object');
|
||||
|
||||
// Should have the new API methods
|
||||
expect(typeof threatScorer.scoreRequest).toBe('function');
|
||||
});
|
||||
|
||||
test('should export configuration functions', () => {
|
||||
expect(configureDefaultThreatScorer).toBeDefined();
|
||||
expect(typeof configureDefaultThreatScorer).toBe('function');
|
||||
|
||||
expect(createThreatScorer).toBeDefined();
|
||||
expect(typeof createThreatScorer).toBe('function');
|
||||
});
|
||||
});
|
||||
|
||||
describe('threatScorer functionality', () => {
|
||||
test('should score a simple request', async () => {
|
||||
const mockRequest = {
|
||||
headers: { 'user-agent': 'test-browser' },
|
||||
method: 'GET',
|
||||
url: '/test'
|
||||
};
|
||||
|
||||
const result = await threatScorer.scoreRequest(mockRequest);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(typeof result.totalScore).toBe('number');
|
||||
expect(typeof result.confidence).toBe('number');
|
||||
expect(['allow', 'challenge', 'block']).toContain(result.riskLevel);
|
||||
expect(Array.isArray(result.signalsTriggered)).toBe(true);
|
||||
expect(typeof result.processingTimeMs).toBe('number');
|
||||
});
|
||||
|
||||
test('should handle disabled scoring', async () => {
|
||||
const disabledConfig = {
|
||||
enabled: false,
|
||||
thresholds: { ALLOW: 20, CHALLENGE: 60, BLOCK: 100 },
|
||||
signalWeights: {}
|
||||
};
|
||||
|
||||
const disabledScorer = createThreatScorer(disabledConfig);
|
||||
|
||||
const mockRequest = {
|
||||
headers: { 'user-agent': 'test-browser' },
|
||||
method: 'GET',
|
||||
url: '/test'
|
||||
};
|
||||
|
||||
const result = await disabledScorer.scoreRequest(mockRequest);
|
||||
|
||||
expect(result.riskLevel).toBe('allow');
|
||||
expect(result.totalScore).toBe(0);
|
||||
});
|
||||
|
||||
test('should require configuration for default scorer', async () => {
|
||||
// Test that unconfigured scorer behaves correctly
|
||||
const unconfiguredScorer = createThreatScorer({ enabled: false, thresholds: {} });
|
||||
|
||||
const mockRequest = {
|
||||
headers: { 'user-agent': 'test-browser' },
|
||||
method: 'GET',
|
||||
url: '/test'
|
||||
};
|
||||
|
||||
// Unconfigured/disabled scorer should return allow with 0 score
|
||||
const result = await unconfiguredScorer.scoreRequest(mockRequest);
|
||||
expect(result.riskLevel).toBe('allow');
|
||||
expect(result.totalScore).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('threat scoring configuration', () => {
|
||||
test('should create scorer with custom config', () => {
|
||||
const customConfig = {
|
||||
enabled: true,
|
||||
thresholds: {
|
||||
ALLOW: 10,
|
||||
CHALLENGE: 30,
|
||||
BLOCK: 50
|
||||
},
|
||||
signalWeights: {
|
||||
BLACKLISTED_IP: { weight: 100, confidence: 1.0 }
|
||||
},
|
||||
enableBotVerification: true
|
||||
};
|
||||
|
||||
const customScorer = createThreatScorer(customConfig);
|
||||
expect(customScorer).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,270 +0,0 @@
|
|||
import { parseDuration, formatDuration, isValidDurationString } from '../dist/utils/time.js';
|
||||
|
||||
describe('Time utilities', () => {
|
||||
describe('parseDuration', () => {
|
||||
describe('numeric inputs', () => {
|
||||
test('should parse positive numbers as milliseconds', () => {
|
||||
expect(parseDuration(1000)).toBe(1000);
|
||||
expect(parseDuration(5000)).toBe(5000);
|
||||
expect(parseDuration(0)).toBe(0);
|
||||
});
|
||||
|
||||
test('should throw error for negative numbers', () => {
|
||||
expect(() => parseDuration(-1000)).toThrow('Duration cannot be negative');
|
||||
expect(() => parseDuration(-1)).toThrow('Duration cannot be negative');
|
||||
});
|
||||
|
||||
test('should throw error for numbers exceeding MAX_SAFE_INTEGER', () => {
|
||||
expect(() => parseDuration(Number.MAX_SAFE_INTEGER + 1)).toThrow('Duration too large');
|
||||
});
|
||||
});
|
||||
|
||||
describe('string inputs', () => {
|
||||
describe('valid duration strings', () => {
|
||||
test('should parse seconds correctly', () => {
|
||||
expect(parseDuration('5s')).toBe(5000);
|
||||
expect(parseDuration('30s')).toBe(30000);
|
||||
expect(parseDuration('1s')).toBe(1000);
|
||||
});
|
||||
|
||||
test('should parse minutes correctly', () => {
|
||||
expect(parseDuration('5m')).toBe(300000);
|
||||
expect(parseDuration('1m')).toBe(60000);
|
||||
expect(parseDuration('10m')).toBe(600000);
|
||||
});
|
||||
|
||||
test('should parse hours correctly', () => {
|
||||
expect(parseDuration('1h')).toBe(3600000);
|
||||
expect(parseDuration('2h')).toBe(7200000);
|
||||
});
|
||||
|
||||
test('should parse days correctly', () => {
|
||||
expect(parseDuration('1d')).toBe(86400000);
|
||||
expect(parseDuration('2d')).toBe(172800000);
|
||||
});
|
||||
|
||||
test('should parse decimal values', () => {
|
||||
expect(parseDuration('1.5s')).toBe(1500);
|
||||
expect(parseDuration('2.5m')).toBe(150000);
|
||||
});
|
||||
|
||||
test('should be case-insensitive for units', () => {
|
||||
expect(parseDuration('5S')).toBe(5000);
|
||||
expect(parseDuration('5M')).toBe(300000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('numeric strings', () => {
|
||||
test('should parse numeric strings as milliseconds', () => {
|
||||
expect(parseDuration('1000')).toBe(1000);
|
||||
expect(parseDuration('5000')).toBe(5000);
|
||||
});
|
||||
|
||||
test('should throw error for negative numeric strings', () => {
|
||||
expect(() => parseDuration('-1000')).toThrow('Duration cannot be negative');
|
||||
});
|
||||
});
|
||||
|
||||
describe('invalid inputs', () => {
|
||||
test('should throw error for empty string', () => {
|
||||
expect(() => parseDuration('')).toThrow('Duration cannot be empty');
|
||||
expect(() => parseDuration(' ')).toThrow('Duration cannot be empty');
|
||||
});
|
||||
|
||||
test('should throw error for invalid format', () => {
|
||||
expect(() => parseDuration('abc')).toThrow('Invalid duration format');
|
||||
expect(() => parseDuration('5x')).toThrow('Invalid duration format');
|
||||
expect(() => parseDuration('s5')).toThrow('Invalid duration format');
|
||||
});
|
||||
|
||||
test('should throw error for negative duration values', () => {
|
||||
expect(() => parseDuration('-5s')).toThrow('Invalid duration format');
|
||||
expect(() => parseDuration('-10m')).toThrow('Invalid duration format');
|
||||
});
|
||||
|
||||
describe('edge case validation', () => {
|
||||
test('should handle duration strings that exceed MAX_SAFE_INTEGER when multiplied', () => {
|
||||
// Test a very large number that when multiplied by day multiplier exceeds MAX_SAFE_INTEGER
|
||||
const largeValue = Math.floor(Number.MAX_SAFE_INTEGER / (24 * 60 * 60 * 1000)) + 1;
|
||||
expect(() => parseDuration(`${largeValue}d`)).toThrow('Duration too large');
|
||||
});
|
||||
|
||||
test('should handle duration strings with whitespace around units', () => {
|
||||
expect(parseDuration('5 s')).toBe(5000);
|
||||
expect(parseDuration('10 m')).toBe(600000);
|
||||
expect(parseDuration('2 h')).toBe(7200000);
|
||||
expect(parseDuration('1 d')).toBe(86400000);
|
||||
});
|
||||
|
||||
test('should throw error for duration strings with negative values in unit format', () => {
|
||||
expect(() => parseDuration('-1.5s')).toThrow('Invalid duration format');
|
||||
expect(() => parseDuration('-10m')).toThrow('Invalid duration format');
|
||||
expect(() => parseDuration('-2.5h')).toThrow('Invalid duration format');
|
||||
});
|
||||
|
||||
test('should handle various decimal formats', () => {
|
||||
expect(parseDuration('0.5s')).toBe(500);
|
||||
expect(parseDuration('0.1m')).toBe(6000);
|
||||
expect(parseDuration('1.0h')).toBe(3600000);
|
||||
expect(parseDuration('0.25d')).toBe(21600000);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('invalid types', () => {
|
||||
test('should throw error for non-string/number inputs', () => {
|
||||
expect(() => parseDuration(null)).toThrow('Duration must be a string or number');
|
||||
expect(() => parseDuration(undefined)).toThrow('Duration must be a string or number');
|
||||
expect(() => parseDuration({})).toThrow('Duration must be a string or number');
|
||||
expect(() => parseDuration([])).toThrow('Duration must be a string or number');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatDuration', () => {
|
||||
test('should format milliseconds to ms for values < 1000', () => {
|
||||
expect(formatDuration(0)).toBe('0ms');
|
||||
expect(formatDuration(999)).toBe('999ms');
|
||||
expect(formatDuration(500)).toBe('500ms');
|
||||
});
|
||||
|
||||
test('should format to seconds', () => {
|
||||
expect(formatDuration(1000)).toBe('1s');
|
||||
expect(formatDuration(5000)).toBe('5s');
|
||||
expect(formatDuration(59000)).toBe('59s');
|
||||
});
|
||||
|
||||
test('should format to minutes', () => {
|
||||
expect(formatDuration(60000)).toBe('1m');
|
||||
expect(formatDuration(300000)).toBe('5m');
|
||||
expect(formatDuration(3540000)).toBe('59m');
|
||||
});
|
||||
|
||||
test('should format to hours', () => {
|
||||
expect(formatDuration(3600000)).toBe('1h');
|
||||
expect(formatDuration(7200000)).toBe('2h');
|
||||
expect(formatDuration(86340000)).toBe('23h');
|
||||
});
|
||||
|
||||
test('should format to days', () => {
|
||||
expect(formatDuration(86400000)).toBe('1d');
|
||||
expect(formatDuration(172800000)).toBe('2d');
|
||||
expect(formatDuration(604800000)).toBe('7d');
|
||||
});
|
||||
|
||||
test('should throw error for negative values', () => {
|
||||
expect(() => formatDuration(-1000)).toThrow('Duration cannot be negative');
|
||||
});
|
||||
|
||||
test('should format edge case durations correctly', () => {
|
||||
expect(formatDuration(1)).toBe('1ms');
|
||||
expect(formatDuration(999)).toBe('999ms');
|
||||
expect(formatDuration(1001)).toBe('1s');
|
||||
expect(formatDuration(59999)).toBe('59s');
|
||||
expect(formatDuration(60001)).toBe('1m');
|
||||
expect(formatDuration(3599999)).toBe('59m');
|
||||
expect(formatDuration(3600001)).toBe('1h');
|
||||
expect(formatDuration(86399999)).toBe('23h');
|
||||
expect(formatDuration(86400001)).toBe('1d');
|
||||
});
|
||||
|
||||
test('should format very large durations to days', () => {
|
||||
expect(formatDuration(Number.MAX_SAFE_INTEGER)).toBe(`${Math.floor(Number.MAX_SAFE_INTEGER / (24 * 60 * 60 * 1000))}d`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidDurationString', () => {
|
||||
test('should return true for valid duration formats', () => {
|
||||
expect(isValidDurationString('5s')).toBe(true);
|
||||
expect(isValidDurationString('10m')).toBe(true);
|
||||
expect(isValidDurationString('2h')).toBe(true);
|
||||
expect(isValidDurationString('1d')).toBe(true);
|
||||
expect(isValidDurationString('1000')).toBe(true);
|
||||
});
|
||||
|
||||
test('should return false for invalid formats', () => {
|
||||
expect(isValidDurationString('abc')).toBe(false);
|
||||
expect(isValidDurationString('5x')).toBe(false);
|
||||
expect(isValidDurationString('')).toBe(false);
|
||||
expect(isValidDurationString('-5s')).toBe(false);
|
||||
});
|
||||
|
||||
test('should return false for various invalid input types', () => {
|
||||
expect(isValidDurationString(null)).toBe(false);
|
||||
expect(isValidDurationString(undefined)).toBe(false);
|
||||
expect(isValidDurationString({})).toBe(false);
|
||||
expect(isValidDurationString([])).toBe(false);
|
||||
expect(isValidDurationString(true)).toBe(false);
|
||||
expect(isValidDurationString(false)).toBe(false);
|
||||
});
|
||||
|
||||
test('should return false for durations that would overflow', () => {
|
||||
const largeValue = Math.floor(Number.MAX_SAFE_INTEGER / (24 * 60 * 60 * 1000)) + 1;
|
||||
expect(isValidDurationString(`${largeValue}d`)).toBe(false);
|
||||
// Test overflow in unit-based durations (not pure numeric strings which have different validation)
|
||||
expect(isValidDurationString(`${Number.MAX_SAFE_INTEGER}d`)).toBe(false);
|
||||
});
|
||||
|
||||
test('should return false for edge case invalid formats', () => {
|
||||
expect(isValidDurationString('5.5.5s')).toBe(false);
|
||||
expect(isValidDurationString('5ss')).toBe(false);
|
||||
expect(isValidDurationString('s')).toBe(false);
|
||||
expect(isValidDurationString('5')).toBe(true); // Valid numeric string
|
||||
expect(isValidDurationString('5.m')).toBe(false);
|
||||
expect(isValidDurationString('.5m')).toBe(false);
|
||||
});
|
||||
|
||||
test('should handle decimal validation correctly', () => {
|
||||
expect(isValidDurationString('1.5s')).toBe(true);
|
||||
expect(isValidDurationString('0.5m')).toBe(true);
|
||||
expect(isValidDurationString('2.0h')).toBe(true);
|
||||
expect(isValidDurationString('1.25d')).toBe(true);
|
||||
});
|
||||
|
||||
test('should validate case insensitive units', () => {
|
||||
expect(isValidDurationString('5S')).toBe(true);
|
||||
expect(isValidDurationString('5M')).toBe(true);
|
||||
expect(isValidDurationString('5H')).toBe(true);
|
||||
expect(isValidDurationString('5D')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('integration tests', () => {
|
||||
test('parseDuration and formatDuration should work together', () => {
|
||||
const testValues = ['5s', '10m', '2h', '1d'];
|
||||
|
||||
testValues.forEach(value => {
|
||||
const parsed = parseDuration(value);
|
||||
const formatted = formatDuration(parsed);
|
||||
expect(formatted).toBe(value);
|
||||
});
|
||||
});
|
||||
|
||||
test('should handle round-trip conversion with decimal values', () => {
|
||||
// Note: formatDuration floors values, so we test known good round-trips
|
||||
const testCases = [
|
||||
{ input: '1.5s', parsed: 1500, formatted: '1s' },
|
||||
{ input: '2.5m', parsed: 150000, formatted: '2m' },
|
||||
{ input: '1.5h', parsed: 5400000, formatted: '1h' }
|
||||
];
|
||||
|
||||
testCases.forEach(({ input, parsed, formatted }) => {
|
||||
expect(parseDuration(input)).toBe(parsed);
|
||||
expect(formatDuration(parsed)).toBe(formatted);
|
||||
});
|
||||
});
|
||||
|
||||
test('should validate and format edge boundary values', () => {
|
||||
// Test values at unit boundaries
|
||||
expect(formatDuration(999)).toBe('999ms');
|
||||
expect(formatDuration(1000)).toBe('1s');
|
||||
expect(formatDuration(59999)).toBe('59s');
|
||||
expect(formatDuration(60000)).toBe('1m');
|
||||
expect(formatDuration(3599999)).toBe('59m');
|
||||
expect(formatDuration(3600000)).toBe('1h');
|
||||
expect(formatDuration(86399999)).toBe('23h');
|
||||
expect(formatDuration(86400000)).toBe('1d');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -18,4 +18,4 @@ EXPOSE 3000
|
|||
ENV NODE_ENV=production
|
||||
|
||||
# Run the application
|
||||
CMD ["npm", "run", "daemon-r"]
|
||||
CMD ["npm", "run", "daemon"]
|
||||
202
README.md
202
README.md
|
|
@ -1,18 +1,17 @@
|
|||
# Checkpoint Security Gateway
|
||||
# Checkpoint
|
||||
|
||||
> High-performance, TypeScript-based security gateway with advanced threat detection, behavioral analysis, and adaptive protection.
|
||||
> Secure, extensible, high-performance Node.js middleware server for proof-of-work security, IP filtering, reverse proxying, and real-time analytics.
|
||||
|
||||
**Features:**
|
||||
|
||||
- 🔐 **Checkpoint Security:** Proof-of-work (PoW) and proof-of-space-time (PoST) challenges for suspicious traffic
|
||||
- 🛡️ **Web Application Firewall:** Advanced pattern matching against SQL injection, XSS, command injection, and more
|
||||
- 🌎 **IP & Geo-Filtering:** Block or allow traffic based on country, continent, or ASN using MaxMind GeoIP2
|
||||
- 🔀 **Reverse Proxy:** High-performance request forwarding with WebSocket support
|
||||
- 🧠 **Behavioral Detection:** ML-inspired pattern recognition with adaptive scoring
|
||||
- 📊 **Threat Scoring:** Real-time risk assessment with configurable thresholds
|
||||
- 🤖 **Bot Verification:** Identifies and handles good bots vs malicious automation
|
||||
- 🧩 **Plugin Architecture:** Modular design for easy extension and customization
|
||||
- 📂 **Data Persistence:** Secure token storage with LevelDB + TTL and HMAC protection
|
||||
- 🔐 **Checkpoint Security:** Enforce proof-of-work (PoW) and proof-of-space-time (PoST) challenges before granting access.
|
||||
- 🌎 **IP & Geo-Blocking:** Block or allow traffic based on country, continent, or ASN using MaxMind GeoIP2.
|
||||
- 🔀 **Reverse Proxy:** Route incoming requests to backend services based on hostname mappings.
|
||||
- 📊 **Real-time Stats:** Collect detailed metrics and browse via built-in web UI or API.
|
||||
- 🧩 **Plugin Architecture:** Easily extend and customize via modular plugins.
|
||||
- 🛠️ **Flexible Configuration:** Manage settings in TOML files and via environment variables.
|
||||
- ⚙️ **Daemon & PM2 Support:** Run as a background service with built-in daemon mode or PM2.
|
||||
- 📂 **Data Persistence:** Secure token storage with LevelDB + TTL and HMAC protection.
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
|
|
@ -21,184 +20,63 @@
|
|||
git clone https://git.caileb.com/Caileb/Checkpoint.git
|
||||
cd Checkpoint
|
||||
```
|
||||
|
||||
2. **Install dependencies**
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
3. **Set up configuration files**
|
||||
```bash
|
||||
cp config/*.toml.example config/*.toml
|
||||
3. **Set up environment variables** (optional)
|
||||
Create a `.env` file in the project root:
|
||||
```ini
|
||||
MAXMIND_ACCOUNT_ID=your_account_id
|
||||
MAXMIND_LICENSE_KEY=your_license_key
|
||||
PORT=8080 # Default: 3000
|
||||
```
|
||||
|
||||
4. **Configure your settings**
|
||||
- Edit TOML files in `config/` directory
|
||||
- Set proxy mappings in `proxy.toml`
|
||||
- Configure security rules in `waf.toml`
|
||||
- Adjust thresholds in `threat-scoring.toml`
|
||||
|
||||
5. **Development mode**
|
||||
4. **Development mode**
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
6. **Production mode**
|
||||
5. **Start the server**
|
||||
```bash
|
||||
npm start
|
||||
```
|
||||
|
||||
7. **Daemonize with PM2**
|
||||
6. **Daemonize**
|
||||
```bash
|
||||
npm run daemon # Start in background
|
||||
npm run stop # Stop daemon
|
||||
npm run restart # Restart daemon
|
||||
npm run logs # View logs
|
||||
npm run logs # Show logs
|
||||
```
|
||||
Or use PM2 directly:
|
||||
```bash
|
||||
pm2 start index.js --name checkpoint
|
||||
```
|
||||
|
||||
## ⚙️ Configuration
|
||||
|
||||
All settings are stored in TOML files within the `config/` directory:
|
||||
All core settings are stored in the `config/` directory as TOML files:
|
||||
|
||||
- `checkpoint.toml` — Proof-of-work parameters, token storage, exclusion rules
|
||||
- `waf.toml` — Web Application Firewall rules, scoring, and bot verification
|
||||
- `behavioral-detection.toml` — Pattern detection rules and correlations
|
||||
- `proxy.toml` — Hostname-to-backend mappings, timeouts, and body size limits
|
||||
- `ipfilter.toml` — Geographic and network filtering with MaxMind integration
|
||||
- `threat-scoring.toml` — Advanced scoring thresholds and feature weights
|
||||
- `checkpoint.toml` — PoW/PoST parameters, tokens, exclusions, interstitial templates.
|
||||
- `ipfilter.toml` — Country, continent, ASN filtering rules and custom block pages.
|
||||
- `proxy.toml` — Hostname-to-backend mappings and timeouts.
|
||||
- `stats.toml` — Metrics TTL and paths for UI/API.
|
||||
|
||||
### Environment Variables
|
||||
Override any setting via environment variables or by editing these files directly.
|
||||
|
||||
- `PORT` — Server port (default: 3000)
|
||||
- `NODE_ENV` — Environment mode (production/development)
|
||||
- `MAXMIND_ACCOUNT_ID` — MaxMind account ID for GeoIP databases
|
||||
- `MAXMIND_LICENSE_KEY` — MaxMind license key
|
||||
- `MAX_BODY_SIZE` — Request body size limit (default: 10mb)
|
||||
- `MAX_BODY_SIZE_MB` — WAF body size limit in MB (default: 10)
|
||||
|
||||
## 📂 Project Structure
|
||||
## 📂 Directory Structure
|
||||
|
||||
```plaintext
|
||||
.
|
||||
├── config/ # TOML configuration files
|
||||
├── data/ # Runtime data (secrets, downloads)
|
||||
├── data/ # Runtime data (secrets, snapshots)
|
||||
├── db/ # LevelDB token stores
|
||||
├── plugins/ # Plugin modules (checkpoint, ipfilter, proxy, stats)
|
||||
├── pages/ # Static assets and UI templates
|
||||
│ ├── interstitial/ # Proof-of-work challenge pages
|
||||
│ ├── ipfilter/ # Custom geo-block pages
|
||||
│ └── dashboard/ # Admin dashboard (if enabled)
|
||||
├── src/ # TypeScript source code
|
||||
│ ├── plugins/ # Plugin modules
|
||||
│ │ ├── ipfilter.ts # Geographic filtering
|
||||
│ │ └── waf.ts # Web Application Firewall
|
||||
│ ├── utils/ # Utility modules
|
||||
│ │ ├── behavioral-detection.ts
|
||||
│ │ ├── behavioral-middleware.ts
|
||||
│ │ ├── bot-verification.ts
|
||||
│ │ ├── cache-utils.ts
|
||||
│ │ ├── logs.ts
|
||||
│ │ ├── network.ts
|
||||
│ │ ├── performance.ts
|
||||
│ │ ├── plugins.ts
|
||||
│ │ ├── proof.ts
|
||||
│ │ ├── threat-scoring/
|
||||
│ │ └── time.ts
|
||||
│ ├── checkpoint.ts # Checkpoint security middleware
|
||||
│ ├── index.ts # Main application entry
|
||||
│ └── proxy.ts # Reverse proxy implementation
|
||||
├── dist/ # Compiled JavaScript (generated)
|
||||
├── .tests/ # Test files
|
||||
├── docker-compose-synology.yml
|
||||
├── Dockerfile
|
||||
├── jest.config.cjs
|
||||
├── package.json
|
||||
├── tsconfig.json
|
||||
└── README.md
|
||||
│ ├── ipfilter/ # Custom block pages
|
||||
│ └── stats/ # Statistics web UI
|
||||
├── utils/ # Internal utilities (logging, network, proof, time)
|
||||
├── index.js # Core server & plugin loader
|
||||
├── checkpoint.js # Checkpoint security middleware
|
||||
├── package.json # Project metadata & scripts
|
||||
└── README.md # This file
|
||||
```
|
||||
|
||||
## 🏗️ Architecture
|
||||
|
||||
The gateway processes requests through a layered security pipeline:
|
||||
|
||||
1. **Pre-filtering** — Request exclusion rules
|
||||
2. **IP Filter** — Geographic and ASN-based blocking
|
||||
3. **WAF** — Pattern matching and attack detection
|
||||
4. **Behavioral Detection** — Cross-request pattern analysis
|
||||
5. **Threat Scoring** — Aggregate risk assessment
|
||||
6. **Checkpoint** — Challenge suspicious requests
|
||||
7. **Proxy** — Forward legitimate traffic to backends
|
||||
|
||||
## 🔒 Security Features
|
||||
|
||||
### Web Application Firewall
|
||||
- SQL injection detection with evasion handling
|
||||
- XSS prevention across multiple vectors
|
||||
- Command injection blocking
|
||||
- Path traversal protection
|
||||
- XXE and SSRF prevention
|
||||
- Bot detection and verification
|
||||
|
||||
### Behavioral Analysis
|
||||
- Request pattern tracking
|
||||
- Rate limit enforcement
|
||||
- Geo-velocity detection
|
||||
- User agent consistency checks
|
||||
- Automated attack pattern recognition
|
||||
|
||||
### Threat Scoring Engine
|
||||
- Real-time risk calculation
|
||||
- Adaptive thresholds
|
||||
- Feature extraction from multiple sources
|
||||
- Configurable scoring weights
|
||||
- Automatic severity classification
|
||||
|
||||
## 📊 Default Security Thresholds
|
||||
|
||||
**Critical Threats (Immediate Block):**
|
||||
- `javascript:` URLs — Score: 100+
|
||||
- `<script>` tags — Score: 80+
|
||||
- Command injection — Score: 90+
|
||||
- SQL injection — Score: 70+
|
||||
|
||||
**Action Thresholds:**
|
||||
- Allow: 0-15 (normal traffic)
|
||||
- Challenge: 16-80 (suspicious)
|
||||
- Block: 80+ (malicious)
|
||||
|
||||
## 🚢 Deployment
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker build -t Checkpoint .
|
||||
docker run -d -p 3000:3000 -v $(pwd)/config:/app/config Checkpoint
|
||||
```
|
||||
|
||||
### Docker Compose (Synology)
|
||||
|
||||
```bash
|
||||
docker-compose -f docker-compose-synology.yml up -d
|
||||
```
|
||||
|
||||
### PM2 Process Manager
|
||||
|
||||
```bash
|
||||
npm run daemon # Start with PM2
|
||||
pm2 save # Save process list
|
||||
pm2 startup # Generate startup script
|
||||
```
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
```bash
|
||||
npm test # Run all tests
|
||||
npm run test:watch # Watch mode
|
||||
npm run test:coverage # Coverage report
|
||||
```
|
||||
|
||||
## 📈 Performance
|
||||
|
||||
- Handles 10,000+ requests/second
|
||||
- Sub-millisecond security decisions
|
||||
- Efficient caching and connection pooling
|
||||
- WebSocket support with proper cleanup
|
||||
963
checkpoint.js
Normal file
963
checkpoint.js
Normal file
|
|
@ -0,0 +1,963 @@
|
|||
import { registerPlugin, loadConfig, rootDir } from './index.js';
|
||||
import crypto from 'crypto';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { promises as fsPromises } from 'fs';
|
||||
import { dirname, join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { Level } from 'level';
|
||||
import cookie from 'cookie';
|
||||
import { parseDuration } from './utils/time.js';
|
||||
import { getRealIP } from './utils/network.js';
|
||||
import ttl from 'level-ttl';
|
||||
import { Readable } from 'stream';
|
||||
import {
|
||||
challengeStore,
|
||||
generateRequestID as proofGenerateRequestID,
|
||||
getChallengeParams,
|
||||
deleteChallenge,
|
||||
verifyPoW,
|
||||
verifyPoS,
|
||||
} from './utils/proof.js';
|
||||
import express from 'express';
|
||||
// Import recordEvent dynamically to avoid circular dependency issues
|
||||
let recordEvent;
|
||||
let statsLoadPromise = import('./plugins/stats.js')
|
||||
.then((stats) => {
|
||||
recordEvent = stats.recordEvent;
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('Failed to import stats module:', err);
|
||||
recordEvent = null;
|
||||
});
|
||||
|
||||
function sanitizePath(inputPath) {
|
||||
let pathOnly = inputPath.replace(/[\x00-\x1F\x7F]/g, '');
|
||||
|
||||
pathOnly = pathOnly.replace(/[<>;"'`|]/g, '');
|
||||
|
||||
const parts = pathOnly.split('/').filter((seg) => seg && seg !== '.' && seg !== '..');
|
||||
|
||||
return '/' + parts.map((seg) => encodeURIComponent(seg)).join('/');
|
||||
}
|
||||
|
||||
const checkpointConfig = {};
|
||||
let hmacSecret = null;
|
||||
const usedNonces = new Map();
|
||||
const ipRateLimit = new Map();
|
||||
|
||||
const tokenCache = new Map();
|
||||
|
||||
let db;
|
||||
|
||||
const tokenExpirations = new Map();
|
||||
|
||||
let interstitialTemplate = null;
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
function simpleTemplate(str) {
|
||||
return function (data) {
|
||||
return str.replace(/\{\{\s*([^{}]+?)\s*\}\}/g, (_, key) => {
|
||||
let value = data;
|
||||
|
||||
for (const part of key.trim().split('.')) {
|
||||
value = value?.[part];
|
||||
if (value == null) break;
|
||||
}
|
||||
return value != null ? String(value) : '';
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
async function initConfig() {
|
||||
await loadConfig('checkpoint', checkpointConfig);
|
||||
|
||||
// Handle new nested configuration structure
|
||||
// Map nested structure to flat structure for internal use
|
||||
checkpointConfig.Enabled = checkpointConfig.Core.Enabled;
|
||||
checkpointConfig.CookieName = checkpointConfig.Core.CookieName;
|
||||
checkpointConfig.CookieDomain = checkpointConfig.Core.CookieDomain;
|
||||
checkpointConfig.SanitizeURLs = checkpointConfig.Core.SanitizeURLs;
|
||||
|
||||
// Proof of Work settings
|
||||
checkpointConfig.Difficulty = checkpointConfig.ProofOfWork.Difficulty;
|
||||
checkpointConfig.SaltLength = checkpointConfig.ProofOfWork.SaltLength;
|
||||
checkpointConfig.ChallengeExpiration = parseDuration(
|
||||
checkpointConfig.ProofOfWork.ChallengeExpiration,
|
||||
);
|
||||
checkpointConfig.MaxAttemptsPerHour = checkpointConfig.ProofOfWork.MaxAttemptsPerHour;
|
||||
|
||||
// Proof of Space-Time settings
|
||||
checkpointConfig.CheckPoSTimes = checkpointConfig.ProofOfSpaceTime.Enabled;
|
||||
checkpointConfig.PoSTimeConsistencyRatio = checkpointConfig.ProofOfSpaceTime.ConsistencyRatio;
|
||||
|
||||
// Token settings
|
||||
checkpointConfig.TokenExpiration = parseDuration(checkpointConfig.Token.Expiration);
|
||||
checkpointConfig.MaxNonceAge = parseDuration(checkpointConfig.Token.MaxNonceAge);
|
||||
|
||||
// Storage settings
|
||||
checkpointConfig.SecretConfigPath = checkpointConfig.Storage.SecretPath;
|
||||
checkpointConfig.TokenStoreDBPath = checkpointConfig.Storage.TokenDBPath;
|
||||
checkpointConfig.InterstitialPaths = checkpointConfig.Storage.InterstitialTemplates;
|
||||
|
||||
// Process exclusions
|
||||
checkpointConfig.ExclusionRules = checkpointConfig.Exclusion || [];
|
||||
|
||||
// Process bypass keys
|
||||
checkpointConfig.BypassQueryKeys = [];
|
||||
checkpointConfig.BypassHeaderKeys = [];
|
||||
checkpointConfig.BypassKeys.forEach((key) => {
|
||||
if (key.Type === 'query') {
|
||||
checkpointConfig.BypassQueryKeys.push({
|
||||
Key: key.Key,
|
||||
Value: key.Value,
|
||||
Domains: key.Hosts || [],
|
||||
});
|
||||
} else if (key.Type === 'header') {
|
||||
checkpointConfig.BypassHeaderKeys.push({
|
||||
Name: key.Key,
|
||||
Value: key.Value,
|
||||
Domains: key.Hosts || [],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Extension handling
|
||||
checkpointConfig.HTMLCheckpointIncludedExtensions = checkpointConfig.Extensions?.IncludeOnly || [];
|
||||
checkpointConfig.HTMLCheckpointExcludedExtensions = checkpointConfig.Extensions?.Exclude || [];
|
||||
|
||||
// Remove legacy arrays
|
||||
checkpointConfig.HTMLCheckpointExclusions = [];
|
||||
checkpointConfig.UserAgentValidationExclusions = [];
|
||||
checkpointConfig.UserAgentRequiredPrefixes = {};
|
||||
checkpointConfig.ReverseProxyMappings = {};
|
||||
}
|
||||
|
||||
function addReadStreamSupport(dbInstance) {
|
||||
if (!dbInstance.createReadStream) {
|
||||
dbInstance.createReadStream = (opts) =>
|
||||
Readable.from(
|
||||
(async function* () {
|
||||
for await (const [key, value] of dbInstance.iterator(opts)) {
|
||||
yield { key, value };
|
||||
}
|
||||
})(),
|
||||
);
|
||||
}
|
||||
return dbInstance;
|
||||
}
|
||||
|
||||
function initTokenStore() {
|
||||
try {
|
||||
const storePath = join(rootDir, checkpointConfig.TokenStoreDBPath || 'db/tokenstore');
|
||||
fs.mkdirSync(storePath, { recursive: true });
|
||||
|
||||
let rawDB = new Level(storePath, { valueEncoding: 'json' });
|
||||
|
||||
addReadStreamSupport(rawDB);
|
||||
db = ttl(rawDB, { defaultTTL: checkpointConfig.TokenExpiration });
|
||||
|
||||
addReadStreamSupport(db);
|
||||
console.log('Token store initialized with TTL');
|
||||
} catch (err) {
|
||||
console.error('Failed to initialize token store:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function getFullClientIP(request) {
|
||||
const ip = getRealIP(request) || '';
|
||||
const h = crypto.createHash('sha256').update(ip).digest();
|
||||
return h.slice(0, 8).toString('hex');
|
||||
}
|
||||
|
||||
function hashUserAgent(ua) {
|
||||
if (!ua) return '';
|
||||
const h = crypto.createHash('sha256').update(ua).digest();
|
||||
return h.slice(0, 8).toString('hex');
|
||||
}
|
||||
|
||||
function extractBrowserFingerprint(request) {
|
||||
const headers = [
|
||||
'sec-ch-ua',
|
||||
'sec-ch-ua-platform',
|
||||
'sec-ch-ua-mobile',
|
||||
'sec-ch-ua-platform-version',
|
||||
'sec-ch-ua-arch',
|
||||
'sec-ch-ua-model',
|
||||
];
|
||||
const parts = headers.map((h) => request.headers.get(h)).filter(Boolean);
|
||||
if (!parts.length) return '';
|
||||
const buf = Buffer.from(parts.join('|'));
|
||||
const h = crypto.createHash('sha256').update(buf).digest();
|
||||
return h.slice(0, 12).toString('hex');
|
||||
}
|
||||
|
||||
async function getInterstitialTemplate() {
|
||||
if (!interstitialTemplate) {
|
||||
for (const p of checkpointConfig.InterstitialPaths) {
|
||||
try {
|
||||
let templatePath = join(__dirname, p);
|
||||
if (fs.existsSync(templatePath)) {
|
||||
const raw = await fsPromises.readFile(templatePath, 'utf8');
|
||||
interstitialTemplate = simpleTemplate(raw);
|
||||
break;
|
||||
}
|
||||
|
||||
templatePath = join(rootDir, p);
|
||||
if (fs.existsSync(templatePath)) {
|
||||
const raw = await fsPromises.readFile(templatePath, 'utf8');
|
||||
interstitialTemplate = simpleTemplate(raw);
|
||||
break;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`Failed to load interstitial template from path ${p}:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
if (!interstitialTemplate) {
|
||||
// Create a minimal fallback template
|
||||
console.warn('Could not find interstitial HTML template, using minimal fallback');
|
||||
interstitialTemplate = simpleTemplate(`
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Security Verification</title>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
</head>
|
||||
<body>
|
||||
<h1>Security Verification Required</h1>
|
||||
<p>Please wait while we verify your request...</p>
|
||||
<div id="verification-data"
|
||||
data-target="{{TargetPath}}"
|
||||
data-request-id="{{RequestID}}">
|
||||
</div>
|
||||
<script src="/js/c.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
`);
|
||||
}
|
||||
}
|
||||
return interstitialTemplate;
|
||||
}
|
||||
|
||||
// Helper function for safe stats recording
|
||||
function safeRecordEvent(metric, data) {
|
||||
// If recordEvent is not yet loaded, try to wait for it
|
||||
if (!recordEvent && statsLoadPromise) {
|
||||
statsLoadPromise.then(() => {
|
||||
if (recordEvent) {
|
||||
try {
|
||||
recordEvent(metric, data);
|
||||
} catch (err) {
|
||||
console.error(`Failed to record ${metric} event:`, err);
|
||||
}
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (typeof recordEvent === 'function') {
|
||||
try {
|
||||
recordEvent(metric, data);
|
||||
} catch (err) {
|
||||
console.error(`Failed to record ${metric} event:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function serveInterstitial(request) {
|
||||
const ip = getRealIP(request);
|
||||
const requestPath = new URL(request.url).pathname;
|
||||
safeRecordEvent('checkpoint.sent', { ip, path: requestPath });
|
||||
let tpl;
|
||||
try {
|
||||
tpl = await getInterstitialTemplate();
|
||||
} catch (err) {
|
||||
console.error('Interstitial template error:', err);
|
||||
return new Response('Security verification required.', {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'text/plain' },
|
||||
});
|
||||
}
|
||||
|
||||
const requestID = proofGenerateRequestID(request, checkpointConfig);
|
||||
const url = new URL(request.url);
|
||||
const host = request.headers.get('host') || url.hostname;
|
||||
const targetPath = url.pathname;
|
||||
const fullURL = request.url;
|
||||
|
||||
const html = tpl({
|
||||
TargetPath: targetPath,
|
||||
RequestID: requestID,
|
||||
Host: host,
|
||||
FullURL: fullURL,
|
||||
});
|
||||
|
||||
return new Response(html, {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'text/html; charset=utf-8' },
|
||||
});
|
||||
}
|
||||
|
||||
async function handleGetCheckpointChallenge(request) {
|
||||
const url = new URL(request.url);
|
||||
const requestID = url.searchParams.get('id');
|
||||
if (!requestID) {
|
||||
return new Response(JSON.stringify({ error: 'Missing request ID' }), {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
const ip = getRealIP(request);
|
||||
const attempts = (ipRateLimit.get(ip) || 0) + 1;
|
||||
ipRateLimit.set(ip, attempts);
|
||||
|
||||
if (attempts > checkpointConfig.MaxAttemptsPerHour) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'Too many challenge requests. Try again later.' }),
|
||||
{
|
||||
status: 429,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const params = getChallengeParams(requestID);
|
||||
if (!params) {
|
||||
return new Response(JSON.stringify({ error: 'Challenge not found or expired' }), {
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
if (ip !== params.ClientIP) {
|
||||
return new Response(JSON.stringify({ error: 'IP address mismatch for challenge' }), {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
const payload = {
|
||||
a: params.Challenge,
|
||||
b: params.Salt,
|
||||
c: params.Difficulty,
|
||||
d: params.PoSSeed,
|
||||
};
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
function calculateTokenHash(token) {
|
||||
const data = `${token.Nonce}:${token.Entropy}:${token.Created.getTime()}`;
|
||||
return crypto.createHash('sha256').update(data).digest('hex');
|
||||
}
|
||||
|
||||
function computeTokenSignature(token) {
|
||||
const copy = { ...token, Signature: '' };
|
||||
const serialized = JSON.stringify(copy);
|
||||
return crypto.createHmac('sha256', hmacSecret).update(serialized).digest('hex');
|
||||
}
|
||||
|
||||
function verifyTokenSignature(token) {
|
||||
if (!token.Signature) return false;
|
||||
const expected = computeTokenSignature(token);
|
||||
try {
|
||||
return crypto.timingSafeEqual(
|
||||
Buffer.from(token.Signature, 'hex'),
|
||||
Buffer.from(expected, 'hex'),
|
||||
);
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function issueToken(request, token) {
|
||||
const tokenHash = calculateTokenHash(token);
|
||||
const storedData = {
|
||||
ClientIPHash: token.ClientIP,
|
||||
UserAgentHash: token.UserAgent,
|
||||
BrowserHint: token.BrowserHint,
|
||||
LastVerified: new Date(token.LastVerified).toISOString(),
|
||||
ExpiresAt: new Date(token.ExpiresAt).toISOString(),
|
||||
};
|
||||
|
||||
try {
|
||||
await addToken(tokenHash, storedData);
|
||||
} catch (err) {
|
||||
console.error('Failed to store token:', err);
|
||||
}
|
||||
|
||||
token.Signature = computeTokenSignature(token);
|
||||
|
||||
const tokenStr = Buffer.from(JSON.stringify(token)).toString('base64');
|
||||
|
||||
const url = new URL(request.url);
|
||||
const cookieDomain = checkpointConfig.CookieDomain || '';
|
||||
const sameSite = cookieDomain ? 'Lax' : 'Strict';
|
||||
const secure = url.protocol === 'https:';
|
||||
const expires = new Date(token.ExpiresAt).toUTCString();
|
||||
|
||||
const domainPart = cookieDomain ? `; Domain=${cookieDomain}` : '';
|
||||
const securePart = secure ? '; Secure' : '';
|
||||
const cookieStr =
|
||||
`${checkpointConfig.CookieName}=${tokenStr}; Path=/` +
|
||||
`${domainPart}; Expires=${expires}; HttpOnly; SameSite=${sameSite}${securePart}`;
|
||||
return new Response(JSON.stringify({ token: tokenStr, expires_at: token.ExpiresAt }), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Set-Cookie': cookieStr,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function handleVerifyCheckpoint(request) {
|
||||
let body;
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch (e) {
|
||||
safeRecordEvent('checkpoint.failure', { reason: 'invalid_json', ip: getRealIP(request) });
|
||||
return new Response(JSON.stringify({ error: 'Invalid JSON' }), {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
const ip = getRealIP(request);
|
||||
const params = getChallengeParams(body.request_id);
|
||||
|
||||
if (!params) {
|
||||
safeRecordEvent('checkpoint.failure', { reason: 'invalid_or_expired_request', ip });
|
||||
return new Response(JSON.stringify({ error: 'Invalid or expired request ID' }), {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
if (ip !== params.ClientIP) {
|
||||
safeRecordEvent('checkpoint.failure', { reason: 'ip_mismatch', ip });
|
||||
return new Response(JSON.stringify({ error: 'IP address mismatch' }), {
|
||||
status: 403,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
const challenge = params.Challenge;
|
||||
const salt = params.Salt;
|
||||
|
||||
if (!body.g || !verifyPoW(challenge, salt, body.g, params.Difficulty)) {
|
||||
safeRecordEvent('checkpoint.failure', { reason: 'invalid_pow', ip });
|
||||
return new Response(JSON.stringify({ error: 'Invalid proof-of-work solution' }), {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
const nonceKey = body.g + challenge;
|
||||
usedNonces.set(nonceKey, Date.now());
|
||||
|
||||
if (body.h?.length === 3 && body.i?.length === 3) {
|
||||
try {
|
||||
verifyPoS(body.h, body.i, checkpointConfig);
|
||||
} catch (e) {
|
||||
safeRecordEvent('checkpoint.failure', { reason: 'invalid_pos', ip });
|
||||
return new Response(JSON.stringify({ error: e.message }), {
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
deleteChallenge(body.request_id);
|
||||
safeRecordEvent('checkpoint.success', { ip });
|
||||
const now = new Date();
|
||||
const expiresAt = new Date(now.getTime() + checkpointConfig.TokenExpiration);
|
||||
|
||||
const token = {
|
||||
Nonce: body.g,
|
||||
ExpiresAt: expiresAt,
|
||||
ClientIP: getFullClientIP(request),
|
||||
UserAgent: hashUserAgent(request.headers.get('user-agent')),
|
||||
BrowserHint: extractBrowserFingerprint(request),
|
||||
Entropy: crypto.randomBytes(8).toString('hex'),
|
||||
Created: now,
|
||||
LastVerified: now,
|
||||
TokenFormat: 2,
|
||||
};
|
||||
|
||||
token.Signature = computeTokenSignature(token);
|
||||
const tokenStr = Buffer.from(JSON.stringify(token)).toString('base64');
|
||||
|
||||
const tokenKey = crypto.createHash('sha256').update(tokenStr).digest('hex');
|
||||
try {
|
||||
await db.put(tokenKey, true);
|
||||
tokenCache.set(tokenKey, true);
|
||||
|
||||
tokenExpirations.set(tokenKey, new Date(token.ExpiresAt).getTime());
|
||||
console.log(`checkpoint: token stored in DB and cache key=${tokenKey}`);
|
||||
} catch (e) {
|
||||
console.error('checkpoint: failed to store token in DB:', e);
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify({ token: tokenStr, expires_at: token.ExpiresAt }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
||||
function generateUpdatedCookie(token, secure) {
|
||||
token.Signature = computeTokenSignature(token);
|
||||
const tokenStr = Buffer.from(JSON.stringify(token)).toString('base64');
|
||||
const cookieDomain = checkpointConfig.CookieDomain || '';
|
||||
const sameSite = cookieDomain ? 'Lax' : 'Strict';
|
||||
const expires = new Date(token.ExpiresAt).toUTCString();
|
||||
|
||||
const domainPart = cookieDomain ? `; Domain=${cookieDomain}` : '';
|
||||
const securePart = secure ? '; Secure' : '';
|
||||
const cookieStr =
|
||||
`${checkpointConfig.CookieName}=${tokenStr}; Path=/` +
|
||||
`${domainPart}; Expires=${expires}; HttpOnly; SameSite=${sameSite}${securePart}`;
|
||||
return cookieStr;
|
||||
}
|
||||
|
||||
async function validateToken(tokenStr, request) {
|
||||
if (!tokenStr) return false;
|
||||
let token;
|
||||
try {
|
||||
token = JSON.parse(Buffer.from(tokenStr, 'base64').toString());
|
||||
} catch {
|
||||
console.log('checkpoint: invalid token format');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (Date.now() > new Date(token.ExpiresAt).getTime()) {
|
||||
console.log('checkpoint: token expired');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!verifyTokenSignature(token)) {
|
||||
console.log('checkpoint: invalid token signature');
|
||||
return false;
|
||||
}
|
||||
|
||||
const tokenKey = crypto.createHash('sha256').update(tokenStr).digest('hex');
|
||||
|
||||
if (tokenCache.has(tokenKey)) return true;
|
||||
|
||||
try {
|
||||
await db.get(tokenKey);
|
||||
tokenCache.set(tokenKey, true);
|
||||
|
||||
tokenExpirations.set(tokenKey, new Date(token.ExpiresAt).getTime());
|
||||
return true;
|
||||
} catch {
|
||||
console.log('checkpoint: token not found in DB');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTokenRedirect(request) {
|
||||
const url = new URL(request.url);
|
||||
const tokenStr = url.searchParams.get('token');
|
||||
if (!tokenStr) return undefined;
|
||||
|
||||
let token;
|
||||
try {
|
||||
token = JSON.parse(Buffer.from(tokenStr, 'base64').toString());
|
||||
|
||||
if (Date.now() > new Date(token.ExpiresAt).getTime()) {
|
||||
console.log('checkpoint: token in URL parameter expired');
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (!verifyTokenSignature(token)) {
|
||||
console.log('checkpoint: invalid token signature in URL parameter');
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const tokenKey = crypto.createHash('sha256').update(tokenStr).digest('hex');
|
||||
try {
|
||||
await db.get(tokenKey);
|
||||
} catch {
|
||||
console.log('checkpoint: token in URL parameter not found in DB');
|
||||
return undefined;
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('checkpoint: invalid token format in URL parameter', e);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const expires = new Date(token.ExpiresAt).toUTCString();
|
||||
const cookieDomain = checkpointConfig.CookieDomain || '';
|
||||
const sameSite = cookieDomain ? 'Lax' : 'Strict';
|
||||
const securePart = url.protocol === 'https:' ? '; Secure' : '';
|
||||
const domainPart = cookieDomain ? `; Domain=${cookieDomain}` : '';
|
||||
const cookieStr =
|
||||
`${checkpointConfig.CookieName}=${tokenStr}; Path=/` +
|
||||
`${domainPart}; Expires=${expires}; HttpOnly; SameSite=${sameSite}${securePart}`;
|
||||
|
||||
url.searchParams.delete('token');
|
||||
const cleanUrl = url.pathname + (url.search || '');
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: {
|
||||
'Set-Cookie': cookieStr,
|
||||
Location: cleanUrl,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function CheckpointMiddleware() {
|
||||
// Return Express-compatible middleware
|
||||
return {
|
||||
middleware: [
|
||||
// Add body parser middleware for JSON
|
||||
express.json({ limit: '10mb' }),
|
||||
// Main checkpoint middleware
|
||||
async (req, res, next) => {
|
||||
// Check if checkpoint is enabled
|
||||
if (checkpointConfig.Enabled === false) {
|
||||
return next();
|
||||
}
|
||||
|
||||
// Convert Express request to the format expected by checkpoint logic
|
||||
const request = {
|
||||
url: `${req.protocol}://${req.get('host')}${req.originalUrl}`,
|
||||
method: req.method,
|
||||
headers: {
|
||||
get: (name) => req.get(name),
|
||||
entries: () => Object.entries(req.headers).map(([k, v]) => [k, Array.isArray(v) ? v.join(', ') : v])
|
||||
},
|
||||
json: () => Promise.resolve(req.body)
|
||||
};
|
||||
|
||||
const urlObj = new URL(request.url);
|
||||
const host = request.headers.get('host')?.split(':')[0];
|
||||
const userAgent = request.headers.get('user-agent') || '';
|
||||
|
||||
// 1) Bypass via query keys
|
||||
for (const { Key, Value, Domains } of checkpointConfig.BypassQueryKeys) {
|
||||
if (urlObj.searchParams.get(Key) === Value) {
|
||||
if (!Array.isArray(Domains) || Domains.length === 0 || Domains.includes(host)) {
|
||||
return next();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Bypass via header keys
|
||||
for (const { Name, Value, Domains } of checkpointConfig.BypassHeaderKeys) {
|
||||
// Get header value case-insensitively by checking all headers
|
||||
let headerVal = null;
|
||||
const headersMap = Object.fromEntries([...request.headers.entries()].map(([k, v]) => [k.toLowerCase(), v]));
|
||||
headerVal = headersMap[Name.toLowerCase()] || request.headers.get(Name);
|
||||
|
||||
console.log(`DEBUG - Checking header ${Name}: received="${headerVal}", expected="${Value}", domains=${JSON.stringify(Domains)}`);
|
||||
|
||||
if (headerVal === Value) {
|
||||
console.log(`DEBUG - Header value matched for ${Name}`);
|
||||
if (!Array.isArray(Domains) || Domains.length === 0 || Domains.includes(host)) {
|
||||
console.log(`DEBUG - Domain check passed for ${host}`);
|
||||
return next();
|
||||
} else {
|
||||
console.log(`DEBUG - Domain check failed: ${host} not in ${JSON.stringify(Domains)}`);
|
||||
}
|
||||
} else {
|
||||
console.log(`DEBUG - Header value mismatch for ${Name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle token redirect for URL-token login
|
||||
const tokenResponse = await handleTokenRedirect(request);
|
||||
if (tokenResponse) {
|
||||
// Convert Response to Express response
|
||||
res.status(tokenResponse.status);
|
||||
tokenResponse.headers.forEach((value, key) => {
|
||||
res.setHeader(key, value);
|
||||
});
|
||||
const body = await tokenResponse.text();
|
||||
return res.send(body);
|
||||
}
|
||||
|
||||
// Setup request context
|
||||
const url = new URL(request.url);
|
||||
let path = url.pathname;
|
||||
if (checkpointConfig.SanitizeURLs) {
|
||||
path = sanitizePath(path);
|
||||
}
|
||||
const method = request.method;
|
||||
|
||||
// Always allow challenge & verify endpoints
|
||||
if (method === 'GET' && path === '/api/challenge') {
|
||||
const response = await handleGetCheckpointChallenge(request);
|
||||
res.status(response.status);
|
||||
response.headers.forEach((value, key) => {
|
||||
res.setHeader(key, value);
|
||||
});
|
||||
const body = await response.text();
|
||||
return res.send(body);
|
||||
}
|
||||
if (method === 'POST' && path === '/api/verify') {
|
||||
const response = await handleVerifyCheckpoint(request);
|
||||
res.status(response.status);
|
||||
response.headers.forEach((value, key) => {
|
||||
res.setHeader(key, value);
|
||||
});
|
||||
const body = await response.text();
|
||||
return res.send(body);
|
||||
}
|
||||
|
||||
// Check new exclusion rules
|
||||
if (checkpointConfig.ExclusionRules && checkpointConfig.ExclusionRules.length > 0) {
|
||||
for (const rule of checkpointConfig.ExclusionRules) {
|
||||
// Check if path matches
|
||||
if (!rule.Path || !path.startsWith(rule.Path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if host matches (if specified)
|
||||
if (rule.Hosts && rule.Hosts.length > 0 && !rule.Hosts.includes(host)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if user agent matches (if specified)
|
||||
if (rule.UserAgents && rule.UserAgents.length > 0) {
|
||||
const matchesUA = rule.UserAgents.some((ua) => userAgent.includes(ua));
|
||||
if (!matchesUA) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// All conditions match - exclude this request
|
||||
return next();
|
||||
}
|
||||
}
|
||||
|
||||
// Only checkpoint requests explicitly accepting 'text/html'
|
||||
const acceptHeader = request.headers.get('accept') || '';
|
||||
if (!acceptHeader.toLowerCase().includes('text/html')) {
|
||||
return next();
|
||||
}
|
||||
|
||||
// Validate session token
|
||||
const cookies = cookie.parse(request.headers.get('cookie') || '');
|
||||
const tokenCookie = cookies[checkpointConfig.CookieName];
|
||||
const validation = await validateToken(tokenCookie, request);
|
||||
if (validation) {
|
||||
// Active session: bypass checkpoint
|
||||
return next();
|
||||
}
|
||||
|
||||
// Log new checkpoint flow
|
||||
console.log(`checkpoint: incoming ${method} ${request.url}`);
|
||||
console.log(`checkpoint: tokenCookie=${tokenCookie}`);
|
||||
console.log(`checkpoint: validateToken => ${validation}`);
|
||||
|
||||
// Serve interstitial challenge
|
||||
const response = await serveInterstitial(request);
|
||||
res.status(response.status);
|
||||
response.headers.forEach((value, key) => {
|
||||
res.setHeader(key, value);
|
||||
});
|
||||
const body = await response.text();
|
||||
return res.send(body);
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
async function addToken(tokenHash, data) {
|
||||
if (!db) return;
|
||||
try {
|
||||
const ttlMs = checkpointConfig.TokenExpiration;
|
||||
|
||||
await db.put(tokenHash, data);
|
||||
|
||||
tokenExpirations.set(tokenHash, Date.now() + ttlMs);
|
||||
} catch (err) {
|
||||
console.error('Error adding token:', err);
|
||||
}
|
||||
}
|
||||
|
||||
async function updateTokenVerification(tokenHash) {
|
||||
if (!db) return;
|
||||
try {
|
||||
const data = await db.get(tokenHash);
|
||||
data.LastVerified = new Date().toISOString();
|
||||
await db.put(tokenHash, data);
|
||||
} catch (err) {
|
||||
console.error('Error updating token verification:', err);
|
||||
}
|
||||
}
|
||||
|
||||
async function lookupTokenData(tokenHash) {
|
||||
if (!db) return { data: null, found: false };
|
||||
try {
|
||||
const expireTime = tokenExpirations.get(tokenHash);
|
||||
if (!expireTime || expireTime <= Date.now()) {
|
||||
if (expireTime) {
|
||||
tokenExpirations.delete(tokenHash);
|
||||
try {
|
||||
await db.del(tokenHash);
|
||||
} catch (e) {}
|
||||
}
|
||||
return { data: null, found: false };
|
||||
}
|
||||
|
||||
const data = await db.get(tokenHash);
|
||||
return { data, found: true };
|
||||
} catch (err) {
|
||||
if (err.code === 'LEVEL_NOT_FOUND') return { data: null, found: false };
|
||||
console.error('Error looking up token:', err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function closeTokenStore() {
|
||||
if (db) await db.close();
|
||||
}
|
||||
|
||||
function startCleanupTimer() {
|
||||
// Cleanup expired data hourly
|
||||
setInterval(() => {
|
||||
cleanupExpiredData();
|
||||
}, 3600000);
|
||||
// Cleanup expired challenges at the challenge expiration interval
|
||||
const challengeInterval = checkpointConfig.ChallengeExpiration || 60000;
|
||||
setInterval(() => {
|
||||
cleanupExpiredChallenges();
|
||||
}, challengeInterval);
|
||||
}
|
||||
|
||||
function cleanupExpiredData() {
|
||||
const now = Date.now();
|
||||
let count = 0;
|
||||
|
||||
try {
|
||||
for (const [nonce, ts] of usedNonces.entries()) {
|
||||
if (now - ts > checkpointConfig.MaxNonceAge) {
|
||||
usedNonces.delete(nonce);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
if (count) console.log(`Checkpoint: cleaned up ${count} expired nonces.`);
|
||||
} catch (err) {
|
||||
console.error('Error cleaning up nonces:', err);
|
||||
}
|
||||
|
||||
// Clean up expired tokens from cache
|
||||
let tokenCacheCount = 0;
|
||||
try {
|
||||
for (const [tokenKey, _] of tokenCache.entries()) {
|
||||
const expireTime = tokenExpirations.get(tokenKey);
|
||||
if (!expireTime || expireTime <= now) {
|
||||
tokenCache.delete(tokenKey);
|
||||
tokenExpirations.delete(tokenKey);
|
||||
tokenCacheCount++;
|
||||
}
|
||||
}
|
||||
if (tokenCacheCount)
|
||||
console.log(`Checkpoint: cleaned up ${tokenCacheCount} expired tokens from cache.`);
|
||||
} catch (err) {
|
||||
console.error('Error cleaning up token cache:', err);
|
||||
}
|
||||
|
||||
try {
|
||||
ipRateLimit.clear();
|
||||
console.log('Checkpoint: IP rate limits reset.');
|
||||
} catch (err) {
|
||||
console.error('Error resetting IP rate limits:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupExpiredChallenges() {
|
||||
const now = Date.now();
|
||||
let count = 0;
|
||||
for (const [id, params] of challengeStore.entries()) {
|
||||
if (params.ExpiresAt && params.ExpiresAt < now) {
|
||||
// Record failure for expired challenges that were never completed
|
||||
safeRecordEvent('checkpoint.failure', {
|
||||
reason: 'challenge_expired',
|
||||
ip: params.ClientIP,
|
||||
challenge_id: id.substring(0, 8), // Include partial ID for debugging
|
||||
age_ms: now - params.CreatedAt, // How long the challenge existed
|
||||
expiry_ms: checkpointConfig.ChallengeExpiration, // Configured expiry time
|
||||
});
|
||||
challengeStore.delete(id);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
if (count) console.log(`Checkpoint: cleaned up ${count} expired challenges.`);
|
||||
}
|
||||
|
||||
async function initSecret() {
|
||||
try {
|
||||
if (!checkpointConfig.SecretConfigPath) {
|
||||
checkpointConfig.SecretConfigPath = join(rootDir, 'data', 'checkpoint_secret.json');
|
||||
}
|
||||
|
||||
const secretPath = checkpointConfig.SecretConfigPath;
|
||||
const exists = fs.existsSync(secretPath);
|
||||
|
||||
if (exists) {
|
||||
const loaded = loadSecretFromFile();
|
||||
if (loaded) {
|
||||
hmacSecret = loaded;
|
||||
console.log(`Loaded existing HMAC secret from ${secretPath}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
hmacSecret = crypto.randomBytes(32);
|
||||
fs.mkdirSync(path.dirname(secretPath), { recursive: true });
|
||||
|
||||
const secretCfg = {
|
||||
hmac_secret: hmacSecret.toString('base64'),
|
||||
created_at: new Date().toISOString(),
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
|
||||
fs.writeFileSync(secretPath, JSON.stringify(secretCfg), { mode: 0o600 });
|
||||
console.log(`Created and saved new HMAC secret to ${secretPath}`);
|
||||
} catch (err) {
|
||||
console.error('Error initializing secret:', err);
|
||||
|
||||
hmacSecret = crypto.randomBytes(32);
|
||||
}
|
||||
}
|
||||
|
||||
function loadSecretFromFile() {
|
||||
try {
|
||||
const data = fs.readFileSync(checkpointConfig.SecretConfigPath, 'utf8');
|
||||
const cfg = JSON.parse(data);
|
||||
const buf = Buffer.from(cfg.hmac_secret, 'base64');
|
||||
if (buf.length < 16) return null;
|
||||
|
||||
cfg.updated_at = new Date().toISOString();
|
||||
fs.writeFileSync(checkpointConfig.SecretConfigPath, JSON.stringify(cfg), { mode: 0o600 });
|
||||
return buf;
|
||||
} catch (e) {
|
||||
console.warn('Could not load HMAC secret from file:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
(async function initialize() {
|
||||
await initConfig();
|
||||
await initSecret();
|
||||
initTokenStore();
|
||||
startCleanupTimer();
|
||||
|
||||
// Only register plugin if enabled
|
||||
if (checkpointConfig.Enabled !== false) {
|
||||
registerPlugin('checkpoint', CheckpointMiddleware());
|
||||
} else {
|
||||
console.log('Checkpoint plugin disabled via configuration');
|
||||
}
|
||||
})();
|
||||
|
||||
export { checkpointConfig, addToken, updateTokenVerification, lookupTokenData, closeTokenStore };
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
# =============================================================================
|
||||
# BEHAVIORAL DETECTION CONFIGURATION - EXAMPLE
|
||||
# =============================================================================
|
||||
# Copy this file to behavioral-detection.toml and customize for your environment
|
||||
# =============================================================================
|
||||
|
||||
[Core]
|
||||
# Enable or disable the behavioral detection engine
|
||||
Enabled = true
|
||||
|
||||
# Operation mode: "detect" (log only) or "prevent" (actively block/rate limit)
|
||||
Mode = "prevent"
|
||||
|
||||
# Default time window for metrics (milliseconds)
|
||||
DefaultTimeWindow = 300000 # 5 minutes
|
||||
|
||||
# Maximum request history to keep per IP
|
||||
MaxHistoryPerIP = 1000
|
||||
|
||||
# Database cleanup interval (milliseconds)
|
||||
CleanupInterval = 3600000 # 1 hour
|
||||
|
||||
# =============================================================================
|
||||
# EXAMPLE DETECTION RULES
|
||||
# =============================================================================
|
||||
|
||||
[[Rules]]
|
||||
Name = "404 Path Enumeration"
|
||||
Type = "enumeration"
|
||||
Severity = "medium"
|
||||
Description = "Detects rapid 404 responses indicating directory/file scanning"
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "status_code_count"
|
||||
StatusCode = 404
|
||||
Threshold = 15
|
||||
TimeWindow = 60000 # 1 minute
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "unique_paths_by_status"
|
||||
StatusCode = 404
|
||||
Threshold = 10
|
||||
TimeWindow = 60000
|
||||
|
||||
[Rules.Action]
|
||||
Score = 30
|
||||
Tags = ["scanning", "enumeration", "reconnaissance"]
|
||||
RateLimit = { Requests = 10, Window = 60000 }
|
||||
Alert = false
|
||||
|
||||
# Authentication bruteforce rule removed - not applicable for this security system
|
||||
|
||||
[[Rules]]
|
||||
Name = "API Endpoint Enumeration"
|
||||
Type = "enumeration"
|
||||
Severity = "medium"
|
||||
Description = "Scanning for API endpoints"
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "unique_api_paths"
|
||||
PathPrefix = "/api/"
|
||||
Threshold = 20
|
||||
TimeWindow = 60000
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "mixed_http_methods"
|
||||
PathPrefix = "/api/"
|
||||
MinMethods = 3 # GET, POST, PUT, DELETE, etc.
|
||||
TimeWindow = 60000
|
||||
|
||||
[Rules.Action]
|
||||
Score = 25
|
||||
Tags = ["api_abuse", "enumeration"]
|
||||
RateLimit = { Requests = 20, Window = 60000 }
|
||||
|
||||
[[Rules]]
|
||||
Name = "Velocity-Based Scanner"
|
||||
Type = "scanning"
|
||||
Severity = "medium"
|
||||
Description = "High-speed request patterns typical of automated scanners"
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "request_velocity"
|
||||
RequestsPerSecond = 10
|
||||
Duration = 5000 # Sustained for 5 seconds
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "request_regularity"
|
||||
MaxVariance = 0.1 # Very regular timing
|
||||
MinRequests = 20
|
||||
|
||||
[Rules.Action]
|
||||
Score = 35
|
||||
Tags = ["automated_scanner", "bot"]
|
||||
Challenge = true # Show CAPTCHA or similar
|
||||
|
||||
[[Rules]]
|
||||
Name = "Admin Interface Probing"
|
||||
Type = "reconnaissance"
|
||||
Severity = "medium"
|
||||
Description = "Attempts to find admin interfaces"
|
||||
|
||||
[[Rules.Triggers]]
|
||||
Metric = "path_status_combo"
|
||||
PathPattern = "^/(wp-)?admin|^/administrator|^/manage|^/cpanel|^/phpmyadmin"
|
||||
StatusCodes = [200, 301, 302, 403, 404]
|
||||
Threshold = 5
|
||||
TimeWindow = 300000
|
||||
|
||||
[Rules.Action]
|
||||
Score = 25
|
||||
Tags = ["admin_probe", "reconnaissance"]
|
||||
RateLimit = { Requests = 5, Window = 300000 }
|
||||
|
||||
# =============================================================================
|
||||
# CORRELATION RULES EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
[[Correlations]]
|
||||
Name = "Rotating User-Agent Attack"
|
||||
Description = "Same IP using multiple user agents rapidly"
|
||||
|
||||
[Correlations.Conditions]
|
||||
Metric = "unique_user_agents_per_ip"
|
||||
Threshold = 5
|
||||
TimeWindow = 60000
|
||||
|
||||
[Correlations.Action]
|
||||
Score = 20
|
||||
Tags = ["evasion", "user_agent_rotation"]
|
||||
|
||||
# =============================================================================
|
||||
# BEHAVIORAL THRESHOLDS
|
||||
# =============================================================================
|
||||
|
||||
[Thresholds]
|
||||
# Minimum score to trigger any action
|
||||
MinActionScore = 20
|
||||
|
||||
# Score thresholds for different severity levels
|
||||
LowSeverityThreshold = 20
|
||||
MediumSeverityThreshold = 40
|
||||
HighSeverityThreshold = 60
|
||||
CriticalSeverityThreshold = 80
|
||||
|
||||
# =============================================================================
|
||||
# WHITELISTING
|
||||
# =============================================================================
|
||||
|
||||
[Whitelist]
|
||||
# IPs that should never be blocked by behavioral rules
|
||||
TrustedIPs = [
|
||||
"127.0.0.1",
|
||||
"::1"
|
||||
# Add your monitoring service IPs here
|
||||
]
|
||||
|
||||
# User agents to treat with lower sensitivity
|
||||
TrustedUserAgents = [
|
||||
"Googlebot",
|
||||
"bingbot",
|
||||
"Slackbot",
|
||||
"monitoring-bot"
|
||||
]
|
||||
|
||||
# Paths where higher thresholds apply
|
||||
MonitoringPaths = [
|
||||
"/health",
|
||||
"/metrics",
|
||||
"/api/status",
|
||||
"/.well-known/",
|
||||
"/robots.txt",
|
||||
"/sitemap.xml"
|
||||
]
|
||||
|
||||
# =============================================================================
|
||||
# RESPONSE CUSTOMIZATION
|
||||
# =============================================================================
|
||||
|
||||
[Responses]
|
||||
# Custom block message (can include HTML)
|
||||
BlockMessage = """
|
||||
<html>
|
||||
<head><title>Access Denied</title></head>
|
||||
<body>
|
||||
<h1>Access Denied</h1>
|
||||
<p>Your access has been restricted due to suspicious activity.</p>
|
||||
<p>If you believe this is an error, please contact support.</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Rate limit message
|
||||
RateLimitMessage = "Rate limit exceeded. Please slow down your requests."
|
||||
|
||||
# Challenge page URL (for CAPTCHA/verification)
|
||||
ChallengePageURL = "/verify"
|
||||
|
|
@ -20,8 +20,8 @@ AccountID = ""
|
|||
# Can also be set via MAXMIND_LICENSE_KEY environment variable or .env file
|
||||
LicenseKey = ""
|
||||
|
||||
# How often to check for database updates (uses time.ts format: "24h", "5m", etc.)
|
||||
DBUpdateInterval = "12h"
|
||||
# How often to check for database updates (in hours)
|
||||
DBUpdateIntervalHours = 12
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# CACHING SETTINGS
|
||||
|
|
|
|||
|
|
@ -12,9 +12,6 @@
|
|||
# Enable or disable the proxy middleware
|
||||
Enabled = true
|
||||
|
||||
# Maximum body size in MB (default: 10MB if not specified)
|
||||
MaxBodySizeMB = 10
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# TIMEOUT SETTINGS
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -30,8 +27,6 @@ UpstreamTimeoutMs = 30000
|
|||
# -----------------------------------------------------------------------------
|
||||
# Map hostnames to backend service URLs
|
||||
# Format: "hostname" = "backend_url"
|
||||
# Optional: AllowedMethods = ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH", "TRACE", "CONNECT"]
|
||||
# If AllowedMethods is not specified, defaults to ["GET", "HEAD", "POST", "PUT", "OPTIONS"]
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
[[Mapping]]
|
||||
|
|
@ -49,20 +44,12 @@ Target = "http://192.168.1.100:4533"
|
|||
Host = "git.example.com"
|
||||
Target = "http://192.168.1.100:3000"
|
||||
|
||||
[[Mapping]]
|
||||
# Gallery service with DELETE method enabled
|
||||
Host = "gallery.caileb.com"
|
||||
Target = "http://192.168.1.100:8080"
|
||||
AllowedMethods = ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"]
|
||||
|
||||
# [[Mapping]]
|
||||
# API service with specific methods
|
||||
# API service
|
||||
# Host = "api.example.com"
|
||||
# Target = "http://localhost:3001"
|
||||
# AllowedMethods = ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH"]
|
||||
|
||||
# [[Mapping]]
|
||||
# Admin panel (read-only)
|
||||
# Admin panel
|
||||
# Host = "admin.example.com"
|
||||
# Target = "http://localhost:3002"
|
||||
# AllowedMethods = ["GET", "HEAD", "OPTIONS"]
|
||||
31
config/stats.toml.example
Normal file
31
config/stats.toml.example
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# =============================================================================
|
||||
# STATS CONFIGURATION
|
||||
# =============================================================================
|
||||
# This configuration controls the statistics collection and visualization
|
||||
# middleware that tracks events and provides a web UI for viewing metrics.
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# CORE SETTINGS
|
||||
# -----------------------------------------------------------------------------
|
||||
[Core]
|
||||
# Enable or disable the stats plugin
|
||||
Enabled = true
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# STORAGE SETTINGS
|
||||
# -----------------------------------------------------------------------------
|
||||
[Storage]
|
||||
# TTL for stats entries
|
||||
# Format: "30d", "24h", "1h", etc.
|
||||
StatsTTL = "30d"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# WEB UI SETTINGS
|
||||
# -----------------------------------------------------------------------------
|
||||
[WebUI]
|
||||
# Path for stats UI
|
||||
StatsUIPath = "/stats"
|
||||
|
||||
# Path for stats API
|
||||
StatsAPIPath = "/stats/api"
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
# =============================================================================
|
||||
# THREAT SCORING CONFIGURATION - EXAMPLE CONFIG
|
||||
# =============================================================================
|
||||
# Copy this file to threat-scoring.toml and customize for your environment
|
||||
# All included threat signals are fully implemented and tested
|
||||
|
||||
[Core]
|
||||
# Enable or disable threat scoring entirely
|
||||
Enabled = true
|
||||
|
||||
# Enable detailed logging of scoring decisions (for debugging)
|
||||
LogDetailedScores = false
|
||||
|
||||
[Thresholds]
|
||||
# Score thresholds that determine the action taken for each request
|
||||
# Scores are calculated from 0-100+ based on various threat signals
|
||||
|
||||
# Requests with scores <= AllowThreshold are allowed through immediately
|
||||
AllowThreshold = 15 # Conservative - allows more legitimate traffic
|
||||
|
||||
# Requests with scores <= ChallengeThreshold receive a challenge (proof-of-work)
|
||||
ChallengeThreshold = 80 # Much higher - blocking is absolute last resort
|
||||
|
||||
# Requests with scores > ChallengeThreshold are blocked
|
||||
BlockThreshold = 100 # Truly malicious content (javascript:, <script>, etc.)
|
||||
|
||||
[Features]
|
||||
# Enable/disable specific threat analysis features
|
||||
EnableBotVerification = true # Bot verification via DNS + IP ranges
|
||||
EnableGeoAnalysis = true # Geographic analysis based on GeoIP data
|
||||
EnableBehaviorAnalysis = true # Behavioral pattern analysis across requests
|
||||
EnableContentAnalysis = true # Content/WAF analysis for malicious payloads
|
||||
|
||||
# Signal weights for implemented threat detections
|
||||
[SignalWeights]
|
||||
|
||||
# User-Agent Analysis
|
||||
[SignalWeights.ATTACK_TOOL_UA]
|
||||
weight = 30 # Risk score added for suspicious user agents
|
||||
confidence = 0.75 # Confidence in this signal (0.0-1.0)
|
||||
|
||||
[SignalWeights.MISSING_UA]
|
||||
weight = 10 # Risk score for missing user agent
|
||||
confidence = 0.60 # Lower confidence for this signal
|
||||
|
||||
# Web Application Firewall Signals
|
||||
[SignalWeights.SQL_INJECTION]
|
||||
weight = 80 # Very high risk - increased from 60
|
||||
confidence = 0.95 # High confidence in WAF detection
|
||||
|
||||
[SignalWeights.XSS_ATTEMPT]
|
||||
weight = 85 # Extremely high risk - increased from 50
|
||||
confidence = 0.95 # Very high confidence - XSS is critical
|
||||
|
||||
[SignalWeights.COMMAND_INJECTION]
|
||||
weight = 95 # Extreme risk - increased from 65
|
||||
confidence = 0.98 # Near certain malicious
|
||||
|
||||
[SignalWeights.PATH_TRAVERSAL]
|
||||
weight = 70 # High risk - increased from 45
|
||||
confidence = 0.90 # High confidence
|
||||
|
||||
# Enhanced Bot Scoring Configuration
|
||||
[EnhancedBotScoring]
|
||||
# Enhanced bot verification and scoring settings
|
||||
Enabled = true
|
||||
|
||||
# Risk adjustment weights for verified bots (negative values reduce threat scores)
|
||||
[EnhancedBotScoring.Weights]
|
||||
baseVerificationWeight = 15 # Base weight for bot verification
|
||||
ipRangeWeight = 20 # Weight for IP range verification
|
||||
dnsWeight = 25 # Weight for DNS verification
|
||||
combinedWeight = 35 # Weight when both DNS + IP match
|
||||
majorSearchEngineWeight = 10 # Additional weight for major search engines
|
||||
|
||||
# Confidence thresholds for trust level determination
|
||||
[EnhancedBotScoring.Thresholds]
|
||||
verifiedLevel = 0.9 # Threshold for verified bot (90% confidence)
|
||||
highLevel = 0.8 # High confidence threshold
|
||||
mediumLevel = 0.7 # Medium confidence threshold
|
||||
lowLevel = 0.5 # Low confidence threshold
|
||||
|
||||
# Maximum risk reduction that can be applied (prevents abuse)
|
||||
maxRiskReduction = 50
|
||||
|
||||
# Cache TTL Settings
|
||||
[Cache]
|
||||
BotVerificationTTL = "1h" # How long to cache bot verification results
|
||||
IPScoreTTL = "30m" # How long to cache IP threat scores
|
||||
SessionBehaviorTTL = "2h" # How long to cache session behavior data
|
||||
|
|
@ -1,340 +0,0 @@
|
|||
# =============================================================================
|
||||
# WEB APPLICATION FIREWALL (WAF) CONFIGURATION - EXAMPLE
|
||||
# =============================================================================
|
||||
# Copy this file to waf.toml and customize for your environment
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# CORE SETTINGS
|
||||
# -----------------------------------------------------------------------------
|
||||
[Core]
|
||||
# Enable or disable the WAF entirely
|
||||
Enabled = true
|
||||
|
||||
# Log all WAF detections (even if not blocked)
|
||||
LogAllDetections = true
|
||||
|
||||
# Maximum request body size to analyze (in bytes)
|
||||
MaxBodySize = 10485760 # 10MB
|
||||
|
||||
# WAF operation mode: "detect" or "prevent"
|
||||
# detect = log only, prevent = actively block
|
||||
Mode = "prevent"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# DETECTION SETTINGS
|
||||
# -----------------------------------------------------------------------------
|
||||
[Detection]
|
||||
# Enable specific attack detection categories
|
||||
SQLInjection = true
|
||||
XSS = true
|
||||
CommandInjection = true
|
||||
PathTraversal = true
|
||||
LFI_RFI = true
|
||||
NoSQLInjection = true
|
||||
XXE = true
|
||||
LDAPInjection = true
|
||||
SSRF = true
|
||||
XMLRPCAttacks = true
|
||||
|
||||
# Sensitivity levels: low, medium, high
|
||||
Sensitivity = "medium"
|
||||
|
||||
# Paranoia level (1-4)
|
||||
ParanoiaLevel = 2
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# SCORING CONFIGURATION
|
||||
# -----------------------------------------------------------------------------
|
||||
[Scoring]
|
||||
# Base scores for each attack type - significantly increased for aggressive detection
|
||||
SQLInjection = 80 # Increased from 35
|
||||
XSS = 90 # Increased from 30 - XSS is extremely dangerous
|
||||
CommandInjection = 100 # Increased from 40 - most dangerous
|
||||
PathTraversal = 70 # Increased from 25
|
||||
LFI_RFI = 80 # Increased from 35
|
||||
NoSQLInjection = 60 # Increased from 30
|
||||
XXE = 80 # Increased from 35
|
||||
LDAPInjection = 50 # Increased from 30
|
||||
SSRF = 75 # Increased from 35
|
||||
XMLRPCAttacks = 45 # Increased from 25
|
||||
|
||||
# Score modifiers based on confidence
|
||||
HighConfidenceMultiplier = 1.2
|
||||
MediumConfidenceMultiplier = 1.0
|
||||
LowConfidenceMultiplier = 0.8
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# RATE LIMITING
|
||||
# -----------------------------------------------------------------------------
|
||||
[RateLimit]
|
||||
# Maximum WAF detections per IP in the time window
|
||||
MaxDetectionsPerIP = 5 # More aggressive - reduced from 10
|
||||
|
||||
# Time window for rate limiting (in seconds)
|
||||
TimeWindow = 600 # 10 minutes - increased window
|
||||
|
||||
# Action when rate limit exceeded: "block" or "challenge"
|
||||
RateLimitAction = "block" # Changed from challenge to block
|
||||
|
||||
# Decay factor for repeated offenses
|
||||
DecayFactor = 0.8 # More aggressive decay
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ADVANCED DETECTION
|
||||
# -----------------------------------------------------------------------------
|
||||
[Advanced]
|
||||
# Enable machine learning-based detection
|
||||
MLDetection = false
|
||||
|
||||
# Enable payload deobfuscation
|
||||
Deobfuscation = true
|
||||
MaxDeobfuscationLevels = 3
|
||||
|
||||
# Enable response analysis (detect info leakage)
|
||||
ResponseAnalysis = true
|
||||
|
||||
# Enable timing attack detection
|
||||
TimingAnalysis = false
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# CUSTOM RULES EXAMPLES
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
[[CustomRules]]
|
||||
Name = "WordPress Admin Probe"
|
||||
Pattern = "(?i)/wp-admin/(admin-ajax\\.php|post\\.php)"
|
||||
Category = "reconnaissance"
|
||||
Score = 15
|
||||
Enabled = true
|
||||
Action = "log"
|
||||
Field = "uri_path"
|
||||
|
||||
[[CustomRules]]
|
||||
Name = "Block Headless Browsers"
|
||||
Field = "user_agent"
|
||||
Pattern = "(?i)HeadlessChrome/"
|
||||
Category = "bad_bot"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# Example of blocking specific paths on specific hosts
|
||||
[[CustomRules]]
|
||||
Name = "Block Setup Endpoint"
|
||||
Field = "uri_path"
|
||||
Pattern = "(?i)/setup"
|
||||
Category = "access_control"
|
||||
Score = 100
|
||||
Enabled = false # Disabled by default
|
||||
Action = "block"
|
||||
Hosts = ["example.com"]
|
||||
|
||||
# Example of chained conditions (both must match)
|
||||
[[CustomRules]]
|
||||
Name = "Chained Demo Rule"
|
||||
Category = "demo"
|
||||
Score = 25
|
||||
Enabled = false # Disabled by default
|
||||
Action = "block"
|
||||
|
||||
[[CustomRules.Conditions]]
|
||||
Field = "uri_query"
|
||||
Pattern = "(?i)debug=true"
|
||||
|
||||
[[CustomRules.Conditions]]
|
||||
Field = "user_agent"
|
||||
Pattern = "(?i)curl"
|
||||
|
||||
# Block javascript: protocol in any part of the URL - CRITICAL
|
||||
[[CustomRules]]
|
||||
Name = "Block JavaScript Protocol"
|
||||
Field = "uri"
|
||||
Pattern = "(?i)javascript:"
|
||||
Category = "xss"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# Block dangerous data: URLs
|
||||
[[CustomRules]]
|
||||
Name = "Block Data URL XSS"
|
||||
Field = "uri"
|
||||
Pattern = "(?i)data:.*text/html"
|
||||
Category = "xss"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# Block data: URLs with JavaScript
|
||||
[[CustomRules]]
|
||||
Name = "Block Data URL JavaScript"
|
||||
Field = "uri"
|
||||
Pattern = "(?i)data:.*javascript"
|
||||
Category = "xss"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# Block vbscript: protocol
|
||||
[[CustomRules]]
|
||||
Name = "Block VBScript Protocol"
|
||||
Field = "uri"
|
||||
Pattern = "(?i)vbscript:"
|
||||
Category = "xss"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# Block any script tags in URL parameters
|
||||
[[CustomRules]]
|
||||
Name = "Block Script Tags in Query"
|
||||
Field = "uri_query"
|
||||
Pattern = "(?i)<script"
|
||||
Category = "xss"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# Block SQL injection keywords in query
|
||||
[[CustomRules]]
|
||||
Name = "Block SQL Keywords"
|
||||
Field = "uri_query"
|
||||
Pattern = "(?i)(union.*select|insert.*into|delete.*from|drop.*table)"
|
||||
Category = "sql_injection"
|
||||
Score = 100
|
||||
Enabled = true
|
||||
Action = "block"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# WHITELIST / EXCEPTIONS
|
||||
# -----------------------------------------------------------------------------
|
||||
[Exceptions]
|
||||
# Paths to exclude from WAF analysis
|
||||
ExcludedPaths = [
|
||||
"/api/upload",
|
||||
"/static/",
|
||||
"/assets/",
|
||||
"/health",
|
||||
"/metrics"
|
||||
]
|
||||
|
||||
# Parameter names to exclude from analysis
|
||||
ExcludedParameters = [
|
||||
"utm_source",
|
||||
"utm_medium",
|
||||
"utm_campaign",
|
||||
"ref",
|
||||
"callback"
|
||||
]
|
||||
|
||||
# Known good User-Agents to reduce false positives
|
||||
TrustedUserAgents = [
|
||||
"GoogleBot",
|
||||
"BingBot",
|
||||
"monitoring-system"
|
||||
]
|
||||
|
||||
# IP addresses to exclude from WAF analysis
|
||||
TrustedIPs = [
|
||||
"127.0.0.1",
|
||||
"::1"
|
||||
]
|
||||
|
||||
# Content types to skip
|
||||
SkipContentTypes = [
|
||||
"image/",
|
||||
"video/",
|
||||
"audio/",
|
||||
"font/",
|
||||
"application/pdf"
|
||||
]
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# FALSE POSITIVE REDUCTION
|
||||
# -----------------------------------------------------------------------------
|
||||
[FalsePositive]
|
||||
# Common false positive patterns to ignore
|
||||
IgnorePatterns = [
|
||||
# Legitimate base64 in JSON (e.g., image data)
|
||||
"\"data:image\\/[^;]+;base64,",
|
||||
# Markdown code blocks
|
||||
"```[a-z]*\\n",
|
||||
# Common API tokens (not actual secrets)
|
||||
"token=[a-f0-9]{32}",
|
||||
# Timestamps
|
||||
"\\d{10,13}"
|
||||
]
|
||||
|
||||
# Context-aware detection
|
||||
ContextualDetection = true
|
||||
|
||||
# Authentication features removed - not applicable for this security system
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# BOT VERIFICATION
|
||||
# -----------------------------------------------------------------------------
|
||||
[BotVerification]
|
||||
# Enable comprehensive bot verification using IP ranges and DNS
|
||||
Enabled = true
|
||||
|
||||
# Allow verified legitimate bots (Googlebot, Bingbot, etc.) to bypass WAF analysis
|
||||
# When true, verified bots get 90% threat score reduction
|
||||
AllowVerifiedBots = true
|
||||
|
||||
# Block requests that claim to be bots but fail verification
|
||||
# When true, fake bot user agents get +50 threat score penalty
|
||||
BlockUnverifiedBots = true
|
||||
|
||||
# Enable DNS verification (reverse DNS + forward DNS confirmation)
|
||||
EnableDNSVerification = true
|
||||
|
||||
# Enable IP range verification using official bot IP ranges
|
||||
EnableIPRangeVerification = true
|
||||
|
||||
# DNS lookup timeout
|
||||
DNSTimeout = "5s"
|
||||
|
||||
# Minimum confidence score required to trust a bot (0.0-1.0)
|
||||
# Higher values = more strict verification
|
||||
MinimumConfidence = 0.8
|
||||
|
||||
# Bot source definitions with user agent patterns and IP range sources
|
||||
[[BotVerification.BotSources]]
|
||||
name = "googlebot"
|
||||
userAgentPattern = "Googlebot/\\d+\\.\\d+"
|
||||
ipRangeURL = "https://developers.google.com/static/search/apis/ipranges/googlebot.json"
|
||||
dnsVerificationDomain = "googlebot.com"
|
||||
updateInterval = "24h"
|
||||
enabled = true
|
||||
|
||||
[[BotVerification.BotSources]]
|
||||
name = "bingbot"
|
||||
userAgentPattern = "bingbot/\\d+\\.\\d+"
|
||||
ipRangeURL = "https://www.bing.com/toolbox/bingbot-ips.txt"
|
||||
dnsVerificationDomain = "search.msn.com"
|
||||
updateInterval = "24h"
|
||||
enabled = true
|
||||
|
||||
[[BotVerification.BotSources]]
|
||||
name = "slurp"
|
||||
userAgentPattern = "Slurp"
|
||||
ipRangeURL = "https://help.yahoo.com/slurpbot-ips.txt"
|
||||
dnsVerificationDomain = "crawl.yahoo.net"
|
||||
updateInterval = "2d"
|
||||
enabled = false
|
||||
|
||||
[[BotVerification.BotSources]]
|
||||
name = "duckduckbot"
|
||||
userAgentPattern = "DuckDuckBot/\\d+\\.\\d+"
|
||||
ipRangeURL = "https://duckduckgo.com/duckduckbot-ips.txt"
|
||||
updateInterval = "3d"
|
||||
enabled = false
|
||||
|
||||
[[BotVerification.BotSources]]
|
||||
name = "facebookexternalhit"
|
||||
userAgentPattern = "facebookexternalhit/\\d+\\.\\d+"
|
||||
ipRangeURL = "https://developers.facebook.com/docs/sharing/webmasters/crawler-ips"
|
||||
dnsVerificationDomain = "facebook.com"
|
||||
updateInterval = "24h"
|
||||
enabled = false
|
||||
242
index.js
Normal file
242
index.js
Normal file
|
|
@ -0,0 +1,242 @@
|
|||
import { mkdir, readFile } from 'fs/promises';
|
||||
import { writeFileSync, readFileSync, unlinkSync, existsSync } from 'fs';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { secureImportModule } from './utils/plugins.js';
|
||||
import * as logs from './utils/logs.js';
|
||||
import express from 'express';
|
||||
import { createServer } from 'http';
|
||||
import { spawn } from 'child_process';
|
||||
|
||||
// Load environment variables from .env file
|
||||
import dotenv from 'dotenv';
|
||||
dotenv.config();
|
||||
|
||||
// Stop daemon: if run with -k, kill the running process and exit.
|
||||
if (process.argv.includes('-k')) {
|
||||
const pidFile = join(dirname(fileURLToPath(import.meta.url)), 'checkpoint.pid');
|
||||
if (existsSync(pidFile)) {
|
||||
const pid = parseInt(readFileSync(pidFile, 'utf8'), 10);
|
||||
try {
|
||||
process.kill(pid);
|
||||
unlinkSync(pidFile);
|
||||
console.log(`Stopped daemon (pid ${pid})`);
|
||||
} catch (err) {
|
||||
console.error(`Failed to stop pid ${pid}: ${err}`);
|
||||
}
|
||||
} else {
|
||||
console.error(`No pid file found at ${pidFile}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Daemonize: if run with -d, kill any existing daemon, then re-spawn detached, write pid file, and exit parent.
|
||||
if (process.argv.includes('-d')) {
|
||||
const pidFile = join(dirname(fileURLToPath(import.meta.url)), 'checkpoint.pid');
|
||||
// If already running, stop the old daemon
|
||||
if (existsSync(pidFile)) {
|
||||
const oldPid = parseInt(readFileSync(pidFile, 'utf8'), 10);
|
||||
try {
|
||||
process.kill(oldPid);
|
||||
console.log(`Stopped old daemon (pid ${oldPid})`);
|
||||
} catch (e) {
|
||||
console.error(`Failed to stop old daemon (pid ${oldPid}): ${e}`);
|
||||
}
|
||||
try {
|
||||
unlinkSync(pidFile);
|
||||
} catch {}
|
||||
}
|
||||
// Spawn new background process
|
||||
const args = process.argv.slice(1).filter((arg) => arg !== '-d');
|
||||
const cp = spawn(process.argv[0], args, {
|
||||
detached: true,
|
||||
stdio: 'ignore'
|
||||
});
|
||||
cp.unref();
|
||||
writeFileSync(pidFile, cp.pid.toString(), 'utf8');
|
||||
console.log(`Daemonized (pid ${cp.pid}), pid stored in ${pidFile}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Disable console.log in production to suppress output in daemon mode
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
console.log = () => {};
|
||||
}
|
||||
|
||||
const pluginRegistry = [];
|
||||
export function registerPlugin(pluginName, handler) {
|
||||
pluginRegistry.push({ name: pluginName, handler });
|
||||
}
|
||||
/**
|
||||
* Return the array of middleware handlers in registration order.
|
||||
*/
|
||||
export function loadPlugins() {
|
||||
return pluginRegistry.map((item) => item.handler);
|
||||
}
|
||||
/**
|
||||
* Return the names of all registered plugins.
|
||||
*/
|
||||
export function getRegisteredPluginNames() {
|
||||
return pluginRegistry.map((item) => item.name);
|
||||
}
|
||||
/**
|
||||
* Freeze plugin registry to prevent further registration and log the final set.
|
||||
*/
|
||||
export function freezePlugins() {
|
||||
Object.freeze(pluginRegistry);
|
||||
pluginRegistry.forEach((item) => Object.freeze(item));
|
||||
logs.msg('Plugin registration frozen');
|
||||
}
|
||||
|
||||
// Determine root directory for config loading
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
export const rootDir = __dirname;
|
||||
|
||||
export async function loadConfig(name, target) {
|
||||
const configPath = join(rootDir, 'config', `${name}.toml`);
|
||||
const txt = await readFile(configPath, 'utf8');
|
||||
const { default: toml } = await import('@iarna/toml');
|
||||
Object.assign(target, toml.parse(txt));
|
||||
logs.config(name, 'loaded');
|
||||
}
|
||||
|
||||
async function initDataDirectories() {
|
||||
logs.section('INIT');
|
||||
const directories = [join(rootDir, 'data'), join(rootDir, 'db'), join(rootDir, 'config')];
|
||||
for (const dirPath of directories) {
|
||||
try {
|
||||
await mkdir(dirPath, { recursive: true });
|
||||
} catch {}
|
||||
}
|
||||
logs.init('Data directories are now in place');
|
||||
}
|
||||
|
||||
function staticFileMiddleware() {
|
||||
const router = express.Router();
|
||||
router.use('/webfont', express.static(join(rootDir, 'pages/interstitial/webfont'), {
|
||||
maxAge: '7d'
|
||||
}));
|
||||
router.use('/js', express.static(join(rootDir, 'pages/interstitial/js'), {
|
||||
maxAge: '7d'
|
||||
}));
|
||||
return router;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await initDataDirectories();
|
||||
|
||||
logs.section('CONFIG');
|
||||
logs.config('checkpoint', 'loaded');
|
||||
logs.config('ipfilter', 'loaded');
|
||||
logs.config('proxy', 'loaded');
|
||||
logs.config('stats', 'loaded');
|
||||
|
||||
logs.section('OPERATIONS');
|
||||
|
||||
const app = express();
|
||||
const server = createServer(app);
|
||||
|
||||
// Trust proxy headers (important for proper protocol detection)
|
||||
app.set('trust proxy', true);
|
||||
|
||||
try {
|
||||
await secureImportModule('checkpoint.js');
|
||||
} catch (e) {
|
||||
logs.error('checkpoint', `Failed to load checkpoint plugin: ${e}`);
|
||||
}
|
||||
try {
|
||||
await secureImportModule('plugins/ipfilter.js');
|
||||
} catch (e) {
|
||||
logs.error('ipfilter', `Failed to load IP filter plugin: ${e}`);
|
||||
}
|
||||
try {
|
||||
await secureImportModule('plugins/proxy.js');
|
||||
} catch (e) {
|
||||
logs.error('proxy', `Failed to load proxy plugin: ${e}`);
|
||||
}
|
||||
try {
|
||||
await secureImportModule('plugins/stats.js');
|
||||
} catch (e) {
|
||||
logs.error('stats', `Failed to load stats plugin: ${e}`);
|
||||
}
|
||||
|
||||
// Register static middleware
|
||||
app.use(staticFileMiddleware());
|
||||
|
||||
logs.section('PLUGINS');
|
||||
// Ensure ipfilter runs first by moving it to front of the registry
|
||||
const ipIndex = pluginRegistry.findIndex((item) => item.name === 'ipfilter');
|
||||
if (ipIndex > 0) {
|
||||
const [ipEntry] = pluginRegistry.splice(ipIndex, 1);
|
||||
pluginRegistry.unshift(ipEntry);
|
||||
}
|
||||
pluginRegistry.forEach((item) => logs.msg(item.name));
|
||||
logs.section('SYSTEM');
|
||||
freezePlugins();
|
||||
|
||||
// Apply all plugin middlewares to Express
|
||||
const middlewareHandlers = loadPlugins();
|
||||
middlewareHandlers.forEach(handler => {
|
||||
if (handler && handler.middleware) {
|
||||
// If plugin exports an object with middleware property
|
||||
if (Array.isArray(handler.middleware)) {
|
||||
// If middleware is an array, apply each one
|
||||
handler.middleware.forEach(mw => app.use(mw));
|
||||
} else {
|
||||
// Single middleware
|
||||
app.use(handler.middleware);
|
||||
}
|
||||
} else if (typeof handler === 'function') {
|
||||
// Legacy function-style handlers (shouldn't exist anymore)
|
||||
logs.warn('server', 'Found legacy function-style plugin handler');
|
||||
app.use(handler);
|
||||
}
|
||||
});
|
||||
|
||||
// 404 handler
|
||||
app.use((req, res) => {
|
||||
res.status(404).send('Not Found');
|
||||
});
|
||||
|
||||
// Error handler
|
||||
app.use((err, req, res, next) => {
|
||||
logs.error('server', `Server error: ${err.message}`);
|
||||
res.status(500).send(`Server Error: ${err.message}`);
|
||||
});
|
||||
|
||||
// Handle WebSocket upgrades using http-proxy-middleware instances
|
||||
server.on('upgrade', (req, socket, head) => {
|
||||
const hostname = req.headers.host?.split(':')[0];
|
||||
if (!hostname) {
|
||||
logs.error('websocket', 'Upgrade request without host header, destroying socket.');
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
logs.server(`WebSocket upgrade request for ${hostname}${req.url}`);
|
||||
|
||||
import('./plugins/proxy.js').then(proxyModule => {
|
||||
const hpmInstance = proxyModule.getHpmInstance(hostname);
|
||||
|
||||
if (hpmInstance && typeof hpmInstance.upgrade === 'function') {
|
||||
logs.server(`Attempting to upgrade WebSocket for ${hostname} using HPM instance.`);
|
||||
hpmInstance.upgrade(req, socket, head);
|
||||
} else {
|
||||
logs.error('websocket', `No HPM instance or upgrade method found for ${hostname}${req.url}`);
|
||||
socket.destroy();
|
||||
}
|
||||
}).catch(err => {
|
||||
logs.error('websocket', `Error importing proxy module for upgrade: ${err.message}`);
|
||||
socket.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
logs.section('SERVER');
|
||||
const portNumber = Number(process.env.PORT || 3000);
|
||||
server.listen(portNumber, () => {
|
||||
logs.server(`🚀 Server is up and running on port ${portNumber}...`);
|
||||
logs.section('REQ LOGS');
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
/** @type {import('jest').Config} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest/presets/default-esm',
|
||||
extensionsToTreatAsEsm: ['.ts'],
|
||||
testEnvironment: 'node',
|
||||
transform: {
|
||||
'^.+\\.tsx?$': ['ts-jest', {
|
||||
useESM: true,
|
||||
}],
|
||||
'^.+\\.jsx?$': ['ts-jest', {
|
||||
useESM: true,
|
||||
}],
|
||||
},
|
||||
testMatch: [
|
||||
'**/.tests/**/*.test.js'
|
||||
],
|
||||
collectCoverage: true,
|
||||
collectCoverageFrom: [
|
||||
'dist/**/*.js', // Include all JS files in dist directory
|
||||
'!dist/**/*.test.js', // Exclude test files
|
||||
'!dist/**/*.spec.js', // Exclude spec files
|
||||
'!dist/**/node_modules/**' // Exclude node_modules
|
||||
],
|
||||
coverageDirectory: 'coverage',
|
||||
coverageReporters: ['text', 'lcov', 'html'],
|
||||
|
||||
// Practical 75% global coverage threshold
|
||||
coverageThreshold: {
|
||||
global: {
|
||||
statements: 75,
|
||||
branches: 75,
|
||||
functions: 75,
|
||||
lines: 75
|
||||
}
|
||||
},
|
||||
|
||||
setupFilesAfterEnv: ['./.tests/setup.js'],
|
||||
globalTeardown: './.tests/teardown.js',
|
||||
testTimeout: 10000,
|
||||
verbose: true,
|
||||
|
||||
// Additional configuration to handle async operations
|
||||
forceExit: true,
|
||||
detectOpenHandles: false
|
||||
};
|
||||
5919
package-lock.json
generated
5919
package-lock.json
generated
File diff suppressed because it is too large
Load diff
36
package.json
36
package.json
|
|
@ -3,49 +3,27 @@
|
|||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "npm run build && node dist/index.js",
|
||||
"dev": "npx tsx src/index.ts",
|
||||
"build": "npm run clean && npm run compile && npm run copy-config && npm run copy-pages",
|
||||
"compile": "npx tsc",
|
||||
"copy-config": "copyfiles -u 1 \"config/**/*\" dist/",
|
||||
"copy-pages": "copyfiles -u 1 \"pages/**/*\" dist/ || exit 0",
|
||||
"clean": "rimraf dist",
|
||||
"typecheck": "npx tsc --noEmit",
|
||||
"test": "node --experimental-vm-modules ./node_modules/.bin/jest",
|
||||
"test:watch": "node --experimental-vm-modules ./node_modules/.bin/jest --watch",
|
||||
"test:coverage": "node --experimental-vm-modules ./node_modules/.bin/jest --coverage",
|
||||
"daemon": "npm run build && pm2 start dist/index.js --name checkpoint",
|
||||
"daemon-r": "npm run build && pm2-runtime start dist/index.js --name checkpoint",
|
||||
"start": "node index.js",
|
||||
"dev": "nodemon index.js",
|
||||
"daemon": "pm2-runtime start index.js --name checkpoint",
|
||||
"stop": "pm2 stop checkpoint",
|
||||
"logs": "pm2 logs checkpoint",
|
||||
"reload": "pm2 delete checkpoint && npm run daemon && npx pm2 reload checkpoint --update-env",
|
||||
"restart": "pm2 delete checkpoint && npm run daemon && npx pm2 reload checkpoint --update-env"
|
||||
"restart": "pm2 restart checkpoint",
|
||||
"logs": "pm2 logs checkpoint"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@types/express": "^4.17.23",
|
||||
"@types/jest": "^30.0.0",
|
||||
"@types/node": "^24.0.3",
|
||||
"copyfiles": "^2.4.1",
|
||||
"jest": "^29.7.0",
|
||||
"nodemon": "^3.0.2",
|
||||
"prettier": "^2.8.8",
|
||||
"rimraf": "^6.0.1",
|
||||
"ts-jest": "^29.4.0",
|
||||
"tsx": "^4.7.0",
|
||||
"typescript": "^5.8.3"
|
||||
"prettier": "^2.8.8"
|
||||
},
|
||||
"dependencies": {
|
||||
"@iarna/toml": "^2.2.5",
|
||||
"cookie": "^1.0.2",
|
||||
"dotenv": "^16.5.0",
|
||||
"express": "^4.18.2",
|
||||
"http-proxy": "^1.18.1",
|
||||
"http-proxy-middleware": "^2.0.6",
|
||||
"level": "^10.0.0",
|
||||
"level-ttl": "^3.1.1",
|
||||
"maxmind": "^4.3.25",
|
||||
"pm2": "^6.0.5",
|
||||
"pm2": "^5.3.0",
|
||||
"string-dsa": "^2.1.0",
|
||||
"tar-stream": "^3.1.7",
|
||||
"ws": "^8.16.0"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ let isModalOpen = false;
|
|||
let pendingAction = null; // Can be 'success' or 'error'
|
||||
let storedErrorMessage = '';
|
||||
let storedRedirectUrl = '';
|
||||
const REDIRECT_DELAY = 1488;
|
||||
// let redirectToken = ''; // This was defined but not used, removing for now.
|
||||
const REDIRECT_DELAY = 1488; // Moved for wider accessibility
|
||||
|
||||
function workerFunction() {
|
||||
self.onmessage = function (e) {
|
||||
|
|
@ -235,6 +236,11 @@ document.addEventListener('DOMContentLoaded', function () {
|
|||
errorDetails.style.display = 'block';
|
||||
}
|
||||
// Ensure any running workers are stopped on error
|
||||
// This might need to be called from within Verifier's scope or Verifier needs a public method
|
||||
// For now, if terminateWorkers is global or accessible, it would be called.
|
||||
// However, terminateWorkers is defined within Verifier. This needs careful handling.
|
||||
// Let's assume for now the original placement inside Verifier handles termination on actual error progression.
|
||||
// The primary goal here is to show the UI error state.
|
||||
}
|
||||
|
||||
function initVerification() {
|
||||
|
|
|
|||
|
|
@ -1,104 +1 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Access Denied - Datacenter Restriction</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: linear-gradient(135deg, #4facfe 0%, #00f2fe 100%);
|
||||
color: #333;
|
||||
line-height: 1.6;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.container {
|
||||
max-width: 500px;
|
||||
margin: 0 20px;
|
||||
}
|
||||
.card {
|
||||
background: white;
|
||||
border-radius: 16px;
|
||||
box-shadow: 0 10px 30px rgba(0,0,0,0.2);
|
||||
padding: 40px;
|
||||
text-align: center;
|
||||
}
|
||||
.icon {
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
margin: 0 auto 20px;
|
||||
background: #4facfe;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.icon svg {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
fill: white;
|
||||
}
|
||||
h1 {
|
||||
font-size: 24px;
|
||||
margin-bottom: 16px;
|
||||
color: #2c3e50;
|
||||
}
|
||||
.message {
|
||||
color: #666;
|
||||
margin-bottom: 24px;
|
||||
font-size: 16px;
|
||||
}
|
||||
.reason {
|
||||
background: #f8f9fa;
|
||||
padding: 16px;
|
||||
border-radius: 8px;
|
||||
border-left: 4px solid #4facfe;
|
||||
margin: 20px 0;
|
||||
text-align: left;
|
||||
}
|
||||
.reason-title {
|
||||
font-weight: 600;
|
||||
color: #2c3e50;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.contact {
|
||||
margin-top: 24px;
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
}
|
||||
.server-icon {
|
||||
font-size: 32px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="card">
|
||||
<div class="server-icon">🖥️</div>
|
||||
|
||||
<div class="icon">
|
||||
<svg viewBox="0 0 24 24">
|
||||
<path d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-2 15l-5-5 1.41-1.41L10 14.17l7.59-7.59L19 8l-9 9z"/>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<h1>Access Denied</h1>
|
||||
<p class="message">Access from datacenter or cloud hosting providers is restricted.</p>
|
||||
|
||||
<div class="reason">
|
||||
<div class="reason-title">Datacenter Policy</div>
|
||||
<div>This service blocks access from known datacenter, cloud hosting, and VPN provider IP ranges to prevent automated abuse and maintain service quality for residential users.</div>
|
||||
</div>
|
||||
|
||||
<div class="contact">
|
||||
<p>If you're accessing this service legitimately and need datacenter access for business purposes, please contact our support team for whitelisting consideration.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Blocked (Datacenter)
|
||||
|
|
|
|||
|
|
@ -1,98 +1 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Access Denied - Geographic Restriction</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: #333;
|
||||
line-height: 1.6;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.container {
|
||||
max-width: 500px;
|
||||
margin: 0 20px;
|
||||
}
|
||||
.card {
|
||||
background: white;
|
||||
border-radius: 16px;
|
||||
box-shadow: 0 10px 30px rgba(0,0,0,0.2);
|
||||
padding: 40px;
|
||||
text-align: center;
|
||||
}
|
||||
.icon {
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
margin: 0 auto 20px;
|
||||
background: #ff6b6b;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.icon svg {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
fill: white;
|
||||
}
|
||||
h1 {
|
||||
font-size: 24px;
|
||||
margin-bottom: 16px;
|
||||
color: #2c3e50;
|
||||
}
|
||||
.message {
|
||||
color: #666;
|
||||
margin-bottom: 24px;
|
||||
font-size: 16px;
|
||||
}
|
||||
.reason {
|
||||
background: #f8f9fa;
|
||||
padding: 16px;
|
||||
border-radius: 8px;
|
||||
border-left: 4px solid #ff6b6b;
|
||||
margin: 20px 0;
|
||||
text-align: left;
|
||||
}
|
||||
.reason-title {
|
||||
font-weight: 600;
|
||||
color: #2c3e50;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.contact {
|
||||
margin-top: 24px;
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="card">
|
||||
<div class="icon">
|
||||
<svg viewBox="0 0 24 24">
|
||||
<path d="M12 2C8.13 2 5 5.13 5 9c0 5.25 7 13 7 13s7-7.75 7-13c0-3.87-3.13-7-7-7zm0 9.5c-1.38 0-2.5-1.12-2.5-2.5s1.12-2.5 2.5-2.5 2.5 1.12 2.5 2.5-1.12 2.5-2.5 2.5z"/>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<h1>Access Denied</h1>
|
||||
<p class="message">We're sorry, but access from your location is currently restricted.</p>
|
||||
|
||||
<div class="reason">
|
||||
<div class="reason-title">Geographic Restriction</div>
|
||||
<div>This service has geographic access restrictions in place. Access from your location or network provider is not permitted at this time.</div>
|
||||
</div>
|
||||
|
||||
<div class="contact">
|
||||
<p>If you believe this is an error or you have questions about access, please contact support.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Blocked (Default)
|
||||
|
|
|
|||
|
|
@ -1,104 +1 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Access Denied - Geographic Restriction</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||
background: linear-gradient(135deg, #ff9a56 0%, #ff6b6b 100%);
|
||||
color: #333;
|
||||
line-height: 1.6;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.container {
|
||||
max-width: 500px;
|
||||
margin: 0 20px;
|
||||
}
|
||||
.card {
|
||||
background: white;
|
||||
border-radius: 16px;
|
||||
box-shadow: 0 10px 30px rgba(0,0,0,0.2);
|
||||
padding: 40px;
|
||||
text-align: center;
|
||||
}
|
||||
.icon {
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
margin: 0 auto 20px;
|
||||
background: #ff6b6b;
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.icon svg {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
fill: white;
|
||||
}
|
||||
h1 {
|
||||
font-size: 24px;
|
||||
margin-bottom: 16px;
|
||||
color: #2c3e50;
|
||||
}
|
||||
.message {
|
||||
color: #666;
|
||||
margin-bottom: 24px;
|
||||
font-size: 16px;
|
||||
}
|
||||
.reason {
|
||||
background: #f8f9fa;
|
||||
padding: 16px;
|
||||
border-radius: 8px;
|
||||
border-left: 4px solid #ff6b6b;
|
||||
margin: 20px 0;
|
||||
text-align: left;
|
||||
}
|
||||
.reason-title {
|
||||
font-weight: 600;
|
||||
color: #2c3e50;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.contact {
|
||||
margin-top: 24px;
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
}
|
||||
.flag {
|
||||
font-size: 32px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="card">
|
||||
<div class="flag">🇮🇳</div>
|
||||
|
||||
<div class="icon">
|
||||
<svg viewBox="0 0 24 24">
|
||||
<path d="M12 2C8.13 2 5 5.13 5 9c0 5.25 7 13 7 13s7-7.75 7-13c0-3.87-3.13-7-7-7zm0 9.5c-1.38 0-2.5-1.12-2.5-2.5s1.12-2.5 2.5-2.5 2.5 1.12 2.5 2.5-1.12 2.5-2.5 2.5z"/>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<h1>Access Denied</h1>
|
||||
<p class="message">Access from India is currently restricted due to security policies.</p>
|
||||
|
||||
<div class="reason">
|
||||
<div class="reason-title">Regional Security Policy</div>
|
||||
<div>This service has implemented geographic access restrictions. Access from Indian IP addresses is currently blocked due to security and compliance requirements.</div>
|
||||
</div>
|
||||
|
||||
<div class="contact">
|
||||
<p>If you believe this restriction affects you unfairly or you have questions about access policies, please contact our support team.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Blocked (India)
|
||||
|
|
|
|||
1274
pages/stats/stats.html
Normal file
1274
pages/stats/stats.html
Normal file
File diff suppressed because it is too large
Load diff
469
plugins/ipfilter.js
Normal file
469
plugins/ipfilter.js
Normal file
|
|
@ -0,0 +1,469 @@
|
|||
import { registerPlugin, loadConfig, rootDir } from '../index.js';
|
||||
import fs from 'fs';
|
||||
import { dirname, join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import maxmind from 'maxmind';
|
||||
import { AhoCorasick } from 'string-dsa';
|
||||
import { getRealIP } from '../utils/network.js';
|
||||
import { createGunzip } from 'zlib';
|
||||
import tarStream from 'tar-stream';
|
||||
import { Buffer } from 'buffer';
|
||||
import * as logs from '../utils/logs.js';
|
||||
import { recordEvent } from './stats.js';
|
||||
|
||||
const cfg = {};
|
||||
await loadConfig('ipfilter', cfg);
|
||||
|
||||
// Map configuration to internal structure
|
||||
const enabled = cfg.Core.Enabled;
|
||||
const accountId = cfg.Core.AccountID || process.env.MAXMIND_ACCOUNT_ID;
|
||||
const licenseKey = cfg.Core.LicenseKey || process.env.MAXMIND_LICENSE_KEY;
|
||||
const dbUpdateInterval = cfg.Core.DBUpdateIntervalHours;
|
||||
|
||||
const ipBlockCacheTTL = cfg.Cache.IPBlockCacheTTLSec * 1000;
|
||||
const ipBlockCacheMaxEntries = cfg.Cache.IPBlockCacheMaxEntries;
|
||||
|
||||
const blockedCountryCodes = new Set(cfg.Blocking.CountryCodes);
|
||||
const blockedContinentCodes = new Set(cfg.Blocking.ContinentCodes);
|
||||
const defaultBlockPage = cfg.Blocking.DefaultBlockPage;
|
||||
|
||||
// Process ASN blocks
|
||||
const blockedASNs = {};
|
||||
const asnGroupBlockPages = {};
|
||||
for (const [group, config] of Object.entries(cfg.ASN || {})) {
|
||||
blockedASNs[group] = config.Numbers || [];
|
||||
asnGroupBlockPages[group] = config.BlockPage;
|
||||
}
|
||||
|
||||
// Process ASN name blocks
|
||||
const blockedASNNames = {};
|
||||
for (const [group, config] of Object.entries(cfg.ASNNames || {})) {
|
||||
blockedASNNames[group] = config.Patterns || [];
|
||||
if (config.BlockPage) {
|
||||
asnGroupBlockPages[group] = config.BlockPage;
|
||||
}
|
||||
}
|
||||
|
||||
const countryBlockPages = cfg.CountryBlockPages || {};
|
||||
const continentBlockPages = cfg.ContinentBlockPages || {};
|
||||
|
||||
const ipBlockCache = new Map();
|
||||
|
||||
const blockPageCache = new Map();
|
||||
async function loadBlockPage(filePath) {
|
||||
if (!blockPageCache.has(filePath)) {
|
||||
try {
|
||||
const txt = await fs.promises.readFile(filePath, 'utf8');
|
||||
blockPageCache.set(filePath, txt);
|
||||
} catch {
|
||||
blockPageCache.set(filePath, null);
|
||||
}
|
||||
}
|
||||
return blockPageCache.get(filePath);
|
||||
}
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const geoIPCountryDBPath = join(rootDir, 'data/GeoLite2-Country.mmdb');
|
||||
const geoIPASNDBPath = join(rootDir, 'data/GeoLite2-ASN.mmdb');
|
||||
const updateTimestampPath = join(rootDir, 'data/ipfilter_update.json');
|
||||
|
||||
let geoipCountryReader, geoipASNReader;
|
||||
|
||||
let isReloading = false;
|
||||
let reloadLock = Promise.resolve();
|
||||
|
||||
async function getLastUpdateTimestamp() {
|
||||
try {
|
||||
if (fs.existsSync(updateTimestampPath)) {
|
||||
const data = await fs.promises.readFile(updateTimestampPath, 'utf8');
|
||||
const json = JSON.parse(data);
|
||||
return json.lastUpdated || 0;
|
||||
}
|
||||
} catch (err) {
|
||||
logs.warn('ipfilter', `Failed to read last update timestamp: ${err}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function saveUpdateTimestamp() {
|
||||
try {
|
||||
const timestamp = Date.now();
|
||||
await fs.promises.writeFile(
|
||||
updateTimestampPath,
|
||||
JSON.stringify({ lastUpdated: timestamp }),
|
||||
'utf8',
|
||||
);
|
||||
return timestamp;
|
||||
} catch (err) {
|
||||
logs.error('ipfilter', `Failed to save update timestamp: ${err}`);
|
||||
return Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the update timestamp file exists on first run
|
||||
if (!fs.existsSync(updateTimestampPath)) {
|
||||
try {
|
||||
await saveUpdateTimestamp();
|
||||
} catch (err) {
|
||||
logs.error('ipfilter', `Failed to initialize update timestamp file: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Download GeoIP databases if missing
|
||||
async function downloadGeoIPDatabases() {
|
||||
if (!licenseKey || !accountId) {
|
||||
logs.warn(
|
||||
'ipfilter',
|
||||
'No MaxMind credentials found; skipping GeoIP database download. Please set MAXMIND_ACCOUNT_ID and MAXMIND_LICENSE_KEY environment variables or add AccountID and LicenseKey to config/ipfilter.toml',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const editions = [
|
||||
{ id: 'GeoLite2-Country', filePath: geoIPCountryDBPath },
|
||||
{ id: 'GeoLite2-ASN', filePath: geoIPASNDBPath },
|
||||
];
|
||||
for (const { id, filePath } of editions) {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
logs.plugin('ipfilter', `Downloading ${id} database...`);
|
||||
const url = `https://download.maxmind.com/app/geoip_download?edition_id=${id}&license_key=${licenseKey}&suffix=tar.gz`;
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) {
|
||||
logs.error(
|
||||
'ipfilter',
|
||||
`Failed to download ${id} database: ${res.status} ${res.statusText}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const tempTar = join(rootDir, 'data', `${id}.tar.gz`);
|
||||
// write response body into a .tar.gz file
|
||||
const arrayBuf = await res.arrayBuffer();
|
||||
await fs.promises.writeFile(tempTar, Buffer.from(arrayBuf));
|
||||
// extract .mmdb files from the downloaded tar.gz
|
||||
const extract = tarStream.extract();
|
||||
extract.on('entry', (header, stream, next) => {
|
||||
if (header.name.endsWith('.mmdb')) {
|
||||
const filename = header.name.split('/').pop();
|
||||
const outPath = join(rootDir, 'data', filename);
|
||||
const ws = fs.createWriteStream(outPath);
|
||||
stream
|
||||
.pipe(ws)
|
||||
.on('finish', next)
|
||||
.on('error', (err) => {
|
||||
logs.error('ipfilter', `Extraction error: ${err}`);
|
||||
next();
|
||||
});
|
||||
} else {
|
||||
stream.resume();
|
||||
next();
|
||||
}
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
fs.createReadStream(tempTar)
|
||||
.pipe(createGunzip())
|
||||
.pipe(extract)
|
||||
.on('finish', resolve)
|
||||
.on('error', reject);
|
||||
});
|
||||
await fs.promises.unlink(tempTar);
|
||||
logs.plugin('ipfilter', `${id} database downloaded and extracted.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await downloadGeoIPDatabases();
|
||||
|
||||
async function loadGeoDatabases() {
|
||||
if (isReloading) {
|
||||
await reloadLock;
|
||||
return true;
|
||||
}
|
||||
|
||||
isReloading = true;
|
||||
let lockResolve;
|
||||
reloadLock = new Promise((resolve) => {
|
||||
lockResolve = resolve;
|
||||
});
|
||||
|
||||
try {
|
||||
const countryStats = fs.statSync(geoIPCountryDBPath);
|
||||
const asnStats = fs.statSync(geoIPASNDBPath);
|
||||
|
||||
if (countryStats.size > 1024 && asnStats.size > 1024) {
|
||||
logs.plugin('ipfilter', 'Initializing GeoIP databases from disk...');
|
||||
const newCountryReader = await maxmind.open(geoIPCountryDBPath);
|
||||
const newASNReader = await maxmind.open(geoIPASNDBPath);
|
||||
|
||||
try {
|
||||
const testIP = '8.8.8.8';
|
||||
const countryTest = newCountryReader.get(testIP);
|
||||
const asnTest = newASNReader.get(testIP);
|
||||
|
||||
if (!countryTest || !asnTest) {
|
||||
throw new Error('Database validation failed: test lookups returned empty results');
|
||||
}
|
||||
} catch (validationErr) {
|
||||
logs.error('ipfilter', `GeoIP database validation failed: ${validationErr}`);
|
||||
|
||||
try {
|
||||
await newCountryReader.close();
|
||||
} catch (e) {}
|
||||
try {
|
||||
await newASNReader.close();
|
||||
} catch (e) {}
|
||||
throw new Error('Database validation failed');
|
||||
}
|
||||
|
||||
const oldCountryReader = geoipCountryReader;
|
||||
const oldASNReader = geoipASNReader;
|
||||
|
||||
geoipCountryReader = newCountryReader;
|
||||
geoipASNReader = newASNReader;
|
||||
if (oldCountryReader || oldASNReader) {
|
||||
logs.plugin('ipfilter', 'GeoIP databases reloaded and active');
|
||||
} else {
|
||||
logs.plugin('ipfilter', 'GeoIP databases loaded and active');
|
||||
}
|
||||
|
||||
ipBlockCache.clear();
|
||||
|
||||
await saveUpdateTimestamp();
|
||||
|
||||
if (oldCountryReader || oldASNReader) {
|
||||
setTimeout(async () => {
|
||||
if (oldCountryReader) {
|
||||
try {
|
||||
await oldCountryReader.close();
|
||||
} catch (e) {}
|
||||
}
|
||||
if (oldASNReader) {
|
||||
try {
|
||||
await oldASNReader.close();
|
||||
} catch (e) {}
|
||||
}
|
||||
logs.plugin('ipfilter', 'Old GeoIP database instances closed successfully');
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
return true;
|
||||
} else {
|
||||
logs.warn(
|
||||
'ipfilter',
|
||||
'GeoIP database files are empty or too small. IP filtering will be disabled.',
|
||||
);
|
||||
return false;
|
||||
}
|
||||
} catch (err) {
|
||||
logs.error('ipfilter', `Failed to load GeoIP databases: ${err}`);
|
||||
return false;
|
||||
} finally {
|
||||
isReloading = false;
|
||||
lockResolve();
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAndUpdateDatabases() {
|
||||
if (isReloading) return false;
|
||||
|
||||
const lastUpdate = await getLastUpdateTimestamp();
|
||||
const now = Date.now();
|
||||
const hoursSinceUpdate = (now - lastUpdate) / (1000 * 60 * 60);
|
||||
|
||||
if (hoursSinceUpdate >= dbUpdateInterval) {
|
||||
logs.plugin(
|
||||
'ipfilter',
|
||||
`GeoIP databases last updated ${hoursSinceUpdate.toFixed(1)} hours ago, reloading...`,
|
||||
);
|
||||
return await loadGeoDatabases();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function startPeriodicDatabaseUpdates() {
|
||||
// Calculate interval in milliseconds
|
||||
const intervalMs = dbUpdateInterval * 60 * 60 * 1000;
|
||||
|
||||
// Schedule periodic updates
|
||||
setInterval(async () => {
|
||||
try {
|
||||
await checkAndUpdateDatabases();
|
||||
} catch (err) {
|
||||
logs.error('ipfilter', `Failed during periodic database update: ${err}`);
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
logs.plugin('ipfilter', `Scheduled GeoIP database updates every ${dbUpdateInterval} hours`);
|
||||
}
|
||||
|
||||
await loadGeoDatabases();
|
||||
|
||||
startPeriodicDatabaseUpdates();
|
||||
|
||||
const asnNameMatchers = new Map();
|
||||
for (const [group, names] of Object.entries(blockedASNNames)) {
|
||||
asnNameMatchers.set(group, new AhoCorasick(names));
|
||||
}
|
||||
|
||||
function cacheAndReturn(ip, blocked, blockType, blockValue, customPage, asnOrgName) {
|
||||
const expiresAt = Date.now() + ipBlockCacheTTL;
|
||||
ipBlockCache.set(ip, { blocked, blockType, blockValue, customPage, asnOrgName, expiresAt });
|
||||
// Enforce maximum cache size
|
||||
if (ipBlockCacheMaxEntries > 0 && ipBlockCache.size > ipBlockCacheMaxEntries) {
|
||||
// Remove the oldest entry (first key in insertion order)
|
||||
const oldestKey = ipBlockCache.keys().next().value;
|
||||
ipBlockCache.delete(oldestKey);
|
||||
}
|
||||
return [blocked, blockType, blockValue, customPage, asnOrgName];
|
||||
}
|
||||
|
||||
function isBlockedIPExtended(ip) {
|
||||
const now = Date.now();
|
||||
const entry = ipBlockCache.get(ip);
|
||||
if (entry) {
|
||||
if (entry.expiresAt > now) {
|
||||
// Refresh recency by re-inserting entry
|
||||
ipBlockCache.delete(ip);
|
||||
ipBlockCache.set(ip, entry);
|
||||
return [entry.blocked, entry.blockType, entry.blockValue, entry.customPage, entry.asnOrgName];
|
||||
} else {
|
||||
// Entry expired, remove it
|
||||
ipBlockCache.delete(ip);
|
||||
}
|
||||
}
|
||||
|
||||
const countryReader = geoipCountryReader;
|
||||
const asnReader = geoipASNReader;
|
||||
|
||||
if (!countryReader || !asnReader) {
|
||||
return [false, '', '', '', ''];
|
||||
}
|
||||
|
||||
let countryInfo;
|
||||
try {
|
||||
countryInfo = countryReader.get(ip);
|
||||
} catch (e) {}
|
||||
if (countryInfo?.country && blockedCountryCodes.has(countryInfo.country.iso_code)) {
|
||||
const page = countryBlockPages[countryInfo.country.iso_code] || defaultBlockPage;
|
||||
return cacheAndReturn(ip, true, 'country', countryInfo.country.iso_code, page, '');
|
||||
}
|
||||
|
||||
if (countryInfo?.continent && blockedContinentCodes.has(countryInfo.continent.code)) {
|
||||
const page = continentBlockPages[countryInfo.continent.code] || defaultBlockPage;
|
||||
return cacheAndReturn(ip, true, 'continent', countryInfo.continent.code, page, '');
|
||||
}
|
||||
|
||||
let asnInfo;
|
||||
try {
|
||||
asnInfo = asnReader.get(ip);
|
||||
} catch (e) {}
|
||||
if (asnInfo?.autonomous_system_number) {
|
||||
const asn = asnInfo.autonomous_system_number;
|
||||
const orgName = asnInfo.autonomous_system_organization || '';
|
||||
|
||||
for (const [group, arr] of Object.entries(blockedASNs)) {
|
||||
if (arr.includes(asn)) {
|
||||
const page = asnGroupBlockPages[group] || defaultBlockPage;
|
||||
return cacheAndReturn(ip, true, 'asn', group, page, orgName);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [group, matcher] of asnNameMatchers.entries()) {
|
||||
const matches = matcher.find(orgName);
|
||||
if (matches.length) {
|
||||
const page = asnGroupBlockPages[group] || defaultBlockPage;
|
||||
return cacheAndReturn(ip, true, 'asn', group, page, orgName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cacheAndReturn(ip, false, '', '', '', '');
|
||||
}
|
||||
|
||||
function IPBlockMiddleware() {
|
||||
return {
|
||||
middleware: async (req, res, next) => {
|
||||
// Convert Express request to the format expected by ipfilter logic
|
||||
const request = {
|
||||
url: `${req.protocol}://${req.get('host')}${req.originalUrl}`,
|
||||
headers: {
|
||||
get: (name) => req.get(name),
|
||||
entries: () => Object.entries(req.headers).map(([k, v]) => [k, Array.isArray(v) ? v.join(', ') : v])
|
||||
}
|
||||
};
|
||||
|
||||
const clientIP = getRealIP(request);
|
||||
logs.plugin('ipfilter', `Incoming request from IP: ${clientIP}`);
|
||||
const [blocked, blockType, blockValue, customPage, asnOrgName] = isBlockedIPExtended(clientIP);
|
||||
|
||||
if (blocked) {
|
||||
recordEvent('ipfilter.block', {
|
||||
type: blockType,
|
||||
value: blockValue,
|
||||
asn_org: asnOrgName,
|
||||
ip: clientIP, // Include the IP address for stats
|
||||
});
|
||||
const url = new URL(request.url);
|
||||
|
||||
if (url.pathname.startsWith('/api')) {
|
||||
return res.status(403).json({
|
||||
error: 'Access denied from your location or network.',
|
||||
reason: 'geoip',
|
||||
type: blockType,
|
||||
value: blockValue,
|
||||
asn_org: asnOrgName,
|
||||
});
|
||||
}
|
||||
|
||||
// Normalize page paths by stripping leading slash
|
||||
const cleanCustomPage = customPage.replace(/^\/+/, '');
|
||||
const cleanDefaultPage = defaultBlockPage.replace(/^\/+/, '');
|
||||
|
||||
let html = '';
|
||||
logs.plugin(
|
||||
'ipfilter',
|
||||
`Block pages: custom="${cleanCustomPage}", default="${cleanDefaultPage}"`,
|
||||
);
|
||||
logs.plugin('ipfilter', 'Searching for block page in the following locations:');
|
||||
const paths = [
|
||||
// allow absolute paths relative to project root first
|
||||
join(rootDir, cleanCustomPage),
|
||||
];
|
||||
// Fallback to default block page if custom page isn't found
|
||||
if (customPage !== defaultBlockPage) {
|
||||
paths.push(
|
||||
// check default page at root directory
|
||||
join(rootDir, cleanDefaultPage),
|
||||
);
|
||||
}
|
||||
|
||||
for (const p of paths) {
|
||||
logs.plugin('ipfilter', `Trying block page at: ${p}`);
|
||||
const content = await loadBlockPage(p);
|
||||
logs.plugin('ipfilter', `Load result for ${p}: ${content ? 'FOUND' : 'NOT FOUND'}`);
|
||||
if (content) {
|
||||
html = content;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (html) {
|
||||
const output = html.replace('{{.ASNName}}', asnOrgName || 'Blocked Network');
|
||||
return res.status(403).type('html').send(output);
|
||||
} else {
|
||||
return res.status(403).type('text').send('Access denied from your location or network.');
|
||||
}
|
||||
}
|
||||
|
||||
return next();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if (enabled) {
|
||||
registerPlugin('ipfilter', IPBlockMiddleware());
|
||||
} else {
|
||||
logs.plugin('ipfilter', 'IP filter plugin disabled via config');
|
||||
}
|
||||
|
||||
export { checkAndUpdateDatabases, loadGeoDatabases };
|
||||
116
plugins/proxy.js
Normal file
116
plugins/proxy.js
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import { registerPlugin, loadConfig } from '../index.js';
|
||||
import * as logs from '../utils/logs.js';
|
||||
import { createProxyMiddleware } from 'http-proxy-middleware';
|
||||
import express from 'express';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname } from 'path';
|
||||
import { createRequire } from 'module';
|
||||
|
||||
// Setup require for ESM modules
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
// Monkey patch the ws module to prevent "write after end" errors
|
||||
// Based on https://stackoverflow.com/questions/27769842/write-after-end-error-in-node-js-webserver/33591429
|
||||
try {
|
||||
const ws = require('ws');
|
||||
const originalClose = ws.Sender.prototype.close;
|
||||
|
||||
// Override the close method to check if the socket is already closed
|
||||
ws.Sender.prototype.close = function(code, data, mask, cb) {
|
||||
if (this._socket && (this._socket.destroyed || !this._socket.writable)) {
|
||||
logs.plugin('proxy', 'WebSocket close called on already closed socket - ignoring');
|
||||
if (typeof cb === 'function') cb();
|
||||
return;
|
||||
}
|
||||
return originalClose.call(this, code, data, mask, cb);
|
||||
};
|
||||
logs.plugin('proxy', 'Monkey patched ws module to prevent write after end errors');
|
||||
} catch (err) {
|
||||
logs.error('proxy', `Failed to monkey patch ws module: ${err.message}`);
|
||||
}
|
||||
|
||||
const proxyConfig = {};
|
||||
await loadConfig('proxy', proxyConfig);
|
||||
|
||||
const enabled = proxyConfig.Core.Enabled;
|
||||
const upstreamTimeout = proxyConfig.Timeouts.UpstreamTimeoutMs;
|
||||
|
||||
const proxyMappings = {};
|
||||
proxyConfig.Mapping.forEach(mapping => {
|
||||
proxyMappings[mapping.Host] = mapping.Target;
|
||||
});
|
||||
|
||||
logs.plugin('proxy', `Proxy mappings loaded: ${JSON.stringify(proxyMappings)}`);
|
||||
|
||||
// Store for http-proxy-middleware instances
|
||||
const hpmInstances = {};
|
||||
|
||||
function createProxyForHost(target) {
|
||||
const proxyOptions = {
|
||||
target,
|
||||
changeOrigin: true,
|
||||
ws: true,
|
||||
logLevel: 'info',
|
||||
timeout: upstreamTimeout,
|
||||
onError: (err, req, res, _target) => {
|
||||
const targetInfo = _target && _target.href ? _target.href : (typeof _target === 'string' ? _target : 'N/A');
|
||||
logs.error('proxy', `[HPM onError] Proxy error for ${req.method} ${req.url} to ${targetInfo}: ${err.message} (Code: ${err.code || 'N/A'})`);
|
||||
if (res && typeof res.writeHead === 'function') {
|
||||
if (!res.headersSent) {
|
||||
res.writeHead(502, { 'Content-Type': 'text/plain' });
|
||||
res.end('Bad Gateway');
|
||||
} else if (typeof res.destroy === 'function' && !res.destroyed) {
|
||||
res.destroy();
|
||||
}
|
||||
} else if (res && typeof res.end === 'function' && res.writable && !res.destroyed) {
|
||||
logs.plugin('proxy', `[HPM onError] Client WebSocket socket for ${req.url} attempting to end due to proxy error: ${err.message}.`);
|
||||
res.end();
|
||||
}
|
||||
},
|
||||
followRedirects: false,
|
||||
preserveHeaderKeyCase: true,
|
||||
autoRewrite: true,
|
||||
protocolRewrite: 'http',
|
||||
cookieDomainRewrite: { "*": "" }
|
||||
};
|
||||
|
||||
return createProxyMiddleware(proxyOptions);
|
||||
}
|
||||
|
||||
function proxyMiddleware() {
|
||||
const router = express.Router();
|
||||
|
||||
router.use('/api/challenge', (req, res, next) => next('route'));
|
||||
router.use('/api/verify', (req, res, next) => next('route'));
|
||||
router.use('/webfont/', (req, res, next) => next('route'));
|
||||
router.use('/js/', (req, res, next) => next('route'));
|
||||
|
||||
Object.entries(proxyMappings).forEach(([host, target]) => {
|
||||
hpmInstances[host] = createProxyForHost(target);
|
||||
});
|
||||
|
||||
router.use((req, res, next) => {
|
||||
const hostname = req.hostname || req.headers.host?.split(':')[0];
|
||||
const proxyInstance = hpmInstances[hostname];
|
||||
|
||||
if (proxyInstance) {
|
||||
proxyInstance(req, res, next);
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
});
|
||||
|
||||
return { middleware: router };
|
||||
}
|
||||
|
||||
export function getHpmInstance(hostname) {
|
||||
return hpmInstances[hostname];
|
||||
}
|
||||
|
||||
if (enabled) {
|
||||
registerPlugin('proxy', proxyMiddleware());
|
||||
} else {
|
||||
logs.plugin('proxy', 'Proxy plugin disabled via config');
|
||||
}
|
||||
134
plugins/stats.js
Normal file
134
plugins/stats.js
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import { registerPlugin, rootDir, loadConfig } from '../index.js';
|
||||
import { Level } from 'level';
|
||||
import ttl from 'level-ttl';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { Readable } from 'stream';
|
||||
import cookie from 'cookie';
|
||||
import { getRealIP } from '../utils/network.js';
|
||||
import { parseDuration } from '../utils/time.js';
|
||||
|
||||
// Load stats configuration
|
||||
const statsConfig = {};
|
||||
await loadConfig('stats', statsConfig);
|
||||
|
||||
// Map configuration to internal structure
|
||||
const enabled = statsConfig.Core.Enabled;
|
||||
const statsTTL = parseDuration(statsConfig.Storage.StatsTTL);
|
||||
const statsUIPath = statsConfig.WebUI.StatsUIPath;
|
||||
const statsAPIPath = statsConfig.WebUI.StatsAPIPath;
|
||||
|
||||
// Determine __dirname for ES modules
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/**
|
||||
* Adds createReadStream support to LevelDB instances using async iterator.
|
||||
*/
|
||||
function addReadStreamSupport(dbInstance) {
|
||||
if (!dbInstance.createReadStream) {
|
||||
dbInstance.createReadStream = (opts) =>
|
||||
Readable.from(
|
||||
(async function* () {
|
||||
for await (const [key, value] of dbInstance.iterator(opts)) {
|
||||
yield { key, value };
|
||||
}
|
||||
})(),
|
||||
);
|
||||
}
|
||||
return dbInstance;
|
||||
}
|
||||
|
||||
// Initialize LevelDB for stats under db/stats with TTL and stream support
|
||||
const statsDBPath = path.join(rootDir, 'db', 'stats');
|
||||
await fs.mkdir(statsDBPath, { recursive: true });
|
||||
let rawStatsDB = new Level(statsDBPath, { valueEncoding: 'json' });
|
||||
rawStatsDB = addReadStreamSupport(rawStatsDB);
|
||||
const statsDB = ttl(rawStatsDB, { defaultTTL: statsTTL });
|
||||
addReadStreamSupport(statsDB);
|
||||
|
||||
/**
|
||||
* Record a stat event with a metric name and optional data.
|
||||
* @param {string} metric
|
||||
* @param {object} data
|
||||
*/
|
||||
function recordEvent(metric, data = {}) {
|
||||
// Skip if statsDB is not initialized
|
||||
if (typeof statsDB === 'undefined' || !statsDB || typeof statsDB.put !== 'function') {
|
||||
console.warn(`stats: cannot record "${metric}", statsDB not available`);
|
||||
return;
|
||||
}
|
||||
const timestamp = Date.now();
|
||||
// key includes metric and timestamp and a random suffix to avoid collisions
|
||||
const key = `${metric}:${timestamp}:${Math.random().toString(36).slice(2, 8)}`;
|
||||
try {
|
||||
// Use callback form to avoid promise chaining
|
||||
statsDB.put(key, { timestamp, metric, ...data }, (err) => {
|
||||
if (err) console.error('stats: failed to record event', err);
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('stats: failed to record event', err);
|
||||
}
|
||||
}
|
||||
|
||||
// Handler for serving the stats HTML UI
|
||||
async function handleStatsPage(req, res) {
|
||||
const url = new URL(`${req.protocol}://${req.get('host')}${req.originalUrl}`);
|
||||
if (url.pathname !== statsUIPath) return false;
|
||||
try {
|
||||
// Load the stats UI from pages/stats/stats.html in the project root
|
||||
const statsHtmlPath = path.join(rootDir, 'pages', 'stats', 'stats.html');
|
||||
const html = await fs.readFile(statsHtmlPath, 'utf8');
|
||||
res.status(200).type('html').send(html);
|
||||
return true;
|
||||
} catch (e) {
|
||||
res.status(404).send('Stats UI not found');
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Handler for stats API
|
||||
async function handleStatsAPI(req, res) {
|
||||
const url = new URL(`${req.protocol}://${req.get('host')}${req.originalUrl}`);
|
||||
if (url.pathname !== statsAPIPath) return false;
|
||||
const metric = url.searchParams.get('metric');
|
||||
const start = parseInt(url.searchParams.get('start') || '0', 10);
|
||||
const end = parseInt(url.searchParams.get('end') || `${Date.now()}`, 10);
|
||||
const result = [];
|
||||
// Iterate over keys for this metric in the time range
|
||||
for await (const [key, value] of statsDB.iterator({
|
||||
gte: `${metric}:${start}`,
|
||||
lte: `${metric}:${end}\uffff`,
|
||||
})) {
|
||||
result.push(value);
|
||||
}
|
||||
res.status(200).json(result);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Middleware for stats plugin
|
||||
function StatsMiddleware() {
|
||||
return {
|
||||
middleware: async (req, res, next) => {
|
||||
// Always serve stats UI and API first, bypassing auth
|
||||
const pageHandled = await handleStatsPage(req, res);
|
||||
if (pageHandled) return;
|
||||
|
||||
const apiHandled = await handleStatsAPI(req, res);
|
||||
if (apiHandled) return;
|
||||
|
||||
// For any other routes, do not handle
|
||||
return next();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Register the stats plugin
|
||||
if (enabled) {
|
||||
registerPlugin('stats', StatsMiddleware());
|
||||
} else {
|
||||
console.log('Stats plugin disabled via config');
|
||||
}
|
||||
|
||||
// Export recordEvent for other plugins to use
|
||||
export { recordEvent };
|
||||
1452
src/checkpoint.ts
1452
src/checkpoint.ts
File diff suppressed because it is too large
Load diff
782
src/index.ts
782
src/index.ts
|
|
@ -1,782 +0,0 @@
|
|||
import { mkdir, readFile } from 'fs/promises';
|
||||
import { existsSync, readdirSync } from 'fs';
|
||||
import { join, dirname, basename } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { secureImportModule } from './utils/plugins.js';
|
||||
import * as logs from './utils/logs.js';
|
||||
import express, { Request, Response, NextFunction, Router } from 'express';
|
||||
import { createServer, Server } from 'http';
|
||||
import { Socket } from 'net';
|
||||
|
||||
// Load environment variables from .env file
|
||||
import * as dotenv from 'dotenv';
|
||||
dotenv.config();
|
||||
|
||||
// Order of critical plugins that must load before others
|
||||
// Proxy is registered dynamically (see PROXY section in main())
|
||||
const PLUGIN_LOAD_ORDER: readonly string[] = ['ipfilter', 'waf'] as const;
|
||||
|
||||
// Type definitions for the system
|
||||
interface PluginRegistration {
|
||||
readonly name: string;
|
||||
readonly handler: PluginHandler;
|
||||
}
|
||||
|
||||
interface PluginHandler {
|
||||
readonly middleware?: PluginMiddleware | PluginMiddleware[];
|
||||
readonly initializationComplete?: Promise<void>;
|
||||
readonly handleUpgrade?: (req: Request, socket: Socket, head: Buffer) => void;
|
||||
}
|
||||
|
||||
type PluginMiddleware = (req: Request, res: Response, next: NextFunction) => void;
|
||||
|
||||
interface PluginInfo {
|
||||
readonly name: string;
|
||||
readonly path: string;
|
||||
}
|
||||
|
||||
interface ExclusionRule {
|
||||
readonly Path: string;
|
||||
readonly Hosts?: readonly string[];
|
||||
readonly UserAgents?: readonly string[];
|
||||
}
|
||||
|
||||
interface CompiledExclusionRule extends ExclusionRule {
|
||||
readonly pathStartsWith: string;
|
||||
readonly hostsSet: Set<string> | null;
|
||||
readonly userAgentPatterns: readonly RegExp[];
|
||||
}
|
||||
|
||||
interface CheckpointConfig {
|
||||
readonly Core?: {
|
||||
readonly Enabled?: boolean;
|
||||
};
|
||||
readonly Exclusion?: readonly ExclusionRule[];
|
||||
}
|
||||
|
||||
interface AppConfigs {
|
||||
checkpoint?: CheckpointConfig;
|
||||
[configName: string]: unknown;
|
||||
}
|
||||
|
||||
// Type-safe interfaces for threat scoring TOML configuration
|
||||
interface ThreatScoringTomlConfig {
|
||||
readonly Core?: {
|
||||
readonly Enabled?: boolean;
|
||||
readonly LogDetailedScores?: boolean;
|
||||
};
|
||||
readonly Thresholds?: {
|
||||
readonly AllowThreshold?: number;
|
||||
readonly ChallengeThreshold?: number;
|
||||
readonly BlockThreshold?: number;
|
||||
};
|
||||
readonly SignalWeights?: {
|
||||
readonly ATTACK_TOOL_UA?: {
|
||||
readonly weight?: number;
|
||||
readonly confidence?: number;
|
||||
};
|
||||
readonly MISSING_UA?: {
|
||||
readonly weight?: number;
|
||||
readonly confidence?: number;
|
||||
};
|
||||
readonly SQL_INJECTION?: {
|
||||
readonly weight?: number;
|
||||
readonly confidence?: number;
|
||||
};
|
||||
readonly XSS_ATTEMPT?: {
|
||||
readonly weight?: number;
|
||||
readonly confidence?: number;
|
||||
};
|
||||
readonly COMMAND_INJECTION?: {
|
||||
readonly weight?: number;
|
||||
readonly confidence?: number;
|
||||
};
|
||||
readonly PATH_TRAVERSAL?: {
|
||||
readonly weight?: number;
|
||||
readonly confidence?: number;
|
||||
};
|
||||
};
|
||||
readonly Features?: {
|
||||
readonly EnableBotVerification?: boolean;
|
||||
readonly EnableGeoAnalysis?: boolean;
|
||||
readonly EnableBehaviorAnalysis?: boolean;
|
||||
readonly EnableContentAnalysis?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
// Type-safe configuration transformation
|
||||
function transformThreatScoringConfig(tomlConfig: ThreatScoringTomlConfig): {
|
||||
enabled: boolean;
|
||||
thresholds: {
|
||||
ALLOW: number;
|
||||
CHALLENGE: number;
|
||||
BLOCK: number;
|
||||
};
|
||||
signalWeights: {
|
||||
ATTACK_TOOL_UA: { weight: number; confidence: number };
|
||||
MISSING_UA: { weight: number; confidence: number };
|
||||
SQL_INJECTION: { weight: number; confidence: number };
|
||||
XSS_ATTEMPT: { weight: number; confidence: number };
|
||||
COMMAND_INJECTION: { weight: number; confidence: number };
|
||||
PATH_TRAVERSAL: { weight: number; confidence: number };
|
||||
};
|
||||
enableBotVerification: boolean;
|
||||
enableGeoAnalysis: boolean;
|
||||
enableBehaviorAnalysis: boolean;
|
||||
enableContentAnalysis: boolean;
|
||||
logDetailedScores: boolean;
|
||||
} {
|
||||
return {
|
||||
enabled: tomlConfig.Core?.Enabled ?? false,
|
||||
thresholds: {
|
||||
ALLOW: tomlConfig.Thresholds?.AllowThreshold ?? 20,
|
||||
CHALLENGE: tomlConfig.Thresholds?.ChallengeThreshold ?? 60,
|
||||
BLOCK: tomlConfig.Thresholds?.BlockThreshold ?? 100
|
||||
},
|
||||
signalWeights: {
|
||||
ATTACK_TOOL_UA: {
|
||||
weight: tomlConfig.SignalWeights?.ATTACK_TOOL_UA?.weight ?? 30,
|
||||
confidence: tomlConfig.SignalWeights?.ATTACK_TOOL_UA?.confidence ?? 0.75
|
||||
},
|
||||
MISSING_UA: {
|
||||
weight: tomlConfig.SignalWeights?.MISSING_UA?.weight ?? 10,
|
||||
confidence: tomlConfig.SignalWeights?.MISSING_UA?.confidence ?? 0.60
|
||||
},
|
||||
SQL_INJECTION: {
|
||||
weight: tomlConfig.SignalWeights?.SQL_INJECTION?.weight ?? 60,
|
||||
confidence: tomlConfig.SignalWeights?.SQL_INJECTION?.confidence ?? 0.92
|
||||
},
|
||||
XSS_ATTEMPT: {
|
||||
weight: tomlConfig.SignalWeights?.XSS_ATTEMPT?.weight ?? 50,
|
||||
confidence: tomlConfig.SignalWeights?.XSS_ATTEMPT?.confidence ?? 0.88
|
||||
},
|
||||
COMMAND_INJECTION: {
|
||||
weight: tomlConfig.SignalWeights?.COMMAND_INJECTION?.weight ?? 65,
|
||||
confidence: tomlConfig.SignalWeights?.COMMAND_INJECTION?.confidence ?? 0.95
|
||||
},
|
||||
PATH_TRAVERSAL: {
|
||||
weight: tomlConfig.SignalWeights?.PATH_TRAVERSAL?.weight ?? 45,
|
||||
confidence: tomlConfig.SignalWeights?.PATH_TRAVERSAL?.confidence ?? 0.85
|
||||
}
|
||||
},
|
||||
enableBotVerification: tomlConfig.Features?.EnableBotVerification ?? false,
|
||||
enableGeoAnalysis: tomlConfig.Features?.EnableGeoAnalysis ?? false,
|
||||
enableBehaviorAnalysis: tomlConfig.Features?.EnableBehaviorAnalysis ?? false,
|
||||
enableContentAnalysis: tomlConfig.Features?.EnableContentAnalysis ?? false,
|
||||
logDetailedScores: tomlConfig.Core?.LogDetailedScores ?? false
|
||||
};
|
||||
}
|
||||
|
||||
// Extend Express Request to include our custom properties
|
||||
declare global {
|
||||
namespace Express {
|
||||
interface Request {
|
||||
isWebSocketRequest?: boolean;
|
||||
_excluded?: boolean;
|
||||
}
|
||||
|
||||
interface Locals {
|
||||
_excluded?: boolean;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Command-line argument handling - use pm2 for process management
|
||||
if (process.argv.includes('-k') || process.argv.includes('-d')) {
|
||||
console.error('Command-line daemonization is deprecated. Use pm2 instead:');
|
||||
console.error(' npm run daemon # Start as daemon');
|
||||
console.error(' npm run stop # Stop daemon');
|
||||
console.error(' npm run restart # Restart daemon');
|
||||
console.error(' npm run logs # View logs');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Disable console.log in production to suppress output in daemon mode
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
console.log = (): void => {};
|
||||
}
|
||||
|
||||
const pluginRegistry: PluginRegistration[] = [];
|
||||
|
||||
export function registerPlugin(pluginName: string, handler: PluginHandler): void {
|
||||
if (typeof pluginName !== 'string' || !pluginName.trim()) {
|
||||
throw new Error('Plugin name must be a non-empty string');
|
||||
}
|
||||
|
||||
if (!handler || typeof handler !== 'object') {
|
||||
throw new Error('Plugin handler must be an object');
|
||||
}
|
||||
|
||||
// Check for duplicate registration
|
||||
if (pluginRegistry.some(p => p.name === pluginName)) {
|
||||
throw new Error(`Plugin '${pluginName}' is already registered`);
|
||||
}
|
||||
|
||||
pluginRegistry.push({ name: pluginName, handler });
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the array of middleware handlers in registration order.
|
||||
*/
|
||||
export function loadPlugins(): readonly PluginHandler[] {
|
||||
return pluginRegistry.map((item) => item.handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the names of all registered plugins.
|
||||
*/
|
||||
export function getRegisteredPluginNames(): readonly string[] {
|
||||
return pluginRegistry.map((item) => item.name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeze plugin registry to prevent further registration and log the final set.
|
||||
*/
|
||||
export function freezePlugins(): void {
|
||||
Object.freeze(pluginRegistry);
|
||||
pluginRegistry.forEach((item) => Object.freeze(item));
|
||||
logs.msg('Plugin registration frozen');
|
||||
}
|
||||
|
||||
// Determine root directory for config loading
|
||||
let _dirname: string;
|
||||
try {
|
||||
_dirname = dirname(fileURLToPath(import.meta.url));
|
||||
} catch (error) {
|
||||
// Fallback for test environments or cases where import.meta.url isn't available
|
||||
_dirname = process.cwd();
|
||||
}
|
||||
|
||||
// Ensure _dirname is valid
|
||||
if (!_dirname) {
|
||||
_dirname = process.cwd();
|
||||
}
|
||||
|
||||
export const rootDir: string = _dirname.endsWith('/dist') || _dirname.endsWith('\\dist') ?
|
||||
dirname(_dirname) :
|
||||
(_dirname.endsWith('/src') || _dirname.endsWith('\\src') ? dirname(_dirname) : _dirname);
|
||||
|
||||
export async function loadConfig<T extends Record<string, unknown>>(
|
||||
name: string,
|
||||
target: T
|
||||
): Promise<void> {
|
||||
if (typeof name !== 'string' || !name.trim()) {
|
||||
throw new Error('Config name must be a non-empty string');
|
||||
}
|
||||
|
||||
if (!target || typeof target !== 'object') {
|
||||
throw new Error('Config target must be an object');
|
||||
}
|
||||
|
||||
const configPath = join(rootDir, 'config', `${name}.toml`);
|
||||
|
||||
try {
|
||||
const txt = await readFile(configPath, 'utf8');
|
||||
const toml = await import('@iarna/toml');
|
||||
const parsed = toml.parse(txt) as Partial<T>;
|
||||
Object.assign(target, parsed);
|
||||
} catch (error) {
|
||||
const err = error as Error;
|
||||
throw new Error(`Failed to load config '${name}': ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Discover all config files in the config directory
|
||||
function discoverConfigs(): string[] {
|
||||
try {
|
||||
const configDir = join(rootDir, 'config');
|
||||
if (!existsSync(configDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return readdirSync(configDir)
|
||||
.filter(file => file.endsWith('.toml') && !file.includes('.example'))
|
||||
.map(file => basename(file, '.toml'))
|
||||
.sort();
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Discover all plugin files in the plugins directory
|
||||
function discoverPlugins(): PluginInfo[] {
|
||||
try {
|
||||
// Look for plugins in the correct directory based on execution context
|
||||
const isCompiledMode = _dirname.endsWith('/dist') || _dirname.endsWith('\\dist');
|
||||
const pluginsDir = isCompiledMode ?
|
||||
join(_dirname, 'plugins') : // dist/plugins when running compiled
|
||||
join(rootDir, 'src', 'plugins'); // src/plugins when running source
|
||||
|
||||
if (!existsSync(pluginsDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const fileExt = isCompiledMode ? '.js' : '.ts';
|
||||
const relativePathPrefix = isCompiledMode ? 'dist/plugins' : 'src/plugins';
|
||||
|
||||
const allPlugins: PluginInfo[] = readdirSync(pluginsDir)
|
||||
.filter(file => file.endsWith(fileExt))
|
||||
.map(file => ({
|
||||
name: basename(file, fileExt),
|
||||
path: join(relativePathPrefix, file)
|
||||
}));
|
||||
|
||||
// Sort by load order, then alphabetically
|
||||
const ordered: PluginInfo[] = [];
|
||||
const remaining = [...allPlugins];
|
||||
|
||||
PLUGIN_LOAD_ORDER.forEach(name => {
|
||||
const idx = remaining.findIndex(p => p.name === name);
|
||||
if (idx >= 0) {
|
||||
ordered.push(...remaining.splice(idx, 1));
|
||||
}
|
||||
});
|
||||
|
||||
return [...ordered, ...remaining.sort((a, b) => a.name.localeCompare(b.name))];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function initDataDirectories(): Promise<void> {
|
||||
logs.section('INIT');
|
||||
const directories = [
|
||||
join(rootDir, 'data'),
|
||||
join(rootDir, 'db'),
|
||||
join(rootDir, 'config')
|
||||
];
|
||||
|
||||
for (const dirPath of directories) {
|
||||
try {
|
||||
await mkdir(dirPath, { recursive: true });
|
||||
} catch {
|
||||
// Ignore errors if directory already exists
|
||||
}
|
||||
}
|
||||
logs.init('Data directories are now in place');
|
||||
}
|
||||
|
||||
function staticFileMiddleware(): Router {
|
||||
const router = express.Router();
|
||||
|
||||
// Validate static directories exist before serving
|
||||
const webfontPath = join(rootDir, 'pages/interstitial/webfont');
|
||||
const jsPath = join(rootDir, 'pages/interstitial/js');
|
||||
|
||||
if (existsSync(webfontPath)) {
|
||||
router.use('/webfont', express.static(webfontPath, {
|
||||
maxAge: '7d'
|
||||
}));
|
||||
}
|
||||
|
||||
if (existsSync(jsPath)) {
|
||||
router.use('/js', express.static(jsPath, {
|
||||
maxAge: '7d'
|
||||
}));
|
||||
}
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
await initDataDirectories();
|
||||
|
||||
logs.section('CONFIG');
|
||||
|
||||
// Dynamically discover and load all config files
|
||||
const configNames = discoverConfigs();
|
||||
const configs: AppConfigs = {};
|
||||
|
||||
for (const configName of configNames) {
|
||||
configs[configName] = {};
|
||||
try {
|
||||
await loadConfig(configName, configs[configName] as Record<string, unknown>);
|
||||
logs.config(configName, 'loaded');
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.error('config', `Failed to load ${configName} config: ${error.message}`);
|
||||
// Don't exit on config error - plugin might work without config
|
||||
}
|
||||
}
|
||||
|
||||
const earlyCheckpointConfig = configs.checkpoint as CheckpointConfig || {};
|
||||
|
||||
// Initialize threat scoring system if threat-scoring config exists
|
||||
logs.section('THREAT SCORING');
|
||||
if (configs['threat-scoring']) {
|
||||
try {
|
||||
const { configureDefaultThreatScorer } = await import('./utils/threat-scoring.js');
|
||||
const threatConfig = configs['threat-scoring'] as ThreatScoringTomlConfig;
|
||||
|
||||
// Transform config structure to match ThreatScoringConfig interface
|
||||
const scoringConfig = transformThreatScoringConfig(threatConfig);
|
||||
|
||||
configureDefaultThreatScorer(scoringConfig);
|
||||
logs.msg('Threat scoring system initialized');
|
||||
} catch (e) {
|
||||
const error = e as Error;
|
||||
logs.error('threat-scoring', `Failed to initialize threat scoring: ${error.message}`);
|
||||
}
|
||||
} else {
|
||||
logs.msg('Threat scoring disabled - no config file found');
|
||||
}
|
||||
|
||||
const app = express();
|
||||
|
||||
// Disable Express default header so our headers plugin can set its own value
|
||||
app.disable('x-powered-by');
|
||||
|
||||
// Global header applied to all responses handled by Express
|
||||
app.use((_req: Request, res: Response, next: NextFunction) => {
|
||||
// Only set if not already set
|
||||
if (!res.headersSent) {
|
||||
res.setHeader('X-Powered-By', 'Checkpoint (https://git.caileb.com/Caileb/Checkpoint)');
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
// Hold proxy plugin module for WebSocket upgrade forwarding
|
||||
let proxyPluginModule: PluginHandler | undefined;
|
||||
|
||||
// Trust proxy headers (important for proper protocol detection)
|
||||
app.set('trust proxy', true);
|
||||
|
||||
// WebSocket requests bypass body parsing
|
||||
app.use((req: Request, _res: Response, next: NextFunction) => {
|
||||
const upgradeHeader = req.headers.upgrade;
|
||||
const connectionHeader = req.headers.connection;
|
||||
|
||||
if (upgradeHeader === 'websocket' ||
|
||||
(connectionHeader && connectionHeader.toLowerCase().includes('upgrade'))) {
|
||||
req.isWebSocketRequest = true;
|
||||
return next();
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
const bodyLimit = process.env.MAX_BODY_SIZE || '10mb';
|
||||
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
if (req.isWebSocketRequest) return next();
|
||||
express.json({ limit: bodyLimit })(req, res, next);
|
||||
});
|
||||
|
||||
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
if (req.isWebSocketRequest) return next();
|
||||
express.urlencoded({ extended: true, limit: bodyLimit })(req, res, next);
|
||||
});
|
||||
|
||||
// Load plugins
|
||||
|
||||
// Load behavioral detection middleware
|
||||
logs.section('BEHAVIORAL DETECTION');
|
||||
try {
|
||||
await import('./utils/behavioral-middleware.js');
|
||||
logs.msg('Behavioral detection middleware loaded');
|
||||
} catch (e) {
|
||||
const error = e as Error;
|
||||
logs.error('behavioral', `Failed to load behavioral detection: ${error.message}`);
|
||||
}
|
||||
|
||||
// CRITICAL: Load checkpoint middleware directly (since it's not in plugins directory)
|
||||
logs.section('CHECKPOINT');
|
||||
try {
|
||||
await import('./checkpoint.js');
|
||||
logs.msg('Checkpoint middleware loaded');
|
||||
} catch (e) {
|
||||
const error = e as Error;
|
||||
logs.error('checkpoint', `Failed to load checkpoint middleware: ${error.message}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PROXY (dynamic registration)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
logs.section('PROXY');
|
||||
try {
|
||||
const {
|
||||
getProxyMiddleware,
|
||||
handleUpgrade: proxyHandleUpgrade,
|
||||
isProxyEnabled
|
||||
} = await import('./proxy.js');
|
||||
|
||||
if (typeof isProxyEnabled === 'function' && isProxyEnabled()) {
|
||||
const proxyMw = getProxyMiddleware();
|
||||
if (proxyMw) {
|
||||
registerPlugin('proxy', {
|
||||
middleware: proxyMw,
|
||||
handleUpgrade: proxyHandleUpgrade
|
||||
});
|
||||
proxyPluginModule = {
|
||||
middleware: proxyMw,
|
||||
handleUpgrade: proxyHandleUpgrade
|
||||
};
|
||||
logs.msg('Proxy middleware enabled and registered');
|
||||
} else {
|
||||
logs.msg('Proxy middleware disabled via configuration');
|
||||
}
|
||||
} else {
|
||||
logs.msg('Proxy disabled via configuration');
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.error('proxy', `Failed to initialize proxy: ${error.message}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Discover and load all plugins from the plugins directory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const plugins = discoverPlugins();
|
||||
|
||||
for (const plugin of plugins) {
|
||||
// Create section header based on plugin name
|
||||
const sectionName = plugin.name.toUpperCase().replace(/-/g, ' ');
|
||||
logs.section(sectionName);
|
||||
|
||||
try {
|
||||
const module = await secureImportModule(plugin.path) as PluginHandler;
|
||||
|
||||
// Wait for plugin initialization if it exports an init promise
|
||||
if (module.initializationComplete) {
|
||||
await module.initializationComplete;
|
||||
}
|
||||
} catch (e) {
|
||||
const error = e as Error;
|
||||
logs.error(plugin.name, `Failed to load ${plugin.name} plugin: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Register static middleware
|
||||
app.use(staticFileMiddleware());
|
||||
|
||||
logs.section('PLUGINS');
|
||||
// Display all registered plugins
|
||||
const registeredPluginNames = getRegisteredPluginNames();
|
||||
registeredPluginNames.forEach(name => logs.msg(name));
|
||||
|
||||
logs.section('SYSTEM');
|
||||
freezePlugins();
|
||||
|
||||
// Use pre-loaded checkpoint config for exclusion rules
|
||||
const checkpointConfig = earlyCheckpointConfig;
|
||||
const exclusionRules = checkpointConfig.Exclusion || [];
|
||||
|
||||
// Pre-compile patterns once at startup for better performance
|
||||
const compiledExclusionPatterns: CompiledExclusionRule[] = exclusionRules.map(rule => ({
|
||||
...rule,
|
||||
pathStartsWith: rule.Path, // Cache for faster comparison
|
||||
hostsSet: rule.Hosts ? new Set(rule.Hosts) : null, // Use Set for O(1) lookup
|
||||
userAgentPatterns: (rule.UserAgents || []).map(pattern => {
|
||||
try {
|
||||
return new RegExp(pattern, 'i');
|
||||
} catch {
|
||||
logs.error('config', `Invalid UserAgent regex pattern: ${pattern}`);
|
||||
// Return a pattern that never matches if the regex is invalid
|
||||
return /(?!)/;
|
||||
}
|
||||
})
|
||||
}));
|
||||
|
||||
// Create exclusion pre-check middleware that runs BEFORE all plugins
|
||||
// CRITICAL: This middleware determines which requests bypass security processing
|
||||
// Breaking this logic will either block legitimate traffic or let malicious traffic through
|
||||
app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
// Skip exclusion check if checkpoint is disabled
|
||||
if (!checkpointConfig.Core?.Enabled) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const pathname = req.path;
|
||||
const hostname = req.hostname;
|
||||
const userAgent = req.headers['user-agent'] || '';
|
||||
|
||||
// Validate inputs to prevent bypasses through malformed data
|
||||
if (typeof pathname !== 'string' || typeof hostname !== 'string') {
|
||||
logs.error('server', 'Invalid pathname or hostname in request');
|
||||
return next();
|
||||
}
|
||||
|
||||
// Process exclusion rules with optimized data structures for better performance
|
||||
const shouldExclude = compiledExclusionPatterns.some(rule => {
|
||||
// Check path match first (most likely to fail, so fail fast)
|
||||
if (!pathname.startsWith(rule.pathStartsWith)) return false;
|
||||
|
||||
// Check host match using Set for O(1) lookup
|
||||
if (rule.hostsSet && !rule.hostsSet.has(hostname)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check user agent match using pre-compiled patterns
|
||||
if (rule.userAgentPatterns.length > 0) {
|
||||
return rule.userAgentPatterns.some(pattern => {
|
||||
try {
|
||||
return pattern.test(userAgent);
|
||||
} catch {
|
||||
// If regex test fails, don't exclude (fail secure)
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return true; // No UA restrictions, so it matches
|
||||
});
|
||||
|
||||
if (shouldExclude) {
|
||||
// Mark request as excluded so plugins can skip processing
|
||||
req._excluded = true;
|
||||
res.locals._excluded = true;
|
||||
logs.server(`Pre-excluded request from ${req.ip} to ${pathname}`);
|
||||
}
|
||||
|
||||
next();
|
||||
});
|
||||
|
||||
// Apply all plugin middlewares to Express
|
||||
const middlewareHandlers = loadPlugins();
|
||||
middlewareHandlers.forEach(handler => {
|
||||
// Validate plugin interface
|
||||
if (!handler || typeof handler !== 'object') {
|
||||
logs.error('server', 'Invalid plugin: must export an object with middleware property');
|
||||
return;
|
||||
}
|
||||
|
||||
if (handler.middleware) {
|
||||
// If plugin exports an object with middleware property
|
||||
if (Array.isArray(handler.middleware)) {
|
||||
// If middleware is an array, apply each one
|
||||
handler.middleware.forEach(mw => {
|
||||
if (typeof mw === 'function') {
|
||||
app.use(mw);
|
||||
} else {
|
||||
logs.error('server', 'Invalid middleware function in array');
|
||||
}
|
||||
});
|
||||
} else if (typeof handler.middleware === 'function') {
|
||||
// Single middleware
|
||||
app.use(handler.middleware);
|
||||
} else {
|
||||
logs.error('server', 'Middleware must be a function or array of functions');
|
||||
}
|
||||
} else {
|
||||
logs.error('server', 'Plugin missing required middleware property');
|
||||
}
|
||||
});
|
||||
|
||||
// Basic test route for middleware testing
|
||||
app.get('/', (req: Request, res: Response) => {
|
||||
res.json({
|
||||
message: 'Checkpoint Security Gateway',
|
||||
timestamp: new Date().toISOString(),
|
||||
ip: req.ip || 'unknown',
|
||||
userAgent: req.headers['user-agent'] || 'unknown'
|
||||
});
|
||||
});
|
||||
|
||||
// 404 handler
|
||||
app.use((_req: Request, res: Response) => {
|
||||
res.status(404).send('Not Found');
|
||||
});
|
||||
|
||||
// Error handler
|
||||
app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => {
|
||||
logs.error('server', `Server error: ${err.message}`);
|
||||
res.status(500).send(`Server Error: ${err.message}`);
|
||||
});
|
||||
|
||||
logs.section('SERVER');
|
||||
const portNumber = Number(process.env.PORT || 3000);
|
||||
|
||||
// Validate port number
|
||||
if (isNaN(portNumber) || portNumber < 1 || portNumber > 65535) {
|
||||
throw new Error(`Invalid port number: ${process.env.PORT}`);
|
||||
}
|
||||
|
||||
const server: Server = createServer(app);
|
||||
|
||||
// Track active sockets for proper shutdown handling
|
||||
const activeSockets = new Set<Socket>();
|
||||
let isShuttingDown = false;
|
||||
|
||||
// Extend socket timeout to prevent premature disconnections
|
||||
server.on('connection', (socket: Socket) => {
|
||||
// Track this socket
|
||||
activeSockets.add(socket);
|
||||
socket.on('close', () => activeSockets.delete(socket));
|
||||
|
||||
// Set longer socket timeouts to avoid connection issues
|
||||
socket.setTimeout(120000); // 2 minutes timeout
|
||||
socket.setKeepAlive(true, 60000); // Keep-alive every 60 seconds
|
||||
|
||||
socket.on('error', (err: Error) => {
|
||||
logs.error('server', `Socket error: ${err.message}`);
|
||||
// Don't destroy socket on error, just let it handle itself
|
||||
});
|
||||
});
|
||||
|
||||
// Better WebSocket upgrade handling
|
||||
server.on('upgrade', (req: Request, socket: Socket, head: Buffer) => {
|
||||
// Mark this as a WebSocket request
|
||||
req.isWebSocketRequest = true;
|
||||
|
||||
// WebSocket upgrade events for diagnostic purposes
|
||||
logs.server(`WebSocket upgrade request to ${req.url || 'unknown'}`);
|
||||
|
||||
// Add keep-alive to prevent socket timeouts
|
||||
socket.setKeepAlive(true, 30000);
|
||||
|
||||
// Socket error handling for upgrades
|
||||
socket.on('error', (err: Error) => {
|
||||
logs.error('server', `WebSocket upgrade socket error: ${err.message}`);
|
||||
if (!socket.destroyed) {
|
||||
socket.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
// Forward upgrade to proxy plugin
|
||||
if (proxyPluginModule && typeof proxyPluginModule.handleUpgrade === 'function') {
|
||||
proxyPluginModule.handleUpgrade(req, socket, head);
|
||||
} else {
|
||||
socket.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(portNumber, () => {
|
||||
logs.server(`🚀 Server is up and running on port ${portNumber}...`);
|
||||
logs.section('REQ LOGS');
|
||||
});
|
||||
|
||||
// Graceful shutdown handling
|
||||
const shutdownHandler = (signal: string): void => {
|
||||
if (isShuttingDown) {
|
||||
console.log('Shutdown already in progress, please wait...');
|
||||
return;
|
||||
}
|
||||
isShuttingDown = true;
|
||||
console.log(`\n📡 Received ${signal}, shutting down gracefully...`);
|
||||
|
||||
// Destroy all active sockets to ensure server.close completes
|
||||
activeSockets.forEach((sock) => sock.destroy());
|
||||
|
||||
server.close(() => {
|
||||
console.log('✅ HTTP server closed');
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
// Force exit if still hanging
|
||||
setTimeout(() => {
|
||||
console.error('Forcing shutdown after timeout');
|
||||
process.exit(1);
|
||||
}, 10000);
|
||||
};
|
||||
|
||||
process.on('SIGINT', () => shutdownHandler('SIGINT'));
|
||||
process.on('SIGTERM', () => shutdownHandler('SIGTERM'));
|
||||
}
|
||||
|
||||
// Skip auto-execution during tests
|
||||
if (process.env.NODE_ENV !== 'test' && process.env.JEST_WORKER_ID === undefined) {
|
||||
main().catch((error: Error) => {
|
||||
console.error('Fatal error during startup:', error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
1270
src/plugins/waf.ts
1270
src/plugins/waf.ts
File diff suppressed because it is too large
Load diff
525
src/proxy.ts
525
src/proxy.ts
|
|
@ -1,525 +0,0 @@
|
|||
import { loadConfig } from './index.js';
|
||||
import { parseDuration } from './utils/time.js';
|
||||
import * as logs from './utils/logs.js';
|
||||
import express from 'express';
|
||||
import { IncomingMessage } from 'http';
|
||||
import { Socket } from 'net';
|
||||
|
||||
// @ts-ignore - http-proxy-middleware doesn't have perfect TypeScript definitions
|
||||
import { createProxyMiddleware, Options as ProxyOptions } from 'http-proxy-middleware';
|
||||
|
||||
// ==================== SECURITY-HARDENED TYPE DEFINITIONS ====================
|
||||
|
||||
interface ProxyCoreConfig {
|
||||
Enabled: boolean;
|
||||
MaxBodySizeMB?: number;
|
||||
}
|
||||
|
||||
interface ProxyTimeoutsConfig {
|
||||
UpstreamTimeoutMs: number;
|
||||
}
|
||||
|
||||
interface ProxyMappingConfig {
|
||||
Host: string;
|
||||
Target: string;
|
||||
AllowedMethods?: string[];
|
||||
}
|
||||
|
||||
interface ProxyConfiguration {
|
||||
Core: ProxyCoreConfig;
|
||||
Timeouts: ProxyTimeoutsConfig;
|
||||
Mapping: ProxyMappingConfig[];
|
||||
}
|
||||
|
||||
interface ProxyInstance {
|
||||
(req: express.Request, res: express.Response, next: express.NextFunction): void;
|
||||
upgrade?: (req: IncomingMessage, socket: Socket, head: Buffer) => void;
|
||||
}
|
||||
|
||||
interface ProxyErrorWithCode extends Error {
|
||||
code?: string;
|
||||
}
|
||||
|
||||
interface ExpressRequest {
|
||||
method?: string;
|
||||
path: string;
|
||||
headers: express.Request['headers'];
|
||||
hostname?: string;
|
||||
body?: any;
|
||||
isWebSocketRequest?: boolean;
|
||||
}
|
||||
|
||||
interface ExpressResponse {
|
||||
headersSent: boolean;
|
||||
writeHead(statusCode: number, headers?: Record<string, string>): void;
|
||||
end(data?: string): void;
|
||||
}
|
||||
|
||||
// ==================== SECURITY CONSTANTS ====================
|
||||
|
||||
const SECURITY_LIMITS = {
|
||||
MAX_PROXY_MAPPINGS: 100,
|
||||
MAX_HOST_LENGTH: 253, // RFC 1035 limit
|
||||
MAX_TARGET_LENGTH: 2000,
|
||||
MAX_UPSTREAM_TIMEOUT: parseDuration('5m'), // 5 minutes
|
||||
MIN_UPSTREAM_TIMEOUT: parseDuration('1s'), // 1 second
|
||||
SOCKET_TIMEOUT: parseDuration('30s'), // 30 seconds
|
||||
WEBSOCKET_TIMEOUT: 0, // No timeout for WebSockets
|
||||
MAX_METHODS_PER_HOST: 20, // Maximum allowed methods per host
|
||||
} as const;
|
||||
|
||||
const BLOCKED_INTERNAL_PATHS = [
|
||||
'/api/challenge',
|
||||
'/api/verify',
|
||||
] as const;
|
||||
|
||||
|
||||
|
||||
// Valid HTTP methods that can be configured
|
||||
const VALID_HTTP_METHODS = [
|
||||
'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH'
|
||||
] as const;
|
||||
|
||||
const DEFAULT_ALLOWED_METHODS = ['GET', 'HEAD', 'POST', 'PUT', 'OPTIONS'] as const;
|
||||
|
||||
// Proxy configuration - loaded during initialization to avoid race conditions
|
||||
let proxyConfig: ProxyConfiguration = {
|
||||
Core: { Enabled: false },
|
||||
Timeouts: { UpstreamTimeoutMs: 30000 },
|
||||
Mapping: []
|
||||
};
|
||||
|
||||
/**
|
||||
* SECURITY VALIDATION: Initialize proxy configuration with comprehensive error handling
|
||||
* Prevents SSRF attacks and ensures safe defaults
|
||||
*/
|
||||
async function initializeProxy(): Promise<void> {
|
||||
try {
|
||||
const loadedConfig: any = {};
|
||||
await loadConfig('proxy', loadedConfig);
|
||||
|
||||
// Validate and sanitize loaded configuration
|
||||
proxyConfig = {
|
||||
Core: {
|
||||
Enabled: Boolean(loadedConfig.Core?.Enabled)
|
||||
},
|
||||
Timeouts: {
|
||||
UpstreamTimeoutMs: Math.max(
|
||||
SECURITY_LIMITS.MIN_UPSTREAM_TIMEOUT,
|
||||
Math.min(SECURITY_LIMITS.MAX_UPSTREAM_TIMEOUT, Number(loadedConfig.Timeouts?.UpstreamTimeoutMs) || 30000)
|
||||
)
|
||||
},
|
||||
Mapping: []
|
||||
};
|
||||
|
||||
// Safely process proxy mappings with comprehensive validation
|
||||
if (Array.isArray(loadedConfig.Mapping)) {
|
||||
const validMappings = loadedConfig.Mapping
|
||||
.filter((mapping: any) => mapping && typeof mapping === 'object')
|
||||
.slice(0, SECURITY_LIMITS.MAX_PROXY_MAPPINGS)
|
||||
.map((mapping: any) => ({
|
||||
Host: validateHost(mapping.Host),
|
||||
Target: validateTarget(mapping.Target),
|
||||
AllowedMethods: validateAllowedMethods(mapping.AllowedMethods)
|
||||
}))
|
||||
.filter((mapping: ProxyMappingConfig) => mapping.Host && mapping.Target && mapping.AllowedMethods && mapping.AllowedMethods.length > 0);
|
||||
|
||||
proxyConfig.Mapping = validMappings;
|
||||
}
|
||||
|
||||
logs.server('Proxy configuration loaded and validated successfully');
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
logs.error('proxy', `Failed to load proxy config: ${errorMessage}`);
|
||||
// proxyConfig already has safe defaults
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* SECURITY CRITICAL: Validate host to prevent header injection and SSRF
|
||||
*/
|
||||
function validateHost(host: unknown): string {
|
||||
if (typeof host !== 'string' || !host) return '';
|
||||
|
||||
// Sanitize and validate host
|
||||
const sanitizedHost = host.toLowerCase().trim().slice(0, SECURITY_LIMITS.MAX_HOST_LENGTH);
|
||||
|
||||
const hostRegex = /^[a-z0-9_]([a-z0-9\-_]{0,61}[a-z0-9_])?(\.[a-z0-9_]([a-z0-9\-_]{0,61}[a-z0-9_])?)*$/;
|
||||
|
||||
if (!hostRegex.test(sanitizedHost)) {
|
||||
logs.warn('proxy', `Invalid host format rejected: ${host}`);
|
||||
return '';
|
||||
}
|
||||
|
||||
return sanitizedHost;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate target URL for reverse proxy use (allows internal IPs for local services)
|
||||
*/
|
||||
function validateTarget(target: unknown): string {
|
||||
if (typeof target !== 'string' || !target) return '';
|
||||
|
||||
try {
|
||||
const url = new URL(target);
|
||||
|
||||
// Only allow HTTP and HTTPS protocols
|
||||
if (!['http:', 'https:'].includes(url.protocol)) {
|
||||
logs.warn('proxy', `Invalid protocol rejected: ${url.protocol}`);
|
||||
return '';
|
||||
}
|
||||
|
||||
// For reverse proxy use case, we WANT to allow internal IPs
|
||||
// This is the whole point - forwarding to local backend services
|
||||
|
||||
// Limit target URL length for safety
|
||||
const sanitizedTarget = target.slice(0, SECURITY_LIMITS.MAX_TARGET_LENGTH);
|
||||
return sanitizedTarget;
|
||||
} catch (error) {
|
||||
logs.warn('proxy', `Invalid target URL rejected: ${target}`);
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate allowed HTTP methods for a host
|
||||
*/
|
||||
function validateAllowedMethods(methods: unknown): string[] {
|
||||
// If no methods specified, use defaults
|
||||
if (!methods) {
|
||||
return [...DEFAULT_ALLOWED_METHODS];
|
||||
}
|
||||
|
||||
// Ensure methods is an array
|
||||
if (!Array.isArray(methods)) {
|
||||
logs.warn('proxy', `Invalid AllowedMethods format, using defaults: ${methods}`);
|
||||
return [...DEFAULT_ALLOWED_METHODS];
|
||||
}
|
||||
|
||||
// Validate and sanitize each method
|
||||
const validMethods = methods
|
||||
.filter((method: any) => typeof method === 'string')
|
||||
.map((method: string) => method.toUpperCase().trim())
|
||||
.filter((method: string) => VALID_HTTP_METHODS.includes(method as any))
|
||||
.slice(0, SECURITY_LIMITS.MAX_METHODS_PER_HOST);
|
||||
|
||||
// Remove duplicates
|
||||
const uniqueMethods = [...new Set(validMethods)];
|
||||
|
||||
// If no valid methods remain, use defaults
|
||||
if (uniqueMethods.length === 0) {
|
||||
logs.warn('proxy', `No valid methods found, using defaults: ${methods}`);
|
||||
return [...DEFAULT_ALLOWED_METHODS];
|
||||
}
|
||||
|
||||
return uniqueMethods;
|
||||
}
|
||||
|
||||
// Initialize configuration on module load
|
||||
await initializeProxy();
|
||||
|
||||
const enabled = proxyConfig.Core.Enabled;
|
||||
const upstreamTimeout = proxyConfig.Timeouts.UpstreamTimeoutMs;
|
||||
|
||||
// Process proxy mappings with error handling and optimization
|
||||
const proxyMappings = new Map<string, string>(); // Use Map for O(1) lookups
|
||||
const allowedMethods = new Map<string, Set<string>>(); // Store allowed methods per host
|
||||
|
||||
try {
|
||||
proxyConfig.Mapping.forEach(mapping => {
|
||||
if (mapping.Host && mapping.Target && mapping.AllowedMethods) {
|
||||
const normalizedHost = mapping.Host.toLowerCase();
|
||||
proxyMappings.set(normalizedHost, mapping.Target);
|
||||
allowedMethods.set(normalizedHost, new Set(mapping.AllowedMethods));
|
||||
} else {
|
||||
logs.warn('proxy', `Invalid proxy mapping: ${JSON.stringify(mapping)}`);
|
||||
}
|
||||
});
|
||||
|
||||
logs.server(`Proxy mappings loaded: ${proxyMappings.size} hosts configured`);
|
||||
} catch (error) {
|
||||
logs.error('proxy', `Failed to process proxy mappings: ${error}`);
|
||||
}
|
||||
|
||||
// Store for http-proxy-middleware instances
|
||||
const hpmInstances = new Map<string, ProxyInstance>();
|
||||
|
||||
/**
|
||||
* SECURITY ENGINE: Create secure proxy instance with comprehensive error handling
|
||||
*/
|
||||
function createProxyForHost(target: string): ProxyInstance {
|
||||
const proxyOptions: ProxyOptions = {
|
||||
target,
|
||||
changeOrigin: true,
|
||||
ws: true,
|
||||
logLevel: 'warn',
|
||||
timeout: upstreamTimeout,
|
||||
proxyTimeout: upstreamTimeout,
|
||||
secure: false,
|
||||
followRedirects: false,
|
||||
|
||||
onProxyReqWs: (proxyReq: any, req: IncomingMessage, socket: Socket) => {
|
||||
logs.server(`WebSocket proxying: ${req.url}`);
|
||||
|
||||
try {
|
||||
// Optimize socket settings
|
||||
socket.setNoDelay(true); // Disable Nagle's algorithm for real-time
|
||||
socket.setKeepAlive(true, SECURITY_LIMITS.SOCKET_TIMEOUT);
|
||||
socket.setTimeout(SECURITY_LIMITS.WEBSOCKET_TIMEOUT); // Disable timeout for WebSockets
|
||||
|
||||
// --- IMPORTANT ---
|
||||
// Do **not** aggressively destroy either side of the connection here.
|
||||
// http-proxy manages cleanup itself and premature destruction is what
|
||||
// leads to `ERR_STREAM_WRITE_AFTER_END` when it later tries to flush
|
||||
// handshake data (see ws-incoming.js in http-proxy).
|
||||
|
||||
// Still surface errors so they are visible for troubleshooting.
|
||||
proxyReq.on('error', (error: Error) => {
|
||||
logs.error('proxy', `WebSocket proxy request error: ${error.message}`);
|
||||
});
|
||||
|
||||
socket.on('error', (error: Error) => {
|
||||
logs.error('proxy', `WebSocket socket error during proxy: ${error.message}`);
|
||||
});
|
||||
} catch (error) {
|
||||
logs.error('proxy', `Error in WebSocket proxy setup: ${error}`);
|
||||
socket.destroy();
|
||||
}
|
||||
},
|
||||
|
||||
onError: (error: ProxyErrorWithCode, _req: any, res: any) => {
|
||||
logs.error('proxy', `Proxy error: ${error.message} (${error.code || 'NO_CODE'})`);
|
||||
|
||||
try {
|
||||
// Handle regular HTTP errors
|
||||
if (res && !res.headersSent && typeof res.writeHead === 'function') {
|
||||
// Send appropriate error based on error code
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
res.writeHead(503, { 'Content-Type': 'text/plain; charset=utf-8' });
|
||||
res.end('Service Unavailable');
|
||||
} else if (error.code === 'ETIMEDOUT') {
|
||||
res.writeHead(504, { 'Content-Type': 'text/plain; charset=utf-8' });
|
||||
res.end('Gateway Timeout');
|
||||
} else if (error.code === 'ENOTFOUND') {
|
||||
res.writeHead(502, { 'Content-Type': 'text/plain; charset=utf-8' });
|
||||
res.end('Bad Gateway - Host Not Found');
|
||||
} else {
|
||||
res.writeHead(502, { 'Content-Type': 'text/plain; charset=utf-8' });
|
||||
res.end('Bad Gateway');
|
||||
}
|
||||
}
|
||||
} catch (responseError) {
|
||||
logs.error('proxy', `Error sending proxy error response: ${responseError}`);
|
||||
}
|
||||
},
|
||||
|
||||
preserveHeaderKeyCase: true,
|
||||
autoRewrite: true,
|
||||
xfwd: true,
|
||||
cookieDomainRewrite: false,
|
||||
|
||||
// Ensure custom X-Powered-By header on proxied responses
|
||||
onProxyRes: (proxyRes: any) => {
|
||||
proxyRes.headers['x-powered-by'] = 'Checkpoint (https://git.caileb.com/Caileb/Checkpoint)';
|
||||
},
|
||||
|
||||
// Optimized POST body handling with security validation
|
||||
onProxyReq: (proxyReq: any, req: any) => {
|
||||
try {
|
||||
// Skip WebSocket upgrade requests
|
||||
if (req.headers?.upgrade === 'websocket' || req.isWebSocketRequest) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Special handling for requests with parsed bodies
|
||||
if ((req.method === 'POST' || req.method === 'PUT' || req.method === 'PATCH' || req.method === 'DELETE') &&
|
||||
req.body && Object.keys(req.body).length > 0) {
|
||||
const contentType = req.headers?.['content-type'] || '';
|
||||
let bodyData: string | undefined;
|
||||
|
||||
try {
|
||||
if (contentType.includes('application/json')) {
|
||||
bodyData = JSON.stringify(req.body);
|
||||
proxyReq.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
} else if (contentType.includes('application/x-www-form-urlencoded')) {
|
||||
bodyData = new URLSearchParams(req.body).toString();
|
||||
proxyReq.setHeader('Content-Type', 'application/x-www-form-urlencoded');
|
||||
} else {
|
||||
// For other content types, try to handle gracefully
|
||||
bodyData = typeof req.body === 'string' ? req.body : JSON.stringify(req.body);
|
||||
}
|
||||
|
||||
const maxBodySize = proxyConfig.Core?.MaxBodySizeMB ? proxyConfig.Core.MaxBodySizeMB * 1024 * 1024 : 10 * 1024 * 1024;
|
||||
if (bodyData && bodyData.length > maxBodySize) {
|
||||
logs.warn('proxy', `Request body too large: ${bodyData.length} bytes (max: ${maxBodySize})`);
|
||||
bodyData = bodyData.slice(0, maxBodySize);
|
||||
}
|
||||
|
||||
// Update content-length and write body
|
||||
if (bodyData) {
|
||||
proxyReq.setHeader('Content-Length', Buffer.byteLength(bodyData, 'utf8'));
|
||||
proxyReq.write(bodyData);
|
||||
}
|
||||
} catch (bodyError) {
|
||||
logs.error('proxy', `Error processing request body: ${bodyError}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logs.error('proxy', `Error in proxy request handler: ${error}`);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
return createProxyMiddleware(proxyOptions) as ProxyInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* SECURITY ENGINE: Create proxy middleware with path validation and security controls
|
||||
*/
|
||||
function createProxyRouter(): express.Router {
|
||||
const router = express.Router();
|
||||
|
||||
// Pre-create proxy instances for all configured hosts
|
||||
let instanceCount = 0;
|
||||
proxyMappings.forEach((target, host) => {
|
||||
try {
|
||||
const proxyInstance = createProxyForHost(target);
|
||||
hpmInstances.set(host, proxyInstance);
|
||||
instanceCount++;
|
||||
logs.server(`Proxy: Created proxy instance for ${host} -> ${target}`);
|
||||
} catch (error) {
|
||||
logs.error('proxy', `Failed to create proxy for ${host}: ${error}`);
|
||||
}
|
||||
});
|
||||
|
||||
logs.server(`Proxy: Initialized ${instanceCount} proxy instances`);
|
||||
|
||||
// Main proxy middleware with optimized host lookup and security controls
|
||||
router.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
|
||||
try {
|
||||
// Security: Block access to internal/sensitive paths
|
||||
const pathname = req.path;
|
||||
|
||||
// Use early return for better performance and security
|
||||
for (const blockedPath of BLOCKED_INTERNAL_PATHS) {
|
||||
if (pathname.startsWith(blockedPath)) {
|
||||
return next();
|
||||
}
|
||||
}
|
||||
|
||||
// Extract and validate hostname
|
||||
const fullHost = req.headers.host || req.hostname || 'localhost';
|
||||
const hostParts = fullHost.split(':');
|
||||
const hostname = hostParts[0]?.toLowerCase();
|
||||
|
||||
// Security: Validate hostname format
|
||||
if (!hostname || hostname.length > SECURITY_LIMITS.MAX_HOST_LENGTH) {
|
||||
logs.warn('proxy', `Invalid hostname rejected: ${hostname}`);
|
||||
return next();
|
||||
}
|
||||
|
||||
// Look up proxy instance
|
||||
const proxyInstance = hpmInstances.get(hostname);
|
||||
|
||||
if (proxyInstance) {
|
||||
// Check if the HTTP method is allowed for this host
|
||||
const requestMethod = req.method?.toUpperCase() || 'GET';
|
||||
const hostAllowedMethods = allowedMethods.get(hostname);
|
||||
|
||||
if (hostAllowedMethods && !hostAllowedMethods.has(requestMethod)) {
|
||||
logs.warn('proxy', `Method ${requestMethod} not allowed for host ${hostname}`);
|
||||
return res.status(405).set('Allow', Array.from(hostAllowedMethods).join(', ')).send('Method Not Allowed');
|
||||
}
|
||||
|
||||
// Enhanced logging for DELETE operations
|
||||
if (requestMethod === 'DELETE') {
|
||||
logs.server(`DELETE request forwarded: ${hostname}${req.path} -> ${proxyMappings.get(hostname)}`);
|
||||
}
|
||||
|
||||
proxyInstance(req, res, next);
|
||||
} else {
|
||||
// No proxy mapping found, continue to next middleware
|
||||
next();
|
||||
}
|
||||
} catch (error) {
|
||||
logs.error('proxy', `Error in proxy middleware: ${error}`);
|
||||
next();
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the proxy middleware - returns null if proxy is disabled
|
||||
*/
|
||||
export function getProxyMiddleware(): express.Router | null {
|
||||
if (!enabled) {
|
||||
logs.server('Proxy: Disabled via configuration');
|
||||
return null;
|
||||
}
|
||||
|
||||
return createProxyRouter();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if proxy is enabled
|
||||
*/
|
||||
export function isProxyEnabled(): boolean {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* SECURITY ENGINE: Optimized WebSocket upgrade handler with comprehensive validation
|
||||
* Handles WebSocket connections securely with proper error handling and resource cleanup
|
||||
*/
|
||||
export function handleUpgrade(req: IncomingMessage, socket: Socket, head: Buffer): void {
|
||||
try {
|
||||
// Security: Validate request and socket
|
||||
if (!req || !socket || socket.destroyed) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract and validate hostname
|
||||
const fullHost = req.headers.host || '';
|
||||
const hostParts = fullHost.split(':');
|
||||
const hostname = hostParts[0]?.toLowerCase();
|
||||
|
||||
if (!hostname || hostname.length > SECURITY_LIMITS.MAX_HOST_LENGTH) {
|
||||
logs.warn('proxy', `Invalid WebSocket hostname rejected: ${hostname}`);
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
|
||||
// Look up proxy instance
|
||||
const proxyInstance = hpmInstances.get(hostname);
|
||||
|
||||
if (proxyInstance && typeof proxyInstance.upgrade === 'function') {
|
||||
// Security: Set socket timeout for WebSocket upgrades
|
||||
socket.setTimeout(SECURITY_LIMITS.SOCKET_TIMEOUT);
|
||||
|
||||
try {
|
||||
proxyInstance.upgrade(req, socket, head);
|
||||
} catch (upgradeError) {
|
||||
logs.error('proxy', `WebSocket upgrade error: ${upgradeError}`);
|
||||
socket.destroy();
|
||||
}
|
||||
} else {
|
||||
logs.warn('proxy', `No WebSocket proxy found for hostname: ${hostname}`);
|
||||
socket.destroy();
|
||||
}
|
||||
} catch (error) {
|
||||
logs.error('proxy', `Error in WebSocket upgrade handler: ${error}`);
|
||||
socket.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
// Export types for external use
|
||||
export type {
|
||||
ProxyConfiguration,
|
||||
ProxyMappingConfig,
|
||||
ProxyInstance,
|
||||
ExpressRequest,
|
||||
ExpressResponse
|
||||
};
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,207 +0,0 @@
|
|||
import { registerPlugin } from '../index.js';
|
||||
import { behavioralDetection } from './behavioral-detection.js';
|
||||
import { getRealIP } from './network.js';
|
||||
import { parseDuration } from './time.js';
|
||||
import * as logs from './logs.js';
|
||||
import { Request, Response, NextFunction } from 'express';
|
||||
|
||||
// Pre-computed durations to avoid parsing overhead in hot paths
|
||||
const DEFAULT_RATE_LIMIT_WINDOW = parseDuration('1m');
|
||||
const DEFAULT_RATE_LIMIT_RESET = parseDuration('1m');
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface BehavioralResponse {
|
||||
readonly status: number;
|
||||
readonly responseTime: number;
|
||||
}
|
||||
|
||||
interface BlockStatus {
|
||||
readonly blocked: boolean;
|
||||
readonly reason?: string;
|
||||
}
|
||||
|
||||
interface RateLimit {
|
||||
readonly exceeded?: boolean;
|
||||
readonly requests?: number;
|
||||
readonly limit?: number;
|
||||
readonly window?: number;
|
||||
readonly resetTime?: number;
|
||||
}
|
||||
|
||||
interface BehavioralPattern {
|
||||
readonly name: string;
|
||||
readonly score: number;
|
||||
}
|
||||
|
||||
interface BehavioralAnalysis {
|
||||
readonly totalScore: number;
|
||||
readonly patterns: readonly BehavioralPattern[];
|
||||
}
|
||||
|
||||
interface BehavioralMiddlewarePlugin {
|
||||
readonly name: string;
|
||||
readonly priority: number;
|
||||
readonly middleware: (req: Request, res: Response, next: NextFunction) => Promise<void> | void;
|
||||
}
|
||||
|
||||
// Extend Express Response locals to include behavioral signals
|
||||
declare global {
|
||||
namespace Express {
|
||||
interface Locals {
|
||||
behavioralSignals?: BehavioralAnalysis;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// BEHAVIORAL DETECTION MIDDLEWARE
|
||||
// =============================================================================
|
||||
// Captures response status codes and integrates with behavioral detection
|
||||
// =============================================================================
|
||||
|
||||
function BehavioralDetectionMiddleware(): BehavioralMiddlewarePlugin {
|
||||
return {
|
||||
name: 'behavioral-detection',
|
||||
priority: 90, // Run after WAF but before final response
|
||||
middleware: async (req: Request, res: Response, next: NextFunction): Promise<void> => {
|
||||
// Skip if behavioral detection is disabled
|
||||
if (!behavioralDetection.config.enabled) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const clientIP = getRealIP(req);
|
||||
const originalEnd = res.end;
|
||||
const originalJson = res.json;
|
||||
const originalSend = res.send;
|
||||
const startTime = Date.now();
|
||||
|
||||
// Function to capture response and analyze
|
||||
const captureResponse = async (): Promise<string | void> => {
|
||||
const responseTime = Date.now() - startTime;
|
||||
|
||||
// Create response object for behavioral analysis
|
||||
const response: BehavioralResponse = {
|
||||
status: res.statusCode,
|
||||
responseTime
|
||||
};
|
||||
|
||||
try {
|
||||
// Log that we're processing this request
|
||||
logs.plugin('behavioral', `Processing response for ${clientIP} - Status: ${res.statusCode}`);
|
||||
|
||||
// Perform behavioural analysis first so internal metrics are updated even if
|
||||
// we cannot mutate the outgoing response anymore.
|
||||
const analysis: BehavioralAnalysis = await behavioralDetection.analyzeRequest(clientIP, req, response);
|
||||
|
||||
// Store behavioural signals for checkpoint integration regardless of whether
|
||||
// headers can be altered.
|
||||
if (res.locals) {
|
||||
res.locals.behavioralSignals = analysis;
|
||||
}
|
||||
|
||||
// If the response has already been sent we must NOT attempt to change
|
||||
// status or headers – doing so triggers the repeated
|
||||
// "Cannot set headers after they are sent to the client" error.
|
||||
if (res.headersSent) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if IP is blocked **before** we send the response so we can return
|
||||
// the appropriate status and headers.
|
||||
const blockStatus: BlockStatus = await behavioralDetection.isBlocked(clientIP);
|
||||
if (blockStatus.blocked) {
|
||||
logs.plugin('behavioral', `Blocked IP ${clientIP} attempted access: ${blockStatus.reason || 'unknown reason'}`);
|
||||
|
||||
res.status(403);
|
||||
res.setHeader('X-Behavioral-Block', 'true');
|
||||
res.setHeader('X-Block-Reason', blockStatus.reason || 'suspicious activity');
|
||||
return behavioralDetection.config.Responses?.BlockMessage ||
|
||||
'Access denied due to suspicious activity';
|
||||
}
|
||||
|
||||
// Check rate limits
|
||||
const rateLimit: RateLimit | null = await behavioralDetection.getRateLimit(clientIP);
|
||||
if (rateLimit && rateLimit.exceeded) {
|
||||
const requests = rateLimit.requests || 0;
|
||||
const limit = rateLimit.limit || 100;
|
||||
const window = rateLimit.window || DEFAULT_RATE_LIMIT_WINDOW;
|
||||
const resetTime = rateLimit.resetTime || Date.now() + window;
|
||||
|
||||
logs.plugin('behavioral', `Rate limit exceeded for ${clientIP}: ${requests}/${limit} in ${window}ms`);
|
||||
|
||||
res.status(429);
|
||||
res.setHeader('X-RateLimit-Limit', String(limit));
|
||||
res.setHeader('X-RateLimit-Remaining', String(Math.max(0, limit - requests)));
|
||||
res.setHeader('X-RateLimit-Reset', String(resetTime));
|
||||
res.setHeader('X-RateLimit-Window', String(window));
|
||||
res.setHeader('Retry-After', String(Math.ceil(window / 1000)));
|
||||
|
||||
return behavioralDetection.config.Responses?.RateLimitMessage ||
|
||||
'Rate limit exceeded. Please slow down your requests.';
|
||||
} else if (rateLimit) {
|
||||
// Set rate-limit headers even when the client is below the threshold
|
||||
const requests = rateLimit.requests || 0;
|
||||
const limit = rateLimit.limit || 100;
|
||||
const resetTime = rateLimit.resetTime || Date.now() + DEFAULT_RATE_LIMIT_RESET;
|
||||
|
||||
res.setHeader('X-RateLimit-Limit', String(limit));
|
||||
res.setHeader('X-RateLimit-Remaining', String(Math.max(0, limit - requests)));
|
||||
res.setHeader('X-RateLimit-Reset', String(resetTime));
|
||||
}
|
||||
|
||||
// Attach behavioural debug headers if we still can.
|
||||
if (analysis.patterns.length > 0) {
|
||||
res.setHeader('X-Behavioral-Score', String(analysis.totalScore));
|
||||
res.setHeader('X-Behavioral-Patterns', analysis.patterns.map(p => p.name).join(', '));
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.error('behavioral', `Error in behavioral analysis: ${error.message}`);
|
||||
// Fail open – do not block the response chain on analysis errors
|
||||
}
|
||||
};
|
||||
|
||||
// Override response methods to capture status with proper typing
|
||||
res.end = function(this: Response, ...args: any[]) {
|
||||
// Capture response asynchronously without blocking
|
||||
setImmediate(() => {
|
||||
captureResponse().catch((err: Error) => {
|
||||
logs.error('behavioral', `Error in async capture: ${err.message}`);
|
||||
});
|
||||
});
|
||||
return (originalEnd as any).apply(this, args);
|
||||
};
|
||||
|
||||
res.json = function(this: Response, ...args: any[]) {
|
||||
// Capture response asynchronously without blocking
|
||||
setImmediate(() => {
|
||||
captureResponse().catch((err: Error) => {
|
||||
logs.error('behavioral', `Error in async capture: ${err.message}`);
|
||||
});
|
||||
});
|
||||
return (originalJson as any).apply(this, args);
|
||||
};
|
||||
|
||||
res.send = function(this: Response, ...args: any[]) {
|
||||
// Capture response asynchronously without blocking
|
||||
setImmediate(() => {
|
||||
captureResponse().catch((err: Error) => {
|
||||
logs.error('behavioral', `Error in async capture: ${err.message}`);
|
||||
});
|
||||
});
|
||||
return (originalSend as any).apply(this, args);
|
||||
};
|
||||
|
||||
next();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Register the plugin
|
||||
registerPlugin('behavioral-detection', BehavioralDetectionMiddleware());
|
||||
|
||||
export default BehavioralDetectionMiddleware;
|
||||
|
|
@ -1,285 +0,0 @@
|
|||
import { TimedDownloadManager, type TimedDownloadSource } from './timed-downloads.js';
|
||||
import { type DurationInput } from './time.js';
|
||||
import { validateCIDR, isValidIP, ipToCIDR } from './ip-validation.js';
|
||||
import * as logs from './logs.js';
|
||||
|
||||
// ==================== TYPE DEFINITIONS ====================
|
||||
|
||||
export interface BotSource {
|
||||
readonly name: string;
|
||||
readonly url: string;
|
||||
readonly updateInterval: DurationInput; // Uses time.ts format: "24h", "5m", etc.
|
||||
readonly dnsVerificationDomain?: string;
|
||||
readonly enabled: boolean;
|
||||
}
|
||||
|
||||
export interface IPRange {
|
||||
readonly cidr: string;
|
||||
readonly ipv4?: boolean;
|
||||
readonly ipv6?: boolean;
|
||||
}
|
||||
|
||||
export interface BotIPRanges {
|
||||
readonly botName: string;
|
||||
readonly ranges: readonly IPRange[];
|
||||
readonly lastUpdated: number;
|
||||
readonly source: string;
|
||||
}
|
||||
|
||||
// ==================== UNIVERSAL PARSER ====================
|
||||
|
||||
/**
|
||||
* Universal parser that extracts IP ranges from any format and converts to CIDR list
|
||||
*/
|
||||
class UniversalRangeParser {
|
||||
static parse(data: string): readonly IPRange[] {
|
||||
const ranges: IPRange[] = [];
|
||||
const trimmed = data.trim();
|
||||
|
||||
logs.plugin('bot-range-downloader', `Parsing ${trimmed.length} bytes of data`);
|
||||
|
||||
// Try JSON parsing first
|
||||
let parsedFromJSON = false;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
|
||||
// Handle Google's JSON format: { "prefixes": [{"ipv4Prefix": "..."}, {"ipv6Prefix": "..."}] }
|
||||
if (parsed.prefixes && Array.isArray(parsed.prefixes)) {
|
||||
for (const prefix of parsed.prefixes) {
|
||||
if (prefix.ipv4Prefix) {
|
||||
const cidrResult = validateCIDR(prefix.ipv4Prefix);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr: prefix.ipv4Prefix,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
}
|
||||
if (prefix.ipv6Prefix) {
|
||||
const cidrResult = validateCIDR(prefix.ipv6Prefix);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr: prefix.ipv6Prefix,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
parsedFromJSON = true;
|
||||
}
|
||||
|
||||
// Handle Microsoft/generic JSON format: { "ranges": ["...", "..."] }
|
||||
else if (parsed.ranges && Array.isArray(parsed.ranges)) {
|
||||
for (const range of parsed.ranges) {
|
||||
if (typeof range === 'string') {
|
||||
const cidrResult = validateCIDR(range);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr: range,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
parsedFromJSON = true;
|
||||
}
|
||||
|
||||
// Handle simple JSON array: ["...", "..."]
|
||||
else if (Array.isArray(parsed)) {
|
||||
for (const item of parsed) {
|
||||
if (typeof item === 'string') {
|
||||
// Check if it's already CIDR or needs conversion
|
||||
if (item.includes('/')) {
|
||||
const cidrResult = validateCIDR(item);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr: item,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
} else if (isValidIP(item)) {
|
||||
// Convert single IP to CIDR notation
|
||||
const cidr = ipToCIDR(item);
|
||||
if (cidr) {
|
||||
const cidrResult = validateCIDR(cidr);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
parsedFromJSON = true;
|
||||
}
|
||||
|
||||
} catch {
|
||||
// Not JSON, continue with text parsing
|
||||
}
|
||||
|
||||
// If we successfully parsed JSON, return those results
|
||||
if (parsedFromJSON) {
|
||||
logs.plugin('bot-range-downloader', `Parsed ${ranges.length} ranges from JSON format`);
|
||||
return ranges.slice(0, 100000);
|
||||
}
|
||||
|
||||
// Text-based parsing - handle both CIDR lists and IP lists
|
||||
const lines = trimmed.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
const cleaned = line.trim();
|
||||
|
||||
// Skip empty lines and comments
|
||||
if (!cleaned || cleaned.startsWith('#') || cleaned.startsWith('//')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if line contains CIDR notation
|
||||
if (cleaned.includes('/')) {
|
||||
const cidrResult = validateCIDR(cleaned);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr: cleaned,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
}
|
||||
// Check if line is a single IP address
|
||||
else if (isValidIP(cleaned)) {
|
||||
// Convert single IP to CIDR notation
|
||||
const cidr = ipToCIDR(cleaned);
|
||||
if (cidr) {
|
||||
const cidrResult = validateCIDR(cidr);
|
||||
if (cidrResult.valid) {
|
||||
ranges.push({
|
||||
cidr,
|
||||
ipv4: cidrResult.type === 'ipv4',
|
||||
ipv6: cidrResult.type !== 'ipv4'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logs.plugin('bot-range-downloader', `Parsed ${ranges.length} ranges from text format`);
|
||||
return ranges.slice(0, 100000);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== BOT RANGE DOWNLOADER ====================
|
||||
|
||||
export class BotRangeDownloader {
|
||||
private readonly downloadManager: TimedDownloadManager;
|
||||
|
||||
constructor() {
|
||||
this.downloadManager = new TimedDownloadManager('bot-ranges');
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts bot source to generic timed download source
|
||||
*/
|
||||
private createTimedDownloadSource(botSource: BotSource): TimedDownloadSource {
|
||||
return {
|
||||
name: botSource.name,
|
||||
url: botSource.url,
|
||||
updateInterval: botSource.updateInterval,
|
||||
enabled: botSource.enabled,
|
||||
parser: {
|
||||
format: 'custom',
|
||||
parseFunction: (data: string) => {
|
||||
const ranges = UniversalRangeParser.parse(data);
|
||||
return {
|
||||
ranges,
|
||||
lastUpdated: Date.now(),
|
||||
source: botSource.url
|
||||
};
|
||||
},
|
||||
},
|
||||
validator: {
|
||||
maxSize: 50 * 1024 * 1024, // 50MB max
|
||||
maxEntries: 100000,
|
||||
validationFunction: (data: unknown): boolean => {
|
||||
return !!(data && typeof data === 'object' &&
|
||||
'ranges' in data && Array.isArray((data as any).ranges) &&
|
||||
(data as any).ranges.length > 0);
|
||||
},
|
||||
},
|
||||
headers: {
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'User-Agent': 'Checkpoint-Security-Gateway/1.0 (Bot Range Downloader)',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads bot ranges using the universal parser
|
||||
*/
|
||||
async downloadBotRanges(botSource: BotSource): Promise<{ success: boolean; ranges?: readonly IPRange[]; error?: string }> {
|
||||
const timedSource = this.createTimedDownloadSource(botSource);
|
||||
const result = await this.downloadManager.downloadFromSource(timedSource);
|
||||
|
||||
if (result.success && result.data) {
|
||||
const parsedData = result.data as { ranges: readonly IPRange[] };
|
||||
return {
|
||||
success: true,
|
||||
ranges: parsedData.ranges,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads bot ranges from disk
|
||||
*/
|
||||
async loadBotRanges(botName: string): Promise<BotIPRanges | null> {
|
||||
const downloadedData = await this.downloadManager.loadDownloadedData(botName);
|
||||
if (!downloadedData) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = downloadedData.data as { ranges: readonly IPRange[] };
|
||||
|
||||
return {
|
||||
botName,
|
||||
ranges: data.ranges,
|
||||
lastUpdated: downloadedData.lastUpdated,
|
||||
source: downloadedData.source,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if bot ranges need updating
|
||||
*/
|
||||
async needsUpdate(botSource: BotSource): Promise<boolean> {
|
||||
const timedSource = this.createTimedDownloadSource(botSource);
|
||||
return await this.downloadManager.needsUpdate(timedSource);
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts periodic updates for bot sources
|
||||
*/
|
||||
startPeriodicUpdates(botSources: readonly BotSource[]): void {
|
||||
const timedSources = botSources.map(source => this.createTimedDownloadSource(source));
|
||||
this.downloadManager.startPeriodicUpdates(timedSources);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates all bot sources that need updating
|
||||
*/
|
||||
async updateAllSources(botSources: readonly BotSource[]): Promise<void> {
|
||||
const timedSources = botSources.map(source => this.createTimedDownloadSource(source));
|
||||
await this.downloadManager.updateAllSources(timedSources);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,465 +0,0 @@
|
|||
import { createRequire } from 'module';
|
||||
import { promisify } from 'util';
|
||||
import { BotRangeDownloader, type BotSource, type IPRange } from './bot-range-downloader.js';
|
||||
import { getRealIP, type NetworkRequest } from './network.js';
|
||||
import { VERIFIED_GOOD_BOTS } from './threat-scoring/constants.js';
|
||||
import { parseDuration } from './time.js';
|
||||
import { CacheUtils, TTLCacheCleaner } from './cache-utils.js';
|
||||
import * as logs from './logs.js';
|
||||
|
||||
// Node.js dns module (Node.js 18+ compatible)
|
||||
const require = createRequire(import.meta.url);
|
||||
const dns = require('dns');
|
||||
const dnsReverse = promisify(dns.reverse);
|
||||
const dnsResolve4 = promisify(dns.resolve4);
|
||||
const dnsResolve6 = promisify(dns.resolve6);
|
||||
|
||||
// ==================== TYPE DEFINITIONS ====================
|
||||
|
||||
export interface BotVerificationResult {
|
||||
readonly isVerifiedBot: boolean;
|
||||
readonly botName: string | null;
|
||||
readonly verificationMethod: 'ip_range' | 'dns_reverse' | 'user_agent' | 'combined';
|
||||
readonly confidence: number; // 0-1
|
||||
readonly details: {
|
||||
readonly userAgentMatch?: boolean;
|
||||
readonly ipRangeMatch?: boolean;
|
||||
readonly dnsVerified?: boolean;
|
||||
readonly reverseDnsHostname?: string;
|
||||
readonly matchedRange?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BotVerificationConfig {
|
||||
readonly enableDNSVerification: boolean;
|
||||
readonly enableIPRangeVerification: boolean;
|
||||
readonly dnsTimeout: number;
|
||||
readonly sources: readonly BotSource[];
|
||||
readonly minimumConfidence: number;
|
||||
readonly weights: {
|
||||
readonly userAgentMatch: number;
|
||||
readonly ipRangeMatch: number;
|
||||
readonly dnsVerification: number;
|
||||
};
|
||||
}
|
||||
|
||||
// ==================== SECURITY CONSTANTS ====================
|
||||
|
||||
const VERIFICATION_LIMITS = {
|
||||
DNS_TIMEOUT: parseDuration('5s'), // 5 seconds for DNS lookups
|
||||
MAX_DNS_QUERIES: 10, // Max concurrent DNS queries
|
||||
IP_CACHE_TTL: parseDuration('1h'), // 1 hour cache for IP verifications
|
||||
DNS_CACHE_TTL: parseDuration('30m'), // 30 minutes cache for DNS verifications
|
||||
MAX_CACHE_SIZE: 10000, // Max entries in verification cache
|
||||
} as const;
|
||||
|
||||
// Bot sources should come from config only - no hardcoded defaults
|
||||
|
||||
// ==================== UTILITY FUNCTIONS ====================
|
||||
|
||||
/**
|
||||
* Checks if an IP address falls within a CIDR range
|
||||
*/
|
||||
function ipInRange(ip: string, cidr: string): boolean {
|
||||
try {
|
||||
// Simple CIDR check implementation
|
||||
const [rangeIP, prefixLength] = cidr.split('/');
|
||||
if (!rangeIP || !prefixLength) return false;
|
||||
|
||||
const prefix = parseInt(prefixLength, 10);
|
||||
|
||||
if (ip.includes('.') && rangeIP.includes('.')) {
|
||||
// IPv4 check
|
||||
return ipv4InRange(ip, rangeIP, prefix);
|
||||
} else if (ip.includes(':') && rangeIP.includes(':')) {
|
||||
// IPv6 check (simplified)
|
||||
return ipv6InRange(ip, rangeIP, prefix);
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* IPv4 CIDR check
|
||||
*/
|
||||
function ipv4InRange(ip: string, rangeIP: string, prefix: number): boolean {
|
||||
try {
|
||||
const ipNum = ipv4ToNumber(ip);
|
||||
const rangeNum = ipv4ToNumber(rangeIP);
|
||||
const mask = (0xffffffff << (32 - prefix)) >>> 0;
|
||||
|
||||
return (ipNum & mask) === (rangeNum & mask);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert IPv4 to number
|
||||
*/
|
||||
function ipv4ToNumber(ip: string): number {
|
||||
const parts = ip.split('.');
|
||||
if (parts.length !== 4) throw new Error('Invalid IPv4');
|
||||
|
||||
return parts.reduce((acc, part) => {
|
||||
const num = parseInt(part, 10);
|
||||
if (isNaN(num) || num < 0 || num > 255) throw new Error('Invalid IPv4 octet');
|
||||
return (acc << 8) + num;
|
||||
}, 0) >>> 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* IPv6 CIDR check (simplified implementation)
|
||||
*/
|
||||
function ipv6InRange(ip: string, rangeIP: string, prefix: number): boolean {
|
||||
try {
|
||||
// This is a simplified IPv6 implementation
|
||||
// For production, you'd want a more robust IPv6 CIDR library
|
||||
const ipHex = ipv6ToHex(ip);
|
||||
const rangeHex = ipv6ToHex(rangeIP);
|
||||
|
||||
const hexChars = Math.floor(prefix / 4);
|
||||
const partialBits = prefix % 4;
|
||||
|
||||
// Compare full hex characters
|
||||
if (ipHex.slice(0, hexChars) !== rangeHex.slice(0, hexChars)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check partial bits if needed
|
||||
if (partialBits > 0 && hexChars < ipHex.length) {
|
||||
const ipChar = parseInt(ipHex[hexChars] || '0', 16);
|
||||
const rangeChar = parseInt(rangeHex[hexChars] || '0', 16);
|
||||
const mask = (0xf << (4 - partialBits)) & 0xf;
|
||||
|
||||
return (ipChar & mask) === (rangeChar & mask);
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert IPv6 to normalized hex string (simplified)
|
||||
*/
|
||||
function ipv6ToHex(ip: string): string {
|
||||
// This is a very simplified IPv6 normalization
|
||||
// For production, use a proper IPv6 library
|
||||
return ip.replace(/:/g, '').toLowerCase().padEnd(32, '0');
|
||||
}
|
||||
|
||||
// ==================== BOT VERIFICATION ENGINE ====================
|
||||
|
||||
export class BotVerificationEngine {
|
||||
private readonly downloader: BotRangeDownloader;
|
||||
private readonly config: BotVerificationConfig;
|
||||
private readonly ipRangeCache = new Map<string, { ranges: readonly IPRange[]; timestamp: number }>();
|
||||
private readonly verificationCache = new Map<string, import('./cache-utils.js').TTLCacheEntry>();
|
||||
private readonly dnsQueue = new Set<string>(); // Track ongoing DNS queries
|
||||
|
||||
constructor(config: BotVerificationConfig) {
|
||||
this.downloader = new BotRangeDownloader();
|
||||
this.config = config;
|
||||
|
||||
this.initializeBotRanges();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize bot ranges and start periodic updates
|
||||
*/
|
||||
private async initializeBotRanges(): Promise<void> {
|
||||
try {
|
||||
// Load existing ranges from disk first
|
||||
for (const source of this.config.sources) {
|
||||
if (!source.enabled) continue;
|
||||
|
||||
const existing = await this.downloader.loadBotRanges(source.name);
|
||||
if (existing) {
|
||||
this.ipRangeCache.set(source.name, {
|
||||
ranges: existing.ranges,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
logs.plugin('bot-verification', `Loaded ${existing.ranges.length} cached ranges for ${source.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Start periodic downloads
|
||||
this.downloader.startPeriodicUpdates(this.config.sources);
|
||||
|
||||
// Download any that need updating
|
||||
for (const source of this.config.sources) {
|
||||
if (source.enabled && await this.downloader.needsUpdate(source)) {
|
||||
const result = await this.downloader.downloadBotRanges(source);
|
||||
if (result.success && result.ranges) {
|
||||
this.ipRangeCache.set(source.name, {
|
||||
ranges: result.ranges,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logs.error('bot-verification', `Failed to initialize bot ranges: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies if a request comes from a legitimate bot
|
||||
*/
|
||||
async verifyBot(request: NetworkRequest, userAgent?: string): Promise<BotVerificationResult> {
|
||||
try {
|
||||
const clientIP = getRealIP(request);
|
||||
const ua = userAgent || String((request.headers as any)?.['user-agent'] || '');
|
||||
const cacheKey = `${clientIP}:${ua}`;
|
||||
|
||||
// Check cache first
|
||||
const cachedResult = CacheUtils.safeGet<BotVerificationResult>(this.verificationCache, cacheKey);
|
||||
if (cachedResult) {
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
// Perform verification
|
||||
const result = await this.performVerification(clientIP, ua);
|
||||
|
||||
// Cache result
|
||||
if (this.verificationCache.size >= VERIFICATION_LIMITS.MAX_CACHE_SIZE) {
|
||||
TTLCacheCleaner.cleanup(this.verificationCache, { maxSize: VERIFICATION_LIMITS.MAX_CACHE_SIZE });
|
||||
}
|
||||
this.verificationCache.set(cacheKey, CacheUtils.createTTLEntry(result, VERIFICATION_LIMITS.IP_CACHE_TTL));
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
logs.error('bot-verification', `Bot verification failed: ${error}`);
|
||||
return this.createNegativeResult();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs the actual bot verification
|
||||
*/
|
||||
private async performVerification(clientIP: string, userAgent: string): Promise<BotVerificationResult> {
|
||||
let userAgentMatch = false;
|
||||
let ipRangeMatch = false;
|
||||
let dnsVerified = false;
|
||||
let reverseDnsHostname: string | undefined;
|
||||
let matchedRange: string | undefined;
|
||||
let botName: string | null = null;
|
||||
let verificationMethod: BotVerificationResult['verificationMethod'] = 'user_agent';
|
||||
|
||||
// 1. Check user agent patterns first
|
||||
for (const [name, botInfo] of Object.entries(VERIFIED_GOOD_BOTS)) {
|
||||
if (this.testUserAgentPattern(userAgent, botInfo.pattern)) {
|
||||
userAgentMatch = true;
|
||||
botName = name;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If no user agent match, this is likely not a bot
|
||||
if (!userAgentMatch) {
|
||||
return this.createNegativeResult();
|
||||
}
|
||||
|
||||
// 2. Check IP range verification if enabled
|
||||
if (this.config.enableIPRangeVerification && botName) {
|
||||
const rangeResult = await this.checkIPRanges(clientIP, botName);
|
||||
if (rangeResult.match) {
|
||||
ipRangeMatch = true;
|
||||
matchedRange = rangeResult.range;
|
||||
verificationMethod = 'ip_range';
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Check DNS verification if enabled
|
||||
if (this.config.enableDNSVerification && botName) {
|
||||
const botConfig = VERIFIED_GOOD_BOTS[botName];
|
||||
const source = this.config.sources.find(s => s.name === botName);
|
||||
|
||||
if (botConfig?.verifyDNS && source?.dnsVerificationDomain) {
|
||||
const dnsResult = await this.verifyDNS(clientIP, source.dnsVerificationDomain);
|
||||
if (dnsResult.verified) {
|
||||
dnsVerified = true;
|
||||
reverseDnsHostname = dnsResult.hostname;
|
||||
if (!ipRangeMatch) {
|
||||
verificationMethod = 'dns_reverse';
|
||||
} else {
|
||||
verificationMethod = 'combined';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate confidence based on verification methods (from config)
|
||||
let confidence = 0;
|
||||
if (userAgentMatch) confidence += this.config.weights.userAgentMatch;
|
||||
if (ipRangeMatch) confidence += this.config.weights.ipRangeMatch;
|
||||
if (dnsVerified) confidence += this.config.weights.dnsVerification;
|
||||
|
||||
const isVerified = confidence >= this.config.minimumConfidence;
|
||||
|
||||
return {
|
||||
isVerifiedBot: isVerified,
|
||||
botName: isVerified ? botName : null,
|
||||
verificationMethod,
|
||||
confidence: Math.min(1, confidence),
|
||||
details: {
|
||||
userAgentMatch,
|
||||
ipRangeMatch,
|
||||
dnsVerified,
|
||||
reverseDnsHostname,
|
||||
matchedRange,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests user agent against pattern with timeout protection
|
||||
*/
|
||||
private testUserAgentPattern(userAgent: string, pattern: RegExp): boolean {
|
||||
try {
|
||||
return pattern.test(userAgent);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if IP is in known bot ranges
|
||||
*/
|
||||
private async checkIPRanges(clientIP: string, botName: string): Promise<{ match: boolean; range?: string }> {
|
||||
try {
|
||||
const cached = this.ipRangeCache.get(botName);
|
||||
if (!cached) {
|
||||
// Try to load from disk
|
||||
const saved = await this.downloader.loadBotRanges(botName);
|
||||
if (saved) {
|
||||
this.ipRangeCache.set(botName, {
|
||||
ranges: saved.ranges,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
return this.checkIPInRanges(clientIP, saved.ranges);
|
||||
}
|
||||
return { match: false };
|
||||
}
|
||||
|
||||
return this.checkIPInRanges(clientIP, cached.ranges);
|
||||
} catch {
|
||||
return { match: false };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if IP is in the provided ranges
|
||||
*/
|
||||
private checkIPInRanges(clientIP: string, ranges: readonly IPRange[]): { match: boolean; range?: string } {
|
||||
for (const range of ranges) {
|
||||
if (ipInRange(clientIP, range.cidr)) {
|
||||
return { match: true, range: range.cidr };
|
||||
}
|
||||
}
|
||||
return { match: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies bot via reverse DNS lookup
|
||||
*/
|
||||
private async verifyDNS(clientIP: string, expectedDomain: string): Promise<{ verified: boolean; hostname?: string }> {
|
||||
// Prevent too many concurrent DNS queries
|
||||
if (this.dnsQueue.size >= VERIFICATION_LIMITS.MAX_DNS_QUERIES) {
|
||||
return { verified: false };
|
||||
}
|
||||
|
||||
const queryKey = clientIP;
|
||||
if (this.dnsQueue.has(queryKey)) {
|
||||
return { verified: false };
|
||||
}
|
||||
|
||||
this.dnsQueue.add(queryKey);
|
||||
|
||||
try {
|
||||
// Step 1: Reverse DNS lookup
|
||||
const hostnames = await Promise.race([
|
||||
dnsReverse(clientIP),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error('DNS timeout')), this.config.dnsTimeout)
|
||||
),
|
||||
]);
|
||||
|
||||
if (!hostnames || hostnames.length === 0) {
|
||||
return { verified: false };
|
||||
}
|
||||
|
||||
// Step 2: Check if hostname matches expected domain
|
||||
const hostname = hostnames[0];
|
||||
if (!hostname.endsWith(`.${expectedDomain}`)) {
|
||||
return { verified: false, hostname };
|
||||
}
|
||||
|
||||
// Step 3: Forward DNS lookup to verify
|
||||
const forwardIPs = await Promise.race([
|
||||
this.resolveHostname(hostname),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(() => reject(new Error('DNS timeout')), this.config.dnsTimeout)
|
||||
),
|
||||
]);
|
||||
|
||||
// Step 4: Check if forward lookup matches original IP
|
||||
const verified = forwardIPs.includes(clientIP);
|
||||
return { verified, hostname };
|
||||
|
||||
} catch (error) {
|
||||
logs.warn('bot-verification', `DNS verification failed for ${clientIP}: ${error}`);
|
||||
return { verified: false };
|
||||
} finally {
|
||||
this.dnsQueue.delete(queryKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves hostname to IP addresses (both IPv4 and IPv6)
|
||||
*/
|
||||
private async resolveHostname(hostname: string): Promise<string[]> {
|
||||
const results: string[] = [];
|
||||
|
||||
try {
|
||||
const ipv4 = await dnsResolve4(hostname);
|
||||
results.push(...ipv4);
|
||||
} catch {
|
||||
// IPv4 resolution failed, continue
|
||||
}
|
||||
|
||||
try {
|
||||
const ipv6 = await dnsResolve6(hostname);
|
||||
results.push(...ipv6);
|
||||
} catch {
|
||||
// IPv6 resolution failed, continue
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a negative verification result
|
||||
*/
|
||||
private createNegativeResult(): BotVerificationResult {
|
||||
return {
|
||||
isVerifiedBot: false,
|
||||
botName: null,
|
||||
verificationMethod: 'user_agent',
|
||||
confidence: 0,
|
||||
details: {
|
||||
userAgentMatch: false,
|
||||
ipRangeMatch: false,
|
||||
dnsVerified: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Bot verification engine should be initialized with config from TOML files
|
||||
// No hardcoded singleton instances
|
||||
|
|
@ -1,278 +0,0 @@
|
|||
// =============================================================================
|
||||
// CENTRALIZED CACHE CLEANUP UTILITY
|
||||
// =============================================================================
|
||||
// Consolidates all cache cleanup logic to prevent duplication
|
||||
|
||||
import { parseDuration } from './time.js';
|
||||
|
||||
export interface CacheEntry<T = unknown> {
|
||||
readonly value: T;
|
||||
readonly timestamp: number;
|
||||
readonly ttl?: number;
|
||||
}
|
||||
|
||||
export interface CacheOptions {
|
||||
readonly maxSize?: number;
|
||||
readonly defaultTTL?: number;
|
||||
readonly cleanupRatio?: number; // What percentage to clean when over limit (0.0-1.0)
|
||||
}
|
||||
|
||||
export interface CacheCleanupResult {
|
||||
readonly expired: number;
|
||||
readonly overflow: number;
|
||||
readonly total: number;
|
||||
}
|
||||
|
||||
export interface TTLCacheEntry {
|
||||
readonly data: unknown;
|
||||
readonly expires: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic TTL-based cache cleaner
|
||||
*/
|
||||
export class TTLCacheCleaner {
|
||||
/**
|
||||
* Cleans expired entries from a Map-based cache with TTL entries
|
||||
* @param cache - Map cache to clean
|
||||
* @param now - Current timestamp (defaults to Date.now())
|
||||
* @returns Number of entries removed
|
||||
*/
|
||||
static cleanExpired<K>(
|
||||
cache: Map<K, TTLCacheEntry>,
|
||||
now: number = Date.now()
|
||||
): number {
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, entry] of cache.entries()) {
|
||||
if (now >= entry.expires) {
|
||||
cache.delete(key);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans cache by removing oldest entries when over size limit
|
||||
* @param cache - Map cache to clean
|
||||
* @param maxSize - Maximum allowed size
|
||||
* @param cleanupRatio - What percentage to remove (default 0.25 = 25%)
|
||||
* @returns Number of entries removed
|
||||
*/
|
||||
static cleanOverflow<K>(
|
||||
cache: Map<K, TTLCacheEntry>,
|
||||
maxSize: number,
|
||||
cleanupRatio: number = 0.25
|
||||
): number {
|
||||
if (cache.size <= maxSize) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const targetSize = Math.floor(maxSize * (1 - cleanupRatio));
|
||||
const toRemove = cache.size - targetSize;
|
||||
|
||||
// Remove oldest entries (based on Map insertion order)
|
||||
let removed = 0;
|
||||
for (const key of cache.keys()) {
|
||||
if (removed >= toRemove) break;
|
||||
cache.delete(key);
|
||||
removed++;
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Comprehensive cache cleanup (expired + overflow)
|
||||
*/
|
||||
static cleanup<K>(
|
||||
cache: Map<K, TTLCacheEntry>,
|
||||
options: CacheOptions = {}
|
||||
): CacheCleanupResult {
|
||||
const { maxSize = 10000, cleanupRatio = 0.25 } = options;
|
||||
const now = Date.now();
|
||||
|
||||
const expired = this.cleanExpired(cache, now);
|
||||
const overflow = this.cleanOverflow(cache, maxSize, cleanupRatio);
|
||||
|
||||
return {
|
||||
expired,
|
||||
overflow,
|
||||
total: expired + overflow
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic timestamped cache cleaner (for caches with timestamp fields)
|
||||
*/
|
||||
export class TimestampCacheCleaner {
|
||||
/**
|
||||
* Cleans expired entries from cache with custom timestamp/TTL logic
|
||||
*/
|
||||
static cleanExpired<K, T extends { timestamp?: number; lastReset?: number }>(
|
||||
cache: Map<K, T>,
|
||||
ttlMs: number,
|
||||
timestampField: 'timestamp' | 'lastReset' = 'timestamp',
|
||||
now: number = Date.now()
|
||||
): number {
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, entry] of cache.entries()) {
|
||||
const entryTime = entry[timestampField];
|
||||
if (!entryTime || (now - entryTime) > ttlMs) {
|
||||
cache.delete(key);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans cache entries with custom expiration logic
|
||||
*/
|
||||
static cleanWithCustomLogic<K, T>(
|
||||
cache: Map<K, T>,
|
||||
shouldExpire: (key: K, value: T, now: number) => boolean,
|
||||
now: number = Date.now()
|
||||
): number {
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, entry] of cache.entries()) {
|
||||
if (shouldExpire(key, entry, now)) {
|
||||
cache.delete(key);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Specialized cleaner for rate limiting caches
|
||||
*/
|
||||
export class RateLimitCacheCleaner {
|
||||
static cleanExpiredRateLimits<K>(
|
||||
cache: Map<K, { count: number; lastReset: number }>,
|
||||
windowMs: number,
|
||||
now: number = Date.now()
|
||||
): number {
|
||||
return TimestampCacheCleaner.cleanExpired(cache, windowMs, 'lastReset', now);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Specialized cleaner for reputation caches
|
||||
*/
|
||||
export class ReputationCacheCleaner {
|
||||
static cleanExpiredReputation<K>(
|
||||
cache: Map<K, { reputation: unknown; timestamp: number }>,
|
||||
ttlMs: number,
|
||||
now: number = Date.now()
|
||||
): number {
|
||||
return TimestampCacheCleaner.cleanExpired(cache, ttlMs, 'timestamp', now);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* High-level cache manager for common patterns
|
||||
*/
|
||||
export class CacheManager {
|
||||
private cleanupTimers: Map<string, NodeJS.Timeout> = new Map();
|
||||
|
||||
/**
|
||||
* Sets up automatic cleanup for a cache
|
||||
*/
|
||||
setupPeriodicCleanup<K>(
|
||||
cacheName: string,
|
||||
cache: Map<K, TTLCacheEntry>,
|
||||
options: CacheOptions & { interval?: string } = {}
|
||||
): void {
|
||||
const { interval = '5m', maxSize = 10000 } = options;
|
||||
const intervalMs = parseDuration(interval);
|
||||
|
||||
const timer = setInterval(() => {
|
||||
const result = TTLCacheCleaner.cleanup(cache, { maxSize });
|
||||
if (result.total > 0) {
|
||||
console.log(`Cache ${cacheName}: cleaned ${result.expired} expired + ${result.overflow} overflow entries`);
|
||||
}
|
||||
}, intervalMs);
|
||||
|
||||
// Store timer so it can be cleared later
|
||||
this.cleanupTimers.set(cacheName, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops periodic cleanup for a cache
|
||||
*/
|
||||
stopPeriodicCleanup(cacheName: string): void {
|
||||
const timer = this.cleanupTimers.get(cacheName);
|
||||
if (timer) {
|
||||
clearInterval(timer);
|
||||
this.cleanupTimers.delete(cacheName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops all periodic cleanups
|
||||
*/
|
||||
stopAllCleanups(): void {
|
||||
for (const [_name, timer] of this.cleanupTimers.entries()) {
|
||||
clearInterval(timer);
|
||||
}
|
||||
this.cleanupTimers.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton cache manager
|
||||
export const cacheManager = new CacheManager();
|
||||
|
||||
/**
|
||||
* Utility functions for common cache operations
|
||||
*/
|
||||
export const CacheUtils = {
|
||||
/**
|
||||
* Creates a TTL cache entry
|
||||
*/
|
||||
createTTLEntry<T>(value: T, ttlMs: number): TTLCacheEntry {
|
||||
return {
|
||||
data: value,
|
||||
expires: Date.now() + ttlMs
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* Checks if TTL entry is expired
|
||||
*/
|
||||
isExpired(entry: TTLCacheEntry, now: number = Date.now()): boolean {
|
||||
return now >= entry.expires;
|
||||
},
|
||||
|
||||
/**
|
||||
* Gets remaining TTL for an entry
|
||||
*/
|
||||
getRemainingTTL(entry: TTLCacheEntry, now: number = Date.now()): number {
|
||||
return Math.max(0, entry.expires - now);
|
||||
},
|
||||
|
||||
/**
|
||||
* Safely gets cache entry, returning null if expired
|
||||
*/
|
||||
safeGet<T>(cache: Map<string, TTLCacheEntry>, key: string): T | null {
|
||||
const entry = cache.get(key);
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (this.isExpired(entry)) {
|
||||
cache.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.data as T;
|
||||
}
|
||||
};
|
||||
|
|
@ -1,279 +0,0 @@
|
|||
import { BotVerificationEngine, type BotVerificationResult } from './bot-verification.js';
|
||||
import { type NetworkRequest } from './network.js';
|
||||
import * as logs from './logs.js';
|
||||
|
||||
// ==================== TYPE DEFINITIONS ====================
|
||||
|
||||
export interface EnhancedBotAnalysis {
|
||||
readonly isVerifiedBot: boolean;
|
||||
readonly verification: BotVerificationResult;
|
||||
readonly riskAdjustment: number; // Negative for reduced risk, positive for increased
|
||||
readonly trustLevel: 'none' | 'low' | 'medium' | 'high' | 'verified';
|
||||
}
|
||||
|
||||
export interface EnhancedBotScoringConfig {
|
||||
readonly enabled: boolean;
|
||||
readonly weights: {
|
||||
readonly baseVerificationWeight: number;
|
||||
readonly ipRangeWeight: number;
|
||||
readonly dnsWeight: number;
|
||||
readonly combinedWeight: number;
|
||||
readonly majorSearchEngineWeight: number;
|
||||
};
|
||||
readonly thresholds: {
|
||||
readonly verifiedLevel: number;
|
||||
readonly highLevel: number;
|
||||
readonly mediumLevel: number;
|
||||
readonly lowLevel: number;
|
||||
};
|
||||
readonly maxRiskReduction: number;
|
||||
}
|
||||
|
||||
// ==================== ENHANCED BOT SCORING ====================
|
||||
|
||||
export class EnhancedBotScorer {
|
||||
private readonly botEngine: BotVerificationEngine;
|
||||
private readonly config: EnhancedBotScoringConfig;
|
||||
|
||||
constructor(botEngine: BotVerificationEngine, config: EnhancedBotScoringConfig) {
|
||||
this.botEngine = botEngine;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs enhanced bot verification and calculates appropriate risk adjustments
|
||||
* This can be used in conjunction with or instead of the basic user-agent checking
|
||||
*/
|
||||
async performEnhancedBotAnalysis(
|
||||
request: NetworkRequest,
|
||||
userAgent?: string
|
||||
): Promise<EnhancedBotAnalysis> {
|
||||
try {
|
||||
if (!this.config.enabled) {
|
||||
return this.createNegativeAnalysis();
|
||||
}
|
||||
|
||||
// Perform comprehensive bot verification
|
||||
const verification = await this.botEngine.verifyBot(request, userAgent);
|
||||
|
||||
// Calculate risk adjustment based on verification results
|
||||
const riskAdjustment = this.calculateRiskAdjustment(verification);
|
||||
|
||||
// Determine trust level
|
||||
const trustLevel = this.determineTrustLevel(verification);
|
||||
|
||||
// Log if it's a verified bot
|
||||
if (verification.isVerifiedBot) {
|
||||
logs.plugin('enhanced-bot',
|
||||
`Verified bot: ${verification.botName} (${verification.verificationMethod}, confidence: ${verification.confidence})`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
isVerifiedBot: verification.isVerifiedBot,
|
||||
verification,
|
||||
riskAdjustment,
|
||||
trustLevel,
|
||||
};
|
||||
} catch (error) {
|
||||
logs.error('enhanced-bot', `Enhanced bot analysis failed: ${error}`);
|
||||
return this.createNegativeAnalysis();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates risk score adjustment based on bot verification results
|
||||
*/
|
||||
private calculateRiskAdjustment(verification: BotVerificationResult): number {
|
||||
if (!verification.isVerifiedBot) {
|
||||
return 0; // No adjustment for unverified requests
|
||||
}
|
||||
|
||||
let adjustment = 0;
|
||||
|
||||
// Base adjustment for verified bot
|
||||
adjustment -= this.config.weights.baseVerificationWeight;
|
||||
|
||||
// Additional adjustments based on verification method and confidence
|
||||
switch (verification.verificationMethod) {
|
||||
case 'user_agent':
|
||||
// User agent only - minimal reduction
|
||||
adjustment -= this.config.weights.baseVerificationWeight * 0.3;
|
||||
break;
|
||||
case 'ip_range':
|
||||
// IP range verified - good reduction
|
||||
adjustment -= this.config.weights.ipRangeWeight;
|
||||
break;
|
||||
case 'dns_reverse':
|
||||
// DNS verified - excellent reduction
|
||||
adjustment -= this.config.weights.dnsWeight;
|
||||
break;
|
||||
case 'combined':
|
||||
// Multiple verification methods - maximum reduction
|
||||
adjustment -= this.config.weights.combinedWeight;
|
||||
break;
|
||||
}
|
||||
|
||||
// Scale by confidence
|
||||
adjustment = Math.floor(adjustment * verification.confidence);
|
||||
|
||||
// Known major search engines get additional trust
|
||||
if (verification.botName === 'googlebot' || verification.botName === 'bingbot') {
|
||||
adjustment -= this.config.weights.majorSearchEngineWeight;
|
||||
}
|
||||
|
||||
// Cap the maximum reduction to prevent abuse
|
||||
return Math.max(adjustment, -this.config.maxRiskReduction);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines trust level based on verification results
|
||||
*/
|
||||
private determineTrustLevel(verification: BotVerificationResult): EnhancedBotAnalysis['trustLevel'] {
|
||||
if (!verification.isVerifiedBot) {
|
||||
return 'none';
|
||||
}
|
||||
|
||||
const confidence = verification.confidence;
|
||||
const details = verification.details;
|
||||
|
||||
// Verified with high confidence and multiple methods
|
||||
if (confidence >= this.config.thresholds.verifiedLevel && (details.ipRangeMatch || details.dnsVerified)) {
|
||||
return 'verified';
|
||||
}
|
||||
|
||||
// Good verification with IP or DNS
|
||||
if (confidence >= this.config.thresholds.highLevel && (details.ipRangeMatch || details.dnsVerified)) {
|
||||
return 'high';
|
||||
}
|
||||
|
||||
// Decent verification
|
||||
if (confidence >= this.config.thresholds.mediumLevel) {
|
||||
return 'medium';
|
||||
}
|
||||
|
||||
// Basic verification (user agent only)
|
||||
if (confidence >= this.config.thresholds.lowLevel) {
|
||||
return 'low';
|
||||
}
|
||||
|
||||
return 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a negative analysis result
|
||||
*/
|
||||
private createNegativeAnalysis(): EnhancedBotAnalysis {
|
||||
return {
|
||||
isVerifiedBot: false,
|
||||
verification: {
|
||||
isVerifiedBot: false,
|
||||
botName: null,
|
||||
verificationMethod: 'user_agent',
|
||||
confidence: 0,
|
||||
details: {
|
||||
userAgentMatch: false,
|
||||
ipRangeMatch: false,
|
||||
dnsVerified: false,
|
||||
},
|
||||
},
|
||||
riskAdjustment: 0,
|
||||
trustLevel: 'none',
|
||||
};
|
||||
}
|
||||
|
||||
// ==================== INTEGRATION HELPERS ====================
|
||||
|
||||
/**
|
||||
* Helper function to integrate enhanced bot analysis into existing threat scoring
|
||||
* Returns the risk adjustment that should be applied to the base threat score
|
||||
*/
|
||||
async getBotRiskAdjustment(
|
||||
request: NetworkRequest,
|
||||
userAgent?: string
|
||||
): Promise<number> {
|
||||
const analysis = await this.performEnhancedBotAnalysis(request, userAgent);
|
||||
return analysis.riskAdjustment;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to check if a request is from a verified bot
|
||||
*/
|
||||
async isVerifiedBot(
|
||||
request: NetworkRequest,
|
||||
userAgent?: string
|
||||
): Promise<boolean> {
|
||||
const analysis = await this.performEnhancedBotAnalysis(request, userAgent);
|
||||
return analysis.isVerifiedBot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get bot information for logging/headers
|
||||
*/
|
||||
async getBotInfo(
|
||||
request: NetworkRequest,
|
||||
userAgent?: string
|
||||
): Promise<{ name: string | null; verified: boolean; method: string }> {
|
||||
const analysis = await this.performEnhancedBotAnalysis(request, userAgent);
|
||||
return {
|
||||
name: analysis.verification.botName,
|
||||
verified: analysis.isVerifiedBot,
|
||||
method: analysis.verification.verificationMethod,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== CONVENIENCE FUNCTIONS ====================
|
||||
// Note: These require configured instances - no singletons
|
||||
|
||||
/**
|
||||
* Creates an enhanced bot scorer with the provided configuration
|
||||
*/
|
||||
export function createEnhancedBotScorer(
|
||||
botEngine: BotVerificationEngine,
|
||||
config: EnhancedBotScoringConfig
|
||||
): EnhancedBotScorer {
|
||||
return new EnhancedBotScorer(botEngine, config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Default enhanced bot scorer for convenience (requires configuration)
|
||||
*/
|
||||
let defaultEnhancedScorer: EnhancedBotScorer | null = null;
|
||||
|
||||
export function configureDefaultEnhancedBotScorer(
|
||||
botEngine: BotVerificationEngine,
|
||||
config: EnhancedBotScoringConfig
|
||||
): void {
|
||||
defaultEnhancedScorer = new EnhancedBotScorer(botEngine, config);
|
||||
}
|
||||
|
||||
export const enhancedBotScoring = {
|
||||
performEnhancedBotAnalysis: async (request: NetworkRequest, userAgent?: string): Promise<EnhancedBotAnalysis> => {
|
||||
if (!defaultEnhancedScorer) {
|
||||
throw new Error('Default enhanced bot scorer not configured. Call configureDefaultEnhancedBotScorer() first.');
|
||||
}
|
||||
return defaultEnhancedScorer.performEnhancedBotAnalysis(request, userAgent);
|
||||
},
|
||||
|
||||
getBotRiskAdjustment: async (request: NetworkRequest, userAgent?: string): Promise<number> => {
|
||||
if (!defaultEnhancedScorer) {
|
||||
throw new Error('Default enhanced bot scorer not configured. Call configureDefaultEnhancedBotScorer() first.');
|
||||
}
|
||||
return defaultEnhancedScorer.getBotRiskAdjustment(request, userAgent);
|
||||
},
|
||||
|
||||
isVerifiedBot: async (request: NetworkRequest, userAgent?: string): Promise<boolean> => {
|
||||
if (!defaultEnhancedScorer) {
|
||||
throw new Error('Default enhanced bot scorer not configured. Call configureDefaultEnhancedBotScorer() first.');
|
||||
}
|
||||
return defaultEnhancedScorer.isVerifiedBot(request, userAgent);
|
||||
},
|
||||
|
||||
getBotInfo: async (request: NetworkRequest, userAgent?: string): Promise<{ name: string | null; verified: boolean; method: string }> => {
|
||||
if (!defaultEnhancedScorer) {
|
||||
throw new Error('Default enhanced bot scorer not configured. Call configureDefaultEnhancedBotScorer() first.');
|
||||
}
|
||||
return defaultEnhancedScorer.getBotInfo(request, userAgent);
|
||||
}
|
||||
};
|
||||
|
|
@ -1,227 +0,0 @@
|
|||
// =============================================================================
|
||||
// CENTRALIZED IP VALIDATION UTILITY
|
||||
// =============================================================================
|
||||
// Consolidates all IP validation logic to prevent security inconsistencies
|
||||
|
||||
// Security constants
|
||||
const MAX_IP_LENGTH = 45; // Max IPv6 length
|
||||
const MIN_IP_LENGTH = 7; // Min IPv4 length (0.0.0.0)
|
||||
|
||||
// Comprehensive IP patterns (ReDoS-safe)
|
||||
const IP_PATTERNS = {
|
||||
// IPv4 pattern (strict)
|
||||
IPV4: /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/,
|
||||
// IPv6 pattern (simplified but secure)
|
||||
IPV6_FULL: /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$/,
|
||||
IPV6_LOOPBACK: /^::1$/,
|
||||
IPV6_ANY: /^::$/,
|
||||
// IPv6 compressed forms
|
||||
IPV6_COMPRESSED: /^[0-9a-fA-F:]+::?[0-9a-fA-F:]*$/,
|
||||
// IPv4-mapped IPv6
|
||||
IPV6_MAPPED: /^::ffff:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/
|
||||
} as const;
|
||||
|
||||
// Security patterns to detect injection attempts
|
||||
const DANGEROUS_PATTERNS = [
|
||||
/[<>\"'`]/, // HTML/JS injection
|
||||
/[;|&$]/, // Command injection
|
||||
/\.\./, // Path traversal
|
||||
/\/\*/, // SQL comment
|
||||
/--/, // SQL comment
|
||||
/[\x00-\x1f\x7f-\x9f]/, // Control characters
|
||||
] as const;
|
||||
|
||||
export interface IPValidationResult {
|
||||
readonly valid: boolean;
|
||||
readonly ip?: string;
|
||||
readonly type?: 'ipv4' | 'ipv6' | 'ipv6-mapped';
|
||||
readonly error?: string;
|
||||
}
|
||||
|
||||
export interface IPValidationOptions {
|
||||
readonly allowEmpty?: boolean;
|
||||
readonly allowMapped?: boolean; // Allow IPv4-mapped IPv6
|
||||
readonly strict?: boolean; // Strict validation (no special cases)
|
||||
}
|
||||
|
||||
/**
|
||||
* Comprehensive IP address validation with security checks
|
||||
* @param ip - The IP address to validate
|
||||
* @param options - Validation options
|
||||
* @returns Validation result with type information
|
||||
*/
|
||||
export function validateIPAddress(ip: unknown, options: IPValidationOptions = {}): IPValidationResult {
|
||||
// Type check
|
||||
if (typeof ip !== 'string') {
|
||||
return { valid: false, error: 'IP address must be a string' };
|
||||
}
|
||||
|
||||
// Handle empty input
|
||||
if (ip.length === 0) {
|
||||
if (options.allowEmpty) {
|
||||
return { valid: true, ip: '' };
|
||||
}
|
||||
return { valid: false, error: 'IP address cannot be empty' };
|
||||
}
|
||||
|
||||
// Length validation
|
||||
if (ip.length < MIN_IP_LENGTH || ip.length > MAX_IP_LENGTH) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `IP address length must be between ${MIN_IP_LENGTH} and ${MAX_IP_LENGTH} characters`
|
||||
};
|
||||
}
|
||||
|
||||
// Clean input
|
||||
const cleanIP = ip.trim();
|
||||
|
||||
// Security injection checks
|
||||
for (const pattern of DANGEROUS_PATTERNS) {
|
||||
if (pattern.test(cleanIP)) {
|
||||
return { valid: false, error: 'IP address contains dangerous characters' };
|
||||
}
|
||||
}
|
||||
|
||||
// Additional malformed checks
|
||||
if (cleanIP.includes('..') || cleanIP.includes(':::')) {
|
||||
return { valid: false, error: 'Malformed IP address' };
|
||||
}
|
||||
|
||||
// IPv4 validation
|
||||
if (cleanIP.includes('.')) {
|
||||
if (IP_PATTERNS.IPV4.test(cleanIP)) {
|
||||
return { valid: true, ip: cleanIP, type: 'ipv4' };
|
||||
}
|
||||
return { valid: false, error: 'Invalid IPv4 address format' };
|
||||
}
|
||||
|
||||
// IPv6 validation
|
||||
if (cleanIP.includes(':')) {
|
||||
// Check for IPv4-mapped IPv6 first
|
||||
if (IP_PATTERNS.IPV6_MAPPED.test(cleanIP)) {
|
||||
if (options.allowMapped !== false) {
|
||||
return { valid: true, ip: cleanIP, type: 'ipv6-mapped' };
|
||||
}
|
||||
return { valid: false, error: 'IPv4-mapped IPv6 not allowed' };
|
||||
}
|
||||
|
||||
// Standard IPv6 patterns
|
||||
if (IP_PATTERNS.IPV6_FULL.test(cleanIP) ||
|
||||
IP_PATTERNS.IPV6_LOOPBACK.test(cleanIP) ||
|
||||
IP_PATTERNS.IPV6_ANY.test(cleanIP)) {
|
||||
return { valid: true, ip: cleanIP, type: 'ipv6' };
|
||||
}
|
||||
|
||||
// Compressed IPv6 (more permissive)
|
||||
if (!options.strict && IP_PATTERNS.IPV6_COMPRESSED.test(cleanIP)) {
|
||||
return { valid: true, ip: cleanIP, type: 'ipv6' };
|
||||
}
|
||||
|
||||
return { valid: false, error: 'Invalid IPv6 address format' };
|
||||
}
|
||||
|
||||
return { valid: false, error: 'Invalid IP address format' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates and returns IP address, throwing on invalid input
|
||||
* @param ip - The IP address to validate
|
||||
* @param options - Validation options
|
||||
* @returns The validated IP address
|
||||
* @throws Error if IP is invalid
|
||||
*/
|
||||
export function requireValidIP(ip: unknown, options: IPValidationOptions = {}): string {
|
||||
const result = validateIPAddress(ip, options);
|
||||
if (!result.valid) {
|
||||
throw new Error(result.error || 'Invalid IP address');
|
||||
}
|
||||
// TypeScript assertion: when valid is true, ip is always defined
|
||||
return result.ip as string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if input is a valid IP address (boolean check)
|
||||
* @param ip - The IP address to check
|
||||
* @param options - Validation options
|
||||
* @returns True if valid IP address
|
||||
*/
|
||||
export function isValidIP(ip: unknown, options: IPValidationOptions = {}): boolean {
|
||||
return validateIPAddress(ip, options).valid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets IP address type
|
||||
* @param ip - The IP address to analyze
|
||||
* @returns IP type or null if invalid
|
||||
*/
|
||||
export function getIPType(ip: unknown): 'ipv4' | 'ipv6' | 'ipv6-mapped' | null {
|
||||
const result = validateIPAddress(ip);
|
||||
// TypeScript assertion: when valid is true, type is always defined
|
||||
return result.valid ? (result.type as 'ipv4' | 'ipv6' | 'ipv6-mapped') : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates CIDR notation
|
||||
* @param cidr - CIDR string to validate
|
||||
* @returns Validation result with prefix information
|
||||
*/
|
||||
export function validateCIDR(cidr: unknown): { valid: boolean; ip?: string; prefix?: number; type?: string; error?: string } {
|
||||
if (typeof cidr !== 'string') {
|
||||
return { valid: false, error: 'CIDR must be a string' };
|
||||
}
|
||||
|
||||
const trimmed = cidr.trim();
|
||||
const parts = trimmed.split('/');
|
||||
|
||||
if (parts.length !== 2) {
|
||||
return { valid: false, error: 'CIDR must contain exactly one "/" character' };
|
||||
}
|
||||
|
||||
const [ip, prefixStr] = parts;
|
||||
|
||||
// Validate IP part
|
||||
const ipResult = validateIPAddress(ip);
|
||||
if (!ipResult.valid) {
|
||||
return { valid: false, error: `Invalid IP in CIDR: ${ipResult.error}` };
|
||||
}
|
||||
|
||||
// Validate prefix part - ensure prefixStr is defined
|
||||
if (!prefixStr) {
|
||||
return { valid: false, error: 'CIDR prefix is missing' };
|
||||
}
|
||||
|
||||
const prefix = parseInt(prefixStr, 10);
|
||||
if (isNaN(prefix)) {
|
||||
return { valid: false, error: 'CIDR prefix must be a number' };
|
||||
}
|
||||
|
||||
// Check prefix bounds based on IP type
|
||||
const ipType = ipResult.type as 'ipv4' | 'ipv6' | 'ipv6-mapped';
|
||||
const maxPrefix = ipType === 'ipv4' ? 32 : 128;
|
||||
if (prefix < 0 || prefix > maxPrefix) {
|
||||
return { valid: false, error: `CIDR prefix must be between 0 and ${maxPrefix} for ${ipType}` };
|
||||
}
|
||||
|
||||
return {
|
||||
valid: true,
|
||||
// TypeScript assertions: when valid is true, these are always defined
|
||||
ip: ipResult.ip as string,
|
||||
prefix,
|
||||
type: ipType
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts single IP to CIDR notation
|
||||
* @param ip - IP address to convert
|
||||
* @returns CIDR string or null if invalid
|
||||
*/
|
||||
export function ipToCIDR(ip: unknown): string | null {
|
||||
const result = validateIPAddress(ip);
|
||||
if (!result.valid || !result.ip || !result.type) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prefix = result.type === 'ipv4' ? 32 : 128;
|
||||
return `${result.ip}/${prefix}`;
|
||||
}
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
// Logging categories for type safety
|
||||
export type LogCategory =
|
||||
| 'checkpoint'
|
||||
| 'waf'
|
||||
| 'ipfilter'
|
||||
| 'proxy'
|
||||
| 'behavioral'
|
||||
| 'threat-scoring'
|
||||
| 'network'
|
||||
| 'server'
|
||||
| 'config'
|
||||
| 'db'
|
||||
| 'plugin'
|
||||
| 'performance'
|
||||
| string; // Allow custom categories
|
||||
|
||||
// Type for async operations
|
||||
export type AsyncOperation<T> = () => Promise<T>;
|
||||
export type SyncOperation<T> = () => T;
|
||||
|
||||
// Type for errors with message property
|
||||
interface ErrorWithMessage {
|
||||
message: string;
|
||||
}
|
||||
|
||||
// Track seen configs to avoid duplicate logs
|
||||
const seenConfigs = new Set<string>();
|
||||
|
||||
export function init(msg: string): void {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
// Utility function to handle common async operations with consistent error logging
|
||||
export async function safeAsync<T>(
|
||||
operation: AsyncOperation<T>,
|
||||
context: LogCategory,
|
||||
errorMessage: string,
|
||||
fallback: T | null = null
|
||||
): Promise<T | null> {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
error(context, `${errorMessage}: ${errMsg}`);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
// Utility function for synchronous operations with error handling
|
||||
export function safeSync<T>(
|
||||
operation: SyncOperation<T>,
|
||||
context: LogCategory,
|
||||
errorMessage: string,
|
||||
fallback: T | null = null
|
||||
): T | null {
|
||||
try {
|
||||
return operation();
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
error(context, `${errorMessage}: ${errMsg}`);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
export function plugin(_name: string, msg: string): void {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function config(name: string, msg: string): void {
|
||||
if (!seenConfigs.has(name)) {
|
||||
console.log(`Config ${msg} for ${name}`);
|
||||
seenConfigs.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
export function db(msg: string): void {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function server(msg: string): void {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function section(title: string): void {
|
||||
console.log(`\n=== ${title.toUpperCase()} ===`);
|
||||
}
|
||||
|
||||
export function warn(_category: LogCategory, msg: string): void {
|
||||
console.warn(`WARNING: ${msg}`);
|
||||
}
|
||||
|
||||
export function error(_category: LogCategory, msg: string): void {
|
||||
console.error(`ERROR: ${msg}`);
|
||||
}
|
||||
|
||||
// General message function for bullet items
|
||||
export function msg(message: string): void {
|
||||
console.log(message);
|
||||
}
|
||||
|
||||
// Re-export common types for convenience
|
||||
export type { ErrorWithMessage };
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
import * as logs from './logs.js';
|
||||
|
||||
// Type definitions for different request styles
|
||||
interface ExpressHeaders {
|
||||
[key: string]: string | string[] | undefined;
|
||||
'x-forwarded-for'?: string;
|
||||
'x-real-ip'?: string;
|
||||
'x-forwarded-proto'?: string;
|
||||
host?: string;
|
||||
'x-forwarded-host'?: string;
|
||||
}
|
||||
|
||||
interface FetchHeaders {
|
||||
get(name: string): string | null;
|
||||
}
|
||||
|
||||
interface ExpressConnection {
|
||||
remoteAddress?: string;
|
||||
}
|
||||
|
||||
interface ExpressSocket {
|
||||
remoteAddress?: string;
|
||||
}
|
||||
|
||||
interface ExpressRequest {
|
||||
url?: string;
|
||||
secure?: boolean;
|
||||
headers: ExpressHeaders;
|
||||
connection?: ExpressConnection;
|
||||
socket?: ExpressSocket;
|
||||
ip?: string;
|
||||
}
|
||||
|
||||
interface FetchRequest {
|
||||
url?: string;
|
||||
headers: FetchHeaders;
|
||||
}
|
||||
|
||||
interface ServerInfo {
|
||||
remoteAddress?: string;
|
||||
}
|
||||
|
||||
// Union type for both request styles
|
||||
export type NetworkRequest = ExpressRequest | FetchRequest;
|
||||
|
||||
// Type guard to check if headers support get() method
|
||||
function isFetchHeaders(headers: ExpressHeaders | FetchHeaders): headers is FetchHeaders {
|
||||
return typeof (headers as FetchHeaders).get === 'function';
|
||||
}
|
||||
|
||||
// Helper function to safely create URL objects from Express requests
|
||||
export function getRequestURL(request: NetworkRequest): URL | null {
|
||||
if (!request.url) return null;
|
||||
|
||||
// If it's already a complete URL, use it as is
|
||||
if (request.url.startsWith('http://') || request.url.startsWith('https://')) {
|
||||
return new URL(request.url);
|
||||
}
|
||||
|
||||
// For Express requests, construct a complete URL
|
||||
const expressReq = request as ExpressRequest;
|
||||
const protocol = expressReq.secure || expressReq.headers['x-forwarded-proto'] === 'https' ? 'https:' : 'http:';
|
||||
|
||||
let host: string;
|
||||
if (isFetchHeaders(request.headers)) {
|
||||
host = request.headers.get('host') || request.headers.get('x-forwarded-host') || 'localhost';
|
||||
} else {
|
||||
const headers = request.headers as ExpressHeaders;
|
||||
host = headers.host || headers['x-forwarded-host'] || 'localhost';
|
||||
// Handle array values
|
||||
if (Array.isArray(host)) {
|
||||
host = host[0] || 'localhost';
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return new URL(request.url, `${protocol}//${host}`);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
logs.warn('network', `Failed to parse URL ${request.url}: ${errorMessage}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getRealIP(request: NetworkRequest, server?: ServerInfo): string {
|
||||
// Handle both Express req.headers and fetch-style request.headers.get()
|
||||
let ip: string | undefined;
|
||||
|
||||
if (isFetchHeaders(request.headers)) {
|
||||
// Fetch-style Request object
|
||||
ip = request.headers.get('x-forwarded-for') || request.headers.get('x-real-ip') || undefined;
|
||||
} else {
|
||||
// Express request object
|
||||
const headers = request.headers as ExpressHeaders;
|
||||
const forwardedFor = headers['x-forwarded-for'];
|
||||
const realIp = headers['x-real-ip'];
|
||||
|
||||
// Handle both string and array values
|
||||
if (forwardedFor) {
|
||||
ip = Array.isArray(forwardedFor) ? forwardedFor[0] : forwardedFor;
|
||||
} else if (realIp) {
|
||||
ip = Array.isArray(realIp) ? realIp[0] : realIp;
|
||||
}
|
||||
}
|
||||
|
||||
if (ip?.includes(',')) {
|
||||
ip = ip.split(',')[0]?.trim();
|
||||
}
|
||||
|
||||
if (!ip && server) {
|
||||
ip = server.remoteAddress;
|
||||
}
|
||||
|
||||
const expressReq = request as ExpressRequest;
|
||||
if (!ip && expressReq.connection) {
|
||||
// Express-style connection
|
||||
ip = expressReq.connection.remoteAddress || expressReq.socket?.remoteAddress;
|
||||
}
|
||||
|
||||
if (!ip && expressReq.ip) {
|
||||
// Express provides req.ip
|
||||
ip = expressReq.ip;
|
||||
}
|
||||
|
||||
if (!ip) {
|
||||
// Fallback to URL hostname
|
||||
const url = getRequestURL(request);
|
||||
ip = url?.hostname || '127.0.0.1';
|
||||
}
|
||||
|
||||
// Clean IPv6 mapped IPv4 addresses
|
||||
if (ip?.startsWith('::ffff:')) {
|
||||
ip = ip.slice(7);
|
||||
}
|
||||
|
||||
return ip;
|
||||
}
|
||||
|
||||
// Export types for use in other modules
|
||||
export type { ExpressRequest, FetchRequest, ExpressHeaders, FetchHeaders, ServerInfo };
|
||||
|
|
@ -1,449 +0,0 @@
|
|||
// =============================================================================
|
||||
// CENTRALIZED PATTERN MATCHING UTILITY
|
||||
// =============================================================================
|
||||
// Consolidates all pattern matching logic to prevent duplication
|
||||
|
||||
// @ts-ignore - string-dsa doesn't have TypeScript definitions
|
||||
import { AhoCorasick } from 'string-dsa';
|
||||
import * as logs from './logs.js';
|
||||
|
||||
export interface PatternMatcher {
|
||||
find(text: string): string[];
|
||||
}
|
||||
|
||||
export interface PatternMatchResult {
|
||||
readonly matched: boolean;
|
||||
readonly matches: readonly string[];
|
||||
readonly matchCount: number;
|
||||
}
|
||||
|
||||
export interface RegexMatchResult {
|
||||
readonly matched: boolean;
|
||||
readonly pattern?: string;
|
||||
readonly match?: string;
|
||||
}
|
||||
|
||||
export interface PatternCollection {
|
||||
readonly name: string;
|
||||
readonly patterns: readonly string[];
|
||||
readonly description?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Centralized Aho-Corasick pattern matcher
|
||||
*/
|
||||
export class AhoCorasickPatternMatcher {
|
||||
private matcher: PatternMatcher | null = null;
|
||||
private readonly patterns: readonly string[];
|
||||
private readonly name: string;
|
||||
|
||||
constructor(name: string, patterns: readonly string[]) {
|
||||
this.name = name;
|
||||
this.patterns = patterns;
|
||||
this.initialize();
|
||||
}
|
||||
|
||||
private initialize(): void {
|
||||
try {
|
||||
if (this.patterns.length === 0) {
|
||||
logs.warn('pattern-matching', `No patterns provided for matcher ${this.name}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.patterns.length > 10000) {
|
||||
logs.warn('pattern-matching', `Too many patterns for ${this.name}: ${this.patterns.length}, truncating to 10000`);
|
||||
this.matcher = new AhoCorasick(this.patterns.slice(0, 10000)) as PatternMatcher;
|
||||
} else {
|
||||
this.matcher = new AhoCorasick(this.patterns) as PatternMatcher;
|
||||
}
|
||||
|
||||
logs.plugin('pattern-matching', `Initialized ${this.name} matcher with ${this.patterns.length} patterns`);
|
||||
} catch (error) {
|
||||
logs.error('pattern-matching', `Failed to initialize ${this.name} matcher: ${error}`);
|
||||
this.matcher = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds pattern matches in text
|
||||
*/
|
||||
find(text: string): PatternMatchResult {
|
||||
if (!this.matcher || !text) {
|
||||
return { matched: false, matches: [], matchCount: 0 };
|
||||
}
|
||||
|
||||
try {
|
||||
const matches = this.matcher.find(text.toLowerCase());
|
||||
return {
|
||||
matched: matches.length > 0,
|
||||
matches: matches.slice(0, 100), // Limit matches to prevent memory issues
|
||||
matchCount: matches.length
|
||||
};
|
||||
} catch (error) {
|
||||
logs.warn('pattern-matching', `Pattern matching failed for ${this.name}: ${error}`);
|
||||
return { matched: false, matches: [], matchCount: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if text contains any patterns
|
||||
*/
|
||||
hasMatch(text: string): boolean {
|
||||
return this.find(text).matched;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets first match found
|
||||
*/
|
||||
getFirstMatch(text: string): string | null {
|
||||
const result = this.find(text);
|
||||
return result.matches.length > 0 ? (result.matches[0] || null) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reinitializes the matcher (useful for pattern updates)
|
||||
*/
|
||||
reinitialize(): void {
|
||||
this.initialize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets pattern count
|
||||
*/
|
||||
getPatternCount(): number {
|
||||
return this.patterns.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if matcher is ready
|
||||
*/
|
||||
isReady(): boolean {
|
||||
return this.matcher !== null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Centralized regex pattern matcher
|
||||
*/
|
||||
export class RegexPatternMatcher {
|
||||
private readonly patterns: Map<string, RegExp> = new Map();
|
||||
private readonly name: string;
|
||||
|
||||
constructor(name: string, patterns: Record<string, string> = {}) {
|
||||
this.name = name;
|
||||
this.compilePatterns(patterns);
|
||||
}
|
||||
|
||||
private compilePatterns(patterns: Record<string, string>): void {
|
||||
let compiled = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const [name, pattern] of Object.entries(patterns)) {
|
||||
try {
|
||||
// Validate pattern length to prevent ReDoS
|
||||
if (pattern.length > 500) {
|
||||
logs.warn('pattern-matching', `Pattern ${name} too long: ${pattern.length} chars, skipping`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
this.patterns.set(name, new RegExp(pattern, 'i'));
|
||||
compiled++;
|
||||
} catch (error) {
|
||||
logs.error('pattern-matching', `Failed to compile regex pattern ${name}: ${error}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
logs.plugin('pattern-matching', `${this.name}: compiled ${compiled} patterns, ${failed} failed`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests text against a specific pattern
|
||||
*/
|
||||
test(patternName: string, text: string): RegexMatchResult {
|
||||
const pattern = this.patterns.get(patternName);
|
||||
if (!pattern) {
|
||||
return { matched: false };
|
||||
}
|
||||
|
||||
try {
|
||||
const match = pattern.exec(text);
|
||||
return {
|
||||
matched: match !== null,
|
||||
pattern: patternName,
|
||||
match: match ? match[0] : undefined
|
||||
};
|
||||
} catch (error) {
|
||||
logs.warn('pattern-matching', `Regex test failed for ${patternName}: ${error}`);
|
||||
return { matched: false };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests text against all patterns
|
||||
*/
|
||||
testAll(text: string): RegexMatchResult[] {
|
||||
const results: RegexMatchResult[] = [];
|
||||
|
||||
for (const patternName of this.patterns.keys()) {
|
||||
const result = this.test(patternName, text);
|
||||
if (result.matched) {
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if any pattern matches
|
||||
*/
|
||||
hasAnyMatch(text: string): boolean {
|
||||
for (const pattern of this.patterns.values()) {
|
||||
try {
|
||||
if (pattern.test(text)) {
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
// Continue with other patterns
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new pattern
|
||||
*/
|
||||
addPattern(name: string, pattern: string): boolean {
|
||||
try {
|
||||
if (pattern.length > 500) {
|
||||
logs.warn('pattern-matching', `Pattern ${name} too long, rejecting`);
|
||||
return false;
|
||||
}
|
||||
|
||||
this.patterns.set(name, new RegExp(pattern, 'i'));
|
||||
return true;
|
||||
} catch (error) {
|
||||
logs.error('pattern-matching', `Failed to add pattern ${name}: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a pattern
|
||||
*/
|
||||
removePattern(name: string): boolean {
|
||||
return this.patterns.delete(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets pattern count
|
||||
*/
|
||||
getPatternCount(): number {
|
||||
return this.patterns.size;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pattern matcher factory for common use cases
|
||||
*/
|
||||
export class PatternMatcherFactory {
|
||||
private static ahoCorasickMatchers: Map<string, AhoCorasickPatternMatcher> = new Map();
|
||||
private static regexMatchers: Map<string, RegexPatternMatcher> = new Map();
|
||||
|
||||
/**
|
||||
* Creates or gets an Aho-Corasick matcher
|
||||
*/
|
||||
static getAhoCorasickMatcher(name: string, patterns: readonly string[]): AhoCorasickPatternMatcher {
|
||||
if (!this.ahoCorasickMatchers.has(name)) {
|
||||
this.ahoCorasickMatchers.set(name, new AhoCorasickPatternMatcher(name, patterns));
|
||||
}
|
||||
return this.ahoCorasickMatchers.get(name)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates or gets a regex matcher
|
||||
*/
|
||||
static getRegexMatcher(name: string, patterns: Record<string, string> = {}): RegexPatternMatcher {
|
||||
if (!this.regexMatchers.has(name)) {
|
||||
this.regexMatchers.set(name, new RegexPatternMatcher(name, patterns));
|
||||
}
|
||||
return this.regexMatchers.get(name)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a matcher
|
||||
*/
|
||||
static removeMatcher(name: string): void {
|
||||
this.ahoCorasickMatchers.delete(name);
|
||||
this.regexMatchers.delete(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all matchers
|
||||
*/
|
||||
static clearAll(): void {
|
||||
this.ahoCorasickMatchers.clear();
|
||||
this.regexMatchers.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all matcher names
|
||||
*/
|
||||
static getMatcherNames(): { ahoCorasick: string[]; regex: string[] } {
|
||||
return {
|
||||
ahoCorasick: Array.from(this.ahoCorasickMatchers.keys()),
|
||||
regex: Array.from(this.regexMatchers.keys())
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Common pattern collections for reuse
|
||||
*/
|
||||
export const CommonPatterns = {
|
||||
// Attack tool patterns
|
||||
ATTACK_TOOLS: [
|
||||
'sqlmap', 'nikto', 'nmap', 'burpsuite', 'w3af', 'acunetix',
|
||||
'nessus', 'openvas', 'gobuster', 'dirbuster', 'wfuzz', 'ffuf',
|
||||
'hydra', 'medusa', 'masscan', 'zmap', 'metasploit', 'burp suite',
|
||||
'scanner', 'exploit', 'payload', 'injection', 'vulnerability'
|
||||
],
|
||||
|
||||
// Suspicious bot patterns
|
||||
SUSPICIOUS_BOTS: [
|
||||
'bot', 'crawler', 'spider', 'scraper', 'scanner', 'harvest',
|
||||
'extract', 'collect', 'gather', 'fetch'
|
||||
],
|
||||
|
||||
// SQL injection patterns
|
||||
SQL_INJECTION: [
|
||||
'union select', 'insert into', 'delete from', 'drop table', 'select * from',
|
||||
"' or '1'='1", "' or 1=1", "admin'--", "' union select", "'; drop table",
|
||||
'union all select', 'group_concat', 'version()', 'database()', 'user()',
|
||||
'information_schema', 'pg_sleep', 'waitfor delay', 'benchmark(',
|
||||
'extractvalue', 'updatexml', 'load_file', 'into outfile',
|
||||
// More aggressive patterns
|
||||
'exec sp_', 'exec xp_', 'execute immediate', 'dbms_',
|
||||
'; shutdown', '; exec', '; execute', '; xp_cmdshell', '; sp_',
|
||||
'cast(', 'convert(', 'concat(', 'substring(', 'ascii(', 'char(',
|
||||
'hex(', 'unhex(', 'md5(', 'sha1(', 'sha2(', 'encode(', 'decode(',
|
||||
'compress(', 'uncompress(', 'aes_encrypt(', 'aes_decrypt(', 'des_encrypt(',
|
||||
'sleep(', 'benchmark(', 'pg_sleep(', 'waitfor delay', 'dbms_lock.sleep',
|
||||
'randomblob(', 'load_extension(', 'sql', 'mysql', 'mssql', 'oracle',
|
||||
'sqlite_', 'pragma ', 'attach database', 'create table', 'alter table',
|
||||
'update set', 'bulk insert', 'openrowset', 'opendatasource', 'openquery',
|
||||
'xtype', 'sysobjects', 'syscolumns', 'sysusers', 'systables',
|
||||
'all_tables', 'user_tables', 'user_tab_columns', 'table_schema',
|
||||
'column_name', 'table_name', 'schema_name', 'database_name',
|
||||
'@@version', '@@datadir', '@@hostname', '@@basedir', 'session_user',
|
||||
'current_user', 'system_user', 'user_name()', 'suser_name()',
|
||||
'is_srvrolemember', 'is_member', 'has_dbaccess', 'has_perms_by_name'
|
||||
],
|
||||
|
||||
// XSS patterns
|
||||
XSS: [
|
||||
'<script>', '</script>', 'javascript:', 'document.cookie', 'document.write',
|
||||
'alert(', 'prompt(', 'confirm(', 'onload=', 'onerror=', 'onclick=',
|
||||
'<iframe', '<object', '<embed', '<svg', 'onmouseover=', 'onfocus=',
|
||||
'eval(', 'unescape(', 'fromcharcode(', 'expression(', 'vbscript:',
|
||||
// ... existing code ...
|
||||
// Add more aggressive XSS patterns
|
||||
'<script', 'script>', 'javascript:', 'data:text/html', 'data:application',
|
||||
'ondblclick=', 'onmouseenter=', 'onmouseleave=', 'onmousemove=', 'onkeydown=',
|
||||
'onkeypress=', 'onkeyup=', 'onsubmit=', 'onreset=', 'onblur=', 'onchange=',
|
||||
'onsearch=', 'onselect=', 'ontoggle=', 'ondrag=', 'ondrop=', 'oninput=',
|
||||
'oninvalid=', 'onpaste=', 'oncopy=', 'oncut=', 'onwheel=', 'ontouchstart=',
|
||||
'ontouchend=', 'ontouchmove=', 'onpointerdown=', 'onpointerup=', 'onpointermove=',
|
||||
'srcdoc=', '<applet', '<base', '<meta', '<link', 'import(', 'constructor.',
|
||||
'prototype.', '__proto__', 'contenteditable', 'designmode', 'javascript://',
|
||||
'vbs:', 'vbscript://', 'data:text/javascript', 'behavior:', 'mhtml:',
|
||||
'-moz-binding', 'xlink:href', 'autofocus', 'onfocusin=', 'onfocusout=',
|
||||
'onhashchange=', 'onmessage=', 'onoffline=', 'ononline=', 'onpagehide=',
|
||||
'onpageshow=', 'onpopstate=', 'onresize=', 'onstorage=', 'onunload=',
|
||||
'onbeforeunload=', 'onanimationstart=', 'onanimationend=', 'onanimationiteration=',
|
||||
'ontransitionend=', '<style', 'style=', '@import'
|
||||
],
|
||||
|
||||
// Command injection patterns
|
||||
COMMAND_INJECTION: [
|
||||
'rm -rf', 'wget http', 'curl http', '| nc', '| netcat', '| sh',
|
||||
'/bin/sh', '/bin/bash', 'cat /etc/passwd', '$(', '`', 'powershell',
|
||||
'cmd.exe', 'system(', 'exec(', 'shell_exec', 'passthru', 'popen',
|
||||
// More dangerous patterns
|
||||
'; ls', '; dir', '; cat', '; type', '; more', '; less', '; head', '; tail',
|
||||
'; ps', '; kill', '; pkill', '; killall', '; timeout', '; sleep',
|
||||
'; uname', '; id', '; whoami', '; groups', '; users', '; w', '; who',
|
||||
'; netstat', '; ss', '; ifconfig', '; ip addr', '; arp', '; route',
|
||||
'; ping', '; traceroute', '; nslookup', '; dig', '; host', '; whois',
|
||||
'; ssh', '; telnet', '; ftp', '; tftp', '; scp', '; rsync', '; rcp',
|
||||
'; chmod', '; chown', '; chgrp', '; umask', '; touch', '; mkdir',
|
||||
'; cp', '; mv', '; ln', '; dd', '; tar', '; zip', '; unzip', '; gzip',
|
||||
'; find', '; locate', '; grep', '; egrep', '; fgrep', '; sed', '; awk',
|
||||
'; perl', '; python', '; ruby', '; php', '; node', '; java', '; gcc',
|
||||
'; make', '; cmake', '; apt', '; yum', '; dnf', '; pacman', '; brew',
|
||||
'; systemctl', '; service', '; init', '; cron', '; at', '; batch',
|
||||
'; mount', '; umount', '; fdisk', '; parted', '; mkfs', '; fsck',
|
||||
'; iptables', '; firewall-cmd', '; ufw', '; fail2ban', '; tcpdump',
|
||||
'; nmap', '; masscan', '; zmap', '; nikto', '; sqlmap', '; metasploit',
|
||||
'& ', '&& ', '|| ', '| ', '; ', '\n', '\r\n', '%0a', '%0d',
|
||||
'eval ', 'assert ', 'preg_replace', 'create_function', 'include ',
|
||||
'require ', 'require_once ', 'include_once ', 'file_get_contents',
|
||||
'file_put_contents', 'fopen', 'fwrite', 'fputs', 'file', 'readfile',
|
||||
'highlight_file', 'show_source', 'proc_open', 'pcntl_exec',
|
||||
'dl(', 'expect ', 'popen(', 'proc_', 'shellexec', 'pcntl_',
|
||||
'posix_', 'getenv', 'putenv', 'setenv', 'mail(', 'mb_send_mail'
|
||||
],
|
||||
|
||||
// Path traversal patterns
|
||||
PATH_TRAVERSAL: [
|
||||
'../../../', '/etc/passwd', '/etc/shadow', '/windows/system32',
|
||||
'..\\..\\..\\', 'boot.ini', '..%2f', '%2e%2e%2f', '..%5c', '%2e%2e%5c'
|
||||
]
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Utility functions for pattern matching
|
||||
*/
|
||||
export const PatternUtils = {
|
||||
/**
|
||||
* Creates a pattern collection
|
||||
*/
|
||||
createCollection(name: string, patterns: readonly string[], description?: string): PatternCollection {
|
||||
return { name, patterns, description };
|
||||
},
|
||||
|
||||
/**
|
||||
* Merges multiple pattern collections
|
||||
*/
|
||||
mergeCollections(...collections: PatternCollection[]): PatternCollection {
|
||||
const allPatterns = collections.flatMap(c => c.patterns);
|
||||
const uniquePatterns = Array.from(new Set(allPatterns));
|
||||
const names = collections.map(c => c.name).join('+');
|
||||
|
||||
return {
|
||||
name: names,
|
||||
patterns: uniquePatterns,
|
||||
description: `Merged collection: ${names}`
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* Validates pattern array
|
||||
*/
|
||||
validatePatterns(patterns: readonly string[]): { valid: readonly string[]; invalid: readonly string[] } {
|
||||
const valid: string[] = [];
|
||||
const invalid: string[] = [];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
if (typeof pattern === 'string' && pattern.length > 0 && pattern.length <= 200) {
|
||||
valid.push(pattern);
|
||||
} else {
|
||||
invalid.push(pattern);
|
||||
}
|
||||
}
|
||||
|
||||
return { valid, invalid };
|
||||
}
|
||||
};
|
||||
|
|
@ -1,510 +0,0 @@
|
|||
// Performance optimization utilities shared across plugin
|
||||
|
||||
import { parseDuration } from './time.js';
|
||||
|
||||
// Performance utilities for plugin development - provide sensible defaults
|
||||
// These are internal utilities, not user-configurable
|
||||
|
||||
// Default values for performance utilities
|
||||
const DEFAULT_RATE_LIMITER_WINDOW = parseDuration('1m');
|
||||
const DEFAULT_RATE_LIMITER_CLEANUP = parseDuration('1m');
|
||||
const DEFAULT_BATCH_FLUSH_INTERVAL = parseDuration('1s');
|
||||
const DEFAULT_CONNECTION_TIMEOUT = parseDuration('30s');
|
||||
|
||||
// Type definitions for performance utilities
|
||||
export interface CacheOptions {
|
||||
maxSize?: number;
|
||||
ttl?: number | null;
|
||||
}
|
||||
|
||||
export interface RateLimiterOptions {
|
||||
windowMs?: number;
|
||||
maxRequests?: number;
|
||||
}
|
||||
|
||||
export interface BatchProcessorOptions {
|
||||
batchSize?: number;
|
||||
flushInterval?: number;
|
||||
}
|
||||
|
||||
export interface MemoizeOptions {
|
||||
maxSize?: number;
|
||||
ttl?: number;
|
||||
}
|
||||
|
||||
export interface ConnectionPoolOptions {
|
||||
maxConnections?: number;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export interface PoolStats {
|
||||
available: number;
|
||||
inUse: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface PoolData<T> {
|
||||
connections: T[];
|
||||
inUse: Set<T>;
|
||||
}
|
||||
|
||||
export interface Connection {
|
||||
host: string;
|
||||
created: number;
|
||||
}
|
||||
|
||||
// Type aliases for function types
|
||||
export type ObjectFactory<T> = () => T;
|
||||
export type ObjectReset<T> = (obj: T) => void;
|
||||
export type BatchProcessorFunction<T> = (batch: T[]) => Promise<void>;
|
||||
export type DebouncedFunction<T extends unknown[]> = (...args: T) => void;
|
||||
export type ThrottledFunction<T extends unknown[]> = (...args: T) => void;
|
||||
export type MemoizedFunction<T extends unknown[], R> = (...args: T) => R;
|
||||
|
||||
/**
|
||||
* LRU (Least Recently Used) cache implementation with size limits
|
||||
* Prevents memory leaks by automatically evicting oldest entries
|
||||
*/
|
||||
export class LRUCache<K = string, V = unknown> {
|
||||
private readonly maxSize: number;
|
||||
private readonly ttl: number | null;
|
||||
private readonly cache = new Map<K, V>();
|
||||
private readonly accessOrder = new Map<K, number>(); // Track access times
|
||||
|
||||
constructor(maxSize: number = 10000, ttl: number | null = null) {
|
||||
this.maxSize = maxSize;
|
||||
this.ttl = ttl; // Time to live in milliseconds
|
||||
}
|
||||
|
||||
set(key: K, value: V): void {
|
||||
// Delete if at capacity
|
||||
if (this.cache.size >= this.maxSize) {
|
||||
const oldestKey = this.cache.keys().next().value;
|
||||
if (oldestKey !== undefined) {
|
||||
this.cache.delete(oldestKey);
|
||||
this.accessOrder.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
// Add/update entry
|
||||
this.cache.delete(key);
|
||||
this.cache.set(key, value);
|
||||
this.accessOrder.set(key, Date.now());
|
||||
}
|
||||
|
||||
get(key: K): V | undefined {
|
||||
if (!this.cache.has(key)) return undefined;
|
||||
|
||||
// Check TTL if configured
|
||||
if (this.ttl) {
|
||||
const accessTime = this.accessOrder.get(key);
|
||||
if (accessTime && Date.now() - accessTime > this.ttl) {
|
||||
this.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Move to end (most recently used)
|
||||
const value = this.cache.get(key);
|
||||
if (value !== undefined) {
|
||||
this.cache.delete(key);
|
||||
this.cache.set(key, value);
|
||||
this.accessOrder.set(key, Date.now());
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
has(key: K): boolean {
|
||||
if (this.ttl) {
|
||||
const accessTime = this.accessOrder.get(key) || 0;
|
||||
const age = Date.now() - accessTime;
|
||||
if (age > this.ttl) {
|
||||
this.delete(key);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return this.cache.has(key);
|
||||
}
|
||||
|
||||
delete(key: K): boolean {
|
||||
this.accessOrder.delete(key);
|
||||
return this.cache.delete(key);
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.cache.clear();
|
||||
this.accessOrder.clear();
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.cache.size;
|
||||
}
|
||||
|
||||
// Clean expired entries
|
||||
cleanup(): number {
|
||||
if (!this.ttl) return 0;
|
||||
|
||||
const now = Date.now();
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, timestamp] of this.accessOrder.entries()) {
|
||||
if (now - timestamp > this.ttl) {
|
||||
this.delete(key);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate limiter with sliding window and automatic cleanup
|
||||
*/
|
||||
export class RateLimiter {
|
||||
private readonly windowMs: number;
|
||||
private readonly maxRequests: number;
|
||||
private readonly requests = new Map<string, number[]>();
|
||||
private cleanupInterval: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(windowMs: number = DEFAULT_RATE_LIMITER_WINDOW, maxRequests: number = 100, cleanupIntervalMs: number = DEFAULT_RATE_LIMITER_CLEANUP) {
|
||||
this.windowMs = windowMs;
|
||||
this.maxRequests = maxRequests;
|
||||
|
||||
// Automatic cleanup with configured interval
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
this.cleanup();
|
||||
}, cleanupIntervalMs);
|
||||
}
|
||||
|
||||
isAllowed(identifier: string): boolean {
|
||||
const now = Date.now();
|
||||
const userRequests = this.requests.get(identifier) || [];
|
||||
|
||||
// Remove old requests outside the window
|
||||
const validRequests = userRequests.filter(timestamp => now - timestamp < this.windowMs);
|
||||
|
||||
if (validRequests.length >= this.maxRequests) {
|
||||
this.requests.set(identifier, validRequests);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Add new request
|
||||
validRequests.push(now);
|
||||
this.requests.set(identifier, validRequests);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
cleanup(): number {
|
||||
const now = Date.now();
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [identifier, timestamps] of this.requests.entries()) {
|
||||
const validRequests = timestamps.filter(t => now - t < this.windowMs);
|
||||
|
||||
if (validRequests.length === 0) {
|
||||
this.requests.delete(identifier);
|
||||
cleaned++;
|
||||
} else {
|
||||
this.requests.set(identifier, validRequests);
|
||||
}
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
this.cleanupInterval = null;
|
||||
}
|
||||
this.requests.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Object pool for reusing expensive objects
|
||||
*/
|
||||
export class ObjectPool<T> {
|
||||
private readonly factory: ObjectFactory<T>;
|
||||
private readonly reset: ObjectReset<T>;
|
||||
private readonly maxSize: number;
|
||||
private available: T[] = [];
|
||||
private readonly inUse = new Set<T>();
|
||||
|
||||
constructor(factory: ObjectFactory<T>, reset: ObjectReset<T>, maxSize: number = 100) {
|
||||
this.factory = factory;
|
||||
this.reset = reset;
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
acquire(): T {
|
||||
let obj: T;
|
||||
|
||||
if (this.available.length > 0) {
|
||||
obj = this.available.pop()!;
|
||||
} else {
|
||||
obj = this.factory();
|
||||
}
|
||||
|
||||
this.inUse.add(obj);
|
||||
return obj;
|
||||
}
|
||||
|
||||
release(obj: T): void {
|
||||
if (!this.inUse.has(obj)) return;
|
||||
|
||||
this.inUse.delete(obj);
|
||||
|
||||
if (this.available.length < this.maxSize) {
|
||||
this.reset(obj);
|
||||
this.available.push(obj);
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.available = [];
|
||||
this.inUse.clear();
|
||||
}
|
||||
|
||||
get size(): PoolStats {
|
||||
return {
|
||||
available: this.available.length,
|
||||
inUse: this.inUse.size,
|
||||
total: this.available.length + this.inUse.size
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch processor for aggregating operations
|
||||
*/
|
||||
export class BatchProcessor<T> {
|
||||
private readonly processor: BatchProcessorFunction<T>;
|
||||
private readonly batchSize: number;
|
||||
private readonly flushInterval: number;
|
||||
private queue: T[] = [];
|
||||
private processing = false;
|
||||
private intervalId: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(processor: BatchProcessorFunction<T>, options: BatchProcessorOptions = {}) {
|
||||
this.processor = processor;
|
||||
this.batchSize = options.batchSize || 100;
|
||||
this.flushInterval = options.flushInterval || DEFAULT_BATCH_FLUSH_INTERVAL;
|
||||
|
||||
// Auto-flush on interval
|
||||
this.intervalId = setInterval(() => {
|
||||
this.flush();
|
||||
}, this.flushInterval);
|
||||
}
|
||||
|
||||
add(item: T): void {
|
||||
this.queue.push(item);
|
||||
|
||||
if (this.queue.length >= this.batchSize) {
|
||||
this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
if (this.processing || this.queue.length === 0) return;
|
||||
|
||||
this.processing = true;
|
||||
const batch = this.queue.splice(0, this.batchSize);
|
||||
|
||||
try {
|
||||
await this.processor(batch);
|
||||
} catch (err) {
|
||||
console.error('Batch processing error:', err);
|
||||
} finally {
|
||||
this.processing = false;
|
||||
}
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.intervalId) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
}
|
||||
this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Debounce function for reducing function call frequency
|
||||
*/
|
||||
export function debounce<T extends unknown[]>(
|
||||
func: (...args: T) => void,
|
||||
wait: number
|
||||
): DebouncedFunction<T> {
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
|
||||
return function executedFunction(...args: T): void {
|
||||
const later = (): void => {
|
||||
timeout = undefined;
|
||||
func(...args);
|
||||
};
|
||||
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
timeout = setTimeout(later, wait);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttle function for limiting function execution rate
|
||||
*/
|
||||
export function throttle<T extends unknown[]>(
|
||||
func: (...args: T) => void,
|
||||
limit: number
|
||||
): ThrottledFunction<T> {
|
||||
let inThrottle = false;
|
||||
|
||||
return function(...args: T): void {
|
||||
if (!inThrottle) {
|
||||
func(...args);
|
||||
inThrottle = true;
|
||||
setTimeout(() => {
|
||||
inThrottle = false;
|
||||
}, limit);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Memoize function results with optional TTL
|
||||
*/
|
||||
export function memoize<T extends unknown[], R>(
|
||||
func: (...args: T) => R,
|
||||
options: MemoizeOptions = {}
|
||||
): MemoizedFunction<T, R> {
|
||||
const cache = new LRUCache<string, R>(options.maxSize || 1000, options.ttl);
|
||||
|
||||
return function(...args: T): R {
|
||||
const key = JSON.stringify(args);
|
||||
|
||||
if (cache.has(key)) {
|
||||
const cached = cache.get(key);
|
||||
if (cached !== undefined) {
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
|
||||
const result = func(...args);
|
||||
cache.set(key, result);
|
||||
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Efficient string search using Set for O(1) lookups
|
||||
*/
|
||||
export class StringMatcher {
|
||||
private readonly patterns: Set<string>;
|
||||
|
||||
constructor(patterns: string[]) {
|
||||
this.patterns = new Set(patterns.map(p => p.toLowerCase()));
|
||||
}
|
||||
|
||||
contains(text: string): boolean {
|
||||
return this.patterns.has(text.toLowerCase());
|
||||
}
|
||||
|
||||
containsAny(texts: string[]): boolean {
|
||||
return texts.some(text => this.contains(text));
|
||||
}
|
||||
|
||||
add(pattern: string): void {
|
||||
this.patterns.add(pattern.toLowerCase());
|
||||
}
|
||||
|
||||
remove(pattern: string): boolean {
|
||||
return this.patterns.delete(pattern.toLowerCase());
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.patterns.size;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection pool for reusing network connections
|
||||
*/
|
||||
export class ConnectionPool<T extends Connection = Connection> {
|
||||
private readonly maxConnections: number;
|
||||
private readonly connectionTimeoutMs: number;
|
||||
private readonly pools = new Map<string, PoolData<T>>(); // host -> connections
|
||||
|
||||
constructor(options: ConnectionPoolOptions = {}) {
|
||||
this.maxConnections = options.maxConnections || 50;
|
||||
this.connectionTimeoutMs = options.timeout || DEFAULT_CONNECTION_TIMEOUT;
|
||||
}
|
||||
|
||||
// Getter for subclasses to access connection timeout
|
||||
protected get connectionTimeout(): number {
|
||||
return this.connectionTimeoutMs;
|
||||
}
|
||||
|
||||
getConnection(host: string): T | null {
|
||||
if (!this.pools.has(host)) {
|
||||
this.pools.set(host, {
|
||||
connections: [],
|
||||
inUse: new Set()
|
||||
});
|
||||
}
|
||||
|
||||
const pool = this.pools.get(host)!;
|
||||
|
||||
// Reuse existing connection
|
||||
if (pool.connections.length > 0) {
|
||||
const conn = pool.connections.pop()!;
|
||||
pool.inUse.add(conn);
|
||||
return conn;
|
||||
}
|
||||
|
||||
// Create new connection if under limit
|
||||
if (pool.inUse.size < this.maxConnections) {
|
||||
const conn = this.createConnection(host);
|
||||
pool.inUse.add(conn);
|
||||
return conn;
|
||||
}
|
||||
|
||||
return null; // Pool exhausted
|
||||
}
|
||||
|
||||
releaseConnection(host: string, conn: T): void {
|
||||
const pool = this.pools.get(host);
|
||||
if (!pool || !pool.inUse.has(conn)) return;
|
||||
|
||||
pool.inUse.delete(conn);
|
||||
|
||||
if (pool.connections.length < this.maxConnections / 2) {
|
||||
pool.connections.push(conn);
|
||||
} else {
|
||||
this.closeConnection(conn);
|
||||
}
|
||||
}
|
||||
|
||||
protected createConnection(host: string): T {
|
||||
// Override in subclass
|
||||
return { host, created: Date.now() } as T;
|
||||
}
|
||||
|
||||
protected closeConnection(_conn: T): void {
|
||||
// Override in subclass
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
for (const [_host, pool] of this.pools.entries()) {
|
||||
pool.connections.forEach(conn => this.closeConnection(conn));
|
||||
pool.inUse.forEach(conn => this.closeConnection(conn));
|
||||
}
|
||||
this.pools.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Note: All types are already exported above
|
||||
|
|
@ -1,182 +0,0 @@
|
|||
// =============================================================================
|
||||
// SECURE PLUGIN SYSTEM - TYPESCRIPT VERSION
|
||||
// =============================================================================
|
||||
// Enhanced security for module imports with comprehensive path validation
|
||||
// Prevents path traversal, validates file extensions, and enforces application boundaries
|
||||
|
||||
import { resolve, extname, sep, isAbsolute, normalize } from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { rootDir } from '../index.js';
|
||||
|
||||
// Type definitions for secure plugin system
|
||||
export interface PluginModule {
|
||||
readonly [key: string]: unknown;
|
||||
}
|
||||
|
||||
// Security constants for module validation
|
||||
const ALLOWED_EXTENSIONS = new Set(['.js', '.mjs']);
|
||||
const MAX_PATH_LENGTH = 1024; // Reasonable path length limit
|
||||
const MAX_PATH_DEPTH = 20; // Maximum directory depth
|
||||
const BLOCKED_PATTERNS = [
|
||||
/\.\./, // Directory traversal
|
||||
/\/\/+/, // Double slashes
|
||||
/\0/, // Null bytes
|
||||
/[\x00-\x1f]/, // Control characters
|
||||
/node_modules/i, // Prevent node_modules access
|
||||
/package\.json/i, // Prevent package.json access
|
||||
/\.env/i, // Prevent environment file access
|
||||
] as const;
|
||||
|
||||
// Input validation with zero trust approach
|
||||
function validateModulePath(relPath: unknown): string {
|
||||
// Type validation
|
||||
if (typeof relPath !== 'string') {
|
||||
throw new Error('Module path must be a string');
|
||||
}
|
||||
|
||||
// Length validation
|
||||
if (relPath.length === 0) {
|
||||
throw new Error('Module path cannot be empty');
|
||||
}
|
||||
|
||||
if (relPath.length > MAX_PATH_LENGTH) {
|
||||
throw new Error(`Module path too long: ${relPath.length} > ${MAX_PATH_LENGTH}`);
|
||||
}
|
||||
|
||||
// Security pattern validation
|
||||
for (const pattern of BLOCKED_PATTERNS) {
|
||||
if (pattern.test(relPath)) {
|
||||
throw new Error(`Module path contains blocked pattern: ${relPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize path to prevent encoding bypasses
|
||||
const normalizedPath = normalize(relPath);
|
||||
|
||||
// Validate path depth
|
||||
const pathSegments = normalizedPath.split(sep).filter(segment => segment !== '');
|
||||
if (pathSegments.length > MAX_PATH_DEPTH) {
|
||||
throw new Error(`Module path too deep: ${pathSegments.length} > ${MAX_PATH_DEPTH}`);
|
||||
}
|
||||
|
||||
return normalizedPath;
|
||||
}
|
||||
|
||||
function validateFileExtension(filePath: string): void {
|
||||
const ext = extname(filePath).toLowerCase();
|
||||
|
||||
if (!ALLOWED_EXTENSIONS.has(ext as any)) {
|
||||
throw new Error(`Only ${Array.from(ALLOWED_EXTENSIONS).join(', ')} files can be imported: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateRootDirectory(): string {
|
||||
if (typeof rootDir !== 'string' || rootDir.length === 0) {
|
||||
throw new Error('Invalid application root directory');
|
||||
}
|
||||
|
||||
return normalize(rootDir);
|
||||
}
|
||||
|
||||
function validateResolvedPath(absPath: string, rootDir: string): void {
|
||||
const normalizedAbsPath = normalize(absPath);
|
||||
const normalizedRootDir = normalize(rootDir);
|
||||
|
||||
// Ensure the resolved path is within the application root
|
||||
if (!normalizedAbsPath.startsWith(normalizedRootDir + sep) && normalizedAbsPath !== normalizedRootDir) {
|
||||
throw new Error(`Module path outside of application root: ${normalizedAbsPath}`);
|
||||
}
|
||||
|
||||
// Additional security check for symbolic link traversal
|
||||
try {
|
||||
const relativePath = normalizedAbsPath.substring(normalizedRootDir.length);
|
||||
if (relativePath.includes('..')) {
|
||||
throw new Error(`Path traversal detected in resolved path: ${normalizedAbsPath}`);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Path validation failed: ${error instanceof Error ? error.message : 'unknown'}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Securely import a JavaScript module from within the application root.
|
||||
* Enhanced with comprehensive security validation and TypeScript safety.
|
||||
* Prevents path traversal, validates extensions, and enforces application boundaries.
|
||||
*
|
||||
* @param relPath - The relative path to the module from the application root
|
||||
* @returns Promise that resolves to the imported module
|
||||
* @throws Error if the path is invalid, unsafe, or outside application boundaries
|
||||
*/
|
||||
export async function secureImportModule(relPath: unknown): Promise<PluginModule> {
|
||||
try {
|
||||
// Validate and normalize the input path
|
||||
const validatedPath = validateModulePath(relPath);
|
||||
|
||||
// Security check: reject absolute paths
|
||||
if (isAbsolute(validatedPath)) {
|
||||
throw new Error('Absolute paths are not allowed for module imports');
|
||||
}
|
||||
|
||||
// Validate file extension
|
||||
validateFileExtension(validatedPath);
|
||||
|
||||
// Validate root directory
|
||||
const validatedRootDir = validateRootDirectory();
|
||||
|
||||
// Resolve the absolute path
|
||||
const absPath = resolve(validatedRootDir, validatedPath);
|
||||
|
||||
// Validate the resolved path is within application boundaries
|
||||
validateResolvedPath(absPath, validatedRootDir);
|
||||
|
||||
// Convert to file URL for secure import
|
||||
const url = pathToFileURL(absPath).href;
|
||||
|
||||
// Perform the actual import with error handling
|
||||
try {
|
||||
const importedModule = await import(url);
|
||||
|
||||
// Validate the imported module
|
||||
if (!importedModule || typeof importedModule !== 'object') {
|
||||
throw new Error(`Invalid module structure: ${validatedPath}`);
|
||||
}
|
||||
|
||||
return importedModule as PluginModule;
|
||||
|
||||
} catch (importError) {
|
||||
// Provide more context for import failures
|
||||
throw new Error(`Failed to import module ${validatedPath}: ${importError instanceof Error ? importError.message : 'unknown error'}`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// Re-throw with additional context while preventing information leakage
|
||||
if (error instanceof Error) {
|
||||
throw new Error(`Module import failed: ${error.message}`);
|
||||
} else {
|
||||
throw new Error('Module import failed due to unknown error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if an imported module has a specific export
|
||||
* @param module - The imported module
|
||||
* @param exportName - The name of the export to check
|
||||
* @returns True if the export exists
|
||||
*/
|
||||
export function hasExport(module: PluginModule, exportName: string): boolean {
|
||||
return exportName in module && module[exportName] !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely extract a specific export from a module with type checking
|
||||
* @param module - The imported module
|
||||
* @param exportName - The name of the export to extract
|
||||
* @returns The export value or undefined if not found
|
||||
*/
|
||||
export function getExport<T = unknown>(module: PluginModule, exportName: string): T | undefined {
|
||||
if (hasExport(module, exportName)) {
|
||||
return module[exportName] as T;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -1,306 +0,0 @@
|
|||
import * as crypto from 'crypto';
|
||||
import { getRealIP, type NetworkRequest } from './network.js';
|
||||
import { parseDuration } from './time.js';
|
||||
|
||||
// Type definitions for secure proof operations
|
||||
export interface ChallengeData {
|
||||
readonly challenge: string;
|
||||
readonly salt: string;
|
||||
}
|
||||
|
||||
export interface ChallengeParams {
|
||||
readonly Challenge: string;
|
||||
readonly Salt: string;
|
||||
readonly Difficulty: number;
|
||||
readonly ExpiresAt: number;
|
||||
readonly CreatedAt: number;
|
||||
readonly ClientIP: string;
|
||||
readonly PoSSeed: string;
|
||||
}
|
||||
|
||||
export interface CheckpointConfig {
|
||||
readonly SaltLength: number;
|
||||
readonly Difficulty: number;
|
||||
readonly ChallengeExpiration: number;
|
||||
readonly CheckPoSTimes: boolean;
|
||||
readonly PoSTimeConsistencyRatio: number;
|
||||
}
|
||||
|
||||
// Security constants - prevent DoS attacks while respecting user config
|
||||
const ABSOLUTE_MAX_SALT_LENGTH = 1024; // 1KB - prevents memory exhaustion
|
||||
const ABSOLUTE_MAX_DIFFICULTY = 64; // Reasonable upper bound for crypto safety
|
||||
const ABSOLUTE_MIN_DIFFICULTY = 1; // Must be at least 1
|
||||
const ABSOLUTE_MAX_DURATION = parseDuration('365d'); // 1 year - prevents overflow
|
||||
const EXPECTED_POS_TIMES_LENGTH = 3; // Protocol requirement
|
||||
const EXPECTED_POS_HASHES_LENGTH = 3; // Protocol requirement
|
||||
const EXPECTED_HASH_LENGTH = 64; // SHA-256 hex length
|
||||
const ABSOLUTE_MAX_INPUT_LENGTH = 100000; // 100KB - prevents DoS
|
||||
const ABSOLUTE_MAX_REQUEST_ID_LENGTH = 64; // Reasonable hex string limit
|
||||
|
||||
// Input validation functions - zero trust approach
|
||||
function validateHexString(value: unknown, paramName: string, maxLength: number): string {
|
||||
if (typeof value !== 'string') {
|
||||
throw new Error(`${paramName} must be a string`);
|
||||
}
|
||||
if (value.length === 0) {
|
||||
throw new Error(`${paramName} cannot be empty`);
|
||||
}
|
||||
if (value.length > maxLength) {
|
||||
throw new Error(`${paramName} exceeds maximum length of ${maxLength}`);
|
||||
}
|
||||
if (!/^[0-9a-fA-F]+$/.test(value)) {
|
||||
throw new Error(`${paramName} must be a valid hexadecimal string`);
|
||||
}
|
||||
return value.toLowerCase();
|
||||
}
|
||||
|
||||
function validatePositiveInteger(value: unknown, paramName: string, min: number, max: number): number {
|
||||
if (typeof value !== 'number' || !Number.isInteger(value)) {
|
||||
throw new Error(`${paramName} must be an integer`);
|
||||
}
|
||||
if (value < min || value > max) {
|
||||
throw new Error(`${paramName} must be between ${min} and ${max}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateTimesArray(value: unknown, paramName: string): number[] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new Error(`${paramName} must be an array`);
|
||||
}
|
||||
if (value.length !== EXPECTED_POS_TIMES_LENGTH) {
|
||||
throw new Error(`${paramName} must have exactly ${EXPECTED_POS_TIMES_LENGTH} elements`);
|
||||
}
|
||||
|
||||
const validatedTimes: number[] = [];
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const time = value[i];
|
||||
if (typeof time !== 'number' || !Number.isFinite(time) || time < 0) {
|
||||
throw new Error(`${paramName}[${i}] must be a non-negative finite number`);
|
||||
}
|
||||
if (time > 10000000) { // 10M ms = ~3 hours - generous but prevents DoS
|
||||
throw new Error(`${paramName}[${i}] exceeds maximum allowed value`);
|
||||
}
|
||||
validatedTimes.push(time);
|
||||
}
|
||||
return validatedTimes;
|
||||
}
|
||||
|
||||
function validateHashesArray(value: unknown, paramName: string): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
throw new Error(`${paramName} must be an array`);
|
||||
}
|
||||
if (value.length !== EXPECTED_POS_HASHES_LENGTH) {
|
||||
throw new Error(`${paramName} must have exactly ${EXPECTED_POS_HASHES_LENGTH} elements`);
|
||||
}
|
||||
|
||||
const validatedHashes: string[] = [];
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const hash = validateHexString(value[i], `${paramName}[${i}]`, EXPECTED_HASH_LENGTH);
|
||||
if (hash.length !== EXPECTED_HASH_LENGTH) {
|
||||
throw new Error(`${paramName}[${i}] must be exactly ${EXPECTED_HASH_LENGTH} characters`);
|
||||
}
|
||||
validatedHashes.push(hash);
|
||||
}
|
||||
return validatedHashes;
|
||||
}
|
||||
|
||||
function validateCheckpointConfig(config: unknown): CheckpointConfig {
|
||||
if (!config || typeof config !== 'object') {
|
||||
throw new Error('CheckpointConfig must be an object');
|
||||
}
|
||||
|
||||
const cfg = config as Record<string, unknown>;
|
||||
|
||||
// Validate user's salt length - allow generous range but prevent memory exhaustion
|
||||
const saltLength = validatePositiveInteger(cfg.SaltLength, 'SaltLength', 1, ABSOLUTE_MAX_SALT_LENGTH);
|
||||
|
||||
// Respect user's difficulty settings completely - they know their security needs
|
||||
const difficulty = validatePositiveInteger(cfg.Difficulty, 'Difficulty', ABSOLUTE_MIN_DIFFICULTY, ABSOLUTE_MAX_DIFFICULTY);
|
||||
|
||||
// Respect user's expiration settings - they control their own security/usability balance
|
||||
const challengeExpiration = validatePositiveInteger(cfg.ChallengeExpiration, 'ChallengeExpiration', 1000, ABSOLUTE_MAX_DURATION);
|
||||
|
||||
// Validate consistency ratio - prevent divide by zero but allow user control
|
||||
const consistencyRatio = typeof cfg.PoSTimeConsistencyRatio === 'number' && cfg.PoSTimeConsistencyRatio > 0 && cfg.PoSTimeConsistencyRatio <= 1000
|
||||
? cfg.PoSTimeConsistencyRatio : 2.0;
|
||||
|
||||
return {
|
||||
SaltLength: saltLength,
|
||||
Difficulty: difficulty,
|
||||
ChallengeExpiration: challengeExpiration,
|
||||
CheckPoSTimes: typeof cfg.CheckPoSTimes === 'boolean' ? cfg.CheckPoSTimes : false,
|
||||
PoSTimeConsistencyRatio: consistencyRatio
|
||||
};
|
||||
}
|
||||
|
||||
function validateNetworkRequest(request: unknown): NetworkRequest {
|
||||
if (!request || typeof request !== 'object') {
|
||||
throw new Error('Request must be an object');
|
||||
}
|
||||
|
||||
const req = request as Record<string, unknown>;
|
||||
|
||||
// Validate headers object exists
|
||||
if (!req.headers || typeof req.headers !== 'object') {
|
||||
throw new Error('Request must have headers object');
|
||||
}
|
||||
|
||||
// Basic validation - ensure it has the minimal structure for a NetworkRequest
|
||||
return request as NetworkRequest;
|
||||
}
|
||||
|
||||
function generateChallenge(checkpointConfig: unknown): ChallengeData {
|
||||
const validatedConfig = validateCheckpointConfig(checkpointConfig);
|
||||
|
||||
const challenge = crypto.randomBytes(16).toString('hex');
|
||||
const salt = crypto.randomBytes(validatedConfig.SaltLength).toString('hex');
|
||||
|
||||
return { challenge, salt };
|
||||
}
|
||||
|
||||
function calculateHash(input: unknown): string {
|
||||
if (typeof input !== 'string') {
|
||||
throw new Error('Hash input must be a string');
|
||||
}
|
||||
if (input.length === 0) {
|
||||
throw new Error('Hash input cannot be empty');
|
||||
}
|
||||
if (input.length > ABSOLUTE_MAX_INPUT_LENGTH) { // Prevent DoS via massive strings
|
||||
throw new Error(`Hash input exceeds maximum length of ${ABSOLUTE_MAX_INPUT_LENGTH}`);
|
||||
}
|
||||
|
||||
return crypto.createHash('sha256').update(input).digest('hex');
|
||||
}
|
||||
|
||||
export function verifyPoW(
|
||||
challenge: unknown,
|
||||
salt: unknown,
|
||||
nonce: unknown,
|
||||
difficulty: unknown
|
||||
): boolean {
|
||||
// Validate all user-provided inputs with zero trust
|
||||
const validatedChallenge = validateHexString(challenge, 'challenge', ABSOLUTE_MAX_INPUT_LENGTH);
|
||||
const validatedSalt = validateHexString(salt, 'salt', ABSOLUTE_MAX_INPUT_LENGTH);
|
||||
const validatedNonce = validateHexString(nonce, 'nonce', ABSOLUTE_MAX_INPUT_LENGTH);
|
||||
const validatedDifficulty = validatePositiveInteger(difficulty, 'difficulty', ABSOLUTE_MIN_DIFFICULTY, ABSOLUTE_MAX_DIFFICULTY);
|
||||
|
||||
// Perform cryptographic operation with validated inputs
|
||||
const hash = calculateHash(validatedChallenge + validatedSalt + validatedNonce);
|
||||
const requiredPrefix = '0'.repeat(validatedDifficulty);
|
||||
|
||||
return hash.startsWith(requiredPrefix);
|
||||
}
|
||||
|
||||
export function checkPoSTimes(times: unknown, enableCheck: unknown, ratio: unknown): void {
|
||||
const validatedTimes = validateTimesArray(times, 'times');
|
||||
const validatedEnableCheck = typeof enableCheck === 'boolean' ? enableCheck : false;
|
||||
const validatedRatio = typeof ratio === 'number' && ratio > 0 ? ratio : 2.0;
|
||||
|
||||
if (!validatedEnableCheck) {
|
||||
return; // Skip check if disabled
|
||||
}
|
||||
|
||||
const minTime = Math.min(...validatedTimes);
|
||||
const maxTime = Math.max(...validatedTimes);
|
||||
|
||||
if (minTime === 0) {
|
||||
throw new Error('PoS run times cannot be zero');
|
||||
}
|
||||
|
||||
const actualRatio = maxTime / minTime;
|
||||
if (actualRatio > validatedRatio) {
|
||||
throw new Error(`PoS run times inconsistent (ratio ${actualRatio.toFixed(2)} > ${validatedRatio})`);
|
||||
}
|
||||
}
|
||||
|
||||
// Secure in-memory storage with automatic cleanup
|
||||
export const challengeStore = new Map<string, ChallengeParams>();
|
||||
|
||||
// Cleanup expired challenges to prevent memory exhaustion
|
||||
function cleanupExpiredChallenges(): void {
|
||||
const now = Date.now();
|
||||
for (const [requestId, params] of Array.from(challengeStore.entries())) {
|
||||
if (params.ExpiresAt < now) {
|
||||
challengeStore.delete(requestId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run cleanup every 5 minutes
|
||||
setInterval(cleanupExpiredChallenges, parseDuration('5m'));
|
||||
|
||||
export function generateRequestID(request: unknown, checkpointConfig: unknown): string {
|
||||
const validatedConfig = validateCheckpointConfig(checkpointConfig);
|
||||
const validatedRequest = validateNetworkRequest(request);
|
||||
const { challenge, salt } = generateChallenge(validatedConfig);
|
||||
|
||||
const posSeed = crypto.randomBytes(32).toString('hex');
|
||||
const requestId = crypto.randomBytes(16).toString('hex');
|
||||
|
||||
const params: ChallengeParams = {
|
||||
Challenge: challenge,
|
||||
Salt: salt,
|
||||
Difficulty: validatedConfig.Difficulty,
|
||||
ExpiresAt: Date.now() + validatedConfig.ChallengeExpiration,
|
||||
CreatedAt: Date.now(),
|
||||
ClientIP: getRealIP(validatedRequest),
|
||||
PoSSeed: posSeed,
|
||||
};
|
||||
|
||||
challengeStore.set(requestId, params);
|
||||
return requestId;
|
||||
}
|
||||
|
||||
export function getChallengeParams(requestId: unknown): ChallengeParams | undefined {
|
||||
if (typeof requestId !== 'string') {
|
||||
throw new Error('Request ID must be a string');
|
||||
}
|
||||
if (requestId.length > ABSOLUTE_MAX_REQUEST_ID_LENGTH) {
|
||||
throw new Error(`Request ID exceeds maximum length of ${ABSOLUTE_MAX_REQUEST_ID_LENGTH}`);
|
||||
}
|
||||
if (requestId.length !== 32) { // Expected length for hex-encoded 16 bytes
|
||||
throw new Error('Invalid request ID format');
|
||||
}
|
||||
if (!/^[0-9a-fA-F]+$/.test(requestId)) {
|
||||
throw new Error('Request ID must be hexadecimal');
|
||||
}
|
||||
|
||||
return challengeStore.get(requestId);
|
||||
}
|
||||
|
||||
export function deleteChallenge(requestId: unknown): boolean {
|
||||
if (typeof requestId !== 'string') {
|
||||
throw new Error('Request ID must be a string');
|
||||
}
|
||||
|
||||
return challengeStore.delete(requestId);
|
||||
}
|
||||
|
||||
export function verifyPoS(
|
||||
hashes: unknown,
|
||||
times: unknown,
|
||||
checkpointConfig: unknown
|
||||
): void {
|
||||
// Validate all user inputs with zero trust
|
||||
const validatedHashes = validateHashesArray(hashes, 'hashes');
|
||||
const validatedTimes = validateTimesArray(times, 'times');
|
||||
const validatedConfig = validateCheckpointConfig(checkpointConfig);
|
||||
|
||||
// Verify hash consistency - all must match
|
||||
const firstHash = validatedHashes[0];
|
||||
for (let i = 1; i < validatedHashes.length; i++) {
|
||||
if (validatedHashes[i] !== firstHash) {
|
||||
throw new Error('PoS hashes do not match');
|
||||
}
|
||||
}
|
||||
|
||||
// Validate timing consistency
|
||||
checkPoSTimes(validatedTimes, validatedConfig.CheckPoSTimes, validatedConfig.PoSTimeConsistencyRatio);
|
||||
}
|
||||
|
||||
// Export for testing
|
||||
export {
|
||||
calculateHash,
|
||||
generateChallenge
|
||||
};
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
// =============================================================================
|
||||
// THREAT SCORING ENGINE V2.0 - BACKWARD COMPATIBILITY LAYER (TYPESCRIPT)
|
||||
// =============================================================================
|
||||
// This file maintains backward compatibility by re-exporting from the refactored modules
|
||||
// Provides type-safe access to threat scoring functionality
|
||||
// =============================================================================
|
||||
|
||||
export { threatScorer, configureDefaultThreatScorer, createThreatScorer, type ThreatScore, type ThreatScoringConfig } from './threat-scoring/index.js';
|
||||
|
|
@ -1,480 +0,0 @@
|
|||
// =============================================================================
|
||||
// GEO ANALYSIS (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface GeoLocation {
|
||||
readonly lat: number;
|
||||
readonly lon: number;
|
||||
}
|
||||
|
||||
interface GeoData {
|
||||
readonly country?: string;
|
||||
readonly continent?: string;
|
||||
readonly latitude?: number;
|
||||
readonly longitude?: number;
|
||||
readonly asn?: number;
|
||||
readonly isp?: string;
|
||||
readonly datacenter?: boolean;
|
||||
readonly city?: string;
|
||||
readonly region?: string;
|
||||
readonly timezone?: string;
|
||||
}
|
||||
|
||||
interface GeoFeatures {
|
||||
readonly country: string | null;
|
||||
readonly isHighRisk: boolean;
|
||||
readonly isDatacenter: boolean;
|
||||
readonly location: GeoLocation | null;
|
||||
readonly geoScore: number;
|
||||
readonly countryRisk: number;
|
||||
readonly continent?: string;
|
||||
readonly asn?: number;
|
||||
readonly isp?: string;
|
||||
}
|
||||
|
||||
interface DistanceCalculationResult {
|
||||
readonly distance: number;
|
||||
readonly unit: 'km' | 'miles';
|
||||
readonly formula: 'haversine';
|
||||
readonly accuracy: 'high' | 'medium' | 'low';
|
||||
}
|
||||
|
||||
interface CountryRiskProfile {
|
||||
readonly code: string;
|
||||
readonly name: string;
|
||||
readonly riskLevel: 'low' | 'medium' | 'high' | 'critical';
|
||||
readonly score: number;
|
||||
readonly reasons: readonly string[];
|
||||
}
|
||||
|
||||
// Geographic analysis configuration
|
||||
interface GeoAnalysisConfig {
|
||||
readonly earthRadiusKm: number;
|
||||
readonly earthRadiusMiles: number;
|
||||
readonly coordinatePrecision: number;
|
||||
readonly maxValidLatitude: number;
|
||||
readonly maxValidLongitude: number;
|
||||
readonly datacenterASNs: readonly number[];
|
||||
readonly highRiskCountries: readonly string[];
|
||||
readonly mediumRiskCountries: readonly string[];
|
||||
}
|
||||
|
||||
// Configuration constants
|
||||
const GEO_CONFIG: GeoAnalysisConfig = {
|
||||
earthRadiusKm: 6371, // Earth's radius in kilometers
|
||||
earthRadiusMiles: 3959, // Earth's radius in miles
|
||||
coordinatePrecision: 6, // Decimal places for coordinates
|
||||
maxValidLatitude: 90, // Maximum valid latitude
|
||||
maxValidLongitude: 180, // Maximum valid longitude
|
||||
datacenterASNs: [
|
||||
13335, 15169, 16509, 8075, // Cloudflare, Google, Amazon, Microsoft
|
||||
32934, 54113, 394711 // Facebook, Fastly, Alibaba
|
||||
],
|
||||
highRiskCountries: [
|
||||
'CN', 'RU', 'KP', 'IR', 'SY', 'AF', 'IQ', 'LY', 'SO', 'SS'
|
||||
],
|
||||
mediumRiskCountries: [
|
||||
'PK', 'BD', 'NG', 'VE', 'MM', 'KH', 'LA', 'UZ', 'TM'
|
||||
]
|
||||
} as const;
|
||||
|
||||
// Country risk profiles for detailed analysis
|
||||
const COUNTRY_RISK_PROFILES: Record<string, CountryRiskProfile> = {
|
||||
'CN': {
|
||||
code: 'CN',
|
||||
name: 'China',
|
||||
riskLevel: 'high',
|
||||
score: 75,
|
||||
reasons: ['state_sponsored_attacks', 'high_malware_volume', 'censorship_infrastructure']
|
||||
},
|
||||
'RU': {
|
||||
code: 'RU',
|
||||
name: 'Russia',
|
||||
riskLevel: 'high',
|
||||
score: 80,
|
||||
reasons: ['cybercrime_hub', 'ransomware_operations', 'state_sponsored_attacks']
|
||||
},
|
||||
'KP': {
|
||||
code: 'KP',
|
||||
name: 'North Korea',
|
||||
riskLevel: 'critical',
|
||||
score: 95,
|
||||
reasons: ['state_sponsored_attacks', 'sanctions_evasion', 'cryptocurrency_theft']
|
||||
},
|
||||
'IR': {
|
||||
code: 'IR',
|
||||
name: 'Iran',
|
||||
riskLevel: 'high',
|
||||
score: 70,
|
||||
reasons: ['state_sponsored_attacks', 'sanctions_evasion', 'regional_threats']
|
||||
}
|
||||
} as const;
|
||||
|
||||
// =============================================================================
|
||||
// MAIN ANALYSIS FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Analyzes geographic data and extracts security-relevant features
|
||||
* @param geoData - Geographic information from IP geolocation
|
||||
* @returns Comprehensive geographic feature analysis
|
||||
*/
|
||||
export function analyzeGeoData(geoData: GeoData | null): GeoFeatures {
|
||||
// Default features for invalid or missing geo data
|
||||
const defaultFeatures: GeoFeatures = {
|
||||
country: null,
|
||||
isHighRisk: false,
|
||||
isDatacenter: false,
|
||||
location: null,
|
||||
geoScore: 0,
|
||||
countryRisk: 0
|
||||
};
|
||||
|
||||
// Return defaults if no geo data provided
|
||||
if (!geoData || typeof geoData !== 'object') {
|
||||
return defaultFeatures;
|
||||
}
|
||||
|
||||
try {
|
||||
// Extract and validate country information
|
||||
const country = validateCountryCode(geoData.country);
|
||||
const countryRisk = calculateCountryRisk(country);
|
||||
|
||||
// Check if this is a datacenter/hosting provider
|
||||
const isDatacenter = checkDatacenterSource(geoData);
|
||||
|
||||
// Extract and validate location coordinates
|
||||
const location = extractLocation(geoData);
|
||||
|
||||
// Calculate overall geographic risk score
|
||||
const geoScore = calculateGeoScore(countryRisk.score, isDatacenter, geoData);
|
||||
|
||||
const features: GeoFeatures = {
|
||||
country,
|
||||
isHighRisk: countryRisk.isHighRisk,
|
||||
isDatacenter,
|
||||
location,
|
||||
geoScore: Math.round(geoScore * 100) / 100, // Round to 2 decimal places
|
||||
countryRisk: countryRisk.score,
|
||||
continent: geoData.continent || undefined,
|
||||
asn: geoData.asn || undefined,
|
||||
isp: geoData.isp || undefined
|
||||
};
|
||||
|
||||
return features;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to analyze geo data:', error.message);
|
||||
return defaultFeatures;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the great-circle distance between two geographic points
|
||||
* Uses the Haversine formula for high accuracy
|
||||
*
|
||||
* @param loc1 - First location coordinates
|
||||
* @param loc2 - Second location coordinates
|
||||
* @param unit - Distance unit ('km' or 'miles')
|
||||
* @returns Distance in specified units or null if invalid
|
||||
*/
|
||||
export function calculateDistance(
|
||||
loc1: GeoLocation | null,
|
||||
loc2: GeoLocation | null,
|
||||
unit: 'km' | 'miles' = 'km'
|
||||
): number | null {
|
||||
// Input validation
|
||||
if (!isValidLocation(loc1) || !isValidLocation(loc2)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// Select Earth radius based on desired unit
|
||||
const earthRadius = unit === 'km' ? GEO_CONFIG.earthRadiusKm : GEO_CONFIG.earthRadiusMiles;
|
||||
|
||||
// Convert coordinates to radians
|
||||
const lat1Rad = toRadians(loc1!.lat);
|
||||
const lon1Rad = toRadians(loc1!.lon);
|
||||
const lat2Rad = toRadians(loc2!.lat);
|
||||
const lon2Rad = toRadians(loc2!.lon);
|
||||
|
||||
// Calculate differences
|
||||
const dLat = lat2Rad - lat1Rad;
|
||||
const dLon = lon2Rad - lon1Rad;
|
||||
|
||||
// Haversine formula calculation
|
||||
const a = Math.sin(dLat / 2) * Math.sin(dLat / 2) +
|
||||
Math.cos(lat1Rad) * Math.cos(lat2Rad) *
|
||||
Math.sin(dLon / 2) * Math.sin(dLon / 2);
|
||||
|
||||
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
||||
const distance = earthRadius * c;
|
||||
|
||||
// Round to appropriate precision and ensure non-negative
|
||||
return Math.max(0, Math.round(distance * 1000) / 1000); // 3 decimal places
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to calculate distance:', error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced distance calculation with detailed results
|
||||
* @param loc1 - First location
|
||||
* @param loc2 - Second location
|
||||
* @param unit - Distance unit
|
||||
* @returns Detailed distance calculation result
|
||||
*/
|
||||
export function calculateDistanceDetailed(
|
||||
loc1: GeoLocation | null,
|
||||
loc2: GeoLocation | null,
|
||||
unit: 'km' | 'miles' = 'km'
|
||||
): DistanceCalculationResult | null {
|
||||
const distance = calculateDistance(loc1, loc2, unit);
|
||||
|
||||
if (distance === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Determine accuracy based on coordinate precision
|
||||
const accuracy = determineCalculationAccuracy(loc1!, loc2!);
|
||||
|
||||
return {
|
||||
distance,
|
||||
unit,
|
||||
formula: 'haversine',
|
||||
accuracy
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// HELPER FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Validates and normalizes country code
|
||||
* @param country - Country code to validate
|
||||
* @returns Valid country code or null
|
||||
*/
|
||||
function validateCountryCode(country: string | undefined): string | null {
|
||||
if (!country || typeof country !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Normalize to uppercase and trim
|
||||
const normalized = country.trim().toUpperCase();
|
||||
|
||||
// Validate ISO 3166-1 alpha-2 format (2 letters)
|
||||
if (!/^[A-Z]{2}$/.test(normalized)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates country-based risk assessment
|
||||
* @param country - Country code
|
||||
* @returns Risk assessment with score and classification
|
||||
*/
|
||||
function calculateCountryRisk(country: string | null): { score: number; isHighRisk: boolean; profile?: CountryRiskProfile } {
|
||||
if (!country) {
|
||||
return { score: 0, isHighRisk: false };
|
||||
}
|
||||
|
||||
// Check for detailed risk profile
|
||||
const profile = COUNTRY_RISK_PROFILES[country];
|
||||
if (profile) {
|
||||
return {
|
||||
score: profile.score,
|
||||
isHighRisk: profile.riskLevel === 'high' || profile.riskLevel === 'critical',
|
||||
profile
|
||||
};
|
||||
}
|
||||
|
||||
// Check high-risk countries list
|
||||
if (GEO_CONFIG.highRiskCountries.includes(country)) {
|
||||
return { score: 65, isHighRisk: true };
|
||||
}
|
||||
|
||||
// Check medium-risk countries list
|
||||
if (GEO_CONFIG.mediumRiskCountries.includes(country)) {
|
||||
return { score: 35, isHighRisk: false };
|
||||
}
|
||||
|
||||
// Default low risk for unclassified countries
|
||||
return { score: 10, isHighRisk: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the source appears to be a datacenter or hosting provider
|
||||
* @param geoData - Geographic data
|
||||
* @returns True if likely datacenter source
|
||||
*/
|
||||
function checkDatacenterSource(geoData: GeoData): boolean {
|
||||
// Check explicit datacenter flag
|
||||
if (geoData.datacenter === true) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check known datacenter ASNs
|
||||
if (geoData.asn && GEO_CONFIG.datacenterASNs.includes(geoData.asn)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check ISP name for datacenter indicators
|
||||
if (geoData.isp && typeof geoData.isp === 'string') {
|
||||
const ispLower = geoData.isp.toLowerCase();
|
||||
const datacenterIndicators = [
|
||||
'amazon', 'aws', 'google', 'microsoft', 'azure', 'cloudflare',
|
||||
'digitalocean', 'linode', 'vultr', 'hetzner', 'ovh',
|
||||
'datacenter', 'hosting', 'cloud', 'server', 'vps'
|
||||
];
|
||||
|
||||
return datacenterIndicators.some(indicator => ispLower.includes(indicator));
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts and validates location coordinates
|
||||
* @param geoData - Geographic data
|
||||
* @returns Valid location or null
|
||||
*/
|
||||
function extractLocation(geoData: GeoData): GeoLocation | null {
|
||||
const { latitude, longitude } = geoData;
|
||||
|
||||
// Check if coordinates are present and numeric
|
||||
if (typeof latitude !== 'number' || typeof longitude !== 'number') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Validate coordinate ranges
|
||||
if (!isValidCoordinate(latitude, longitude)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Round to appropriate precision
|
||||
const precision = Math.pow(10, GEO_CONFIG.coordinatePrecision);
|
||||
|
||||
return {
|
||||
lat: Math.round(latitude * precision) / precision,
|
||||
lon: Math.round(longitude * precision) / precision
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates overall geographic risk score
|
||||
* @param countryRisk - Country risk score
|
||||
* @param isDatacenter - Whether source is datacenter
|
||||
* @param geoData - Additional geographic data
|
||||
* @returns Composite geographic risk score
|
||||
*/
|
||||
function calculateGeoScore(countryRisk: number, isDatacenter: boolean, geoData: GeoData): number {
|
||||
let score = countryRisk * 0.7; // Country risk is primary factor
|
||||
|
||||
// Datacenter sources get moderate risk boost
|
||||
if (isDatacenter) {
|
||||
score += 15;
|
||||
}
|
||||
|
||||
// ASN-based adjustments
|
||||
if (geoData.asn) {
|
||||
// Known malicious ASNs (simplified list)
|
||||
const maliciousASNs = [4134, 4837, 9808]; // Example ASNs
|
||||
if (maliciousASNs.includes(geoData.asn)) {
|
||||
score += 20;
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure score stays within valid range
|
||||
return Math.max(0, Math.min(100, score));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates geographic coordinates
|
||||
* @param lat - Latitude
|
||||
* @param lon - Longitude
|
||||
* @returns True if coordinates are valid
|
||||
*/
|
||||
function isValidCoordinate(lat: number, lon: number): boolean {
|
||||
return lat >= -GEO_CONFIG.maxValidLatitude &&
|
||||
lat <= GEO_CONFIG.maxValidLatitude &&
|
||||
lon >= -GEO_CONFIG.maxValidLongitude &&
|
||||
lon <= GEO_CONFIG.maxValidLongitude &&
|
||||
!isNaN(lat) && !isNaN(lon) &&
|
||||
isFinite(lat) && isFinite(lon);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates location object
|
||||
* @param location - Location to validate
|
||||
* @returns True if location is valid
|
||||
*/
|
||||
function isValidLocation(location: GeoLocation | null): location is GeoLocation {
|
||||
return location !== null &&
|
||||
typeof location === 'object' &&
|
||||
typeof location.lat === 'number' &&
|
||||
typeof location.lon === 'number' &&
|
||||
isValidCoordinate(location.lat, location.lon);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines accuracy of distance calculation based on coordinate precision
|
||||
* @param loc1 - First location
|
||||
* @param loc2 - Second location
|
||||
* @returns Accuracy classification
|
||||
*/
|
||||
function determineCalculationAccuracy(loc1: GeoLocation, loc2: GeoLocation): 'high' | 'medium' | 'low' {
|
||||
// Calculate decimal places in coordinates
|
||||
const lat1Decimals = countDecimalPlaces(loc1.lat);
|
||||
const lon1Decimals = countDecimalPlaces(loc1.lon);
|
||||
const lat2Decimals = countDecimalPlaces(loc2.lat);
|
||||
const lon2Decimals = countDecimalPlaces(loc2.lon);
|
||||
|
||||
const minPrecision = Math.min(lat1Decimals, lon1Decimals, lat2Decimals, lon2Decimals);
|
||||
|
||||
if (minPrecision >= 4) return 'high'; // ~11m accuracy
|
||||
if (minPrecision >= 2) return 'medium'; // ~1.1km accuracy
|
||||
return 'low'; // ~111km accuracy
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts decimal places in a number
|
||||
* @param num - Number to analyze
|
||||
* @returns Number of decimal places
|
||||
*/
|
||||
function countDecimalPlaces(num: number): number {
|
||||
if (Math.floor(num) === num) return 0;
|
||||
const str = num.toString();
|
||||
const decimalIndex = str.indexOf('.');
|
||||
return decimalIndex >= 0 ? str.length - decimalIndex - 1 : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts degrees to radians
|
||||
* @param degrees - Angle in degrees
|
||||
* @returns Angle in radians
|
||||
*/
|
||||
function toRadians(degrees: number): number {
|
||||
return degrees * (Math.PI / 180);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// EXPORT TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
export type {
|
||||
GeoData,
|
||||
GeoFeatures,
|
||||
GeoLocation,
|
||||
DistanceCalculationResult,
|
||||
CountryRiskProfile,
|
||||
GeoAnalysisConfig
|
||||
};
|
||||
|
|
@ -1,349 +0,0 @@
|
|||
// =============================================================================
|
||||
// HEADER ANALYSIS - SECURE TYPESCRIPT VERSION
|
||||
// =============================================================================
|
||||
// Comprehensive HTTP header security analysis with injection prevention
|
||||
// Handles completely user-controlled header data with zero trust validation
|
||||
|
||||
import { checkUAConsistency } from './user-agent.js';
|
||||
import { detectEncodingLevels } from './patterns.js';
|
||||
|
||||
// Type definitions for secure header analysis
|
||||
export interface HeaderFeatures {
|
||||
readonly headerCount: number;
|
||||
readonly hasStandardHeaders: boolean;
|
||||
readonly headerAnomalies: number;
|
||||
readonly suspiciousHeaders: readonly string[];
|
||||
readonly missingExpectedHeaders: readonly string[];
|
||||
readonly riskScore: number;
|
||||
readonly validationErrors: readonly string[];
|
||||
}
|
||||
|
||||
interface HeaderData {
|
||||
readonly name: string;
|
||||
readonly value: string;
|
||||
readonly normalizedName: string;
|
||||
}
|
||||
|
||||
// Security constants for header validation
|
||||
const MAX_HEADER_COUNT = 100; // Reasonable limit for headers
|
||||
const MAX_HEADER_NAME_LENGTH = 128; // HTTP spec recommends this
|
||||
const MAX_HEADER_VALUE_LENGTH = 8192; // 8KB per header value
|
||||
const MAX_TOTAL_HEADER_SIZE = 32768; // 32KB total headers
|
||||
const MAX_SUSPICIOUS_HEADERS = 20; // Limit suspicious header collection
|
||||
const MAX_VALIDATION_ERRORS = 15; // Prevent memory exhaustion
|
||||
|
||||
// Expected standard headers for legitimate requests
|
||||
const EXPECTED_HEADERS = ['host', 'user-agent', 'accept'] as const;
|
||||
|
||||
// Suspicious header patterns that indicate attacks or spoofing
|
||||
const SUSPICIOUS_PATTERNS = [
|
||||
'x-forwarded-for-for', // Double forwarding attempt
|
||||
'x-originating-ip', // IP spoofing attempt
|
||||
'x-remote-ip', // Remote IP manipulation
|
||||
'x-remote-addr', // Address manipulation
|
||||
'x-proxy-id', // Proxy identification spoofing
|
||||
'via-via', // Double via header
|
||||
'x-cluster-client-ip', // Cluster IP spoofing
|
||||
'x-forwarded-proto-proto', // Protocol spoofing
|
||||
'x-injection-test', // Obvious injection test
|
||||
'x-hack', // Obvious attack attempt
|
||||
'x-exploit' // Exploitation attempt
|
||||
] as const;
|
||||
|
||||
// Headers that should be checked for consistency in forwarding scenarios
|
||||
const FORWARDED_HEADERS = ['x-forwarded-for', 'x-real-ip', 'x-forwarded-host', 'cf-connecting-ip'] as const;
|
||||
|
||||
// Input validation functions with zero trust approach
|
||||
function validateHeaders(headers: unknown): Record<string, unknown> {
|
||||
if (!headers || typeof headers !== 'object') {
|
||||
throw new Error('Headers must be an object');
|
||||
}
|
||||
|
||||
return headers as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function validateHeaderName(name: unknown): string {
|
||||
if (typeof name !== 'string') {
|
||||
throw new Error('Header name must be a string');
|
||||
}
|
||||
|
||||
if (name.length === 0 || name.length > MAX_HEADER_NAME_LENGTH) {
|
||||
throw new Error(`Header name length must be between 1 and ${MAX_HEADER_NAME_LENGTH} characters`);
|
||||
}
|
||||
|
||||
// Check for control characters and invalid header name chars
|
||||
if (/[\x00-\x1f\x7f-\x9f\s:]/i.test(name)) {
|
||||
throw new Error('Header name contains invalid characters');
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
function validateHeaderValue(value: unknown): string {
|
||||
if (value === null || value === undefined) {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (typeof value !== 'string') {
|
||||
// Convert to string but validate the result
|
||||
const stringValue = String(value);
|
||||
if (stringValue.length > MAX_HEADER_VALUE_LENGTH) {
|
||||
throw new Error(`Header value too long: ${stringValue.length} > ${MAX_HEADER_VALUE_LENGTH}`);
|
||||
}
|
||||
return stringValue;
|
||||
}
|
||||
|
||||
if (value.length > MAX_HEADER_VALUE_LENGTH) {
|
||||
throw new Error(`Header value too long: ${value.length} > ${MAX_HEADER_VALUE_LENGTH}`);
|
||||
}
|
||||
|
||||
// Check for obvious injection attempts
|
||||
if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/i.test(value)) {
|
||||
throw new Error('Header value contains control characters');
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function extractSafeHeaderEntries(headers: unknown): HeaderData[] {
|
||||
const validatedHeaders = validateHeaders(headers);
|
||||
const entries: HeaderData[] = [];
|
||||
let totalSize = 0;
|
||||
|
||||
try {
|
||||
// Handle different header object types safely
|
||||
let headerEntries: [string, unknown][];
|
||||
|
||||
if (typeof (validatedHeaders as any).entries === 'function') {
|
||||
// Headers object with entries() method (like fetch Headers)
|
||||
headerEntries = Array.from((validatedHeaders as any).entries());
|
||||
} else {
|
||||
// Plain object (like Express headers)
|
||||
headerEntries = Object.entries(validatedHeaders);
|
||||
}
|
||||
|
||||
// Limit the number of headers to prevent DoS
|
||||
if (headerEntries.length > MAX_HEADER_COUNT) {
|
||||
headerEntries = headerEntries.slice(0, MAX_HEADER_COUNT);
|
||||
}
|
||||
|
||||
for (const [rawName, rawValue] of headerEntries) {
|
||||
try {
|
||||
const name = validateHeaderName(rawName);
|
||||
const value = validateHeaderValue(rawValue);
|
||||
const normalizedName = name.toLowerCase();
|
||||
|
||||
// Check total header size to prevent memory exhaustion
|
||||
totalSize += name.length + value.length;
|
||||
if (totalSize > MAX_TOTAL_HEADER_SIZE) {
|
||||
break; // Stop processing if headers too large
|
||||
}
|
||||
|
||||
entries.push({
|
||||
name,
|
||||
value,
|
||||
normalizedName
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
// Skip invalid headers but continue processing
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// If extraction fails, return empty array
|
||||
return [];
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
// Safe header access functions with type checking
|
||||
export function hasHeader(headers: unknown, name: string): boolean {
|
||||
try {
|
||||
const validatedHeaders = validateHeaders(headers);
|
||||
const lowerName = name.toLowerCase();
|
||||
|
||||
if (typeof (validatedHeaders as any).has === 'function') {
|
||||
// Headers object with has() method
|
||||
return (validatedHeaders as any).has(name) || (validatedHeaders as any).has(lowerName);
|
||||
}
|
||||
|
||||
// Plain object - check both cases
|
||||
return (validatedHeaders as any)[name] !== undefined ||
|
||||
(validatedHeaders as any)[lowerName] !== undefined;
|
||||
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function getHeader(headers: unknown, name: string): string | null {
|
||||
try {
|
||||
const validatedHeaders = validateHeaders(headers);
|
||||
const lowerName = name.toLowerCase();
|
||||
|
||||
if (typeof (validatedHeaders as any).get === 'function') {
|
||||
// Headers object with get() method
|
||||
const value = (validatedHeaders as any).get(name) || (validatedHeaders as any).get(lowerName);
|
||||
return value ? validateHeaderValue(value) : null;
|
||||
}
|
||||
|
||||
// Plain object - check both cases
|
||||
const value = (validatedHeaders as any)[name] || (validatedHeaders as any)[lowerName];
|
||||
return value ? validateHeaderValue(value) : null;
|
||||
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getHeaderEntries(headers: unknown): readonly HeaderData[] {
|
||||
return extractSafeHeaderEntries(headers);
|
||||
}
|
||||
|
||||
// Enhanced header spoofing detection with validation
|
||||
export function detectHeaderSpoofing(headers: unknown): boolean {
|
||||
try {
|
||||
const forwardedValues = new Set<string>();
|
||||
|
||||
for (const headerName of FORWARDED_HEADERS) {
|
||||
const value = getHeader(headers, headerName);
|
||||
if (value && value.length > 0) {
|
||||
// Normalize the value for comparison
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (normalized.length > 0) {
|
||||
forwardedValues.add(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Multiple different forwarded values indicate potential spoofing
|
||||
// But allow for legitimate proxy chains (limit to reasonable number)
|
||||
return forwardedValues.size > 3;
|
||||
|
||||
} catch (error) {
|
||||
// If analysis fails, assume no spoofing but log the issue
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Main header analysis function with comprehensive security
|
||||
export function extractHeaderFeatures(headers: unknown): HeaderFeatures {
|
||||
const validationErrors: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Initialize safe default values
|
||||
let headerCount = 0;
|
||||
let hasStandardHeaders = true;
|
||||
let headerAnomalies = 0;
|
||||
const suspiciousHeaders: string[] = [];
|
||||
const missingExpectedHeaders: string[] = [];
|
||||
|
||||
try {
|
||||
// Extract headers safely with validation
|
||||
const headerEntries = extractSafeHeaderEntries(headers);
|
||||
headerCount = headerEntries.length;
|
||||
|
||||
// Check for reasonable header count
|
||||
if (headerCount === 0) {
|
||||
validationErrors.push('no_headers_found');
|
||||
riskScore += 30; // Medium risk for missing headers
|
||||
} else if (headerCount > 50) {
|
||||
validationErrors.push('excessive_header_count');
|
||||
riskScore += 20; // Low-medium risk for too many headers
|
||||
}
|
||||
|
||||
// Check for standard browser headers
|
||||
for (const expectedHeader of EXPECTED_HEADERS) {
|
||||
if (!hasHeader(headers, expectedHeader)) {
|
||||
hasStandardHeaders = false;
|
||||
missingExpectedHeaders.push(expectedHeader);
|
||||
headerAnomalies++;
|
||||
riskScore += 15; // Low risk per missing header
|
||||
}
|
||||
}
|
||||
|
||||
// Check for suspicious header patterns
|
||||
for (const headerData of headerEntries) {
|
||||
const { name, value, normalizedName } = headerData;
|
||||
|
||||
// Check suspicious patterns in header names
|
||||
for (const pattern of SUSPICIOUS_PATTERNS) {
|
||||
if (normalizedName.includes(pattern)) {
|
||||
suspiciousHeaders.push(name);
|
||||
headerAnomalies++;
|
||||
riskScore += 25; // Medium risk for suspicious headers
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for encoding attacks in header values
|
||||
try {
|
||||
const encodingLevels = detectEncodingLevels(value);
|
||||
if (encodingLevels > 2) {
|
||||
headerAnomalies++;
|
||||
riskScore += 20; // Medium risk for encoding attacks
|
||||
validationErrors.push('excessive_encoding_detected');
|
||||
}
|
||||
} catch (error) {
|
||||
validationErrors.push('encoding_analysis_failed');
|
||||
riskScore += 10; // Small penalty for analysis failure
|
||||
}
|
||||
|
||||
// Limit suspicious headers collection
|
||||
if (suspiciousHeaders.length >= MAX_SUSPICIOUS_HEADERS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for header spoofing
|
||||
try {
|
||||
if (detectHeaderSpoofing(headers)) {
|
||||
headerAnomalies += 2;
|
||||
riskScore += 35; // High risk for spoofing attempts
|
||||
validationErrors.push('header_spoofing_detected');
|
||||
}
|
||||
} catch (error) {
|
||||
validationErrors.push('spoofing_detection_failed');
|
||||
riskScore += 10;
|
||||
}
|
||||
|
||||
// Check User-Agent consistency with Client Hints
|
||||
try {
|
||||
const userAgent = getHeader(headers, 'user-agent');
|
||||
const secChUa = getHeader(headers, 'sec-ch-ua');
|
||||
|
||||
if (userAgent && secChUa && !checkUAConsistency(userAgent, secChUa)) {
|
||||
headerAnomalies++;
|
||||
riskScore += 25; // Medium risk for UA inconsistency
|
||||
validationErrors.push('user_agent_inconsistency');
|
||||
}
|
||||
} catch (error) {
|
||||
validationErrors.push('ua_consistency_check_failed');
|
||||
riskScore += 5; // Small penalty
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// Critical validation failure
|
||||
validationErrors.push('header_validation_failed');
|
||||
riskScore = 100; // Maximum risk for validation failure
|
||||
headerAnomalies = 999; // Indicate severe anomaly
|
||||
}
|
||||
|
||||
// Cap risk score and limit validation errors
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
const limitedErrors = validationErrors.slice(0, MAX_VALIDATION_ERRORS);
|
||||
const limitedSuspiciousHeaders = suspiciousHeaders.slice(0, MAX_SUSPICIOUS_HEADERS);
|
||||
|
||||
return {
|
||||
headerCount,
|
||||
hasStandardHeaders,
|
||||
headerAnomalies,
|
||||
suspiciousHeaders: limitedSuspiciousHeaders,
|
||||
missingExpectedHeaders,
|
||||
riskScore: finalRiskScore,
|
||||
validationErrors: limitedErrors
|
||||
};
|
||||
}
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
// =============================================================================
|
||||
// ANALYZER EXPORTS (TypeScript)
|
||||
// =============================================================================
|
||||
// Central export hub for all threat analysis functions
|
||||
// Provides a clean interface for accessing all security analyzers
|
||||
|
||||
// =============================================================================
|
||||
// FUNCTION EXPORTS
|
||||
// =============================================================================
|
||||
|
||||
// User-Agent analysis functions
|
||||
export {
|
||||
analyzeUserAgentAdvanced,
|
||||
checkUAConsistency
|
||||
} from './user-agent.js';
|
||||
|
||||
// Geographic analysis functions
|
||||
export {
|
||||
analyzeGeoData,
|
||||
calculateDistance,
|
||||
calculateDistanceDetailed
|
||||
} from './geo.js';
|
||||
|
||||
// Header analysis functions
|
||||
export {
|
||||
extractHeaderFeatures,
|
||||
detectHeaderSpoofing,
|
||||
hasHeader,
|
||||
getHeader,
|
||||
getHeaderEntries
|
||||
} from './headers.js';
|
||||
|
||||
// Pattern analysis functions
|
||||
export {
|
||||
detectAutomation,
|
||||
calculateEntropy,
|
||||
detectEncodingLevels
|
||||
} from './patterns.js';
|
||||
|
||||
// =============================================================================
|
||||
// TYPE EXPORTS
|
||||
// =============================================================================
|
||||
// Re-export available types from converted TypeScript modules
|
||||
|
||||
// User-Agent types (available types only)
|
||||
export type {
|
||||
UserAgentFeatures,
|
||||
UserAgentConsistencyResult
|
||||
} from './user-agent.js';
|
||||
|
||||
// Geographic types
|
||||
export type {
|
||||
GeoData,
|
||||
GeoFeatures,
|
||||
GeoLocation,
|
||||
DistanceCalculationResult,
|
||||
CountryRiskProfile,
|
||||
GeoAnalysisConfig
|
||||
} from './geo.js';
|
||||
|
||||
// Header types (available types only)
|
||||
export type {
|
||||
HeaderFeatures
|
||||
} from './headers.js';
|
||||
|
||||
// =============================================================================
|
||||
// UTILITY FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Gets a list of all available analyzer categories
|
||||
* @returns Array of analyzer category names
|
||||
*/
|
||||
export function getAnalyzerCategories(): readonly string[] {
|
||||
return ['userAgent', 'geo', 'headers', 'patterns'] as const;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the available analyzer functions by category
|
||||
* @returns Object with arrays of function names by category
|
||||
*/
|
||||
export function getAnalyzersByCategory(): Record<string, readonly string[]> {
|
||||
return {
|
||||
userAgent: ['analyzeUserAgentAdvanced', 'checkUAConsistency'],
|
||||
geo: ['analyzeGeoData', 'calculateDistance', 'calculateDistanceDetailed'],
|
||||
headers: ['extractHeaderFeatures', 'detectHeaderSpoofing', 'hasHeader', 'getHeader', 'getHeaderEntries'],
|
||||
patterns: ['detectAutomation', 'calculateEntropy', 'detectEncodingLevels']
|
||||
} as const;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that all required analyzers are available
|
||||
* @returns True if all analyzers are properly loaded
|
||||
*/
|
||||
export function validateAnalyzers(): boolean {
|
||||
try {
|
||||
// Basic validation - extensible for future enhancements
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Analyzer validation failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
// =============================================================================
|
||||
// METRIC NORMALIZATION UTILITIES
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Normalizes a metric value to a 0-1 range based on min/max bounds
|
||||
* @param value - The value to normalize
|
||||
* @param min - The minimum expected value
|
||||
* @param max - The maximum expected value
|
||||
* @returns Normalized value between 0 and 1
|
||||
*/
|
||||
export function normalizeMetricValue(value: number, min: number, max: number): number {
|
||||
if (typeof value !== 'number' || isNaN(value)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (typeof min !== 'number' || typeof max !== 'number' || isNaN(min) || isNaN(max)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (max <= min) {
|
||||
return value >= max ? 1 : 0;
|
||||
}
|
||||
|
||||
// Clamp value to bounds and normalize
|
||||
const clampedValue = Math.max(min, Math.min(max, value));
|
||||
return (clampedValue - min) / (max - min);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a score using sigmoid function for smoother transitions
|
||||
* @param value - The value to normalize
|
||||
* @param midpoint - The midpoint where the function equals 0.5
|
||||
* @param steepness - How steep the transition is (higher = steeper)
|
||||
* @returns Normalized value between 0 and 1
|
||||
*/
|
||||
export function sigmoidNormalize(value: number, midpoint: number = 50, steepness: number = 0.1): number {
|
||||
if (typeof value !== 'number' || isNaN(value)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1 / (1 + Math.exp(-steepness * (value - midpoint)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a confidence score based on multiple factors
|
||||
* @param primaryScore - The primary score (0-100)
|
||||
* @param evidenceCount - Number of pieces of evidence
|
||||
* @param timeRecency - How recent the evidence is (0-1, 1 = very recent)
|
||||
* @returns Normalized confidence score (0-1)
|
||||
*/
|
||||
export function normalizeConfidence(primaryScore: number, evidenceCount: number, timeRecency: number = 1): number {
|
||||
const normalizedPrimary = normalizeMetricValue(primaryScore, 0, 100);
|
||||
const evidenceBonus = Math.min(evidenceCount * 0.1, 0.3); // Max 30% bonus
|
||||
const recencyFactor = Math.max(0.5, timeRecency); // Minimum 50% even for old data
|
||||
|
||||
return Math.min(1, (normalizedPrimary + evidenceBonus) * recencyFactor);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies logarithmic normalization for values that grow exponentially
|
||||
* @param value - The value to normalize
|
||||
* @param maxValue - The maximum expected value
|
||||
* @returns Normalized value between 0 and 1
|
||||
*/
|
||||
export function logNormalize(value: number, maxValue: number = 1000): number {
|
||||
if (typeof value !== 'number' || isNaN(value) || value <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (typeof maxValue !== 'number' || isNaN(maxValue) || maxValue <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const logValue = Math.log(value + 1);
|
||||
const logMax = Math.log(maxValue + 1);
|
||||
|
||||
return Math.min(1, logValue / logMax);
|
||||
}
|
||||
|
|
@ -1,560 +0,0 @@
|
|||
// =============================================================================
|
||||
// PATTERN ANALYSIS (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface RequestHistoryEntry {
|
||||
readonly timestamp: number;
|
||||
readonly method?: string;
|
||||
readonly path?: string;
|
||||
readonly userAgent?: string;
|
||||
readonly responseTime?: number;
|
||||
readonly statusCode?: number;
|
||||
}
|
||||
|
||||
interface AutomationAnalysis {
|
||||
readonly score: number;
|
||||
readonly confidence: number;
|
||||
readonly indicators: readonly string[];
|
||||
readonly statistics: RequestStatistics;
|
||||
}
|
||||
|
||||
interface RequestStatistics {
|
||||
readonly avgInterval: number;
|
||||
readonly stdDev: number;
|
||||
readonly coefficientOfVariation: number;
|
||||
readonly totalRequests: number;
|
||||
readonly timeSpan: number;
|
||||
}
|
||||
|
||||
interface EntropyAnalysis {
|
||||
readonly entropy: number;
|
||||
readonly classification: 'very_low' | 'low' | 'medium' | 'high' | 'very_high';
|
||||
readonly randomness: number;
|
||||
readonly characterDistribution: Record<string, number>;
|
||||
}
|
||||
|
||||
interface EncodingAnalysis {
|
||||
readonly levels: number;
|
||||
readonly originalString: string;
|
||||
readonly decodedString: string;
|
||||
readonly encodingTypes: readonly string[];
|
||||
readonly isSuspicious: boolean;
|
||||
}
|
||||
|
||||
interface PatternAnalysisConfig {
|
||||
readonly automationThresholds: {
|
||||
readonly highConfidence: number;
|
||||
readonly mediumConfidence: number;
|
||||
readonly lowConfidence: number;
|
||||
};
|
||||
readonly intervalThresholds: {
|
||||
readonly veryFast: number;
|
||||
readonly fast: number;
|
||||
readonly normal: number;
|
||||
};
|
||||
readonly entropyThresholds: {
|
||||
readonly veryLow: number;
|
||||
readonly low: number;
|
||||
readonly medium: number;
|
||||
readonly high: number;
|
||||
};
|
||||
readonly maxEncodingLevels: number;
|
||||
readonly minHistorySize: number;
|
||||
}
|
||||
|
||||
// Configuration constants
|
||||
const PATTERN_CONFIG: PatternAnalysisConfig = {
|
||||
automationThresholds: {
|
||||
highConfidence: 0.1, // CV < 0.1 = high automation confidence
|
||||
mediumConfidence: 0.2, // CV < 0.2 = medium automation confidence
|
||||
lowConfidence: 0.3 // CV < 0.3 = low automation confidence
|
||||
},
|
||||
intervalThresholds: {
|
||||
veryFast: 1000, // < 1 second intervals
|
||||
fast: 2000, // < 2 second intervals
|
||||
normal: 5000 // < 5 second intervals
|
||||
},
|
||||
entropyThresholds: {
|
||||
veryLow: 1.0, // Very predictable
|
||||
low: 2.0, // Low randomness
|
||||
medium: 3.5, // Medium randomness
|
||||
high: 4.5 // High randomness
|
||||
},
|
||||
maxEncodingLevels: 5, // Maximum encoding levels to check
|
||||
minHistorySize: 5 // Minimum history entries for automation detection
|
||||
} as const;
|
||||
|
||||
// =============================================================================
|
||||
// AUTOMATION DETECTION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Detects automation patterns in request history
|
||||
* Analyzes timing intervals and consistency to identify bot-like behavior
|
||||
*
|
||||
* @param history - Array of request history entries
|
||||
* @returns Automation detection score (0-1) where 1 = highly likely automation
|
||||
*/
|
||||
export function detectAutomation(history: readonly RequestHistoryEntry[]): number {
|
||||
// Input validation
|
||||
if (!Array.isArray(history) || history.length < PATTERN_CONFIG.minHistorySize) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
// Validate history entries
|
||||
const validHistory = history.filter(entry =>
|
||||
entry &&
|
||||
typeof entry.timestamp === 'number' &&
|
||||
entry.timestamp > 0 &&
|
||||
isFinite(entry.timestamp)
|
||||
);
|
||||
|
||||
if (validHistory.length < PATTERN_CONFIG.minHistorySize) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Calculate request intervals
|
||||
const intervals = calculateIntervals(validHistory);
|
||||
|
||||
if (intervals.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Calculate statistical measures
|
||||
const statistics = calculateStatistics(intervals);
|
||||
|
||||
// Determine automation score based on coefficient of variation and intervals
|
||||
return calculateAutomationScore(statistics);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to detect automation patterns:', error.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced automation detection with detailed analysis
|
||||
* @param history - Request history entries
|
||||
* @returns Detailed automation analysis
|
||||
*/
|
||||
export function detectAutomationAdvanced(history: readonly RequestHistoryEntry[]): AutomationAnalysis {
|
||||
const score = detectAutomation(history);
|
||||
|
||||
if (score === 0 || !Array.isArray(history) || history.length < PATTERN_CONFIG.minHistorySize) {
|
||||
return {
|
||||
score: 0,
|
||||
confidence: 0,
|
||||
indicators: [],
|
||||
statistics: {
|
||||
avgInterval: 0,
|
||||
stdDev: 0,
|
||||
coefficientOfVariation: 0,
|
||||
totalRequests: history?.length || 0,
|
||||
timeSpan: 0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const validHistory = history.filter(entry =>
|
||||
entry && typeof entry.timestamp === 'number' && entry.timestamp > 0
|
||||
);
|
||||
|
||||
const intervals = calculateIntervals(validHistory);
|
||||
const statistics = calculateStatistics(intervals);
|
||||
const indicators = identifyAutomationIndicators(statistics, validHistory);
|
||||
const confidence = calculateConfidence(statistics, indicators.length);
|
||||
|
||||
return {
|
||||
score,
|
||||
confidence,
|
||||
indicators,
|
||||
statistics: {
|
||||
...statistics,
|
||||
totalRequests: validHistory.length,
|
||||
timeSpan: validHistory.length > 1
|
||||
? validHistory[validHistory.length - 1].timestamp - validHistory[0].timestamp
|
||||
: 0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// ENTROPY CALCULATION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Calculates Shannon entropy of a string to measure randomness
|
||||
* Higher entropy indicates more randomness, lower entropy indicates patterns
|
||||
*
|
||||
* @param str - String to analyze
|
||||
* @returns Entropy value (bits)
|
||||
*/
|
||||
export function calculateEntropy(str: string): number {
|
||||
// Input validation
|
||||
if (!str || typeof str !== 'string' || str.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
// Count character frequencies
|
||||
const charCounts: Record<string, number> = {};
|
||||
for (const char of str) {
|
||||
charCounts[char] = (charCounts[char] || 0) + 1;
|
||||
}
|
||||
|
||||
// Calculate Shannon entropy
|
||||
let entropy = 0;
|
||||
const len = str.length;
|
||||
|
||||
for (const count of Object.values(charCounts)) {
|
||||
if (count > 0) {
|
||||
const probability = count / len;
|
||||
entropy -= probability * Math.log2(probability);
|
||||
}
|
||||
}
|
||||
|
||||
// Round to 6 decimal places for consistency
|
||||
return Math.round(entropy * 1000000) / 1000000;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to calculate entropy:', error.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced entropy analysis with classification
|
||||
* @param str - String to analyze
|
||||
* @returns Detailed entropy analysis
|
||||
*/
|
||||
export function calculateEntropyAdvanced(str: string): EntropyAnalysis {
|
||||
const entropy = calculateEntropy(str);
|
||||
|
||||
if (!str || typeof str !== 'string') {
|
||||
return {
|
||||
entropy: 0,
|
||||
classification: 'very_low',
|
||||
randomness: 0,
|
||||
characterDistribution: {}
|
||||
};
|
||||
}
|
||||
|
||||
// Count character frequencies for distribution analysis
|
||||
const charCounts: Record<string, number> = {};
|
||||
for (const char of str) {
|
||||
charCounts[char] = (charCounts[char] || 0) + 1;
|
||||
}
|
||||
|
||||
// Classify entropy level
|
||||
const classification = classifyEntropy(entropy);
|
||||
|
||||
// Calculate randomness percentage (0-100)
|
||||
const maxEntropy = Math.log2(Math.min(str.length, 256)); // Max possible entropy
|
||||
const randomness = maxEntropy > 0 ? Math.min(100, (entropy / maxEntropy) * 100) : 0;
|
||||
|
||||
return {
|
||||
entropy,
|
||||
classification,
|
||||
randomness: Math.round(randomness * 100) / 100,
|
||||
characterDistribution: charCounts
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// ENCODING LEVEL DETECTION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Detects how many levels of URL encoding are applied to a string
|
||||
* Multiple encoding levels can indicate obfuscation attempts
|
||||
*
|
||||
* @param str - String to analyze
|
||||
* @returns Number of encoding levels detected
|
||||
*/
|
||||
export function detectEncodingLevels(str: string): number {
|
||||
// Input validation
|
||||
if (!str || typeof str !== 'string') {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
let levels = 0;
|
||||
let current = str;
|
||||
let previous = '';
|
||||
|
||||
// Iteratively decode until no more changes or max levels reached
|
||||
while (current !== previous && levels < PATTERN_CONFIG.maxEncodingLevels) {
|
||||
previous = current;
|
||||
|
||||
try {
|
||||
const decoded = decodeURIComponent(current);
|
||||
if (decoded !== current && isValidDecoding(decoded)) {
|
||||
current = decoded;
|
||||
levels++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} catch (decodeError) {
|
||||
// Stop if decoding fails
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return levels;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to detect encoding levels:', error.message);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced encoding analysis with detailed results
|
||||
* @param str - String to analyze
|
||||
* @returns Detailed encoding analysis
|
||||
*/
|
||||
export function detectEncodingLevelsAdvanced(str: string): EncodingAnalysis {
|
||||
if (!str || typeof str !== 'string') {
|
||||
return {
|
||||
levels: 0,
|
||||
originalString: '',
|
||||
decodedString: '',
|
||||
encodingTypes: [],
|
||||
isSuspicious: false
|
||||
};
|
||||
}
|
||||
|
||||
const levels = detectEncodingLevels(str);
|
||||
let current = str;
|
||||
let previous = '';
|
||||
const encodingTypes: string[] = [];
|
||||
|
||||
// Track encoding types detected
|
||||
for (let i = 0; i < levels; i++) {
|
||||
previous = current;
|
||||
try {
|
||||
current = decodeURIComponent(current);
|
||||
if (current !== previous) {
|
||||
encodingTypes.push('uri_component');
|
||||
}
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Determine if encoding pattern is suspicious
|
||||
const isSuspicious = levels > 2 || (levels > 1 && str.length > 100);
|
||||
|
||||
return {
|
||||
levels,
|
||||
originalString: str,
|
||||
decodedString: current,
|
||||
encodingTypes,
|
||||
isSuspicious
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// HELPER FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Calculates intervals between consecutive requests
|
||||
* @param history - Sorted request history
|
||||
* @returns Array of intervals in milliseconds
|
||||
*/
|
||||
function calculateIntervals(history: readonly RequestHistoryEntry[]): number[] {
|
||||
const intervals: number[] = [];
|
||||
|
||||
for (let i = 1; i < history.length; i++) {
|
||||
const current = history[i];
|
||||
const previous = history[i - 1];
|
||||
|
||||
if (current && previous &&
|
||||
typeof current.timestamp === 'number' &&
|
||||
typeof previous.timestamp === 'number') {
|
||||
const interval = current.timestamp - previous.timestamp;
|
||||
if (interval > 0 && isFinite(interval)) {
|
||||
intervals.push(interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return intervals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates statistical measures for request intervals
|
||||
* @param intervals - Array of time intervals
|
||||
* @returns Statistical measures
|
||||
*/
|
||||
function calculateStatistics(intervals: readonly number[]): RequestStatistics {
|
||||
if (intervals.length === 0) {
|
||||
return {
|
||||
avgInterval: 0,
|
||||
stdDev: 0,
|
||||
coefficientOfVariation: 0,
|
||||
totalRequests: 0,
|
||||
timeSpan: 0
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate average interval
|
||||
const avgInterval = intervals.reduce((sum, interval) => sum + interval, 0) / intervals.length;
|
||||
|
||||
// Calculate standard deviation
|
||||
const variance = intervals.reduce((acc, interval) =>
|
||||
acc + Math.pow(interval - avgInterval, 2), 0) / intervals.length;
|
||||
const stdDev = Math.sqrt(variance);
|
||||
|
||||
// Calculate coefficient of variation (CV)
|
||||
const coefficientOfVariation = avgInterval > 0 ? stdDev / avgInterval : 0;
|
||||
|
||||
return {
|
||||
avgInterval: Math.round(avgInterval * 100) / 100,
|
||||
stdDev: Math.round(stdDev * 100) / 100,
|
||||
coefficientOfVariation: Math.round(coefficientOfVariation * 1000) / 1000,
|
||||
totalRequests: intervals.length + 1, // +1 because intervals = requests - 1
|
||||
timeSpan: intervals.reduce((sum, interval) => sum + interval, 0)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates automation score based on statistical measures
|
||||
* @param statistics - Request interval statistics
|
||||
* @returns Automation score (0-1)
|
||||
*/
|
||||
function calculateAutomationScore(statistics: RequestStatistics): number {
|
||||
const { coefficientOfVariation, avgInterval } = statistics;
|
||||
|
||||
// Low CV with fast intervals indicates high automation probability
|
||||
if (coefficientOfVariation < PATTERN_CONFIG.automationThresholds.highConfidence &&
|
||||
avgInterval < PATTERN_CONFIG.intervalThresholds.veryFast) {
|
||||
return 0.9;
|
||||
}
|
||||
|
||||
if (coefficientOfVariation < PATTERN_CONFIG.automationThresholds.mediumConfidence &&
|
||||
avgInterval < PATTERN_CONFIG.intervalThresholds.fast) {
|
||||
return 0.7;
|
||||
}
|
||||
|
||||
if (coefficientOfVariation < PATTERN_CONFIG.automationThresholds.lowConfidence &&
|
||||
avgInterval < PATTERN_CONFIG.intervalThresholds.normal) {
|
||||
return 0.5;
|
||||
}
|
||||
|
||||
// Additional scoring for very consistent patterns regardless of speed
|
||||
if (coefficientOfVariation < 0.05) {
|
||||
return 0.6; // Very consistent timing is suspicious
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Identifies specific automation indicators
|
||||
* @param statistics - Request statistics
|
||||
* @param history - Request history
|
||||
* @returns Array of automation indicators
|
||||
*/
|
||||
function identifyAutomationIndicators(
|
||||
statistics: RequestStatistics,
|
||||
history: readonly RequestHistoryEntry[]
|
||||
): string[] {
|
||||
const indicators: string[] = [];
|
||||
|
||||
if (statistics.coefficientOfVariation < 0.05) {
|
||||
indicators.push('extremely_consistent_timing');
|
||||
}
|
||||
|
||||
if (statistics.avgInterval < 500) {
|
||||
indicators.push('very_fast_requests');
|
||||
}
|
||||
|
||||
if (statistics.totalRequests > 50 && statistics.timeSpan < 60000) {
|
||||
indicators.push('high_request_volume');
|
||||
}
|
||||
|
||||
// Check for identical user agents
|
||||
const userAgents = new Set(history.map(entry => entry.userAgent).filter(Boolean));
|
||||
if (userAgents.size === 1 && history.length > 10) {
|
||||
indicators.push('identical_user_agents');
|
||||
}
|
||||
|
||||
return indicators;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates confidence in automation detection
|
||||
* @param statistics - Request statistics
|
||||
* @param indicatorCount - Number of indicators found
|
||||
* @returns Confidence score (0-1)
|
||||
*/
|
||||
function calculateConfidence(statistics: RequestStatistics, indicatorCount: number): number {
|
||||
let confidence = 0;
|
||||
|
||||
// Base confidence from coefficient of variation
|
||||
if (statistics.coefficientOfVariation < 0.05) confidence += 0.4;
|
||||
else if (statistics.coefficientOfVariation < 0.1) confidence += 0.3;
|
||||
else if (statistics.coefficientOfVariation < 0.2) confidence += 0.2;
|
||||
|
||||
// Additional confidence from sample size
|
||||
if (statistics.totalRequests > 20) confidence += 0.2;
|
||||
else if (statistics.totalRequests > 10) confidence += 0.1;
|
||||
|
||||
// Confidence from multiple indicators
|
||||
confidence += Math.min(0.4, indicatorCount * 0.1);
|
||||
|
||||
return Math.min(1, Math.round(confidence * 100) / 100);
|
||||
}
|
||||
|
||||
/**
|
||||
* Classifies entropy level
|
||||
* @param entropy - Entropy value
|
||||
* @returns Classification level
|
||||
*/
|
||||
function classifyEntropy(entropy: number): 'very_low' | 'low' | 'medium' | 'high' | 'very_high' {
|
||||
if (entropy < PATTERN_CONFIG.entropyThresholds.veryLow) return 'very_low';
|
||||
if (entropy < PATTERN_CONFIG.entropyThresholds.low) return 'low';
|
||||
if (entropy < PATTERN_CONFIG.entropyThresholds.medium) return 'medium';
|
||||
if (entropy < PATTERN_CONFIG.entropyThresholds.high) return 'high';
|
||||
return 'very_high';
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that decoded string is reasonable
|
||||
* @param decoded - Decoded string
|
||||
* @returns True if decoding appears valid
|
||||
*/
|
||||
function isValidDecoding(decoded: string): boolean {
|
||||
// Check for common invalid decode patterns
|
||||
if (decoded.includes('\u0000') || decoded.includes('\uFFFD')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check for reasonable character distribution
|
||||
const controlChars = decoded.match(/[\x00-\x1F\x7F-\x9F]/g);
|
||||
if (controlChars && controlChars.length > decoded.length * 0.1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// EXPORT TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
export type {
|
||||
RequestHistoryEntry,
|
||||
AutomationAnalysis,
|
||||
RequestStatistics,
|
||||
EntropyAnalysis,
|
||||
EncodingAnalysis,
|
||||
PatternAnalysisConfig
|
||||
};
|
||||
|
|
@ -1,452 +0,0 @@
|
|||
// =============================================================================
|
||||
// USER AGENT ANALYSIS - SECURE TYPESCRIPT VERSION
|
||||
// =============================================================================
|
||||
// Comprehensive User-Agent string analysis with ReDoS protection and type safety
|
||||
// Handles completely user-controlled input with zero trust validation
|
||||
|
||||
import { matchAttackTools, matchSuspiciousBots } from '../pattern-matcher.js';
|
||||
import { VERIFIED_GOOD_BOTS, type BotInfo, type VerifiedGoodBots } from '../constants.js';
|
||||
|
||||
// Type definitions for user-agent analysis
|
||||
export interface UserAgentFeatures {
|
||||
readonly isAttackTool: boolean;
|
||||
readonly isMissing: boolean;
|
||||
readonly isMalformed: boolean;
|
||||
readonly isSuspiciousBot: boolean;
|
||||
readonly isVerifiedGoodBot: boolean;
|
||||
readonly botType: string | null;
|
||||
readonly anomalies: readonly string[];
|
||||
readonly entropy: number;
|
||||
readonly length: number;
|
||||
readonly riskScore: number;
|
||||
}
|
||||
|
||||
export interface UserAgentConsistencyResult {
|
||||
readonly isConsistent: boolean;
|
||||
readonly inconsistencies: readonly string[];
|
||||
}
|
||||
|
||||
// Security constants for user-agent validation
|
||||
const MAX_USER_AGENT_LENGTH = 2048; // 2KB - generous but realistic (normal UAs ~100-500 chars)
|
||||
const MIN_NORMAL_UA_LENGTH = 10; // Legitimate UAs are usually longer
|
||||
const MAX_ENTROPY_THRESHOLD = 5.5; // High entropy indicates randomness
|
||||
const REGEX_TIMEOUT_MS = 100; // Prevent ReDoS attacks
|
||||
const MAX_ANOMALIES_TRACKED = 50; // Prevent memory exhaustion
|
||||
|
||||
// Safe regex patterns with ReDoS protection
|
||||
const SAFE_PATTERNS = {
|
||||
// Pre-compiled patterns to avoid runtime compilation from user input
|
||||
ENCODED_CHARS: /[%\\]x/g,
|
||||
MULTIPLE_SPACES: /\s{3,}/g,
|
||||
MULTIPLE_SEMICOLONS: /;{3,}/g,
|
||||
CONTROL_CHARS: /[\x00-\x1F\x7F]/g,
|
||||
LEGACY_MOZILLA: /mozilla\/4\.0/i,
|
||||
VERSION_PATTERN: /\d+\.\d+\.\d+\.\d+\.\d+/g,
|
||||
PARENTHESES_OPEN: /\(/g,
|
||||
PARENTHESES_CLOSE: /\)/g
|
||||
} as const;
|
||||
|
||||
// Input validation functions with zero trust approach
|
||||
function validateUserAgentInput(userAgent: unknown, paramName: string): string {
|
||||
if (typeof userAgent !== 'string') {
|
||||
throw new Error(`${paramName} must be a string`);
|
||||
}
|
||||
|
||||
if (userAgent.length > MAX_USER_AGENT_LENGTH) {
|
||||
throw new Error(`${paramName} exceeds maximum length of ${MAX_USER_AGENT_LENGTH} characters`);
|
||||
}
|
||||
|
||||
return userAgent;
|
||||
}
|
||||
|
||||
function validateSecChUaInput(secChUa: unknown): string | null {
|
||||
if (secChUa === null || secChUa === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof secChUa !== 'string') {
|
||||
throw new Error('Sec-CH-UA must be a string or null');
|
||||
}
|
||||
|
||||
if (secChUa.length > MAX_USER_AGENT_LENGTH) {
|
||||
throw new Error(`Sec-CH-UA exceeds maximum length of ${MAX_USER_AGENT_LENGTH} characters`);
|
||||
}
|
||||
|
||||
return secChUa;
|
||||
}
|
||||
|
||||
// ReDoS-safe regex execution with timeout
|
||||
function safeRegexTest(pattern: RegExp, input: string, timeoutMs: number = REGEX_TIMEOUT_MS): boolean {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Reset regex state to prevent stateful regex issues
|
||||
pattern.lastIndex = 0;
|
||||
|
||||
// Check if execution takes too long (ReDoS protection)
|
||||
const result = pattern.test(input);
|
||||
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
throw new Error('Regex execution timeout - possible ReDoS attack');
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes('timeout')) {
|
||||
throw error; // Re-throw timeout errors
|
||||
}
|
||||
// For other regex errors, assume no match (fail safe)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Safe pattern matching with bounds checking
|
||||
function safePatternCount(pattern: RegExp, input: string): number {
|
||||
try {
|
||||
const matches = input.match(pattern);
|
||||
return matches ? Math.min(matches.length, 1000) : 0; // Cap at 1000 to prevent DoS
|
||||
} catch {
|
||||
return 0; // Fail safe on regex errors
|
||||
}
|
||||
}
|
||||
|
||||
// Entropy calculation with bounds checking and DoS protection
|
||||
function calculateEntropy(input: string): number {
|
||||
if (!input || input.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Limit analysis to first 800 chars to prevent DoS (normal UAs are ~100-500 chars)
|
||||
const analysisString = input.length > 800 ? input.substring(0, 800) : input;
|
||||
|
||||
const charCounts = new Map<string, number>();
|
||||
|
||||
// Count character frequencies with bounds checking
|
||||
for (let i = 0; i < analysisString.length; i++) {
|
||||
const char = analysisString.charAt(i);
|
||||
if (!char) continue; // Skip if somehow empty
|
||||
|
||||
const currentCount = charCounts.get(char) ?? 0;
|
||||
|
||||
if (currentCount > 100) {
|
||||
// Skip if character appears too frequently (DoS protection)
|
||||
continue;
|
||||
}
|
||||
|
||||
charCounts.set(char, currentCount + 1);
|
||||
|
||||
// Prevent memory exhaustion from too many unique characters
|
||||
if (charCounts.size > 256) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (charCounts.size === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let entropy = 0;
|
||||
const totalLength = analysisString.length;
|
||||
|
||||
for (const count of Array.from(charCounts.values())) {
|
||||
if (count > 0) {
|
||||
const probability = count / totalLength;
|
||||
entropy -= probability * Math.log2(probability);
|
||||
}
|
||||
}
|
||||
|
||||
return Math.min(entropy, 10); // Cap entropy to prevent overflow
|
||||
}
|
||||
|
||||
// Malformed user-agent detection with ReDoS protection
|
||||
function detectMalformedUA(userAgent: string): boolean {
|
||||
try {
|
||||
// Check parentheses balance with safe counting
|
||||
const openParens = safePatternCount(SAFE_PATTERNS.PARENTHESES_OPEN, userAgent);
|
||||
const closeParens = safePatternCount(SAFE_PATTERNS.PARENTHESES_CLOSE, userAgent);
|
||||
|
||||
if (openParens !== closeParens) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for invalid version formats with timeout protection
|
||||
if (safeRegexTest(SAFE_PATTERNS.VERSION_PATTERN, userAgent)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for multiple consecutive spaces or semicolons
|
||||
if (safeRegexTest(SAFE_PATTERNS.MULTIPLE_SPACES, userAgent) ||
|
||||
safeRegexTest(SAFE_PATTERNS.MULTIPLE_SEMICOLONS, userAgent)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check for control characters
|
||||
if (safeRegexTest(SAFE_PATTERNS.CONTROL_CHARS, userAgent)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
// If malformation detection fails, assume malformed for safety
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Safe bot detection with pattern timeout protection
|
||||
function detectVerifiedBot(userAgent: string, verifiedBots: VerifiedGoodBots): { isBot: boolean; botType: string | null } {
|
||||
try {
|
||||
for (const [botName, botConfig] of Object.entries(verifiedBots)) {
|
||||
if (safeRegexTest(botConfig.pattern, userAgent)) {
|
||||
return { isBot: true, botType: botName };
|
||||
}
|
||||
}
|
||||
return { isBot: false, botType: null };
|
||||
} catch {
|
||||
// On error, assume not a verified bot (fail safe)
|
||||
return { isBot: false, botType: null };
|
||||
}
|
||||
}
|
||||
|
||||
// Main user-agent analysis function with comprehensive validation
|
||||
export function analyzeUserAgentAdvanced(userAgent: unknown): UserAgentFeatures {
|
||||
// Validate input with zero trust
|
||||
let validatedUA: string;
|
||||
try {
|
||||
validatedUA = validateUserAgentInput(userAgent, 'userAgent');
|
||||
} catch (error) {
|
||||
// If validation fails, return safe defaults
|
||||
return {
|
||||
isAttackTool: false,
|
||||
isMissing: true,
|
||||
isMalformed: true,
|
||||
isSuspiciousBot: false,
|
||||
isVerifiedGoodBot: false,
|
||||
botType: null,
|
||||
anomalies: ['validation_failed'],
|
||||
entropy: 0,
|
||||
length: 0,
|
||||
riskScore: 100 // High risk for invalid input
|
||||
};
|
||||
}
|
||||
|
||||
const anomalies: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Handle missing or empty user agent
|
||||
if (!validatedUA || validatedUA.trim() === '') {
|
||||
return {
|
||||
isAttackTool: false,
|
||||
isMissing: true,
|
||||
isMalformed: false,
|
||||
isSuspiciousBot: false,
|
||||
isVerifiedGoodBot: false,
|
||||
botType: null,
|
||||
anomalies: ['missing_user_agent'],
|
||||
entropy: 0,
|
||||
length: validatedUA.length,
|
||||
riskScore: 50 // Medium risk for missing UA
|
||||
};
|
||||
}
|
||||
|
||||
const uaLower = validatedUA.toLowerCase();
|
||||
const uaLength = validatedUA.length;
|
||||
|
||||
// Attack tool detection with safe pattern matching
|
||||
let isAttackTool = false;
|
||||
try {
|
||||
if (matchAttackTools(uaLower)) {
|
||||
isAttackTool = true;
|
||||
anomalies.push('attack_tool_detected');
|
||||
riskScore += 80; // High risk
|
||||
}
|
||||
} catch {
|
||||
// If attack tool detection fails, log anomaly but continue
|
||||
anomalies.push('attack_tool_detection_failed');
|
||||
}
|
||||
|
||||
// Suspicious bot detection with safe pattern matching
|
||||
let isSuspiciousBot = false;
|
||||
try {
|
||||
if (matchSuspiciousBots(uaLower)) {
|
||||
isSuspiciousBot = true;
|
||||
anomalies.push('suspicious_bot_pattern');
|
||||
riskScore += 30; // Medium risk
|
||||
}
|
||||
} catch {
|
||||
anomalies.push('bot_detection_failed');
|
||||
}
|
||||
|
||||
// Verified good bot detection with timeout protection
|
||||
const botDetection = detectVerifiedBot(validatedUA, VERIFIED_GOOD_BOTS);
|
||||
const isVerifiedGoodBot = botDetection.isBot;
|
||||
const botType = botDetection.botType;
|
||||
|
||||
if (isVerifiedGoodBot) {
|
||||
riskScore = Math.max(0, riskScore - 20); // Reduce risk for verified bots
|
||||
|
||||
// Note: Enhanced bot verification with IP ranges and DNS is available
|
||||
// via the botVerificationEngine in src/utils/bot-verification.ts
|
||||
// This can be integrated for more robust bot verification beyond user-agent patterns
|
||||
}
|
||||
|
||||
// Entropy calculation with DoS protection
|
||||
let entropy = 0;
|
||||
try {
|
||||
entropy = calculateEntropy(validatedUA);
|
||||
if (entropy > MAX_ENTROPY_THRESHOLD) {
|
||||
anomalies.push('high_entropy_ua');
|
||||
riskScore += 25;
|
||||
}
|
||||
} catch {
|
||||
anomalies.push('entropy_calculation_failed');
|
||||
riskScore += 10; // Small penalty for analysis failure
|
||||
}
|
||||
|
||||
// Malformation detection with ReDoS protection
|
||||
let isMalformed = false;
|
||||
try {
|
||||
isMalformed = detectMalformedUA(validatedUA);
|
||||
if (isMalformed) {
|
||||
anomalies.push('malformed_user_agent');
|
||||
riskScore += 40;
|
||||
}
|
||||
} catch {
|
||||
anomalies.push('malformation_detection_failed');
|
||||
isMalformed = true; // Assume malformed on detection failure
|
||||
riskScore += 30;
|
||||
}
|
||||
|
||||
// Additional anomaly detection with safe patterns
|
||||
try {
|
||||
// Legacy Mozilla spoofing
|
||||
if (safeRegexTest(SAFE_PATTERNS.LEGACY_MOZILLA, validatedUA) && !validatedUA.toLowerCase().includes('msie')) {
|
||||
anomalies.push('legacy_mozilla_spoof');
|
||||
riskScore += 15;
|
||||
}
|
||||
|
||||
// Suspiciously short user agents
|
||||
if (uaLength < MIN_NORMAL_UA_LENGTH) {
|
||||
anomalies.push('suspiciously_short_ua');
|
||||
riskScore += 20;
|
||||
}
|
||||
|
||||
// Encoded characters
|
||||
if (safeRegexTest(SAFE_PATTERNS.ENCODED_CHARS, validatedUA)) {
|
||||
anomalies.push('encoded_characters_in_ua');
|
||||
riskScore += 25;
|
||||
}
|
||||
|
||||
// Extremely long user agents (potential DoS or attack)
|
||||
if (uaLength > 800) {
|
||||
anomalies.push('suspiciously_long_ua');
|
||||
riskScore += 30;
|
||||
}
|
||||
|
||||
} catch {
|
||||
anomalies.push('anomaly_detection_failed');
|
||||
riskScore += 10;
|
||||
}
|
||||
|
||||
// Limit anomalies to prevent memory exhaustion
|
||||
const limitedAnomalies = anomalies.slice(0, MAX_ANOMALIES_TRACKED);
|
||||
|
||||
// Cap risk score to valid range
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
|
||||
return {
|
||||
isAttackTool,
|
||||
isMissing: false,
|
||||
isMalformed,
|
||||
isSuspiciousBot,
|
||||
isVerifiedGoodBot,
|
||||
botType,
|
||||
anomalies: limitedAnomalies,
|
||||
entropy,
|
||||
length: uaLength,
|
||||
riskScore: finalRiskScore
|
||||
};
|
||||
}
|
||||
|
||||
// User-Agent consistency checking with comprehensive validation
|
||||
export function checkUAConsistency(userAgent: unknown, secChUa: unknown): UserAgentConsistencyResult {
|
||||
try {
|
||||
// Validate inputs with zero trust
|
||||
const validatedUA = userAgent ? validateUserAgentInput(userAgent, 'userAgent') : null;
|
||||
const validatedSecChUa = validateSecChUaInput(secChUa);
|
||||
|
||||
const inconsistencies: string[] = [];
|
||||
|
||||
// If either is missing, that's not necessarily inconsistent
|
||||
if (!validatedUA || !validatedSecChUa) {
|
||||
return {
|
||||
isConsistent: true,
|
||||
inconsistencies: []
|
||||
};
|
||||
}
|
||||
|
||||
const uaLower = validatedUA.toLowerCase();
|
||||
const secChUaLower = validatedSecChUa.toLowerCase();
|
||||
|
||||
// Browser detection with safe string operations
|
||||
const uaBrowsers = {
|
||||
chrome: uaLower.includes('chrome/'),
|
||||
firefox: uaLower.includes('firefox/'),
|
||||
edge: uaLower.includes('edg/'),
|
||||
safari: uaLower.includes('safari/') && !uaLower.includes('chrome/')
|
||||
};
|
||||
|
||||
const secChBrowsers = {
|
||||
chrome: secChUaLower.includes('chrome'),
|
||||
firefox: secChUaLower.includes('firefox'),
|
||||
edge: secChUaLower.includes('edge'),
|
||||
safari: secChUaLower.includes('safari')
|
||||
};
|
||||
|
||||
// Check for inconsistencies
|
||||
if (uaBrowsers.chrome && !secChBrowsers.chrome) {
|
||||
inconsistencies.push('chrome_ua_mismatch');
|
||||
}
|
||||
|
||||
if (uaBrowsers.firefox && !secChBrowsers.firefox) {
|
||||
inconsistencies.push('firefox_ua_mismatch');
|
||||
}
|
||||
|
||||
if (uaBrowsers.edge && !secChBrowsers.edge) {
|
||||
inconsistencies.push('edge_ua_mismatch');
|
||||
}
|
||||
|
||||
if (uaBrowsers.safari && !secChBrowsers.safari) {
|
||||
inconsistencies.push('safari_ua_mismatch');
|
||||
}
|
||||
|
||||
// Check for completely different browsers
|
||||
const uaHasBrowser = Object.values(uaBrowsers).some(Boolean);
|
||||
const secChHasBrowser = Object.values(secChBrowsers).some(Boolean);
|
||||
|
||||
if (uaHasBrowser && secChHasBrowser) {
|
||||
const hasAnyMatch = Object.keys(uaBrowsers).some(browser =>
|
||||
uaBrowsers[browser as keyof typeof uaBrowsers] &&
|
||||
secChBrowsers[browser as keyof typeof secChBrowsers]
|
||||
);
|
||||
|
||||
if (!hasAnyMatch) {
|
||||
inconsistencies.push('completely_different_browsers');
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
isConsistent: inconsistencies.length === 0,
|
||||
inconsistencies
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
// On validation error, assume inconsistent for security
|
||||
return {
|
||||
isConsistent: false,
|
||||
inconsistencies: ['validation_error']
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Export types for use in other modules
|
||||
export type { BotInfo, VerifiedGoodBots };
|
||||
|
|
@ -1,548 +0,0 @@
|
|||
// =============================================================================
|
||||
// CACHE MANAGEMENT FOR THREAT SCORING (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
import { CACHE_CONFIG } from './constants.js';
|
||||
import { parseDuration } from '../time.js';
|
||||
|
||||
// Pre-computed durations for hot path cache operations
|
||||
const REQUEST_HISTORY_TTL = parseDuration('30m');
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface CachedEntry<T> {
|
||||
readonly data: T;
|
||||
readonly timestamp: number;
|
||||
readonly ttl?: number;
|
||||
}
|
||||
|
||||
interface RequestHistoryEntry {
|
||||
readonly timestamp: number;
|
||||
readonly method?: string;
|
||||
readonly path?: string;
|
||||
readonly userAgent?: string;
|
||||
readonly score?: number;
|
||||
readonly responseTime?: number;
|
||||
readonly statusCode?: number;
|
||||
}
|
||||
|
||||
interface CachedRequestHistory {
|
||||
readonly history: readonly RequestHistoryEntry[];
|
||||
readonly timestamp: number;
|
||||
}
|
||||
|
||||
interface IPScoreEntry {
|
||||
readonly score: number;
|
||||
readonly confidence: number;
|
||||
readonly lastCalculated: number;
|
||||
readonly components: Record<string, number>;
|
||||
}
|
||||
|
||||
interface SessionEntry {
|
||||
readonly sessionId: string;
|
||||
readonly startTime: number;
|
||||
readonly lastActivity: number;
|
||||
readonly requestCount: number;
|
||||
readonly behaviorScore: number;
|
||||
readonly flags: readonly string[];
|
||||
}
|
||||
|
||||
interface BehaviorEntry {
|
||||
readonly patterns: Record<string, unknown>;
|
||||
readonly anomalies: readonly string[];
|
||||
readonly riskScore: number;
|
||||
readonly lastUpdated: number;
|
||||
readonly requestPattern: Record<string, number>;
|
||||
}
|
||||
|
||||
interface VerifiedBotEntry {
|
||||
readonly botName: string;
|
||||
readonly verified: boolean;
|
||||
readonly verificationMethod: 'dns' | 'user_agent' | 'signature' | 'manual';
|
||||
readonly lastVerified: number;
|
||||
readonly trustScore: number;
|
||||
}
|
||||
|
||||
interface CacheStats {
|
||||
readonly ipScore: number;
|
||||
readonly session: number;
|
||||
readonly behavior: number;
|
||||
readonly verifiedBots: number;
|
||||
}
|
||||
|
||||
interface CacheCleanupResult {
|
||||
readonly beforeSize: CacheStats;
|
||||
readonly afterSize: CacheStats;
|
||||
readonly totalCleaned: number;
|
||||
readonly emergencyTriggered: boolean;
|
||||
}
|
||||
|
||||
// Generic cache interface for type safety
|
||||
interface TypedCache<T> {
|
||||
get(key: string): T | undefined;
|
||||
set(key: string, value: T): void;
|
||||
delete(key: string): boolean;
|
||||
has(key: string): boolean;
|
||||
clear(): void;
|
||||
readonly size: number;
|
||||
[Symbol.iterator](): IterableIterator<[string, T]>;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// CACHE MANAGER CLASS
|
||||
// =============================================================================
|
||||
|
||||
export class CacheManager {
|
||||
// Type-safe cache instances
|
||||
private readonly ipScoreCache: TypedCache<CachedEntry<IPScoreEntry>>;
|
||||
private readonly sessionCache: TypedCache<CachedEntry<SessionEntry>>;
|
||||
private readonly behaviorCache: TypedCache<CachedEntry<BehaviorEntry | CachedRequestHistory>>;
|
||||
private readonly verifiedBotsCache: TypedCache<CachedEntry<VerifiedBotEntry>>;
|
||||
|
||||
// Cleanup timer reference for proper disposal
|
||||
private cleanupTimer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor() {
|
||||
// Initialize in-memory caches with size limits
|
||||
this.ipScoreCache = new Map<string, CachedEntry<IPScoreEntry>>() as TypedCache<CachedEntry<IPScoreEntry>>;
|
||||
this.sessionCache = new Map<string, CachedEntry<SessionEntry>>() as TypedCache<CachedEntry<SessionEntry>>;
|
||||
this.behaviorCache = new Map<string, CachedEntry<BehaviorEntry | CachedRequestHistory>>() as TypedCache<CachedEntry<BehaviorEntry | CachedRequestHistory>>;
|
||||
this.verifiedBotsCache = new Map<string, CachedEntry<VerifiedBotEntry>>() as TypedCache<CachedEntry<VerifiedBotEntry>>;
|
||||
|
||||
// Start cache cleanup timer
|
||||
this.startCacheCleanup();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// CACHE LIFECYCLE MANAGEMENT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Starts the cache cleanup timer - CRITICAL for memory stability
|
||||
* This prevents memory leaks under high load by periodically cleaning expired entries
|
||||
*/
|
||||
private startCacheCleanup(): void {
|
||||
// CRITICAL: This timer prevents memory leaks under high load
|
||||
// If this cleanup stops running, the system will eventually crash due to memory exhaustion
|
||||
// The cleanup interval affects both memory usage and performance - too frequent = CPU waste,
|
||||
// too infrequent = memory problems
|
||||
this.cleanupTimer = setInterval(() => {
|
||||
this.cleanupCaches();
|
||||
}, CACHE_CONFIG.CACHE_CLEANUP_INTERVAL);
|
||||
|
||||
// Ensure cleanup timer doesn't keep process alive
|
||||
if (this.cleanupTimer.unref) {
|
||||
this.cleanupTimer.unref();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the cache cleanup timer and clears all caches
|
||||
* Should be called during application shutdown
|
||||
*/
|
||||
public destroy(): void {
|
||||
if (this.cleanupTimer) {
|
||||
clearInterval(this.cleanupTimer);
|
||||
this.cleanupTimer = null;
|
||||
}
|
||||
|
||||
// Clear all caches
|
||||
this.ipScoreCache.clear();
|
||||
this.sessionCache.clear();
|
||||
this.behaviorCache.clear();
|
||||
this.verifiedBotsCache.clear();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// CACHE CLEANUP OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Performs comprehensive cache cleanup to prevent memory exhaustion
|
||||
* @returns Cleanup statistics
|
||||
*/
|
||||
public cleanupCaches(): CacheCleanupResult {
|
||||
const beforeSize: CacheStats = {
|
||||
ipScore: this.ipScoreCache.size,
|
||||
session: this.sessionCache.size,
|
||||
behavior: this.behaviorCache.size,
|
||||
verifiedBots: this.verifiedBotsCache.size
|
||||
};
|
||||
|
||||
// Clean each cache using the optimized cleanup method
|
||||
this.cleanupCache(this.ipScoreCache);
|
||||
this.cleanupCache(this.sessionCache);
|
||||
this.cleanupCache(this.behaviorCache);
|
||||
this.cleanupCache(this.verifiedBotsCache);
|
||||
|
||||
const afterSize: CacheStats = {
|
||||
ipScore: this.ipScoreCache.size,
|
||||
session: this.sessionCache.size,
|
||||
behavior: this.behaviorCache.size,
|
||||
verifiedBots: this.verifiedBotsCache.size
|
||||
};
|
||||
|
||||
const totalCleaned = Object.keys(beforeSize).reduce((total, key) => {
|
||||
const beforeCount = beforeSize[key as keyof CacheStats];
|
||||
const afterCount = afterSize[key as keyof CacheStats];
|
||||
return total + (beforeCount - afterCount);
|
||||
}, 0);
|
||||
|
||||
let emergencyTriggered = false;
|
||||
|
||||
if (totalCleaned > 0) {
|
||||
console.log(`Threat scorer: cleaned ${totalCleaned} expired cache entries`);
|
||||
}
|
||||
|
||||
// Emergency cleanup if caches are still too large
|
||||
// This prevents memory exhaustion under extreme load
|
||||
if (this.ipScoreCache.size > CACHE_CONFIG.MAX_CACHE_SIZE * CACHE_CONFIG.EMERGENCY_CLEANUP_THRESHOLD) {
|
||||
console.warn('Threat scorer: Emergency cleanup triggered - system under high load');
|
||||
this.emergencyCleanup();
|
||||
emergencyTriggered = true;
|
||||
}
|
||||
|
||||
return {
|
||||
beforeSize,
|
||||
afterSize,
|
||||
totalCleaned,
|
||||
emergencyTriggered
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Optimized cache cleanup - removes oldest entries when cache exceeds size limit
|
||||
* Maps maintain insertion order, so we can efficiently remove oldest entries
|
||||
*/
|
||||
private cleanupCache<T>(cache: TypedCache<T>): number {
|
||||
if (cache.size <= CACHE_CONFIG.MAX_CACHE_SIZE) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const excess = cache.size - CACHE_CONFIG.MAX_CACHE_SIZE;
|
||||
let removed = 0;
|
||||
|
||||
// Remove oldest entries (Maps maintain insertion order)
|
||||
const cacheAsMap = cache as unknown as Map<string, T>;
|
||||
for (const [key] of Array.from(cacheAsMap.entries())) {
|
||||
if (removed >= excess) {
|
||||
break;
|
||||
}
|
||||
cache.delete(key);
|
||||
removed++;
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Emergency cleanup for extreme memory pressure
|
||||
* Aggressively reduces cache sizes to prevent system crashes
|
||||
*/
|
||||
private emergencyCleanup(): void {
|
||||
// Aggressively reduce cache sizes to 25% of max
|
||||
const targetSize = Math.floor(CACHE_CONFIG.MAX_CACHE_SIZE * CACHE_CONFIG.EMERGENCY_CLEANUP_TARGET);
|
||||
|
||||
// Clean each cache individually to avoid type issues
|
||||
this.emergencyCleanupCache(this.ipScoreCache, targetSize);
|
||||
this.emergencyCleanupCache(this.sessionCache, targetSize);
|
||||
this.emergencyCleanupCache(this.behaviorCache, targetSize);
|
||||
this.emergencyCleanupCache(this.verifiedBotsCache, targetSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method for emergency cleanup of individual cache
|
||||
*/
|
||||
private emergencyCleanupCache<T>(cache: TypedCache<T>, targetSize: number): void {
|
||||
if (cache.size <= targetSize) {
|
||||
return;
|
||||
}
|
||||
|
||||
const toRemove = cache.size - targetSize;
|
||||
let removed = 0;
|
||||
|
||||
// Clear the cache if we need to remove too many entries (emergency scenario)
|
||||
if (toRemove > cache.size * 0.8) {
|
||||
cache.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, remove oldest entries using the Map's iteration order
|
||||
const cacheAsMap = cache as unknown as Map<string, T>;
|
||||
const keysToDelete: string[] = [];
|
||||
|
||||
for (const [key] of Array.from(cacheAsMap.entries())) {
|
||||
if (keysToDelete.length >= toRemove) {
|
||||
break;
|
||||
}
|
||||
keysToDelete.push(key);
|
||||
}
|
||||
|
||||
for (const key of keysToDelete) {
|
||||
cache.delete(key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// IP SCORE CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached IP score if still valid
|
||||
*/
|
||||
public getCachedIPScore(ip: string): IPScoreEntry | null {
|
||||
if (!ip || typeof ip !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.ipScoreCache.get(ip);
|
||||
if (cached && this.isEntryValid(cached)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches IP score with optional TTL
|
||||
*/
|
||||
public setCachedIPScore(ip: string, scoreData: IPScoreEntry, ttlMs?: number): void {
|
||||
if (!ip || typeof ip !== 'string' || !scoreData) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<IPScoreEntry> = {
|
||||
data: scoreData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.ipScoreCache.set(ip, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// SESSION CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached session data if still valid
|
||||
*/
|
||||
public getCachedSession(sessionId: string): SessionEntry | null {
|
||||
if (!sessionId || typeof sessionId !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.sessionCache.get(sessionId);
|
||||
if (cached && this.isEntryValid(cached)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches session data with optional TTL
|
||||
*/
|
||||
public setCachedSession(sessionId: string, sessionData: SessionEntry, ttlMs?: number): void {
|
||||
if (!sessionId || typeof sessionId !== 'string' || !sessionData) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<SessionEntry> = {
|
||||
data: sessionData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.sessionCache.set(sessionId, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// BEHAVIOR CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached behavior data if still valid
|
||||
*/
|
||||
public getCachedBehavior(key: string): BehaviorEntry | null {
|
||||
if (!key || typeof key !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.behaviorCache.get(key);
|
||||
if (cached && this.isEntryValid(cached) && this.isBehaviorEntry(cached.data)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches behavior data with optional TTL
|
||||
*/
|
||||
public setCachedBehavior(key: string, behaviorData: BehaviorEntry, ttlMs?: number): void {
|
||||
if (!key || typeof key !== 'string' || !behaviorData) {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<BehaviorEntry> = {
|
||||
data: behaviorData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.behaviorCache.set(key, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// REQUEST HISTORY CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached request history if still valid
|
||||
*/
|
||||
public getCachedRequestHistory(ip: string, cutoff: number): readonly RequestHistoryEntry[] | null {
|
||||
if (!ip || typeof ip !== 'string' || typeof cutoff !== 'number') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cacheKey = `history:${ip}`;
|
||||
const cached = this.behaviorCache.get(cacheKey);
|
||||
|
||||
if (cached && cached.timestamp > cutoff && this.isRequestHistoryEntry(cached.data)) {
|
||||
return cached.data.history.filter(h => h.timestamp > cutoff);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches request history with automatic TTL
|
||||
*/
|
||||
public setCachedRequestHistory(ip: string, history: readonly RequestHistoryEntry[]): void {
|
||||
if (!ip || typeof ip !== 'string' || !Array.isArray(history)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const cacheKey = `history:${ip}`;
|
||||
const cachedHistory: CachedRequestHistory = {
|
||||
history,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
|
||||
const entry: CachedEntry<CachedRequestHistory> = {
|
||||
data: cachedHistory,
|
||||
timestamp: Date.now(),
|
||||
ttl: REQUEST_HISTORY_TTL // 30 minutes TTL for request history
|
||||
};
|
||||
|
||||
this.behaviorCache.set(cacheKey, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// VERIFIED BOTS CACHE OPERATIONS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Retrieves cached bot verification if still valid
|
||||
*/
|
||||
public getCachedBotVerification(userAgent: string): VerifiedBotEntry | null {
|
||||
if (!userAgent || typeof userAgent !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cached = this.verifiedBotsCache.get(userAgent);
|
||||
if (cached && this.isEntryValid(cached)) {
|
||||
return cached.data;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caches bot verification with TTL from configuration
|
||||
*/
|
||||
public setCachedBotVerification(userAgent: string, botData: VerifiedBotEntry, ttlMs: number): void {
|
||||
if (!userAgent || typeof userAgent !== 'string' || !botData || typeof ttlMs !== 'number') {
|
||||
return;
|
||||
}
|
||||
|
||||
const entry: CachedEntry<VerifiedBotEntry> = {
|
||||
data: botData,
|
||||
timestamp: Date.now(),
|
||||
ttl: ttlMs
|
||||
};
|
||||
|
||||
this.verifiedBotsCache.set(userAgent, entry);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// CACHE STATISTICS AND MONITORING
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Gets current cache statistics for monitoring
|
||||
*/
|
||||
public getCacheStats(): CacheStats & { totalEntries: number; memoryPressure: boolean } {
|
||||
const stats: CacheStats = {
|
||||
ipScore: this.ipScoreCache.size,
|
||||
session: this.sessionCache.size,
|
||||
behavior: this.behaviorCache.size,
|
||||
verifiedBots: this.verifiedBotsCache.size
|
||||
};
|
||||
|
||||
const totalEntries = Object.values(stats).reduce((sum, count) => sum + count, 0);
|
||||
const memoryPressure = totalEntries > (CACHE_CONFIG.MAX_CACHE_SIZE * 4 * CACHE_CONFIG.EMERGENCY_CLEANUP_THRESHOLD);
|
||||
|
||||
return {
|
||||
...stats,
|
||||
totalEntries,
|
||||
memoryPressure
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all caches - use with caution
|
||||
*/
|
||||
public clearAllCaches(): void {
|
||||
this.ipScoreCache.clear();
|
||||
this.sessionCache.clear();
|
||||
this.behaviorCache.clear();
|
||||
this.verifiedBotsCache.clear();
|
||||
|
||||
console.log('Threat scorer: All caches cleared');
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// UTILITY METHODS
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Checks if a cached entry is still valid based on TTL
|
||||
*/
|
||||
private isEntryValid<T>(entry: CachedEntry<T>): boolean {
|
||||
if (!entry.ttl) {
|
||||
return true; // No TTL means it doesn't expire
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
return (now - entry.timestamp) < entry.ttl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if cached data is BehaviorEntry
|
||||
*/
|
||||
private isBehaviorEntry(data: BehaviorEntry | CachedRequestHistory): data is BehaviorEntry {
|
||||
return 'patterns' in data && 'anomalies' in data && 'riskScore' in data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if cached data is CachedRequestHistory
|
||||
*/
|
||||
private isRequestHistoryEntry(data: BehaviorEntry | CachedRequestHistory): data is CachedRequestHistory {
|
||||
return 'history' in data && Array.isArray((data as CachedRequestHistory).history);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
// =============================================================================
|
||||
// THREAT SCORING ENGINE CONSTANTS & CONFIGURATION
|
||||
// =============================================================================
|
||||
|
||||
import { parseDuration } from '../time.js';
|
||||
|
||||
// Type definitions for threat scoring system
|
||||
export interface ThreatThresholds {
|
||||
readonly ALLOW: number;
|
||||
readonly CHALLENGE: number;
|
||||
readonly BLOCK: number;
|
||||
}
|
||||
|
||||
export interface SignalWeight {
|
||||
readonly weight: number;
|
||||
readonly confidence: number;
|
||||
}
|
||||
|
||||
export interface SignalWeights {
|
||||
// User-Agent signals (implemented)
|
||||
readonly ATTACK_TOOL_UA: SignalWeight;
|
||||
readonly MISSING_UA: SignalWeight;
|
||||
|
||||
// WAF signals (implemented via WAF plugin)
|
||||
readonly SQL_INJECTION: SignalWeight;
|
||||
readonly XSS_ATTEMPT: SignalWeight;
|
||||
readonly COMMAND_INJECTION: SignalWeight;
|
||||
readonly PATH_TRAVERSAL: SignalWeight;
|
||||
}
|
||||
|
||||
export interface StaticWhitelist {
|
||||
readonly extensions: ReadonlySet<string>;
|
||||
readonly paths: ReadonlySet<string>;
|
||||
readonly patterns: readonly RegExp[];
|
||||
}
|
||||
|
||||
export interface BotInfo {
|
||||
readonly pattern: RegExp;
|
||||
readonly verifyDNS: boolean;
|
||||
}
|
||||
|
||||
export interface VerifiedGoodBots {
|
||||
readonly [botName: string]: BotInfo;
|
||||
}
|
||||
|
||||
export interface CacheConfig {
|
||||
readonly MAX_CACHE_SIZE: number;
|
||||
readonly CACHE_CLEANUP_INTERVAL: number;
|
||||
readonly EMERGENCY_CLEANUP_THRESHOLD: number;
|
||||
readonly EMERGENCY_CLEANUP_TARGET: number;
|
||||
}
|
||||
|
||||
export interface DbTtlConfig {
|
||||
readonly THREAT_DB_TTL: number;
|
||||
readonly BEHAVIOR_DB_TTL: number;
|
||||
}
|
||||
|
||||
// Attack pattern types
|
||||
export type AttackToolPattern = string;
|
||||
export type SuspiciousBotPattern = string;
|
||||
|
||||
// All threat score thresholds should come from user configuration
|
||||
// No hardcoded defaults - configuration required
|
||||
|
||||
// Attack tool patterns for Aho-Corasick matching
|
||||
export const ATTACK_TOOL_PATTERNS: readonly AttackToolPattern[] = [
|
||||
'sqlmap', 'nikto', 'nmap', 'burpsuite', 'w3af', 'acunetix',
|
||||
'nessus', 'openvas', 'gobuster', 'dirbuster', 'wfuzz', 'ffuf',
|
||||
'hydra', 'medusa', 'masscan', 'zmap', 'metasploit', 'burp suite',
|
||||
'scanner', 'exploit', 'payload', 'injection', 'vulnerability'
|
||||
] as const;
|
||||
|
||||
// Suspicious bot patterns
|
||||
export const SUSPICIOUS_BOT_PATTERNS: readonly SuspiciousBotPattern[] = [
|
||||
'bot', 'crawler', 'spider', 'scraper', 'scanner', 'harvest',
|
||||
'extract', 'collect', 'gather', 'fetch'
|
||||
] as const;
|
||||
|
||||
// Signal weights should come from user configuration
|
||||
// No hardcoded signal weights - configuration required
|
||||
|
||||
// Paths and extensions that should never trigger scoring
|
||||
export const STATIC_WHITELIST: StaticWhitelist = {
|
||||
extensions: new Set([
|
||||
'.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.webp',
|
||||
'.woff', '.woff2', '.ttf', '.eot', '.pdf', '.mp4', '.mp3', '.zip', '.avif',
|
||||
'.bmp', '.tiff', '.webm', '.mov', '.avi', '.flv', '.map', '.txt', '.xml'
|
||||
]) as ReadonlySet<string>,
|
||||
paths: new Set([
|
||||
'/static/', '/assets/', '/images/', '/img/', '/css/', '/js/', '/fonts/',
|
||||
'/webfont/', '/favicon', '/media/', '/uploads/', '/.well-known/'
|
||||
]) as ReadonlySet<string>,
|
||||
patterns: [
|
||||
/^\/[a-f0-9]{32}\.(css|js)$/i, // Hashed asset files
|
||||
/^\/build\/[^\/]+\.(css|js)$/i, // Build output files
|
||||
/^\/dist\/[^\/]+\.(css|js)$/i, // Distribution files
|
||||
] as const
|
||||
} as const;
|
||||
|
||||
// Known good bots that should be treated favorably
|
||||
export const VERIFIED_GOOD_BOTS: VerifiedGoodBots = {
|
||||
// Search engines
|
||||
'googlebot': { pattern: /Googlebot\/\d+\.\d+/i, verifyDNS: true },
|
||||
'bingbot': { pattern: /bingbot\/\d+\.\d+/i, verifyDNS: true },
|
||||
'slurp': { pattern: /Slurp/i, verifyDNS: true },
|
||||
'duckduckbot': { pattern: /DuckDuckBot\/\d+\.\d+/i, verifyDNS: false },
|
||||
'baiduspider': { pattern: /Baiduspider\/\d+\.\d+/i, verifyDNS: true },
|
||||
'yandexbot': { pattern: /YandexBot\/\d+\.\d+/i, verifyDNS: true },
|
||||
|
||||
// Social media
|
||||
'facebookexternalhit': { pattern: /facebookexternalhit\/\d+\.\d+/i, verifyDNS: false },
|
||||
'twitterbot': { pattern: /Twitterbot\/\d+\.\d+/i, verifyDNS: false },
|
||||
'linkedinbot': { pattern: /LinkedInBot\/\d+\.\d+/i, verifyDNS: false },
|
||||
|
||||
// Monitoring services
|
||||
'uptimerobot': { pattern: /UptimeRobot\/\d+\.\d+/i, verifyDNS: false },
|
||||
'pingdom': { pattern: /Pingdom\.com_bot/i, verifyDNS: false }
|
||||
} as const;
|
||||
|
||||
// Cache configuration
|
||||
export const CACHE_CONFIG: CacheConfig = {
|
||||
MAX_CACHE_SIZE: 10000,
|
||||
CACHE_CLEANUP_INTERVAL: parseDuration('5m'), // 5 minutes
|
||||
EMERGENCY_CLEANUP_THRESHOLD: 1.5, // 150% of max size
|
||||
EMERGENCY_CLEANUP_TARGET: 0.25 // Reduce to 25% of max
|
||||
} as const;
|
||||
|
||||
// Database TTL configuration
|
||||
export const DB_TTL_CONFIG: DbTtlConfig = {
|
||||
THREAT_DB_TTL: parseDuration('1h'), // 1 hour
|
||||
BEHAVIOR_DB_TTL: parseDuration('24h') // 24 hours
|
||||
} as const;
|
||||
|
||||
// Type utility to get signal weight names as union type
|
||||
export type SignalWeightName = keyof SignalWeights;
|
||||
|
||||
// Type utility to get attack tool patterns as literal types
|
||||
export type AttackToolPatterns = typeof ATTACK_TOOL_PATTERNS[number];
|
||||
export type SuspiciousBotPatterns = typeof SUSPICIOUS_BOT_PATTERNS[number];
|
||||
|
||||
// Note: All interface types are already exported above
|
||||
|
|
@ -1,503 +0,0 @@
|
|||
// =============================================================================
|
||||
// DATABASE OPERATIONS FOR THREAT SCORING (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
import { Level } from 'level';
|
||||
// @ts-ignore - level-ttl doesn't have TypeScript definitions
|
||||
import ttl from 'level-ttl';
|
||||
import { rootDir } from '../../index.js';
|
||||
import { join } from 'path';
|
||||
import { Readable } from 'stream';
|
||||
import * as fs from 'fs';
|
||||
import { DB_TTL_CONFIG } from './constants.js';
|
||||
|
||||
// Import types from the main threat scoring module
|
||||
// Local type definitions for database operations
|
||||
type ThreatFeatures = Record<string, any>;
|
||||
type AssessmentData = Record<string, any>;
|
||||
type SanitizedFeatures = Record<string, any>;
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface DatabaseOperation {
|
||||
readonly type: 'put' | 'del';
|
||||
readonly key: string;
|
||||
readonly value?: unknown;
|
||||
}
|
||||
|
||||
interface ThreatAssessment {
|
||||
readonly score: number;
|
||||
readonly action: 'allow' | 'challenge' | 'block';
|
||||
readonly features: Record<string, unknown>;
|
||||
readonly scoreComponents: Record<string, number>;
|
||||
readonly confidence: number;
|
||||
readonly timestamp: number;
|
||||
}
|
||||
|
||||
interface BehaviorData {
|
||||
readonly lastScore: number;
|
||||
readonly lastSeen: number;
|
||||
readonly features: Record<string, unknown>;
|
||||
readonly requestCount: number;
|
||||
}
|
||||
|
||||
interface ReputationData {
|
||||
score: number;
|
||||
incidents: number;
|
||||
blacklisted: boolean;
|
||||
tags: string[];
|
||||
notes?: string;
|
||||
firstSeen?: number;
|
||||
lastUpdate: number;
|
||||
source: 'static_migration' | 'dynamic' | 'manual';
|
||||
migrated?: boolean;
|
||||
}
|
||||
|
||||
interface RequestHistoryEntry {
|
||||
readonly timestamp: number;
|
||||
readonly method?: string;
|
||||
readonly path?: string;
|
||||
readonly userAgent?: string;
|
||||
readonly score?: number;
|
||||
}
|
||||
|
||||
interface MigrationRecord {
|
||||
readonly completed: number;
|
||||
readonly count: number;
|
||||
}
|
||||
|
||||
interface StaticReputationEntry {
|
||||
readonly score?: number;
|
||||
readonly incidents?: number;
|
||||
readonly blacklisted?: boolean;
|
||||
readonly tags?: readonly string[];
|
||||
readonly notes?: string;
|
||||
}
|
||||
|
||||
interface LevelDatabase {
|
||||
put(key: string, value: unknown): Promise<void>;
|
||||
get(key: string): Promise<unknown>;
|
||||
del(key: string): Promise<void>;
|
||||
batch(operations: readonly DatabaseOperation[]): Promise<void>;
|
||||
createReadStream(options?: DatabaseStreamOptions): AsyncIterable<DatabaseEntry>;
|
||||
iterator(options?: DatabaseStreamOptions): AsyncIterable<[string, unknown]>;
|
||||
}
|
||||
|
||||
interface DatabaseStreamOptions {
|
||||
readonly gte?: string;
|
||||
readonly lte?: string;
|
||||
readonly limit?: number;
|
||||
readonly reverse?: boolean;
|
||||
}
|
||||
|
||||
interface DatabaseEntry {
|
||||
readonly key: string;
|
||||
readonly value: unknown;
|
||||
}
|
||||
|
||||
type SanitizeFeaturesFunction = (features: Record<string, unknown> | ThreatFeatures) => SanitizedFeatures;
|
||||
|
||||
// =============================================================================
|
||||
// DATABASE INITIALIZATION
|
||||
// =============================================================================
|
||||
|
||||
// Database paths
|
||||
const threatDBPath = join(rootDir, 'db', 'threats');
|
||||
const behaviorDBPath = join(rootDir, 'db', 'behavior');
|
||||
|
||||
// Ensure database directories exist
|
||||
fs.mkdirSync(threatDBPath, { recursive: true });
|
||||
fs.mkdirSync(behaviorDBPath, { recursive: true });
|
||||
|
||||
// Add read stream support for LevelDB
|
||||
function addReadStreamSupport(dbInstance: any): LevelDatabase {
|
||||
if (!dbInstance.createReadStream) {
|
||||
dbInstance.createReadStream = (opts?: DatabaseStreamOptions): AsyncIterable<DatabaseEntry> =>
|
||||
Readable.from((async function* () {
|
||||
for await (const [key, value] of dbInstance.iterator(opts)) {
|
||||
yield { key, value };
|
||||
}
|
||||
})());
|
||||
}
|
||||
return dbInstance as LevelDatabase;
|
||||
}
|
||||
|
||||
// Initialize databases with proper TTL and stream support
|
||||
const rawThreatDB = addReadStreamSupport(new Level(threatDBPath, { valueEncoding: 'json' }));
|
||||
export const threatDB: LevelDatabase = addReadStreamSupport(
|
||||
ttl(rawThreatDB, { defaultTTL: DB_TTL_CONFIG.THREAT_DB_TTL })
|
||||
);
|
||||
|
||||
const rawBehaviorDB = addReadStreamSupport(new Level(behaviorDBPath, { valueEncoding: 'json' }));
|
||||
export const behaviorDB: LevelDatabase = addReadStreamSupport(
|
||||
ttl(rawBehaviorDB, { defaultTTL: DB_TTL_CONFIG.BEHAVIOR_DB_TTL })
|
||||
);
|
||||
|
||||
// =============================================================================
|
||||
// DATABASE OPERATIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Stores a threat assessment in the database with automatic TTL
|
||||
* @param clientIP - The IP address being assessed
|
||||
* @param assessment - The threat assessment data
|
||||
*/
|
||||
export async function storeAssessment(clientIP: string, assessment: ThreatAssessment | AssessmentData): Promise<void> {
|
||||
try {
|
||||
// Input validation
|
||||
if (!clientIP || typeof clientIP !== 'string') {
|
||||
throw new Error('Invalid client IP provided');
|
||||
}
|
||||
|
||||
if (!assessment || typeof assessment !== 'object') {
|
||||
throw new Error('Invalid assessment data provided');
|
||||
}
|
||||
|
||||
const key = `assessment:${clientIP}:${Date.now()}`;
|
||||
|
||||
// Store assessment with TTL to prevent unbounded growth
|
||||
await threatDB.put(key, assessment);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
// CRITICAL: Database errors should not crash the threat scorer
|
||||
// Log the error but continue processing - the system can function without
|
||||
// storing assessments, though learning capabilities will be reduced
|
||||
console.error('Failed to store threat assessment:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates behavioral models based on observed client behavior
|
||||
* @param clientIP - The IP address to update
|
||||
* @param features - Extracted threat features
|
||||
* @param score - Calculated threat score
|
||||
* @param sanitizeFeatures - Function to sanitize features for storage
|
||||
*/
|
||||
export async function updateBehavioralModels(
|
||||
clientIP: string,
|
||||
features: Record<string, unknown> | ThreatFeatures,
|
||||
score: number,
|
||||
sanitizeFeatures: SanitizeFeaturesFunction
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Input validation
|
||||
if (!clientIP || typeof clientIP !== 'string') {
|
||||
throw new Error('Invalid client IP provided');
|
||||
}
|
||||
|
||||
if (typeof score !== 'number' || score < 0 || score > 100) {
|
||||
throw new Error('Invalid threat score provided');
|
||||
}
|
||||
|
||||
// Batch database operations for better performance
|
||||
const operations: DatabaseOperation[] = [];
|
||||
|
||||
// Update IP behavior history
|
||||
const behaviorKey = `behavior:${clientIP}`;
|
||||
const existingBehavior = await getBehaviorData(clientIP);
|
||||
|
||||
const behaviorData: BehaviorData = {
|
||||
lastScore: score,
|
||||
lastSeen: Date.now(),
|
||||
features: sanitizeFeatures(features) as unknown as Record<string, unknown>,
|
||||
requestCount: (existingBehavior?.requestCount || 0) + 1
|
||||
};
|
||||
|
||||
operations.push({
|
||||
type: 'put',
|
||||
key: behaviorKey,
|
||||
value: behaviorData
|
||||
});
|
||||
|
||||
// Update reputation based on observed behavior (automatic reputation management)
|
||||
await updateIPReputation(clientIP, score, features as ThreatFeatures, operations);
|
||||
|
||||
// Execute batch operation if we have operations to perform
|
||||
if (operations.length > 0) {
|
||||
await behaviorDB.batch(operations);
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
// Log but don't throw - behavioral model updates shouldn't crash the system
|
||||
console.error('Failed to update behavioral models:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatic IP reputation management based on observed behavior
|
||||
* @param clientIP - The IP address to update
|
||||
* @param score - Current threat score
|
||||
* @param features - Threat features detected
|
||||
* @param operations - Array to append database operations to
|
||||
*/
|
||||
export async function updateIPReputation(
|
||||
clientIP: string,
|
||||
score: number,
|
||||
features: ThreatFeatures,
|
||||
operations: DatabaseOperation[]
|
||||
): Promise<void> {
|
||||
try {
|
||||
const currentRep: ReputationData = await getReputationData(clientIP) || {
|
||||
score: 0,
|
||||
incidents: 0,
|
||||
blacklisted: false,
|
||||
tags: [],
|
||||
firstSeen: Date.now(),
|
||||
lastUpdate: Date.now(),
|
||||
source: 'dynamic'
|
||||
};
|
||||
|
||||
let reputationChanged = false;
|
||||
const now = Date.now();
|
||||
|
||||
// Automatic reputation scoring based on behavior
|
||||
if (score >= 90) {
|
||||
// Critical threat - significant reputation penalty
|
||||
currentRep.score = Math.min(100, currentRep.score + 25);
|
||||
currentRep.incidents += 1;
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'critical_threat']));
|
||||
reputationChanged = true;
|
||||
} else if (score >= 75) {
|
||||
// High threat - moderate reputation penalty
|
||||
currentRep.score = Math.min(100, currentRep.score + 15);
|
||||
currentRep.incidents += 1;
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'high_threat']));
|
||||
reputationChanged = true;
|
||||
} else if (score >= 50) {
|
||||
// Medium threat - small reputation penalty
|
||||
currentRep.score = Math.min(100, currentRep.score + 5);
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'medium_threat']));
|
||||
reputationChanged = true;
|
||||
} else if (score <= 10) {
|
||||
// Very low threat - slowly improve reputation for good behavior
|
||||
currentRep.score = Math.max(0, currentRep.score - 1);
|
||||
if (currentRep.score === 0) {
|
||||
currentRep.tags = currentRep.tags.filter(tag => !tag.includes('threat'));
|
||||
}
|
||||
reputationChanged = true;
|
||||
}
|
||||
|
||||
// Add specific behavior tags for detailed tracking
|
||||
if (features.userAgent?.isAttackTool) {
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'attack_tool']));
|
||||
currentRep.score = Math.min(100, currentRep.score + 20);
|
||||
reputationChanged = true;
|
||||
}
|
||||
|
||||
if (features.pattern?.patternAnomalies?.includes('enumeration_detected')) {
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'enumeration']));
|
||||
currentRep.score = Math.min(100, currentRep.score + 10);
|
||||
reputationChanged = true;
|
||||
}
|
||||
|
||||
if (features.pattern?.patternAnomalies?.includes('bruteforce_detected')) {
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'bruteforce']));
|
||||
currentRep.score = Math.min(100, currentRep.score + 15);
|
||||
reputationChanged = true;
|
||||
}
|
||||
|
||||
if (features.velocity?.impossibleTravel) {
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'impossible_travel']));
|
||||
currentRep.score = Math.min(100, currentRep.score + 12);
|
||||
reputationChanged = true;
|
||||
}
|
||||
|
||||
// Automatic blacklisting for consistently bad actors
|
||||
if (currentRep.score >= 80 && currentRep.incidents >= 5) {
|
||||
currentRep.blacklisted = true;
|
||||
currentRep.tags = Array.from(new Set([...currentRep.tags, 'auto_blacklisted']));
|
||||
reputationChanged = true;
|
||||
console.log(`Threat scorer: Auto-blacklisted ${clientIP} (score: ${currentRep.score}, incidents: ${currentRep.incidents})`);
|
||||
}
|
||||
|
||||
// Automatic reputation decay over time (good IPs recover slowly)
|
||||
const daysSinceLastUpdate = (now - currentRep.lastUpdate) / (1000 * 60 * 60 * 24);
|
||||
if (daysSinceLastUpdate > 7 && currentRep.score > 0) {
|
||||
// Decay reputation by 1 point per week for inactive IPs
|
||||
const decayAmount = Math.floor(daysSinceLastUpdate / 7);
|
||||
currentRep.score = Math.max(0, currentRep.score - decayAmount);
|
||||
if (currentRep.score < 50) {
|
||||
currentRep.blacklisted = false; // Unblacklist if score drops
|
||||
}
|
||||
reputationChanged = true;
|
||||
}
|
||||
|
||||
// Only update database if reputation actually changed
|
||||
if (reputationChanged) {
|
||||
currentRep.lastUpdate = now;
|
||||
operations.push({
|
||||
type: 'put',
|
||||
key: `reputation:${clientIP}`,
|
||||
value: currentRep
|
||||
});
|
||||
|
||||
console.log(`Threat scorer: Updated reputation for ${clientIP}: score=${currentRep.score}, incidents=${currentRep.incidents}, tags=[${currentRep.tags.join(', ')}]`);
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.error('Failed to update IP reputation:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// HELPER METHODS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Retrieves behavioral data for a specific IP address
|
||||
* @param clientIP - The IP address to look up
|
||||
* @returns Behavioral data or null if not found
|
||||
*/
|
||||
export async function getBehaviorData(clientIP: string): Promise<BehaviorData | null> {
|
||||
try {
|
||||
if (!clientIP || typeof clientIP !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await behaviorDB.get(`behavior:${clientIP}`);
|
||||
return data as BehaviorData;
|
||||
} catch (err) {
|
||||
return null; // Key doesn't exist or database error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves reputation data for a specific IP address
|
||||
* @param clientIP - The IP address to look up
|
||||
* @returns Reputation data or null if not found
|
||||
*/
|
||||
export async function getReputationData(clientIP: string): Promise<ReputationData | null> {
|
||||
try {
|
||||
if (!clientIP || typeof clientIP !== 'string') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = await threatDB.get(`reputation:${clientIP}`);
|
||||
return data as ReputationData;
|
||||
} catch (err) {
|
||||
return null; // Key doesn't exist or database error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets request history from database within a specific time window
|
||||
* @param ip - The IP address to get history for
|
||||
* @param timeWindow - Time window in milliseconds
|
||||
* @returns Array of request history entries
|
||||
*/
|
||||
export async function getRequestHistory(ip: string, timeWindow: number): Promise<RequestHistoryEntry[]> {
|
||||
const history: RequestHistoryEntry[] = [];
|
||||
|
||||
// Input validation
|
||||
if (!ip || typeof ip !== 'string') {
|
||||
return history;
|
||||
}
|
||||
|
||||
if (typeof timeWindow !== 'number' || timeWindow <= 0) {
|
||||
return history;
|
||||
}
|
||||
|
||||
const cutoff = Date.now() - timeWindow;
|
||||
|
||||
try {
|
||||
// Get from database
|
||||
const stream = threatDB.createReadStream({
|
||||
gte: `request:${ip}:${cutoff}`,
|
||||
lte: `request:${ip}:${Date.now()}`,
|
||||
limit: 1000
|
||||
});
|
||||
|
||||
for await (const { value } of stream) {
|
||||
const entry = value as RequestHistoryEntry;
|
||||
if (entry.timestamp && entry.timestamp > cutoff) {
|
||||
history.push(entry);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to get request history:', error.message);
|
||||
}
|
||||
|
||||
return history;
|
||||
}
|
||||
|
||||
/**
|
||||
* One-time migration of static IP reputation data to database
|
||||
* Safely migrates existing JSON reputation data to the new database format
|
||||
*/
|
||||
export async function migrateStaticReputationData(): Promise<void> {
|
||||
try {
|
||||
const ipReputationPath = join(rootDir, 'data', 'ip-reputation.json');
|
||||
|
||||
if (!fs.existsSync(ipReputationPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we've already migrated
|
||||
const migrationKey = 'reputation:migration:completed';
|
||||
try {
|
||||
await threatDB.get(migrationKey);
|
||||
return; // Already migrated
|
||||
} catch (err) {
|
||||
// Not migrated yet, proceed
|
||||
}
|
||||
|
||||
console.log('Threat scorer: Migrating static IP reputation data to database...');
|
||||
|
||||
const staticDataRaw = fs.readFileSync(ipReputationPath, 'utf8');
|
||||
const staticData = JSON.parse(staticDataRaw) as Record<string, StaticReputationEntry>;
|
||||
const operations: DatabaseOperation[] = [];
|
||||
|
||||
for (const [ip, repData] of Object.entries(staticData)) {
|
||||
// Validate IP format (basic validation)
|
||||
if (!ip || typeof ip !== 'string') {
|
||||
console.warn(`Skipping invalid IP during migration: ${ip}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const migratedData: ReputationData = {
|
||||
score: repData.score || 0,
|
||||
incidents: repData.incidents || 0,
|
||||
blacklisted: repData.blacklisted || false,
|
||||
tags: Array.isArray(repData.tags) ? [...repData.tags] : [],
|
||||
notes: repData.notes || '',
|
||||
lastUpdate: Date.now(),
|
||||
source: 'static_migration',
|
||||
migrated: true
|
||||
};
|
||||
|
||||
operations.push({
|
||||
type: 'put',
|
||||
key: `reputation:${ip}`,
|
||||
value: migratedData
|
||||
});
|
||||
}
|
||||
|
||||
// Mark migration as complete
|
||||
const migrationRecord: MigrationRecord = {
|
||||
completed: Date.now(),
|
||||
count: operations.length
|
||||
};
|
||||
|
||||
operations.push({
|
||||
type: 'put',
|
||||
key: migrationKey,
|
||||
value: migrationRecord
|
||||
});
|
||||
|
||||
if (operations.length > 1) {
|
||||
await threatDB.batch(operations);
|
||||
console.log(`Threat scorer: Migrated ${operations.length - 1} IP reputation records to database`);
|
||||
|
||||
// Optionally archive the static file
|
||||
const archivePath = ipReputationPath + '.migrated';
|
||||
fs.renameSync(ipReputationPath, archivePath);
|
||||
console.log(`Threat scorer: Static IP reputation file archived to ${archivePath}`);
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.error('Failed to migrate static IP reputation data:', error.message);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,472 +0,0 @@
|
|||
// =============================================================================
|
||||
// BEHAVIORAL FEATURE EXTRACTION - SECURE TYPESCRIPT VERSION
|
||||
// =============================================================================
|
||||
// Comprehensive behavioral pattern analysis with security hardening
|
||||
// Handles completely user-controlled behavioral data with zero trust validation
|
||||
|
||||
import { behavioralDetection } from '../../behavioral-detection.js';
|
||||
import { getRequestHistory } from '../database.js';
|
||||
import { detectAutomation } from '../analyzers/index.js';
|
||||
import { randomBytes } from 'crypto';
|
||||
import type { NetworkRequest } from '../../network.js';
|
||||
import { requireValidIP } from '../../ip-validation.js';
|
||||
|
||||
// Type definitions for secure behavioral analysis
|
||||
export interface RequestPatternFeatures {
|
||||
readonly enumerationScore: number;
|
||||
readonly crawlingScore: number;
|
||||
readonly bruteForceScore: number;
|
||||
readonly scanningScore: number;
|
||||
readonly automationScore: number;
|
||||
readonly patternAnomalies: readonly string[];
|
||||
readonly riskScore: number;
|
||||
readonly validationErrors: readonly string[];
|
||||
}
|
||||
|
||||
export interface SessionBehaviorFeatures {
|
||||
readonly sessionAge: number;
|
||||
readonly requestCount: number;
|
||||
readonly uniqueEndpoints: number;
|
||||
readonly suspiciousBehavior: boolean;
|
||||
readonly sessionAnomalies: readonly string[];
|
||||
readonly riskScore: number;
|
||||
readonly validationErrors: readonly string[];
|
||||
}
|
||||
|
||||
interface BehavioralPattern {
|
||||
readonly type: string;
|
||||
readonly score: number;
|
||||
}
|
||||
|
||||
// Security constants for behavioral validation
|
||||
const MAX_PATTERN_ANOMALIES = 20; // Prevent memory exhaustion
|
||||
const MAX_SESSION_ANOMALIES = 15; // Limit session anomaly collection
|
||||
const MAX_VALIDATION_ERRORS = 10; // Prevent error collection bloat
|
||||
const MAX_SESSION_ID_LENGTH = 256; // Reasonable session ID limit
|
||||
const MIN_SESSION_ID_LENGTH = 8; // Minimum for security
|
||||
const MAX_COOKIE_LENGTH = 4096; // Standard cookie size limit
|
||||
const MAX_HEADER_VALUE_LENGTH = 8192; // HTTP header limit
|
||||
const COOKIE_PARSE_TIMEOUT = 50; // 50ms timeout for cookie parsing
|
||||
const MAX_SCORE_VALUE = 100; // Maximum behavioral score
|
||||
const MIN_SCORE_VALUE = 0; // Minimum behavioral score
|
||||
|
||||
// Valid session ID pattern (alphanumeric + common safe characters)
|
||||
const SESSION_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
|
||||
|
||||
// Input validation functions with zero trust approach
|
||||
|
||||
function validateNetworkRequest(request: unknown): NetworkRequest {
|
||||
if (!request || typeof request !== 'object') {
|
||||
throw new Error('Request must be an object');
|
||||
}
|
||||
|
||||
const req = request as Record<string, unknown>;
|
||||
|
||||
// Validate headers exist and are an object
|
||||
if (!req.headers || typeof req.headers !== 'object') {
|
||||
throw new Error('Request must have headers object');
|
||||
}
|
||||
|
||||
return request as NetworkRequest;
|
||||
}
|
||||
|
||||
function validateResponse(response: unknown): Record<string, unknown> {
|
||||
if (!response || typeof response !== 'object') {
|
||||
// Return safe default if no response provided
|
||||
return { status: 200 };
|
||||
}
|
||||
|
||||
const resp = response as Record<string, unknown>;
|
||||
|
||||
// Validate status code if present
|
||||
if (resp.status !== undefined) {
|
||||
if (typeof resp.status !== 'number' || resp.status < 100 || resp.status > 599) {
|
||||
throw new Error('Invalid response status code');
|
||||
}
|
||||
}
|
||||
|
||||
return resp;
|
||||
}
|
||||
|
||||
function validateSessionId(sessionId: unknown): string {
|
||||
if (!sessionId) {
|
||||
throw new Error('Session ID is required');
|
||||
}
|
||||
|
||||
if (typeof sessionId !== 'string') {
|
||||
throw new Error('Session ID must be a string');
|
||||
}
|
||||
|
||||
if (sessionId.length < MIN_SESSION_ID_LENGTH || sessionId.length > MAX_SESSION_ID_LENGTH) {
|
||||
throw new Error(`Session ID length must be between ${MIN_SESSION_ID_LENGTH} and ${MAX_SESSION_ID_LENGTH} characters`);
|
||||
}
|
||||
|
||||
if (!SESSION_ID_PATTERN.test(sessionId)) {
|
||||
throw new Error('Session ID contains invalid characters');
|
||||
}
|
||||
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
function validateBehavioralScore(score: unknown): number {
|
||||
if (typeof score !== 'number') {
|
||||
return 0; // Default to 0 for invalid scores
|
||||
}
|
||||
|
||||
if (!Number.isFinite(score)) {
|
||||
return 0; // Handle NaN and Infinity
|
||||
}
|
||||
|
||||
// Clamp score to valid range
|
||||
return Math.max(MIN_SCORE_VALUE, Math.min(MAX_SCORE_VALUE, score));
|
||||
}
|
||||
|
||||
function validateBehavioralPattern(pattern: unknown): BehavioralPattern | null {
|
||||
if (!pattern || typeof pattern !== 'object') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const p = pattern as Record<string, unknown>;
|
||||
|
||||
if (typeof p.type !== 'string' || p.type.length === 0 || p.type.length > 50) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const validatedScore = validateBehavioralScore(p.score);
|
||||
|
||||
return {
|
||||
type: p.type,
|
||||
score: validatedScore
|
||||
};
|
||||
}
|
||||
|
||||
function normalizePatternScore(score: number): number {
|
||||
// Normalize behavioral scores to 0-1 range
|
||||
return Math.max(0, Math.min(1, score / 50));
|
||||
}
|
||||
|
||||
// Safe cookie parsing with timeout protection
|
||||
function parseCookieValue(cookieString: string, name: string): string | null {
|
||||
if (!cookieString || cookieString.length > MAX_COOKIE_LENGTH) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Simple cookie parsing with timeout protection
|
||||
const cookies = cookieString.split(';');
|
||||
|
||||
for (const cookie of cookies) {
|
||||
// Timeout protection
|
||||
if (Date.now() - startTime > COOKIE_PARSE_TIMEOUT) {
|
||||
break;
|
||||
}
|
||||
|
||||
const [cookieName, ...cookieValueParts] = cookie.split('=');
|
||||
if (cookieName?.trim() === name) {
|
||||
const value = cookieValueParts.join('=').trim();
|
||||
return value.length > 0 ? value : null;
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// Parsing error - return null
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// Safe header value extraction
|
||||
function getHeaderValue(headers: Record<string, unknown>, name: string): string | null {
|
||||
const value = headers[name] || headers[name.toLowerCase()];
|
||||
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof value !== 'string') {
|
||||
const stringValue = String(value);
|
||||
if (stringValue.length > MAX_HEADER_VALUE_LENGTH) {
|
||||
return null;
|
||||
}
|
||||
return stringValue;
|
||||
}
|
||||
|
||||
if (value.length > MAX_HEADER_VALUE_LENGTH) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
// Secure request pattern feature extraction
|
||||
export async function extractRequestPatternFeatures(
|
||||
ip: unknown,
|
||||
request: unknown,
|
||||
response?: unknown
|
||||
): Promise<RequestPatternFeatures> {
|
||||
const validationErrors: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Initialize safe default values
|
||||
let enumerationScore = 0;
|
||||
let crawlingScore = 0;
|
||||
let bruteForceScore = 0;
|
||||
let scanningScore = 0;
|
||||
let automationScore = 0;
|
||||
const patternAnomalies: string[] = [];
|
||||
|
||||
try {
|
||||
// Validate inputs with zero trust
|
||||
const validatedIP = requireValidIP(ip);
|
||||
const validatedRequest = validateNetworkRequest(request);
|
||||
const validatedResponse = validateResponse(response);
|
||||
|
||||
// Perform behavioral analysis with error handling
|
||||
try {
|
||||
const behavioralAnalysis = await behavioralDetection.analyzeRequest(
|
||||
validatedIP,
|
||||
validatedRequest,
|
||||
validatedResponse
|
||||
);
|
||||
|
||||
// Validate and process behavioral patterns
|
||||
if (behavioralAnalysis && Array.isArray(behavioralAnalysis.patterns)) {
|
||||
for (const rawPattern of behavioralAnalysis.patterns) {
|
||||
const pattern = validateBehavioralPattern(rawPattern);
|
||||
if (!pattern) {
|
||||
continue; // Skip invalid patterns
|
||||
}
|
||||
|
||||
const normalizedScore = normalizePatternScore(pattern.score);
|
||||
|
||||
switch (pattern.type) {
|
||||
case 'enumeration':
|
||||
enumerationScore = Math.max(enumerationScore, normalizedScore);
|
||||
if (!patternAnomalies.includes('enumeration_detected')) {
|
||||
patternAnomalies.push('enumeration_detected');
|
||||
}
|
||||
riskScore += normalizedScore * 30; // High risk for enumeration
|
||||
break;
|
||||
|
||||
case 'bruteforce':
|
||||
bruteForceScore = Math.max(bruteForceScore, normalizedScore);
|
||||
if (!patternAnomalies.includes('bruteforce_detected')) {
|
||||
patternAnomalies.push('bruteforce_detected');
|
||||
}
|
||||
riskScore += normalizedScore * 40; // Very high risk for brute force
|
||||
break;
|
||||
|
||||
case 'scanning':
|
||||
scanningScore = Math.max(scanningScore, normalizedScore);
|
||||
if (!patternAnomalies.includes('scanning_detected')) {
|
||||
patternAnomalies.push('scanning_detected');
|
||||
}
|
||||
riskScore += normalizedScore * 35; // High risk for scanning
|
||||
break;
|
||||
|
||||
case 'abuse':
|
||||
crawlingScore = Math.max(crawlingScore, normalizedScore);
|
||||
if (!patternAnomalies.includes('abuse_detected')) {
|
||||
patternAnomalies.push('abuse_detected');
|
||||
}
|
||||
riskScore += normalizedScore * 25; // Medium-high risk for abuse
|
||||
break;
|
||||
}
|
||||
|
||||
// Limit pattern anomalies to prevent memory exhaustion
|
||||
if (patternAnomalies.length >= MAX_PATTERN_ANOMALIES) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (behavioralError) {
|
||||
validationErrors.push('behavioral_analysis_failed');
|
||||
riskScore += 20; // Medium penalty for analysis failure
|
||||
}
|
||||
|
||||
// Detect automation with error handling
|
||||
try {
|
||||
const history = await getRequestHistory(validatedIP, 300000); // Last 5 minutes
|
||||
const rawAutomationScore = detectAutomation(history);
|
||||
automationScore = validateBehavioralScore(rawAutomationScore);
|
||||
|
||||
if (automationScore > 0.7) {
|
||||
patternAnomalies.push('automation_detected');
|
||||
riskScore += automationScore * 30; // Risk based on automation level
|
||||
}
|
||||
|
||||
} catch (automationError) {
|
||||
validationErrors.push('automation_detection_failed');
|
||||
riskScore += 10; // Small penalty for detection failure
|
||||
}
|
||||
|
||||
} catch (validationError) {
|
||||
// Critical validation failure
|
||||
validationErrors.push('input_validation_failed');
|
||||
riskScore = 100; // Maximum risk for validation failure
|
||||
}
|
||||
|
||||
// Cap risk score and limit collections
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
const limitedErrors = validationErrors.slice(0, MAX_VALIDATION_ERRORS);
|
||||
const limitedAnomalies = patternAnomalies.slice(0, MAX_PATTERN_ANOMALIES);
|
||||
|
||||
return {
|
||||
enumerationScore,
|
||||
crawlingScore,
|
||||
bruteForceScore,
|
||||
scanningScore,
|
||||
automationScore,
|
||||
patternAnomalies: limitedAnomalies,
|
||||
riskScore: finalRiskScore,
|
||||
validationErrors: limitedErrors
|
||||
};
|
||||
}
|
||||
|
||||
// Secure session behavior feature extraction
|
||||
export async function extractSessionBehaviorFeatures(
|
||||
sessionId: unknown,
|
||||
request: unknown
|
||||
): Promise<SessionBehaviorFeatures> {
|
||||
const validationErrors: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Initialize safe default values
|
||||
let sessionAge = 0;
|
||||
let requestCount = 0;
|
||||
let uniqueEndpoints = 0;
|
||||
let suspiciousBehavior = false;
|
||||
const sessionAnomalies: string[] = [];
|
||||
|
||||
try {
|
||||
// Handle missing session ID
|
||||
if (!sessionId) {
|
||||
sessionAnomalies.push('missing_session');
|
||||
validationErrors.push('session_id_missing');
|
||||
riskScore += 25; // Medium risk for missing session
|
||||
|
||||
return {
|
||||
sessionAge,
|
||||
requestCount,
|
||||
uniqueEndpoints,
|
||||
suspiciousBehavior,
|
||||
sessionAnomalies: sessionAnomalies.slice(0, MAX_SESSION_ANOMALIES),
|
||||
riskScore,
|
||||
validationErrors: validationErrors.slice(0, MAX_VALIDATION_ERRORS)
|
||||
};
|
||||
}
|
||||
|
||||
// Validate inputs
|
||||
const validatedSessionId = validateSessionId(sessionId);
|
||||
const validatedRequest = validateNetworkRequest(request);
|
||||
|
||||
// Safely extract headers
|
||||
const headers = validatedRequest.headers as Record<string, unknown>;
|
||||
|
||||
// Check for session hijacking indicators
|
||||
try {
|
||||
const secFetchSite = getHeaderValue(headers, 'sec-fetch-site');
|
||||
const referer = getHeaderValue(headers, 'referer');
|
||||
|
||||
if (secFetchSite === 'cross-site' && !referer) {
|
||||
sessionAnomalies.push('cross_site_no_referer');
|
||||
suspiciousBehavior = true;
|
||||
riskScore += 30; // High risk for potential session hijacking
|
||||
}
|
||||
|
||||
} catch (headerError) {
|
||||
validationErrors.push('header_analysis_failed');
|
||||
riskScore += 5; // Small penalty
|
||||
}
|
||||
|
||||
// Check for session manipulation in cookies
|
||||
try {
|
||||
const cookieHeader = getHeaderValue(headers, 'cookie');
|
||||
if (cookieHeader) {
|
||||
// Count session ID occurrences safely
|
||||
const sessionIdCount = (cookieHeader.match(/session_id=/g) || []).length;
|
||||
if (sessionIdCount > 1) {
|
||||
sessionAnomalies.push('multiple_session_ids');
|
||||
suspiciousBehavior = true;
|
||||
riskScore += 40; // High risk for session manipulation
|
||||
}
|
||||
|
||||
// Check for session ID in unexpected places
|
||||
if (cookieHeader.includes('session_id=') && cookieHeader.includes('sid=')) {
|
||||
sessionAnomalies.push('duplicate_session_mechanisms');
|
||||
suspiciousBehavior = true;
|
||||
riskScore += 25; // Medium-high risk
|
||||
}
|
||||
}
|
||||
|
||||
} catch (cookieError) {
|
||||
validationErrors.push('cookie_analysis_failed');
|
||||
riskScore += 5; // Small penalty
|
||||
}
|
||||
|
||||
// Additional session validation
|
||||
if (validatedSessionId.length > 128) {
|
||||
sessionAnomalies.push('oversized_session_id');
|
||||
suspiciousBehavior = true;
|
||||
riskScore += 20; // Medium risk
|
||||
}
|
||||
|
||||
} catch (validationError) {
|
||||
// Critical validation failure
|
||||
validationErrors.push('session_validation_failed');
|
||||
riskScore = 100; // Maximum risk for validation failure
|
||||
suspiciousBehavior = true;
|
||||
}
|
||||
|
||||
// Cap risk score and limit collections
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
const limitedErrors = validationErrors.slice(0, MAX_VALIDATION_ERRORS);
|
||||
const limitedAnomalies = sessionAnomalies.slice(0, MAX_SESSION_ANOMALIES);
|
||||
|
||||
return {
|
||||
sessionAge,
|
||||
requestCount,
|
||||
uniqueEndpoints,
|
||||
suspiciousBehavior,
|
||||
sessionAnomalies: limitedAnomalies,
|
||||
riskScore: finalRiskScore,
|
||||
validationErrors: limitedErrors
|
||||
};
|
||||
}
|
||||
|
||||
// Secure session ID extraction and generation
|
||||
export function getSessionId(request: unknown): string {
|
||||
try {
|
||||
const validatedRequest = validateNetworkRequest(request);
|
||||
const headers = validatedRequest.headers as Record<string, unknown>;
|
||||
|
||||
// Extract session ID from cookies safely
|
||||
const cookieHeader = getHeaderValue(headers, 'cookie');
|
||||
if (cookieHeader) {
|
||||
const sessionId = parseCookieValue(cookieHeader, 'session_id');
|
||||
if (sessionId) {
|
||||
try {
|
||||
// Validate extracted session ID
|
||||
return validateSessionId(sessionId);
|
||||
} catch (error) {
|
||||
// Invalid session ID - generate new one
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// Extraction failed - generate new session ID
|
||||
}
|
||||
|
||||
// Generate new session ID with error handling
|
||||
try {
|
||||
return randomBytes(16).toString('hex');
|
||||
} catch (cryptoError) {
|
||||
// Fallback to timestamp-based ID if crypto fails
|
||||
return `fallback_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,450 +0,0 @@
|
|||
// =============================================================================
|
||||
// CONTENT FEATURE EXTRACTION - SECURE TYPESCRIPT VERSION
|
||||
// =============================================================================
|
||||
// Comprehensive content analysis with JSON bomb protection and ReDoS prevention
|
||||
// Handles completely user-controlled request bodies and URL parameters with zero trust
|
||||
|
||||
import { calculateEntropy, detectEncodingLevels } from '../analyzers/index.js';
|
||||
import type { NetworkRequest } from '../../network.js';
|
||||
|
||||
// Type definitions for secure content analysis
|
||||
export interface PayloadFeatures {
|
||||
readonly payloadSize: number;
|
||||
readonly hasSQLPatterns: boolean;
|
||||
readonly hasXSSPatterns: boolean;
|
||||
readonly hasCommandPatterns: boolean;
|
||||
readonly hasPathTraversal: boolean;
|
||||
readonly encodingLevels: number;
|
||||
readonly entropy: number;
|
||||
readonly suspiciousPatterns: readonly string[];
|
||||
readonly riskScore: number;
|
||||
readonly processingErrors: readonly string[];
|
||||
}
|
||||
|
||||
export interface NormalizedWAFSignals {
|
||||
readonly sqlInjection: boolean;
|
||||
readonly xss: boolean;
|
||||
readonly commandInjection: boolean;
|
||||
readonly pathTraversal: boolean;
|
||||
readonly totalViolations: number;
|
||||
}
|
||||
|
||||
// Security constants for content processing
|
||||
const MAX_URL_LENGTH = 8192; // 8KB max URL length
|
||||
const MAX_QUERY_STRING_LENGTH = 4096; // 4KB max query string
|
||||
const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10MB max body size
|
||||
const MAX_ENCODING_LEVELS = 10; // Prevent infinite decoding loops
|
||||
const REGEX_TIMEOUT_MS = 50; // Prevent ReDoS attacks (shorter for content analysis)
|
||||
const MAX_SUSPICIOUS_PATTERNS = 100; // Prevent memory exhaustion
|
||||
const MAX_JSON_STRINGIFY_SIZE = 1024 * 1024; // 1MB max for JSON.stringify
|
||||
|
||||
// Safe regex patterns with ReDoS protection
|
||||
const SAFE_CONTENT_PATTERNS = {
|
||||
// SQL injection patterns (simplified to prevent ReDoS)
|
||||
SQL_KEYWORDS: /\b(union|select|insert|update|delete|drop|create|alter|exec|script)\b/gi,
|
||||
SQL_CHARS: /--|\/\*|\*\//g,
|
||||
|
||||
// XSS patterns (simplified and safe)
|
||||
XSS_TAGS: /<\/?[a-z][^>]*>/gi,
|
||||
XSS_EVENTS: /\bon[a-z]+\s*=/gi,
|
||||
XSS_JAVASCRIPT: /javascript\s*:/gi,
|
||||
XSS_SCRIPT: /<script[^>]*>/gi,
|
||||
|
||||
// Command injection patterns
|
||||
COMMAND_CHARS: /[;&|`]/g,
|
||||
COMMAND_VARS: /\$\([^)]*\)/g,
|
||||
ENCODED_NEWLINES: /%0[ad]/gi,
|
||||
|
||||
// Path traversal patterns
|
||||
PATH_DOTS: /\.\.[\\/]/g,
|
||||
ENCODED_DOTS: /%2e%2e|%252e%252e/gi
|
||||
} as const;
|
||||
|
||||
// Input validation functions with zero trust approach
|
||||
function validateRequestInput(request: unknown): NetworkRequest & { body?: unknown } {
|
||||
if (!request || typeof request !== 'object') {
|
||||
throw new Error('Request must be an object');
|
||||
}
|
||||
|
||||
return request as NetworkRequest & { body?: unknown };
|
||||
}
|
||||
|
||||
function validateAndSanitizeURL(url: unknown): string {
|
||||
if (typeof url !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (url.length > MAX_URL_LENGTH) {
|
||||
throw new Error(`URL exceeds maximum length of ${MAX_URL_LENGTH} characters`);
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
function validateRequestBody(body: unknown): unknown {
|
||||
if (body === null || body === undefined) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if it's a string
|
||||
if (typeof body === 'string') {
|
||||
if (body.length > MAX_BODY_SIZE) {
|
||||
throw new Error(`Request body string exceeds maximum size of ${MAX_BODY_SIZE} characters`);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
// For objects, we'll validate during JSON.stringify with size limits
|
||||
return body;
|
||||
}
|
||||
|
||||
// Safe JSON.stringify with protection against circular references and size limits
|
||||
function safeJSONStringify(obj: unknown, maxSize: number = MAX_JSON_STRINGIFY_SIZE): string {
|
||||
if (obj === null || obj === undefined) {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (typeof obj === 'string') {
|
||||
return obj;
|
||||
}
|
||||
|
||||
try {
|
||||
// Use a replacer to detect circular references and limit depth
|
||||
const seen = new WeakSet();
|
||||
let depth = 0;
|
||||
const maxDepth = 50; // Prevent deeply nested JSON bombs
|
||||
|
||||
const replacer = (_key: string, value: unknown): unknown => {
|
||||
if (depth++ > maxDepth) {
|
||||
return '[Max Depth Exceeded]';
|
||||
}
|
||||
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
if (seen.has(value)) {
|
||||
return '[Circular Reference]';
|
||||
}
|
||||
seen.add(value);
|
||||
}
|
||||
|
||||
depth--;
|
||||
return value;
|
||||
};
|
||||
|
||||
const jsonString = JSON.stringify(obj, replacer);
|
||||
|
||||
if (jsonString.length > maxSize) {
|
||||
throw new Error(`JSON string exceeds maximum size of ${maxSize} characters`);
|
||||
}
|
||||
|
||||
return jsonString;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes('maximum size')) {
|
||||
throw error; // Re-throw size errors
|
||||
}
|
||||
// For other JSON errors (circular refs, etc.), return safe fallback
|
||||
return '[JSON Serialization Error]';
|
||||
}
|
||||
}
|
||||
|
||||
// ReDoS-safe pattern matching with timeout protection
|
||||
function safePatternTest(pattern: RegExp, input: string, timeoutMs: number = REGEX_TIMEOUT_MS): boolean {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Reset regex state
|
||||
pattern.lastIndex = 0;
|
||||
|
||||
// Limit input size for regex processing to prevent catastrophic backtracking
|
||||
const limitedInput = input.length > 10000 ? input.substring(0, 10000) : input;
|
||||
|
||||
const result = pattern.test(limitedInput);
|
||||
|
||||
if (Date.now() - startTime > timeoutMs) {
|
||||
throw new Error('Regex execution timeout - possible ReDoS attack');
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes('timeout')) {
|
||||
throw error; // Re-throw timeout errors for logging
|
||||
}
|
||||
// For other regex errors, assume no match (fail safe)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Secure content analysis with comprehensive validation
|
||||
function analyzeContentSafely(content: string, _contentType: string): {
|
||||
hasSQLPatterns: boolean;
|
||||
hasXSSPatterns: boolean;
|
||||
hasCommandPatterns: boolean;
|
||||
hasPathTraversal: boolean;
|
||||
suspiciousPatterns: string[];
|
||||
encodingLevels: number;
|
||||
entropy: number;
|
||||
processingErrors: string[];
|
||||
} {
|
||||
const suspiciousPatterns: string[] = [];
|
||||
const processingErrors: string[] = [];
|
||||
let hasSQLPatterns = false;
|
||||
let hasXSSPatterns = false;
|
||||
let hasCommandPatterns = false;
|
||||
let hasPathTraversal = false;
|
||||
let encodingLevels = 0;
|
||||
let entropy = 0;
|
||||
|
||||
try {
|
||||
// SQL injection detection with safe patterns
|
||||
try {
|
||||
if (safePatternTest(SAFE_CONTENT_PATTERNS.SQL_KEYWORDS, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.SQL_CHARS, content)) {
|
||||
hasSQLPatterns = true;
|
||||
suspiciousPatterns.push('sql_keywords');
|
||||
}
|
||||
} catch (error) {
|
||||
processingErrors.push('sql_detection_timeout');
|
||||
}
|
||||
|
||||
// XSS detection with safe patterns
|
||||
try {
|
||||
if (safePatternTest(SAFE_CONTENT_PATTERNS.XSS_TAGS, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.XSS_EVENTS, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.XSS_JAVASCRIPT, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.XSS_SCRIPT, content)) {
|
||||
hasXSSPatterns = true;
|
||||
suspiciousPatterns.push('xss_patterns');
|
||||
}
|
||||
} catch (error) {
|
||||
processingErrors.push('xss_detection_timeout');
|
||||
}
|
||||
|
||||
// Command injection detection with safe patterns
|
||||
try {
|
||||
if (safePatternTest(SAFE_CONTENT_PATTERNS.COMMAND_CHARS, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.COMMAND_VARS, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.ENCODED_NEWLINES, content)) {
|
||||
hasCommandPatterns = true;
|
||||
suspiciousPatterns.push('command_chars');
|
||||
}
|
||||
} catch (error) {
|
||||
processingErrors.push('command_detection_timeout');
|
||||
}
|
||||
|
||||
// Path traversal detection with safe patterns
|
||||
try {
|
||||
if (safePatternTest(SAFE_CONTENT_PATTERNS.PATH_DOTS, content) ||
|
||||
safePatternTest(SAFE_CONTENT_PATTERNS.ENCODED_DOTS, content)) {
|
||||
hasPathTraversal = true;
|
||||
suspiciousPatterns.push('path_traversal');
|
||||
}
|
||||
} catch (error) {
|
||||
processingErrors.push('path_detection_timeout');
|
||||
}
|
||||
|
||||
// Safe encoding level detection
|
||||
try {
|
||||
encodingLevels = Math.min(detectEncodingLevels(content), MAX_ENCODING_LEVELS);
|
||||
} catch (error) {
|
||||
processingErrors.push('encoding_detection_failed');
|
||||
encodingLevels = 0;
|
||||
}
|
||||
|
||||
// Safe entropy calculation
|
||||
try {
|
||||
entropy = calculateEntropy(content);
|
||||
} catch (error) {
|
||||
processingErrors.push('entropy_calculation_failed');
|
||||
entropy = 0;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
processingErrors.push('general_analysis_error');
|
||||
}
|
||||
|
||||
return {
|
||||
hasSQLPatterns,
|
||||
hasXSSPatterns,
|
||||
hasCommandPatterns,
|
||||
hasPathTraversal,
|
||||
suspiciousPatterns: suspiciousPatterns.slice(0, MAX_SUSPICIOUS_PATTERNS),
|
||||
encodingLevels,
|
||||
entropy,
|
||||
processingErrors
|
||||
};
|
||||
}
|
||||
|
||||
// Main payload extraction function with comprehensive security
|
||||
export async function extractPayloadFeatures(request: unknown): Promise<PayloadFeatures> {
|
||||
const processingErrors: string[] = [];
|
||||
let payloadSize = 0;
|
||||
let hasSQLPatterns = false;
|
||||
let hasXSSPatterns = false;
|
||||
let hasCommandPatterns = false;
|
||||
let hasPathTraversal = false;
|
||||
let encodingLevels = 0;
|
||||
let entropy = 0;
|
||||
let allSuspiciousPatterns: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
try {
|
||||
// Validate request input with zero trust
|
||||
const validatedRequest = validateRequestInput(request);
|
||||
|
||||
// Analyze URL parameters with validation
|
||||
try {
|
||||
const url = validateAndSanitizeURL(validatedRequest.url);
|
||||
|
||||
if (url && url.includes('?')) {
|
||||
const urlParts = url.split('?');
|
||||
if (urlParts.length > 1) {
|
||||
const queryString = urlParts[1];
|
||||
|
||||
if (queryString && queryString.length > MAX_QUERY_STRING_LENGTH) {
|
||||
processingErrors.push('query_string_too_large');
|
||||
riskScore += 30;
|
||||
} else if (queryString) {
|
||||
payloadSize += queryString.length;
|
||||
|
||||
const urlAnalysis = analyzeContentSafely(queryString, 'query_string');
|
||||
|
||||
if (urlAnalysis.hasSQLPatterns) hasSQLPatterns = true;
|
||||
if (urlAnalysis.hasXSSPatterns) hasXSSPatterns = true;
|
||||
if (urlAnalysis.hasCommandPatterns) hasCommandPatterns = true;
|
||||
if (urlAnalysis.hasPathTraversal) hasPathTraversal = true;
|
||||
|
||||
encodingLevels = Math.max(encodingLevels, urlAnalysis.encodingLevels);
|
||||
entropy = Math.max(entropy, urlAnalysis.entropy);
|
||||
allSuspiciousPatterns.push(...urlAnalysis.suspiciousPatterns);
|
||||
processingErrors.push(...urlAnalysis.processingErrors);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
processingErrors.push('url_analysis_failed');
|
||||
riskScore += 20;
|
||||
}
|
||||
|
||||
// Analyze request body with comprehensive validation
|
||||
try {
|
||||
const validatedBody = validateRequestBody(validatedRequest.body);
|
||||
|
||||
if (validatedBody !== null && validatedBody !== undefined) {
|
||||
let bodyStr: string;
|
||||
|
||||
try {
|
||||
bodyStr = typeof validatedBody === 'string'
|
||||
? validatedBody
|
||||
: safeJSONStringify(validatedBody);
|
||||
} catch (error) {
|
||||
processingErrors.push('json_stringify_failed');
|
||||
riskScore += 25;
|
||||
bodyStr = '[Body Processing Failed]';
|
||||
}
|
||||
|
||||
if (bodyStr.length > MAX_BODY_SIZE) {
|
||||
processingErrors.push('body_too_large');
|
||||
riskScore += 40;
|
||||
} else {
|
||||
payloadSize += bodyStr.length;
|
||||
|
||||
const bodyAnalysis = analyzeContentSafely(bodyStr, 'request_body');
|
||||
|
||||
if (bodyAnalysis.hasSQLPatterns) hasSQLPatterns = true;
|
||||
if (bodyAnalysis.hasXSSPatterns) hasXSSPatterns = true;
|
||||
if (bodyAnalysis.hasCommandPatterns) hasCommandPatterns = true;
|
||||
if (bodyAnalysis.hasPathTraversal) hasPathTraversal = true;
|
||||
|
||||
encodingLevels = Math.max(encodingLevels, bodyAnalysis.encodingLevels);
|
||||
entropy = Math.max(entropy, bodyAnalysis.entropy);
|
||||
|
||||
// Merge patterns, avoiding duplicates
|
||||
for (const pattern of bodyAnalysis.suspiciousPatterns) {
|
||||
if (!allSuspiciousPatterns.includes(pattern)) {
|
||||
allSuspiciousPatterns.push(pattern);
|
||||
}
|
||||
}
|
||||
|
||||
processingErrors.push(...bodyAnalysis.processingErrors);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
processingErrors.push('body_analysis_failed');
|
||||
riskScore += 30;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
processingErrors.push('request_validation_failed');
|
||||
riskScore = 100; // Maximum risk for validation failure
|
||||
}
|
||||
|
||||
// Calculate risk score based on findings - MUCH MORE AGGRESSIVE
|
||||
if (hasSQLPatterns) riskScore += 80; // Increased from 50
|
||||
if (hasXSSPatterns) riskScore += 85; // Increased from 45 - XSS is critical
|
||||
if (hasCommandPatterns) riskScore += 90; // Increased from 55 - most dangerous
|
||||
if (hasPathTraversal) riskScore += 70; // Increased from 40
|
||||
if (encodingLevels > 3) riskScore += 30; // Increased from 20 - likely evasion
|
||||
if (encodingLevels > 5) riskScore += 50; // Very suspicious encoding depth
|
||||
if (entropy > 6.0) riskScore += 25; // Increased from 15
|
||||
if (payloadSize > 1024 * 1024) riskScore += 20; // Increased from 10
|
||||
|
||||
// Limit collections to prevent memory exhaustion
|
||||
const limitedPatterns = allSuspiciousPatterns.slice(0, MAX_SUSPICIOUS_PATTERNS);
|
||||
const limitedErrors = processingErrors.slice(0, 20);
|
||||
|
||||
// Cap risk score
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
|
||||
return {
|
||||
payloadSize,
|
||||
hasSQLPatterns,
|
||||
hasXSSPatterns,
|
||||
hasCommandPatterns,
|
||||
hasPathTraversal,
|
||||
encodingLevels,
|
||||
entropy,
|
||||
suspiciousPatterns: limitedPatterns,
|
||||
riskScore: finalRiskScore,
|
||||
processingErrors: limitedErrors
|
||||
};
|
||||
}
|
||||
|
||||
// Secure WAF signal normalization with input validation
|
||||
export function normalizeWAFSignals(wafSignals: unknown): NormalizedWAFSignals {
|
||||
const defaultSignals: NormalizedWAFSignals = {
|
||||
sqlInjection: false,
|
||||
xss: false,
|
||||
commandInjection: false,
|
||||
pathTraversal: false,
|
||||
totalViolations: 0
|
||||
};
|
||||
|
||||
// Validate input
|
||||
if (!wafSignals || typeof wafSignals !== 'object') {
|
||||
return defaultSignals;
|
||||
}
|
||||
|
||||
try {
|
||||
const signals = wafSignals as Record<string, unknown>;
|
||||
|
||||
// Safely extract boolean signals
|
||||
const sqlInjection = Boolean(signals.sqlInjection || signals.sql_injection);
|
||||
const xss = Boolean(signals.xss || signals.xssAttempt);
|
||||
const commandInjection = Boolean(signals.commandInjection || signals.command_injection);
|
||||
const pathTraversal = Boolean(signals.pathTraversal || signals.path_traversal);
|
||||
|
||||
// Count total violations
|
||||
const totalViolations = [sqlInjection, xss, commandInjection, pathTraversal].filter(Boolean).length;
|
||||
|
||||
return {
|
||||
sqlInjection,
|
||||
xss,
|
||||
commandInjection,
|
||||
pathTraversal,
|
||||
totalViolations
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
// On any error, return safe defaults
|
||||
return defaultSignals;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
// =============================================================================
|
||||
// FEATURE EXTRACTOR EXPORTS (TypeScript)
|
||||
// =============================================================================
|
||||
// Central export hub for all feature extraction functions used in threat scoring
|
||||
// This module provides a clean interface for accessing all feature extractors
|
||||
|
||||
// Network-based feature extractors
|
||||
export {
|
||||
extractIPReputationFeatures,
|
||||
extractNetworkAnomalyFeatures
|
||||
} from './network.js';
|
||||
|
||||
// Behavioral feature extractors
|
||||
export {
|
||||
extractRequestPatternFeatures,
|
||||
extractSessionBehaviorFeatures,
|
||||
getSessionId
|
||||
} from './behavioral.js';
|
||||
|
||||
// Content-based feature extractors
|
||||
export {
|
||||
extractPayloadFeatures,
|
||||
normalizeWAFSignals
|
||||
} from './content.js';
|
||||
|
||||
// Temporal feature extractors
|
||||
export {
|
||||
extractTimingFeatures,
|
||||
extractVelocityFeatures
|
||||
} from './temporal.js';
|
||||
|
||||
// Header analysis features
|
||||
export {
|
||||
extractHeaderFeatures
|
||||
} from '../analyzers/headers.js';
|
||||
|
||||
// =============================================================================
|
||||
// UTILITY FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Gets a list of all available feature extractor categories
|
||||
* @returns Array of feature extractor category names
|
||||
*/
|
||||
export function getFeatureExtractorCategories(): readonly string[] {
|
||||
return ['network', 'behavioral', 'content', 'temporal', 'headers'] as const;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that all required feature extractors are available
|
||||
* @returns True if all extractors are properly loaded
|
||||
*/
|
||||
export function validateFeatureExtractors(): boolean {
|
||||
try {
|
||||
// Basic validation - just check if we can access the module
|
||||
// More detailed validation can be done when the modules are converted to TypeScript
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Feature extractor validation failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
// Basic type definitions for feature extractor functions until modules are converted
|
||||
|
||||
export type FeatureExtractorFunction = (...args: any[]) => Promise<unknown> | unknown;
|
||||
|
||||
export interface FeatureExtractorCategories {
|
||||
readonly network: readonly string[];
|
||||
readonly behavioral: readonly string[];
|
||||
readonly content: readonly string[];
|
||||
readonly temporal: readonly string[];
|
||||
readonly headers: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the available feature extractors by category
|
||||
* @returns Object with arrays of extractor names by category
|
||||
*/
|
||||
export function getFeatureExtractorsByCategory(): FeatureExtractorCategories {
|
||||
return {
|
||||
network: ['extractIPReputationFeatures', 'extractNetworkAnomalyFeatures'],
|
||||
behavioral: ['extractRequestPatternFeatures', 'extractSessionBehaviorFeatures', 'getSessionId'],
|
||||
content: ['extractPayloadFeatures', 'normalizeWAFSignals'],
|
||||
temporal: ['extractTimingFeatures', 'extractVelocityFeatures'],
|
||||
headers: ['extractHeaderFeatures']
|
||||
} as const;
|
||||
}
|
||||
|
|
@ -1,312 +0,0 @@
|
|||
// =============================================================================
|
||||
// NETWORK FEATURE EXTRACTION - SECURE TYPESCRIPT VERSION
|
||||
// =============================================================================
|
||||
// Comprehensive network analysis with IP validation and header spoofing protection
|
||||
// Handles completely user-controlled network data with zero trust validation
|
||||
|
||||
import { getReputationData } from '../database.js';
|
||||
import { detectHeaderSpoofing } from '../analyzers/index.js';
|
||||
import { requireValidIP } from '../../ip-validation.js';
|
||||
import type { NetworkRequest } from '../../network.js';
|
||||
|
||||
// Type definitions for secure network analysis
|
||||
export interface IPReputationFeatures {
|
||||
readonly isBlacklisted: boolean;
|
||||
readonly reputationScore: number;
|
||||
readonly asnRisk: number;
|
||||
readonly previousIncidents: number;
|
||||
readonly reputationSource: string;
|
||||
readonly riskScore: number;
|
||||
readonly validationErrors: readonly string[];
|
||||
}
|
||||
|
||||
export interface NetworkAnomalyFeatures {
|
||||
readonly portScanningBehavior: boolean;
|
||||
readonly unusualProtocol: boolean;
|
||||
readonly spoofedHeaders: boolean;
|
||||
readonly connectionAnomalies: number;
|
||||
readonly riskScore: number;
|
||||
readonly detectionErrors: readonly string[];
|
||||
}
|
||||
|
||||
interface DatabaseReputationData {
|
||||
readonly score?: number;
|
||||
readonly incidents?: number;
|
||||
readonly blacklisted?: boolean;
|
||||
readonly source?: string;
|
||||
readonly migrated?: boolean;
|
||||
}
|
||||
|
||||
interface ConnectionData {
|
||||
readonly uniquePorts: number;
|
||||
readonly protocols: readonly string[];
|
||||
}
|
||||
|
||||
// Security constants for network validation
|
||||
const MAX_REPUTATION_SCORE = 100;
|
||||
const MIN_REPUTATION_SCORE = -100;
|
||||
const MAX_INCIDENTS = 1000000; // Reasonable upper bound
|
||||
const MAX_UNIQUE_PORTS = 65535; // Max possible ports
|
||||
const MAX_PROTOCOLS = 100; // Reasonable protocol limit
|
||||
const MAX_VALIDATION_ERRORS = 20; // Prevent memory exhaustion
|
||||
|
||||
function validateNetworkRequest(request: unknown): NetworkRequest {
|
||||
if (!request || typeof request !== 'object') {
|
||||
throw new Error('Request must be an object');
|
||||
}
|
||||
|
||||
const req = request as Record<string, unknown>;
|
||||
|
||||
// Validate headers exist
|
||||
if (!req.headers || typeof req.headers !== 'object') {
|
||||
throw new Error('Request must have headers object');
|
||||
}
|
||||
|
||||
return request as NetworkRequest;
|
||||
}
|
||||
|
||||
function validateDatabaseReputationData(data: unknown): DatabaseReputationData {
|
||||
if (!data || typeof data !== 'object') {
|
||||
return {}; // Return empty object for missing data
|
||||
}
|
||||
|
||||
const dbData = data as Record<string, unknown>;
|
||||
|
||||
// Build validated object (not assigning to readonly properties)
|
||||
const validated: Record<string, unknown> = {};
|
||||
|
||||
// Validate score
|
||||
if (typeof dbData.score === 'number' &&
|
||||
dbData.score >= MIN_REPUTATION_SCORE &&
|
||||
dbData.score <= MAX_REPUTATION_SCORE &&
|
||||
Number.isFinite(dbData.score)) {
|
||||
validated.score = dbData.score;
|
||||
}
|
||||
|
||||
// Validate incidents
|
||||
if (typeof dbData.incidents === 'number' &&
|
||||
dbData.incidents >= 0 &&
|
||||
dbData.incidents <= MAX_INCIDENTS &&
|
||||
Number.isInteger(dbData.incidents)) {
|
||||
validated.incidents = dbData.incidents;
|
||||
}
|
||||
|
||||
// Validate blacklisted flag
|
||||
if (typeof dbData.blacklisted === 'boolean') {
|
||||
validated.blacklisted = dbData.blacklisted;
|
||||
}
|
||||
|
||||
// Validate source
|
||||
if (typeof dbData.source === 'string' && dbData.source.length <= 100) {
|
||||
validated.source = dbData.source;
|
||||
}
|
||||
|
||||
// Validate migrated flag
|
||||
if (typeof dbData.migrated === 'boolean') {
|
||||
validated.migrated = dbData.migrated;
|
||||
}
|
||||
|
||||
return validated as DatabaseReputationData;
|
||||
}
|
||||
|
||||
function validateConnectionData(data: unknown): ConnectionData {
|
||||
const defaultData: ConnectionData = {
|
||||
uniquePorts: 0,
|
||||
protocols: []
|
||||
};
|
||||
|
||||
if (!data || typeof data !== 'object') {
|
||||
return defaultData;
|
||||
}
|
||||
|
||||
const connData = data as Record<string, unknown>;
|
||||
|
||||
// Validate uniquePorts
|
||||
let uniquePorts = 0;
|
||||
if (typeof connData.uniquePorts === 'number' &&
|
||||
connData.uniquePorts >= 0 &&
|
||||
connData.uniquePorts <= MAX_UNIQUE_PORTS &&
|
||||
Number.isInteger(connData.uniquePorts)) {
|
||||
uniquePorts = connData.uniquePorts;
|
||||
}
|
||||
|
||||
// Validate protocols array
|
||||
let protocols: string[] = [];
|
||||
if (Array.isArray(connData.protocols)) {
|
||||
protocols = connData.protocols
|
||||
.filter((p): p is string => typeof p === 'string' && p.length <= 20)
|
||||
.slice(0, MAX_PROTOCOLS); // Limit array size
|
||||
}
|
||||
|
||||
return {
|
||||
uniquePorts,
|
||||
protocols
|
||||
};
|
||||
}
|
||||
|
||||
// Secure IP reputation extraction with comprehensive validation
|
||||
export async function extractIPReputationFeatures(ip: unknown): Promise<IPReputationFeatures> {
|
||||
const validationErrors: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Mutable working values
|
||||
let isBlacklisted = false;
|
||||
let reputationScore = 0;
|
||||
let asnRisk = 0;
|
||||
let previousIncidents = 0;
|
||||
let reputationSource = 'none';
|
||||
|
||||
try {
|
||||
// Use centralized IP validation
|
||||
const validatedIP = requireValidIP(ip);
|
||||
|
||||
// Check database reputation with error handling
|
||||
try {
|
||||
const dbReputation = await getReputationData(validatedIP);
|
||||
const validatedDbData = validateDatabaseReputationData(dbReputation);
|
||||
|
||||
if (validatedDbData.score !== undefined) {
|
||||
reputationScore = validatedDbData.score;
|
||||
riskScore += Math.max(0, validatedDbData.score); // Only positive scores add risk
|
||||
}
|
||||
|
||||
if (validatedDbData.incidents !== undefined) {
|
||||
previousIncidents = validatedDbData.incidents;
|
||||
if (validatedDbData.incidents > 0) {
|
||||
riskScore += Math.min(20, validatedDbData.incidents * 2); // Cap incident-based risk
|
||||
}
|
||||
}
|
||||
|
||||
if (validatedDbData.blacklisted !== undefined) {
|
||||
isBlacklisted = validatedDbData.blacklisted;
|
||||
if (validatedDbData.blacklisted) {
|
||||
riskScore += 80; // High risk for blacklisted IPs
|
||||
}
|
||||
}
|
||||
|
||||
if (validatedDbData.source !== undefined) {
|
||||
reputationSource = validatedDbData.source;
|
||||
}
|
||||
|
||||
// Safe logging with validated data
|
||||
if (validatedDbData.migrated) {
|
||||
console.log(`Threat scorer: Using migrated reputation data for ${validatedIP}: score=${reputationScore}`);
|
||||
} else if (reputationScore !== 0 || previousIncidents > 0) {
|
||||
console.log(`Threat scorer: Using dynamic reputation for ${validatedIP}: score=${reputationScore}, incidents=${previousIncidents}`);
|
||||
}
|
||||
|
||||
} catch (dbError) {
|
||||
// Database errors are normal for clean IPs
|
||||
console.log(`Threat scorer: No reputation history found for ${validatedIP} (clean IP)`);
|
||||
validationErrors.push('reputation_lookup_failed');
|
||||
}
|
||||
|
||||
} catch (ipError) {
|
||||
// IP validation failed - high risk
|
||||
validationErrors.push('ip_validation_failed');
|
||||
riskScore = 100; // Maximum risk for invalid IP
|
||||
reputationSource = 'validation_error';
|
||||
}
|
||||
|
||||
// Cap risk score and limit validation errors
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
const limitedErrors = validationErrors.slice(0, MAX_VALIDATION_ERRORS);
|
||||
|
||||
return {
|
||||
isBlacklisted,
|
||||
reputationScore,
|
||||
asnRisk,
|
||||
previousIncidents,
|
||||
reputationSource,
|
||||
riskScore: finalRiskScore,
|
||||
validationErrors: limitedErrors
|
||||
};
|
||||
}
|
||||
|
||||
// Secure network anomaly detection with validation
|
||||
export async function extractNetworkAnomalyFeatures(ip: unknown, request: unknown): Promise<NetworkAnomalyFeatures> {
|
||||
const detectionErrors: string[] = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Mutable working values
|
||||
let portScanningBehavior = false;
|
||||
let unusualProtocol = false;
|
||||
let spoofedHeaders = false;
|
||||
let connectionAnomalies = 0;
|
||||
|
||||
try {
|
||||
// Use centralized IP validation
|
||||
const validatedIP = requireValidIP(ip);
|
||||
const validatedRequest = validateNetworkRequest(request);
|
||||
|
||||
// Check for port scanning patterns with error handling
|
||||
try {
|
||||
const recentConnections = await getRecentConnections(validatedIP);
|
||||
const validatedConnData = validateConnectionData(recentConnections);
|
||||
|
||||
if (validatedConnData.uniquePorts > 10) {
|
||||
portScanningBehavior = true;
|
||||
connectionAnomalies++;
|
||||
riskScore += 40; // High risk for port scanning
|
||||
}
|
||||
|
||||
// Check for unusual protocol patterns
|
||||
if (validatedConnData.protocols.length > 5) {
|
||||
unusualProtocol = true;
|
||||
connectionAnomalies++;
|
||||
riskScore += 20; // Medium risk for unusual protocols
|
||||
}
|
||||
|
||||
} catch (connError) {
|
||||
detectionErrors.push('connection_analysis_failed');
|
||||
riskScore += 10; // Small penalty for analysis failure
|
||||
}
|
||||
|
||||
// Check for header spoofing with error handling
|
||||
try {
|
||||
if (detectHeaderSpoofing(validatedRequest.headers)) {
|
||||
spoofedHeaders = true;
|
||||
connectionAnomalies++;
|
||||
riskScore += 35; // High risk for header spoofing
|
||||
}
|
||||
} catch (headerError) {
|
||||
detectionErrors.push('header_spoofing_check_failed');
|
||||
riskScore += 10; // Small penalty for detection failure
|
||||
}
|
||||
|
||||
} catch (validationError) {
|
||||
// Input validation failed - high risk
|
||||
detectionErrors.push('input_validation_failed');
|
||||
riskScore = 100; // Maximum risk for validation failure
|
||||
connectionAnomalies = 999; // Indicate severe anomaly
|
||||
}
|
||||
|
||||
// Cap risk score and limit detection errors
|
||||
const finalRiskScore = Math.max(0, Math.min(100, riskScore));
|
||||
const limitedErrors = detectionErrors.slice(0, MAX_VALIDATION_ERRORS);
|
||||
|
||||
return {
|
||||
portScanningBehavior,
|
||||
unusualProtocol,
|
||||
spoofedHeaders,
|
||||
connectionAnomalies,
|
||||
riskScore: finalRiskScore,
|
||||
detectionErrors: limitedErrors
|
||||
};
|
||||
}
|
||||
|
||||
async function getRecentConnections(ip: string): Promise<ConnectionData> {
|
||||
try {
|
||||
// Track actual connection data in production environment
|
||||
return {
|
||||
uniquePorts: 0,
|
||||
protocols: ['http', 'https']
|
||||
};
|
||||
} catch (error) {
|
||||
console.warn(`Connection data retrieval failed for ${ip}:`, error);
|
||||
return {
|
||||
uniquePorts: 0,
|
||||
protocols: []
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,446 +0,0 @@
|
|||
// =============================================================================
|
||||
// TEMPORAL FEATURE EXTRACTION (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
import { getRequestHistory, behaviorDB } from '../database.js';
|
||||
import { calculateDistance } from '../analyzers/index.js';
|
||||
import { parseDuration } from '../../time.js';
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface TimingFeatures {
|
||||
readonly requestRate: number;
|
||||
readonly burstBehavior: boolean;
|
||||
readonly timingAnomalies: number;
|
||||
readonly isNightTime: boolean;
|
||||
readonly isWeekend: boolean;
|
||||
readonly requestSpacing?: number;
|
||||
readonly peakHourActivity?: boolean;
|
||||
}
|
||||
|
||||
interface VelocityFeatures {
|
||||
readonly impossibleTravel: boolean;
|
||||
readonly rapidLocationChange: boolean;
|
||||
readonly travelVelocity: number;
|
||||
readonly geoAnomalies: readonly string[];
|
||||
readonly distanceTraveled?: number;
|
||||
readonly timeElapsed?: number;
|
||||
}
|
||||
|
||||
interface GeoLocation {
|
||||
readonly lat: number;
|
||||
readonly lon: number;
|
||||
}
|
||||
|
||||
interface GeoData {
|
||||
readonly latitude?: number;
|
||||
readonly longitude?: number;
|
||||
readonly country?: string;
|
||||
readonly continent?: string;
|
||||
readonly asn?: number;
|
||||
readonly isp?: string;
|
||||
}
|
||||
|
||||
interface RequestHistoryEntry {
|
||||
readonly timestamp: number;
|
||||
readonly method?: string;
|
||||
readonly path?: string;
|
||||
readonly userAgent?: string;
|
||||
readonly score?: number;
|
||||
}
|
||||
|
||||
interface BehaviorData {
|
||||
readonly lastLocation?: GeoLocation;
|
||||
readonly lastSeen?: number;
|
||||
readonly requestCount?: number;
|
||||
readonly [key: string]: unknown;
|
||||
}
|
||||
|
||||
interface TimingAnalysisConfig {
|
||||
readonly historyWindowMs: number;
|
||||
readonly burstThreshold: number;
|
||||
readonly minRequestsForBurst: number;
|
||||
readonly nightStartHour: number;
|
||||
readonly nightEndHour: number;
|
||||
readonly maxCommercialFlightSpeed: number;
|
||||
readonly rapidMovementThreshold: number;
|
||||
readonly rapidMovementTimeWindow: number;
|
||||
}
|
||||
|
||||
// Configuration constants
|
||||
const TIMING_CONFIG: TimingAnalysisConfig = {
|
||||
historyWindowMs: parseDuration('5m'), // 5 minutes
|
||||
burstThreshold: 0.6, // 60% of intervals must be short for burst detection
|
||||
minRequestsForBurst: 10, // Minimum requests needed for burst analysis
|
||||
nightStartHour: 2, // 2 AM
|
||||
nightEndHour: 6, // 6 AM
|
||||
maxCommercialFlightSpeed: 900, // km/h
|
||||
rapidMovementThreshold: 200, // km/h
|
||||
rapidMovementTimeWindow: 3600 // 1 hour in seconds
|
||||
} as const;
|
||||
|
||||
// =============================================================================
|
||||
// TIMING FEATURE EXTRACTION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Extracts timing-based features from request patterns
|
||||
* Analyzes request frequency, burst behavior, and temporal anomalies
|
||||
*
|
||||
* @param ip - Client IP address for history lookup
|
||||
* @param timestamp - Current request timestamp
|
||||
* @returns Promise resolving to timing features
|
||||
*/
|
||||
export async function extractTimingFeatures(ip: string, timestamp: number): Promise<TimingFeatures> {
|
||||
// Input validation
|
||||
if (!ip || typeof ip !== 'string') {
|
||||
throw new Error('Invalid IP address provided to extractTimingFeatures');
|
||||
}
|
||||
|
||||
if (!timestamp || typeof timestamp !== 'number' || timestamp <= 0) {
|
||||
throw new Error('Invalid timestamp provided to extractTimingFeatures');
|
||||
}
|
||||
|
||||
const features: TimingFeatures = {
|
||||
requestRate: 0,
|
||||
burstBehavior: false,
|
||||
timingAnomalies: 0,
|
||||
isNightTime: false,
|
||||
isWeekend: false
|
||||
};
|
||||
|
||||
try {
|
||||
// Get request history for timing analysis
|
||||
const history = await getRequestHistory(ip, TIMING_CONFIG.historyWindowMs);
|
||||
|
||||
if (!Array.isArray(history) || history.length === 0) {
|
||||
return features;
|
||||
}
|
||||
|
||||
// Calculate request rate (requests per minute)
|
||||
const oldestRequest = Math.min(...history.map(h => h.timestamp));
|
||||
const timeSpan = Math.max(timestamp - oldestRequest, 1000); // Avoid division by zero
|
||||
const requestRate = (history.length / timeSpan) * 60000; // Convert to per minute
|
||||
|
||||
// Apply reasonable bounds to request rate
|
||||
const boundedRequestRate = Math.min(requestRate, 1000); // Cap at 1000 requests/minute
|
||||
|
||||
const updatedFeatures: TimingFeatures = {
|
||||
...features,
|
||||
requestRate: Math.round(boundedRequestRate * 100) / 100, // Round to 2 decimal places
|
||||
requestSpacing: timeSpan / history.length
|
||||
};
|
||||
|
||||
// Detect burst behavior
|
||||
if (history.length >= TIMING_CONFIG.minRequestsForBurst) {
|
||||
const burstAnalysis = analyzeBurstBehavior(history, timestamp);
|
||||
Object.assign(updatedFeatures, {
|
||||
burstBehavior: burstAnalysis.isBurst,
|
||||
timingAnomalies: updatedFeatures.timingAnomalies + (burstAnalysis.isBurst ? 1 : 0)
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze temporal patterns
|
||||
const temporalAnalysis = analyzeTemporalPatterns(timestamp);
|
||||
Object.assign(updatedFeatures, {
|
||||
isNightTime: temporalAnalysis.isNightTime,
|
||||
isWeekend: temporalAnalysis.isWeekend,
|
||||
peakHourActivity: temporalAnalysis.isPeakHour,
|
||||
timingAnomalies: updatedFeatures.timingAnomalies + temporalAnalysis.anomalyCount
|
||||
});
|
||||
|
||||
return updatedFeatures;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn(`Failed to extract timing features for IP ${ip}:`, error.message);
|
||||
return features;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyzes request patterns for burst behavior
|
||||
* @param history - Array of request history entries
|
||||
* @param currentTimestamp - Current request timestamp
|
||||
* @returns Burst analysis results
|
||||
*/
|
||||
function analyzeBurstBehavior(
|
||||
history: readonly RequestHistoryEntry[],
|
||||
_currentTimestamp: number
|
||||
): { isBurst: boolean; shortIntervalRatio: number } {
|
||||
if (history.length < 2) {
|
||||
return { isBurst: false, shortIntervalRatio: 0 };
|
||||
}
|
||||
|
||||
// Calculate intervals between consecutive requests
|
||||
const intervals: number[] = [];
|
||||
const sortedHistory = [...history].sort((a, b) => a.timestamp - b.timestamp);
|
||||
|
||||
for (let i = 1; i < sortedHistory.length; i++) {
|
||||
const current = sortedHistory[i];
|
||||
const previous = sortedHistory[i - 1];
|
||||
if (current && previous && current.timestamp && previous.timestamp) {
|
||||
const interval = current.timestamp - previous.timestamp;
|
||||
if (interval > 0) { // Only include positive intervals
|
||||
intervals.push(interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (intervals.length === 0) {
|
||||
return { isBurst: false, shortIntervalRatio: 0 };
|
||||
}
|
||||
|
||||
// Calculate average interval
|
||||
const avgInterval = intervals.reduce((sum, interval) => sum + interval, 0) / intervals.length;
|
||||
|
||||
// Define "short" intervals as those significantly below average
|
||||
const shortIntervalThreshold = avgInterval * 0.2;
|
||||
const shortIntervals = intervals.filter(interval => interval < shortIntervalThreshold);
|
||||
const shortIntervalRatio = shortIntervals.length / intervals.length;
|
||||
|
||||
// Burst detected if majority of intervals are short
|
||||
const isBurst = shortIntervalRatio > TIMING_CONFIG.burstThreshold;
|
||||
|
||||
return { isBurst, shortIntervalRatio };
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyzes temporal patterns for unusual timing
|
||||
* @param timestamp - Current request timestamp
|
||||
* @returns Temporal analysis results
|
||||
*/
|
||||
function analyzeTemporalPatterns(timestamp: number): {
|
||||
isNightTime: boolean;
|
||||
isWeekend: boolean;
|
||||
isPeakHour: boolean;
|
||||
anomalyCount: number;
|
||||
} {
|
||||
const date = new Date(timestamp);
|
||||
const hour = date.getHours();
|
||||
const day = date.getDay();
|
||||
|
||||
let anomalyCount = 0;
|
||||
|
||||
// Night time detection (2 AM - 6 AM)
|
||||
const isNightTime = hour >= TIMING_CONFIG.nightStartHour && hour <= TIMING_CONFIG.nightEndHour;
|
||||
if (isNightTime) {
|
||||
anomalyCount++;
|
||||
}
|
||||
|
||||
// Weekend detection (Saturday = 6, Sunday = 0)
|
||||
const isWeekend = day === 0 || day === 6;
|
||||
|
||||
// Peak hour detection (9 AM - 5 PM on weekdays)
|
||||
const isPeakHour = !isWeekend && hour >= 9 && hour <= 17;
|
||||
|
||||
return {
|
||||
isNightTime,
|
||||
isWeekend,
|
||||
isPeakHour,
|
||||
anomalyCount
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// VELOCITY FEATURE EXTRACTION
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Extracts velocity-based features from geographic data
|
||||
* Detects impossible travel and rapid location changes
|
||||
*
|
||||
* @param ip - Client IP address for behavior tracking
|
||||
* @param geoData - Geographic location data
|
||||
* @returns Promise resolving to velocity features
|
||||
*/
|
||||
export async function extractVelocityFeatures(ip: string, geoData: GeoData | null): Promise<VelocityFeatures> {
|
||||
// Input validation
|
||||
if (!ip || typeof ip !== 'string') {
|
||||
throw new Error('Invalid IP address provided to extractVelocityFeatures');
|
||||
}
|
||||
|
||||
// Use mutable object during construction
|
||||
const features = {
|
||||
impossibleTravel: false,
|
||||
rapidLocationChange: false,
|
||||
travelVelocity: 0,
|
||||
geoAnomalies: [] as string[]
|
||||
};
|
||||
|
||||
// Return early if no geo data or incomplete coordinates
|
||||
if (!geoData ||
|
||||
typeof geoData.latitude !== 'number' ||
|
||||
typeof geoData.longitude !== 'number' ||
|
||||
!isValidCoordinate(geoData.latitude, geoData.longitude)) {
|
||||
return features;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get previous location data from behavior database
|
||||
const behaviorKey = `behavior:${ip}`;
|
||||
const behaviorData = await getBehaviorData(behaviorKey);
|
||||
|
||||
if (behaviorData?.lastLocation && behaviorData.lastSeen) {
|
||||
const velocityAnalysis = analyzeVelocity(
|
||||
behaviorData.lastLocation,
|
||||
{ lat: geoData.latitude, lon: geoData.longitude },
|
||||
behaviorData.lastSeen,
|
||||
Date.now()
|
||||
);
|
||||
|
||||
// Return new object with velocity analysis results
|
||||
return {
|
||||
impossibleTravel: velocityAnalysis.impossibleTravel ?? features.impossibleTravel,
|
||||
rapidLocationChange: velocityAnalysis.rapidLocationChange ?? features.rapidLocationChange,
|
||||
travelVelocity: velocityAnalysis.travelVelocity ?? features.travelVelocity,
|
||||
geoAnomalies: velocityAnalysis.geoAnomalies ? [...velocityAnalysis.geoAnomalies] : features.geoAnomalies,
|
||||
distanceTraveled: velocityAnalysis.distanceTraveled,
|
||||
timeElapsed: velocityAnalysis.timeElapsed
|
||||
};
|
||||
}
|
||||
|
||||
// Store current location for future comparisons
|
||||
await updateLocationData(behaviorKey, geoData, behaviorData);
|
||||
|
||||
return features;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn(`Failed to extract velocity features for IP ${ip}:`, error.message);
|
||||
return features;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates geographic coordinates
|
||||
* @param lat - Latitude
|
||||
* @param lon - Longitude
|
||||
* @returns True if coordinates are valid
|
||||
*/
|
||||
function isValidCoordinate(lat: number, lon: number): boolean {
|
||||
return lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets behavior data from database with proper error handling
|
||||
* @param behaviorKey - Database key for behavior data
|
||||
* @returns Behavior data or null
|
||||
*/
|
||||
async function getBehaviorData(behaviorKey: string): Promise<BehaviorData | null> {
|
||||
try {
|
||||
const data = await behaviorDB.get(behaviorKey);
|
||||
return data as BehaviorData;
|
||||
} catch (err) {
|
||||
// Key doesn't exist or database error
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyzes velocity between two geographic points
|
||||
* @param lastLocation - Previous location
|
||||
* @param currentLocation - Current location
|
||||
* @param lastTimestamp - Previous timestamp
|
||||
* @param currentTimestamp - Current timestamp
|
||||
* @returns Velocity analysis results
|
||||
*/
|
||||
function analyzeVelocity(
|
||||
lastLocation: GeoLocation,
|
||||
currentLocation: GeoLocation,
|
||||
lastTimestamp: number,
|
||||
currentTimestamp: number
|
||||
): Partial<VelocityFeatures> {
|
||||
const features: {
|
||||
impossibleTravel?: boolean;
|
||||
rapidLocationChange?: boolean;
|
||||
travelVelocity?: number;
|
||||
geoAnomalies?: string[];
|
||||
distanceTraveled?: number;
|
||||
timeElapsed?: number;
|
||||
} = {
|
||||
geoAnomalies: []
|
||||
};
|
||||
|
||||
// Calculate distance between locations
|
||||
const distance = calculateDistance(lastLocation, currentLocation);
|
||||
|
||||
if (distance === null || distance < 0) {
|
||||
return features;
|
||||
}
|
||||
|
||||
// Calculate time difference in seconds
|
||||
const timeDiffSeconds = Math.max((currentTimestamp - lastTimestamp) / 1000, 1);
|
||||
|
||||
// Calculate velocity in km/h
|
||||
const velocityKmh = (distance / timeDiffSeconds) * 3600;
|
||||
|
||||
// Apply reasonable bounds to velocity
|
||||
const boundedVelocity = Math.min(velocityKmh, 50000); // Cap at 50,000 km/h (orbital speeds)
|
||||
|
||||
features.travelVelocity = Math.round(boundedVelocity * 100) / 100; // Round to 2 decimal places
|
||||
features.distanceTraveled = Math.round(distance * 100) / 100;
|
||||
features.timeElapsed = Math.round(timeDiffSeconds);
|
||||
|
||||
const anomalies: string[] = features.geoAnomalies || [];
|
||||
|
||||
// Impossible travel detection (faster than commercial flight)
|
||||
if (boundedVelocity > TIMING_CONFIG.maxCommercialFlightSpeed) {
|
||||
features.impossibleTravel = true;
|
||||
anomalies.push('impossible_travel_speed');
|
||||
}
|
||||
|
||||
// Rapid location change detection
|
||||
if (boundedVelocity > TIMING_CONFIG.rapidMovementThreshold &&
|
||||
timeDiffSeconds < TIMING_CONFIG.rapidMovementTimeWindow) {
|
||||
features.rapidLocationChange = true;
|
||||
anomalies.push('rapid_location_change');
|
||||
}
|
||||
|
||||
// Additional velocity-based anomalies
|
||||
if (boundedVelocity > 2000) { // Faster than supersonic aircraft
|
||||
anomalies.push('supersonic_travel');
|
||||
}
|
||||
|
||||
if (distance > 20000) { // Distance greater than half Earth's circumference
|
||||
anomalies.push('extreme_distance');
|
||||
}
|
||||
|
||||
features.geoAnomalies = anomalies;
|
||||
|
||||
return features;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates location data in behavior database
|
||||
* @param behaviorKey - Database key
|
||||
* @param geoData - Current geographic data
|
||||
* @param existingData - Existing behavior data
|
||||
*/
|
||||
async function updateLocationData(
|
||||
behaviorKey: string,
|
||||
geoData: GeoData,
|
||||
existingData: BehaviorData | null
|
||||
): Promise<void> {
|
||||
try {
|
||||
const updatedData: BehaviorData = {
|
||||
...existingData,
|
||||
lastLocation: {
|
||||
lat: geoData.latitude!,
|
||||
lon: geoData.longitude!
|
||||
},
|
||||
lastSeen: Date.now()
|
||||
};
|
||||
|
||||
await behaviorDB.put(behaviorKey, updatedData);
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
console.warn('Failed to update location data:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// EXPORT TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
export type { TimingFeatures, VelocityFeatures, GeoData, GeoLocation };
|
||||
|
|
@ -1,437 +0,0 @@
|
|||
// =============================================================================
|
||||
// THREAT SCORING ENGINE (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
import { STATIC_WHITELIST, type ThreatThresholds, type SignalWeights } from './constants.js';
|
||||
import { type IncomingHttpHeaders } from 'http';
|
||||
import type { NetworkRequest } from '../network.js';
|
||||
import * as logs from '../logs.js';
|
||||
import { performance } from 'perf_hooks';
|
||||
|
||||
// Simple utility functions
|
||||
function performSecurityChecks(ip: string): string {
|
||||
if (typeof ip !== 'string' || ip.length === 0 || ip.length > 45) {
|
||||
throw new Error('Invalid IP address');
|
||||
}
|
||||
return ip.trim();
|
||||
}
|
||||
|
||||
function normalizeMetricValue(value: number, min: number, max: number): number {
|
||||
if (typeof value !== 'number' || isNaN(value)) return 0;
|
||||
if (max <= min) return value >= max ? 1 : 0;
|
||||
const clampedValue = Math.max(min, Math.min(max, value));
|
||||
return (clampedValue - min) / (max - min);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
export interface ThreatScore {
|
||||
readonly totalScore: number;
|
||||
readonly confidence: number;
|
||||
readonly riskLevel: 'allow' | 'challenge' | 'block';
|
||||
readonly components: {
|
||||
readonly behaviorScore: number;
|
||||
readonly contentScore: number;
|
||||
readonly networkScore: number;
|
||||
readonly anomalyScore: number;
|
||||
};
|
||||
readonly signalsTriggered: readonly string[];
|
||||
readonly normalizedFeatures: Record<string, number>;
|
||||
readonly processingTimeMs: number;
|
||||
}
|
||||
|
||||
export interface ThreatScoringConfig {
|
||||
readonly enabled: boolean;
|
||||
readonly thresholds: ThreatThresholds;
|
||||
readonly signalWeights: SignalWeights;
|
||||
readonly enableBotVerification?: boolean;
|
||||
readonly enableGeoAnalysis?: boolean;
|
||||
readonly enableBehaviorAnalysis?: boolean;
|
||||
readonly enableContentAnalysis?: boolean;
|
||||
readonly logDetailedScores?: boolean;
|
||||
}
|
||||
|
||||
interface RequestMetadata {
|
||||
readonly startTime: number;
|
||||
readonly ip: string;
|
||||
readonly userAgent?: string;
|
||||
readonly method: string;
|
||||
readonly path: string;
|
||||
readonly headers: IncomingHttpHeaders;
|
||||
readonly body?: string;
|
||||
readonly sessionId?: string;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// THREAT SCORING ENGINE
|
||||
// =============================================================================
|
||||
|
||||
export class ThreatScorer {
|
||||
private readonly config: ThreatScoringConfig;
|
||||
|
||||
constructor(config: ThreatScoringConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs comprehensive threat scoring on a request
|
||||
*/
|
||||
public async scoreRequest(request: NetworkRequest): Promise<ThreatScore> {
|
||||
const startTime = performance.now();
|
||||
|
||||
try {
|
||||
// Check if scoring is enabled
|
||||
if (!this.config.enabled) {
|
||||
return this.createAllowScore(startTime);
|
||||
}
|
||||
|
||||
// Extract request metadata
|
||||
const metadata = this.extractRequestMetadata(request, startTime);
|
||||
|
||||
// Validate input and perform security checks
|
||||
performSecurityChecks(metadata.ip);
|
||||
|
||||
// Check static whitelist (quick path for assets)
|
||||
if (this.isWhitelisted(metadata.path)) {
|
||||
return this.createAllowScore(startTime);
|
||||
}
|
||||
|
||||
// Perform threat analysis
|
||||
const score = this.performBasicThreatAnalysis(metadata);
|
||||
|
||||
return score;
|
||||
|
||||
} catch (error) {
|
||||
logs.error('threat-scorer', `Error scoring request: ${error}`);
|
||||
return this.createErrorScore(startTime);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract basic metadata from request
|
||||
*/
|
||||
private extractRequestMetadata(request: NetworkRequest, startTime: number): RequestMetadata {
|
||||
const headers = request.headers || {};
|
||||
const userAgent = this.extractUserAgent(headers);
|
||||
const ip = this.extractClientIP(request);
|
||||
|
||||
return {
|
||||
startTime,
|
||||
ip,
|
||||
userAgent,
|
||||
method: (request as any).method || 'GET',
|
||||
path: this.extractPath(request),
|
||||
headers: headers as IncomingHttpHeaders,
|
||||
body: (request as any).body,
|
||||
sessionId: this.extractSessionId(headers)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract user agent from headers
|
||||
*/
|
||||
private extractUserAgent(headers: any): string {
|
||||
if (headers && typeof headers.get === 'function') {
|
||||
return headers.get('user-agent') || '';
|
||||
}
|
||||
if (headers && typeof headers === 'object') {
|
||||
return headers['user-agent'] || '';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract client IP from request
|
||||
*/
|
||||
private extractClientIP(request: NetworkRequest): string {
|
||||
// Try common IP extraction methods
|
||||
const headers = request.headers;
|
||||
if (headers) {
|
||||
if (typeof headers.get === 'function') {
|
||||
return headers.get('x-forwarded-for') ||
|
||||
headers.get('x-real-ip') ||
|
||||
headers.get('cf-connecting-ip') || '127.0.0.1';
|
||||
}
|
||||
if (typeof headers === 'object') {
|
||||
const h = headers as any;
|
||||
return h['x-forwarded-for'] || h['x-real-ip'] || h['cf-connecting-ip'] || '127.0.0.1';
|
||||
}
|
||||
}
|
||||
return '127.0.0.1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract path from request
|
||||
*/
|
||||
private extractPath(request: NetworkRequest): string {
|
||||
if ((request as any).url) {
|
||||
try {
|
||||
const url = new URL((request as any).url, 'http://localhost');
|
||||
return url.pathname;
|
||||
} catch {
|
||||
return (request as any).url || '/';
|
||||
}
|
||||
}
|
||||
return '/';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract session ID from headers
|
||||
*/
|
||||
private extractSessionId(headers: any): string | undefined {
|
||||
// Basic session ID extraction from cookies
|
||||
if (headers && headers.cookie) {
|
||||
const cookies = headers.cookie.split(';');
|
||||
for (const cookie of cookies) {
|
||||
const [name, value] = cookie.trim().split('=');
|
||||
if (name && name.toLowerCase().includes('session')) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if path is in static whitelist
|
||||
*/
|
||||
private isWhitelisted(path: string): boolean {
|
||||
// Check static file extensions
|
||||
for (const ext of STATIC_WHITELIST.extensions) {
|
||||
if (path.endsWith(ext)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check whitelisted paths
|
||||
for (const whitelistPath of STATIC_WHITELIST.paths) {
|
||||
if (path.startsWith(whitelistPath)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check patterns
|
||||
for (const pattern of STATIC_WHITELIST.patterns) {
|
||||
if (pattern.test(path)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform basic threat analysis (simplified version)
|
||||
*/
|
||||
private performBasicThreatAnalysis(metadata: RequestMetadata): ThreatScore {
|
||||
const startTime = performance.now();
|
||||
const signalsTriggered: string[] = [];
|
||||
let totalScore = 0;
|
||||
|
||||
const components = {
|
||||
networkScore: 0,
|
||||
behaviorScore: 0,
|
||||
contentScore: 0,
|
||||
anomalyScore: 0
|
||||
};
|
||||
|
||||
// Basic checks
|
||||
if (!metadata.userAgent || metadata.userAgent.length === 0) {
|
||||
components.anomalyScore += this.config.signalWeights.MISSING_UA?.weight || 10;
|
||||
signalsTriggered.push('MISSING_UA');
|
||||
}
|
||||
|
||||
// WAF signal integration - use WAF results if available
|
||||
const wafSignals = this.extractWAFSignals(metadata);
|
||||
if (wafSignals) {
|
||||
const wafScore = this.calculateWAFScore(wafSignals, signalsTriggered);
|
||||
components.contentScore += wafScore;
|
||||
}
|
||||
|
||||
totalScore = components.networkScore + components.behaviorScore +
|
||||
components.contentScore + components.anomalyScore;
|
||||
|
||||
// Determine risk level
|
||||
const riskLevel = this.determineRiskLevel(totalScore);
|
||||
|
||||
// Calculate confidence (simplified)
|
||||
const confidence = Math.min(0.8, signalsTriggered.length * 0.2 + 0.3);
|
||||
|
||||
const processingTimeMs = performance.now() - startTime;
|
||||
|
||||
return {
|
||||
totalScore,
|
||||
confidence,
|
||||
riskLevel,
|
||||
components,
|
||||
signalsTriggered,
|
||||
normalizedFeatures: {
|
||||
networkRisk: normalizeMetricValue(components.networkScore, 0, 100),
|
||||
behaviorRisk: normalizeMetricValue(components.behaviorScore, 0, 100),
|
||||
contentRisk: normalizeMetricValue(components.contentScore, 0, 100),
|
||||
anomalyRisk: normalizeMetricValue(components.anomalyScore, 0, 100)
|
||||
},
|
||||
processingTimeMs
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract WAF signals from request metadata
|
||||
*/
|
||||
private extractWAFSignals(metadata: RequestMetadata): Record<string, unknown> | null {
|
||||
// WAF signals are attached to the request object by WAF middleware
|
||||
// Try multiple ways to access them depending on request type
|
||||
const request = metadata as any;
|
||||
|
||||
// Express-style: res.locals.wafSignals (if request has res)
|
||||
if (request.res?.locals?.wafSignals) {
|
||||
return request.res.locals.wafSignals;
|
||||
}
|
||||
|
||||
// Direct attachment: request.wafSignals
|
||||
if (request.wafSignals) {
|
||||
return request.wafSignals;
|
||||
}
|
||||
|
||||
// Headers may contain WAF detection flags
|
||||
if (metadata.headers) {
|
||||
const wafHeader = metadata.headers['x-waf-signals'] || metadata.headers['X-WAF-Signals'];
|
||||
if (wafHeader && typeof wafHeader === 'string') {
|
||||
try {
|
||||
return JSON.parse(wafHeader);
|
||||
} catch {
|
||||
// Invalid JSON, ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate threat score from WAF signals
|
||||
*/
|
||||
private calculateWAFScore(wafSignals: Record<string, unknown>, signalsTriggered: string[]): number {
|
||||
let score = 0;
|
||||
|
||||
// Map WAF detections to configured signal weights
|
||||
if (wafSignals.sqlInjection || wafSignals.sql_injection) {
|
||||
score += this.config.signalWeights.SQL_INJECTION?.weight || 80;
|
||||
signalsTriggered.push('SQL_INJECTION');
|
||||
}
|
||||
|
||||
if (wafSignals.xss || wafSignals.xssAttempt) {
|
||||
score += this.config.signalWeights.XSS_ATTEMPT?.weight || 85;
|
||||
signalsTriggered.push('XSS_ATTEMPT');
|
||||
}
|
||||
|
||||
if (wafSignals.commandInjection || wafSignals.command_injection) {
|
||||
score += this.config.signalWeights.COMMAND_INJECTION?.weight || 95;
|
||||
signalsTriggered.push('COMMAND_INJECTION');
|
||||
}
|
||||
|
||||
if (wafSignals.pathTraversal || wafSignals.path_traversal) {
|
||||
score += this.config.signalWeights.PATH_TRAVERSAL?.weight || 70;
|
||||
signalsTriggered.push('PATH_TRAVERSAL');
|
||||
}
|
||||
|
||||
// Handle unverified bot detection - CRITICAL for fake bots
|
||||
if (wafSignals.unverified_bot) {
|
||||
score += 50; // High penalty for fake bot user agents
|
||||
signalsTriggered.push('UNVERIFIED_BOT');
|
||||
}
|
||||
|
||||
// Handle WAF attack tool detection in user agents
|
||||
const detectedAttacks = wafSignals.detected_attacks;
|
||||
if (Array.isArray(detectedAttacks)) {
|
||||
if (detectedAttacks.includes('attack_tool_user_agent')) {
|
||||
score += this.config.signalWeights.ATTACK_TOOL_UA?.weight || 30;
|
||||
signalsTriggered.push('ATTACK_TOOL_UA');
|
||||
}
|
||||
|
||||
// Additional detection for unverified bots via attack list
|
||||
if (detectedAttacks.includes('unverified_bot')) {
|
||||
score += 50;
|
||||
signalsTriggered.push('UNVERIFIED_BOT');
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines risk level based on score and configured thresholds
|
||||
*/
|
||||
private determineRiskLevel(score: number): 'allow' | 'challenge' | 'block' {
|
||||
if (score <= this.config.thresholds.ALLOW) return 'allow';
|
||||
if (score <= this.config.thresholds.CHALLENGE) return 'challenge';
|
||||
return 'block';
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an allow score for whitelisted or disabled requests
|
||||
*/
|
||||
private createAllowScore(startTime: number): ThreatScore {
|
||||
return {
|
||||
totalScore: 0,
|
||||
confidence: 1.0,
|
||||
riskLevel: 'allow',
|
||||
components: {
|
||||
behaviorScore: 0,
|
||||
contentScore: 0,
|
||||
networkScore: 0,
|
||||
anomalyScore: 0
|
||||
},
|
||||
signalsTriggered: [],
|
||||
normalizedFeatures: {},
|
||||
processingTimeMs: performance.now() - startTime
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an error score when threat analysis fails
|
||||
*/
|
||||
private createErrorScore(startTime: number): ThreatScore {
|
||||
return {
|
||||
totalScore: 0,
|
||||
confidence: 0,
|
||||
riskLevel: 'allow', // Fail open
|
||||
components: {
|
||||
behaviorScore: 0,
|
||||
contentScore: 0,
|
||||
networkScore: 0,
|
||||
anomalyScore: 0
|
||||
},
|
||||
signalsTriggered: ['ERROR'],
|
||||
normalizedFeatures: {},
|
||||
processingTimeMs: performance.now() - startTime
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates and configures a threat scorer instance
|
||||
*/
|
||||
export function createThreatScorer(config: ThreatScoringConfig): ThreatScorer {
|
||||
return new ThreatScorer(config);
|
||||
}
|
||||
|
||||
// Default threat scorer for convenience (requires configuration)
|
||||
let defaultScorer: ThreatScorer | null = null;
|
||||
|
||||
export function configureDefaultThreatScorer(config: ThreatScoringConfig): void {
|
||||
defaultScorer = new ThreatScorer(config);
|
||||
}
|
||||
|
||||
export const threatScorer = {
|
||||
scoreRequest: async (request: NetworkRequest): Promise<ThreatScore> => {
|
||||
if (!defaultScorer) {
|
||||
throw new Error('Default threat scorer not configured. Call configureDefaultThreatScorer() first.');
|
||||
}
|
||||
return defaultScorer.scoreRequest(request);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -1,185 +0,0 @@
|
|||
// =============================================================================
|
||||
// PATTERN MATCHING FOR THREAT SCORING (TypeScript)
|
||||
// =============================================================================
|
||||
|
||||
// @ts-ignore - string-dsa doesn't have TypeScript definitions
|
||||
import { AhoCorasick } from 'string-dsa';
|
||||
import { ATTACK_TOOL_PATTERNS, SUSPICIOUS_BOT_PATTERNS } from './constants.js';
|
||||
import * as logs from '../logs.js';
|
||||
|
||||
// =============================================================================
|
||||
// TYPE DEFINITIONS
|
||||
// =============================================================================
|
||||
|
||||
interface AhoCorasickMatcher {
|
||||
find(text: string): readonly string[] | null;
|
||||
}
|
||||
|
||||
interface AhoCorasickMatchers {
|
||||
attackTools: AhoCorasickMatcher | null;
|
||||
suspiciousBotPatterns: AhoCorasickMatcher | null;
|
||||
}
|
||||
|
||||
interface ReadonlyAhoCorasickMatchers {
|
||||
readonly attackTools: AhoCorasickMatcher | null;
|
||||
readonly suspiciousBotPatterns: AhoCorasickMatcher | null;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// PATTERN MATCHING IMPLEMENTATION
|
||||
// =============================================================================
|
||||
|
||||
// Pre-compiled Aho-Corasick matchers for ultra-fast pattern matching
|
||||
// CRITICAL: These provide 10-100x performance improvement over individual string.includes() calls
|
||||
const internalMatchers: AhoCorasickMatchers = {
|
||||
attackTools: null,
|
||||
suspiciousBotPatterns: null
|
||||
};
|
||||
|
||||
// Initialize Aho-Corasick matchers once at startup
|
||||
function initializeAhoCorasickMatchers(): void {
|
||||
try {
|
||||
internalMatchers.attackTools = new AhoCorasick(ATTACK_TOOL_PATTERNS) as AhoCorasickMatcher;
|
||||
internalMatchers.suspiciousBotPatterns = new AhoCorasick(SUSPICIOUS_BOT_PATTERNS) as AhoCorasickMatcher;
|
||||
|
||||
logs.plugin('threat-scoring', 'Initialized Aho-Corasick matchers for ultra-fast pattern matching');
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.error('threat-scoring', `Failed to initialize Aho-Corasick matchers: ${error.message}`);
|
||||
|
||||
// Set to null so we can fall back to traditional methods
|
||||
(Object.keys(internalMatchers) as Array<keyof AhoCorasickMatchers>).forEach(key => {
|
||||
internalMatchers[key] = null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize matchers at module load
|
||||
initializeAhoCorasickMatchers();
|
||||
|
||||
// =============================================================================
|
||||
// EXPORTED MATCHER FUNCTIONS
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Checks if the given text contains patterns associated with attack tools
|
||||
* @param text - The text to search for attack tool patterns
|
||||
* @returns true if attack tool patterns are found, false otherwise
|
||||
*/
|
||||
export function matchAttackTools(text: unknown): boolean {
|
||||
// Type guard: ensure we have a string
|
||||
if (!text || typeof text !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Use Aho-Corasick for performance if available
|
||||
if (internalMatchers.attackTools) {
|
||||
try {
|
||||
const matches = internalMatchers.attackTools.find(text.toLowerCase());
|
||||
return matches !== null && matches.length > 0;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.warn('threat-scoring', `Aho-Corasick attack tool matching failed: ${error.message}`);
|
||||
// Fall through to traditional method
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to traditional method if Aho-Corasick fails
|
||||
const lowerText = text.toLowerCase();
|
||||
return ATTACK_TOOL_PATTERNS.some((pattern: string) => lowerText.includes(pattern));
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the given text contains patterns associated with suspicious bots
|
||||
* @param text - The text to search for suspicious bot patterns
|
||||
* @returns true if suspicious bot patterns are found, false otherwise
|
||||
*/
|
||||
export function matchSuspiciousBots(text: unknown): boolean {
|
||||
// Type guard: ensure we have a string
|
||||
if (!text || typeof text !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Use Aho-Corasick for performance if available
|
||||
if (internalMatchers.suspiciousBotPatterns) {
|
||||
try {
|
||||
const matches = internalMatchers.suspiciousBotPatterns.find(text.toLowerCase());
|
||||
return matches !== null && matches.length > 0;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.warn('threat-scoring', `Aho-Corasick suspicious bot matching failed: ${error.message}`);
|
||||
// Fall through to traditional method
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to traditional method if Aho-Corasick fails
|
||||
const lowerText = text.toLowerCase();
|
||||
return SUSPICIOUS_BOT_PATTERNS.some((pattern: string) => lowerText.includes(pattern));
|
||||
}
|
||||
|
||||
/**
|
||||
* Advanced pattern matching with detailed results
|
||||
* @param text - The text to analyze
|
||||
* @param patterns - Array of patterns to search for
|
||||
* @returns Array of matched patterns with positions
|
||||
*/
|
||||
export function findDetailedMatches(
|
||||
text: unknown,
|
||||
patterns: readonly string[]
|
||||
): readonly { pattern: string; position: number }[] {
|
||||
if (!text || typeof text !== 'string') {
|
||||
return [];
|
||||
}
|
||||
|
||||
const results: { pattern: string; position: number }[] = [];
|
||||
const lowerText = text.toLowerCase();
|
||||
|
||||
patterns.forEach(pattern => {
|
||||
const position = lowerText.indexOf(pattern.toLowerCase());
|
||||
if (position !== -1) {
|
||||
results.push({ pattern, position });
|
||||
}
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current status of Aho-Corasick matchers
|
||||
* @returns Status object indicating which matchers are available
|
||||
*/
|
||||
export function getMatcherStatus(): {
|
||||
readonly attackToolsAvailable: boolean;
|
||||
readonly suspiciousBotsAvailable: boolean;
|
||||
readonly fallbackMode: boolean;
|
||||
} {
|
||||
const attackToolsAvailable = internalMatchers.attackTools !== null;
|
||||
const suspiciousBotsAvailable = internalMatchers.suspiciousBotPatterns !== null;
|
||||
|
||||
return {
|
||||
attackToolsAvailable,
|
||||
suspiciousBotsAvailable,
|
||||
fallbackMode: !attackToolsAvailable || !suspiciousBotsAvailable
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reinitializes the Aho-Corasick matchers (useful for recovery after errors)
|
||||
*/
|
||||
export function reinitializeMatchers(): boolean {
|
||||
try {
|
||||
initializeAhoCorasickMatchers();
|
||||
const status = getMatcherStatus();
|
||||
return status.attackToolsAvailable && status.suspiciousBotsAvailable;
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logs.error('threat-scoring', `Failed to reinitialize matchers: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-export the matchers for testing/debugging (readonly for safety)
|
||||
export const ahoCorasickMatchers: ReadonlyAhoCorasickMatchers = Object.freeze({
|
||||
get attackTools() { return internalMatchers.attackTools; },
|
||||
get suspiciousBotPatterns() { return internalMatchers.suspiciousBotPatterns; }
|
||||
});
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
// =============================================================================
|
||||
// THREAT SCORING SECURITY UTILITIES
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Performs basic security validation on IP addresses to prevent injection attacks
|
||||
* @param ip - The IP address to validate
|
||||
* @returns The validated IP address
|
||||
* @throws Error if IP is invalid or malicious
|
||||
*/
|
||||
export function performSecurityChecks(ip: string): string {
|
||||
if (typeof ip !== 'string') {
|
||||
throw new Error('IP address must be a string');
|
||||
}
|
||||
|
||||
// Remove any whitespace
|
||||
const cleanIP = ip.trim();
|
||||
|
||||
// Basic length check to prevent extremely long inputs
|
||||
if (cleanIP.length > 45) { // Max IPv6 length
|
||||
throw new Error('IP address too long');
|
||||
}
|
||||
|
||||
if (cleanIP.length === 0) {
|
||||
throw new Error('IP address cannot be empty');
|
||||
}
|
||||
|
||||
// Basic IPv4 pattern check
|
||||
const ipv4Pattern = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
|
||||
|
||||
// Basic IPv6 pattern check (simplified)
|
||||
const ipv6Pattern = /^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$|^::1$|^::$/;
|
||||
|
||||
// Check for common injection patterns
|
||||
const dangerousPatterns = [
|
||||
/[<>\"'`]/, // HTML/JS injection
|
||||
/[;|&$]/, // Command injection
|
||||
/\.\./, // Path traversal
|
||||
/\/\*/, // SQL comment
|
||||
/--/, // SQL comment
|
||||
];
|
||||
|
||||
for (const pattern of dangerousPatterns) {
|
||||
if (pattern.test(cleanIP)) {
|
||||
throw new Error('IP address contains dangerous characters');
|
||||
}
|
||||
}
|
||||
|
||||
// Validate IP format
|
||||
if (!ipv4Pattern.test(cleanIP) && !ipv6Pattern.test(cleanIP)) {
|
||||
// Allow some common internal formats like ::ffff:192.168.1.1
|
||||
if (!/^::ffff:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/.test(cleanIP)) {
|
||||
throw new Error('Invalid IP address format');
|
||||
}
|
||||
}
|
||||
|
||||
return cleanIP;
|
||||
}
|
||||
|
|
@ -1,148 +0,0 @@
|
|||
// Duration parsing utility with error handling and validation
|
||||
// CRITICAL: Used throughout the system for parsing configuration timeouts
|
||||
// Incorrect parsing can lead to system instability or security bypasses
|
||||
|
||||
// Type definitions for duration parsing
|
||||
export type DurationUnit = 's' | 'm' | 'h' | 'd';
|
||||
export type DurationInput = string | number;
|
||||
export type DurationString = `${number}${DurationUnit}`;
|
||||
|
||||
// Interface for duration multipliers
|
||||
interface DurationMultipliers {
|
||||
readonly s: number; // seconds
|
||||
readonly m: number; // minutes
|
||||
readonly h: number; // hours
|
||||
readonly d: number; // days
|
||||
}
|
||||
|
||||
// Constants for duration conversion
|
||||
const DURATION_MULTIPLIERS: DurationMultipliers = {
|
||||
s: 1000, // seconds
|
||||
m: 60 * 1000, // minutes
|
||||
h: 60 * 60 * 1000, // hours
|
||||
d: 24 * 60 * 60 * 1000 // days
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Parse duration strings into milliseconds
|
||||
* Supports formats like: "1s", "5m", "2h", "1d", "30000" (raw ms)
|
||||
*
|
||||
* @param input - Duration string or milliseconds
|
||||
* @returns Duration in milliseconds
|
||||
* @throws Error if input format is invalid
|
||||
*/
|
||||
export function parseDuration(input: DurationInput): number {
|
||||
// Handle numeric input (already in milliseconds)
|
||||
if (typeof input === 'number') {
|
||||
if (input < 0) {
|
||||
throw new Error('Duration cannot be negative');
|
||||
}
|
||||
if (input > Number.MAX_SAFE_INTEGER) {
|
||||
throw new Error('Duration too large');
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
if (typeof input !== 'string') {
|
||||
throw new Error('Duration must be a string or number');
|
||||
}
|
||||
|
||||
// Handle empty or invalid input
|
||||
const trimmed = input.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error('Duration cannot be empty');
|
||||
}
|
||||
|
||||
// Parse numeric-only strings as milliseconds
|
||||
const numericValue = parseInt(trimmed, 10);
|
||||
if (trimmed === numericValue.toString()) {
|
||||
if (numericValue < 0) {
|
||||
throw new Error('Duration cannot be negative');
|
||||
}
|
||||
return numericValue;
|
||||
}
|
||||
|
||||
// Parse duration with unit suffix
|
||||
const match = trimmed.match(/^(\d+(?:\.\d+)?)\s*([smhd])$/i);
|
||||
if (!match) {
|
||||
throw new Error(`Invalid duration format: ${input}. Use formats like "1s", "5m", "2h", "1d"`);
|
||||
}
|
||||
|
||||
const valueMatch = match[1];
|
||||
const unitMatch = match[2];
|
||||
|
||||
if (!valueMatch || !unitMatch) {
|
||||
throw new Error(`Invalid duration format: ${input}. Missing value or unit`);
|
||||
}
|
||||
|
||||
const value = parseFloat(valueMatch);
|
||||
|
||||
const unit = unitMatch.toLowerCase() as DurationUnit;
|
||||
|
||||
if (value < 0) {
|
||||
throw new Error('Duration cannot be negative');
|
||||
}
|
||||
|
||||
// Type-safe unit validation
|
||||
if (!(unit in DURATION_MULTIPLIERS)) {
|
||||
throw new Error(`Invalid duration unit: ${unit}. Use s, m, h, or d`);
|
||||
}
|
||||
|
||||
const result = value * DURATION_MULTIPLIERS[unit];
|
||||
|
||||
if (result > Number.MAX_SAFE_INTEGER) {
|
||||
throw new Error('Duration too large');
|
||||
}
|
||||
|
||||
return Math.floor(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Format milliseconds back to human-readable duration string
|
||||
* @param milliseconds - Duration in milliseconds
|
||||
* @returns Human-readable duration string
|
||||
*/
|
||||
export function formatDuration(milliseconds: number): string {
|
||||
if (milliseconds < 0) {
|
||||
throw new Error('Duration cannot be negative');
|
||||
}
|
||||
|
||||
// Return raw milliseconds for very small values
|
||||
if (milliseconds < 1000) {
|
||||
return `${milliseconds}ms`;
|
||||
}
|
||||
|
||||
// Find the largest appropriate unit
|
||||
const units: Array<[DurationUnit, number]> = [
|
||||
['d', DURATION_MULTIPLIERS.d],
|
||||
['h', DURATION_MULTIPLIERS.h],
|
||||
['m', DURATION_MULTIPLIERS.m],
|
||||
['s', DURATION_MULTIPLIERS.s],
|
||||
];
|
||||
|
||||
for (const [unit, multiplier] of units) {
|
||||
if (milliseconds >= multiplier) {
|
||||
const value = Math.floor(milliseconds / multiplier);
|
||||
return `${value}${unit}`;
|
||||
}
|
||||
}
|
||||
|
||||
return `${milliseconds}ms`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if a string is a valid duration string
|
||||
* @param input - String to check
|
||||
* @returns True if the string is a valid duration format
|
||||
*/
|
||||
export function isValidDurationString(input: string): input is DurationString {
|
||||
try {
|
||||
parseDuration(input);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Export types for use in other modules
|
||||
export type { DurationMultipliers };
|
||||
|
|
@ -1,374 +0,0 @@
|
|||
import { promises as fsPromises } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { rootDir } from '../index.js';
|
||||
import { parseDuration, type DurationInput } from './time.js';
|
||||
import * as logs from './logs.js';
|
||||
|
||||
// ==================== TYPE DEFINITIONS ====================
|
||||
|
||||
export interface TimedDownloadSource {
|
||||
readonly name: string;
|
||||
readonly url: string;
|
||||
readonly updateInterval: DurationInput; // Uses time.ts format: "24h", "5m", etc.
|
||||
readonly enabled: boolean;
|
||||
readonly parser?: DataParser;
|
||||
readonly validator?: DataValidator;
|
||||
readonly headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface DataParser {
|
||||
readonly format: 'json' | 'text' | 'custom';
|
||||
readonly parseFunction?: (data: string) => unknown;
|
||||
}
|
||||
|
||||
export interface DataValidator {
|
||||
readonly maxSize?: number;
|
||||
readonly maxEntries?: number;
|
||||
readonly validationFunction?: (data: unknown) => boolean;
|
||||
}
|
||||
|
||||
export interface DownloadResult {
|
||||
readonly success: boolean;
|
||||
readonly data?: unknown;
|
||||
readonly error?: string;
|
||||
readonly lastUpdated: number;
|
||||
}
|
||||
|
||||
export interface DownloadedData {
|
||||
readonly sourceName: string;
|
||||
readonly data: unknown;
|
||||
readonly lastUpdated: number;
|
||||
readonly source: string;
|
||||
}
|
||||
|
||||
// ==================== SECURITY CONSTANTS ====================
|
||||
|
||||
const SECURITY_LIMITS = {
|
||||
MAX_DOWNLOAD_SIZE: 50 * 1024 * 1024, // 50MB max download
|
||||
MAX_RESPONSE_TIME: parseDuration('30s'), // 30 seconds timeout
|
||||
MIN_UPDATE_INTERVAL: parseDuration('1m'), // Minimum 1 minute between updates
|
||||
MAX_UPDATE_INTERVAL: parseDuration('7d'), // Maximum 1 week between updates
|
||||
MAX_SOURCES: 100, // Maximum number of sources
|
||||
} as const;
|
||||
|
||||
// ==================== DOWNLOAD MANAGER ====================
|
||||
|
||||
export class TimedDownloadManager {
|
||||
private readonly dataDir: string;
|
||||
private readonly updateTimestampPath: string;
|
||||
private readonly updatePromises: Map<string, Promise<DownloadResult>> = new Map();
|
||||
private readonly scheduledUpdates: Map<string, NodeJS.Timeout> = new Map();
|
||||
private readonly parsedIntervals: Map<DurationInput, number> = new Map();
|
||||
|
||||
constructor(subdirectory: string = 'downloads') {
|
||||
this.dataDir = join(rootDir, 'data', subdirectory);
|
||||
this.updateTimestampPath = join(this.dataDir, 'update-timestamps.json');
|
||||
this.ensureDataDirectory();
|
||||
}
|
||||
|
||||
private async ensureDataDirectory(): Promise<void> {
|
||||
try {
|
||||
await fsPromises.mkdir(this.dataDir, { recursive: true });
|
||||
} catch (error) {
|
||||
logs.error('timed-downloads', `Failed to create data directory: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets parsed interval with caching to avoid repeated parsing overhead
|
||||
*/
|
||||
private getParsedInterval(interval: DurationInput): number {
|
||||
if (!this.parsedIntervals.has(interval)) {
|
||||
this.parsedIntervals.set(interval, parseDuration(interval));
|
||||
}
|
||||
return this.parsedIntervals.get(interval)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and parses data from a source
|
||||
*/
|
||||
async downloadFromSource(source: TimedDownloadSource): Promise<DownloadResult> {
|
||||
// Prevent concurrent downloads of the same source
|
||||
if (this.updatePromises.has(source.name)) {
|
||||
return await this.updatePromises.get(source.name)!;
|
||||
}
|
||||
|
||||
const downloadPromise = this.performDownload(source);
|
||||
this.updatePromises.set(source.name, downloadPromise);
|
||||
|
||||
try {
|
||||
return await downloadPromise;
|
||||
} finally {
|
||||
this.updatePromises.delete(source.name);
|
||||
}
|
||||
}
|
||||
|
||||
private async performDownload(source: TimedDownloadSource): Promise<DownloadResult> {
|
||||
const now = Date.now();
|
||||
|
||||
try {
|
||||
logs.plugin('timed-downloads', `Downloading ${source.name} from ${source.url}`);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), SECURITY_LIMITS.MAX_RESPONSE_TIME);
|
||||
|
||||
const headers = {
|
||||
'User-Agent': 'Checkpoint-Security-Gateway/1.0',
|
||||
...source.headers,
|
||||
};
|
||||
|
||||
const response = await fetch(source.url, {
|
||||
signal: controller.signal,
|
||||
headers,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
return {
|
||||
success: false,
|
||||
error: `HTTP ${response.status}: ${response.statusText}`,
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get('content-length');
|
||||
const maxSize = source.validator?.maxSize || SECURITY_LIMITS.MAX_DOWNLOAD_SIZE;
|
||||
|
||||
if (contentLength && parseInt(contentLength) > maxSize) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Response too large: ${contentLength} bytes`,
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
const rawData = await response.text();
|
||||
|
||||
if (rawData.length > maxSize) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Response too large: ${rawData.length} bytes`,
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
// Parse data based on format
|
||||
let parsedData: unknown;
|
||||
try {
|
||||
parsedData = this.parseData(rawData, source.parser);
|
||||
} catch (parseError) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Parse error: ${parseError instanceof Error ? parseError.message : 'Unknown parse error'}`,
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
// Validate parsed data
|
||||
if (source.validator?.validationFunction && !source.validator.validationFunction(parsedData)) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'Data validation failed',
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
// Save to file
|
||||
const downloadedData: DownloadedData = {
|
||||
sourceName: source.name,
|
||||
data: parsedData,
|
||||
lastUpdated: now,
|
||||
source: source.url,
|
||||
};
|
||||
|
||||
await this.saveDownloadedData(source.name, downloadedData);
|
||||
await this.updateTimestamp(source.name);
|
||||
|
||||
logs.plugin('timed-downloads', `Successfully downloaded and saved ${source.name}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: parsedData,
|
||||
lastUpdated: now,
|
||||
};
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
logs.error('timed-downloads', `Failed to download ${source.name}: ${errorMessage}`);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses raw data based on parser configuration
|
||||
*/
|
||||
private parseData(rawData: string, parser?: DataParser): unknown {
|
||||
if (!parser) {
|
||||
return rawData; // Return raw text if no parser specified
|
||||
}
|
||||
|
||||
switch (parser.format) {
|
||||
case 'json':
|
||||
return JSON.parse(rawData);
|
||||
case 'text':
|
||||
return rawData;
|
||||
case 'custom':
|
||||
if (parser.parseFunction) {
|
||||
return parser.parseFunction(rawData);
|
||||
}
|
||||
return rawData;
|
||||
default:
|
||||
return rawData;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves downloaded data to disk
|
||||
*/
|
||||
private async saveDownloadedData(sourceName: string, data: DownloadedData): Promise<void> {
|
||||
const filePath = join(this.dataDir, `${sourceName}.json`);
|
||||
try {
|
||||
await fsPromises.writeFile(filePath, JSON.stringify(data, null, 2), 'utf8');
|
||||
} catch (error) {
|
||||
logs.error('timed-downloads', `Failed to save data for ${sourceName}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads downloaded data from disk
|
||||
*/
|
||||
async loadDownloadedData(sourceName: string): Promise<DownloadedData | null> {
|
||||
const filePath = join(this.dataDir, `${sourceName}.json`);
|
||||
try {
|
||||
const fileData = await fsPromises.readFile(filePath, 'utf8');
|
||||
return JSON.parse(fileData) as DownloadedData;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a source needs updating based on its interval
|
||||
*/
|
||||
async needsUpdate(source: TimedDownloadSource): Promise<boolean> {
|
||||
try {
|
||||
const timestamps = await this.getUpdateTimestamps();
|
||||
const lastUpdate = timestamps[source.name] || 0;
|
||||
const intervalMs = this.getParsedInterval(source.updateInterval);
|
||||
const elapsed = Date.now() - lastUpdate;
|
||||
|
||||
return elapsed >= intervalMs;
|
||||
} catch {
|
||||
return true; // Update on error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates timestamp for a source
|
||||
*/
|
||||
private async updateTimestamp(sourceName: string): Promise<void> {
|
||||
try {
|
||||
const timestamps = await this.getUpdateTimestamps();
|
||||
timestamps[sourceName] = Date.now();
|
||||
await fsPromises.writeFile(this.updateTimestampPath, JSON.stringify(timestamps, null, 2), 'utf8');
|
||||
} catch (error) {
|
||||
logs.error('timed-downloads', `Failed to update timestamp for ${sourceName}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all update timestamps
|
||||
*/
|
||||
private async getUpdateTimestamps(): Promise<Record<string, number>> {
|
||||
try {
|
||||
const data = await fsPromises.readFile(this.updateTimestampPath, 'utf8');
|
||||
return JSON.parse(data);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts periodic updates for sources
|
||||
*/
|
||||
startPeriodicUpdates(sources: readonly TimedDownloadSource[]): void {
|
||||
// Clear any existing scheduled updates
|
||||
this.stopPeriodicUpdates();
|
||||
|
||||
for (const source of sources) {
|
||||
if (!source.enabled) continue;
|
||||
|
||||
try {
|
||||
const intervalMs = this.getParsedInterval(source.updateInterval);
|
||||
|
||||
// Validate interval bounds
|
||||
const boundedInterval = Math.max(
|
||||
SECURITY_LIMITS.MIN_UPDATE_INTERVAL,
|
||||
Math.min(SECURITY_LIMITS.MAX_UPDATE_INTERVAL, intervalMs)
|
||||
);
|
||||
|
||||
const timeoutId = setInterval(async () => {
|
||||
try {
|
||||
if (await this.needsUpdate(source)) {
|
||||
await this.downloadFromSource(source);
|
||||
}
|
||||
} catch (error) {
|
||||
logs.error('timed-downloads', `Periodic update failed for ${source.name}: ${error}`);
|
||||
}
|
||||
}, boundedInterval);
|
||||
|
||||
this.scheduledUpdates.set(source.name, timeoutId);
|
||||
logs.plugin('timed-downloads', `Scheduled updates for ${source.name} every ${source.updateInterval}`);
|
||||
} catch (error) {
|
||||
logs.error('timed-downloads', `Failed to schedule updates for ${source.name}: ${error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops all periodic updates
|
||||
*/
|
||||
stopPeriodicUpdates(): void {
|
||||
for (const [sourceName, timeoutId] of this.scheduledUpdates.entries()) {
|
||||
clearInterval(timeoutId);
|
||||
logs.plugin('timed-downloads', `Stopped periodic updates for ${sourceName}`);
|
||||
}
|
||||
this.scheduledUpdates.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates all sources that need updating
|
||||
*/
|
||||
async updateAllSources(sources: readonly TimedDownloadSource[]): Promise<void> {
|
||||
const updatePromises: Promise<DownloadResult>[] = [];
|
||||
|
||||
for (const source of sources) {
|
||||
if (source.enabled && await this.needsUpdate(source)) {
|
||||
updatePromises.push(this.downloadFromSource(source));
|
||||
}
|
||||
}
|
||||
|
||||
if (updatePromises.length > 0) {
|
||||
logs.plugin('timed-downloads', `Updating ${updatePromises.length} sources...`);
|
||||
const results = await Promise.allSettled(updatePromises);
|
||||
|
||||
let successCount = 0;
|
||||
let failureCount = 0;
|
||||
|
||||
results.forEach((result) => {
|
||||
if (result.status === 'fulfilled' && result.value.success) {
|
||||
successCount++;
|
||||
} else {
|
||||
failureCount++;
|
||||
}
|
||||
});
|
||||
|
||||
logs.plugin('timed-downloads', `Update complete: ${successCount} successful, ${failureCount} failed`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
// Target modern JavaScript
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"downlevelIteration": true,
|
||||
|
||||
// Enable ES modules
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
|
||||
// Output settings - clean separation of source and build
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"preserveConstEnums": true,
|
||||
"removeComments": false,
|
||||
|
||||
// Type checking
|
||||
"strict": true,
|
||||
"noImplicitAny": true,
|
||||
"strictNullChecks": true,
|
||||
"strictFunctionTypes": true,
|
||||
"strictBindCallApply": true,
|
||||
"strictPropertyInitialization": true,
|
||||
"noImplicitThis": true,
|
||||
"alwaysStrict": true,
|
||||
|
||||
// Additional checks
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noImplicitReturns": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
|
||||
// Interop with JavaScript
|
||||
"allowJs": false,
|
||||
"checkJs": false,
|
||||
"maxNodeModuleJsDepth": 0,
|
||||
|
||||
// Emit - minimal output for cleaner project
|
||||
"declaration": false,
|
||||
"declarationMap": false,
|
||||
"sourceMap": false,
|
||||
"inlineSources": false,
|
||||
|
||||
// Advanced
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
|
||||
// Path mapping for source files
|
||||
"baseUrl": "./src",
|
||||
"paths": {
|
||||
"@utils/*": ["utils/*"],
|
||||
"@plugins/*": ["plugins/*"],
|
||||
"@types/*": ["types/*"]
|
||||
}
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist",
|
||||
".tests",
|
||||
"pages",
|
||||
"data",
|
||||
"db",
|
||||
"config",
|
||||
"**/*.js"
|
||||
],
|
||||
"ts-node": {
|
||||
"esm": true,
|
||||
"experimentalSpecifierResolution": "node"
|
||||
}
|
||||
}
|
||||
41
utils/logs.js
Normal file
41
utils/logs.js
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
const seenConfigs = new Set();
|
||||
|
||||
export function init(msg) {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function plugin(_name, msg) {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function config(name, msg) {
|
||||
if (!seenConfigs.has(name)) {
|
||||
console.log(`Config ${msg} for ${name}`);
|
||||
seenConfigs.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
export function db(msg) {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function server(msg) {
|
||||
console.log(msg);
|
||||
}
|
||||
|
||||
export function section(title) {
|
||||
console.log(`\n=== ${title.toUpperCase()} ===`);
|
||||
}
|
||||
|
||||
export function warn(_category, msg) {
|
||||
console.warn(`WARNING: ${msg}`);
|
||||
}
|
||||
|
||||
export function error(_category, msg) {
|
||||
console.error(`ERROR: ${msg}`);
|
||||
}
|
||||
|
||||
// General message function for bullet items
|
||||
export function msg(msg) {
|
||||
console.log(msg);
|
||||
}
|
||||
13
utils/network.js
Normal file
13
utils/network.js
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
export function getRealIP(request, server) {
|
||||
let ip = request.headers.get('x-forwarded-for') || request.headers.get('x-real-ip');
|
||||
if (ip?.includes(',')) ip = ip.split(',')[0].trim();
|
||||
if (!ip && server) {
|
||||
ip = server.remoteAddress;
|
||||
}
|
||||
if (!ip) {
|
||||
const url = new URL(request.url);
|
||||
ip = url.hostname;
|
||||
}
|
||||
if (ip?.startsWith('::ffff:')) ip = ip.slice(7);
|
||||
return ip;
|
||||
}
|
||||
28
utils/plugins.js
Normal file
28
utils/plugins.js
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import { resolve, extname, sep, isAbsolute } from 'path';
|
||||
import { pathToFileURL } from 'url';
|
||||
import { rootDir } from '../index.js';
|
||||
|
||||
/**
|
||||
* Securely import a JavaScript module from within the application root.
|
||||
* Prevents path traversal and disallows non-.js extensions.
|
||||
*
|
||||
* @param {string} relPath - The relative path to the module from the application root.
|
||||
* @returns {Promise<any>} The imported module.
|
||||
*/
|
||||
export async function secureImportModule(relPath) {
|
||||
if (isAbsolute(relPath)) {
|
||||
throw new Error('Absolute paths are not allowed for module imports');
|
||||
}
|
||||
if (relPath.includes('..')) {
|
||||
throw new Error('Relative paths containing .. are not allowed for module imports');
|
||||
}
|
||||
if (extname(relPath) !== '.js') {
|
||||
throw new Error(`Only .js files can be imported: ${relPath}`);
|
||||
}
|
||||
const absPath = resolve(rootDir, relPath);
|
||||
if (!absPath.startsWith(rootDir + sep)) {
|
||||
throw new Error(`Module path outside of application root: ${relPath}`);
|
||||
}
|
||||
const url = pathToFileURL(absPath).href;
|
||||
return import(url);
|
||||
}
|
||||
72
utils/proof.js
Normal file
72
utils/proof.js
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import crypto from 'crypto';
|
||||
import { getRealIP } from './network.js';
|
||||
|
||||
export function generateChallenge(checkpointConfig) {
|
||||
const challenge = crypto.randomBytes(16).toString('hex');
|
||||
const salt = crypto.randomBytes(checkpointConfig.SaltLength).toString('hex');
|
||||
return { challenge, salt };
|
||||
}
|
||||
|
||||
export function calculateHash(input) {
|
||||
return crypto.createHash('sha256').update(input).digest('hex');
|
||||
}
|
||||
|
||||
export function verifyPoW(challenge, salt, nonce, difficulty) {
|
||||
const hash = calculateHash(challenge + salt + nonce);
|
||||
return hash.startsWith('0'.repeat(difficulty));
|
||||
}
|
||||
|
||||
export function checkPoSTimes(times, enableCheck, ratio) {
|
||||
if (!Array.isArray(times) || times.length !== 3) {
|
||||
throw new Error('Invalid PoS run times length');
|
||||
}
|
||||
const minT = Math.min(...times);
|
||||
const maxT = Math.max(...times);
|
||||
if (enableCheck && maxT > minT * ratio) {
|
||||
throw new Error(`PoS run times inconsistent (ratio ${maxT / minT} > ${ratio})`);
|
||||
}
|
||||
}
|
||||
|
||||
export const challengeStore = new Map();
|
||||
|
||||
export function generateRequestID(request, checkpointConfig) {
|
||||
const { challenge, salt } = generateChallenge(checkpointConfig);
|
||||
const posSeed = crypto.randomBytes(32).toString('hex');
|
||||
const requestID = crypto.randomBytes(16).toString('hex');
|
||||
const params = {
|
||||
Challenge: challenge,
|
||||
Salt: salt,
|
||||
Difficulty: checkpointConfig.Difficulty,
|
||||
ExpiresAt: Date.now() + checkpointConfig.ChallengeExpiration,
|
||||
CreatedAt: Date.now(),
|
||||
ClientIP: getRealIP(request),
|
||||
PoSSeed: posSeed,
|
||||
};
|
||||
challengeStore.set(requestID, params);
|
||||
return requestID;
|
||||
}
|
||||
|
||||
export function getChallengeParams(requestID) {
|
||||
return challengeStore.get(requestID);
|
||||
}
|
||||
|
||||
export function deleteChallenge(requestID) {
|
||||
challengeStore.delete(requestID);
|
||||
}
|
||||
|
||||
export function verifyPoS(hashes, times, checkpointConfig) {
|
||||
if (!Array.isArray(hashes) || hashes.length !== 3) {
|
||||
throw new Error('Invalid PoS hashes length');
|
||||
}
|
||||
if (!Array.isArray(times) || times.length !== 3) {
|
||||
throw new Error('Invalid PoS run times length');
|
||||
}
|
||||
if (hashes[0] !== hashes[1] || hashes[1] !== hashes[2]) {
|
||||
throw new Error('PoS hashes do not match');
|
||||
}
|
||||
if (hashes[0].length !== 64) {
|
||||
throw new Error('Invalid PoS hash length');
|
||||
}
|
||||
|
||||
checkPoSTimes(times, checkpointConfig.CheckPoSTimes, checkpointConfig.PoSTimeConsistencyRatio);
|
||||
}
|
||||
20
utils/time.js
Normal file
20
utils/time.js
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
export function parseDuration(str) {
|
||||
if (!str) return 0;
|
||||
const m = /^([0-9]+)(ms|s|m|h|d)$/.exec(str);
|
||||
if (!m) return 0;
|
||||
const val = parseInt(m[1], 10);
|
||||
switch (m[2]) {
|
||||
case 'ms':
|
||||
return val;
|
||||
case 's':
|
||||
return val * 1000;
|
||||
case 'm':
|
||||
return val * 60 * 1000;
|
||||
case 'h':
|
||||
return val * 60 * 60 * 1000;
|
||||
case 'd':
|
||||
return val * 24 * 60 * 60 * 1000;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue