Updates
This commit is contained in:
Iliyan Angelov
491
ETB-API/analytics_predictive_insights/ml/anomaly_detection.py
Normal file
491
ETB-API/analytics_predictive_insights/ml/anomaly_detection.py
Normal file
@@ -0,0 +1,491 @@
|
||||
"""
|
||||
ML-based anomaly detection for incident management
|
||||
Implements various anomaly detection algorithms for identifying unusual patterns
|
||||
"""
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from typing import Dict, List, Tuple, Optional, Any
|
||||
from datetime import datetime, timedelta
|
||||
from sklearn.ensemble import IsolationForest
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sklearn.cluster import DBSCAN
|
||||
from sklearn.decomposition import PCA
|
||||
from scipy import stats
|
||||
import logging
|
||||
|
||||
from django.utils import timezone
|
||||
from django.db.models import Q, Avg, Count, Sum
|
||||
from incident_intelligence.models import Incident
|
||||
from ..models import AnomalyDetection, PredictiveModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AnomalyDetector:
|
||||
"""Base class for anomaly detection algorithms"""
|
||||
|
||||
def __init__(self, model_config: Dict[str, Any] = None):
|
||||
self.model_config = model_config or {}
|
||||
self.scaler = StandardScaler()
|
||||
self.is_fitted = False
|
||||
|
||||
def fit(self, data: pd.DataFrame) -> None:
|
||||
"""Fit the anomaly detection model"""
|
||||
raise NotImplementedError
|
||||
|
||||
def predict(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Predict anomalies in the data"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_anomaly_scores(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Get anomaly scores for the data"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class StatisticalAnomalyDetector(AnomalyDetector):
|
||||
"""Statistical anomaly detection using z-score and IQR methods"""
|
||||
|
||||
def __init__(self, model_config: Dict[str, Any] = None):
|
||||
super().__init__(model_config)
|
||||
self.z_threshold = self.model_config.get('z_threshold', 3.0)
|
||||
self.iqr_multiplier = self.model_config.get('iqr_multiplier', 1.5)
|
||||
self.stats_cache = {}
|
||||
|
||||
def fit(self, data: pd.DataFrame) -> None:
|
||||
"""Calculate statistical parameters for anomaly detection"""
|
||||
for column in data.columns:
|
||||
if data[column].dtype in ['int64', 'float64']:
|
||||
values = data[column].dropna()
|
||||
if len(values) > 0:
|
||||
self.stats_cache[column] = {
|
||||
'mean': values.mean(),
|
||||
'std': values.std(),
|
||||
'q1': values.quantile(0.25),
|
||||
'q3': values.quantile(0.75),
|
||||
'iqr': values.quantile(0.75) - values.quantile(0.25)
|
||||
}
|
||||
self.is_fitted = True
|
||||
|
||||
def predict(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Predict anomalies using statistical methods"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Model must be fitted before prediction")
|
||||
|
||||
anomaly_flags = np.zeros(len(data), dtype=bool)
|
||||
|
||||
for column in data.columns:
|
||||
if column in self.stats_cache and data[column].dtype in ['int64', 'float64']:
|
||||
values = data[column].dropna()
|
||||
if len(values) > 0:
|
||||
stats = self.stats_cache[column]
|
||||
|
||||
# Z-score method
|
||||
z_scores = np.abs((values - stats['mean']) / stats['std'])
|
||||
z_anomalies = z_scores > self.z_threshold
|
||||
|
||||
# IQR method
|
||||
lower_bound = stats['q1'] - self.iqr_multiplier * stats['iqr']
|
||||
upper_bound = stats['q3'] + self.iqr_multiplier * stats['iqr']
|
||||
iqr_anomalies = (values < lower_bound) | (values > upper_bound)
|
||||
|
||||
# Combine both methods
|
||||
column_anomalies = z_anomalies | iqr_anomalies
|
||||
anomaly_flags[values.index] |= column_anomalies
|
||||
|
||||
return anomaly_flags
|
||||
|
||||
def get_anomaly_scores(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Get anomaly scores based on z-scores"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Model must be fitted before prediction")
|
||||
|
||||
scores = np.zeros(len(data))
|
||||
|
||||
for column in data.columns:
|
||||
if column in self.stats_cache and data[column].dtype in ['int64', 'float64']:
|
||||
values = data[column].dropna()
|
||||
if len(values) > 0:
|
||||
stats = self.stats_cache[column]
|
||||
z_scores = np.abs((values - stats['mean']) / stats['std'])
|
||||
scores[values.index] += z_scores
|
||||
|
||||
return scores
|
||||
|
||||
|
||||
class IsolationForestAnomalyDetector(AnomalyDetector):
|
||||
"""Isolation Forest anomaly detection"""
|
||||
|
||||
def __init__(self, model_config: Dict[str, Any] = None):
|
||||
super().__init__(model_config)
|
||||
self.contamination = self.model_config.get('contamination', 0.1)
|
||||
self.n_estimators = self.model_config.get('n_estimators', 100)
|
||||
self.model = IsolationForest(
|
||||
contamination=self.contamination,
|
||||
n_estimators=self.n_estimators,
|
||||
random_state=42
|
||||
)
|
||||
|
||||
def fit(self, data: pd.DataFrame) -> None:
|
||||
"""Fit the Isolation Forest model"""
|
||||
# Select numeric columns only
|
||||
numeric_data = data.select_dtypes(include=[np.number])
|
||||
|
||||
if numeric_data.empty:
|
||||
raise ValueError("No numeric columns found in data")
|
||||
|
||||
# Handle missing values
|
||||
numeric_data = numeric_data.fillna(numeric_data.median())
|
||||
|
||||
# Scale the data
|
||||
scaled_data = self.scaler.fit_transform(numeric_data)
|
||||
|
||||
# Fit the model
|
||||
self.model.fit(scaled_data)
|
||||
self.is_fitted = True
|
||||
|
||||
def predict(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Predict anomalies using Isolation Forest"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Model must be fitted before prediction")
|
||||
|
||||
# Select numeric columns only
|
||||
numeric_data = data.select_dtypes(include=[np.number])
|
||||
|
||||
if numeric_data.empty:
|
||||
return np.zeros(len(data), dtype=bool)
|
||||
|
||||
# Handle missing values
|
||||
numeric_data = numeric_data.fillna(numeric_data.median())
|
||||
|
||||
# Scale the data
|
||||
scaled_data = self.scaler.transform(numeric_data)
|
||||
|
||||
# Predict anomalies (-1 for anomalies, 1 for normal)
|
||||
predictions = self.model.predict(scaled_data)
|
||||
return predictions == -1
|
||||
|
||||
def get_anomaly_scores(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Get anomaly scores from Isolation Forest"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Model must be fitted before prediction")
|
||||
|
||||
# Select numeric columns only
|
||||
numeric_data = data.select_dtypes(include=[np.number])
|
||||
|
||||
if numeric_data.empty:
|
||||
return np.zeros(len(data))
|
||||
|
||||
# Handle missing values
|
||||
numeric_data = numeric_data.fillna(numeric_data.median())
|
||||
|
||||
# Scale the data
|
||||
scaled_data = self.scaler.transform(numeric_data)
|
||||
|
||||
# Get anomaly scores
|
||||
scores = self.model.decision_function(scaled_data)
|
||||
# Convert to positive scores (higher = more anomalous)
|
||||
return -scores
|
||||
|
||||
|
||||
class TemporalAnomalyDetector(AnomalyDetector):
|
||||
"""Temporal anomaly detection for time series data"""
|
||||
|
||||
def __init__(self, model_config: Dict[str, Any] = None):
|
||||
super().__init__(model_config)
|
||||
self.window_size = self.model_config.get('window_size', 24) # hours
|
||||
self.threshold_multiplier = self.model_config.get('threshold_multiplier', 2.0)
|
||||
self.temporal_stats = {}
|
||||
|
||||
def fit(self, data: pd.DataFrame) -> None:
|
||||
"""Calculate temporal statistics for anomaly detection"""
|
||||
if 'timestamp' not in data.columns:
|
||||
raise ValueError("Timestamp column is required for temporal anomaly detection")
|
||||
|
||||
# Sort by timestamp
|
||||
data_sorted = data.sort_values('timestamp')
|
||||
|
||||
# Calculate rolling statistics
|
||||
for column in data_sorted.columns:
|
||||
if column != 'timestamp' and data_sorted[column].dtype in ['int64', 'float64']:
|
||||
# Calculate rolling mean and std
|
||||
rolling_mean = data_sorted[column].rolling(window=self.window_size, min_periods=1).mean()
|
||||
rolling_std = data_sorted[column].rolling(window=self.window_size, min_periods=1).std()
|
||||
|
||||
self.temporal_stats[column] = {
|
||||
'rolling_mean': rolling_mean,
|
||||
'rolling_std': rolling_std
|
||||
}
|
||||
|
||||
self.is_fitted = True
|
||||
|
||||
def predict(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Predict temporal anomalies"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Model must be fitted before prediction")
|
||||
|
||||
if 'timestamp' not in data.columns:
|
||||
return np.zeros(len(data), dtype=bool)
|
||||
|
||||
# Sort by timestamp
|
||||
data_sorted = data.sort_values('timestamp')
|
||||
anomaly_flags = np.zeros(len(data_sorted), dtype=bool)
|
||||
|
||||
for column in data_sorted.columns:
|
||||
if column in self.temporal_stats and column != 'timestamp':
|
||||
values = data_sorted[column]
|
||||
rolling_mean = self.temporal_stats[column]['rolling_mean']
|
||||
rolling_std = self.temporal_stats[column]['rolling_std']
|
||||
|
||||
# Calculate z-scores based on rolling statistics
|
||||
z_scores = np.abs((values - rolling_mean) / (rolling_std + 1e-8))
|
||||
column_anomalies = z_scores > self.threshold_multiplier
|
||||
|
||||
anomaly_flags |= column_anomalies
|
||||
|
||||
return anomaly_flags
|
||||
|
||||
def get_anomaly_scores(self, data: pd.DataFrame) -> np.ndarray:
|
||||
"""Get temporal anomaly scores"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Model must be fitted before prediction")
|
||||
|
||||
if 'timestamp' not in data.columns:
|
||||
return np.zeros(len(data))
|
||||
|
||||
# Sort by timestamp
|
||||
data_sorted = data.sort_values('timestamp')
|
||||
scores = np.zeros(len(data_sorted))
|
||||
|
||||
for column in data_sorted.columns:
|
||||
if column in self.temporal_stats and column != 'timestamp':
|
||||
values = data_sorted[column]
|
||||
rolling_mean = self.temporal_stats[column]['rolling_mean']
|
||||
rolling_std = self.temporal_stats[column]['rolling_std']
|
||||
|
||||
# Calculate z-scores based on rolling statistics
|
||||
z_scores = np.abs((values - rolling_mean) / (rolling_std + 1e-8))
|
||||
scores += z_scores
|
||||
|
||||
return scores
|
||||
|
||||
|
||||
class AnomalyDetectionEngine:
|
||||
"""Main engine for anomaly detection"""
|
||||
|
||||
def __init__(self):
|
||||
self.detectors = {
|
||||
'statistical': StatisticalAnomalyDetector,
|
||||
'isolation_forest': IsolationForestAnomalyDetector,
|
||||
'temporal': TemporalAnomalyDetector
|
||||
}
|
||||
|
||||
def create_detector(self, algorithm_type: str, model_config: Dict[str, Any] = None) -> AnomalyDetector:
|
||||
"""Create an anomaly detector instance"""
|
||||
if algorithm_type not in self.detectors:
|
||||
raise ValueError(f"Unknown algorithm type: {algorithm_type}")
|
||||
|
||||
return self.detectors[algorithm_type](model_config)
|
||||
|
||||
def prepare_incident_data(self, time_window_hours: int = 24) -> pd.DataFrame:
|
||||
"""Prepare incident data for anomaly detection"""
|
||||
end_time = timezone.now()
|
||||
start_time = end_time - timedelta(hours=time_window_hours)
|
||||
|
||||
# Get incidents from the time window
|
||||
incidents = Incident.objects.filter(
|
||||
created_at__gte=start_time,
|
||||
created_at__lte=end_time
|
||||
).values(
|
||||
'id', 'created_at', 'severity', 'category', 'subcategory',
|
||||
'affected_users', 'estimated_downtime', 'status'
|
||||
)
|
||||
|
||||
if not incidents:
|
||||
return pd.DataFrame()
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(list(incidents))
|
||||
|
||||
# Convert datetime to timestamp
|
||||
df['timestamp'] = pd.to_datetime(df['created_at']).astype('int64') // 10**9
|
||||
|
||||
# Encode categorical variables
|
||||
severity_mapping = {'LOW': 1, 'MEDIUM': 2, 'HIGH': 3, 'CRITICAL': 4, 'EMERGENCY': 5}
|
||||
df['severity_encoded'] = df['severity'].map(severity_mapping).fillna(0)
|
||||
|
||||
# Convert estimated_downtime to hours
|
||||
df['downtime_hours'] = df['estimated_downtime'].apply(
|
||||
lambda x: x.total_seconds() / 3600 if x else 0
|
||||
)
|
||||
|
||||
# Create time-based features
|
||||
df['hour_of_day'] = pd.to_datetime(df['created_at']).dt.hour
|
||||
df['day_of_week'] = pd.to_datetime(df['created_at']).dt.dayofweek
|
||||
|
||||
return df
|
||||
|
||||
def detect_anomalies(self, model: PredictiveModel, time_window_hours: int = 24) -> List[Dict[str, Any]]:
|
||||
"""Detect anomalies using the specified model"""
|
||||
try:
|
||||
# Prepare data
|
||||
data = self.prepare_incident_data(time_window_hours)
|
||||
|
||||
if data.empty:
|
||||
logger.warning("No incident data found for anomaly detection")
|
||||
return []
|
||||
|
||||
# Create detector
|
||||
detector = self.create_detector(
|
||||
model.algorithm_type,
|
||||
model.model_config
|
||||
)
|
||||
|
||||
# Fit the model
|
||||
detector.fit(data)
|
||||
|
||||
# Predict anomalies
|
||||
anomaly_flags = detector.predict(data)
|
||||
anomaly_scores = detector.get_anomaly_scores(data)
|
||||
|
||||
# Process results
|
||||
anomalies = []
|
||||
for idx, is_anomaly in enumerate(anomaly_flags):
|
||||
if is_anomaly:
|
||||
incident_data = data.iloc[idx]
|
||||
anomaly_data = {
|
||||
'model': model,
|
||||
'anomaly_type': self._determine_anomaly_type(model.algorithm_type),
|
||||
'severity': self._determine_severity(anomaly_scores[idx]),
|
||||
'confidence_score': min(1.0, max(0.0, anomaly_scores[idx] / 10.0)),
|
||||
'anomaly_score': float(anomaly_scores[idx]),
|
||||
'threshold_used': self._get_threshold(model.algorithm_type, model.model_config),
|
||||
'time_window_start': timezone.now() - timedelta(hours=time_window_hours),
|
||||
'time_window_end': timezone.now(),
|
||||
'description': self._generate_description(incident_data, anomaly_scores[idx]),
|
||||
'affected_services': [incident_data.get('category', 'Unknown')],
|
||||
'affected_metrics': ['incident_frequency', 'severity_distribution'],
|
||||
'metadata': {
|
||||
'incident_id': str(incident_data['id']),
|
||||
'detection_algorithm': model.algorithm_type,
|
||||
'time_window_hours': time_window_hours
|
||||
}
|
||||
}
|
||||
anomalies.append(anomaly_data)
|
||||
|
||||
return anomalies
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in anomaly detection: {str(e)}")
|
||||
return []
|
||||
|
||||
def _determine_anomaly_type(self, algorithm_type: str) -> str:
|
||||
"""Determine anomaly type based on algorithm"""
|
||||
mapping = {
|
||||
'statistical': 'STATISTICAL',
|
||||
'isolation_forest': 'PATTERN',
|
||||
'temporal': 'TEMPORAL'
|
||||
}
|
||||
return mapping.get(algorithm_type, 'STATISTICAL')
|
||||
|
||||
def _determine_severity(self, anomaly_score: float) -> str:
|
||||
"""Determine severity based on anomaly score"""
|
||||
if anomaly_score >= 5.0:
|
||||
return 'CRITICAL'
|
||||
elif anomaly_score >= 3.0:
|
||||
return 'HIGH'
|
||||
elif anomaly_score >= 2.0:
|
||||
return 'MEDIUM'
|
||||
else:
|
||||
return 'LOW'
|
||||
|
||||
def _get_threshold(self, algorithm_type: str, model_config: Dict[str, Any]) -> float:
|
||||
"""Get threshold used for anomaly detection"""
|
||||
if algorithm_type == 'statistical':
|
||||
return model_config.get('z_threshold', 3.0)
|
||||
elif algorithm_type == 'isolation_forest':
|
||||
return model_config.get('contamination', 0.1)
|
||||
elif algorithm_type == 'temporal':
|
||||
return model_config.get('threshold_multiplier', 2.0)
|
||||
return 1.0
|
||||
|
||||
def _generate_description(self, incident_data: pd.Series, anomaly_score: float) -> str:
|
||||
"""Generate description for the anomaly"""
|
||||
severity = incident_data.get('severity', 'Unknown')
|
||||
category = incident_data.get('category', 'Unknown')
|
||||
affected_users = incident_data.get('affected_users', 0)
|
||||
|
||||
return f"Anomalous incident detected: {severity} severity incident in {category} category affecting {affected_users} users. Anomaly score: {anomaly_score:.2f}"
|
||||
|
||||
|
||||
class AnomalyDetectionService:
|
||||
"""Service for managing anomaly detection"""
|
||||
|
||||
def __init__(self):
|
||||
self.engine = AnomalyDetectionEngine()
|
||||
|
||||
def run_anomaly_detection(self, model_id: str = None) -> int:
|
||||
"""Run anomaly detection for all active models or a specific model"""
|
||||
if model_id:
|
||||
models = PredictiveModel.objects.filter(
|
||||
id=model_id,
|
||||
model_type='ANOMALY_DETECTION',
|
||||
status='ACTIVE'
|
||||
)
|
||||
else:
|
||||
models = PredictiveModel.objects.filter(
|
||||
model_type='ANOMALY_DETECTION',
|
||||
status='ACTIVE'
|
||||
)
|
||||
|
||||
total_anomalies = 0
|
||||
|
||||
for model in models:
|
||||
try:
|
||||
# Detect anomalies
|
||||
anomalies = self.engine.detect_anomalies(model)
|
||||
|
||||
# Save anomalies to database
|
||||
for anomaly_data in anomalies:
|
||||
AnomalyDetection.objects.create(**anomaly_data)
|
||||
total_anomalies += 1
|
||||
|
||||
logger.info(f"Detected {len(anomalies)} anomalies using model {model.name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error running anomaly detection for model {model.name}: {str(e)}")
|
||||
|
||||
return total_anomalies
|
||||
|
||||
def get_anomaly_summary(self, time_window_hours: int = 24) -> Dict[str, Any]:
|
||||
"""Get summary of recent anomalies"""
|
||||
end_time = timezone.now()
|
||||
start_time = end_time - timedelta(hours=time_window_hours)
|
||||
|
||||
anomalies = AnomalyDetection.objects.filter(
|
||||
detected_at__gte=start_time,
|
||||
detected_at__lte=end_time
|
||||
)
|
||||
|
||||
return {
|
||||
'total_anomalies': anomalies.count(),
|
||||
'critical_anomalies': anomalies.filter(severity='CRITICAL').count(),
|
||||
'high_anomalies': anomalies.filter(severity='HIGH').count(),
|
||||
'medium_anomalies': anomalies.filter(severity='MEDIUM').count(),
|
||||
'low_anomalies': anomalies.filter(severity='LOW').count(),
|
||||
'unresolved_anomalies': anomalies.filter(
|
||||
status__in=['DETECTED', 'INVESTIGATING']
|
||||
).count(),
|
||||
'false_positive_rate': self._calculate_false_positive_rate(anomalies),
|
||||
'average_confidence': anomalies.aggregate(
|
||||
avg=Avg('confidence_score')
|
||||
)['avg'] or 0.0
|
||||
}
|
||||
|
||||
def _calculate_false_positive_rate(self, anomalies) -> float:
|
||||
"""Calculate false positive rate"""
|
||||
total_anomalies = anomalies.count()
|
||||
if total_anomalies == 0:
|
||||
return 0.0
|
||||
|
||||
false_positives = anomalies.filter(status='FALSE_POSITIVE').count()
|
||||
return (false_positives / total_anomalies) * 100
|
||||
Reference in New Issue
Block a user