""" Views for osint app. """ from django.shortcuts import get_object_or_404, redirect from django.views.generic import ListView, DetailView, UpdateView, TemplateView, CreateView, DeleteView from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin from django.contrib.messages.views import SuccessMessageMixin from django.contrib import messages from django.urls import reverse_lazy from django.utils import timezone from django.db import transaction from django.db.models import Count, Q from django.http import JsonResponse from django.core.management import call_command from django.core.management.base import CommandError import subprocess import threading from reports.models import ScamReport from .models import OSINTTask, OSINTResult, AutoGeneratedReport, SeedWebsite, OSINTKeyword, CrawledContent from .forms import SeedWebsiteForm, OSINTKeywordForm class ModeratorRequiredMixin(UserPassesTestMixin): """Mixin to require moderator role.""" def test_func(self): return self.request.user.is_authenticated and self.request.user.is_moderator() class OSINTTaskListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView): """List OSINT tasks.""" model = OSINTTask template_name = 'osint/task_list.html' context_object_name = 'tasks' paginate_by = 50 def get_queryset(self): status = self.request.GET.get('status', '') queryset = OSINTTask.objects.select_related('report') if status: queryset = queryset.filter(status=status) return queryset.order_by('-created_at') class OSINTTaskDetailView(LoginRequiredMixin, ModeratorRequiredMixin, DetailView): """View OSINT task details.""" model = OSINTTask template_name = 'osint/task_detail.html' context_object_name = 'task' class OSINTResultListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView): """List OSINT results for a report.""" model = OSINTResult template_name = 'osint/result_list.html' context_object_name = 'results' def get_queryset(self): report = get_object_or_404(ScamReport, pk=self.kwargs['report_id']) return OSINTResult.objects.filter(report=report).order_by('-collected_at') def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['report'] = get_object_or_404(ScamReport, pk=self.kwargs['report_id']) return context class AutoReportListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView): """List auto-generated reports for review.""" model = AutoGeneratedReport template_name = 'osint/auto_report_list.html' context_object_name = 'auto_reports' paginate_by = 20 def get_queryset(self): status = self.request.GET.get('status', 'pending') queryset = AutoGeneratedReport.objects.select_related( 'crawled_content', 'reviewed_by', 'report' ).prefetch_related('matched_keywords') if status: queryset = queryset.filter(status=status) return queryset.order_by('-confidence_score', '-created_at') def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['pending_count'] = AutoGeneratedReport.objects.filter(status='pending').count() context['approved_count'] = AutoGeneratedReport.objects.filter(status='approved').count() context['published_count'] = AutoGeneratedReport.objects.filter(status='published').count() context['rejected_count'] = AutoGeneratedReport.objects.filter(status='rejected').count() return context class AutoReportDetailView(LoginRequiredMixin, ModeratorRequiredMixin, DetailView): """View auto-generated report details.""" model = AutoGeneratedReport template_name = 'osint/auto_report_detail.html' context_object_name = 'auto_report' def get_queryset(self): return AutoGeneratedReport.objects.select_related( 'crawled_content', 'crawled_content__seed_website', 'reviewed_by', 'report' ).prefetch_related('matched_keywords') class ApproveAutoReportView(LoginRequiredMixin, ModeratorRequiredMixin, SuccessMessageMixin, UpdateView): """Approve an auto-generated report.""" model = AutoGeneratedReport fields = [] template_name = 'osint/approve_auto_report.html' success_message = "Auto-generated report approved successfully!" def form_valid(self, form): auto_report = form.instance with transaction.atomic(): # Update auto report auto_report.status = 'approved' auto_report.reviewed_by = self.request.user auto_report.reviewed_at = timezone.now() auto_report.save() # Create the actual scam report from reports.models import ScamReport report = ScamReport.objects.create( title=auto_report.title, description=auto_report.description, reported_url=auto_report.source_url, scam_type='other', # Default, can be updated status='verified', verification_score=auto_report.confidence_score, is_public=True, is_anonymous=True, # System-generated is_auto_discovered=True, # Mark as auto-discovered ) auto_report.report = report auto_report.status = 'published' auto_report.published_at = timezone.now() auto_report.save() return super().form_valid(form) def get_success_url(self): return reverse_lazy('osint:auto_report_list') class RejectAutoReportView(LoginRequiredMixin, ModeratorRequiredMixin, SuccessMessageMixin, UpdateView): """Reject an auto-generated report.""" model = AutoGeneratedReport fields = [] template_name = 'osint/reject_auto_report.html' success_message = "Auto-generated report rejected." def form_valid(self, form): auto_report = form.instance auto_report.status = 'rejected' auto_report.reviewed_by = self.request.user auto_report.reviewed_at = timezone.now() auto_report.review_notes = self.request.POST.get('review_notes', '').strip() auto_report.save() return super().form_valid(form) def get_success_url(self): return reverse_lazy('osint:auto_report_list') class AdminRequiredMixin(UserPassesTestMixin): """Mixin to require admin role.""" def test_func(self): return self.request.user.is_authenticated and self.request.user.is_administrator() class OSINTAdminDashboardView(LoginRequiredMixin, AdminRequiredMixin, TemplateView): """Comprehensive OSINT admin dashboard.""" template_name = 'osint/admin_dashboard.html' def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) now = timezone.now() # Seed Website Statistics context['total_seeds'] = SeedWebsite.objects.count() context['active_seeds'] = SeedWebsite.objects.filter(is_active=True).count() context['seed_websites'] = SeedWebsite.objects.all().order_by('-priority', '-last_crawled_at')[:10] # Keyword Statistics context['total_keywords'] = OSINTKeyword.objects.count() context['active_keywords'] = OSINTKeyword.objects.filter(is_active=True).count() context['keywords'] = OSINTKeyword.objects.all().order_by('-is_active', 'name')[:10] # Crawling Statistics context['total_crawled'] = CrawledContent.objects.count() context['potential_scams'] = CrawledContent.objects.filter(has_potential_scam=True).count() context['recent_crawled'] = CrawledContent.objects.order_by('-crawled_at')[:5] # Auto-Report Statistics context['pending_reports'] = AutoGeneratedReport.objects.filter(status='pending').count() context['approved_reports'] = AutoGeneratedReport.objects.filter(status='approved').count() context['published_reports'] = AutoGeneratedReport.objects.filter(status='published').count() context['rejected_reports'] = AutoGeneratedReport.objects.filter(status='rejected').count() context['recent_auto_reports'] = AutoGeneratedReport.objects.order_by('-created_at')[:5] # Overall Statistics context['total_pages_crawled'] = SeedWebsite.objects.aggregate( total=Count('pages_crawled') )['total'] or 0 context['total_matches'] = SeedWebsite.objects.aggregate( total=Count('matches_found') )['total'] or 0 # Seed websites due for crawling due_seeds = [] for seed in SeedWebsite.objects.filter(is_active=True): if not seed.last_crawled_at: due_seeds.append(seed) else: hours_since = (now - seed.last_crawled_at).total_seconds() / 3600 if hours_since >= seed.crawl_interval_hours: due_seeds.append(seed) context['due_for_crawling'] = due_seeds[:5] return context class SeedWebsiteCreateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, CreateView): """Create a new seed website.""" model = SeedWebsite form_class = SeedWebsiteForm template_name = 'osint/seed_website_form.html' success_message = "Seed website created successfully!" def form_valid(self, form): form.instance.created_by = self.request.user return super().form_valid(form) def get_success_url(self): return reverse_lazy('osint:admin_dashboard') class SeedWebsiteUpdateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, UpdateView): """Update a seed website.""" model = SeedWebsite form_class = SeedWebsiteForm template_name = 'osint/seed_website_form.html' success_message = "Seed website updated successfully!" def get_success_url(self): return reverse_lazy('osint:admin_dashboard') class SeedWebsiteDeleteView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, DeleteView): """Delete a seed website.""" model = SeedWebsite template_name = 'osint/seed_website_confirm_delete.html' success_message = "Seed website deleted successfully!" def get_success_url(self): return reverse_lazy('osint:admin_dashboard') class OSINTKeywordCreateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, CreateView): """Create a new OSINT keyword.""" model = OSINTKeyword form_class = OSINTKeywordForm template_name = 'osint/keyword_form.html' success_message = "Keyword created successfully!" def form_valid(self, form): form.instance.created_by = self.request.user return super().form_valid(form) def get_success_url(self): return reverse_lazy('osint:admin_dashboard') class OSINTKeywordUpdateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, UpdateView): """Update an OSINT keyword.""" model = OSINTKeyword form_class = OSINTKeywordForm template_name = 'osint/keyword_form.html' success_message = "Keyword updated successfully!" def get_success_url(self): return reverse_lazy('osint:admin_dashboard') class OSINTKeywordDeleteView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, DeleteView): """Delete an OSINT keyword.""" model = OSINTKeyword template_name = 'osint/keyword_confirm_delete.html' success_message = "Keyword deleted successfully!" def get_success_url(self): return reverse_lazy('osint:admin_dashboard') class StartCrawlingView(LoginRequiredMixin, AdminRequiredMixin, TemplateView): """Start OSINT crawling.""" template_name = 'osint/start_crawling.html' def post(self, request, *args, **kwargs): seed_id = request.POST.get('seed_id') max_pages = request.POST.get('max_pages', 50) delay = request.POST.get('delay', 1.0) def run_crawl(): import sys import os import django from django.db import connections # Ensure Django is set up for this thread django.setup() try: if seed_id: call_command('crawl_osint', '--seed-id', str(seed_id), '--max-pages', str(max_pages), '--delay', str(delay), verbosity=1) else: call_command('crawl_osint', '--all', '--max-pages', str(max_pages), '--delay', str(delay), verbosity=1) except Exception as e: # Log error to a file or database for debugging import traceback error_msg = f"Crawling error: {str(e)}\n{traceback.format_exc()}" print(error_msg, file=sys.stderr) # You could also log to a file or database here finally: # Close database connections connections.close_all() # Run in background thread thread = threading.Thread(target=run_crawl) thread.daemon = True thread.start() messages.success(request, f'Crawling started in background. Check results in a few minutes. (Max pages: {max_pages}, Delay: {delay}s)') return redirect('osint:admin_dashboard') def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['seed_websites'] = SeedWebsite.objects.filter(is_active=True) return context