347 lines
14 KiB
Python
347 lines
14 KiB
Python
"""
|
|
Views for osint app.
|
|
"""
|
|
from django.shortcuts import get_object_or_404, redirect
|
|
from django.views.generic import ListView, DetailView, UpdateView, TemplateView, CreateView, DeleteView
|
|
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
|
|
from django.contrib.messages.views import SuccessMessageMixin
|
|
from django.contrib import messages
|
|
from django.urls import reverse_lazy
|
|
from django.utils import timezone
|
|
from django.db import transaction
|
|
from django.db.models import Count, Q
|
|
from django.http import JsonResponse
|
|
from django.core.management import call_command
|
|
from django.core.management.base import CommandError
|
|
import subprocess
|
|
import threading
|
|
from reports.models import ScamReport
|
|
from .models import OSINTTask, OSINTResult, AutoGeneratedReport, SeedWebsite, OSINTKeyword, CrawledContent
|
|
from .forms import SeedWebsiteForm, OSINTKeywordForm
|
|
|
|
|
|
class ModeratorRequiredMixin(UserPassesTestMixin):
|
|
"""Mixin to require moderator role."""
|
|
def test_func(self):
|
|
return self.request.user.is_authenticated and self.request.user.is_moderator()
|
|
|
|
|
|
class OSINTTaskListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView):
|
|
"""List OSINT tasks."""
|
|
model = OSINTTask
|
|
template_name = 'osint/task_list.html'
|
|
context_object_name = 'tasks'
|
|
paginate_by = 50
|
|
|
|
def get_queryset(self):
|
|
status = self.request.GET.get('status', '')
|
|
queryset = OSINTTask.objects.select_related('report')
|
|
if status:
|
|
queryset = queryset.filter(status=status)
|
|
return queryset.order_by('-created_at')
|
|
|
|
|
|
class OSINTTaskDetailView(LoginRequiredMixin, ModeratorRequiredMixin, DetailView):
|
|
"""View OSINT task details."""
|
|
model = OSINTTask
|
|
template_name = 'osint/task_detail.html'
|
|
context_object_name = 'task'
|
|
|
|
|
|
class OSINTResultListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView):
|
|
"""List OSINT results for a report."""
|
|
model = OSINTResult
|
|
template_name = 'osint/result_list.html'
|
|
context_object_name = 'results'
|
|
|
|
def get_queryset(self):
|
|
report = get_object_or_404(ScamReport, pk=self.kwargs['report_id'])
|
|
return OSINTResult.objects.filter(report=report).order_by('-collected_at')
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
context['report'] = get_object_or_404(ScamReport, pk=self.kwargs['report_id'])
|
|
return context
|
|
|
|
|
|
class AutoReportListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView):
|
|
"""List auto-generated reports for review."""
|
|
model = AutoGeneratedReport
|
|
template_name = 'osint/auto_report_list.html'
|
|
context_object_name = 'auto_reports'
|
|
paginate_by = 20
|
|
|
|
def get_queryset(self):
|
|
status = self.request.GET.get('status', 'pending')
|
|
queryset = AutoGeneratedReport.objects.select_related(
|
|
'crawled_content', 'reviewed_by', 'report'
|
|
).prefetch_related('matched_keywords')
|
|
|
|
if status:
|
|
queryset = queryset.filter(status=status)
|
|
|
|
return queryset.order_by('-confidence_score', '-created_at')
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
context['pending_count'] = AutoGeneratedReport.objects.filter(status='pending').count()
|
|
context['approved_count'] = AutoGeneratedReport.objects.filter(status='approved').count()
|
|
context['published_count'] = AutoGeneratedReport.objects.filter(status='published').count()
|
|
context['rejected_count'] = AutoGeneratedReport.objects.filter(status='rejected').count()
|
|
return context
|
|
|
|
|
|
class AutoReportDetailView(LoginRequiredMixin, ModeratorRequiredMixin, DetailView):
|
|
"""View auto-generated report details."""
|
|
model = AutoGeneratedReport
|
|
template_name = 'osint/auto_report_detail.html'
|
|
context_object_name = 'auto_report'
|
|
|
|
def get_queryset(self):
|
|
return AutoGeneratedReport.objects.select_related(
|
|
'crawled_content', 'crawled_content__seed_website',
|
|
'reviewed_by', 'report'
|
|
).prefetch_related('matched_keywords')
|
|
|
|
|
|
class ApproveAutoReportView(LoginRequiredMixin, ModeratorRequiredMixin, SuccessMessageMixin, UpdateView):
|
|
"""Approve an auto-generated report."""
|
|
model = AutoGeneratedReport
|
|
fields = []
|
|
template_name = 'osint/approve_auto_report.html'
|
|
success_message = "Auto-generated report approved successfully!"
|
|
|
|
def form_valid(self, form):
|
|
auto_report = form.instance
|
|
|
|
with transaction.atomic():
|
|
# Update auto report
|
|
auto_report.status = 'approved'
|
|
auto_report.reviewed_by = self.request.user
|
|
auto_report.reviewed_at = timezone.now()
|
|
auto_report.save()
|
|
|
|
# Create the actual scam report
|
|
from reports.models import ScamReport
|
|
|
|
report = ScamReport.objects.create(
|
|
title=auto_report.title,
|
|
description=auto_report.description,
|
|
reported_url=auto_report.source_url,
|
|
scam_type='other', # Default, can be updated
|
|
status='verified',
|
|
verification_score=auto_report.confidence_score,
|
|
is_public=True,
|
|
is_anonymous=True, # System-generated
|
|
is_auto_discovered=True, # Mark as auto-discovered
|
|
)
|
|
|
|
auto_report.report = report
|
|
auto_report.status = 'published'
|
|
auto_report.published_at = timezone.now()
|
|
auto_report.save()
|
|
|
|
return super().form_valid(form)
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:auto_report_list')
|
|
|
|
|
|
class RejectAutoReportView(LoginRequiredMixin, ModeratorRequiredMixin, SuccessMessageMixin, UpdateView):
|
|
"""Reject an auto-generated report."""
|
|
model = AutoGeneratedReport
|
|
fields = []
|
|
template_name = 'osint/reject_auto_report.html'
|
|
success_message = "Auto-generated report rejected."
|
|
|
|
def form_valid(self, form):
|
|
auto_report = form.instance
|
|
auto_report.status = 'rejected'
|
|
auto_report.reviewed_by = self.request.user
|
|
auto_report.reviewed_at = timezone.now()
|
|
auto_report.review_notes = self.request.POST.get('review_notes', '').strip()
|
|
auto_report.save()
|
|
return super().form_valid(form)
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:auto_report_list')
|
|
|
|
|
|
class AdminRequiredMixin(UserPassesTestMixin):
|
|
"""Mixin to require admin role."""
|
|
def test_func(self):
|
|
return self.request.user.is_authenticated and self.request.user.is_administrator()
|
|
|
|
|
|
class OSINTAdminDashboardView(LoginRequiredMixin, AdminRequiredMixin, TemplateView):
|
|
"""Comprehensive OSINT admin dashboard."""
|
|
template_name = 'osint/admin_dashboard.html'
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
now = timezone.now()
|
|
|
|
# Seed Website Statistics
|
|
context['total_seeds'] = SeedWebsite.objects.count()
|
|
context['active_seeds'] = SeedWebsite.objects.filter(is_active=True).count()
|
|
context['seed_websites'] = SeedWebsite.objects.all().order_by('-priority', '-last_crawled_at')[:10]
|
|
|
|
# Keyword Statistics
|
|
context['total_keywords'] = OSINTKeyword.objects.count()
|
|
context['active_keywords'] = OSINTKeyword.objects.filter(is_active=True).count()
|
|
context['keywords'] = OSINTKeyword.objects.all().order_by('-is_active', 'name')[:10]
|
|
|
|
# Crawling Statistics
|
|
context['total_crawled'] = CrawledContent.objects.count()
|
|
context['potential_scams'] = CrawledContent.objects.filter(has_potential_scam=True).count()
|
|
context['recent_crawled'] = CrawledContent.objects.order_by('-crawled_at')[:5]
|
|
|
|
# Auto-Report Statistics
|
|
context['pending_reports'] = AutoGeneratedReport.objects.filter(status='pending').count()
|
|
context['approved_reports'] = AutoGeneratedReport.objects.filter(status='approved').count()
|
|
context['published_reports'] = AutoGeneratedReport.objects.filter(status='published').count()
|
|
context['rejected_reports'] = AutoGeneratedReport.objects.filter(status='rejected').count()
|
|
context['recent_auto_reports'] = AutoGeneratedReport.objects.order_by('-created_at')[:5]
|
|
|
|
# Overall Statistics
|
|
context['total_pages_crawled'] = SeedWebsite.objects.aggregate(
|
|
total=Count('pages_crawled')
|
|
)['total'] or 0
|
|
context['total_matches'] = SeedWebsite.objects.aggregate(
|
|
total=Count('matches_found')
|
|
)['total'] or 0
|
|
|
|
# Seed websites due for crawling
|
|
due_seeds = []
|
|
for seed in SeedWebsite.objects.filter(is_active=True):
|
|
if not seed.last_crawled_at:
|
|
due_seeds.append(seed)
|
|
else:
|
|
hours_since = (now - seed.last_crawled_at).total_seconds() / 3600
|
|
if hours_since >= seed.crawl_interval_hours:
|
|
due_seeds.append(seed)
|
|
context['due_for_crawling'] = due_seeds[:5]
|
|
|
|
return context
|
|
|
|
|
|
class SeedWebsiteCreateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, CreateView):
|
|
"""Create a new seed website."""
|
|
model = SeedWebsite
|
|
form_class = SeedWebsiteForm
|
|
template_name = 'osint/seed_website_form.html'
|
|
success_message = "Seed website created successfully!"
|
|
|
|
def form_valid(self, form):
|
|
form.instance.created_by = self.request.user
|
|
return super().form_valid(form)
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:admin_dashboard')
|
|
|
|
|
|
class SeedWebsiteUpdateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, UpdateView):
|
|
"""Update a seed website."""
|
|
model = SeedWebsite
|
|
form_class = SeedWebsiteForm
|
|
template_name = 'osint/seed_website_form.html'
|
|
success_message = "Seed website updated successfully!"
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:admin_dashboard')
|
|
|
|
|
|
class SeedWebsiteDeleteView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, DeleteView):
|
|
"""Delete a seed website."""
|
|
model = SeedWebsite
|
|
template_name = 'osint/seed_website_confirm_delete.html'
|
|
success_message = "Seed website deleted successfully!"
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:admin_dashboard')
|
|
|
|
|
|
class OSINTKeywordCreateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, CreateView):
|
|
"""Create a new OSINT keyword."""
|
|
model = OSINTKeyword
|
|
form_class = OSINTKeywordForm
|
|
template_name = 'osint/keyword_form.html'
|
|
success_message = "Keyword created successfully!"
|
|
|
|
def form_valid(self, form):
|
|
form.instance.created_by = self.request.user
|
|
return super().form_valid(form)
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:admin_dashboard')
|
|
|
|
|
|
class OSINTKeywordUpdateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, UpdateView):
|
|
"""Update an OSINT keyword."""
|
|
model = OSINTKeyword
|
|
form_class = OSINTKeywordForm
|
|
template_name = 'osint/keyword_form.html'
|
|
success_message = "Keyword updated successfully!"
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:admin_dashboard')
|
|
|
|
|
|
class OSINTKeywordDeleteView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, DeleteView):
|
|
"""Delete an OSINT keyword."""
|
|
model = OSINTKeyword
|
|
template_name = 'osint/keyword_confirm_delete.html'
|
|
success_message = "Keyword deleted successfully!"
|
|
|
|
def get_success_url(self):
|
|
return reverse_lazy('osint:admin_dashboard')
|
|
|
|
|
|
class StartCrawlingView(LoginRequiredMixin, AdminRequiredMixin, TemplateView):
|
|
"""Start OSINT crawling."""
|
|
template_name = 'osint/start_crawling.html'
|
|
|
|
def post(self, request, *args, **kwargs):
|
|
seed_id = request.POST.get('seed_id')
|
|
max_pages = request.POST.get('max_pages', 50)
|
|
delay = request.POST.get('delay', 1.0)
|
|
|
|
def run_crawl():
|
|
import sys
|
|
import os
|
|
import django
|
|
from django.db import connections
|
|
|
|
# Ensure Django is set up for this thread
|
|
django.setup()
|
|
|
|
try:
|
|
if seed_id:
|
|
call_command('crawl_osint', '--seed-id', str(seed_id),
|
|
'--max-pages', str(max_pages), '--delay', str(delay), verbosity=1)
|
|
else:
|
|
call_command('crawl_osint', '--all',
|
|
'--max-pages', str(max_pages), '--delay', str(delay), verbosity=1)
|
|
except Exception as e:
|
|
# Log error to a file or database for debugging
|
|
import traceback
|
|
error_msg = f"Crawling error: {str(e)}\n{traceback.format_exc()}"
|
|
print(error_msg, file=sys.stderr)
|
|
# You could also log to a file or database here
|
|
finally:
|
|
# Close database connections
|
|
connections.close_all()
|
|
|
|
# Run in background thread
|
|
thread = threading.Thread(target=run_crawl)
|
|
thread.daemon = True
|
|
thread.start()
|
|
|
|
messages.success(request, f'Crawling started in background. Check results in a few minutes. (Max pages: {max_pages}, Delay: {delay}s)')
|
|
return redirect('osint:admin_dashboard')
|
|
|
|
def get_context_data(self, **kwargs):
|
|
context = super().get_context_data(**kwargs)
|
|
context['seed_websites'] = SeedWebsite.objects.filter(is_active=True)
|
|
return context
|