update
This commit is contained in:
Iliyan Angelov
346
osint/views.py
Normal file
346
osint/views.py
Normal file
@@ -0,0 +1,346 @@
|
||||
"""
|
||||
Views for osint app.
|
||||
"""
|
||||
from django.shortcuts import get_object_or_404, redirect
|
||||
from django.views.generic import ListView, DetailView, UpdateView, TemplateView, CreateView, DeleteView
|
||||
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
|
||||
from django.contrib.messages.views import SuccessMessageMixin
|
||||
from django.contrib import messages
|
||||
from django.urls import reverse_lazy
|
||||
from django.utils import timezone
|
||||
from django.db import transaction
|
||||
from django.db.models import Count, Q
|
||||
from django.http import JsonResponse
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import CommandError
|
||||
import subprocess
|
||||
import threading
|
||||
from reports.models import ScamReport
|
||||
from .models import OSINTTask, OSINTResult, AutoGeneratedReport, SeedWebsite, OSINTKeyword, CrawledContent
|
||||
from .forms import SeedWebsiteForm, OSINTKeywordForm
|
||||
|
||||
|
||||
class ModeratorRequiredMixin(UserPassesTestMixin):
|
||||
"""Mixin to require moderator role."""
|
||||
def test_func(self):
|
||||
return self.request.user.is_authenticated and self.request.user.is_moderator()
|
||||
|
||||
|
||||
class OSINTTaskListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView):
|
||||
"""List OSINT tasks."""
|
||||
model = OSINTTask
|
||||
template_name = 'osint/task_list.html'
|
||||
context_object_name = 'tasks'
|
||||
paginate_by = 50
|
||||
|
||||
def get_queryset(self):
|
||||
status = self.request.GET.get('status', '')
|
||||
queryset = OSINTTask.objects.select_related('report')
|
||||
if status:
|
||||
queryset = queryset.filter(status=status)
|
||||
return queryset.order_by('-created_at')
|
||||
|
||||
|
||||
class OSINTTaskDetailView(LoginRequiredMixin, ModeratorRequiredMixin, DetailView):
|
||||
"""View OSINT task details."""
|
||||
model = OSINTTask
|
||||
template_name = 'osint/task_detail.html'
|
||||
context_object_name = 'task'
|
||||
|
||||
|
||||
class OSINTResultListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView):
|
||||
"""List OSINT results for a report."""
|
||||
model = OSINTResult
|
||||
template_name = 'osint/result_list.html'
|
||||
context_object_name = 'results'
|
||||
|
||||
def get_queryset(self):
|
||||
report = get_object_or_404(ScamReport, pk=self.kwargs['report_id'])
|
||||
return OSINTResult.objects.filter(report=report).order_by('-collected_at')
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
context['report'] = get_object_or_404(ScamReport, pk=self.kwargs['report_id'])
|
||||
return context
|
||||
|
||||
|
||||
class AutoReportListView(LoginRequiredMixin, ModeratorRequiredMixin, ListView):
|
||||
"""List auto-generated reports for review."""
|
||||
model = AutoGeneratedReport
|
||||
template_name = 'osint/auto_report_list.html'
|
||||
context_object_name = 'auto_reports'
|
||||
paginate_by = 20
|
||||
|
||||
def get_queryset(self):
|
||||
status = self.request.GET.get('status', 'pending')
|
||||
queryset = AutoGeneratedReport.objects.select_related(
|
||||
'crawled_content', 'reviewed_by', 'report'
|
||||
).prefetch_related('matched_keywords')
|
||||
|
||||
if status:
|
||||
queryset = queryset.filter(status=status)
|
||||
|
||||
return queryset.order_by('-confidence_score', '-created_at')
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
context['pending_count'] = AutoGeneratedReport.objects.filter(status='pending').count()
|
||||
context['approved_count'] = AutoGeneratedReport.objects.filter(status='approved').count()
|
||||
context['published_count'] = AutoGeneratedReport.objects.filter(status='published').count()
|
||||
context['rejected_count'] = AutoGeneratedReport.objects.filter(status='rejected').count()
|
||||
return context
|
||||
|
||||
|
||||
class AutoReportDetailView(LoginRequiredMixin, ModeratorRequiredMixin, DetailView):
|
||||
"""View auto-generated report details."""
|
||||
model = AutoGeneratedReport
|
||||
template_name = 'osint/auto_report_detail.html'
|
||||
context_object_name = 'auto_report'
|
||||
|
||||
def get_queryset(self):
|
||||
return AutoGeneratedReport.objects.select_related(
|
||||
'crawled_content', 'crawled_content__seed_website',
|
||||
'reviewed_by', 'report'
|
||||
).prefetch_related('matched_keywords')
|
||||
|
||||
|
||||
class ApproveAutoReportView(LoginRequiredMixin, ModeratorRequiredMixin, SuccessMessageMixin, UpdateView):
|
||||
"""Approve an auto-generated report."""
|
||||
model = AutoGeneratedReport
|
||||
fields = []
|
||||
template_name = 'osint/approve_auto_report.html'
|
||||
success_message = "Auto-generated report approved successfully!"
|
||||
|
||||
def form_valid(self, form):
|
||||
auto_report = form.instance
|
||||
|
||||
with transaction.atomic():
|
||||
# Update auto report
|
||||
auto_report.status = 'approved'
|
||||
auto_report.reviewed_by = self.request.user
|
||||
auto_report.reviewed_at = timezone.now()
|
||||
auto_report.save()
|
||||
|
||||
# Create the actual scam report
|
||||
from reports.models import ScamReport
|
||||
|
||||
report = ScamReport.objects.create(
|
||||
title=auto_report.title,
|
||||
description=auto_report.description,
|
||||
reported_url=auto_report.source_url,
|
||||
scam_type='other', # Default, can be updated
|
||||
status='verified',
|
||||
verification_score=auto_report.confidence_score,
|
||||
is_public=True,
|
||||
is_anonymous=True, # System-generated
|
||||
is_auto_discovered=True, # Mark as auto-discovered
|
||||
)
|
||||
|
||||
auto_report.report = report
|
||||
auto_report.status = 'published'
|
||||
auto_report.published_at = timezone.now()
|
||||
auto_report.save()
|
||||
|
||||
return super().form_valid(form)
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:auto_report_list')
|
||||
|
||||
|
||||
class RejectAutoReportView(LoginRequiredMixin, ModeratorRequiredMixin, SuccessMessageMixin, UpdateView):
|
||||
"""Reject an auto-generated report."""
|
||||
model = AutoGeneratedReport
|
||||
fields = []
|
||||
template_name = 'osint/reject_auto_report.html'
|
||||
success_message = "Auto-generated report rejected."
|
||||
|
||||
def form_valid(self, form):
|
||||
auto_report = form.instance
|
||||
auto_report.status = 'rejected'
|
||||
auto_report.reviewed_by = self.request.user
|
||||
auto_report.reviewed_at = timezone.now()
|
||||
auto_report.review_notes = self.request.POST.get('review_notes', '').strip()
|
||||
auto_report.save()
|
||||
return super().form_valid(form)
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:auto_report_list')
|
||||
|
||||
|
||||
class AdminRequiredMixin(UserPassesTestMixin):
|
||||
"""Mixin to require admin role."""
|
||||
def test_func(self):
|
||||
return self.request.user.is_authenticated and self.request.user.is_administrator()
|
||||
|
||||
|
||||
class OSINTAdminDashboardView(LoginRequiredMixin, AdminRequiredMixin, TemplateView):
|
||||
"""Comprehensive OSINT admin dashboard."""
|
||||
template_name = 'osint/admin_dashboard.html'
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
now = timezone.now()
|
||||
|
||||
# Seed Website Statistics
|
||||
context['total_seeds'] = SeedWebsite.objects.count()
|
||||
context['active_seeds'] = SeedWebsite.objects.filter(is_active=True).count()
|
||||
context['seed_websites'] = SeedWebsite.objects.all().order_by('-priority', '-last_crawled_at')[:10]
|
||||
|
||||
# Keyword Statistics
|
||||
context['total_keywords'] = OSINTKeyword.objects.count()
|
||||
context['active_keywords'] = OSINTKeyword.objects.filter(is_active=True).count()
|
||||
context['keywords'] = OSINTKeyword.objects.all().order_by('-is_active', 'name')[:10]
|
||||
|
||||
# Crawling Statistics
|
||||
context['total_crawled'] = CrawledContent.objects.count()
|
||||
context['potential_scams'] = CrawledContent.objects.filter(has_potential_scam=True).count()
|
||||
context['recent_crawled'] = CrawledContent.objects.order_by('-crawled_at')[:5]
|
||||
|
||||
# Auto-Report Statistics
|
||||
context['pending_reports'] = AutoGeneratedReport.objects.filter(status='pending').count()
|
||||
context['approved_reports'] = AutoGeneratedReport.objects.filter(status='approved').count()
|
||||
context['published_reports'] = AutoGeneratedReport.objects.filter(status='published').count()
|
||||
context['rejected_reports'] = AutoGeneratedReport.objects.filter(status='rejected').count()
|
||||
context['recent_auto_reports'] = AutoGeneratedReport.objects.order_by('-created_at')[:5]
|
||||
|
||||
# Overall Statistics
|
||||
context['total_pages_crawled'] = SeedWebsite.objects.aggregate(
|
||||
total=Count('pages_crawled')
|
||||
)['total'] or 0
|
||||
context['total_matches'] = SeedWebsite.objects.aggregate(
|
||||
total=Count('matches_found')
|
||||
)['total'] or 0
|
||||
|
||||
# Seed websites due for crawling
|
||||
due_seeds = []
|
||||
for seed in SeedWebsite.objects.filter(is_active=True):
|
||||
if not seed.last_crawled_at:
|
||||
due_seeds.append(seed)
|
||||
else:
|
||||
hours_since = (now - seed.last_crawled_at).total_seconds() / 3600
|
||||
if hours_since >= seed.crawl_interval_hours:
|
||||
due_seeds.append(seed)
|
||||
context['due_for_crawling'] = due_seeds[:5]
|
||||
|
||||
return context
|
||||
|
||||
|
||||
class SeedWebsiteCreateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, CreateView):
|
||||
"""Create a new seed website."""
|
||||
model = SeedWebsite
|
||||
form_class = SeedWebsiteForm
|
||||
template_name = 'osint/seed_website_form.html'
|
||||
success_message = "Seed website created successfully!"
|
||||
|
||||
def form_valid(self, form):
|
||||
form.instance.created_by = self.request.user
|
||||
return super().form_valid(form)
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:admin_dashboard')
|
||||
|
||||
|
||||
class SeedWebsiteUpdateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, UpdateView):
|
||||
"""Update a seed website."""
|
||||
model = SeedWebsite
|
||||
form_class = SeedWebsiteForm
|
||||
template_name = 'osint/seed_website_form.html'
|
||||
success_message = "Seed website updated successfully!"
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:admin_dashboard')
|
||||
|
||||
|
||||
class SeedWebsiteDeleteView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, DeleteView):
|
||||
"""Delete a seed website."""
|
||||
model = SeedWebsite
|
||||
template_name = 'osint/seed_website_confirm_delete.html'
|
||||
success_message = "Seed website deleted successfully!"
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:admin_dashboard')
|
||||
|
||||
|
||||
class OSINTKeywordCreateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, CreateView):
|
||||
"""Create a new OSINT keyword."""
|
||||
model = OSINTKeyword
|
||||
form_class = OSINTKeywordForm
|
||||
template_name = 'osint/keyword_form.html'
|
||||
success_message = "Keyword created successfully!"
|
||||
|
||||
def form_valid(self, form):
|
||||
form.instance.created_by = self.request.user
|
||||
return super().form_valid(form)
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:admin_dashboard')
|
||||
|
||||
|
||||
class OSINTKeywordUpdateView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, UpdateView):
|
||||
"""Update an OSINT keyword."""
|
||||
model = OSINTKeyword
|
||||
form_class = OSINTKeywordForm
|
||||
template_name = 'osint/keyword_form.html'
|
||||
success_message = "Keyword updated successfully!"
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:admin_dashboard')
|
||||
|
||||
|
||||
class OSINTKeywordDeleteView(LoginRequiredMixin, AdminRequiredMixin, SuccessMessageMixin, DeleteView):
|
||||
"""Delete an OSINT keyword."""
|
||||
model = OSINTKeyword
|
||||
template_name = 'osint/keyword_confirm_delete.html'
|
||||
success_message = "Keyword deleted successfully!"
|
||||
|
||||
def get_success_url(self):
|
||||
return reverse_lazy('osint:admin_dashboard')
|
||||
|
||||
|
||||
class StartCrawlingView(LoginRequiredMixin, AdminRequiredMixin, TemplateView):
|
||||
"""Start OSINT crawling."""
|
||||
template_name = 'osint/start_crawling.html'
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
seed_id = request.POST.get('seed_id')
|
||||
max_pages = request.POST.get('max_pages', 50)
|
||||
delay = request.POST.get('delay', 1.0)
|
||||
|
||||
def run_crawl():
|
||||
import sys
|
||||
import os
|
||||
import django
|
||||
from django.db import connections
|
||||
|
||||
# Ensure Django is set up for this thread
|
||||
django.setup()
|
||||
|
||||
try:
|
||||
if seed_id:
|
||||
call_command('crawl_osint', '--seed-id', str(seed_id),
|
||||
'--max-pages', str(max_pages), '--delay', str(delay), verbosity=1)
|
||||
else:
|
||||
call_command('crawl_osint', '--all',
|
||||
'--max-pages', str(max_pages), '--delay', str(delay), verbosity=1)
|
||||
except Exception as e:
|
||||
# Log error to a file or database for debugging
|
||||
import traceback
|
||||
error_msg = f"Crawling error: {str(e)}\n{traceback.format_exc()}"
|
||||
print(error_msg, file=sys.stderr)
|
||||
# You could also log to a file or database here
|
||||
finally:
|
||||
# Close database connections
|
||||
connections.close_all()
|
||||
|
||||
# Run in background thread
|
||||
thread = threading.Thread(target=run_crawl)
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
messages.success(request, f'Crawling started in background. Check results in a few minutes. (Max pages: {max_pages}, Delay: {delay}s)')
|
||||
return redirect('osint:admin_dashboard')
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
context['seed_websites'] = SeedWebsite.objects.filter(is_active=True)
|
||||
return context
|
||||
Reference in New Issue
Block a user