diff --git a/docs/conf.py b/docs/conf.py index b22f1261..e658331f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,6 +54,11 @@ 'sphinx.ext.napoleon' ] +# forum.toggles reads its waffle flags from edx-platform, which is the host +# application and is not installed when the docs are built. Mocking it lets +# autodoc document the module instead of failing to import it. +autodoc_mock_imports = ['openedx'] + # A list of warning types to suppress arbitrary warning messages. suppress_warnings = [ 'image.nonlocal_uri', diff --git a/docs/how-tos/configure_ai_moderation.rst b/docs/how-tos/configure_ai_moderation.rst new file mode 100644 index 00000000..a916c015 --- /dev/null +++ b/docs/how-tos/configure_ai_moderation.rst @@ -0,0 +1,104 @@ +Configure AI moderation +####################### + +Forum can classify new threads and comments as spam and flag them, using an AI +provider of your choosing. Forum ships the interface; you supply a backend that +wraps your provider. + + +1. Write a backend +****************** + +Subclass ``HTTPModerationBackend`` and describe three things: how to +authenticate, what the request body looks like, and where the verdict sits in +the response. + +.. code-block:: python + + from django.conf import settings + + from forum.ai_moderation.backends import HTTPModerationBackend + + + class MyProviderBackend(HTTPModerationBackend): + def classify(self, content): + headers = { + "content-type": "application/json", + "Authorization": f"Bearer {settings.MY_PROVIDER_API_KEY}", + } + payload = { + "model": settings.MY_PROVIDER_MODEL, + "messages": [ + {"role": "system", "content": self.system_message}, + {"role": "user", "content": content}, + ], + } + + response = self.post(payload, headers) + if response is None: + return None + + answer = response["choices"][0]["message"]["content"] + return self.parse_moderation_payload(answer, response) + +``self.post()`` and ``self.parse_moderation_payload()`` handle the endpoint, +timeouts, network failures and parsing; ``self.system_message`` is the prompt. +See ``forum.ai_moderation.backends.base`` for the interface and what +``classify()`` must return. + +If your provider is not a JSON HTTP API, subclass ``BaseModerationBackend`` and +implement ``classify()`` however you like. + +Put the class anywhere the LMS can import. + + +2. Configure it +*************** + +.. code-block:: python + + AI_MODERATION_BACKEND = "myproject.moderation.MyProviderBackend" + AI_MODERATION_API_URL = "https://api.myprovider.example.com/v1/chat/completions" + AI_MODERATION_USER_ID = 42 + + # Read by your backend, named however you like. + MY_PROVIDER_API_KEY = "..." + MY_PROVIDER_MODEL = "..." + +All three forum settings are required and have no defaults. +``AI_MODERATION_USER_ID`` is the user that flagging and deletion are attributed +to; no user is created for you. + +Set them however your deployment sets Django settings -- with Tutor, a plugin +patching ``openedx-lms-production-settings``. Forum declares these settings in +its own plugin settings, so there is nothing to add to edx-platform. + +Optional: + +``AI_MODERATION_SYSTEM_MESSAGE`` + Your own prompt. Defaults to forum's, which asks for the JSON that + ``parse_moderation_payload()`` expects. + +``AI_MODERATION_CONNECTION_TIMEOUT``, ``AI_MODERATION_READ_TIMEOUT`` + Default to 1.0 and 30 seconds. Moderation runs inline with posting, so keep + them low. + +``AI_MODERATION_FLAGGED_CACHE_TTL``, ``AI_MODERATION_FLAGGED_CACHE_PREFIX`` + Spam verdicts are cached by content hash for 24 hours, so an identical + repost costs no second API call. Clean verdicts are never cached. + + +3. Turn it on +************* + +Two course waffle flags, both off by default: + +``discussions.enable_ai_moderation`` + Classify new threads and comments, and flag what comes back spam. + +``discussions.enable_ai_auto_delete_spam`` + Also delete what was flagged. Has no effect on its own. + +Enable them site-wide in Django admin under ``waffle/flag``, or per course with +a "Waffle flag course override" at +``/admin/waffle_utils/waffleflagcourseoverridemodel/``. diff --git a/docs/how-tos/index.rst b/docs/how-tos/index.rst index 5147f808..f2bf79d7 100644 --- a/docs/how-tos/index.rst +++ b/docs/how-tos/index.rst @@ -1,2 +1,7 @@ How-tos ####### + +.. toctree:: + :maxdepth: 1 + + configure_ai_moderation diff --git a/src/forum/admin.py b/src/forum/admin.py index 3d624a8e..c1ff0239 100644 --- a/src/forum/admin.py +++ b/src/forum/admin.py @@ -14,6 +14,7 @@ UserVote, Subscription, MongoContent, + ModerationAuditLog, ) @@ -55,11 +56,12 @@ class CommentThreadAdmin(admin.ModelAdmin): # type: ignore "context", "closed", "pinned", + "is_spam", "created_at", "updated_at", ) search_fields = ("title", "body", "author__username", "course_id") - list_filter = ("thread_type", "context", "closed", "pinned") + list_filter = ("thread_type", "context", "closed", "pinned", "is_spam") @admin.register(Comment) @@ -74,9 +76,10 @@ class CommentAdmin(admin.ModelAdmin): # type: ignore "updated_at", "endorsed", "anonymous", + "is_spam", ) search_fields = ("body", "author__username", "comment_thread__title") - list_filter = ("endorsed", "anonymous") + list_filter = ("endorsed", "anonymous", "is_spam") @admin.register(EditHistory) @@ -152,3 +155,100 @@ class MongoContentAdmin(admin.ModelAdmin): # type: ignore list_display = ("mongo_id", "content_object_id", "content_type") search_fields = ("mongo_id",) + + +@admin.register(ModerationAuditLog) +class ModerationAuditLogAdmin(admin.ModelAdmin): # type: ignore + """Admin interface for ModerationAuditLog model.""" + + list_display = ( + "timestamp", + "classification", + "actions_taken", + "body_preview", + "original_author", + "moderator_override", + "confidence_score", + ) + list_filter = ( + "classification", + "moderator_override", + "timestamp", + ) + search_fields = ( + "original_author__username", + "moderator__username", + "reasoning", + "override_reason", + "body", + ) + readonly_fields = ( + "timestamp", + "body", + "classifier_output", + "reasoning", + "classification", + "actions_taken", + "confidence_score", + "original_author", + ) + fieldsets = ( + ( + "Moderation Decision", + { + "fields": ( + "timestamp", + "classification", + "actions_taken", + "confidence_score", + "reasoning", + ) + }, + ), + ("Content Information", {"fields": ("body",)}), + ("Author Information", {"fields": ("original_author",)}), + ( + "Human Override", + { + "fields": ( + "moderator_override", + "moderator", + "override_reason", + ) + }, + ), + ( + "Technical Details", + { + "fields": ("classifier_output",), + "classes": ("collapse",), + }, + ), + ) + + def body_preview(self, obj): # type: ignore + """Return a truncated preview of the body for list display.""" + if obj.body: + return obj.body[:100] + "..." if len(obj.body) > 100 else obj.body + return "-" + + body_preview.short_description = "Body Preview" # type: ignore + + # pylint: disable=unused-argument + def has_add_permission(self, request): # type: ignore[no-untyped-def] + """Disable adding audit logs manually.""" + return False + + # pylint: disable=unused-argument + def has_delete_permission(self, request, obj=None): # type: ignore[no-untyped-def] + """Disable deleting audit logs to maintain integrity.""" + return False + + def get_queryset(self, request): # type: ignore + """Optimize queryset with related objects.""" + return ( + super() + .get_queryset(request) + .select_related("original_author", "moderator") + .order_by("-timestamp") + ) diff --git a/src/forum/ai_moderation/__init__.py b/src/forum/ai_moderation/__init__.py new file mode 100644 index 00000000..ca6b809f --- /dev/null +++ b/src/forum/ai_moderation/__init__.py @@ -0,0 +1,8 @@ +""" +AI moderation for forum content. + +Classifies each new thread and comment with the configured AI provider, flags +what comes back spam, and soft deletes it when auto-delete is enabled. Both +steps are gated on course waffle flags, and every spam verdict is recorded on a +moderation audit log. +""" diff --git a/src/forum/ai_moderation/backends/__init__.py b/src/forum/ai_moderation/backends/__init__.py new file mode 100644 index 00000000..3070989a --- /dev/null +++ b/src/forum/ai_moderation/backends/__init__.py @@ -0,0 +1,14 @@ +""" +AI moderation backends. + +Each backend adapts one AI provider to the common moderation interface. The +backend in use is chosen with the ``AI_MODERATION_BACKEND`` setting, so nothing +outside this package needs to know which provider is answering. +""" + +from forum.ai_moderation.backends.base import ( + BaseModerationBackend, + HTTPModerationBackend, +) + +__all__ = ["BaseModerationBackend", "HTTPModerationBackend"] diff --git a/src/forum/ai_moderation/backends/base.py b/src/forum/ai_moderation/backends/base.py new file mode 100644 index 00000000..b51eff2b --- /dev/null +++ b/src/forum/ai_moderation/backends/base.py @@ -0,0 +1,188 @@ +""" +Provider-agnostic interface for AI moderation backends. + +A moderation backend is the only part of AI moderation that knows how to talk to +a particular AI provider. It takes a piece of forum content and returns the +common moderation result described by :meth:`BaseModerationBackend.classify`; +everything downstream of that -- caching, flagging, deletion, audit logging -- +is provider independent and lives in :mod:`forum.ai_moderation.service`. +""" + +import json +import logging +from typing import Any, Dict, Optional, Tuple + +import requests +from django.conf import settings + +from forum.ai_moderation.defaults import ( + DEFAULT_CONNECTION_TIMEOUT, + DEFAULT_READ_TIMEOUT, + DEFAULT_REASONING, + DEFAULT_SYSTEM_MESSAGE, +) + +log = logging.getLogger(__name__) + +CLASSIFICATION_SPAM = "spam_or_scam" +CLASSIFICATION_NOT_SPAM = "not_spam" + +# Classifications that count as spam. "spam" is accepted alongside the +# documented "spam_or_scam" because prompts in the wild return either. +SPAM_CLASSIFICATIONS = ("spam", CLASSIFICATION_SPAM) + + +class BaseModerationBackend: + """ + Interface implemented by every AI moderation backend. + """ + + def classify(self, content: str) -> Optional[Dict[str, Any]]: + """ + Classify a piece of forum content. + + Args: + content: The text content to classify. + + Returns: + A moderation result:: + + { + "classification": "spam_or_scam" | "not_spam", + "reasoning": "...", + "confidence_score": float | None, + "full_api_response": , + } + + or None if the provider could not be reached or answered with + something that could not be understood. Returning None must never + raise: a failing classifier degrades moderation, it does not break + posting. + """ + raise NotImplementedError + + +class HTTPModerationBackend(BaseModerationBackend): # pylint: disable=abstract-method + """ + Base class for backends that call an HTTP moderation API. + + It owns the concerns that are the same whichever provider is in use -- + reading the endpoint, prompt and timeouts from Django settings, POSTing + JSON, and turning the classifier's JSON payload into the common moderation + result. Subclasses only describe the provider's request and response shape. + """ + + @property + def api_url(self) -> Optional[str]: + """Endpoint the classifier is served from.""" + return getattr(settings, "AI_MODERATION_API_URL", None) + + @property + def system_message(self) -> str: + """Prompt describing the classification task and its output format.""" + return ( + getattr(settings, "AI_MODERATION_SYSTEM_MESSAGE", None) + or DEFAULT_SYSTEM_MESSAGE + ) + + @property + def timeout(self) -> Tuple[float, float]: + """Connection and read timeouts, in seconds.""" + return ( + getattr( + settings, "AI_MODERATION_CONNECTION_TIMEOUT", DEFAULT_CONNECTION_TIMEOUT + ), + getattr(settings, "AI_MODERATION_READ_TIMEOUT", DEFAULT_READ_TIMEOUT), + ) + + def post(self, payload: Dict[str, Any], headers: Dict[str, str]) -> Optional[Any]: + """ + POST a JSON payload to the configured endpoint and decode the response. + + Returns the decoded JSON body, or None if the request failed. + """ + if not self.api_url: + log.error("AI_MODERATION_API_URL setting is not configured") + return None + + try: + response = requests.post( + self.api_url, + headers=headers, + json=payload, + timeout=self.timeout, + ) + response.raise_for_status() + return response.json() + except ( + requests.RequestException, + requests.Timeout, + requests.ConnectionError, + ) as e: + log.error(f"AI moderation API request failed: {e}") + return None + except ValueError as e: + log.error(f"AI moderation API returned a non-JSON response: {e}") + return None + + def parse_moderation_payload( + self, raw_content: Any, full_api_response: Any + ) -> Optional[Dict[str, Any]]: + """ + Turn the JSON document produced by the classifier into a moderation result. + + Args: + raw_content: The classifier's answer, as a JSON string. Models + routinely wrap it in a Markdown code fence, which is stripped. + full_api_response: The provider's whole response, kept for auditing. + + Returns: + The common moderation result, or None if it could not be parsed. + """ + if not isinstance(raw_content, str) or not raw_content.strip(): + log.error("AI moderation response did not contain any content") + return None + + try: + parsed = json.loads(strip_code_fence(raw_content)) + except json.JSONDecodeError as e: + log.error(f"Failed to parse AI moderation response JSON: {e}") + return None + + if not isinstance(parsed, dict): + log.error( + f"Expected a JSON object from the AI moderation API, got {type(parsed)}" + ) + return None + + return normalize_moderation_result(parsed, full_api_response) + + +def normalize_moderation_result( + parsed: Dict[str, Any], full_api_response: Any +) -> Dict[str, Any]: + """ + Fill in the keys the rest of AI moderation relies on. + + Any additional keys the classifier returned are preserved: they end up on + the audit log, where they are worth having. + """ + result = dict(parsed) + result["classification"] = parsed.get("classification", CLASSIFICATION_NOT_SPAM) + result["reasoning"] = parsed.get("reasoning", DEFAULT_REASONING) + result["confidence_score"] = parsed.get("confidence_score") + result["full_api_response"] = full_api_response + return result + + +def strip_code_fence(text: str) -> str: + """Remove a surrounding Markdown code fence, if the model added one.""" + stripped = text.strip() + if not stripped.startswith("```"): + return stripped + + # Drop the opening fence, which may carry a language hint such as ```json. + lines = stripped.splitlines()[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + return "\n".join(lines).strip() diff --git a/src/forum/ai_moderation/defaults.py b/src/forum/ai_moderation/defaults.py new file mode 100644 index 00000000..397a793e --- /dev/null +++ b/src/forum/ai_moderation/defaults.py @@ -0,0 +1,83 @@ +""" +Provider-independent defaults for AI moderation. + +Every value here is the fallback used when the matching Django setting is not +configured. Nothing in this module may encode the behaviour of a single AI +provider: which provider answers is chosen entirely by AI_MODERATION_BACKEND, +which has no default -- forum ships an interface, not a provider. + +The one thing forum does supply is the prompt, so that standing up a backend +does not also mean writing a spam classifier prompt from scratch. +""" + +# Seconds to wait for the connection to be established, and then for the +# classifier to answer. Moderation runs inline with thread/comment creation, so +# the connect timeout is deliberately short. +DEFAULT_CONNECTION_TIMEOUT = 1.0 +DEFAULT_READ_TIMEOUT = 30 + +# Spam verdicts are cached by content hash so an identical repost does not cost +# a second API call. Clean verdicts are never cached. +DEFAULT_FLAGGED_CACHE_TTL = 60 * 60 * 24 +DEFAULT_FLAGGED_CACHE_PREFIX = "ai_moderation:flagged:v1" + +# Value used for a missing `reasoning` field in a classifier response. +DEFAULT_REASONING = "No reasoning provided" + +DEFAULT_SYSTEM_MESSAGE = """\ +Filter posts from a discussion forum platform to identify and flag content that is likely to be spam or a scam. + +**Instructions**: +- Carefully analyze each post's text for language, links, or patterns typical of spam or scams. +- Use clear reasoning to identify suspicious indicators such as: + * Promotional language or unsolicited commercial content + * Misleading claims or "too good to be true" offers + * Excessive external links (especially non-educational domains) + * Requests for personal information (phone numbers, email, social media) + * Suspicious offers (money, investment, guaranteed results) + * Impersonation of authority figures (course staff, professors) + * Directing users to external communication platforms (WhatsApp, Telegram) + * Cryptocurrency, forex, or investment scheme language + * Urgent pressure tactics ("act now", "limited time") + +- After thoroughly explaining your reasoning and highlighting specific suspicious features, + classify the post as either "spam_or_scam" or "not_spam". +- **Do not make a classification before detailing your reasoning.** Always present your + analysis of the post's content before your final determination. +- If uncertainty exists, explain which factors made detection difficult before concluding. +- Consider legitimate use cases: Course-related external links (.edu domains), genuine help + requests, study group formation. + +**Output Format** (strict JSON, and nothing else): +{ + "reasoning": "[Detailed explanation of why this post may or may not be spam/scam, + referencing specific features of the post. Minimum 2 sentences.]", + "classification": "[spam_or_scam | not_spam]" +} + +**Examples**: + +Example 1 (Spam): +Post: "Hi everyone! I'm Professor Johnson. Contact me on WhatsApp +1-555-0123 for +guaranteed A+ grades. Limited slots!" +Output: +{ + "reasoning": "This post exhibits multiple red flags: (1) Impersonation of a professor + with no verification, (2) request to contact via WhatsApp with phone + number, (3) unrealistic promise of 'guaranteed A+ grades', (4) urgency + tactic 'limited slots'. These are classic patterns of academic scams + targeting students.", + "classification": "spam_or_scam" +} + +Example 2 (Not Spam): +Post: "Can someone explain the difference between merge sort and quick sort? I'm +struggling with the time complexity analysis." +Output: +{ + "reasoning": "This is a legitimate academic question about sorting algorithms. The post + contains no suspicious links, no requests for external contact, no + promotional language, and is directly related to course content. The tone + is appropriate for a learner seeking help.", + "classification": "not_spam" +}""" diff --git a/src/forum/ai_moderation/service.py b/src/forum/ai_moderation/service.py new file mode 100644 index 00000000..a1e7306e --- /dev/null +++ b/src/forum/ai_moderation/service.py @@ -0,0 +1,428 @@ +""" +AI Moderation utilities for forum content. +""" + +import hashlib +import logging +from typing import Any, Dict, Optional + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.cache import cache +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist +from django.utils import timezone +from django.utils.module_loading import import_string +from opaque_keys.edx.keys import CourseKey +from rest_framework.serializers import ValidationError + +from forum.ai_moderation.backends.base import ( + SPAM_CLASSIFICATIONS, + BaseModerationBackend, + CLASSIFICATION_NOT_SPAM, +) +from forum.ai_moderation.defaults import ( + DEFAULT_FLAGGED_CACHE_PREFIX, + DEFAULT_FLAGGED_CACHE_TTL, + DEFAULT_REASONING, +) +from forum.backends.mysql.models import ModerationAuditLog +from forum.utils import ForumV2RequestError + +User = get_user_model() +log = logging.getLogger(__name__) + + +def _get_author_from_content(content_instance: Any) -> Any: + """ + Get author from content instance. + + Args: + content_instance: Dict containing all content related data + Returns: + Author object or user ID + """ + author_id = content_instance.get("author_id") + if author_id: + try: + return User.objects.get(pk=author_id) + except (User.DoesNotExist, ValueError, TypeError): + # If we can't get the User object, return the ID as fallback + return author_id + return None + + +def create_moderation_audit_log( + content_instance: Any, + moderation_result: Dict[str, Any], + actions_taken: list[str], + original_author: Any, +) -> None: + """ + Create an audit log entry for AI moderation decisions. + + Only creates audit logs for spam content to reduce database load. + + Args: + content_instance: The content object (Thread or Comment, dict or model) + moderation_result: Full result from AI moderation + actions_taken: List of actions taken (e.g., ['flagged'], ['flagged', 'soft_deleted']) + original_author: User who created the content + """ + if original_author is None: + original_author = _get_author_from_content(content_instance) + + content_id = str(content_instance.get("_id")) + content_body = content_instance.get("body", "") + + enhanced_moderation_result = moderation_result.copy() + enhanced_moderation_result.update( + { + "content_id": content_id, + "metadata": { + "_id": content_id, + "title": content_instance.get("title", ""), + "body": ( + content_instance.get("body", "")[:200] + "..." + if len(content_instance.get("body", "")) > 200 + else content_instance.get("body", "") + ), + "course_id": content_instance.get("course_id", ""), + "created_at": str(content_instance.get("created_at", "")), + }, + } + ) + + try: + audit_log = ModerationAuditLog( + timestamp=timezone.now(), + body=content_body, # Store full body content + classifier_output=enhanced_moderation_result, + reasoning=moderation_result.get("reasoning", DEFAULT_REASONING), + classification=moderation_result.get("classification", "spam"), + actions_taken=actions_taken, + confidence_score=moderation_result.get("confidence_score"), + original_author=original_author, + ) + audit_log.save() + except (ValueError, TypeError, AttributeError) as db_error: + log.error(f"Failed to create database audit log: {db_error}") + + +class AIModerationService: + """ + Service for AI-based content moderation. + + Waffle Flag "discussion.enable_ai_moderation" controls whether AI moderation is active. + + Content is classified by the moderation backend named in the + AI_MODERATION_BACKEND setting. There is no default: forum defines the + interface and leaves the choice of provider to the deployment. This service + is provider agnostic -- everything it does with a verdict, from caching to + flagging to soft deletion to audit logging, is the same whichever backend + produced it. + """ + + def __init__(self) -> None: + """Initialize the AI moderation service.""" + self._moderation_backend: Optional[BaseModerationBackend] = None + self._moderation_backend_path: Optional[str] = None + + @property + def ai_moderation_user_id(self) -> Optional[Any]: + """User the moderation actions are attributed to.""" + return getattr(settings, "AI_MODERATION_USER_ID", None) + + @property + def flagged_cache_ttl(self) -> int: + """How long a spam verdict stays cached, in seconds.""" + return getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_TTL", DEFAULT_FLAGGED_CACHE_TTL + ) + + @property + def flagged_cache_prefix(self) -> str: + """Key prefix for cached spam verdicts.""" + return getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_PREFIX", DEFAULT_FLAGGED_CACHE_PREFIX + ) + + @property + def moderation_backend_path(self) -> Optional[str]: + """Dotted path of the configured moderation backend, if one is configured.""" + return getattr(settings, "AI_MODERATION_BACKEND", None) + + @property + def moderation_backend(self) -> BaseModerationBackend: + """ + The configured moderation backend. + + Loaded on first use rather than in __init__ so that the module level + service instance does not freeze the setting at import time, and cached + until the configured path changes. + + Raises: + ImproperlyConfigured: if AI_MODERATION_BACKEND is unset, or does not + name a usable BaseModerationBackend. + """ + backend_path = self.moderation_backend_path + if not backend_path: + raise ImproperlyConfigured( + "AI_MODERATION_BACKEND is not configured. Forum provides the " + "moderation interface but no provider: set this to the dotted path " + "of a BaseModerationBackend subclass." + ) + if ( + self._moderation_backend is None + or self._moderation_backend_path != backend_path + ): + self._moderation_backend = self._load_moderation_backend(backend_path) + self._moderation_backend_path = backend_path + return self._moderation_backend + + @staticmethod + def _load_moderation_backend(backend_path: str) -> BaseModerationBackend: + """Import and instantiate the moderation backend at ``backend_path``.""" + try: + backend_class = import_string(backend_path) + except ImportError as e: + raise ImproperlyConfigured( + f"AI_MODERATION_BACKEND '{backend_path}' could not be imported: {e}" + ) from e + + backend = backend_class() + if not isinstance(backend, BaseModerationBackend): + raise ImproperlyConfigured( + f"AI_MODERATION_BACKEND '{backend_path}' is not a subclass of " + f"{BaseModerationBackend.__module__}.{BaseModerationBackend.__name__}" + ) + return backend + + def _classify(self, content: str) -> Optional[Dict[str, Any]]: + """ + Ask the configured backend to classify content. + + Returns the moderation result, or None if the backend is unusable or + failed. Moderation runs inline with posting, so no backend problem is + allowed to propagate out of here. + """ + try: + return self.moderation_backend.classify(content) + except ImproperlyConfigured as e: + log.error(f"AI moderation backend is not usable: {e}") + return None + except Exception: # pylint: disable=broad-except + log.exception( + f"AI moderation backend '{self.moderation_backend_path}' " + f"raised an unexpected error" + ) + return None + + def _cache_key_for_content(self, content: str) -> str: + """Return the cache key for a given message content.""" + normalized = (content or "").strip() + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + return f"{self.flagged_cache_prefix}:{digest}" + + def _get_cached_flagged_result(self, content: str) -> Optional[Dict[str, Any]]: + """Return cached moderation result for flagged content, if present.""" + try: + cached = cache.get(self._cache_key_for_content(content)) + except Exception: # pylint: disable=broad-except, no-else-return + log.exception("AI moderation cache read failed") + return None + return cached if isinstance(cached, dict) else None + + def _set_cached_flagged_result( + self, content: str, moderation_result: Dict[str, Any] + ) -> None: + """Store moderation result for flagged content in cache.""" + try: + cache.set( + self._cache_key_for_content(content), + moderation_result, + timeout=self.flagged_cache_ttl, + ) + except Exception: # pylint: disable=broad-except + log.exception("AI moderation cache write failed") + + def moderate_and_flag_content( + self, + content: str, + content_instance: Any, + course_id: Optional[str] = None, + backend: Optional[Any] = None, + ) -> Dict[str, Any]: + """ + Moderate content and flag as spam and flag abuse if detected. + + Args: + content: The text content to check + content_instance: The content model instance (Thread or Comment) + course_id: Optional course ID for waffle flag checking + backend: Forum storage backend used for the database operations. + This is not the AI moderation backend, which is chosen by the + AI_MODERATION_BACKEND setting. + + Returns: + Dictionary with moderation results and actions taken + """ + result = { + "is_spam": False, + "reasoning": "AI moderation disabled or unavailable", + "classification": CLASSIFICATION_NOT_SPAM, + "actions_taken": ["no_action"], + "flagged": False, + } + # Check if AI moderation is enabled + # pylint: disable=import-outside-toplevel + from forum.toggles import ( + is_ai_moderation_enabled, + is_ai_auto_delete_spam_enabled, + ) + + course_key = CourseKey.from_string(course_id) if course_id else None + if not is_ai_moderation_enabled(course_key): # type: ignore[no-untyped-call] + return result + + # If we've already flagged this exact content before, reuse the cached result + moderation_result = self._get_cached_flagged_result(content) + if moderation_result is None: + moderation_result = self._classify(content) + + if moderation_result is None: + result["reasoning"] = "AI moderation API failed" + log.warning("AI moderation API failed") + return result + + classification = moderation_result.get( + "classification", CLASSIFICATION_NOT_SPAM + ) + reasoning = moderation_result.get("reasoning", DEFAULT_REASONING) + is_spam = classification in SPAM_CLASSIFICATIONS + + # Cache only flagged (spam) results to avoid repeated classifier calls + if is_spam: + self._set_cached_flagged_result(content, moderation_result) + + result.update( + { + "is_spam": is_spam, + "reasoning": reasoning, + "classification": classification, + "moderation_result": moderation_result, + } + ) + + if is_spam: + # Flag content as spam and abuse first + try: + content_instance["is_spam"] = True + + self._mark_as_spam_and_moderate(content_instance, backend) + result["actions_taken"] = ["flagged"] + result["flagged"] = True + except ImproperlyConfigured as e: + log.error(f"Cannot act on AI moderation verdict: {e}") + result["actions_taken"] = ["no_action"] + except (AttributeError, ValueError, TypeError) as e: + log.error(f"Failed to flag content as spam: {e}") + result["actions_taken"] = ["no_action"] + + # Only attempt deletion if flagging succeeded + if is_ai_auto_delete_spam_enabled(course_key) and result["flagged"]: # type: ignore[no-untyped-call] + try: + self._delete_content(content_instance) + result["actions_taken"] = result["actions_taken"] + ["soft_deleted"] # type: ignore[operator] + except (ForumV2RequestError, ObjectDoesNotExist, ValidationError) as e: + log.error(f"Failed to delete content after flagging: {e}") + else: + result["actions_taken"] = ["no_action"] + + # Only create audit log for spam content (or API failures, handled above) + if is_spam: + create_moderation_audit_log( + content_instance, + moderation_result, + result["actions_taken"], # type: ignore[arg-type] + _get_author_from_content(content_instance), + ) + return result + + def _mark_as_spam_and_moderate(self, content_instance: Any, backend: Any) -> None: + """Flag content as abuse using backend methods.""" + content_id = str(content_instance.get("_id")) + content_type = str(content_instance.get("_type")) + extra_data = { + "entity_type": ( + "CommentThread" if content_type == "CommentThread" else "Comment" + ) + } + if not self.ai_moderation_user_id: + raise ImproperlyConfigured( + "AI_MODERATION_USER_ID setting is not configured, so there is no user " + "to attribute AI moderation actions to." + ) + backend.flag_content_as_spam(content_type, content_id) + backend.flag_as_abuse(str(self.ai_moderation_user_id), content_id, **extra_data) + + def _delete_content(self, content_instance: Any) -> None: + """ + Delete content using API layer delete functions. + + Uses the API layer which handles all business logic including: + - Content validation + - Deletion + - Stats updates + - Subscription cleanup (for threads) + - Anonymous content handling + + Args: + content_instance: Dict containing content data including _id, _type, and course_id + """ + # Import here to avoid circular dependency (api modules import from ai_moderation) + # pylint: disable=import-outside-toplevel,cyclic-import + from forum.api.comments import delete_comment + from forum.api.threads import delete_thread + + content_id = str(content_instance.get("_id")) + content_type = str(content_instance.get("_type")) + course_id = content_instance.get("course_id") + + # Use API layer functions which handle all business logic + # Exceptions propagate to caller for proper error handling + if content_type == "CommentThread": + delete_thread(content_id, course_id=course_id) + log.info(f"AI Moderation Deleted CommentThread: {content_id}") + elif content_type == "Comment": + delete_comment(content_id, course_id=course_id) + log.info(f"AI Moderation Deleted Comment: {content_id}") + + +# Global instance +ai_moderation_service = AIModerationService() + + +def moderate_and_flag_spam( + content: str, + content_instance: Any, + course_id: Optional[str] = None, + backend: Optional[Any] = None, +) -> Dict[str, Any]: + """ + Moderate content and flag as spam if detected. + + Args: + content: The text content to moderate + content_instance: The content model instance + course_id: Optional course ID for waffle flag checking + backend: Backend instance for database operations + + Returns: + Dictionary with moderation results and actions taken + + TODO:- + - Add content check for images + """ + return ai_moderation_service.moderate_and_flag_content( + content, content_instance, course_id, backend + ) diff --git a/src/forum/api/comments.py b/src/forum/api/comments.py index c95a1d4b..af8ad77a 100644 --- a/src/forum/api/comments.py +++ b/src/forum/api/comments.py @@ -9,6 +9,7 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework.serializers import ValidationError +from forum.ai_moderation.service import moderate_and_flag_spam from forum.backend import get_backend from forum.backends.mysql.api import MySQLBackend from forum.serializers.comment import CommentSerializer @@ -130,6 +131,15 @@ def create_child_comment( log.error("Forumv2RequestError for create child comment request.") raise ForumV2RequestError("comment is not created") + # AI Moderation: Check for spam after successful creation + try: + moderate_and_flag_spam(body, comment, course_id, backend) + # Get the updated comment after AI moderation; fall back to the + # pre-deletion snapshot if spam auto-delete removed it. + comment = backend.get_comment(comment_id) or comment + except Exception as e: # pylint: disable=broad-except + log.error(f"AI moderation failed for child comment {comment_id}: {e}") + user = backend.get_user(user_id) thread = backend.get_thread(parent_comment["comment_thread_id"]) if user and thread and comment: @@ -292,6 +302,16 @@ def create_parent_comment( log.error("Forumv2RequestError for create parent comment request.") raise ForumV2RequestError("comment is not created") comment = backend.get_comment(comment_id) or {} + + # AI Moderation: Check for spam after successful creation + try: + moderate_and_flag_spam(body, comment, course_id, backend) + # Get the updated comment after AI moderation; fall back to the + # pre-deletion snapshot if spam auto-delete removed it. + comment = backend.get_comment(comment_id) or comment + except Exception as e: # pylint: disable=broad-except + log.error(f"AI moderation failed for parent comment {comment_id}: {e}") + user = backend.get_user(user_id) if user and comment: backend.mark_as_read(user_id, thread_id) diff --git a/src/forum/api/threads.py b/src/forum/api/threads.py index fc089758..72ac07ef 100644 --- a/src/forum/api/threads.py +++ b/src/forum/api/threads.py @@ -8,6 +8,7 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework.serializers import ValidationError +from forum.ai_moderation.service import moderate_and_flag_spam from forum.api.users import mark_thread_as_read from forum.backend import get_backend from forum.backends.mysql.api import MySQLBackend @@ -329,6 +330,16 @@ def create_thread( if not thread: raise ForumV2RequestError(f"Failed to create thread with data: {data}") + # AI Moderation: Check for spam after successful creation + try: + combined_content = f"{title}\n\n{body}" + moderate_and_flag_spam(combined_content, thread, course_id, backend) + # Get the updated thread after AI moderation; fall back to the + # pre-deletion snapshot if spam auto-delete removed it. + thread = backend.get_thread(thread_id) or thread + except Exception as e: # pylint: disable=broad-except + log.error(f"AI moderation failed for thread {thread_id}: {e}") + if not (anonymous or anonymous_to_peers): backend.update_stats_for_course( thread["author_id"], thread["course_id"], threads=1 diff --git a/src/forum/backends/backend.py b/src/forum/backends/backend.py index 2f7a4081..88e3c3f8 100644 --- a/src/forum/backends/backend.py +++ b/src/forum/backends/backend.py @@ -486,3 +486,13 @@ def get_user_post_counts(user_id: str, course_id: str) -> dict[str, int]: def delete_user_posts(user_id: str, course_id: str) -> dict[str, int]: """Delete all threads and comments by user in course. Returns counts before deletion.""" raise NotImplementedError + + @classmethod + def flag_content_as_spam(cls, content_type: str, content_id: str) -> int: + """Flag content as spam. Returns the number of records modified.""" + raise NotImplementedError + + @classmethod + def unflag_content_as_spam(cls, content_type: str, content_id: str) -> int: + """Remove the spam flag from content. Returns the number of records modified.""" + raise NotImplementedError diff --git a/src/forum/backends/mysql/api.py b/src/forum/backends/mysql/api.py index bd4748f1..8b9225dc 100644 --- a/src/forum/backends/mysql/api.py +++ b/src/forum/backends/mysql/api.py @@ -1765,6 +1765,9 @@ def update_comment(comment_id: str, **kwargs: Any) -> int: vote=-1, ) + if "is_spam" in kwargs: + comment.is_spam = kwargs["is_spam"] + comment.updated_at = timezone.now() comment.save() return 1 @@ -1981,6 +1984,9 @@ def update_thread( vote=-1, ) + if "is_spam" in kwargs: + thread.is_spam = kwargs["is_spam"] + thread.updated_at = timezone.now() thread.save() return 1 @@ -2332,3 +2338,40 @@ def delete_user_posts(user_id: str, course_id: str) -> dict[str, int]: Comment.objects.filter(author_id=user_id, course_id=course_id).delete() CommentThread.objects.filter(author_id=user_id, course_id=course_id).delete() return {"thread_count": thread_count, "comment_count": comment_count} + + # AI Moderation Methods for MySQL + @classmethod + def flag_content_as_spam(cls, content_type: str, content_id: str) -> int: + """ + Flag content as spam by adding AI system to abuse flaggers and updating spam fields. + + Args: + content_type: Type of content ('CommentThread' or 'Comment') + content_id: ID of the content to flag + + Returns: + Number of documents modified + """ + # Use existing update methods to add AI system to abuse flaggers and set spam flag + update_data = {"is_spam": True} + if content_type == "CommentThread": + return cls.update_thread(content_id, **update_data) + return cls.update_comment(content_id, **update_data) + + @classmethod + def unflag_content_as_spam(cls, content_type: str, content_id: str) -> int: + """ + Remove spam flag from content. + + Args: + content_type: Type of content ('CommentThread' or 'Comment') + content_id: ID of the content to unflag + + Returns: + Number of documents modified + """ + # Just update the spam flag to False + update_data = {"is_spam": False} + if content_type == "CommentThread": + return cls.update_thread(content_id, **update_data) + return cls.update_comment(content_id, **update_data) diff --git a/src/forum/backends/mysql/models.py b/src/forum/backends/mysql/models.py index 211d1ff5..55d2d649 100644 --- a/src/forum/backends/mysql/models.py +++ b/src/forum/backends/mysql/models.py @@ -125,6 +125,10 @@ class Content(models.Model): updated_at: models.DateTimeField[datetime, datetime] = models.DateTimeField( auto_now=True ) + is_spam: models.BooleanField[bool, bool] = models.BooleanField( + default=False, + help_text="Whether this content has been identified as spam by AI moderation", + ) uservote = GenericRelation( "UserVote", object_id_field="content_object_id", @@ -318,6 +322,7 @@ def to_dict(self) -> dict[str, Any]: "last_activity_at": self.last_activity_at, "edit_history": edit_history, "group_id": self.group_id, + "is_spam": self.is_spam, } def doc_to_hash(self) -> dict[str, Any]: @@ -353,6 +358,9 @@ class Meta: models.Index( fields=["author", "course_id", "anonymous", "anonymous_to_peers"] ), + models.Index(fields=["is_spam"]), + models.Index(fields=["course_id", "is_spam"]), + models.Index(fields=["author", "course_id", "is_spam"]), ] @@ -500,6 +508,7 @@ def to_dict(self) -> dict[str, Any]: "updated_at": self.updated_at, "created_at": self.created_at, "endorsement": endorsement if self.endorsement else None, + "is_spam": self.is_spam, } if edit_history: data["edit_history"] = edit_history @@ -538,6 +547,9 @@ class Meta: models.Index( fields=["author", "course_id", "anonymous", "anonymous_to_peers"] ), + models.Index(fields=["is_spam"]), + models.Index(fields=["course_id", "is_spam"]), + models.Index(fields=["author", "course_id", "is_spam"]), ] @@ -774,3 +786,96 @@ class MongoContent(models.Model): class Meta: app_label = "forum" + + +class ModerationAuditLog(models.Model): + """Audit log for AI moderation decisions on spam content.""" + + # Available actions that can be taken on spam content + ACTION_CHOICES = [ + ("flagged", "Content Flagged"), + ("soft_deleted", "Content Soft Deleted"), + ("no_action", "No Action Taken"), + ] + + # Only spam classifications since we don't store non-spam entries + CLASSIFICATION_CHOICES = [ + ("spam", "Spam"), + ("spam_or_scam", "Spam or Scam"), + ] + + timestamp: models.DateTimeField[datetime, datetime] = models.DateTimeField( + default=timezone.now, help_text="When the moderation decision was made" + ) + body: models.TextField[str, str] = models.TextField( + help_text="The content body that was moderated" + ) + classifier_output: models.JSONField[dict[str, Any], dict[str, Any]] = ( + models.JSONField(help_text="Full output from the AI classifier") + ) + reasoning: models.TextField[str, str] = models.TextField( + help_text="AI reasoning for the decision" + ) + classification: models.CharField[str, str] = models.CharField( + max_length=20, + choices=CLASSIFICATION_CHOICES, + help_text="AI classification result", + ) + actions_taken: models.JSONField[list[str], list[str]] = models.JSONField( + default=list, + help_text="List of actions taken based on moderation (e.g., ['flagged', 'soft_deleted'])", + ) + confidence_score: models.FloatField[Optional[float], float] = models.FloatField( + null=True, blank=True, help_text="AI confidence score if available" + ) + moderator_override: models.BooleanField[bool, bool] = models.BooleanField( + default=False, help_text="Whether a human moderator overrode the AI decision" + ) + override_reason: models.TextField[Optional[str], str] = models.TextField( + blank=True, null=True, help_text="Reason for moderator override" + ) + moderator: models.ForeignKey[User, User] = models.ForeignKey( + User, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="moderation_actions", + help_text="Human moderator who made override", + ) + original_author: models.ForeignKey[User, User] = models.ForeignKey( + User, + on_delete=models.CASCADE, + related_name="moderated_content", + help_text="Original author of the moderated content", + ) + + def to_dict(self) -> dict[str, Any]: + """Return a dictionary representation of the model.""" + return { + "_id": str(self.pk), + "timestamp": self.timestamp.isoformat(), + "body": self.body, + "classifier_output": self.classifier_output, + "reasoning": self.reasoning, + "classification": self.classification, + "actions_taken": self.actions_taken, + "confidence_score": self.confidence_score, + "moderator_override": self.moderator_override, + "override_reason": self.override_reason, + "moderator_id": str(self.moderator.pk) if self.moderator else None, + "moderator_username": self.moderator.username if self.moderator else None, + "original_author_id": str(self.original_author.pk), + "original_author_username": self.original_author.username, + } + + class Meta: + app_label = "forum" + verbose_name = "Moderation Audit Log" + verbose_name_plural = "Moderation Audit Logs" + ordering = ["-timestamp"] + indexes = [ + models.Index(fields=["timestamp"]), + models.Index(fields=["classification"]), + models.Index(fields=["original_author"]), + models.Index(fields=["moderator"]), + ] diff --git a/src/forum/migration_helpers.py b/src/forum/migration_helpers.py index d51d4584..8a574105 100644 --- a/src/forum/migration_helpers.py +++ b/src/forum/migration_helpers.py @@ -122,6 +122,7 @@ def create_or_update_thread(thread_data: dict[str, Any]) -> None: anonymous_to_peers=thread_data.get("anonymous_to_peers", False), closed=thread_data.get("closed", False), pinned=thread_data.get("pinned", False), + is_spam=thread_data.get("is_spam", False), last_activity_at=make_aware(thread_data["last_activity_at"]), commentable_id=thread_data.get("commentable_id"), ) @@ -213,6 +214,7 @@ def create_or_update_comment(comment_data: dict[str, Any]) -> None: endorsed=comment_data.get("endorsed", False), child_count=comment_data.get("child_count", 0), depth=1 if parent else 0, + is_spam=comment_data.get("is_spam", False), ) sort_key = f"{parent.pk}-{comment.pk}" if parent else f"{comment.pk}" # Use QuerySet.update() to preserve original timestamps from MongoDB diff --git a/src/forum/migrations/0006_moderationauditlog_comment_is_spam_and_more.py b/src/forum/migrations/0006_moderationauditlog_comment_is_spam_and_more.py new file mode 100644 index 00000000..6c8edfcb --- /dev/null +++ b/src/forum/migrations/0006_moderationauditlog_comment_is_spam_and_more.py @@ -0,0 +1,191 @@ +# Generated by Django 5.2 on 2026-08-27 10:03 + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("forum", "0005_alter_commentthread_pinned"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ModerationAuditLog", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "timestamp", + models.DateTimeField( + default=django.utils.timezone.now, + help_text="When the moderation decision was made", + ), + ), + ( + "body", + models.TextField(help_text="The content body that was moderated"), + ), + ( + "classifier_output", + models.JSONField(help_text="Full output from the AI classifier"), + ), + ( + "reasoning", + models.TextField(help_text="AI reasoning for the decision"), + ), + ( + "classification", + models.CharField( + choices=[("spam", "Spam"), ("spam_or_scam", "Spam or Scam")], + help_text="AI classification result", + max_length=20, + ), + ), + ( + "actions_taken", + models.JSONField( + default=list, + help_text="List of actions taken based on moderation (e.g., ['flagged', 'soft_deleted'])", + ), + ), + ( + "confidence_score", + models.FloatField( + blank=True, + help_text="AI confidence score if available", + null=True, + ), + ), + ( + "moderator_override", + models.BooleanField( + default=False, + help_text="Whether a human moderator overrode the AI decision", + ), + ), + ( + "override_reason", + models.TextField( + blank=True, help_text="Reason for moderator override", null=True + ), + ), + ], + options={ + "verbose_name": "Moderation Audit Log", + "verbose_name_plural": "Moderation Audit Logs", + "ordering": ["-timestamp"], + }, + ), + migrations.AddField( + model_name="comment", + name="is_spam", + field=models.BooleanField( + default=False, + help_text="Whether this content has been identified as spam by AI moderation", + ), + ), + migrations.AddField( + model_name="commentthread", + name="is_spam", + field=models.BooleanField( + default=False, + help_text="Whether this content has been identified as spam by AI moderation", + ), + ), + migrations.AddIndex( + model_name="comment", + index=models.Index( + fields=["is_spam"], name="forum_comme_is_spam_46c762_idx" + ), + ), + migrations.AddIndex( + model_name="comment", + index=models.Index( + fields=["course_id", "is_spam"], name="forum_comme_course__4a265f_idx" + ), + ), + migrations.AddIndex( + model_name="comment", + index=models.Index( + fields=["author", "course_id", "is_spam"], + name="forum_comme_author__dde6dd_idx", + ), + ), + migrations.AddIndex( + model_name="commentthread", + index=models.Index( + fields=["is_spam"], name="forum_comme_is_spam_0e7304_idx" + ), + ), + migrations.AddIndex( + model_name="commentthread", + index=models.Index( + fields=["course_id", "is_spam"], name="forum_comme_course__2c84e0_idx" + ), + ), + migrations.AddIndex( + model_name="commentthread", + index=models.Index( + fields=["author", "course_id", "is_spam"], + name="forum_comme_author__96f3e5_idx", + ), + ), + migrations.AddField( + model_name="moderationauditlog", + name="moderator", + field=models.ForeignKey( + blank=True, + help_text="Human moderator who made override", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="moderation_actions", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="moderationauditlog", + name="original_author", + field=models.ForeignKey( + help_text="Original author of the moderated content", + on_delete=django.db.models.deletion.CASCADE, + related_name="moderated_content", + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddIndex( + model_name="moderationauditlog", + index=models.Index( + fields=["timestamp"], name="forum_moder_timesta_0d4616_idx" + ), + ), + migrations.AddIndex( + model_name="moderationauditlog", + index=models.Index( + fields=["classification"], name="forum_moder_classif_f477d2_idx" + ), + ), + migrations.AddIndex( + model_name="moderationauditlog", + index=models.Index( + fields=["original_author"], name="forum_moder_origina_c51089_idx" + ), + ), + migrations.AddIndex( + model_name="moderationauditlog", + index=models.Index( + fields=["moderator"], name="forum_moder_moderat_c62a1c_idx" + ), + ), + ] diff --git a/src/forum/serializers/contents.py b/src/forum/serializers/contents.py index a8c5c319..2d8e7abf 100644 --- a/src/forum/serializers/contents.py +++ b/src/forum/serializers/contents.py @@ -55,6 +55,7 @@ class ContentSerializer(serializers.Serializer[dict[str, Any]]): edit_history (list): A list of previous versions of the content. closed (bool): Whether the content is closed for further interactions. type (str): The type of content (e.g., "post", "comment"). + is_spam (bool): Whether the content was flagged as spam by AI moderation. """ id = serializers.CharField(source="_id") @@ -79,6 +80,7 @@ class ContentSerializer(serializers.Serializer[dict[str, Any]]): edit_history = EditHistorySerializer(default=[], many=True) closed = serializers.BooleanField(default=False) type = serializers.CharField() + is_spam = serializers.BooleanField(default=False) def create(self, validated_data: dict[str, Any]) -> Any: """Raise NotImplementedError""" diff --git a/src/forum/settings/common.py b/src/forum/settings/common.py index 772fc9e7..67341c3f 100644 --- a/src/forum/settings/common.py +++ b/src/forum/settings/common.py @@ -4,6 +4,14 @@ from typing import Any +from forum.ai_moderation.defaults import ( + DEFAULT_CONNECTION_TIMEOUT, + DEFAULT_FLAGGED_CACHE_PREFIX, + DEFAULT_FLAGGED_CACHE_TTL, + DEFAULT_READ_TIMEOUT, + DEFAULT_SYSTEM_MESSAGE, +) + def plugin_settings(settings: Any) -> None: """ @@ -50,3 +58,33 @@ def plugin_settings(settings: Any) -> None: # Timezone-awareness is required for mysql fields settings.USE_TZ = getattr(settings, "USE_TZ", True) + + # AI moderation. These run after the deployment's own configuration has been + # read, so every one of them defers to an already configured value; they are + # here to declare the settings and their defaults, not to impose them. + # + # AI_MODERATION_BACKEND has no default on purpose: forum defines the moderation + # interface and ships no provider, so a deployment must name the backend it + # wants. Neither does AI_MODERATION_API_URL, nor AI_MODERATION_USER_ID, which + # decides who flagging and deletion are attributed to -- no user is created for + # you. Whatever else a backend needs is read by that backend and deliberately + # not declared here: the model name, the credential setting, or whatever else + # the provider you wrap happens to want. + settings.AI_MODERATION_BACKEND = getattr(settings, "AI_MODERATION_BACKEND", None) + settings.AI_MODERATION_API_URL = getattr(settings, "AI_MODERATION_API_URL", None) + settings.AI_MODERATION_USER_ID = getattr(settings, "AI_MODERATION_USER_ID", None) + settings.AI_MODERATION_SYSTEM_MESSAGE = getattr( + settings, "AI_MODERATION_SYSTEM_MESSAGE", DEFAULT_SYSTEM_MESSAGE + ) + settings.AI_MODERATION_CONNECTION_TIMEOUT = getattr( + settings, "AI_MODERATION_CONNECTION_TIMEOUT", DEFAULT_CONNECTION_TIMEOUT + ) + settings.AI_MODERATION_READ_TIMEOUT = getattr( + settings, "AI_MODERATION_READ_TIMEOUT", DEFAULT_READ_TIMEOUT + ) + settings.AI_MODERATION_FLAGGED_CACHE_TTL = getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_TTL", DEFAULT_FLAGGED_CACHE_TTL + ) + settings.AI_MODERATION_FLAGGED_CACHE_PREFIX = getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_PREFIX", DEFAULT_FLAGGED_CACHE_PREFIX + ) diff --git a/src/forum/toggles.py b/src/forum/toggles.py new file mode 100644 index 00000000..6e9433ba --- /dev/null +++ b/src/forum/toggles.py @@ -0,0 +1,42 @@ +"""Forum v2 feature toggles.""" + +# pylint: disable=E0401 +from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag # type: ignore[import-not-found] + +DISCUSSION_WAFFLE_FLAG_NAMESPACE = "discussions" + +# .. toggle_name: discussions.enable_ai_moderation +# .. toggle_implementation: CourseWaffleFlag +# .. toggle_default: False +# .. toggle_description: Waffle flag to enable AI moderation for discussions. +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2025-10-29 +# .. toggle_target_removal_date: 2026-06-29 +ENABLE_AI_MODERATION = CourseWaffleFlag( + f"{DISCUSSION_WAFFLE_FLAG_NAMESPACE}.enable_ai_moderation", __name__ +) + +# .. toggle_name: discussions.enable_ai_auto_delete_spam +# .. toggle_implementation: CourseWaffleFlag +# .. toggle_default: False +# .. toggle_description: Waffle flag to enable AI auto delete spam for discussions. +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2026-02-05 +# .. toggle_target_removal_date: 2026-06-29 +ENABLE_AI_AUTO_DELETE_SPAM = CourseWaffleFlag( + f"{DISCUSSION_WAFFLE_FLAG_NAMESPACE}.enable_ai_auto_delete_spam", __name__ +) + + +def is_ai_auto_delete_spam_enabled(course_key): # type: ignore[no-untyped-def] + """ + Check if AI auto delete spam is enabled for the given course. + """ + return ENABLE_AI_AUTO_DELETE_SPAM.is_enabled(course_key) + + +def is_ai_moderation_enabled(course_key): # type: ignore[no-untyped-def] + """ + Check if AI moderation is enabled for the given course. + """ + return ENABLE_AI_MODERATION.is_enabled(course_key) diff --git a/test_utils/moderation.py b/test_utils/moderation.py new file mode 100644 index 00000000..8912a66b --- /dev/null +++ b/test_utils/moderation.py @@ -0,0 +1,60 @@ +""" +An AI moderation backend for tests. + +Forum ships the moderation interface and no provider, so anything that +exercises moderation has to bring a backend of its own. This is one written the +way an Open edX operator would write one: a subclass of ``HTTPModerationBackend`` +that describes a single provider's request and response and is selected by +dotted path. Living outside the ``forum`` package is the point -- it proves the +extension point works from out of tree. + +Its provider is imaginary: a bearer-token JSON API answering +``{"verdict": {"text": ""}}``. +""" + +from typing import Any, Dict, Optional +from unittest.mock import Mock + +from django.conf import settings + +from forum.ai_moderation.backends import HTTPModerationBackend + +STUB_PROVIDER_BACKEND = "test_utils.moderation.StubProviderBackend" + + +class StubProviderBackend(HTTPModerationBackend): + """Classify content with the imaginary provider described above.""" + + @property + def api_key(self) -> Optional[str]: + """Credential for the provider.""" + return getattr(settings, "AI_MODERATION_API_KEY", None) + + def classify(self, content: str) -> Optional[Dict[str, Any]]: + """Classify content, returning the common moderation result.""" + headers = {"content-type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + + response_data = self.post( + {"prompt": self.system_message, "input": content}, headers + ) + if response_data is None: + return None + + return self.parse_moderation_payload( + response_data.get("verdict", {}).get("text"), response_data + ) + + +def stub_provider_response(payload: str) -> Mock: + """ + Build a mocked provider response, in the shape StubProviderBackend reads. + + Args: + payload: The JSON document the classifier answered with. + """ + response = Mock() + response.status_code = 200 + response.json.return_value = {"verdict": {"text": payload}} + return response diff --git a/tests/test_ai_moderation.py b/tests/test_ai_moderation.py new file mode 100644 index 00000000..c5f6f8ec --- /dev/null +++ b/tests/test_ai_moderation.py @@ -0,0 +1,882 @@ +"""Tests for AI moderation functionality.""" + +import sys +from typing import Any, Generator +from unittest.mock import Mock, MagicMock, patch + +import pytest +from django.contrib.auth import get_user_model +from django.core.cache import cache +from django.test import override_settings + +from forum.ai_moderation.service import ( + AIModerationService, + _get_author_from_content, + create_moderation_audit_log, + moderate_and_flag_spam, +) +from forum.backends.mysql.models import Comment, CommentThread, ModerationAuditLog +from forum.utils import ForumV2RequestError +from test_utils.moderation import STUB_PROVIDER_BACKEND, stub_provider_response + +User = get_user_model() + +pytestmark = pytest.mark.django_db + + +# Mock openedx module to prevent import errors +if "openedx" not in sys.modules: + # Create a mock CourseWaffleFlag class + class MockCourseWaffleFlag: + """Mock implementation of openedx CourseWaffleFlag for testing.""" + + def __init__(self, flag_name: str, module_name: str) -> None: + self.flag_name = flag_name + self.module_name = module_name + + def is_enabled(self, _course_key: Any) -> bool: + # This will be overridden by our fixture patches + return False + + mock_openedx = MagicMock() + mock_waffle_utils = MagicMock() + mock_waffle_utils.CourseWaffleFlag = MockCourseWaffleFlag + + sys.modules["openedx"] = mock_openedx + sys.modules["openedx.core"] = MagicMock() + sys.modules["openedx.core.djangoapps"] = MagicMock() + sys.modules["openedx.core.djangoapps.waffle_utils"] = mock_waffle_utils + + +SPAM_RESPONSE = ( + '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' +) +NOT_SPAM_RESPONSE = ( + '{"classification": "not_spam", "reasoning": "This is legitimate content", ' + '"confidence_score": 0.9}' +) + + +@pytest.fixture(autouse=True) +def clear_moderation_cache() -> Generator[None, None, None]: + """Keep cached spam verdicts from leaking between tests.""" + cache.clear() + yield + cache.clear() + + +@pytest.fixture +def mock_ai_moderation_settings() -> Any: + """ + Configure AI moderation against a provider backend. + + Which backend does not matter to any test in this module -- they are about + the provider-agnostic workflow -- but one has to be named, because forum + ships no default. + """ + with override_settings( + AI_MODERATION_BACKEND=STUB_PROVIDER_BACKEND, + AI_MODERATION_API_URL="http://test-api.example.com", + AI_MODERATION_API_KEY="test-api-key", + AI_MODERATION_USER_ID="999", + AI_MODERATION_FLAGGED_CACHE_TTL=60 * 60, + AI_MODERATION_FLAGGED_CACHE_PREFIX="ai_moderation:flagged:v1", + ): + yield + + +@pytest.fixture +def mock_waffle_flags() -> Any: + """Mock waffle flags for AI moderation.""" + # Now we can safely import forum.toggles since openedx is mocked + import forum.toggles # pylint: disable=import-outside-toplevel + + mock_enabled = Mock(return_value=True) + mock_auto_delete = Mock(return_value=True) + + with patch.object( + forum.toggles, "is_ai_moderation_enabled", mock_enabled + ), patch.object(forum.toggles, "is_ai_auto_delete_spam_enabled", mock_auto_delete): + yield {"enabled": mock_enabled, "auto_delete": mock_auto_delete} + + +@pytest.fixture +def ai_service( + mock_ai_moderation_settings: Any, # pylint: disable=redefined-outer-name,unused-argument +) -> AIModerationService: + """Create an AI moderation service instance.""" + return AIModerationService() + + +@pytest.fixture +def sample_thread_content() -> dict[str, Any]: + """Create sample thread content for testing.""" + return { + "_id": "thread123", + "_type": "CommentThread", + "course_id": "course-v1:edX+DemoX+Demo", + "title": "Test Thread", + "body": "This is test content", + "author_id": "1", + "author_username": "testuser", + } + + +@pytest.fixture +def sample_comment_content() -> dict[str, Any]: + """Create sample comment content for testing.""" + return { + "_id": "comment456", + "_type": "Comment", + "course_id": "course-v1:edX+DemoX+Demo", + "body": "This is a test comment", + "author_id": "1", + "author_username": "testuser", + "comment_thread_id": "thread123", + } + + +class TestAIModerationAutoDelete: # pylint: disable=redefined-outer-name,unused-argument + """Tests for AI moderation auto-delete functionality.""" + + def test_auto_delete_triggered_when_enabled( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test that auto-delete is triggered when waffle flag is enabled.""" + # Mock API response indicating spam + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response), patch.object( + ai_service, "_delete_content" + ) as mock_delete: + + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + # Verify auto-delete was called + mock_delete.assert_called_once_with(sample_thread_content) + + # Verify actions_taken includes both flagged and soft_deleted + assert "flagged" in result["actions_taken"] + assert "soft_deleted" in result["actions_taken"] + assert result["is_spam"] is True + + def test_auto_delete_not_triggered_when_disabled( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test that auto-delete is NOT triggered when waffle flag is disabled.""" + # Disable auto-delete flag + mock_waffle_flags["auto_delete"].return_value = False + + # Mock API response indicating spam + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response), patch.object( + ai_service, "_delete_content" + ) as mock_delete: + + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + # Verify auto-delete was NOT called + mock_delete.assert_not_called() + + # Verify actions_taken includes only flagged + assert "flagged" in result["actions_taken"] + assert "soft_deleted" not in result["actions_taken"] + assert result["is_spam"] is True + + def test_auto_delete_not_triggered_for_non_spam( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test that auto-delete is NOT triggered for non-spam content.""" + # Mock API response indicating NOT spam + mock_response = stub_provider_response(NOT_SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response), patch.object( + ai_service, "_delete_content" + ) as mock_delete: + + result = ai_service.moderate_and_flag_content( + "legitimate content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + # Verify auto-delete was NOT called + mock_delete.assert_not_called() + + # Verify no actions taken + assert result["actions_taken"] == ["no_action"] + assert result["is_spam"] is False + + def test_actions_taken_reflects_flagged_only_when_delete_disabled( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_comment_content: dict[str, Any], + ) -> None: + """Test that actions_taken correctly reflects flagging without deletion.""" + # Disable auto-delete + mock_waffle_flags["auto_delete"].return_value = False + + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_comment_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert result["actions_taken"] == ["flagged"] + assert result["flagged"] is True + + def test_actions_taken_reflects_both_when_delete_enabled( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_comment_content: dict[str, Any], + ) -> None: + """Test that actions_taken correctly reflects both flagging and deletion.""" + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response), patch.object( + ai_service, "_delete_content" + ): + + result = ai_service.moderate_and_flag_content( + "spam content", + sample_comment_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert "flagged" in result["actions_taken"] + assert "soft_deleted" in result["actions_taken"] + assert len(result["actions_taken"]) == 2 + + +class TestAIModerationBackendDelegation: # pylint: disable=redefined-outer-name,unused-argument + """Tests that the service delegates classification to the configured backend.""" + + def test_service_calls_backend_classify( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """The service asks the backend to classify, and acts on what it returns.""" + classify = Mock( + return_value={ + "classification": "spam_or_scam", + "reasoning": "Spam detected", + "confidence_score": 0.9, + } + ) + mock_waffle_flags["auto_delete"].return_value = False + + with patch.object(ai_service.moderation_backend, "classify", classify): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=Mock(), + ) + + classify.assert_called_once_with("spam content") + assert result["is_spam"] is True + assert result["actions_taken"] == ["flagged"] + + def test_service_has_no_provider_specific_request_code(self) -> None: + """The service no longer talks to any provider itself.""" + assert not hasattr(AIModerationService, "_make_api_request") + + def test_backend_failure_leaves_content_alone( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """A backend that cannot classify degrades moderation, it does not raise.""" + backend = Mock() + + with patch.object( + ai_service.moderation_backend, "classify", Mock(return_value=None) + ): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert result["is_spam"] is False + assert result["actions_taken"] == ["no_action"] + assert result["reasoning"] == "AI moderation API failed" + backend.flag_content_as_spam.assert_not_called() + + def test_unexpected_backend_error_is_contained( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """An exception from a third party backend must not break posting.""" + with patch.object( + ai_service.moderation_backend, + "classify", + Mock(side_effect=RuntimeError("boom")), + ): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=Mock(), + ) + + assert result["is_spam"] is False + assert result["actions_taken"] == ["no_action"] + + +class TestAIModerationUserId: # pylint: disable=redefined-outer-name,unused-argument + """Tests for attributing moderation actions to AI_MODERATION_USER_ID.""" + + def test_actions_are_attributed_to_configured_user( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Flagging is performed as the configured moderation user.""" + mock_waffle_flags["auto_delete"].return_value = False + backend = Mock() + + with patch("requests.post", return_value=stub_provider_response(SPAM_RESPONSE)): + ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + backend.flag_as_abuse.assert_called_once_with( + "999", "thread123", entity_type="CommentThread" + ) + + def test_missing_user_id_reports_a_configuration_error( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + caplog: pytest.LogCaptureFixture, + ) -> None: + """Without AI_MODERATION_USER_ID nothing is moderated, and it is logged.""" + backend = Mock() + + with override_settings(AI_MODERATION_USER_ID=None), patch( + "requests.post", return_value=stub_provider_response(SPAM_RESPONSE) + ): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert result["is_spam"] is True + assert result["flagged"] is False + assert result["actions_taken"] == ["no_action"] + backend.flag_as_abuse.assert_not_called() + assert "AI_MODERATION_USER_ID" in caplog.text + + +class TestAIModerationCaching: # pylint: disable=redefined-outer-name,unused-argument + """Tests for caching of flagged moderation results.""" + + def test_flagged_result_is_cached_and_reused( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + cache.clear() + mock_waffle_flags["auto_delete"].return_value = False + + # Mock API response indicating spam + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response) as mock_post: + # First call should hit the classifier and then cache + first = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + assert first["is_spam"] is True + assert mock_post.call_count == 1 + + # Second call with identical content should use cached result + second = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + assert second["is_spam"] is True + assert mock_post.call_count == 1 + + +class TestAIModerationErrorHandling: # pylint: disable=redefined-outer-name,unused-argument + """Tests for error handling in AI moderation auto-delete.""" + + def test_deletion_failure_after_successful_flagging( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test that flagging succeeds even if deletion fails.""" + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response), patch.object( + ai_service, + "_delete_content", + side_effect=ForumV2RequestError("Delete failed"), + ): + + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + # Flagging should still succeed + assert result["is_spam"] is True + assert "flagged" in result["actions_taken"] + # soft_deleted should not be in actions since deletion failed + assert "soft_deleted" not in result["actions_taken"] + + def test_flagging_failure_prevents_deletion( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test that if flagging fails, deletion is not attempted.""" + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + backend.flag_content_as_spam.side_effect = ValueError("Flag failed") + + with patch("requests.post", return_value=mock_response), patch.object( + ai_service, "_delete_content" + ) as mock_delete: + + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + # Delete should not be called if flagging fails + mock_delete.assert_not_called() + assert result["actions_taken"] == ["no_action"] + + +class TestDeleteContentMethod: # pylint: disable=redefined-outer-name,protected-access + """Tests for the _delete_content method.""" + + def test_delete_thread_calls_api_correctly( + self, + ai_service: AIModerationService, + sample_thread_content: dict[str, Any], + ) -> None: + """Test that deleting a thread calls the API layer correctly.""" + with patch("forum.api.threads.delete_thread") as mock_delete_thread: + ai_service._delete_content(sample_thread_content) + + mock_delete_thread.assert_called_once_with( + "thread123", + course_id="course-v1:edX+DemoX+Demo", + ) + + def test_delete_comment_calls_api_correctly( + self, + ai_service: AIModerationService, + sample_comment_content: dict[str, Any], + ) -> None: + """Test that deleting a comment calls the API layer correctly.""" + with patch("forum.api.comments.delete_comment") as mock_delete_comment: + ai_service._delete_content(sample_comment_content) + + mock_delete_comment.assert_called_once_with( + "comment456", + course_id="course-v1:edX+DemoX+Demo", + ) + + def test_unknown_content_type_deletes_nothing( + self, + ai_service: AIModerationService, + sample_thread_content: dict[str, Any], + ) -> None: + """Only threads and comments are content AI moderation knows how to delete.""" + content = {**sample_thread_content, "_type": "SomethingElse"} + + with patch("forum.api.threads.delete_thread") as mock_delete_thread, patch( + "forum.api.comments.delete_comment" + ) as mock_delete_comment: + ai_service._delete_content(content) + + mock_delete_thread.assert_not_called() + mock_delete_comment.assert_not_called() + + def test_delete_handles_api_errors( + self, + ai_service: AIModerationService, + sample_thread_content: dict[str, Any], + ) -> None: + """Test that deletion errors propagate to caller.""" + with patch("forum.api.threads.delete_thread") as mock_delete_thread: + mock_delete_thread.side_effect = ForumV2RequestError("API Error") + + # Should raise exception to caller + with pytest.raises(ForumV2RequestError): + ai_service._delete_content(sample_thread_content) + + +class TestModerateAndFlagSpamFunction: # pylint: disable=redefined-outer-name + """Tests for the module-level moderate_and_flag_spam function.""" + + def test_moderate_and_flag_spam_with_auto_delete( # pylint: disable=unused-argument + self, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test the module-level function with auto-delete enabled.""" + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + + with patch("requests.post", return_value=mock_response), patch( + "forum.api.threads.delete_thread" + ): + + result = moderate_and_flag_spam( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert result["is_spam"] is True + assert "flagged" in result["actions_taken"] + assert "soft_deleted" in result["actions_taken"] + + +class TestAuditLogging: # pylint: disable=redefined-outer-name,unused-argument + """Tests for audit logging with auto-delete.""" + + def test_audit_log_created_for_auto_deleted_content( + self, + ai_service: AIModerationService, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Test that audit log is created with correct actions for auto-deleted content.""" + mock_response = stub_provider_response(SPAM_RESPONSE) + + backend = Mock() + user = User.objects.create(username="testuser") + sample_thread_content["author_id"] = str(user.pk) + + with patch("requests.post", return_value=mock_response), patch( + "forum.api.threads.delete_thread" + ): + + ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + # Verify audit log was created + audit_logs = ModerationAuditLog.objects.filter(body="This is test content") + assert audit_logs.exists() + + audit_log = audit_logs.first() + assert audit_log is not None + assert "flagged" in audit_log.actions_taken + assert "soft_deleted" in audit_log.actions_taken + + +class TestModerationDisabled: # pylint: disable=redefined-outer-name,unused-argument + """Tests for the waffle-flag gate in front of everything else.""" + + def test_disabled_moderation_classifies_nothing( + self, + ai_service: AIModerationService, + sample_thread_content: dict[str, Any], + ) -> None: + """With the flag off the classifier is never asked, and nothing is flagged.""" + import forum.toggles # pylint: disable=import-outside-toplevel + + backend = Mock() + with patch.object( + forum.toggles, "is_ai_moderation_enabled", Mock(return_value=False) + ), patch("requests.post") as mock_post: + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + mock_post.assert_not_called() + backend.flag_content_as_spam.assert_not_called() + assert result["is_spam"] is False + assert result["flagged"] is False + assert result["actions_taken"] == ["no_action"] + assert result["reasoning"] == "AI moderation disabled or unavailable" + + +class TestModerationCacheFailures: # pylint: disable=redefined-outer-name,protected-access + """A broken cache degrades moderation; it never breaks posting.""" + + def test_cache_read_failure_falls_back_to_the_classifier( + self, ai_service: AIModerationService + ) -> None: + """An unreachable cache reads as a miss.""" + with patch( + "forum.ai_moderation.service.cache.get", side_effect=Exception("cache down") + ): + assert ai_service._get_cached_flagged_result("some content") is None + + def test_cache_write_failure_is_swallowed( + self, ai_service: AIModerationService + ) -> None: + """A verdict that cannot be cached is still a verdict.""" + with patch( + "forum.ai_moderation.service.cache.set", side_effect=Exception("cache down") + ): + ai_service._set_cached_flagged_result("some content", {"a": 1}) + + +class TestAuditLogAuthorResolution: # pylint: disable=redefined-outer-name + """Tests for working out who wrote the moderated content.""" + + def test_content_without_an_author_resolves_to_none(self) -> None: + """Content that carries no author_id has no author to attribute.""" + assert _get_author_from_content({"_id": "thread123"}) is None + + def test_unknown_author_id_falls_back_to_the_id(self) -> None: + """An author who is no longer a user is recorded by id.""" + assert _get_author_from_content({"author_id": "404404"}) == "404404" + + def test_author_is_resolved_from_the_content_when_not_passed( + self, sample_thread_content: dict[str, Any] + ) -> None: + """create_moderation_audit_log looks the author up when given None.""" + user = User.objects.create(username="unnamed-author") + sample_thread_content["author_id"] = str(user.pk) + + create_moderation_audit_log( + sample_thread_content, + {"classification": "spam", "reasoning": "Spam detected"}, + ["flagged"], + None, + ) + + audit_log = ModerationAuditLog.objects.get(body="This is test content") + assert audit_log.original_author == user + + +class TestWaffleFlagHelpers: # pylint: disable=redefined-outer-name + """The helpers are thin, but they are the only reader of the flags.""" + + def test_helpers_delegate_to_their_flags(self) -> None: + """Each helper asks its own flag about the course it was given.""" + import forum.toggles # pylint: disable=import-outside-toplevel + + course_key = "course-v1:edX+DemoX+Demo" + with patch.object( + forum.toggles.ENABLE_AI_MODERATION, "is_enabled", return_value=True + ) as moderation, patch.object( + forum.toggles.ENABLE_AI_AUTO_DELETE_SPAM, "is_enabled", return_value=False + ) as auto_delete: + assert forum.toggles.is_ai_moderation_enabled(course_key) is True # type: ignore[no-untyped-call] + assert forum.toggles.is_ai_auto_delete_spam_enabled(course_key) is False # type: ignore[no-untyped-call] + + moderation.assert_called_once_with(course_key) + auto_delete.assert_called_once_with(course_key) + + +class TestModerationOnContentCreation: # pylint: disable=redefined-outer-name,unused-argument + """ + The create APIs run moderation inline and answer with what it left behind. + + These go through the real storage backend, so they also cover the case the + service creates for its callers: content that auto-delete has already + removed by the time the API looks for it again. + """ + + COURSE_ID = "course-v1:edX+DemoX+Demo" + + @pytest.fixture(autouse=True) + def moderation_user(self) -> Any: + """AI moderation attributes its actions to AI_MODERATION_USER_ID.""" + return User.objects.create(pk=999, username="ai-moderation") + + @pytest.fixture + def author(self) -> Any: + """The learner doing the posting.""" + return User.objects.create(username="poster") + + def _create_thread(self, author: Any) -> dict[str, Any]: + """Create a thread through the API layer.""" + from forum.api.threads import create_thread # pylint: disable=import-outside-toplevel + + return create_thread( + title="Free followers", + body="Message me on WhatsApp", + course_id=self.COURSE_ID, + user_id=str(author.pk), + ) + + def test_spam_thread_is_flagged_on_creation( + self, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + author: Any, + ) -> None: + """A thread the classifier calls spam comes back flagged.""" + mock_waffle_flags["auto_delete"].return_value = False + + with patch( + "requests.post", return_value=stub_provider_response(SPAM_RESPONSE) + ): + thread = self._create_thread(author) + + assert thread["is_spam"] is True + assert CommentThread.objects.get(pk=thread["id"]).is_spam is True + + def test_clean_thread_is_left_alone( + self, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + author: Any, + ) -> None: + """A thread the classifier clears is created untouched.""" + with patch( + "requests.post", return_value=stub_provider_response(NOT_SPAM_RESPONSE) + ): + thread = self._create_thread(author) + + assert thread["is_spam"] is False + assert ModerationAuditLog.objects.count() == 0 + + def test_auto_deleted_thread_still_returns_a_response( + self, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + author: Any, + ) -> None: + """ + Auto-delete removes the thread mid-request. + + The API answers from the pre-deletion snapshot rather than failing on + the row moderation just deleted. + """ + with patch( + "requests.post", return_value=stub_provider_response(SPAM_RESPONSE) + ): + thread = self._create_thread(author) + + assert thread["is_spam"] is True + assert not CommentThread.objects.filter(pk=thread["id"]).exists() + + def test_classifier_failure_does_not_break_thread_creation( + self, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + author: Any, + ) -> None: + """Moderation blowing up must not cost the learner their post.""" + with patch( + "forum.api.threads.moderate_and_flag_spam", + side_effect=RuntimeError("moderation exploded"), + ): + thread = self._create_thread(author) + + assert thread["is_spam"] is False + assert CommentThread.objects.filter(pk=thread["id"]).exists() + + def test_auto_deleted_comments_still_return_a_response( + self, + mock_ai_moderation_settings: Any, + mock_waffle_flags: dict[str, Mock], + author: Any, + ) -> None: + """The same holds for a response and for a reply to that response.""" + # pylint: disable=import-outside-toplevel + from forum.api.comments import create_child_comment, create_parent_comment + + with patch("requests.post", return_value=stub_provider_response(NOT_SPAM_RESPONSE)): + thread = self._create_thread(author) + response = create_parent_comment( + thread["id"], "A clean response", str(author.pk), self.COURSE_ID, + False, False, + ) + + with patch("requests.post", return_value=stub_provider_response(SPAM_RESPONSE)): + spam_response = create_parent_comment( + thread["id"], "Buy followers", str(author.pk), self.COURSE_ID, + False, False, + ) + spam_reply = create_child_comment( + response["id"], "Buy followers too", str(author.pk), self.COURSE_ID, + False, False, + ) + + for deleted in (spam_response, spam_reply): + assert deleted["is_spam"] is True + assert not Comment.objects.filter(pk=deleted["id"]).exists() diff --git a/tests/test_ai_moderation_backends.py b/tests/test_ai_moderation_backends.py new file mode 100644 index 00000000..7d2683a3 --- /dev/null +++ b/tests/test_ai_moderation_backends.py @@ -0,0 +1,377 @@ +"""Tests for the AI moderation backend interface and backend selection.""" + +# Fixtures are passed to tests by name, which pylint reads as shadowing; one +# applied only for its side effects reads as an unused argument. +# pylint: disable=redefined-outer-name,unused-argument + +import os +import subprocess +import sys +from types import SimpleNamespace +from typing import Any, Dict, Generator, Optional +from unittest.mock import Mock, patch + +import pytest +import requests +from django.core.exceptions import ImproperlyConfigured +from django.test import override_settings + +from forum.ai_moderation.backends import BaseModerationBackend, HTTPModerationBackend +from forum.ai_moderation.backends.base import strip_code_fence +from forum.ai_moderation.defaults import ( + DEFAULT_CONNECTION_TIMEOUT, + DEFAULT_READ_TIMEOUT, + DEFAULT_SYSTEM_MESSAGE, +) +from forum.ai_moderation.service import AIModerationService +from forum.settings.common import plugin_settings +from test_utils.moderation import ( + STUB_PROVIDER_BACKEND, + StubProviderBackend, + stub_provider_response, +) + +SPAM_PAYLOAD = ( + '{"classification": "spam_or_scam", "reasoning": "Spam detected", ' + '"confidence_score": 0.9}' +) + + +class SecondProviderBackend(BaseModerationBackend): + """A second backend, so that switching between two can be tested.""" + + def classify(self, content: str) -> Optional[Dict[str, Any]]: + """Answer nothing; this backend exists only to be selected.""" + return None + + +SECOND_PROVIDER_BACKEND = f"{__name__}.SecondProviderBackend" + + +@pytest.fixture +def stub_provider_settings() -> Generator[None, None, None]: + """Configure the out-of-tree stub provider backend.""" + with override_settings( + AI_MODERATION_BACKEND=STUB_PROVIDER_BACKEND, + AI_MODERATION_API_URL="http://provider.example.com/v1/moderate", + AI_MODERATION_API_KEY="test-api-key", + AI_MODERATION_SYSTEM_MESSAGE="classify this", + AI_MODERATION_CONNECTION_TIMEOUT=0.5, + AI_MODERATION_READ_TIMEOUT=20, + ): + yield + + +class TestBaseModerationBackend: + """Tests for the backend interface itself.""" + + def test_classify_is_not_implemented(self) -> None: + """The interface carries no behaviour of its own.""" + with pytest.raises(NotImplementedError): + BaseModerationBackend().classify("some content") + + def test_http_base_implements_the_interface(self) -> None: + """A backend built on the shared HTTP base satisfies the interface.""" + assert issubclass(HTTPModerationBackend, BaseModerationBackend) + assert isinstance(StubProviderBackend(), BaseModerationBackend) + + def test_http_base_reads_only_provider_neutral_settings(self) -> None: + """ + The shared HTTP base has no opinion about credentials. + + Auth belongs to the subclass -- one provider might send a bearer token, + another an x-api-key header, another an id in the request body. + """ + assert not hasattr(HTTPModerationBackend, "api_key") + + with override_settings( + AI_MODERATION_API_URL="http://provider.example.com", + AI_MODERATION_SYSTEM_MESSAGE=None, + ): + backend = StubProviderBackend() + assert backend.api_url == "http://provider.example.com" + assert backend.system_message == DEFAULT_SYSTEM_MESSAGE + assert backend.timeout == (DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT) + + @pytest.mark.parametrize( + "raw,expected", + [ + ('{"a": 1}', '{"a": 1}'), + ('```json\n{"a": 1}\n```', '{"a": 1}'), + ('```\n{"a": 1}\n```', '{"a": 1}'), + ('```json\n{"a": 1}', '{"a": 1}'), + (' {"a": 1} ', '{"a": 1}'), + ], + ) + def test_strip_code_fence(self, raw: str, expected: str) -> None: + """Models routinely wrap their JSON answer in a Markdown fence.""" + assert strip_code_fence(raw) == expected + + +class TestPluginSettings: + """ + Forum declares the AI moderation settings itself, through its plugin settings, + so that no edx-platform change is needed to configure the feature. + """ + + def test_defaults_are_declared(self) -> None: + """The settings a deployment must fill in are declared, and left empty.""" + site = SimpleNamespace(FEATURES={}) + plugin_settings(site) + + assert site.AI_MODERATION_SYSTEM_MESSAGE == DEFAULT_SYSTEM_MESSAGE + assert site.AI_MODERATION_CONNECTION_TIMEOUT == DEFAULT_CONNECTION_TIMEOUT + assert site.AI_MODERATION_READ_TIMEOUT == DEFAULT_READ_TIMEOUT + + # Nothing is guessed on a deployment's behalf -- least of all a provider. + assert site.AI_MODERATION_BACKEND is None + assert site.AI_MODERATION_API_URL is None + assert site.AI_MODERATION_USER_ID is None + + def test_no_provider_specific_settings_are_declared(self) -> None: + """ + Provider settings belong to the backend that reads them. + + Declaring, say, a bearer token here would bake one provider's auth + scheme into the generic layer. + """ + site = SimpleNamespace(FEATURES={}) + plugin_settings(site) + + assert not hasattr(site, "AI_MODERATION_API_KEY") + assert not hasattr(site, "AI_MODERATION_MODEL") + + def test_configured_values_are_never_overridden(self) -> None: + """ + Plugin settings are applied after a deployment's own configuration is + read, so a site that already chose a provider keeps it. + """ + site = SimpleNamespace( + FEATURES={}, + AI_MODERATION_BACKEND=STUB_PROVIDER_BACKEND, + AI_MODERATION_API_URL="https://provider.example.com/v1/moderate", + AI_MODERATION_SYSTEM_MESSAGE="the deployed prompt", + AI_MODERATION_CONNECTION_TIMEOUT=0.5, + AI_MODERATION_READ_TIMEOUT=20, + AI_MODERATION_USER_ID=758316, + ) + plugin_settings(site) + + assert site.AI_MODERATION_BACKEND == STUB_PROVIDER_BACKEND + assert site.AI_MODERATION_API_URL == "https://provider.example.com/v1/moderate" + assert site.AI_MODERATION_SYSTEM_MESSAGE == "the deployed prompt" + assert site.AI_MODERATION_CONNECTION_TIMEOUT == 0.5 + assert site.AI_MODERATION_READ_TIMEOUT == 20 + assert site.AI_MODERATION_USER_ID == 758316 + + def test_settings_module_imports_before_apps_are_ready(self) -> None: + """ + Plugin settings are imported while Django settings are still being + assembled, so nothing on that path may reach a Django model. A fresh + interpreter is the only honest way to check: this process has long since + imported the service. + """ + result = subprocess.run( + [sys.executable, "-c", "import forum.settings.common"], + env={**os.environ, "DJANGO_SETTINGS_MODULE": "forum.settings.test"}, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +class TestBackendSelection: + """Tests for choosing a backend with AI_MODERATION_BACKEND.""" + + def test_no_provider_is_shipped_by_default(self) -> None: + """ + Forum defines the interface and no provider, so an unconfigured + deployment is told exactly what to set. + """ + service = AIModerationService() + with override_settings(AI_MODERATION_BACKEND=None): + with pytest.raises(ImproperlyConfigured) as excinfo: + _ = service.moderation_backend + + assert "AI_MODERATION_BACKEND" in str(excinfo.value) + assert "BaseModerationBackend" in str(excinfo.value) + + def test_out_of_tree_backend_is_loaded(self, stub_provider_settings: None) -> None: + """The configured dotted path decides which provider is used.""" + service = AIModerationService() + assert isinstance(service.moderation_backend, StubProviderBackend) + + def test_backend_is_reloaded_when_the_setting_changes( + self, stub_provider_settings: None + ) -> None: + """The cached backend does not outlive the setting that chose it.""" + service = AIModerationService() + assert isinstance(service.moderation_backend, StubProviderBackend) + + with override_settings(AI_MODERATION_BACKEND=SECOND_PROVIDER_BACKEND): + assert isinstance(service.moderation_backend, SecondProviderBackend) + + assert isinstance(service.moderation_backend, StubProviderBackend) + + def test_unimportable_backend_reports_a_clear_error(self) -> None: + """A typo in AI_MODERATION_BACKEND says exactly what is wrong.""" + service = AIModerationService() + with override_settings(AI_MODERATION_BACKEND="forum.nope.NoSuchBackend"): + with pytest.raises(ImproperlyConfigured) as excinfo: + _ = service.moderation_backend + + assert "AI_MODERATION_BACKEND" in str(excinfo.value) + assert "forum.nope.NoSuchBackend" in str(excinfo.value) + + def test_backend_must_implement_the_interface(self) -> None: + """Pointing the setting at some other class is a configuration error.""" + service = AIModerationService() + with override_settings( + AI_MODERATION_BACKEND="forum.ai_moderation.service.AIModerationService" + ): + with pytest.raises(ImproperlyConfigured) as excinfo: + _ = service.moderation_backend + + assert "BaseModerationBackend" in str(excinfo.value) + + @pytest.mark.parametrize("backend_path", [None, "forum.nope.NoSuchBackend"]) + def test_misconfiguration_does_not_break_moderation( + self, backend_path: Optional[str] + ) -> None: + """Unset or wrong, a configuration error is logged, not raised at the caller.""" + service = AIModerationService() + classify = service._classify # pylint: disable=protected-access + with override_settings(AI_MODERATION_BACKEND=backend_path): + assert classify("content") is None + + +class TestCustomProviderBackend: + """ + Tests for what an operator gets from the shared HTTP base. + + These exercise StubProviderBackend, whose only provider-specific code is the + request body, the auth header and where the verdict lives in the response. + Everything else -- timeouts, error handling, fence stripping, normalization + -- is inherited. + """ + + def test_request_format(self, stub_provider_settings: None) -> None: + """The backend decides its own body and its own auth scheme.""" + with patch( + "requests.post", return_value=stub_provider_response(SPAM_PAYLOAD) + ) as mock_post: + StubProviderBackend().classify("check me") + + args, kwargs = mock_post.call_args + assert args[0] == "http://provider.example.com/v1/moderate" + assert kwargs["headers"]["Authorization"] == "Bearer test-api-key" + assert kwargs["timeout"] == (0.5, 20) + assert kwargs["json"] == {"prompt": "classify this", "input": "check me"} + + def test_response_is_normalized(self, stub_provider_settings: None) -> None: + """A verdict from anywhere in the response reaches the common result.""" + raw = stub_provider_response(SPAM_PAYLOAD) + with patch("requests.post", return_value=raw): + result = StubProviderBackend().classify("check me") + + assert result is not None + assert result["classification"] == "spam_or_scam" + assert result["reasoning"] == "Spam detected" + assert result["confidence_score"] == 0.9 + assert result["full_api_response"] == raw.json.return_value + + def test_fenced_response_is_parsed(self, stub_provider_settings: None) -> None: + """A JSON answer wrapped in a Markdown fence is still understood.""" + fenced = f"```json\n{SPAM_PAYLOAD}\n```" + with patch("requests.post", return_value=stub_provider_response(fenced)): + result = StubProviderBackend().classify("check me") + + assert result is not None + assert result["classification"] == "spam_or_scam" + + def test_extra_keys_are_preserved(self, stub_provider_settings: None) -> None: + """Anything else the classifier returned is kept for the audit log.""" + payload = '{"classification": "not_spam", "categories": ["none"]}' + with patch("requests.post", return_value=stub_provider_response(payload)): + result = StubProviderBackend().classify("check me") + + assert result is not None + assert result["categories"] == ["none"] + assert result["confidence_score"] is None + + def test_request_failure_returns_none(self, stub_provider_settings: None) -> None: + """Error handling is inherited, not reimplemented per provider.""" + with patch("requests.post", side_effect=requests.ConnectionError("refused")): + assert StubProviderBackend().classify("check me") is None + + def test_default_system_message_is_used_when_unset(self) -> None: + """Standing up a backend does not also mean writing a prompt.""" + with override_settings( + AI_MODERATION_API_URL="http://provider.example.com/v1/moderate", + AI_MODERATION_SYSTEM_MESSAGE=None, + ), patch( + "requests.post", return_value=stub_provider_response(SPAM_PAYLOAD) + ) as mock_post: + StubProviderBackend().classify("check me") + + assert mock_post.call_args.kwargs["json"]["prompt"] == DEFAULT_SYSTEM_MESSAGE + + def test_missing_api_url_is_a_configuration_error(self) -> None: + """No endpoint means no request at all.""" + with override_settings(AI_MODERATION_API_URL=None), patch( + "requests.post" + ) as mock_post: + assert StubProviderBackend().classify("check me") is None + + mock_post.assert_not_called() + + @pytest.mark.parametrize( + "body", + [ + {}, + {"verdict": {}}, + {"verdict": {"text": ""}}, + {"verdict": {"text": "not json"}}, + {"verdict": {"text": "[1, 2, 3]"}}, + ], + ) + def test_unusable_responses_return_none( + self, body: Any, stub_provider_settings: None + ) -> None: + """Anything that is not a parsable verdict fails closed.""" + response = Mock() + response.status_code = 200 + response.json.return_value = body + + with patch("requests.post", return_value=response): + assert StubProviderBackend().classify("check me") is None + + def test_non_json_response_returns_none( + self, stub_provider_settings: None + ) -> None: + """A body that is not JSON at all degrades moderation instead of raising.""" + response = Mock() + response.status_code = 200 + response.json.side_effect = ValueError("not json") + + with patch("requests.post", return_value=response): + assert StubProviderBackend().classify("check me") is None + + def test_timeout_returns_none(self, stub_provider_settings: None) -> None: + """A slow classifier degrades moderation instead of raising.""" + with patch("requests.post", side_effect=requests.Timeout("too slow")): + assert StubProviderBackend().classify("check me") is None + + def test_http_error_returns_none(self, stub_provider_settings: None) -> None: + """A non-2xx answer degrades moderation instead of raising.""" + failed = requests.Response() + failed.status_code = 500 + response = Mock() + response.raise_for_status.side_effect = requests.HTTPError( + "500", response=failed + ) + + with patch("requests.post", return_value=response): + assert StubProviderBackend().classify("check me") is None diff --git a/tests/test_backends/test_mysql/test_api.py b/tests/test_backends/test_mysql/test_api.py index d89a537f..d332d6f0 100644 --- a/tests/test_backends/test_mysql/test_api.py +++ b/tests/test_backends/test_mysql/test_api.py @@ -7,7 +7,7 @@ from django.contrib.auth import get_user_model from forum.backends.mysql.api import MySQLBackend as backend -from forum.backends.mysql.models import AbuseFlagger, CommentThread, CourseStat +from forum.backends.mysql.models import AbuseFlagger, Comment, CommentThread, CourseStat from forum.serializers.thread import ThreadSerializer User = get_user_model() @@ -226,3 +226,83 @@ def test_filter_by_commentable_ids(self) -> None: assert threads["thread_count"] == 2 for thread in threads["collection"]: assert thread["commentable_id"] == "id_2" + + +@pytest.mark.django_db +def test_flag_and_unflag_thread_as_spam() -> None: + """AI moderation marks a thread as spam through the backend, and can undo it.""" + author = User.objects.create(username="spam-thread-author") + thread = CommentThread.objects.create( + author=author, + course_id="course123", + title="Buy followers now", + body="Message me on WhatsApp", + thread_type="discussion", + context="course", + ) + + assert backend.flag_content_as_spam("CommentThread", str(thread.pk)) == 1 + thread.refresh_from_db() + assert thread.is_spam is True + + assert backend.unflag_content_as_spam("CommentThread", str(thread.pk)) == 1 + thread.refresh_from_db() + assert thread.is_spam is False + + +@pytest.mark.django_db +def test_flag_and_unflag_comment_as_spam() -> None: + """Anything that is not a thread is flagged as a comment.""" + author = User.objects.create(username="spam-comment-author") + thread = CommentThread.objects.create( + author=author, + course_id="course123", + title="Test Thread", + body="This is a test thread", + thread_type="discussion", + context="course", + ) + comment = Comment.objects.create( + author=author, + comment_thread=thread, + course_id="course123", + body="Guaranteed returns, DM me", + ) + + assert backend.flag_content_as_spam("Comment", str(comment.pk)) == 1 + comment.refresh_from_db() + assert comment.is_spam is True + + assert backend.unflag_content_as_spam("Comment", str(comment.pk)) == 1 + comment.refresh_from_db() + assert comment.is_spam is False + + +@pytest.mark.django_db +def test_is_spam_is_only_written_when_passed() -> None: + """An update that says nothing about spam leaves the flag alone.""" + author = User.objects.create(username="untouched-author") + thread = CommentThread.objects.create( + author=author, + course_id="course123", + title="Test Thread", + body="This is a test thread", + thread_type="discussion", + context="course", + is_spam=True, + ) + comment = Comment.objects.create( + author=author, + comment_thread=thread, + course_id="course123", + body="A comment", + is_spam=True, + ) + + backend.update_thread(str(thread.pk), title="Edited title") + backend.update_comment(str(comment.pk), body="Edited body") + + thread.refresh_from_db() + comment.refresh_from_db() + assert thread.is_spam is True + assert comment.is_spam is True diff --git a/tests/test_backends/test_mysql/test_models.py b/tests/test_backends/test_mysql/test_models.py index 6598c13a..ddba2c55 100644 --- a/tests/test_backends/test_mysql/test_models.py +++ b/tests/test_backends/test_mysql/test_models.py @@ -17,6 +17,7 @@ ForumUser, HistoricalAbuseFlagger, LastReadTime, + ModerationAuditLog, ReadState, Subscription, UserVote, @@ -1117,3 +1118,59 @@ def test_comment_to_dict_fallback_to_current_username() -> None: comment_dict = comment.to_dict() assert comment_dict["author_username"] == "currentuser" + + +@pytest.mark.django_db +def test_moderation_audit_log_to_dict() -> None: + """A moderation audit log serializes both the verdict and who it concerns.""" + author = User.objects.create(username="spammer") + moderator = User.objects.create(username="moderator") + audit_log = ModerationAuditLog.objects.create( + body="Buy followers now", + classifier_output={"classification": "spam_or_scam"}, + reasoning="Promotional language and an external contact request", + classification="spam_or_scam", + actions_taken=["flagged"], + confidence_score=0.9, + moderator_override=True, + override_reason="Legitimate study group invite", + moderator=moderator, + original_author=author, + ) + + audit_log_dict = audit_log.to_dict() + + assert audit_log_dict["_id"] == str(audit_log.pk) + assert audit_log_dict["timestamp"] == audit_log.timestamp.isoformat() + assert audit_log_dict["body"] == "Buy followers now" + assert audit_log_dict["classifier_output"] == {"classification": "spam_or_scam"} + assert audit_log_dict["classification"] == "spam_or_scam" + assert audit_log_dict["actions_taken"] == ["flagged"] + assert audit_log_dict["confidence_score"] == 0.9 + assert audit_log_dict["moderator_override"] is True + assert audit_log_dict["override_reason"] == "Legitimate study group invite" + assert audit_log_dict["moderator_id"] == str(moderator.pk) + assert audit_log_dict["moderator_username"] == "moderator" + assert audit_log_dict["original_author_id"] == str(author.pk) + assert audit_log_dict["original_author_username"] == "spammer" + + +@pytest.mark.django_db +def test_moderation_audit_log_to_dict_without_a_moderator() -> None: + """Most audit logs are the AI's alone, with no human override.""" + author = User.objects.create(username="another-spammer") + audit_log = ModerationAuditLog.objects.create( + body="Guaranteed returns", + classifier_output={"classification": "spam"}, + reasoning="Investment scheme language", + classification="spam", + actions_taken=["flagged", "soft_deleted"], + original_author=author, + ) + + audit_log_dict = audit_log.to_dict() + + assert audit_log_dict["moderator_id"] is None + assert audit_log_dict["moderator_username"] is None + assert audit_log_dict["confidence_score"] is None + assert audit_log_dict["moderator_override"] is False