diff --git a/handler.py b/handler.py index 7479c23..0773461 100755 --- a/handler.py +++ b/handler.py @@ -1,13 +1,22 @@ +import json from evaluation_function_utils.errors import EvaluationException -from .tools import commands, docs, validate +from .tools import commands, docs, parse, validate from .tools.parse import ParseError -from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response -from .tools.validate import ResBodyValidators, ValidationError +from typing import Any, Optional +from .tools.utils import DocsResponse, ErrorResponse, HandlerResponse, JsonType, Response +from .tools.validate import ( + LegacyReqBodyValidators, + LegacyResBodyValidators, + MuEdReqBodyValidators, + MuEdResBodyValidators, + ValidationError, +) -def handle_command(event: JsonType, command: str) -> HandlerResponse: - """Switch case for handling different command options. + +def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: + """Switch case for handling different command options using legacy schemas. Args: event (JsonType): The AWS Lambda event recieved by the handler. @@ -23,42 +32,179 @@ def handle_command(event: JsonType, command: str) -> HandlerResponse: elif command == "docs-user": return docs.user() - elif command in ("eval", "grade"): - response = commands.evaluate(event) - validator = ResBodyValidators.EVALUATION + body = parse.body(event) + response: Response + validator: LegacyResBodyValidators + + if command in ("eval", "grade"): + validate.body(body, LegacyReqBodyValidators.EVALUATION) + response = commands.evaluate(body) + validator = LegacyResBodyValidators.EVALUATION elif command == "preview": - response = commands.preview(event) - validator = ResBodyValidators.PREVIEW + validate.body(body, LegacyReqBodyValidators.PREVIEW) + response = commands.preview(body) + validator = LegacyResBodyValidators.PREVIEW elif command == "healthcheck": response = commands.healthcheck() - validator = ResBodyValidators.HEALTHCHECK + validator = LegacyResBodyValidators.HEALTHCHECK + else: response = Response( error=ErrorResponse(message=f"Unknown command '{command}'.") ) - validator = ResBodyValidators.EVALUATION + validator = LegacyResBodyValidators.EVALUATION validate.body(response, validator) return response -def handler(event: JsonType, _: JsonType = {}) -> HandlerResponse: +def wrap_muEd_response(body: Any, event: JsonType, status_code: int = 200) -> DocsResponse: + """Wrap a muEd response body in Lambda proxy format with X-Api-Version header. + + Args: + body: The response body to serialise. + event (JsonType): The incoming event (used to resolve the served version). + status_code (int): The HTTP status code. Defaults to 200. + + Returns: + DocsResponse: Proxy-format response with X-Api-Version header set. + """ + requested = (event.get("headers") or {}).get("X-Api-Version") + if requested and requested in commands.SUPPORTED_MUED_VERSIONS: + version = requested + else: + version = commands.SUPPORTED_MUED_VERSIONS[-1] + return DocsResponse( + statusCode=status_code, + headers={"X-Api-Version": version}, + body=json.dumps(body), + isBase64Encoded=False, + ) + + +def check_muEd_version(event: JsonType) -> Optional[HandlerResponse]: + """Check the X-Api-Version header against supported muEd versions. + + Args: + event (JsonType): The AWS Lambda event received by the handler. + + Returns: + Optional[HandlerResponse]: A version-not-supported error response if + the requested version is unsupported, otherwise None. + """ + version = (event.get("headers") or {}).get("X-Api-Version") + if version and version not in commands.SUPPORTED_MUED_VERSIONS: + return { + "title": "API version not supported", + "message": ( + f"The requested API version '{version}' is not supported. " + f"Supported versions are: {commands.SUPPORTED_MUED_VERSIONS}." + ), + "code": "VERSION_NOT_SUPPORTED", + "details": { + "requestedVersion": version, + "supportedVersions": commands.SUPPORTED_MUED_VERSIONS, + }, + } + return None + + +def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: + """Switch case for handling different command options using muEd schemas. + + Args: + event (JsonType): The AWS Lambda event recieved by the handler. + command (str): The name of the function to invoke. + + Returns: + HandlerResponse: The response object returned by the handler. + """ + try: + version_error = check_muEd_version(event) + if version_error: + return wrap_muEd_response(version_error, event, 406) + + if command == "eval": + body = parse.body(event) + validate.body(body, MuEdReqBodyValidators.EVALUATION) + response = commands.evaluate_muEd(body) + validate.body(response, MuEdResBodyValidators.EVALUATION) + + elif command == "healthcheck": + response = commands.healthcheck_muEd() + validate.body(response, MuEdResBodyValidators.HEALTHCHECK) + status_code = 503 if response.get("status") == "UNAVAILABLE" else 200 + return wrap_muEd_response(response, event, status_code) + + else: + error = { + "title": "Not implemented", + "message": f"Unknown command '{command}'.", + "code": "NOT_IMPLEMENTED", + } + return wrap_muEd_response(error, event, 501) + + return wrap_muEd_response(response, event) + + except (ParseError, ValidationError) as e: + error = { + "title": "Bad request", + "message": e.message, + "code": "VALIDATION_ERROR", + "details": {"error": str(e.error_thrown)} if e.error_thrown else None, + } + return wrap_muEd_response(error, event, 400) + + except EvaluationException as e: + detail = str(e) if str(e) else repr(e) + error = {"title": "Internal server error", "message": detail, "code": "INTERNAL_ERROR"} + return wrap_muEd_response(error, event, 500) + + except Exception as e: + detail = str(e) if str(e) else repr(e) + error = {"title": "Internal server error", "message": detail, "code": "INTERNAL_ERROR"} + return wrap_muEd_response(error, event, 500) + + +def handler(event: JsonType, _=None) -> HandlerResponse: """Main function invoked by AWS Lambda to handle incoming requests. Args: event (JsonType): The AWS Lambda event received by the gateway. + _ (JsonType): The AWS Lambda context object (unused). Returns: HandlerResponse: The response to return back to the requestor. """ - headers = event.get("headers", dict()) - command = headers.get("command", "eval") + if _ is None: + _ = {} + + # Normalise path: prefer rawPath (HTTP API v2) over path (REST API v1). + # API Gateway v1 includes the full resource prefix in `path` + # (e.g. /compareExpressions-staging/evaluate), so we match on suffix. + # API Gateway v2 uses `rawPath` at the top level; `path` may be absent. + raw_path = event.get("rawPath") or event.get("path", "/") + if raw_path.endswith("/evaluate/health"): + path = "/evaluate/health" + elif raw_path.endswith("/evaluate"): + path = "/evaluate" + else: + path = raw_path try: - return handle_command(event, command) + if path == "/evaluate": + return handle_muEd_command(event, "eval") + + elif path == "/evaluate/health": + return handle_muEd_command(event, "healthcheck") + + else: + headers = event.get("headers", dict()) + command = headers.get("command", "eval") + return handle_legacy_command(event, command) except (ParseError, ValidationError) as e: error = ErrorResponse(message=e.message, detail=e.error_thrown) diff --git a/requirements.txt b/requirements.txt index 5f5a050..e42e039 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jsonschema +pyyaml requests evaluation-function-utils typing_extensions diff --git a/schemas/__init__.py b/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/schemas/request.json b/schemas/legacy/request.json similarity index 100% rename from schemas/request.json rename to schemas/legacy/request.json diff --git a/schemas/request/eval.json b/schemas/legacy/request/eval.json similarity index 100% rename from schemas/request/eval.json rename to schemas/legacy/request/eval.json diff --git a/schemas/request/preview.json b/schemas/legacy/request/preview.json similarity index 100% rename from schemas/request/preview.json rename to schemas/legacy/request/preview.json diff --git a/schemas/response.json b/schemas/legacy/response.json similarity index 100% rename from schemas/response.json rename to schemas/legacy/response.json diff --git a/schemas/response/eval.json b/schemas/legacy/response/eval.json similarity index 100% rename from schemas/response/eval.json rename to schemas/legacy/response/eval.json diff --git a/schemas/response/healthcheck.json b/schemas/legacy/response/healthcheck.json similarity index 100% rename from schemas/response/healthcheck.json rename to schemas/legacy/response/healthcheck.json diff --git a/schemas/response/preview.json b/schemas/legacy/response/preview.json similarity index 100% rename from schemas/response/preview.json rename to schemas/legacy/response/preview.json diff --git a/schemas/responsev2.json b/schemas/legacy/responsev2.json similarity index 100% rename from schemas/responsev2.json rename to schemas/legacy/responsev2.json diff --git a/schemas/muEd/__init__.py b/schemas/muEd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/schemas/muEd/openapi-v0_1_0.yml b/schemas/muEd/openapi-v0_1_0.yml new file mode 100644 index 0000000..4ec5199 --- /dev/null +++ b/schemas/muEd/openapi-v0_1_0.yml @@ -0,0 +1,1986 @@ +openapi: 3.1.0 +info: + title: µEd API - Educational Microservices + version: 0.1.0 + contact: + name: µEd API Maintainers + description: | + The µEd API ("microservices for education") is a specification for interoperable educational services.

Currently defined endpoints:
- **Evaluate Task**: automatic feedback and grading for student submissions.
- **Chat**: conversational interactions around tasks, submissions, or general + learning questions. +tags: + - name: evaluate + description: Endpoints for evaluating student submissions and generating feedback. + - name: chat + description: Conversational endpoints for educational dialogue. +paths: + /evaluate: + post: + summary: Evaluate a submission and generate feedback + operationId: evaluateSubmission + description: | + Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, and configuration. + tags: + - evaluate + parameters: + - $ref: '#/components/parameters/Authorization' + - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateRequest' + examples: + simpleTextSubmission: + summary: Simple text submission without user or criteria + value: + submission: + submissionId: sub-123 + taskId: task-42 + type: TEXT + format: plain + content: + text: Explain what polymorphism is in object-oriented programming. + submittedAt: '2025-12-16T09:30:00Z' + version: 1 + configuration: null + preSubmissionFeedbackExample: + summary: Pre-submission feedback (non-final) + value: + submission: + submissionId: sub-777 + taskId: task-42 + type: TEXT + format: plain + content: + text: My short answer... + preSubmissionFeedback: + enabled: true + configuration: + llm: + model: gpt-5.2 + temperature: 0.4 + withTaskAndExtras: + summary: With task context, user, criteria and configuration + value: + task: + taskId: task-12 + title: Explain polymorphism + content: + text: Define polymorphism and give at least one example in Java. + learningObjectives: + - Explain the concept of polymorphism. + - Provide an example of subtype polymorphism in Java. + referenceSolution: + text: | + Polymorphism allows the same method call to result in different behavior depending on the object's runtime type. For example, a variable of type Shape can reference a Circle or Rectangle, and calling draw() will invoke the appropriate implementation. + context: + constraints: Answer in 3-6 sentences. + language: en + submission: + submissionId: sub-456 + taskId: task-12 + type: TEXT + format: plain + content: + text: | + Polymorphism means that an object can take many forms, for example subclasses implementing methods differently. + submittedAt: '2025-12-16T10:00:00Z' + version: 2 + user: + userId: user-789 + type: LEARNER + detailPreference: DETAILED + tonePreference: FRIENDLY + languagePreference: en + criteria: + - criterionId: crit-1 + name: Correctness + context: The explanation of polymorphism is conceptually correct. + maxPoints: 10 + - criterionId: crit-2 + name: Clarity + context: The explanation is clear, well-structured, and easy to understand. + maxPoints: 5 + preSubmissionFeedback: + enabled: false + configuration: + llm: + model: gpt-5.2 + temperature: 0.2 + maxTokens: 800 + credentials: + type: JWT + key: Some-Key + enforceRubricStrictness: true + codeSubmissionExample: + summary: Code submission (Python) + value: + submission: + submissionId: sub-code-001 + taskId: task-python-101 + type: CODE + format: python + content: + code: | + def fibonacci(n): + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) + + # Test the function + for i in range(10): + print(fibonacci(i)) + submittedAt: '2025-12-16T11:00:00Z' + version: 1 + codeMultiFileExample: + summary: Code submission with multiple files + value: + submission: + submissionId: sub-code-002 + taskId: task-java-201 + type: CODE + format: java + content: + files: + - path: src/Main.java + content: | + public class Main { + public static void main(String[] args) { + Calculator calc = new Calculator(); + System.out.println(calc.add(2, 3)); + } + } + - path: src/Calculator.java + content: | + public class Calculator { + public int add(int a, int b) { + return a + b; + } + } + entryPoint: src/Main.java + submittedAt: '2025-12-16T11:30:00Z' + version: 1 + codeSympyExample: + summary: Math submission (SymPy/Python) + value: + submission: + submissionId: sub-math-002 + taskId: task-algebra-101 + type: CODE + format: sympy + content: + expression: solve(x**2 - 4, x) + imports: + - from sympy import symbols, solve + - x = symbols('x') + submittedAt: '2025-12-16T12:30:00Z' + version: 1 + mathInlineLatexExample: + summary: Inline math submission (Inline LaTeX) + value: + submission: + submissionId: sub-math-001 + taskId: task-calculus-101 + type: MATH + format: latex + content: + expression: \int_{0}^{\infty} e^{-x^2} dx = \frac{\sqrt{\pi}}{2} + submittedAt: '2025-12-16T12:00:00Z' + version: 1 + mathMathMLExample: + summary: Math submission (MathML) + value: + submission: + submissionId: sub-math-003 + taskId: task-geometry-101 + type: MATH + format: mathml + content: + expression: | + + + a + = + + + b2 + + + c2 + + + + + submittedAt: '2025-12-16T13:00:00Z' + version: 1 + modelUmlExample: + summary: Model submission (UML class diagram - PlantUML) + value: + submission: + submissionId: sub-model-001 + taskId: task-oop-design-101 + type: MODEL + format: uml + content: + model: | + @startuml + abstract class Animal { + +name: String + +speak(): String + } + + class Dog extends Animal { + +speak(): String + } + + class Cat extends Animal { + +speak(): String + } + @enduml + notation: plantuml + diagramType: class + submittedAt: '2025-12-16T14:00:00Z' + version: 1 + modelErExample: + summary: Model submission (ER diagram - JSON structure) + value: + submission: + submissionId: sub-model-002 + taskId: task-database-101 + type: MODEL + format: er + content: + model: + entities: + - name: Student + attributes: + - name: student_id + type: INTEGER + primaryKey: true + - name: name + type: VARCHAR(100) + - name: email + type: VARCHAR(255) + - name: Course + attributes: + - name: course_id + type: INTEGER + primaryKey: true + - name: title + type: VARCHAR(200) + relationships: + - name: enrolls_in + from: Student + to: Course + cardinality: many-to-many + notation: json + submittedAt: '2025-12-16T14:30:00Z' + version: 1 + modelBpmnExample: + summary: Model submission (BPMN process) + value: + submission: + submissionId: sub-model-003 + taskId: task-process-101 + type: MODEL + format: bpmn + content: + model: | + + + + + + + + + + + notation: bpmn-xml + submittedAt: '2025-12-16T15:00:00Z' + version: 1 + textMarkdownExample: + summary: Text submission (Markdown with formatting) + value: + submission: + submissionId: sub-text-002 + taskId: task-essay-101 + type: TEXT + format: markdown + content: + markdown: | + # Introduction to Polymorphism + + Polymorphism is a fundamental concept in **object-oriented programming** that allows objects to be treated as instances of their parent class. + + ## Key Points + + 1. **Subtype polymorphism**: Different classes can be used interchangeably + 2. **Method overriding**: Subclasses provide specific implementations + submittedAt: '2025-12-16T15:30:00Z' + version: 1 + responses: + '200': + description: Successfully generated feedback. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Feedback' + examples: + exampleResponse: + summary: Example feedback response + value: + - feedbackId: fb-1 + title: Clarify your definition + message: Your explanation of polymorphism is generally correct, but it would help to distinguish between subtype polymorphism and parametric polymorphism. + suggestedAction: Add one or two concrete examples of polymorphism in Java, e.g., method overriding. + awardedPoints: 2.5 + criterion: + criterionId: crit-1 + name: Correctness + context: The solution produces correct results for the specified problem. + maxPoints: 10 + target: + artefactType: TEXT + format: plain + locator: + type: span + startIndex: 0 + endIndex: 120 + - feedbackId: fb-2 + title: Overall structure + message: The overall structure of your answer is clear and easy to follow. + '400': + $ref: '#/components/responses/400-BadRequest' + '403': + $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' + '500': + $ref: '#/components/responses/500-InternalError' + '501': + $ref: '#/components/responses/501-NotImplemented' + /evaluate/health: + get: + summary: Health and capabilities of the evaluate service + operationId: getEvaluateHealth + description: | + Returns health information and capabilities of the evaluate service. Clients can use this endpoint to discover whether the service supports optional features such as pre-submission feedback, formative feedback, and summative feedback. + tags: + - evaluate + parameters: + - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' + responses: + '200': + description: Evaluate service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateHealthResponse' + examples: + exampleHealth: + summary: Example healthy service with capabilities + value: + status: OK + message: Service healthy + version: 1.0.0 + capabilities: + supportsEvaluate: true + supportsPreSubmissionFeedback: false + supportsFormativeFeedback: true + supportsSummativeFeedback: true + supportsDataPolicy: PARTIAL + supportedArtefactProfiles: + - type: TEXT + supportedFormats: + - plain + - markdown + - type: CODE + supportedFormats: + - python + - java + - javascript + - type: MATH + supportedFormats: + - latex + - mathml + supportedLanguages: + - en + - de + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' + '501': + description: The server does not implement the health endpoint for evaluate. + $ref: '#/components/responses/501-NotImplemented' + '503': + $ref: '#/components/responses/503-ServiceUnavailable' + /chat: + post: + summary: Chat about tasks, submissions, or learning topics + operationId: chat + description: | + Conversational endpoint for educational chat use cases. A conversation can be grounded in a specific course, task or submission and may use user information to adapt tone and detail. Typical use cases include: asking follow-up questions on feedback, requesting hints, or clarifying concepts. + tags: + - chat + parameters: + - $ref: '#/components/parameters/Authorization' + - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ChatRequest' + examples: + minimalChat: + summary: Minimal chat request + value: + messages: + - role: USER + content: Can you explain polymorphism? + configuration: null + chatWithLlmConfig: + summary: Chat request with optional LLM configuration + value: + conversationId: conv-1001 + user: + userId: user-456 + type: LEARNER + detailPreference: MEDIUM + tonePreference: FRIENDLY + languagePreference: en + messages: + - role: USER + content: Give me a hint for my answer about polymorphism. + context: + task: + taskId: task-12 + title: Explain polymorphism + content: + text: Define polymorphism and give at least one example in Java. + configuration: + type: Java Assistant + llm: + model: gpt-5.2 + temperature: 0.7 + stream: false + credentials: + type: JWT + key: Some-Key + chatWithContext: + summary: Chat request with complex educational context + value: + messages: + - role: USER + content: What should I do for this part? + user: + userId: user-321 + type: LEARNER + detailPreference: DETAILED + tonePreference: NEUTRAL + languagePreference: en + taskProgress: + currentQuestionId: question-321 + timeSpentOnQuestion: 30 minutes + currentPart: + partId: part-1 + timeSpentOnPart: 10 minutes + submission: + type: TEXT + content: + text: outputs= ["Woof!", "Meow!"] + feedback: + - feedbackId: fb-101 + message: Incomplete answer, explain why these outputs occur. + context: + module: + moduleId: module-456 + title: Introduction to Object-Oriented Programming (OOP) + set: + setId: set-789 + title: Fundamentals + question: + questionId: question-321 + title: Understanding Polymorphism + content: | + Answer the questions for the following example of polymorphism in Python. + ```python class Animal: + def speak(self): + pass + + class Dog(Animal): + def speak(self): + return "Woof!" + + class Cat(Animal): + def speak(self): + return "Meow!" + + animals = [Dog(), Cat()] for animal in animals: + print(animal.speak()) + ``` + estimatedTime: 15-25 minutes + parts: + - partId: part-1 + content: | + Looking at the code example, identify which method is being overridden and explain how this demonstrates polymorphism. What output would the code produce and why? + - partId: part-2 + content: | + Write a new class called `Bird` that inherits from `Animal` and overrides the `speak()` method to return "Tweet!". Then add an instance of `Bird` to the animals list. + referenceSolution: + code: | + class Bird(Animal): + def speak(self): + return "Tweet!" + + animals = [Dog(), Cat(), Bird()] + for animal in animals: + print(animal.speak()) + configuration: + type: CS Assistant + llm: + model: gpt-5.2 + temperature: 0.5 + responses: + '200': + description: Successful chat response. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ChatResponse' + examples: + minimalChatResponse: + summary: Minimal chat response + value: + output: + role: ASSISTANT + content: | + Polymorphism is a core concept in object-oriented programming that allows objects of different classes to be treated as objects of a common superclass. It enables a single interface to represent different underlying forms (data types). For example, in Java, you can have a superclass `Animal` with a method `speak()`, and subclasses like `Dog` and `Cat` that provide their own implementations of `speak()`. When you call `speak()` on an `Animal` reference, the appropriate subclass method is invoked based on the actual object type at runtime. + metadata: null + chatWithLlmConfigResponse: + summary: Chat response with LLM configuration + value: + output: + role: ASSISTANT + content: | + Here's a hint for your answer about polymorphism: Focus on how polymorphism allows methods to do different things based on the object that it is acting upon, even when accessed through a common interface. You might want to mention method overriding and how it enables this behavior in object-oriented programming. + metadata: + responseTimeMs: 1800 + type: Java Assistant + createdAt: '2025-12-10T11:15:00Z' + llm: + model: gpt-5.2 + temperature: 0.7 + outputTokens: 78 + chatWithContextResponse: + summary: Chat response with context + value: + output: + role: ASSISTANT + content: | + In the provided code example, the `speak()` method is being overridden in the `Dog` and `Cat` subclasses of the `Animal` superclass. This demonstrates polymorphism because when we call `speak()` on each animal in the `animals` list, the method that gets executed depends on the actual object type (either `Dog` or `Cat`), not the reference type (`Animal`). The output of the code would be: + ``` Woof! Meow! ``` + This happens because each subclass provides its own implementation of the `speak()` method, and at runtime, the correct method is called based on the object's type. + metadata: + responseTimeMs: 2000 + type: CS Assistant + createdAt: '2025-12-10T11:15:00Z' + llm: + model: gpt-5.2 + temperature: 0.5 + outputTokens: 143 + '400': + $ref: '#/components/responses/400-BadRequest-2' + '403': + $ref: '#/components/responses/403-Forbidden' + '406': + $ref: '#/components/responses/406-VersionNotSupported' + '500': + $ref: '#/components/responses/500-InternalError-2' + '501': + $ref: '#/components/responses/501-NotImplemented-2' + /chat/health: + get: + summary: Health and capabilities of the chat service + operationId: getChatHealth + description: | + Returns health information and capabilities of the chat service. Clients can use this endpoint to discover whether the service supports optional features such as user preferences or streaming responses. + tags: + - chat + parameters: + - $ref: '#/components/parameters/X-Request-Id' + - $ref: '#/components/parameters/X-Api-Version' + responses: + '200': + description: Chat service is reachable and reporting capabilities. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ChatHealthResponse' + examples: + exampleHealth: + summary: Example healthy service with capabilities + value: + status: OK + statusMessage: Service healthy + version: 1.0.0 + capabilities: + supportsChat: true + supportsUserPreferences: true + supportsStreaming: true + supportsDataPolicy: NOT_SUPPORTED + supportedLanguages: + - en + - de + supportedModels: + - gpt-4o + - llama-3 + supportedVersions: + - 0.1.0 + '406': + $ref: '#/components/responses/406-VersionNotSupported' + '501': + description: The server does not implement the health endpoint for chat. + $ref: '#/components/responses/501-NotImplemented-2' + '503': + $ref: '#/components/responses/503-ServiceUnavailable-2' +components: + parameters: + Authorization: + in: header + name: Authorization + schema: + type: string + required: false + description: Optional authorization header. + X-Request-Id: + in: header + name: X-Request-Id + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + in: header + name: X-Api-Version + description: | + The µEd API version the client is targeting (e.g. "0.1.0"). If omitted, the server will use the latest version it supports. If the requested version cannot be served, the server returns 406 Version Not Supported. + required: false + schema: + type: string + example: 0.1.0 + schemas: + Task: + type: object + description: | + Task context including the content, learning objectives, optional reference solution, optional context information, and optional metadata. + required: + - title + properties: + taskId: + type: string + description: Optional unique identifier for the task. + title: + type: string + description: Short title or label for the task. + content: + type: + - object + - 'null' + description: Optional content shown to the learner (structure is task-specific). + additionalProperties: true + context: + type: + - object + - 'null' + description: Optional educational context (e.g., course material). + additionalProperties: true + learningObjectives: + type: + - array + - 'null' + description: Optional list of learning objectives addressed by this task. + items: + type: string + referenceSolution: + type: + - object + - 'null' + description: Optional reference or example solution (structure is task-specific). + additionalProperties: true + metadata: + type: + - object + - 'null' + description: Optional metadata such as difficulty, topic, tags, etc. + additionalProperties: true + ArtefactType: + type: string + description: | + High-level type of artefact. Use the 'format' field to specify the exact format (e.g., programming language for CODE, notation for MATH). + enum: + - TEXT + - CODE + - MODEL + - MATH + - OTHER + Submission: + type: object + description: | + A student's submission for a task. The structure of 'content' is intentionally generic and task-specific. + required: + - type + - content + properties: + submissionId: + type: string + description: Optional unique identifier of the submission. + taskId: + type: + - string + - 'null' + description: Optional identifier of the task this submission belongs to. + type: + $ref: '#/components/schemas/ArtefactType' + format: + type: + - string + - 'null' + description: | + Optional format specifier providing additional detail about the artefact. For TEXT: plain_text, markdown, html, rich_text, etc. For CODE: programming language (e.g., python, java, javascript, wolfram, matlab). For MATH: latex, mathml, sympy, wolfram, asciimath, etc. For MODEL: uml, er, bpmn, petri_net, state_machine, etc. Use lowercase values and snakecase. Services should document which formats they support. + content: + type: object + additionalProperties: true + description: | + Logical representation of the submission content. The expected structure depends on the artefact type: - TEXT: { text: string } or { markdown: string } - CODE: { code: string } or { files: [{ path: string, content: string }], entryPoint?: string } - MATH: { expression: string } - MODEL: { model: string | object, notation?: string } + submittedAt: + type: + - string + - 'null' + format: date-time + description: Optional timestamp when the submission was created. + version: + type: + - integer + - 'null' + format: int32 + description: Optional version number (e.g., resubmissions). + UserType: + type: string + description: Type of user interacting with the API. + enum: + - LEARNER + - TEACHER + - EDU_ADMIN + - SYS_ADMIN + - OTHER + Detail: + type: string + description: Level of detail preferred in responses or feedback. + enum: + - BRIEF + - MEDIUM + - DETAILED + Tone: + type: string + description: Preferred tone for responses or feedback. + enum: + - FORMAL + - NEUTRAL + - FRIENDLY + User: + type: object + description: User information including type and optional preferences influencing response tone, detail, and language. + required: + - type + additionalProperties: true + properties: + userId: + type: + - string + - 'null' + description: Optional unique identifier for the user. + type: + $ref: '#/components/schemas/UserType' + preference: + type: object + properties: + detail: + description: Optional preferred level of detail in responses. + $ref: '#/components/schemas/Detail' + tone: + description: Optional preferred tone for responses. + $ref: '#/components/schemas/Tone' + language: + type: + - string + - 'null' + description: Optional preferred language code following ISO 639 language codes (e.g., 'en', 'de'). + additionalProperties: true + taskProgress: + type: + - object + - 'null' + description: Optional information about the user's progress on this task/topic. + additionalProperties: true + NumericGrade: + title: Numeric Grade + type: object + required: + - min + - max + - value + properties: + min: + type: number + description: Minimum value for the numeric range + max: + type: number + description: Maximum value for the numeric range + value: + type: number + description: The actual rating value within the min-max range + LetterOnlyGrade: + title: Letter Only Grade + type: object + required: + - value + properties: + value: + type: string + enum: + - A + - B + - C + - D + - E + - F + - n/a + LetterPlusMinusGrade: + title: Letter +/- grades + type: object + required: + - value + properties: + value: + type: string + enum: + - A+ + - A + - A- + - B+ + - B + - B- + - C+ + - C + - C- + - D+ + - D + - D- + - E+ + - E + - E- + - F + OtherGrade: + title: Other + type: object + required: + - value + properties: + value: + type: string + description: Free-form string rating + Criterion: + type: object + description: A criterion used to assess one dimension of a submission. + required: + - name + properties: + criterionId: + type: string + description: Optional unique identifier of the criterion. + name: + type: string + description: Human-readable name of the criterion. + context: + type: + - string + - object + - 'null' + description: Optional additional context about how to apply this criterion. + additionalProperties: true + gradeConfig: + oneOf: + - $ref: '#/components/schemas/NumericGrade' + - $ref: '#/components/schemas/LetterOnlyGrade' + - $ref: '#/components/schemas/LetterPlusMinusGrade' + - $ref: '#/components/schemas/OtherGrade' + description: Optional configuration for grades for this criterion. + PreSubmissionFeedback: + type: object + description: Optional configuration for pre-submission feedback runs. + required: + - enabled + additionalProperties: true + properties: + enabled: + type: boolean + description: Indicates whether pre-submission feedback is requested. + LLMConfiguration: + type: object + description: | + Optional configuration for an LLM provider. All fields are optional and provider-specific values may be included via additional properties. + additionalProperties: true + properties: + model: + type: + - string + - 'null' + description: Optional model identifier (e.g., 'gpt-4o', 'llama-3'). + temperature: + type: + - number + - 'null' + description: Optional sampling temperature. + maxTokens: + type: + - integer + - 'null' + description: Optional maximum number of tokens to generate. + stream: + type: + - boolean + - 'null' + description: Optional flag indicating whether streaming responses are requested. + credentials: + type: + - object + - 'null' + description: Optional credentials object to be supplied with time-based key via a proxy. + additionalProperties: true + Region: + type: string + description: | + Geographic regions using ISO 3166-1 alpha-2 country codes (e.g., US, GB, DE) or regional groupings (e.g., EEA, EU, APAC). + AnonymizationLevel: + type: string + description: Level of required anonymization. + enum: + - NONE + - PSEUDONYMIZED + - ANONYMIZED + - AGGREGATED + DataPolicy: + type: object + description: | + Declares what downstream services are allowed to do with data associated with this request: which legal regimes apply, what uses are permitted, how long data may be retained, where it may be processed, and what constraints apply (especially for children / sensitive data). + additionalProperties: true + properties: + legal: + type: + - object + - 'null' + description: Legal framework and authority governing this data. + properties: + applicableLaws: + type: array + description: One or more applicable legal regimes. + items: + type: string + enum: + - GDPR + - UK_GDPR + - EPRIVACY + - CCPA_CPRA + - COPPA + - FERPA + - PPRA + - PIPEDA + - LGPD + - POPIA + - APPI + - PIPL + - OTHER + legalBasis: + type: + - array + - 'null' + description: Optional but recommended legal basis for processing. + items: + type: string + enum: + - CONSENT + - CONTRACT + - LEGAL_OBLIGATION + - PUBLIC_TASK + - LEGITIMATE_INTERESTS + - VITAL_INTERESTS + - OTHER + jurisdiction: + type: + - object + - 'null' + description: Geographic constraints for data subjects and processing. + properties: + dataSubjectRegions: + type: + - array + - 'null' + description: Where the data subjects are located. + items: + $ref: '#/components/schemas/Region' + allowedProcessingRegions: + type: + - array + - 'null' + description: Where processing/storage is allowed. + items: + $ref: '#/components/schemas/Region' + disallowedProcessingRegions: + type: + - array + - 'null' + description: Explicit exclusions for processing regions. + items: + $ref: '#/components/schemas/Region' + dataSubject: + type: + - object + - 'null' + description: Information about who this data is about. + properties: + population: + type: + - string + - 'null' + description: Type of population this data concerns. + enum: + - STUDENT + - STAFF + - GUARDIAN + - MIXED + - OTHER + isChildData: + type: + - boolean + - 'null' + description: Indicates whether this data concerns children. If true, then minAge required. + minAge: + type: + - integer + - 'null' + description: Minimum age of data subjects, if relevant/known. + dataCategory: + type: + - object + - 'null' + description: Classification of the type of data. + properties: + classification: + type: + - string + - 'null' + description: Primary classification of the data. + enum: + - ANONYMOUS + - PSEUDONYMOUS + - PERSONAL + - EDUCATION_RECORD + - SENSITIVE + - OTHER + additionalProperties: true + retentionPermission: + type: + - array + - 'null' + description: What retention and secondary uses are permitted. + items: + type: string + enum: + - NEVER + - SECURITY + - LOGGING + - PRODUCT-IMPROVEMENT-NO-SHARE + - PRODUCT-IMPROVEMENT-SHARE-LIMITED + - RESEARCH-CONFIDENTIAL + - PUBLIC + - OTHER + retention: + type: + - object + - 'null' + description: How long data may be retained. + properties: + retentionPeriod: + type: + - string + - 'null' + description: Concrete retention period, following ISO 8601 standard. + deleteOnRequest: + type: + - boolean + - 'null' + description: Whether data must be deleted on user request. + legalHoldAllowed: + type: + - boolean + - 'null' + description: Whether legal holds are permitted on this data. + sharing: + type: + - object + - 'null' + description: Constraints on who can receive this data. + properties: + thirdPartySharing: + type: + - string + - 'null' + description: Third-party sharing policy. + enum: + - PROHIBITED + - ALLOWED + - ALLOWED-LIMITED + subprocessorsAllowed: + type: + - boolean + - 'null' + description: Whether subprocessors are allowed. + allowedRecipients: + type: + - array + - 'null' + description: Categories of allowed recipients. + items: + type: string + enum: + - CONTROLLER-ONLY + - INTERNAL-SERVICES + - NAMED-PARTNERS + - PUBLIC + deidentification: + type: + - object + - 'null' + description: Required deidentification for specific uses. + properties: + requiredForServiceImprovement: + $ref: '#/components/schemas/AnonymizationLevel' + requiredForResearch: + $ref: '#/components/schemas/AnonymizationLevel' + ExecutionPolicy: + type: object + description: | + Declares execution constraints that the server or client consuming this API should apply when triggering calls to external providers (such as LLMs). Covers queue management and response timeouts. + additionalProperties: true + properties: + priority: + type: + - string + - 'null' + description: Request priority for queue management when capacity is constrained. + enum: + - low + - normal + - high + - null + timeout: + type: + - integer + - 'null' + description: Maximum time in milliseconds to wait for a complete response before failing. + minimum: 1 + EvaluateRequest: + type: object + description: | + Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration are optional. + required: + - submission + properties: + task: + description: | + Optional task context that can include prompt content, learning objectives, a reference solution, and additional context metadata. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/Task' + submission: + $ref: '#/components/schemas/Submission' + user: + description: Optional user information including type and preferences that can influence response tone, detail, and language. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/User' + criteria: + type: + - array + - 'null' + description: Optional criteria used for evaluate. + items: + $ref: '#/components/schemas/Criterion' + preSubmissionFeedback: + description: | + Optional settings for pre-submission feedback (non-final). When enabled, the service should avoid committing or finalizing grades. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/PreSubmissionFeedback' + configuration: + description: | + Optional key-value configuration dictionary for provider-specific or experimental parameters. Not standardized. + type: + - object + - 'null' + additionalProperties: true + properties: + llm: + description: Optional LLM configuration used for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/LLMConfiguration' + dataPolicy: + description: Optional data policy governing how this request's data may be processed and retained. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/DataPolicy' + executionPolicy: + description: Optional execution constraints for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/ExecutionPolicy' + FeedbackTarget: + type: object + description: Optional reference to a specific part of the submission. + required: + - artefactType + properties: + artefactType: + $ref: '#/components/schemas/ArtefactType' + format: + type: + - string + - 'null' + description: | + Optional format specifier matching the submission format (e.g., python, latex). + locator: + type: + - object + - 'null' + description: | + Optional locator into the submission content. Standard locator types: - TEXT: { type: "span", startIndex: number, endIndex: number } - CODE: { type: "range", file?: string, startLine: number, endLine: number, startColumn?: number, endColumn?: number } - MATH: { type: "subexpression", path: string } (e.g., path to subexpression) - MODEL: { type: "element", elementId: string, elementType?: string } + additionalProperties: true + Feedback: + type: object + description: | + A single feedback item produced for the submission. It may include suggested actions, optional points, optional criterion linkage, and optional targeting into the submission. + properties: + feedbackId: + type: string + description: Unique identifier of the feedback item. + title: + type: + - string + - 'null' + description: Optional short label for this feedback item. + message: + type: + - string + - 'null' + description: Optional feedback text shown to the learner. + suggestedAction: + type: + - string + - 'null' + description: Optional suggestion for how to act on this feedback. + awardedPoints: + type: + - number + - 'null' + format: double + description: Optional points awarded for this feedback item. + criterion: + type: + - object + - 'null' + description: Optional criterion linked to this feedback item. + allOf: + - $ref: '#/components/schemas/Criterion' + target: + type: + - object + - 'null' + description: Optional target reference inside the submission. + allOf: + - $ref: '#/components/schemas/FeedbackTarget' + ErrorResponse: + type: object + description: Standard error response returned by µEd API services. + required: + - title + properties: + title: + type: string + description: Short, human-readable error title. + message: + type: + - string + - 'null' + description: Optional human-readable error message. + code: + type: + - string + - 'null' + description: Optional application-specific error code. + trace: + type: + - string + - 'null' + description: Optional debug trace or stack trace (should be omitted in production by default). + details: + type: + - object + - 'null' + description: Optional provider-specific details for debugging or programmatic handling. + additionalProperties: true + HealthStatus: + type: string + description: Overall health status of the service. + enum: + - OK + - DEGRADED + - UNAVAILABLE + EvaluateRequirements: + type: object + description: | + Requirements for calling the evaluate endpoint, e.g. whether an Authorization header and/or LLM configuration or credentials (provided via a proxy, preferably time-based and/or signed tokens) are required. + additionalProperties: true + properties: + requiresAuthorizationHeader: + type: + - boolean + - 'null' + description: Optional flag indicating whether an Authorization header is required. + requiresLlmConfiguration: + type: + - boolean + - 'null' + description: Optional flag indicating whether configuration.llm must be provided. + requiresLlmCredentialProxy: + type: + - boolean + - 'null' + description: Optional flag indicating whether configuration.llm.credentials must be provided via a proxy. + DataPolicySupport: + type: string + enum: + - SUPPORTED + - NOT_SUPPORTED + - PARTIAL + description: Indicates whether the service supports data policy configuration. + ArtefactProfile: + type: object + description: | + Describes support for a specific artefact type and its formats. Used in health/capabilities responses to advertise what a service can handle. + required: + - type + properties: + type: + $ref: '#/components/schemas/ArtefactType' + supportedFormats: + type: + - array + - 'null' + description: | + List of supported formats for this artefact type. Use lowercase values. If null or empty, the service accepts any format for this type. + items: + type: string + examples: + - - plain + - markdown + - - python + - java + - javascript + - wolfram + - matlab + - - latex + - mathml + contentSchema: + type: + - object + - 'null' + description: | + Optional JSON Schema describing the expected content structure for this artefact type. Allows services to advertise their exact requirements. + additionalProperties: true + locatorSchema: + type: + - object + - 'null' + description: | + Optional JSON Schema describing the locator structure used for feedback targeting within this artefact type. + additionalProperties: true + EvaluateCapabilities: + type: object + description: Capabilities of the evaluate service. + required: + - supportsEvaluate + - supportsPreSubmissionFeedback + - supportsFormativeFeedback + - supportsSummativeFeedback + - supportsDataPolicy + additionalProperties: true + properties: + supportsEvaluate: + type: boolean + description: Indicates whether the /evaluate endpoint is implemented and usable. + supportsPreSubmissionFeedback: + type: boolean + description: Indicates whether the service supports pre-submission feedback runs. + supportsFormativeFeedback: + type: boolean + description: Indicates whether the service supports qualitative feedback without points. + supportsSummativeFeedback: + type: boolean + description: Indicates whether the service supports feedback with points / grading signals. + supportsDataPolicy: + $ref: '#/components/schemas/DataPolicySupport' + supportedArtefactProfiles: + type: + - array + - 'null' + description: | + Optional list of supported artefact profiles. Each profile specifies an artefact type and the formats supported for that type. + items: + $ref: '#/components/schemas/ArtefactProfile' + supportedLanguages: + type: + - array + - 'null' + description: Optional list of supported language codes (e.g., 'en', 'de'). + items: + type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string + EvaluateHealthResponse: + type: object + description: Health status and capabilities of the evaluate service. + required: + - status + - capabilities + properties: + status: + $ref: '#/components/schemas/HealthStatus' + message: + type: + - string + - 'null' + description: Optional human-readable status message. + version: + type: + - string + - 'null' + description: Optional version of the evaluate service implementation. + requirements: + type: + - object + - 'null' + description: Optional requirements clients must satisfy to use this service. + allOf: + - $ref: '#/components/schemas/EvaluateRequirements' + capabilities: + $ref: '#/components/schemas/EvaluateCapabilities' + Message: + type: object + required: + - role + - content + properties: + role: + type: string + enum: + - USER + - ASSISTANT + - SYSTEM + - TOOL + content: + type: string + ChatRequest: + type: object + description: Request body for the chat endpoint. + required: + - messages + properties: + messages: + type: array + description: List of messages in the conversation (including history). + items: + $ref: '#/components/schemas/Message' + conversationId: + type: + - string + - 'null' + description: Optional identifier for the conversation session. + user: + description: Optional user information to adapt the chat style and level. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/User' + context: + type: + - object + - 'null' + description: Optional educational context (e.g., course material, task context). + additionalProperties: true + configuration: + type: + - object + - 'null' + description: Optional configuration for the model(s). + additionalProperties: true + properties: + type: + type: + - string + - 'null' + description: Optional type for the chatbot or chat model. + llm: + description: Optional LLM configuration used for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/LLMConfiguration' + dataPolicy: + description: Optional data policy governing how this request's data may be processed and retained. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/DataPolicy' + executionPolicy: + description: Optional execution constraints for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/ExecutionPolicy' + ChatResponse: + type: object + description: Response body for the chat endpoint. + required: + - output + properties: + output: + $ref: '#/components/schemas/Message' + description: The generated assistant response. + metadata: + type: + - object + - 'null' + description: Optional metadata about response generation. + additionalProperties: true + ChatCapabilities: + type: object + description: Capabilities of the chat service. + required: + - supportsChat + - supportsDataPolicy + additionalProperties: true + properties: + supportsChat: + type: boolean + description: Indicates whether the /chat endpoint is implemented and usable. + supportsUserPreferences: + type: boolean + description: Indicates whether the service supports adapting to user preferences. + supportsStreaming: + type: boolean + description: Indicates whether the service supports streaming responses. + supportsDataPolicy: + $ref: '#/components/schemas/DataPolicySupport' + supportedLanguages: + type: + - array + - 'null' + description: Optional list of supported language codes. + items: + type: string + supportedModels: + type: + - array + - 'null' + description: Optional list of supported models. + items: + type: string + supportedAPIVersions: + type: + - array + - 'null' + description: | + Optional list of µEd API versions supported by this service implementation (e.g., ["0.1.0"]). Clients can use this to select a compatible X-Api-Version. + items: + type: string + ChatHealthResponse: + type: object + description: Health status and capabilities of the chat service. + required: + - status + - capabilities + properties: + status: + $ref: '#/components/schemas/HealthStatus' + statusMessage: + type: + - string + - 'null' + description: Optional human-readable status message. + version: + type: + - string + - 'null' + description: Optional version of the chat service implementation. + capabilities: + $ref: '#/components/schemas/ChatCapabilities' + responses: + 400-BadRequest: + description: Invalid request (e.g. missing content or invalid schema). + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + validationError: + summary: Example validation error + value: + title: Invalid request + message: submission.content must not be empty. + code: VALIDATION_ERROR + trace: null + details: + field: submission.content + 403-Forbidden: + description: Forbidden (e.g. insufficient permissions or access denied). + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + permissionError: + summary: Example permission error + value: + title: Forbidden + message: You do not have permission to access this resource. + code: PERMISSION_DENIED + trace: null + details: + resource: submission + required_permission: write + 406-VersionNotSupported: + description: | + The requested API version (supplied via X-Api-Version) is not supported by this service. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + versionNotSupported: + summary: Example version not supported error + value: + title: API version not supported + message: 'The requested API version ''0.0'' is not supported. Supported versions are: [''0.1.0''].' + code: VERSION_NOT_SUPPORTED + trace: null + details: + requestedVersion: '0.0' + supportedVersions: + - 0.1.0 + 500-InternalError: + description: Internal server error. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + internalError: + summary: Example internal error + value: + title: Internal server error + message: Unexpected failure while generating feedback. + code: INTERNAL_ERROR + trace: 'java.lang.RuntimeException: ... (stack trace omitted)' + details: + subsystem: llm-provider + 501-NotImplemented: + description: | + The server does not support the evaluate method. This allows service providers to implement only subsets of the µEd API. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + notImplemented: + summary: Example not implemented error + value: + title: Not implemented + message: This service does not implement /evaluate. + code: NOT_IMPLEMENTED + trace: null + details: null + 503-ServiceUnavailable: + description: Service is currently unavailable or unhealthy. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + serviceUnavailable: + summary: Example unavailable error + value: + title: Service unavailable + message: Evaluate service is currently unavailable. + code: SERVICE_UNAVAILABLE + trace: null + details: + reason: Database connection failed + unhealthy: + summary: Example degraded / unavailable service + value: + title: Service unhealthy + status: UNAVAILABLE + message: Database connection failed + version: 1.0.0 + capabilities: + supportsEvaluate: false + supportsPreSubmissionFeedback: false + supportsFormativeFeedback: false + supportsSummativeFeedback: false + supportsDataPolicy: NOT_SUPPORTED + supportedArtefactProfiles: [] + supportedLanguages: [] + 400-BadRequest-2: + description: Invalid request (e.g. missing content or invalid schema). + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + invalidChatRequest: + summary: Example invalid chat request + value: + title: Invalid request + message: messages must contain at least one item. + code: VALIDATION_ERROR + trace: null + details: + field: messages + 500-InternalError-2: + description: Internal server error. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + modelProviderError: + summary: Example model provider error + value: + title: Model provider error + message: LLM provider returned an error while generating a response. + code: LLM_PROVIDER_ERROR + trace: null + details: + provider: openai + 501-NotImplemented-2: + description: | + The server does not support the chat method. This allows service providers to implement only subsets of the µEd API. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + notImplemented: + summary: Example not implemented error + value: + title: Not implemented + message: This service does not implement /chat. + code: NOT_IMPLEMENTED + trace: null + details: null + 503-ServiceUnavailable-2: + description: Service is currently unavailable or unhealthy. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + X-Api-Version: + description: The API version that was used to serve this response. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + serviceUnavailable: + summary: Example unavailable error + value: + title: Service unavailable + message: Chat service is currently unavailable. + code: SERVICE_UNAVAILABLE + trace: null + details: + reason: LLM provider connection failed + serviceUnhealthy: + summary: Example degraded / unavailable service + value: + title: Service unhealthy + message: LLM provider connection failed + code: SERVICE_UNHEALTHY + version: 1.0.0 + capabilities: + supportsChat: false + supportsUserPreferences: false + supportsStreaming: false + supportsDataPolicy: NOT_SUPPORTED + supportedLanguages: [] + supportedModels: [] diff --git a/scripts/generate_mued_types.py b/scripts/generate_mued_types.py new file mode 100644 index 0000000..5599803 --- /dev/null +++ b/scripts/generate_mued_types.py @@ -0,0 +1,43 @@ +import os +import subprocess +import sys +from pathlib import Path + +# --- env vars (both required, no defaults — fail fast like the TS script) --- +version = os.environ.get("MUED_SPEC_VERSION") +if not version: + raise SystemExit("MUED_SPEC_VERSION is not set") + +spec = os.environ.get("MUED_SPEC_URL") +if not spec: + raise SystemExit("MUED_SPEC_URL is not set") + +if not spec.startswith("https://") and not Path(spec).exists(): + raise SystemExit(f"Spec file not found: {spec}") + +# --- output: versioned filename to mirror mued-api-{version}.ts convention --- +out = Path(__file__).parent.parent / "schemas" / "muEd" / f"mued_api_{version}.py" + +print(f"Generating mued types from: {spec}") +subprocess.run( + [ + "datamodel-codegen", + "--input", spec, + "--input-file-type", "openapi", + "--output", str(out), + "--output-model-type", "pydantic_v2.BaseModel", + "--use-annotated", + "--use-standard-collections", + "--target-python-version", "3.8", + "--extra-fields", "allow", + ], + check=True, +) + +# Prepend generated header (mirrors ts-nocheck prepend in TS script) +content = out.read_text() +header = f"# generated by generate_mued_types.py — spec version {version}\n" +if not content.startswith("# generated"): + out.write_text(header + content) + +print(f"Generated: {out}") diff --git a/tests/commands.py b/tests/commands.py deleted file mode 100644 index 59d445b..0000000 --- a/tests/commands.py +++ /dev/null @@ -1,381 +0,0 @@ -import unittest -from typing import Any - -from ..tools import commands, parse, validate -from ..tools.utils import JsonType - - -def evaluation_function( - response: Any, answer: Any, params: JsonType -) -> JsonType: - if params.get("raise", False): - raise ValueError("raised") - - force_true = params.get("force", False) - - return {"is_correct": (response == answer) or force_true} - - -class TestCommandsModule(unittest.TestCase): - def __init__(self, methodName: str = "runTest") -> None: - super().__init__(methodName) - - def setUp(self) -> None: - commands.evaluation_function = evaluation_function - return super().setUp() - - def tearDown(self) -> None: - commands.evaluation_function = None - return super().tearDown() - - def test_valid_eval_command(self): - event = { - "body": {"response": "hello", "answer": "world", "params": {}} - } - response = commands.evaluate(event) - - self.assertIn("result", response) - self.assertIn("is_correct", response["result"]) # type: ignore - - def test_invalid_eval_args_raises_parse_error(self): - event = {"headers": "any", "other": "params"} - - with self.assertRaises(parse.ParseError) as e: - commands.evaluate(event) - - self.assertEqual( - e.exception.message, "No data supplied in request body." - ) - - def test_invalid_eval_schema_raises_validation_error(self): - event = {"body": {"response": "hello", "params": {}}} - - with self.assertRaises(validate.ValidationError) as e: - commands.evaluate(event) - - self.assertEqual( - e.exception.message, - "Failed to validate body against the evaluation schema.", - ) - - def test_single_feedback_case(self): - event = { - "body": { - "response": "hello", - "answer": "hello", - "params": { - "cases": [ - { - "answer": "other", - "feedback": "should be 'other'.", - "mark": 0, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertTrue(result["is_correct"]) - self.assertNotIn("matched_case", result) - self.assertNotIn("feedback", result) - - def test_single_feedback_case_match(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_case_warning_data_structure(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "feedback": "should be 'hello'.", - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertIn("warnings", result) - warning = result["warnings"].pop() # type: ignore - - self.assertDictEqual( - warning, - {"case": 0, "message": "Missing answer/feedback field"}, - ) - - def test_multiple_feedback_cases_single_match(self): - event = { - "body": { - "response": "yes", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - }, - { - "answer": "yes", - "feedback": "should be 'yes'.", - }, - { - "answer": "no", - "feedback": "should be 'no'.", - }, - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 1) - self.assertEqual(result["feedback"], "should be 'yes'.") - - def test_multiple_feedback_cases_multiple_matches(self): - event = { - "body": { - "response": "yes", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "params": {"force": True}, - }, - { - "answer": "yes", - "feedback": "should be 'yes'.", - }, - { - "answer": "no", - "feedback": "should be 'no'.", - }, - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_case_params_overwrite_eval_params(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "force": True, - "cases": [ - { - "answer": "yes", - "feedback": "should be 'yes'.", - "params": {"force": False}, - } - ], - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertTrue(result["is_correct"]) - self.assertNotIn("matched_case", result) - self.assertNotIn("feedback", result) - - def test_invalid_case_entry_doesnt_raise_exception(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "params": {"raise": True}, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertIn("warnings", result) - warning = result["warnings"].pop() # type: ignore - - self.assertDictEqual( - warning, - { - "case": 0, - "message": "An exception was raised while executing " - "the evaluation function.", - "detail": "raised", - }, - ) - - def test_multiple_matched_cases_are_combined_and_warned(self): - event = { - "body": { - "response": "yes", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "params": {"force": True}, - }, - { - "answer": "yes", - "feedback": "should be 'yes'.", - }, - { - "answer": "no", - "feedback": "should be 'no'.", - }, - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertIn("warnings", result) - warning = result["warnings"].pop() # type: ignore - - self.assertDictEqual( - warning, - { - "message": "Cases 0, 1 were matched. " - "Only the first one's feedback was returned", - }, - ) - - def test_overriding_eval_feedback_to_correct_case(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "mark": 1, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertTrue(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_overriding_eval_feedback_to_incorrect_case(self): - event = { - "body": { - "response": "hello", - "answer": "hello", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "mark": 0, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_valid_preview_command(self): - event = {"body": {"response": "hello"}} - - response = commands.preview(event) - result = response["result"] # type: ignore - - self.assertEqual(result["preview"]["latex"], "hello") - - def test_invalid_preview_args_raises_parse_error(self): - event = {"headers": "any", "other": "params"} - - with self.assertRaises(parse.ParseError) as e: - commands.preview(event) - - self.assertEqual( - e.exception.message, "No data supplied in request body." - ) - - def test_invalid_preview_schema_raises_validation_error(self): - event = {"body": {"response": "hello", "answer": "hello"}} - - with self.assertRaises(validate.ValidationError) as e: - commands.preview(event) - - self.assertEqual( - e.exception.message, - "Failed to validate body against the preview schema.", - ) - - def test_healthcheck(self): - response = commands.healthcheck() - - self.assertIn("result", response) - result = response["result"] # type: ignore - - self.assertIn("tests_passed", result) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/commands_test.py b/tests/commands_test.py new file mode 100644 index 0000000..4fe4752 --- /dev/null +++ b/tests/commands_test.py @@ -0,0 +1,360 @@ +import os +import unittest +from pathlib import Path +from typing import Any + +from ..handler import handler +from ..tools import commands +from ..tools.utils import JsonType + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") + + +def evaluation_function( + response: Any, answer: Any, params: JsonType +) -> JsonType: + if params.get("raise", False): + raise ValueError("raised") + + force_true = params.get("force", False) + + return {"is_correct": (response == answer) or force_true} + + +class TestCommandsModule(unittest.TestCase): + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + commands.evaluation_function = evaluation_function + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = None + return super().tearDown() + + def test_valid_eval_command(self): + body = {"response": "hello", "answer": "world", "params": {}} + response = commands.evaluate(body) + + self.assertIn("result", response) + self.assertIn("is_correct", response["result"]) # type: ignore + + def test_invalid_eval_args_raises_parse_error(self): + event = {"headers": {"command": "eval"}, "other": "params"} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], "No data supplied in request body." # type: ignore + ) + + def test_invalid_eval_schema_raises_validation_error(self): + event = {"headers": {"command": "eval"}, "body": {"response": "hello", "params": {}}} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], # type: ignore + "Failed to validate body against the evaluation schema.", + ) + + def test_single_feedback_case(self): + body = { + "response": "hello", + "answer": "hello", + "params": { + "cases": [ + { + "answer": "other", + "feedback": "should be 'other'.", + "mark": 0, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertTrue(result["is_correct"]) + self.assertNotIn("matched_case", result) + self.assertNotIn("feedback", result) + + def test_single_feedback_case_match(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_case_warning_data_structure(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "feedback": "should be 'hello'.", + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertIn("warnings", result) + warning = result["warnings"].pop() # type: ignore + + self.assertDictEqual( + warning, + {"case": 0, "message": "Missing answer/feedback field"}, + ) + + def test_multiple_feedback_cases_single_match(self): + body = { + "response": "yes", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + }, + { + "answer": "yes", + "feedback": "should be 'yes'.", + }, + { + "answer": "no", + "feedback": "should be 'no'.", + }, + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 1) + self.assertEqual(result["feedback"], "should be 'yes'.") + + def test_multiple_feedback_cases_multiple_matches(self): + body = { + "response": "yes", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "params": {"force": True}, + }, + { + "answer": "yes", + "feedback": "should be 'yes'.", + }, + { + "answer": "no", + "feedback": "should be 'no'.", + }, + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_case_params_overwrite_eval_params(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "force": True, + "cases": [ + { + "answer": "yes", + "feedback": "should be 'yes'.", + "params": {"force": False}, + } + ], + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertTrue(result["is_correct"]) + self.assertNotIn("matched_case", result) + self.assertNotIn("feedback", result) + + def test_invalid_case_entry_doesnt_raise_exception(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "params": {"raise": True}, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertIn("warnings", result) + warning = result["warnings"].pop() # type: ignore + + self.assertDictEqual( + warning, + { + "case": 0, + "message": "An exception was raised while executing " + "the evaluation function.", + "detail": "raised", + }, + ) + + def test_first_matching_case_stops_evaluation(self): + body = { + "response": "yes", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "params": {"force": True}, + }, + { + "answer": "yes", + "feedback": "should be 'yes'.", + }, + { + "answer": "no", + "feedback": "should be 'no'.", + }, + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertNotIn("warnings", result) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_overriding_eval_feedback_to_correct_case(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "mark": 1, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertTrue(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_overriding_eval_feedback_to_incorrect_case(self): + body = { + "response": "hello", + "answer": "hola", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "mark": 0, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_valid_preview_command(self): + body = {"response": "hello"} + + response = commands.preview(body) + result = response["result"] # type: ignore + + self.assertEqual(result["preview"], "hello") + + def test_invalid_preview_args_raises_parse_error(self): + event = {"headers": {"command": "preview"}, "other": "params"} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], "No data supplied in request body." # type: ignore + ) + + def test_invalid_preview_schema_raises_validation_error(self): + event = {"headers": {"command": "preview"}, "body": {"response": "hello", "answer": "hello"}} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], # type: ignore + "Failed to validate body against the preview schema.", + ) + + def test_healthcheck(self): + response = commands.healthcheck() + + self.assertIn("result", response) + result = response["result"] # type: ignore + + self.assertIn("tests_passed", result) + + def test_healthcheck_muEd(self): + response = commands.healthcheck_muEd() + + self.assertIn(response["status"], ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = response["capabilities"] + self.assertIn("supportedAPIVersions", capabilities) + self.assertIsInstance(capabilities["supportedAPIVersions"], list) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/docs.py b/tests/docs_test.py similarity index 90% rename from tests/docs.py rename to tests/docs_test.py index 5666af6..185274f 100644 --- a/tests/docs.py +++ b/tests/docs_test.py @@ -1,10 +1,20 @@ import base64 +import os import unittest from ..tools import docs +_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + class TestDocsModule(unittest.TestCase): + def setUp(self): + self._orig_cwd = os.getcwd() + os.chdir(_PROJECT_ROOT) + + def tearDown(self): + os.chdir(self._orig_cwd) + def test_handling_available_doc(self): result = docs.send_file("tests/test_file.md") diff --git a/tests/handling.py b/tests/handling_test.py similarity index 95% rename from tests/handling.py rename to tests/handling_test.py index 8370455..cbcdf4b 100755 --- a/tests/handling.py +++ b/tests/handling_test.py @@ -1,10 +1,14 @@ +import os import unittest +from pathlib import Path from typing import Optional from ..handler import handler from ..tools import commands from ..tools.utils import EvaluationFunctionType +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") + evaluation_function: Optional[ EvaluationFunctionType ] = lambda response, answer, params: {"is_correct": True} @@ -12,10 +16,12 @@ class TestHandlerFunction(unittest.TestCase): def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR commands.evaluation_function = evaluation_function return super().setUp() def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) commands.evaluation_function = None return super().tearDown() diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py new file mode 100644 index 0000000..fb14259 --- /dev/null +++ b/tests/mued_handling_test.py @@ -0,0 +1,467 @@ +import json +import os +import unittest +from pathlib import Path +from typing import Optional + +from ..handler import handler +from ..tools import commands +from ..tools.utils import EvaluationFunctionType + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") + +evaluation_function: Optional[ + EvaluationFunctionType +] = lambda response, answer, params: {"is_correct": True, "feedback": "Well done."} + + +class TestMuEdHandlerFunction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function + commands.evaluation_function = evaluation_function + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = self._orig_eval + return super().tearDown() + + def test_evaluate_returns_feedback_list(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "TEXT", "content": {}}}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) + self.assertIn("awardedPoints", body[0]) + + def test_evaluate_feedback_message(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "TEXT", "content": {"text": "hello"}}}, + } + + response = handler(event) + + body = json.loads(response["body"]) + self.assertEqual(body[0]["message"], "Well done.") + self.assertEqual(body[0]["awardedPoints"], True) + + def test_evaluate_with_task(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "TEXT", "content": {"text": "Polymorphism allows..."}}, + "task": { + "title": "OOP Concepts", + "referenceSolution": {"text": "Polymorphism allows objects..."}, + }, + }, + } + + response = handler(event) + + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + + def test_evaluate_missing_submission_returns_error(self): + event = { + "path": "/evaluate", + "body": {"task": {"title": "Some Task"}}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + + def test_evaluate_invalid_submission_type_returns_error(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "INVALID"}}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + + def test_evaluate_bodyless_event_returns_error(self): + event = {"path": "/evaluate", "random": "metadata"} + + response = handler(event) + + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + self.assertEqual(body["message"], "No data supplied in request body.") + + def test_healthcheck(self): + event = {"path": "/evaluate/health"} + + response = handler(event) + + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + capabilities = body.get("capabilities", {}) + self.assertIn("supportedAPIVersions", capabilities) + self.assertIn("0.1.0", capabilities["supportedAPIVersions"]) + + def test_supported_version_header_is_accepted(self): + event = { + "path": "/evaluate/health", + "headers": {"X-Api-Version": "0.1.0"}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 200) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertIn(body.get("status"), ("OK", "DEGRADED", "UNAVAILABLE")) + + def test_unsupported_version_header_returns_error(self): + event = { + "path": "/evaluate", + "headers": {"X-Api-Version": "99.0.0"}, + "body": {"submission": {"type": "TEXT", "content": {}}}, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 406) + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertEqual(body.get("code"), "VERSION_NOT_SUPPORTED") + self.assertIn("details", body) + self.assertEqual(body["details"]["requestedVersion"], "99.0.0") + self.assertIn("0.1.0", body["details"]["supportedVersions"]) + + def test_absent_version_header_proceeds_normally(self): + event = {"path": "/evaluate/health"} + + response = handler(event) + + self.assertEqual(response["headers"]["X-Api-Version"], "0.1.0") + body = json.loads(response["body"]) + self.assertNotIn("code", body) + + def test_unknown_path_falls_back_to_legacy(self): + event = { + "path": "/unknown", + "body": {"response": "hello", "answer": "world!"}, + "headers": {"command": "eval"}, + } + + response = handler(event) + + self.assertEqual(response.get("command"), "eval") + self.assertIn("result", response) + + +class TestMuEdEvaluateExtraction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_eval = commands.evaluation_function + self.captured: dict = {} + captured = self.captured + + def capturing_eval(response, answer, params): + captured["response"] = response + captured["answer"] = answer + captured["params"] = params + return {"is_correct": True, "feedback": "Captured."} + + commands.evaluation_function = capturing_eval + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = self._orig_eval + return super().tearDown() + + def test_math_submission_extracts_expression(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+1"}}, + }, + } + result = json.loads(handler(event)["body"]) + self.assertEqual(self.captured["response"], "x+1") + self.assertEqual(self.captured["answer"], "x+1") + self.assertEqual(result[0]["awardedPoints"], True) + + def test_text_submission_extracts_text(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "TEXT", "content": {"text": "hello"}}, + "task": {"title": "T", "referenceSolution": {"text": "hello"}}, + }, + } + result = json.loads(handler(event)["body"]) + self.assertEqual(self.captured["response"], "hello") + self.assertEqual(self.captured["answer"], "hello") + self.assertEqual(result[0]["awardedPoints"], True) + + def test_configuration_params_forwarded(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "configuration": {"params": {"strict_syntax": False}}, + }, + } + result = json.loads(handler(event)["body"]) + self.assertEqual(self.captured["params"], {"strict_syntax": False}) + self.assertEqual(result[0]["awardedPoints"], True) + + def test_no_task_answer_is_none(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + }, + } + result = json.loads(handler(event)["body"]) + self.assertIsNone(self.captured["answer"]) + self.assertEqual(result[0]["awardedPoints"], True) + + +class TestMuEdPreviewHandlerFunction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function + self._orig_eval = commands.evaluation_function + commands.preview_function = lambda response, params: { + "preview": {"latex": f"\\text{{{response}}}", "sympy": response} + } + commands.evaluation_function = lambda response, answer, params: { + "is_correct": True, "feedback": "Well done." + } + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.preview_function = self._orig_preview + commands.evaluation_function = self._orig_eval + return super().tearDown() + + def test_preview_returns_feedback_list(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 200) + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(len(body), 1) + + def test_preview_feedback_id_is_preSubmissionFeedback(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + body = json.loads(handler(event)["body"]) + + self.assertNotIn("feedbackId", body[0]) + + def test_preview_contains_preSubmissionFeedback_field(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + body = json.loads(handler(event)["body"]) + + self.assertIn("preSubmissionFeedback", body[0]) + + def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + body = json.loads(handler(event)["body"]) + + preview = body[0]["preSubmissionFeedback"] + self.assertIn("latex", preview) + self.assertIn("sympy", preview) + + def test_preview_missing_submission_returns_error(self): + event = { + "path": "/evaluate", + "body": { + "configuration": {"params": {}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + + def test_preview_bodyless_event_returns_error(self): + event = {"path": "/evaluate", "random": "metadata"} + + response = handler(event) + + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + self.assertEqual(body["message"], "No data supplied in request body.") + + def test_preview_invalid_submission_type_returns_error(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "INVALID", "content": {}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertEqual(response["statusCode"], 400) + body = json.loads(response["body"]) + self.assertEqual(body["code"], "VALIDATION_ERROR") + + def test_presubmission_disabled_runs_normal_evaluation(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": False}, + }, + } + + response = handler(event) + + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertIn("awardedPoints", body[0]) + self.assertNotIn("preSubmissionFeedback", body[0]) + + +class TestMuEdPreviewExtraction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self._orig_preview = commands.preview_function + self.captured: dict = {} + captured = self.captured + + def capturing_preview(response, params): + captured["response"] = response + captured["params"] = params + return {"preview": {"latex": "captured", "sympy": str(response)}} + + commands.preview_function = capturing_preview + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.preview_function = self._orig_preview + return super().tearDown() + + def test_math_submission_extracts_expression(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + handler(event) + + self.assertEqual(self.captured["response"], "x+1") + + def test_text_submission_extracts_text(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "TEXT", "content": {"text": "hello"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + handler(event) + + self.assertEqual(self.captured["response"], "hello") + + def test_configuration_params_forwarded(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "configuration": {"params": {"strict_syntax": False, "is_latex": True}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + handler(event) + + self.assertEqual(self.captured["params"], {"strict_syntax": False, "is_latex": True}) + + def test_no_task_required(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "sin(x)"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + body = json.loads(response["body"]) + self.assertIsInstance(body, list) + self.assertEqual(self.captured["response"], "sin(x)") + + def test_preview_result_propagated(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + body = json.loads(handler(event)["body"]) + + self.assertEqual(body[0]["preSubmissionFeedback"]["latex"], "captured") + self.assertEqual(body[0]["preSubmissionFeedback"]["sympy"], "x+1") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/mued_requests_test.py b/tests/mued_requests_test.py new file mode 100644 index 0000000..23e30ff --- /dev/null +++ b/tests/mued_requests_test.py @@ -0,0 +1,87 @@ +import unittest + +from ..tools import validate +from ..tools.validate import MuEdReqBodyValidators, ValidationError + + +class TestMuEdRequestValidation(unittest.TestCase): + + def test_empty_request_body(self): + body = {} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertEqual( + e.exception.message, + "Failed to validate body against the evaluation schema.", + ) + + def test_missing_submission(self): + body = {"task": {"title": "test task"}} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertIn("submission", e.exception.error_thrown) # type: ignore + + def test_submission_missing_type(self): + body = {"submission": {"content": {"text": "hello"}}} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertIn("type", e.exception.error_thrown) # type: ignore + + def test_invalid_submission_type(self): + body = {"submission": {"type": "INVALID", "content": {}}} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertIn("INVALID", e.exception.error_thrown) # type: ignore + + def test_extra_fields_allowed(self): + # EvaluateRequest allows additional properties — unknown fields should not raise + body = {"submission": {"type": "TEXT", "content": {}}, "unknown_field": "value"} + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + def test_valid_minimal_request(self): + body = {"submission": {"type": "TEXT", "content": {}}} + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + def test_valid_request_with_task(self): + body = { + "submission": { + "type": "TEXT", + "content": {"text": "Explain polymorphism."}, + }, + "task": { + "title": "OOP Concepts", + "referenceSolution": {"text": "Polymorphism allows..."}, + }, + } + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + def test_valid_request_with_all_optional_fields(self): + body = { + "submission": { + "type": "TEXT", + "format": "plain", + "content": {"text": "hello"}, + }, + "task": { + "title": "Test Task", + "referenceSolution": {"text": "answer"}, + "learningObjectives": ["Understand polymorphism"], + }, + "criteria": [ + {"name": "Correctness", "context": "The solution is correct."} + ], + "configuration": {"key": "value"}, + } + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/mued_responses_test.py b/tests/mued_responses_test.py new file mode 100644 index 0000000..73390fc --- /dev/null +++ b/tests/mued_responses_test.py @@ -0,0 +1,54 @@ +import unittest + +from ..tools import validate +from ..tools.validate import MuEdResBodyValidators, ValidationError + + +class TestMuEdResponseValidation(unittest.TestCase): + + def test_non_list_response(self): + body = {} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdResBodyValidators.EVALUATION) + + self.assertEqual( + e.exception.message, + "Failed to validate body against the evaluation schema.", + ) + + def test_valid_empty_list(self): + body = [] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_valid_minimal_feedback(self): + body = [{"message": "Good attempt."}] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_extra_fields_allowed(self): + # Feedback uses extra='allow' — unknown fields should not raise + body = [{"unknown_field": "value"}] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_valid_full_feedback(self): + body = [ + { + "title": "Correctness", + "message": "Your answer is correct.", + "suggestedAction": "Review the concept further.", + "criterion": {"name": "Correctness", "context": "Is the answer correct?"}, + "target": {"artefactType": "TEXT"}, + } + ] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_multiple_feedback_items(self): + body = [ + {"message": "Well structured."}, + {"message": "Good use of examples."}, + ] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/parse.py b/tests/parse_test.py similarity index 100% rename from tests/parse.py rename to tests/parse_test.py diff --git a/tests/requests.py b/tests/requests_test.py similarity index 92% rename from tests/requests.py rename to tests/requests_test.py index 818332f..ed41ad9 100755 --- a/tests/requests.py +++ b/tests/requests_test.py @@ -1,10 +1,20 @@ +import os import unittest +from pathlib import Path from ..tools import validate -from ..tools.validate import ReqBodyValidators, ValidationError +from ..tools.validate import LegacyReqBodyValidators as ReqBodyValidators, ValidationError + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") class TestRequestValidation(unittest.TestCase): + def setUp(self): + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + + def tearDown(self): + os.environ.pop("SCHEMA_DIR", None) + def test_empty_request_body(self): body = {} diff --git a/tests/responses.py b/tests/responses_test.py similarity index 94% rename from tests/responses.py rename to tests/responses_test.py index 010216e..805efda 100755 --- a/tests/responses.py +++ b/tests/responses_test.py @@ -1,10 +1,20 @@ +import os import unittest +from pathlib import Path from ..tools import validate -from ..tools.validate import ResBodyValidators, ValidationError +from ..tools.validate import LegacyResBodyValidators as ResBodyValidators, ValidationError + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") class TestResponseValidation(unittest.TestCase): + def setUp(self): + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + + def tearDown(self): + os.environ.pop("SCHEMA_DIR", None) + def test_empty_response_body(self): body = {} diff --git a/tools/commands.py b/tools/commands.py index 3f4cc60..ee84a54 100644 --- a/tools/commands.py +++ b/tools/commands.py @@ -1,17 +1,15 @@ import warnings -from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict, Union from evaluation_function_utils.errors import EvaluationException from . import healthcheck as health -from . import parse, validate from .utils import ( EvaluationFunctionType, JsonType, PreviewFunctionType, Response, ) -from .validate import ReqBodyValidators try: from ..evaluation import evaluation_function # type: ignore @@ -57,9 +55,13 @@ class CaseResult(NamedTuple): is_correct: bool = False feedback: str = "" + warning: Optional[CaseWarning] = None +SUPPORTED_MUED_VERSIONS: list[str] = ["0.1.0"] + + def healthcheck() -> Response: """Run the healthcheck command for the evaluation function. @@ -70,28 +72,40 @@ def healthcheck() -> Response: return Response(command="healthcheck", result=result) +def healthcheck_muEd() -> Dict: + """Run the healthcheck command and return a muEd EvaluateHealthResponse. + + Returns: + Dict: A spec-compliant EvaluateHealthResponse. + """ + result = health.healthcheck() + status = "OK" if result["tests_passed"] else "DEGRADED" + return { + "status": status, + "capabilities": { + "supportsEvaluate": True, + "supportsPreSubmissionFeedback": True, + "supportsFormativeFeedback": True, + "supportsSummativeFeedback": False, + "supportsDataPolicy": "NOT_SUPPORTED", + "supportedAPIVersions": SUPPORTED_MUED_VERSIONS, + }, + } + + def preview( - event: JsonType, fnc: Optional[PreviewFunctionType] = None + body: JsonType, fnc: Optional[PreviewFunctionType] = None ) -> Response: """Run the preview command for the evaluation function. - Note: - The body of the event is validated against the preview schema - before running the preview function. - Args: - event (JsonType): The dictionary received by the gateway. Must - include a body field which may be a JSON string or a dictionary. + body (JsonType): The validated request body. fnc (Optional[PreviewFunctionType]): A function to override the current preview function (for testing). Defaults to None. Returns: Response: The result given the response and params in the body. """ - body = parse.body(event) - - validate.body(body, ReqBodyValidators.PREVIEW) - params = body.get("params", {}) fnc = fnc or preview_function @@ -100,42 +114,28 @@ def preview( return Response(command="preview", result=result) -def evaluate(event: JsonType) -> Response: - """Run the evaluation command for the evaluation function. - - Note: - The body of the event is validated against the eval schema - before running the evaluation function. - - If cases are included in the params, this function checks for - matching answers and returns the specified feedback. +def _run_evaluation(response: Any, answer: Any, params: Dict) -> Dict: + """Core evaluation logic shared by legacy and muEd command functions. Args: - event (JsonType): The dictionary received by the gateway. Must - include a body field which may be a JSON string or a dictionary. - fnc (Optional[EvaluationFunctionType]): A function to override the - current evaluation function (for testing). Defaults to None. + response (Any): The student's response. + answer (Any): The reference answer. + params (Dict): The evaluation parameters. Returns: - Response: The result given the response and params in the body. + Dict: The raw result from the evaluation function, with case + feedback applied if applicable. """ - body = parse.body(event) - validate.body(body, ReqBodyValidators.EVALUATION) - - params = body.get("params", {}) - if evaluation_function is None: raise EvaluationException("Evaluation function is not defined.") - result = evaluation_function(body["response"], body["answer"], params) + result = evaluation_function(response, answer, params) if result["is_correct"] is False and "cases" in params and len(params["cases"]) > 0: - match, warnings = get_case_feedback( - body["response"], params, params["cases"] - ) + match, case_warnings = get_case_feedback(response, params, params["cases"]) - if warnings: - result["warnings"] = warnings + if case_warnings: + result["warnings"] = case_warnings if match is not None: result["feedback"] = match["feedback"] @@ -146,9 +146,81 @@ def evaluate(event: JsonType) -> Response: if "mark" in match: result["is_correct"] = bool(int(match["mark"])) + return result + + +def evaluate(body: JsonType) -> Response: + """Run the evaluation command for the evaluation function (legacy format). + + Args: + body (JsonType): The validated request body. + + Returns: + Response: The result given the response and params in the body. + """ + params = body.get("params", {}) + result = _run_evaluation(body["response"], body["answer"], params) return Response(command="eval", result=result) +def _extract_muEd_submission(body: JsonType): + """Extract response, params, and content_key from a muEd request body.""" + submission = body["submission"] + sub_type = submission.get("type", "MATH") + _type_key = {"MATH": "expression", "TEXT": "text", "CODE": "code", "MODEL": "model"} + content_key = _type_key.get(sub_type, "value") + response = submission.get("content", {}).get(content_key) + params = body.get("configuration", {}).get("params", {}) + return response, params, content_key + + +def _run_muEd_preview(body: JsonType) -> List[Dict]: + response, params, _ = _extract_muEd_submission(body) + preview_result = preview_function(response, params) + return [{"preSubmissionFeedback": preview_result.get("preview", {})}] + + +def _run_muEd_evaluation(body: JsonType) -> List[Dict]: + response, params, content_key = _extract_muEd_submission(body) + + task = body.get("task") + if task: + ref = task.get("referenceSolution") or {} + answer = ref.get(content_key) + else: + answer = None + + result = _run_evaluation(response, answer, params) + + is_correct = result.get("is_correct", 0) + feedback_text = result.get("feedback", "") + + feedback_item: Dict = { + "message": feedback_text if isinstance(feedback_text, str) else str(feedback_text), + "awardedPoints": int(is_correct), + } + + if result.get("tags"): + feedback_item["tags"] = result["tags"] + + return [feedback_item] + + +def evaluate_muEd(body: JsonType) -> List[Dict]: + """Run the evaluation command for the evaluation function (muEd format). + + Args: + body (JsonType): The validated muEd request body. + + Returns: + List[Dict]: A list of Feedback items. + """ + pre_sub = body.get("preSubmissionFeedback") or {} + if pre_sub.get("enabled"): + return _run_muEd_preview(body) + return _run_muEd_evaluation(body) + + def get_case_feedback( response: Any, params: Dict, diff --git a/tools/smoke_tests.py b/tools/smoke_tests.py index 15e3406..1e0a659 100644 --- a/tools/smoke_tests.py +++ b/tools/smoke_tests.py @@ -64,36 +64,36 @@ def test_load_user_docs(self): self.assertTrue(result["isBase64Encoded"]) def test_load_eval_req_schema(self): - schema = validate.load_validator_from_url( - validate.ReqBodyValidators.EVALUATION + schema = validate.load_validator( + validate.LegacyReqBodyValidators.EVALUATION ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_preview_req_schema(self): - schema = validate.load_validator_from_url( - validate.ReqBodyValidators.PREVIEW + schema = validate.load_validator( + validate.LegacyReqBodyValidators.PREVIEW ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_eval_res_schema(self): - schema = validate.load_validator_from_url( - validate.ResBodyValidators.EVALUATION + schema = validate.load_validator( + validate.LegacyResBodyValidators.EVALUATION ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_preview_res_schema(self): - schema = validate.load_validator_from_url( - validate.ResBodyValidators.PREVIEW + schema = validate.load_validator( + validate.LegacyResBodyValidators.PREVIEW ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_health_res_schema(self): - schema = validate.load_validator_from_url( - validate.ResBodyValidators.HEALTHCHECK + schema = validate.load_validator( + validate.LegacyResBodyValidators.HEALTHCHECK ) self.assertIsInstance(schema, jsonschema.Draft7Validator) diff --git a/tools/validate.py b/tools/validate.py index f82e2f7..6c039c6 100755 --- a/tools/validate.py +++ b/tools/validate.py @@ -1,15 +1,32 @@ import enum -import functools +import json import os from typing import Dict, List, TypedDict, Union import dotenv import jsonschema import jsonschema.exceptions -import json +import yaml + +from .utils import Response dotenv.load_dotenv() +_openapi_spec = None + +# EvaluateResponse is not a named schema in the OpenAPI spec — the evaluate +# endpoint returns an inline array of Feedback items. +_EVALUATE_RESPONSE_SCHEMA = { + "type": "array", + "items": { + "type": "object", + "properties": { + "feedbackId": {"type": "string"}, + }, + "additionalProperties": True, + }, +} + class SchemaErrorThrown(TypedDict): """Detail object returned in the error response for schema exceptions.""" @@ -34,44 +51,80 @@ def __init__( """Enumeration objects for picking which schema to validate against.""" -class ReqBodyValidators(enum.Enum): - """Enum for all request body validators.""" +class LegacyReqBodyValidators(enum.Enum): + """Enum for all legacy request body validators.""" + + ORIGINAL = "legacy/request.json" + EVALUATION = "legacy/request/eval.json" + PREVIEW = "legacy/request/preview.json" + + +class LegacyResBodyValidators(enum.Enum): + """Enum for all legacy response body validators.""" + + ORIGINAL = "legacy/responsev2.json" + EVALUATION = "legacy/response/eval.json" + PREVIEW = "legacy/response/preview.json" + HEALTHCHECK = "legacy/response/healthcheck.json" + + +class MuEdReqBodyValidators(enum.Enum): + """Enum for muEd request body validators.""" + + EVALUATION = "EvaluateRequest" + - ORIGINAL = "request.json" - EVALUATION = "request/eval.json" - PREVIEW = "request/preview.json" +class MuEdResBodyValidators(enum.Enum): + """Enum for muEd response body validators.""" + EVALUATION = "EvaluateResponse" + HEALTHCHECK = "EvaluateHealthResponse" -class ResBodyValidators(enum.Enum): - """Enum for all response body validators.""" - ORIGINAL = "responsev2.json" - EVALUATION = "response/eval.json" - PREVIEW = "response/preview.json" - HEALTHCHECK = "response/healthcheck.json" +BodyValidators = Union[ + LegacyReqBodyValidators, + LegacyResBodyValidators, + MuEdReqBodyValidators, + MuEdResBodyValidators, +] -BodyValidators = Union[ReqBodyValidators, ResBodyValidators] +def _get_openapi_spec() -> dict: + """Load and cache the muEd OpenAPI spec.""" + global _openapi_spec + if _openapi_spec is None: + schema_dir = os.getenv("SCHEMA_DIR") + if schema_dir is not None: + spec_path = os.path.join(schema_dir, "muEd", "openapi-v0_1_0.yml") + else: + spec_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "schemas", "muEd", "openapi-v0_1_0.yml", + ) + with open(spec_path) as f: + _openapi_spec = yaml.safe_load(f) + return _openapi_spec + def load_validator( - validator_enum: BodyValidators, + validator_enum: Union[LegacyReqBodyValidators, LegacyResBodyValidators], ) -> jsonschema.Draft7Validator: """Loads a json schema for body validations. Args: - validator_enum (BodyValidators): The validator enum name. + validator_enum (Union[LegacyReqBodyValidators, LegacyResBodyValidators]): The validator enum name. Raises: - ValueError: Raised if the schema repo URL cannot be found. + RuntimeError: Raised if the schema directory cannot be found. Returns: Draft7Validator: The validator to use. """ schema_dir = os.getenv("SCHEMA_DIR") if schema_dir is None: - raise RuntimeError("No schema path suplied.") + raise RuntimeError("No schema path supplied.") - schema_path = os.path.join(schema_dir,validator_enum.value) + schema_path = os.path.join(schema_dir, validator_enum.value) try: with open(schema_path, "r") as f: @@ -82,25 +135,17 @@ def load_validator( return jsonschema.Draft7Validator(schema) -def body(body: Dict, validator_enum: BodyValidators) -> None: - """Validate the body of a request using the request-respone-schemas. - - Args: - body (Dict): The body object to validate. - validator_enum (BodyValidators): The enum name of the validator to use. - - Raises: - ValidationError: If the validation fails, or the validator could not - be obtained. - """ +def _validate_legacy( + body: Union[Dict, Response], + validator_enum: Union[LegacyReqBodyValidators, LegacyResBodyValidators], +) -> None: try: validator = load_validator(validator_enum) validator.validate(body) - return except jsonschema.exceptions.ValidationError as e: - error_thrown = SchemaErrorThrown( + error_thrown: Union[str, SchemaErrorThrown] = SchemaErrorThrown( message=e.message, schema_path=list(e.absolute_schema_path), instance_path=list(e.absolute_path), @@ -113,3 +158,49 @@ def body(body: Dict, validator_enum: BodyValidators) -> None: f"{validator_enum.name.lower()} schema.", error_thrown, ) + + +def _validate_openapi( + body: Union[Dict, List, Response], + validator_enum: Union[MuEdReqBodyValidators, MuEdResBodyValidators], +) -> None: + spec = _get_openapi_spec() + resolver = jsonschema.RefResolver(base_uri="", referrer=spec) + + schema_name = validator_enum.value + if schema_name == "EvaluateResponse": + schema = _EVALUATE_RESPONSE_SCHEMA + else: + schema = spec["components"]["schemas"][schema_name] + + try: + jsonschema.validate(body, schema, resolver=resolver) + return + + except jsonschema.exceptions.ValidationError as e: + error_thrown: str = e.message + except Exception as e: + error_thrown = str(e) + + raise ValidationError( + "Failed to validate body against the " + f"{validator_enum.name.lower()} schema.", + error_thrown, + ) + + +def body(body: Union[Dict, List, Response], validator_enum: BodyValidators) -> None: + """Validate the body of a request using the request-response schemas. + + Args: + body (Union[Dict, List, Response]): The body object to validate. + validator_enum (BodyValidators): The enum name of the validator to use. + + Raises: + ValidationError: If the validation fails, or the validator could not + be obtained. + """ + if isinstance(validator_enum, (MuEdReqBodyValidators, MuEdResBodyValidators)): + _validate_openapi(body, validator_enum) + else: + _validate_legacy(body, validator_enum)