diff --git a/handler.py b/handler.py index 7479c23..aa1b322 100755 --- a/handler.py +++ b/handler.py @@ -1,13 +1,19 @@ from evaluation_function_utils.errors import EvaluationException -from .tools import commands, docs, validate +from .tools import commands, docs, parse, validate from .tools.parse import ParseError from .tools.utils import ErrorResponse, HandlerResponse, JsonType, Response -from .tools.validate import ResBodyValidators, ValidationError +from .tools.validate import ( + LegacyReqBodyValidators, + LegacyResBodyValidators, + MuEdReqBodyValidators, + MuEdResBodyValidators, + ValidationError, +) -def handle_command(event: JsonType, command: str) -> HandlerResponse: - """Switch case for handling different command options. +def handle_legacy_command(event: JsonType, command: str) -> HandlerResponse: + """Switch case for handling different command options using legacy schemas. Args: event (JsonType): The AWS Lambda event recieved by the handler. @@ -23,42 +29,98 @@ def handle_command(event: JsonType, command: str) -> HandlerResponse: elif command == "docs-user": return docs.user() - elif command in ("eval", "grade"): - response = commands.evaluate(event) - validator = ResBodyValidators.EVALUATION + body = parse.body(event) + response: Response + validator: LegacyResBodyValidators + + if command in ("eval", "grade"): + validate.body(body, LegacyReqBodyValidators.EVALUATION) + response = commands.evaluate(body) + validator = LegacyResBodyValidators.EVALUATION elif command == "preview": - response = commands.preview(event) - validator = ResBodyValidators.PREVIEW + validate.body(body, LegacyReqBodyValidators.PREVIEW) + response = commands.preview(body) + validator = LegacyResBodyValidators.PREVIEW elif command == "healthcheck": response = commands.healthcheck() - validator = ResBodyValidators.HEALTHCHECK + validator = LegacyResBodyValidators.HEALTHCHECK + else: response = Response( error=ErrorResponse(message=f"Unknown command '{command}'.") ) - validator = ResBodyValidators.EVALUATION + validator = LegacyResBodyValidators.EVALUATION validate.body(response, validator) return response -def handler(event: JsonType, _: JsonType = {}) -> HandlerResponse: +def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse: + """Switch case for handling different command options using muEd schemas. + + Args: + event (JsonType): The AWS Lambda event recieved by the handler. + command (str): The name of the function to invoke. + + Returns: + HandlerResponse: The response object returned by the handler. + """ + if command == "eval": + body = parse.body(event) + validate.body(body, MuEdReqBodyValidators.EVALUATION) + response = commands.evaluate_muEd(body) + validate.body(response, MuEdResBodyValidators.EVALUATION) + + elif command == "healthcheck": + response = commands.healthcheck() + + else: + response = Response( + error=ErrorResponse(message=f"Unknown command '{command}'.") + ) + + return response + + +def handler(event: JsonType, _=None) -> HandlerResponse: """Main function invoked by AWS Lambda to handle incoming requests. Args: event (JsonType): The AWS Lambda event received by the gateway. + _ (JsonType): The AWS Lambda context object (unused). Returns: HandlerResponse: The response to return back to the requestor. """ - headers = event.get("headers", dict()) - command = headers.get("command", "eval") + if _ is None: + _ = {} + + # Normalise path: prefer rawPath (HTTP API v2) over path (REST API v1). + # API Gateway v1 includes the full resource prefix in `path` + # (e.g. /compareExpressions-staging/evaluate), so we match on suffix. + # API Gateway v2 uses `rawPath` at the top level; `path` may be absent. + raw_path = event.get("rawPath") or event.get("path", "/") + if raw_path.endswith("/evaluate/health"): + path = "/evaluate/health" + elif raw_path.endswith("/evaluate"): + path = "/evaluate" + else: + path = raw_path try: - return handle_command(event, command) + if path == "/evaluate": + return handle_muEd_command(event, "eval") + + elif path == "/evaluate/health": + return handle_muEd_command(event, "healthcheck") + + else: + headers = event.get("headers", dict()) + command = headers.get("command", "eval") + return handle_legacy_command(event, command) except (ParseError, ValidationError) as e: error = ErrorResponse(message=e.message, detail=e.error_thrown) diff --git a/requirements.txt b/requirements.txt index 5f5a050..e42e039 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ jsonschema +pyyaml requests evaluation-function-utils typing_extensions diff --git a/schemas/__init__.py b/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/schemas/request.json b/schemas/legacy/request.json similarity index 100% rename from schemas/request.json rename to schemas/legacy/request.json diff --git a/schemas/request/eval.json b/schemas/legacy/request/eval.json similarity index 100% rename from schemas/request/eval.json rename to schemas/legacy/request/eval.json diff --git a/schemas/request/preview.json b/schemas/legacy/request/preview.json similarity index 100% rename from schemas/request/preview.json rename to schemas/legacy/request/preview.json diff --git a/schemas/response.json b/schemas/legacy/response.json similarity index 100% rename from schemas/response.json rename to schemas/legacy/response.json diff --git a/schemas/response/eval.json b/schemas/legacy/response/eval.json similarity index 100% rename from schemas/response/eval.json rename to schemas/legacy/response/eval.json diff --git a/schemas/response/healthcheck.json b/schemas/legacy/response/healthcheck.json similarity index 100% rename from schemas/response/healthcheck.json rename to schemas/legacy/response/healthcheck.json diff --git a/schemas/response/preview.json b/schemas/legacy/response/preview.json similarity index 100% rename from schemas/response/preview.json rename to schemas/legacy/response/preview.json diff --git a/schemas/responsev2.json b/schemas/legacy/responsev2.json similarity index 100% rename from schemas/responsev2.json rename to schemas/legacy/responsev2.json diff --git a/schemas/muEd/__init__.py b/schemas/muEd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/schemas/muEd/openapi-v0_1_0.yml b/schemas/muEd/openapi-v0_1_0.yml new file mode 100644 index 0000000..28a8377 --- /dev/null +++ b/schemas/muEd/openapi-v0_1_0.yml @@ -0,0 +1,1917 @@ +openapi: 3.1.0 +info: + title: µEd API - Educational Microservices + version: 0.1.0 + contact: + name: µEd API Maintainers + description: | + The µEd API ("microservices for education") is a specification for interoperable educational services.

Currently defined endpoints:
- **Evaluate Task**: automatic feedback and grading for student submissions.
- **Chat**: conversational interactions around tasks, submissions, or general + learning questions. +tags: + - name: evaluate + description: Endpoints for evaluating student submissions and generating feedback. + - name: chat + description: Conversational endpoints for educational dialogue. +paths: + /evaluate: + post: + summary: Evaluate a submission and generate feedback + operationId: evaluateSubmission + description: | + Generates a list of feedback items for a given student submission. The request can optionally include the task context, user information, criteria to evaluate on, pre-submission feedback options, configuration, and a callback URL for asynchronous result delivery. + tags: + - evaluate + parameters: + - $ref: '#/components/parameters/Authorization' + - $ref: '#/components/parameters/X-Request-Id' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateRequest' + examples: + simpleTextSubmission: + summary: Simple text submission without user or criteria + value: + submission: + submissionId: sub-123 + taskId: task-42 + type: TEXT + format: plain + content: + text: Explain what polymorphism is in object-oriented programming. + submittedAt: '2025-12-16T09:30:00Z' + version: 1 + configuration: null + preSubmissionFeedbackExample: + summary: Pre-submission feedback (non-final) + value: + submission: + submissionId: sub-777 + taskId: task-42 + type: TEXT + format: plain + content: + text: My short answer... + preSubmissionFeedback: + enabled: true + configuration: + llm: + model: gpt-5.2 + temperature: 0.4 + asyncCallbackExample: + summary: Asynchronous processing via callback URL + value: + submission: + submissionId: sub-async-001 + taskId: task-42 + type: TEXT + format: plain + content: + text: Detailed essay answer that may require longer processing. + submittedAt: '2025-12-16T09:45:00Z' + version: 1 + callbackUrl: https://learning-platform.example.com/hooks/evaluate-result + withTaskAndExtras: + summary: With task context, user, criteria and configuration + value: + task: + taskId: task-12 + title: Explain polymorphism + content: + text: Define polymorphism and give at least one example in Java. + learningObjectives: + - Explain the concept of polymorphism. + - Provide an example of subtype polymorphism in Java. + referenceSolution: + text: | + Polymorphism allows the same method call to result in different behavior depending on the object's runtime type. For example, a variable of type Shape can reference a Circle or Rectangle, and calling draw() will invoke the appropriate implementation. + context: + constraints: Answer in 3-6 sentences. + language: en + submission: + submissionId: sub-456 + taskId: task-12 + type: TEXT + format: plain + content: + text: | + Polymorphism means that an object can take many forms, for example subclasses implementing methods differently. + submittedAt: '2025-12-16T10:00:00Z' + version: 2 + user: + userId: user-789 + type: LEARNER + detailPreference: DETAILED + tonePreference: FRIENDLY + languagePreference: en + criteria: + - criterionId: crit-1 + name: Correctness + context: The explanation of polymorphism is conceptually correct. + maxPoints: 10 + - criterionId: crit-2 + name: Clarity + context: The explanation is clear, well-structured, and easy to understand. + maxPoints: 5 + preSubmissionFeedback: + enabled: false + configuration: + llm: + model: gpt-5.2 + temperature: 0.2 + maxTokens: 800 + credentials: + type: JWT + key: Some-Key + enforceRubricStrictness: true + codeSubmissionExample: + summary: Code submission (Python) + value: + submission: + submissionId: sub-code-001 + taskId: task-python-101 + type: CODE + format: python + content: + code: | + def fibonacci(n): + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) + + # Test the function + for i in range(10): + print(fibonacci(i)) + submittedAt: '2025-12-16T11:00:00Z' + version: 1 + codeMultiFileExample: + summary: Code submission with multiple files + value: + submission: + submissionId: sub-code-002 + taskId: task-java-201 + type: CODE + format: java + content: + files: + - path: src/Main.java + content: | + public class Main { + public static void main(String[] args) { + Calculator calc = new Calculator(); + System.out.println(calc.add(2, 3)); + } + } + - path: src/Calculator.java + content: | + public class Calculator { + public int add(int a, int b) { + return a + b; + } + } + entryPoint: src/Main.java + submittedAt: '2025-12-16T11:30:00Z' + version: 1 + codeSympyExample: + summary: Math submission (SymPy/Python) + value: + submission: + submissionId: sub-math-002 + taskId: task-algebra-101 + type: CODE + format: sympy + content: + expression: solve(x**2 - 4, x) + imports: + - from sympy import symbols, solve + - x = symbols('x') + submittedAt: '2025-12-16T12:30:00Z' + version: 1 + mathInlineLatexExample: + summary: Inline math submission (Inline LaTeX) + value: + submission: + submissionId: sub-math-001 + taskId: task-calculus-101 + type: MATH + format: latex + content: + expression: \int_{0}^{\infty} e^{-x^2} dx = \frac{\sqrt{\pi}}{2} + submittedAt: '2025-12-16T12:00:00Z' + version: 1 + mathMathMLExample: + summary: Math submission (MathML) + value: + submission: + submissionId: sub-math-003 + taskId: task-geometry-101 + type: MATH + format: mathml + content: + expression: | + + + a + = + + + b2 + + + c2 + + + + + submittedAt: '2025-12-16T13:00:00Z' + version: 1 + modelUmlExample: + summary: Model submission (UML class diagram - PlantUML) + value: + submission: + submissionId: sub-model-001 + taskId: task-oop-design-101 + type: MODEL + format: uml + content: + model: | + @startuml + abstract class Animal { + +name: String + +speak(): String + } + + class Dog extends Animal { + +speak(): String + } + + class Cat extends Animal { + +speak(): String + } + @enduml + notation: plantuml + diagramType: class + submittedAt: '2025-12-16T14:00:00Z' + version: 1 + modelErExample: + summary: Model submission (ER diagram - JSON structure) + value: + submission: + submissionId: sub-model-002 + taskId: task-database-101 + type: MODEL + format: er + content: + model: + entities: + - name: Student + attributes: + - name: student_id + type: INTEGER + primaryKey: true + - name: name + type: VARCHAR(100) + - name: email + type: VARCHAR(255) + - name: Course + attributes: + - name: course_id + type: INTEGER + primaryKey: true + - name: title + type: VARCHAR(200) + relationships: + - name: enrolls_in + from: Student + to: Course + cardinality: many-to-many + notation: json + submittedAt: '2025-12-16T14:30:00Z' + version: 1 + modelBpmnExample: + summary: Model submission (BPMN process) + value: + submission: + submissionId: sub-model-003 + taskId: task-process-101 + type: MODEL + format: bpmn + content: + model: | + + + + + + + + + + + notation: bpmn-xml + submittedAt: '2025-12-16T15:00:00Z' + version: 1 + textMarkdownExample: + summary: Text submission (Markdown with formatting) + value: + submission: + submissionId: sub-text-002 + taskId: task-essay-101 + type: TEXT + format: markdown + content: + markdown: | + # Introduction to Polymorphism + + Polymorphism is a fundamental concept in **object-oriented programming** that allows objects to be treated as instances of their parent class. + + ## Key Points + + 1. **Subtype polymorphism**: Different classes can be used interchangeably + 2. **Method overriding**: Subclasses provide specific implementations + submittedAt: '2025-12-16T15:30:00Z' + version: 1 + responses: + '200': + description: Successfully generated feedback. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Feedback' + examples: + exampleResponse: + summary: Example feedback response + value: + - feedbackId: fb-1 + title: Clarify your definition + message: Your explanation of polymorphism is generally correct, but it would help to distinguish between subtype polymorphism and parametric polymorphism. + suggestedAction: Add one or two concrete examples of polymorphism in Java, e.g., method overriding. + awardedPoints: 2.5 + criterion: + criterionId: crit-1 + name: Correctness + context: The solution produces correct results for the specified problem. + maxPoints: 10 + target: + artefactType: TEXT + format: plain + locator: + type: span + startIndex: 0 + endIndex: 120 + - feedbackId: fb-2 + title: Overall structure + message: The overall structure of your answer is clear and easy to follow. + '202': + $ref: '#/components/responses/202-Accepted' + '400': + $ref: '#/components/responses/400-BadRequest' + '403': + $ref: '#/components/responses/403-Forbidden' + '500': + $ref: '#/components/responses/500-InternalError' + '501': + $ref: '#/components/responses/501-NotImplemented' + /evaluate/health: + get: + summary: Health and capabilities of the evaluate service + operationId: getEvaluateHealth + description: | + Returns health information and capabilities of the evaluate service. Clients can use this endpoint to discover whether the service supports optional features such as pre-submission feedback, formative feedback, and summative feedback. + tags: + - evaluate + parameters: + - $ref: '#/components/parameters/X-Request-Id' + responses: + '200': + description: Evaluate service is reachable and reporting capabilities. + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateHealthResponse' + examples: + exampleHealth: + summary: Example healthy service with capabilities + value: + status: OK + message: Service healthy + version: 1.0.0 + capabilities: + supportsEvaluate: true + supportsPreSubmissionFeedback: false + supportsFormativeFeedback: true + supportsSummativeFeedback: true + supportsDataPolicy: PARTIAL + supportedArtefactProfiles: + - type: TEXT + supportedFormats: + - plain + - markdown + - type: CODE + supportedFormats: + - python + - java + - javascript + - type: MATH + supportedFormats: + - latex + - mathml + supportedLanguages: + - en + - de + '501': + description: The server does not implement the health endpoint for evaluate. + $ref: '#/components/responses/501-NotImplemented' + '503': + $ref: '#/components/responses/503-ServiceUnavailable' + /chat: + post: + summary: Chat about tasks, submissions, or learning topics + operationId: chat + description: | + Conversational endpoint for educational chat use cases. A conversation can be grounded in a specific course, task or submission and may use user information to adapt tone and detail. Typical use cases include: asking follow-up questions on feedback, requesting hints, or clarifying concepts. + tags: + - chat + parameters: + - $ref: '#/components/parameters/Authorization' + - $ref: '#/components/parameters/X-Request-Id' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ChatRequest' + examples: + minimalChat: + summary: Minimal chat request + value: + messages: + - role: USER + content: Can you explain polymorphism? + configuration: null + chatWithLlmConfig: + summary: Chat request with optional LLM configuration + value: + conversationId: conv-1001 + user: + userId: user-456 + type: LEARNER + detailPreference: MEDIUM + tonePreference: FRIENDLY + languagePreference: en + messages: + - role: USER + content: Give me a hint for my answer about polymorphism. + context: + task: + taskId: task-12 + title: Explain polymorphism + content: + text: Define polymorphism and give at least one example in Java. + configuration: + type: Java Assistant + llm: + model: gpt-5.2 + temperature: 0.7 + stream: false + credentials: + type: JWT + key: Some-Key + chatWithContext: + summary: Chat request with complex educational context + value: + messages: + - role: USER + content: What should I do for this part? + user: + userId: user-321 + type: LEARNER + detailPreference: DETAILED + tonePreference: NEUTRAL + languagePreference: en + taskProgress: + currentQuestionId: question-321 + timeSpentOnQuestion: 30 minutes + currentPart: + partId: part-1 + timeSpentOnPart: 10 minutes + submission: + type: TEXT + content: + text: outputs= ["Woof!", "Meow!"] + feedback: + - feedbackId: fb-101 + message: Incomplete answer, explain why these outputs occur. + context: + module: + moduleId: module-456 + title: Introduction to Object-Oriented Programming (OOP) + set: + setId: set-789 + title: Fundamentals + question: + questionId: question-321 + title: Understanding Polymorphism + content: | + Answer the questions for the following example of polymorphism in Python. + ```python class Animal: + def speak(self): + pass + + class Dog(Animal): + def speak(self): + return "Woof!" + + class Cat(Animal): + def speak(self): + return "Meow!" + + animals = [Dog(), Cat()] for animal in animals: + print(animal.speak()) + ``` + estimatedTime: 15-25 minutes + parts: + - partId: part-1 + content: | + Looking at the code example, identify which method is being overridden and explain how this demonstrates polymorphism. What output would the code produce and why? + - partId: part-2 + content: | + Write a new class called `Bird` that inherits from `Animal` and overrides the `speak()` method to return "Tweet!". Then add an instance of `Bird` to the animals list. + referenceSolution: + code: | + class Bird(Animal): + def speak(self): + return "Tweet!" + + animals = [Dog(), Cat(), Bird()] + for animal in animals: + print(animal.speak()) + configuration: + type: CS Assistant + llm: + model: gpt-5.2 + temperature: 0.5 + responses: + '200': + description: Successful chat response. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ChatResponse' + examples: + minimalChatResponse: + summary: Minimal chat response + value: + output: + role: ASSISTANT + content: | + Polymorphism is a core concept in object-oriented programming that allows objects of different classes to be treated as objects of a common superclass. It enables a single interface to represent different underlying forms (data types). For example, in Java, you can have a superclass `Animal` with a method `speak()`, and subclasses like `Dog` and `Cat` that provide their own implementations of `speak()`. When you call `speak()` on an `Animal` reference, the appropriate subclass method is invoked based on the actual object type at runtime. + metadata: null + chatWithLlmConfigResponse: + summary: Chat response with LLM configuration + value: + output: + role: ASSISTANT + content: | + Here's a hint for your answer about polymorphism: Focus on how polymorphism allows methods to do different things based on the object that it is acting upon, even when accessed through a common interface. You might want to mention method overriding and how it enables this behavior in object-oriented programming. + metadata: + responseTimeMs: 1800 + type: Java Assistant + createdAt: '2025-12-10T11:15:00Z' + llm: + model: gpt-5.2 + temperature: 0.7 + outputTokens: 78 + chatWithContextResponse: + summary: Chat response with context + value: + output: + role: ASSISTANT + content: | + In the provided code example, the `speak()` method is being overridden in the `Dog` and `Cat` subclasses of the `Animal` superclass. This demonstrates polymorphism because when we call `speak()` on each animal in the `animals` list, the method that gets executed depends on the actual object type (either `Dog` or `Cat`), not the reference type (`Animal`). The output of the code would be: + ``` Woof! Meow! ``` + This happens because each subclass provides its own implementation of the `speak()` method, and at runtime, the correct method is called based on the object's type. + metadata: + responseTimeMs: 2000 + type: CS Assistant + createdAt: '2025-12-10T11:15:00Z' + llm: + model: gpt-5.2 + temperature: 0.5 + outputTokens: 143 + '400': + $ref: '#/components/responses/400-BadRequest-2' + '403': + $ref: '#/components/responses/403-Forbidden' + '500': + $ref: '#/components/responses/500-InternalError-2' + '501': + $ref: '#/components/responses/501-NotImplemented-2' + /chat/health: + get: + summary: Health and capabilities of the chat service + operationId: getChatHealth + description: | + Returns health information and capabilities of the chat service. Clients can use this endpoint to discover whether the service supports optional features such as user preferences or streaming responses. + tags: + - chat + parameters: + - $ref: '#/components/parameters/X-Request-Id' + responses: + '200': + description: Chat service is reachable and reporting capabilities. + content: + application/json: + schema: + $ref: '#/components/schemas/ChatHealthResponse' + examples: + exampleHealth: + summary: Example healthy service with capabilities + value: + status: OK + statusMessage: Service healthy + version: 1.0.0 + capabilities: + supportsChat: true + supportsUserPreferences: true + supportsStreaming: true + supportsDataPolicy: NOT_SUPPORTED + supportedLanguages: + - en + - de + supportedModels: + - gpt-4o + - llama-3 + '501': + description: The server does not implement the health endpoint for chat. + $ref: '#/components/responses/501-NotImplemented-2' + '503': + $ref: '#/components/responses/503-ServiceUnavailable-2' +components: + parameters: + Authorization: + in: header + name: Authorization + schema: + type: string + required: false + description: Optional authorization header. + X-Request-Id: + in: header + name: X-Request-Id + description: Request id for tracing this request across services. + schema: + type: string + schemas: + Task: + type: object + description: | + Task context including the content, learning objectives, optional reference solution, optional context information, and optional metadata. + properties: + taskId: + type: string + description: Optional unique identifier for the task. + title: + type: string + description: Short title or label for the task. + content: + type: + - object + - 'null' + description: Optional content shown to the learner (structure is task-specific). + additionalProperties: true + context: + type: + - object + - 'null' + description: Optional educational context (e.g., course material). + additionalProperties: true + learningObjectives: + type: + - array + - 'null' + description: Optional list of learning objectives addressed by this task. + items: + type: string + referenceSolution: + type: + - object + - 'null' + description: Optional reference or example solution (structure is task-specific). + additionalProperties: true + metadata: + type: + - object + - 'null' + description: Optional metadata such as difficulty, topic, tags, etc. + additionalProperties: true + ArtefactType: + type: string + description: | + High-level type of artefact. Use the 'format' field to specify the exact format (e.g., programming language for CODE, notation for MATH). + enum: + - TEXT + - CODE + - MODEL + - MATH + - OTHER + Submission: + type: object + description: | + A student's submission for a task. The structure of 'content' is intentionally generic and task-specific. + required: + - type + - content + properties: + submissionId: + type: string + description: Optional unique identifier of the submission. + taskId: + type: + - string + - 'null' + description: Optional identifier of the task this submission belongs to. + type: + $ref: '#/components/schemas/ArtefactType' + format: + type: + - string + - 'null' + description: | + Optional format specifier providing additional detail about the artefact. For TEXT: plain_text, markdown, html, rich_text, etc. For CODE: programming language (e.g., python, java, javascript, wolfram, matlab). For MATH: latex, mathml, sympy, wolfram, asciimath, etc. For MODEL: uml, er, bpmn, petri_net, state_machine, etc. Use lowercase values and snakecase. Services should document which formats they support. + content: + type: object + additionalProperties: true + description: | + Logical representation of the submission content. The expected structure depends on the artefact type: - TEXT: { text: string } or { markdown: string } - CODE: { code: string } or { files: [{ path: string, content: string }], entryPoint?: string } - MATH: { expression: string } - MODEL: { model: string | object, notation?: string } + supplementaryContent: + type: object + additionalProperties: true + description: Optional additional content for the learner's submission. This could include workings for math tasks, or raw source code for compiled binaries. + submittedAt: + type: + - string + - 'null' + format: date-time + description: Optional timestamp when the submission was created. + version: + type: + - integer + - 'null' + format: int32 + description: Optional version number (e.g., resubmissions). + UserType: + type: string + description: Type of user interacting with the API. + enum: + - LEARNER + - TEACHER + - EDU_ADMIN + - SYS_ADMIN + - OTHER + Detail: + type: string + description: Level of detail preferred in responses or feedback. + enum: + - BRIEF + - MEDIUM + - DETAILED + Tone: + type: string + description: Preferred tone for responses or feedback. + enum: + - FORMAL + - NEUTRAL + - FRIENDLY + User: + type: object + description: User information including type and optional preferences influencing response tone, detail, and language. + required: + - type + additionalProperties: true + properties: + userId: + type: + - string + - 'null' + description: Optional unique identifier for the user. + type: + $ref: '#/components/schemas/UserType' + preference: + type: object + properties: + detail: + description: Optional preferred level of detail in responses. + $ref: '#/components/schemas/Detail' + tone: + description: Optional preferred tone for responses. + $ref: '#/components/schemas/Tone' + language: + type: + - string + - 'null' + description: Optional preferred language code following ISO 639 language codes (e.g., 'en', 'de'). + additionalProperties: true + taskProgress: + type: + - object + - 'null' + description: Optional information about the user's progress on this task/topic. + additionalProperties: true + NumericGrade: + title: Numeric Grade + type: object + required: + - min + - max + - value + properties: + min: + type: number + description: Minimum value for the numeric range + max: + type: number + description: Maximum value for the numeric range + value: + type: number + description: The actual rating value within the min-max range + LetterOnlyGrade: + title: Letter Only Grade + type: object + required: + - value + properties: + value: + type: string + enum: + - A + - B + - C + - D + - E + - F + - n/a + LetterPlusMinusGrade: + title: Letter +/- grades + type: object + required: + - value + properties: + value: + type: string + enum: + - A+ + - A + - A- + - B+ + - B + - B- + - C+ + - C + - C- + - D+ + - D + - D- + - E+ + - E + - E- + - F + OtherGrade: + title: Other + type: object + required: + - value + properties: + value: + type: string + description: Free-form string rating + Criterion: + type: object + description: A criterion used to assess one dimension of a submission. + required: + - name + properties: + criterionId: + type: string + description: Optional unique identifier of the criterion. + name: + type: string + description: Human-readable name of the criterion. + context: + type: + - string + - object + - 'null' + description: Optional additional context about how to apply this criterion. + additionalProperties: true + gradeConfig: + oneOf: + - $ref: '#/components/schemas/NumericGrade' + - $ref: '#/components/schemas/LetterOnlyGrade' + - $ref: '#/components/schemas/LetterPlusMinusGrade' + - $ref: '#/components/schemas/OtherGrade' + description: Optional configuration for grades for this criterion. + PreSubmissionFeedback: + type: object + description: Optional configuration for pre-submission feedback runs. + required: + - enabled + additionalProperties: true + properties: + enabled: + type: boolean + description: Indicates whether pre-submission feedback is requested. + LLMConfiguration: + type: object + description: | + Optional configuration for an LLM provider. All fields are optional and provider-specific values may be included via additional properties. + additionalProperties: true + properties: + model: + type: + - string + - 'null' + description: Optional model identifier (e.g., 'gpt-4o', 'llama-3'). + temperature: + type: + - number + - 'null' + description: Optional sampling temperature. + maxTokens: + type: + - integer + - 'null' + description: Optional maximum number of tokens to generate. + stream: + type: + - boolean + - 'null' + description: Optional flag indicating whether streaming responses are requested. + credentials: + type: + - object + - 'null' + description: Optional credentials object to be supplied with time-based key via a proxy. + additionalProperties: true + Region: + type: string + description: | + Geographic regions using ISO 3166-1 alpha-2 country codes (e.g., US, GB, DE) or regional groupings (e.g., EEA, EU, APAC). + AnonymizationLevel: + type: string + description: Level of required anonymization. + enum: + - NONE + - PSEUDONYMIZED + - ANONYMIZED + - AGGREGATED + DataPolicy: + type: object + description: | + Declares what downstream services are allowed to do with data associated with this request: which legal regimes apply, what uses are permitted, how long data may be retained, where it may be processed, and what constraints apply (especially for children / sensitive data). + additionalProperties: true + properties: + legal: + type: + - object + - 'null' + description: Legal framework and authority governing this data. + properties: + applicableLaws: + type: array + description: One or more applicable legal regimes. + items: + type: string + enum: + - GDPR + - UK_GDPR + - EPRIVACY + - CCPA_CPRA + - COPPA + - FERPA + - PPRA + - PIPEDA + - LGPD + - POPIA + - APPI + - PIPL + - OTHER + legalBasis: + type: + - array + - 'null' + description: Optional but recommended legal basis for processing. + items: + type: string + enum: + - CONSENT + - CONTRACT + - LEGAL_OBLIGATION + - PUBLIC_TASK + - LEGITIMATE_INTERESTS + - VITAL_INTERESTS + - OTHER + jurisdiction: + type: + - object + - 'null' + description: Geographic constraints for data subjects and processing. + properties: + dataSubjectRegions: + type: + - array + - 'null' + description: Where the data subjects are located. + items: + $ref: '#/components/schemas/Region' + allowedProcessingRegions: + type: + - array + - 'null' + description: Where processing/storage is allowed. + items: + $ref: '#/components/schemas/Region' + disallowedProcessingRegions: + type: + - array + - 'null' + description: Explicit exclusions for processing regions. + items: + $ref: '#/components/schemas/Region' + dataSubject: + type: + - object + - 'null' + description: Information about who this data is about. + properties: + population: + type: + - string + - 'null' + description: Type of population this data concerns. + enum: + - STUDENT + - STAFF + - GUARDIAN + - MIXED + - OTHER + isChildData: + type: + - boolean + - 'null' + description: Indicates whether this data concerns children. If true, then minAge required. + minAge: + type: + - integer + - 'null' + description: Minimum age of data subjects, if relevant/known. + dataCategory: + type: + - object + - 'null' + description: Classification of the type of data. + properties: + classification: + type: + - string + - 'null' + description: Primary classification of the data. + enum: + - ANONYMOUS + - PSEUDONYMOUS + - PERSONAL + - EDUCATION_RECORD + - SENSITIVE + - OTHER + additionalProperties: true + retentionPermission: + type: + - array + - 'null' + description: What retention and secondary uses are permitted. + items: + type: string + enum: + - NEVER + - SECURITY + - LOGGING + - PRODUCT-IMPROVEMENT-NO-SHARE + - PRODUCT-IMPROVEMENT-SHARE-LIMITED + - RESEARCH-CONFIDENTIAL + - PUBLIC + - OTHER + retention: + type: + - object + - 'null' + description: How long data may be retained. + properties: + retentionPeriod: + type: + - string + - 'null' + description: Concrete retention period, following ISO 8601 standard. + deleteOnRequest: + type: + - boolean + - 'null' + description: Whether data must be deleted on user request. + legalHoldAllowed: + type: + - boolean + - 'null' + description: Whether legal holds are permitted on this data. + sharing: + type: + - object + - 'null' + description: Constraints on who can receive this data. + properties: + thirdPartySharing: + type: + - string + - 'null' + description: Third-party sharing policy. + enum: + - PROHIBITED + - ALLOWED + - ALLOWED-LIMITED + subprocessorsAllowed: + type: + - boolean + - 'null' + description: Whether subprocessors are allowed. + allowedRecipients: + type: + - array + - 'null' + description: Categories of allowed recipients. + items: + type: string + enum: + - CONTROLLER-ONLY + - INTERNAL-SERVICES + - NAMED-PARTNERS + - PUBLIC + deidentification: + type: + - object + - 'null' + description: Required deidentification for specific uses. + properties: + requiredForServiceImprovement: + $ref: '#/components/schemas/AnonymizationLevel' + requiredForResearch: + $ref: '#/components/schemas/AnonymizationLevel' + ExecutionPolicy: + type: object + description: | + Declares execution constraints that the server or client consuming this API should apply when triggering calls to external providers (such as LLMs). Covers queue management and response timeouts. + additionalProperties: true + properties: + priority: + type: + - string + - 'null' + description: Request priority for queue management when capacity is constrained. + enum: + - low + - normal + - high + - null + timeout: + type: + - integer + - 'null' + description: Maximum time in milliseconds to wait for a complete response before failing. + minimum: 1 + EvaluateRequest: + type: object + description: | + Input for task evaluate service. The submission is mandatory; task, user, criteria, pre-submission feedback options, and configuration, and callback URL are optional. + required: + - submission + properties: + task: + description: | + Optional task context that can include prompt content, learning objectives, a reference solution, and additional context metadata. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/Task' + submission: + $ref: '#/components/schemas/Submission' + user: + description: Optional user information including type and preferences that can influence response tone, detail, and language. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/User' + criteria: + type: + - array + - 'null' + description: Optional criteria used for evaluate. + items: + $ref: '#/components/schemas/Criterion' + preSubmissionFeedback: + description: | + Optional settings for pre-submission feedback (non-final). When enabled, the service should avoid committing or finalizing grades. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/PreSubmissionFeedback' + callbackUrl: + type: + - string + - 'null' + format: uri + description: | + Optional HTTPS callback URL for asynchronous processing. If provided, the service may return 202 Accepted immediately and deliver feedback results to this URL once processing is complete. + configuration: + description: | + Optional key-value configuration dictionary for provider-specific or experimental parameters. Not standardized. + type: + - object + - 'null' + additionalProperties: true + properties: + llm: + description: Optional LLM configuration used for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/LLMConfiguration' + dataPolicy: + description: Optional data policy governing how this request's data may be processed and retained. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/DataPolicy' + executionPolicy: + description: Optional execution constraints for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/ExecutionPolicy' + FeedbackTarget: + type: object + description: Optional reference to a specific part of the submission. + required: + - artefactType + properties: + artefactType: + $ref: '#/components/schemas/ArtefactType' + format: + type: + - string + - 'null' + description: | + Optional format specifier matching the submission format (e.g., python, latex). + locator: + type: + - object + - 'null' + description: | + Optional locator into the submission content. Standard locator types: - TEXT: { type: "span", startIndex: number, endIndex: number } - CODE: { type: "range", file?: string, startLine: number, endLine: number, startColumn?: number, endColumn?: number } - MATH: { type: "subexpression", path: string } (e.g., path to subexpression) - MODEL: { type: "element", elementId: string, elementType?: string } + additionalProperties: true + Feedback: + type: object + description: | + A single feedback item produced for the submission. It may include suggested actions, optional points, optional criterion linkage, and optional targeting into the submission. + properties: + feedbackId: + type: string + description: Unique identifier of the feedback item. + title: + type: + - string + - 'null' + description: Optional short label for this feedback item. + message: + type: + - string + - 'null' + description: Optional feedback text shown to the learner. + suggestedAction: + type: + - string + - 'null' + description: Optional suggestion for how to act on this feedback. + awardedPoints: + type: + - number + - 'null' + format: double + description: Optional points awarded for this feedback item. + criterion: + type: + - object + - 'null' + description: Optional criterion linked to this feedback item. + allOf: + - $ref: '#/components/schemas/Criterion' + target: + type: + - object + - 'null' + description: Optional target reference inside the submission. + allOf: + - $ref: '#/components/schemas/FeedbackTarget' + EvaluateAcceptedResponse: + type: object + description: Acknowledgement that evaluation was accepted for asynchronous processing. + required: + - status + - requestId + properties: + status: + type: string + enum: + - ACCEPTED + description: Indicates that the request has been accepted for asynchronous processing. + requestId: + type: string + description: Identifier to correlate this accepted request with callback delivery. + message: + type: + - string + - 'null' + description: Optional human-readable message about asynchronous processing. + ErrorResponse: + type: object + description: Standard error response returned by µEd API services. + required: + - title + properties: + title: + type: string + description: Short, human-readable error title. + message: + type: + - string + - 'null' + description: Optional human-readable error message. + code: + type: + - string + - 'null' + description: Optional application-specific error code. + trace: + type: + - string + - 'null' + description: Optional debug trace or stack trace (should be omitted in production by default). + details: + type: + - object + - 'null' + description: Optional provider-specific details for debugging or programmatic handling. + additionalProperties: true + HealthStatus: + type: string + description: Overall health status of the service. + enum: + - OK + - DEGRADED + - UNAVAILABLE + EvaluateRequirements: + type: object + description: | + Requirements for calling the evaluate endpoint, e.g. whether an Authorization header and/or LLM configuration or credentials (provided via a proxy, preferably time-based and/or signed tokens) are required. + additionalProperties: true + properties: + requiresAuthorizationHeader: + type: + - boolean + - 'null' + description: Optional flag indicating whether an Authorization header is required. + requiresLlmConfiguration: + type: + - boolean + - 'null' + description: Optional flag indicating whether configuration.llm must be provided. + requiresLlmCredentialProxy: + type: + - boolean + - 'null' + description: Optional flag indicating whether configuration.llm.credentials must be provided via a proxy. + DataPolicySupport: + type: string + enum: + - SUPPORTED + - NOT_SUPPORTED + - PARTIAL + description: Indicates whether the service supports data policy configuration. + ArtefactProfile: + type: object + description: | + Describes support for a specific artefact type and its formats. Used in health/capabilities responses to advertise what a service can handle. + required: + - type + properties: + type: + $ref: '#/components/schemas/ArtefactType' + supportedFormats: + type: + - array + - 'null' + description: | + List of supported formats for this artefact type. Use lowercase values. If null or empty, the service accepts any format for this type. + items: + type: string + examples: + - - plain + - markdown + - - python + - java + - javascript + - wolfram + - matlab + - - latex + - mathml + contentSchema: + type: + - object + - 'null' + description: | + Optional JSON Schema describing the expected content structure for this artefact type. Allows services to advertise their exact requirements. + additionalProperties: true + locatorSchema: + type: + - object + - 'null' + description: | + Optional JSON Schema describing the locator structure used for feedback targeting within this artefact type. + additionalProperties: true + EvaluateCapabilities: + type: object + description: Capabilities of the evaluate service. + required: + - supportsEvaluate + - supportsPreSubmissionFeedback + - supportsFormativeFeedback + - supportsSummativeFeedback + - supportsDataPolicy + additionalProperties: true + properties: + supportsEvaluate: + type: boolean + description: Indicates whether the /evaluate endpoint is implemented and usable. + supportsPreSubmissionFeedback: + type: boolean + description: Indicates whether the service supports pre-submission feedback runs. + supportsFormativeFeedback: + type: boolean + description: Indicates whether the service supports qualitative feedback without points. + supportsSummativeFeedback: + type: boolean + description: Indicates whether the service supports feedback with points / grading signals. + supportsDataPolicy: + $ref: '#/components/schemas/DataPolicySupport' + supportedArtefactProfiles: + type: + - array + - 'null' + description: | + Optional list of supported artefact profiles. Each profile specifies an artefact type and the formats supported for that type. + items: + $ref: '#/components/schemas/ArtefactProfile' + supportedLanguages: + type: + - array + - 'null' + description: Optional list of supported language codes (e.g., 'en', 'de'). + items: + type: string + EvaluateHealthResponse: + type: object + description: Health status and capabilities of the evaluate service. + required: + - status + - capabilities + properties: + status: + $ref: '#/components/schemas/HealthStatus' + message: + type: + - string + - 'null' + description: Optional human-readable status message. + version: + type: + - string + - 'null' + description: Optional version of the evaluate service implementation. + requirements: + type: + - object + - 'null' + description: Optional requirements clients must satisfy to use this service. + allOf: + - $ref: '#/components/schemas/EvaluateRequirements' + capabilities: + $ref: '#/components/schemas/EvaluateCapabilities' + Message: + type: object + required: + - role + - content + properties: + role: + type: string + enum: + - USER + - ASSISTANT + - SYSTEM + - TOOL + content: + type: string + ChatRequest: + type: object + description: Request body for the chat endpoint. + required: + - messages + properties: + messages: + type: array + description: List of messages in the conversation (including history). + items: + $ref: '#/components/schemas/Message' + conversationId: + type: + - string + - 'null' + description: Optional identifier for the conversation session. + user: + description: Optional user information to adapt the chat style and level. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/User' + context: + type: + - object + - 'null' + description: Optional educational context (e.g., course material, task context). + additionalProperties: true + configuration: + type: + - object + - 'null' + description: Optional configuration for the model(s). + additionalProperties: true + properties: + type: + type: + - string + - 'null' + description: Optional type for the chatbot or chat model. + llm: + description: Optional LLM configuration used for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/LLMConfiguration' + dataPolicy: + description: Optional data policy governing how this request's data may be processed and retained. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/DataPolicy' + executionPolicy: + description: Optional execution constraints for this request. + type: + - object + - 'null' + allOf: + - $ref: '#/components/schemas/ExecutionPolicy' + ChatResponse: + type: object + description: Response body for the chat endpoint. + required: + - output + properties: + output: + $ref: '#/components/schemas/Message' + description: The generated assistant response. + metadata: + type: + - object + - 'null' + description: Optional metadata about response generation. + additionalProperties: true + ChatCapabilities: + type: object + description: Capabilities of the chat service. + required: + - supportsChat + - supportsDataPolicy + additionalProperties: true + properties: + supportsChat: + type: boolean + description: Indicates whether the /chat endpoint is implemented and usable. + supportsUserPreferences: + type: boolean + description: Indicates whether the service supports adapting to user preferences. + supportsStreaming: + type: boolean + description: Indicates whether the service supports streaming responses. + supportsDataPolicy: + $ref: '#/components/schemas/DataPolicySupport' + supportedLanguages: + type: + - array + - 'null' + description: Optional list of supported language codes. + items: + type: string + supportedModels: + type: + - array + - 'null' + description: Optional list of supported models. + items: + type: string + ChatHealthResponse: + type: object + description: Health status and capabilities of the chat service. + required: + - status + - capabilities + properties: + status: + $ref: '#/components/schemas/HealthStatus' + statusMessage: + type: + - string + - 'null' + description: Optional human-readable status message. + version: + type: + - string + - 'null' + description: Optional version of the chat service implementation. + capabilities: + $ref: '#/components/schemas/ChatCapabilities' + responses: + 202-Accepted: + description: Request accepted for asynchronous evaluation processing. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateAcceptedResponse' + examples: + asyncAccepted: + summary: Example accepted async request + value: + status: ACCEPTED + requestId: req-7c193f38 + message: Evaluation queued. Results will be sent to callbackUrl. + 400-BadRequest: + description: Invalid request (e.g. missing content or invalid schema). + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + validationError: + summary: Example validation error + value: + title: Invalid request + message: submission.content must not be empty. + code: VALIDATION_ERROR + trace: null + details: + field: submission.content + 403-Forbidden: + description: Forbidden (e.g. insufficient permissions or access denied). + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + permissionError: + summary: Example permission error + value: + title: Forbidden + message: You do not have permission to access this resource. + code: PERMISSION_DENIED + trace: null + details: + resource: submission + required_permission: write + 500-InternalError: + description: Internal server error. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + internalError: + summary: Example internal error + value: + title: Internal server error + message: Unexpected failure while generating feedback. + code: INTERNAL_ERROR + trace: 'java.lang.RuntimeException: ... (stack trace omitted)' + details: + subsystem: llm-provider + 501-NotImplemented: + description: | + The server does not support the evaluate method. This allows service providers to implement only subsets of the µEd API. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + notImplemented: + summary: Example not implemented error + value: + title: Not implemented + message: This service does not implement /evaluate. + code: NOT_IMPLEMENTED + trace: null + details: null + 503-ServiceUnavailable: + description: Service is currently unavailable or unhealthy. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + serviceUnavailable: + summary: Example unavailable error + value: + title: Service unavailable + message: Evaluate service is currently unavailable. + code: SERVICE_UNAVAILABLE + trace: null + details: + reason: Database connection failed + unhealthy: + summary: Example degraded / unavailable service + value: + title: Service unhealthy + status: UNAVAILABLE + message: Database connection failed + version: 1.0.0 + capabilities: + supportsEvaluate: false + supportsPreSubmissionFeedback: false + supportsFormativeFeedback: false + supportsSummativeFeedback: false + supportsDataPolicy: NOT_SUPPORTED + supportedArtefactProfiles: [] + supportedLanguages: [] + 400-BadRequest-2: + description: Invalid request (e.g. missing content or invalid schema). + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + invalidChatRequest: + summary: Example invalid chat request + value: + title: Invalid request + message: messages must contain at least one item. + code: VALIDATION_ERROR + trace: null + details: + field: messages + 500-InternalError-2: + description: Internal server error. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + modelProviderError: + summary: Example model provider error + value: + title: Model provider error + message: LLM provider returned an error while generating a response. + code: LLM_PROVIDER_ERROR + trace: null + details: + provider: openai + 501-NotImplemented-2: + description: | + The server does not support the chat method. This allows service providers to implement only subsets of the µEd API. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + notImplemented: + summary: Example not implemented error + value: + title: Not implemented + message: This service does not implement /chat. + code: NOT_IMPLEMENTED + trace: null + details: null + 503-ServiceUnavailable-2: + description: Service is currently unavailable or unhealthy. + headers: + X-Request-Id: + description: Request id for tracing this request across services. + schema: + type: string + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + serviceUnavailable: + summary: Example unavailable error + value: + title: Service unavailable + message: Chat service is currently unavailable. + code: SERVICE_UNAVAILABLE + trace: null + details: + reason: LLM provider connection failed + serviceUnhealthy: + summary: Example degraded / unavailable service + value: + title: Service unhealthy + message: LLM provider connection failed + code: SERVICE_UNHEALTHY + version: 1.0.0 + capabilities: + supportsChat: false + supportsUserPreferences: false + supportsStreaming: false + supportsDataPolicy: NOT_SUPPORTED + supportedLanguages: [] + supportedModels: [] diff --git a/scripts/generate_mued_types.py b/scripts/generate_mued_types.py new file mode 100644 index 0000000..5599803 --- /dev/null +++ b/scripts/generate_mued_types.py @@ -0,0 +1,43 @@ +import os +import subprocess +import sys +from pathlib import Path + +# --- env vars (both required, no defaults — fail fast like the TS script) --- +version = os.environ.get("MUED_SPEC_VERSION") +if not version: + raise SystemExit("MUED_SPEC_VERSION is not set") + +spec = os.environ.get("MUED_SPEC_URL") +if not spec: + raise SystemExit("MUED_SPEC_URL is not set") + +if not spec.startswith("https://") and not Path(spec).exists(): + raise SystemExit(f"Spec file not found: {spec}") + +# --- output: versioned filename to mirror mued-api-{version}.ts convention --- +out = Path(__file__).parent.parent / "schemas" / "muEd" / f"mued_api_{version}.py" + +print(f"Generating mued types from: {spec}") +subprocess.run( + [ + "datamodel-codegen", + "--input", spec, + "--input-file-type", "openapi", + "--output", str(out), + "--output-model-type", "pydantic_v2.BaseModel", + "--use-annotated", + "--use-standard-collections", + "--target-python-version", "3.8", + "--extra-fields", "allow", + ], + check=True, +) + +# Prepend generated header (mirrors ts-nocheck prepend in TS script) +content = out.read_text() +header = f"# generated by generate_mued_types.py — spec version {version}\n" +if not content.startswith("# generated"): + out.write_text(header + content) + +print(f"Generated: {out}") diff --git a/tests/commands.py b/tests/commands.py deleted file mode 100644 index 59d445b..0000000 --- a/tests/commands.py +++ /dev/null @@ -1,381 +0,0 @@ -import unittest -from typing import Any - -from ..tools import commands, parse, validate -from ..tools.utils import JsonType - - -def evaluation_function( - response: Any, answer: Any, params: JsonType -) -> JsonType: - if params.get("raise", False): - raise ValueError("raised") - - force_true = params.get("force", False) - - return {"is_correct": (response == answer) or force_true} - - -class TestCommandsModule(unittest.TestCase): - def __init__(self, methodName: str = "runTest") -> None: - super().__init__(methodName) - - def setUp(self) -> None: - commands.evaluation_function = evaluation_function - return super().setUp() - - def tearDown(self) -> None: - commands.evaluation_function = None - return super().tearDown() - - def test_valid_eval_command(self): - event = { - "body": {"response": "hello", "answer": "world", "params": {}} - } - response = commands.evaluate(event) - - self.assertIn("result", response) - self.assertIn("is_correct", response["result"]) # type: ignore - - def test_invalid_eval_args_raises_parse_error(self): - event = {"headers": "any", "other": "params"} - - with self.assertRaises(parse.ParseError) as e: - commands.evaluate(event) - - self.assertEqual( - e.exception.message, "No data supplied in request body." - ) - - def test_invalid_eval_schema_raises_validation_error(self): - event = {"body": {"response": "hello", "params": {}}} - - with self.assertRaises(validate.ValidationError) as e: - commands.evaluate(event) - - self.assertEqual( - e.exception.message, - "Failed to validate body against the evaluation schema.", - ) - - def test_single_feedback_case(self): - event = { - "body": { - "response": "hello", - "answer": "hello", - "params": { - "cases": [ - { - "answer": "other", - "feedback": "should be 'other'.", - "mark": 0, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertTrue(result["is_correct"]) - self.assertNotIn("matched_case", result) - self.assertNotIn("feedback", result) - - def test_single_feedback_case_match(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_case_warning_data_structure(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "feedback": "should be 'hello'.", - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertIn("warnings", result) - warning = result["warnings"].pop() # type: ignore - - self.assertDictEqual( - warning, - {"case": 0, "message": "Missing answer/feedback field"}, - ) - - def test_multiple_feedback_cases_single_match(self): - event = { - "body": { - "response": "yes", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - }, - { - "answer": "yes", - "feedback": "should be 'yes'.", - }, - { - "answer": "no", - "feedback": "should be 'no'.", - }, - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 1) - self.assertEqual(result["feedback"], "should be 'yes'.") - - def test_multiple_feedback_cases_multiple_matches(self): - event = { - "body": { - "response": "yes", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "params": {"force": True}, - }, - { - "answer": "yes", - "feedback": "should be 'yes'.", - }, - { - "answer": "no", - "feedback": "should be 'no'.", - }, - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_case_params_overwrite_eval_params(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "force": True, - "cases": [ - { - "answer": "yes", - "feedback": "should be 'yes'.", - "params": {"force": False}, - } - ], - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertTrue(result["is_correct"]) - self.assertNotIn("matched_case", result) - self.assertNotIn("feedback", result) - - def test_invalid_case_entry_doesnt_raise_exception(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "params": {"raise": True}, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertIn("warnings", result) - warning = result["warnings"].pop() # type: ignore - - self.assertDictEqual( - warning, - { - "case": 0, - "message": "An exception was raised while executing " - "the evaluation function.", - "detail": "raised", - }, - ) - - def test_multiple_matched_cases_are_combined_and_warned(self): - event = { - "body": { - "response": "yes", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "params": {"force": True}, - }, - { - "answer": "yes", - "feedback": "should be 'yes'.", - }, - { - "answer": "no", - "feedback": "should be 'no'.", - }, - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertIn("warnings", result) - warning = result["warnings"].pop() # type: ignore - - self.assertDictEqual( - warning, - { - "message": "Cases 0, 1 were matched. " - "Only the first one's feedback was returned", - }, - ) - - def test_overriding_eval_feedback_to_correct_case(self): - event = { - "body": { - "response": "hello", - "answer": "world", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "mark": 1, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertTrue(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_overriding_eval_feedback_to_incorrect_case(self): - event = { - "body": { - "response": "hello", - "answer": "hello", - "params": { - "cases": [ - { - "answer": "hello", - "feedback": "should be 'hello'.", - "mark": 0, - } - ] - }, - } - } - - response = commands.evaluate(event) - result = response["result"] # type: ignore - - self.assertFalse(result["is_correct"]) - self.assertEqual(result["matched_case"], 0) - self.assertEqual(result["feedback"], "should be 'hello'.") - - def test_valid_preview_command(self): - event = {"body": {"response": "hello"}} - - response = commands.preview(event) - result = response["result"] # type: ignore - - self.assertEqual(result["preview"]["latex"], "hello") - - def test_invalid_preview_args_raises_parse_error(self): - event = {"headers": "any", "other": "params"} - - with self.assertRaises(parse.ParseError) as e: - commands.preview(event) - - self.assertEqual( - e.exception.message, "No data supplied in request body." - ) - - def test_invalid_preview_schema_raises_validation_error(self): - event = {"body": {"response": "hello", "answer": "hello"}} - - with self.assertRaises(validate.ValidationError) as e: - commands.preview(event) - - self.assertEqual( - e.exception.message, - "Failed to validate body against the preview schema.", - ) - - def test_healthcheck(self): - response = commands.healthcheck() - - self.assertIn("result", response) - result = response["result"] # type: ignore - - self.assertIn("tests_passed", result) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/commands_test.py b/tests/commands_test.py new file mode 100644 index 0000000..a9f249a --- /dev/null +++ b/tests/commands_test.py @@ -0,0 +1,370 @@ +import os +import unittest +from pathlib import Path +from typing import Any + +from ..handler import handler +from ..tools import commands +from ..tools.utils import JsonType + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") + + +def evaluation_function( + response: Any, answer: Any, params: JsonType +) -> JsonType: + if params.get("raise", False): + raise ValueError("raised") + + force_true = params.get("force", False) + + return {"is_correct": (response == answer) or force_true} + + +class TestCommandsModule(unittest.TestCase): + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + commands.evaluation_function = evaluation_function + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = None + return super().tearDown() + + def test_valid_eval_command(self): + body = {"response": "hello", "answer": "world", "params": {}} + response = commands.evaluate(body) + + self.assertIn("result", response) + self.assertIn("is_correct", response["result"]) # type: ignore + + def test_eval_response_latex_and_simplified_are_none_when_not_returned(self): + body = {"response": "hello", "answer": "world", "params": {}} + response = commands.evaluate(body) + + self.assertNotIn("response_latex", response["result"]) # type: ignore + self.assertNotIn("response_simplified", response["result"]) # type: ignore + + def test_eval_response_latex_and_simplified_passed_through_when_returned(self): + commands.evaluation_function = lambda r, a, p: { + "is_correct": True, + "response_latex": r"x + 1", + "response_simplified": "x + 1", + } + body = {"response": "x+1", "answer": "x+1", "params": {}} + response = commands.evaluate(body) + + self.assertEqual(response["result"]["response_latex"], r"x + 1") # type: ignore + self.assertEqual(response["result"]["response_simplified"], "x + 1") # type: ignore + + def test_invalid_eval_args_raises_parse_error(self): + event = {"headers": {"command": "eval"}, "other": "params"} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], "No data supplied in request body." # type: ignore + ) + + def test_invalid_eval_schema_raises_validation_error(self): + event = {"headers": {"command": "eval"}, "body": {"response": "hello", "params": {}}} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], # type: ignore + "Failed to validate body against the evaluation schema.", + ) + + def test_single_feedback_case(self): + body = { + "response": "hello", + "answer": "hello", + "params": { + "cases": [ + { + "answer": "other", + "feedback": "should be 'other'.", + "mark": 0, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertTrue(result["is_correct"]) + self.assertNotIn("matched_case", result) + self.assertNotIn("feedback", result) + + def test_single_feedback_case_match(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_case_warning_data_structure(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "feedback": "should be 'hello'.", + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertIn("warnings", result) + warning = result["warnings"].pop() # type: ignore + + self.assertDictEqual( + warning, + {"case": 0, "message": "Missing answer/feedback field"}, + ) + + def test_multiple_feedback_cases_single_match(self): + body = { + "response": "yes", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + }, + { + "answer": "yes", + "feedback": "should be 'yes'.", + }, + { + "answer": "no", + "feedback": "should be 'no'.", + }, + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 1) + self.assertEqual(result["feedback"], "should be 'yes'.") + + def test_multiple_feedback_cases_multiple_matches(self): + body = { + "response": "yes", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "params": {"force": True}, + }, + { + "answer": "yes", + "feedback": "should be 'yes'.", + }, + { + "answer": "no", + "feedback": "should be 'no'.", + }, + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_case_params_overwrite_eval_params(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "force": True, + "cases": [ + { + "answer": "yes", + "feedback": "should be 'yes'.", + "params": {"force": False}, + } + ], + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertTrue(result["is_correct"]) + self.assertNotIn("matched_case", result) + self.assertNotIn("feedback", result) + + def test_invalid_case_entry_doesnt_raise_exception(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "params": {"raise": True}, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertIn("warnings", result) + warning = result["warnings"].pop() # type: ignore + + self.assertDictEqual( + warning, + { + "case": 0, + "message": "An exception was raised while executing " + "the evaluation function.", + "detail": "raised", + }, + ) + + def test_first_matching_case_stops_evaluation(self): + body = { + "response": "yes", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "params": {"force": True}, + }, + { + "answer": "yes", + "feedback": "should be 'yes'.", + }, + { + "answer": "no", + "feedback": "should be 'no'.", + }, + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertNotIn("warnings", result) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_overriding_eval_feedback_to_correct_case(self): + body = { + "response": "hello", + "answer": "world", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "mark": 1, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertTrue(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_overriding_eval_feedback_to_incorrect_case(self): + body = { + "response": "hello", + "answer": "hola", + "params": { + "cases": [ + { + "answer": "hello", + "feedback": "should be 'hello'.", + "mark": 0, + } + ] + }, + } + + response = commands.evaluate(body) + result = response["result"] # type: ignore + + self.assertFalse(result["is_correct"]) + self.assertEqual(result["matched_case"], 0) + self.assertEqual(result["feedback"], "should be 'hello'.") + + def test_valid_preview_command(self): + body = {"response": "hello"} + + response = commands.preview(body) + result = response["result"] # type: ignore + + self.assertEqual(result["preview"], "hello") + + def test_invalid_preview_args_raises_parse_error(self): + event = {"headers": {"command": "preview"}, "other": "params"} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], "No data supplied in request body." # type: ignore + ) + + def test_invalid_preview_schema_raises_validation_error(self): + event = {"headers": {"command": "preview"}, "body": {"response": "hello", "answer": "hello"}} + response = handler(event) + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], # type: ignore + "Failed to validate body against the preview schema.", + ) + + def test_healthcheck(self): + response = commands.healthcheck() + + self.assertIn("result", response) + result = response["result"] # type: ignore + + self.assertIn("tests_passed", result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/docs.py b/tests/docs_test.py similarity index 90% rename from tests/docs.py rename to tests/docs_test.py index 5666af6..185274f 100644 --- a/tests/docs.py +++ b/tests/docs_test.py @@ -1,10 +1,20 @@ import base64 +import os import unittest from ..tools import docs +_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + class TestDocsModule(unittest.TestCase): + def setUp(self): + self._orig_cwd = os.getcwd() + os.chdir(_PROJECT_ROOT) + + def tearDown(self): + os.chdir(self._orig_cwd) + def test_handling_available_doc(self): result = docs.send_file("tests/test_file.md") diff --git a/tests/handling.py b/tests/handling_test.py similarity index 95% rename from tests/handling.py rename to tests/handling_test.py index 8370455..cbcdf4b 100755 --- a/tests/handling.py +++ b/tests/handling_test.py @@ -1,10 +1,14 @@ +import os import unittest +from pathlib import Path from typing import Optional from ..handler import handler from ..tools import commands from ..tools.utils import EvaluationFunctionType +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") + evaluation_function: Optional[ EvaluationFunctionType ] = lambda response, answer, params: {"is_correct": True} @@ -12,10 +16,12 @@ class TestHandlerFunction(unittest.TestCase): def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR commands.evaluation_function = evaluation_function return super().setUp() def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) commands.evaluation_function = None return super().tearDown() diff --git a/tests/mued_handling_test.py b/tests/mued_handling_test.py new file mode 100644 index 0000000..0732cdd --- /dev/null +++ b/tests/mued_handling_test.py @@ -0,0 +1,586 @@ +import os +import unittest +from pathlib import Path +from typing import Optional + +from ..handler import handler +from ..tools import commands +from ..tools.utils import EvaluationFunctionType + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") + +evaluation_function: Optional[ + EvaluationFunctionType +] = lambda response, answer, params: {"is_correct": True, "feedback": "Well done."} + + +class TestMuEdHandlerFunction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + commands.evaluation_function = evaluation_function + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = None + return super().tearDown() + + def test_evaluate_returns_feedback_list(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "TEXT", "content": {}}}, + } + + response = handler(event) + + self.assertIsInstance(response, list) + self.assertEqual(len(response), 1) + self.assertIn("awardedPoints", response[0]) + + def test_evaluate_feedback_message(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "TEXT", "content": {"text": "hello"}}}, + } + + response = handler(event) + + self.assertIsInstance(response, list) + self.assertEqual(response[0]["message"], "Well done.") + self.assertEqual(response[0]["awardedPoints"], True) + + def test_evaluate_with_task(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "TEXT", "content": {"text": "Polymorphism allows..."}}, + "task": { + "title": "OOP Concepts", + "referenceSolution": {"text": "Polymorphism allows objects..."}, + }, + }, + } + + response = handler(event) + + self.assertIsInstance(response, list) + + def test_evaluate_missing_submission_returns_error(self): + event = { + "path": "/evaluate", + "body": {"task": {"title": "Some Task"}}, + } + + response = handler(event) + + self.assertIn("error", response) + self.assertIn("submission", response["error"]["detail"]) # type: ignore + + def test_evaluate_invalid_submission_type_returns_error(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "INVALID"}}, + } + + response = handler(event) + + self.assertIn("error", response) + + def test_evaluate_bodyless_event_returns_error(self): + event = {"path": "/evaluate", "random": "metadata"} + + response = handler(event) + + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], # type: ignore + "No data supplied in request body.", + ) + + def test_healthcheck(self): + event = {"path": "/evaluate/health"} + + response = handler(event) + + self.assertEqual(response.get("command"), "healthcheck") + self.assertIn("result", response) + + def test_unknown_path_falls_back_to_legacy(self): + event = { + "path": "/unknown", + "body": {"response": "hello", "answer": "world!"}, + "headers": {"command": "eval"}, + } + + response = handler(event) + + self.assertEqual(response.get("command"), "eval") + self.assertIn("result", response) + + def test_evaluate_response_latex_and_simplified_are_none_when_not_returned(self): + event = { + "path": "/evaluate", + "body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}}, + } + + response = handler(event) + + self.assertIsNone(response[0]["responseLatex"]) # type: ignore + self.assertIsNone(response[0]["responseSimplified"]) # type: ignore + + def test_evaluate_response_latex_and_simplified_populated_when_returned(self): + commands.evaluation_function = lambda r, a, p: { + "is_correct": True, + "feedback": "Well done.", + "response_latex": r"x + 1", + "response_simplified": "x + 1", + } + event = { + "path": "/evaluate", + "body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}}, + } + + response = handler(event) + + self.assertEqual(response[0]["responseLatex"], r"x + 1") # type: ignore + self.assertEqual(response[0]["responseSimplified"], "x + 1") # type: ignore + + +class TestMuEdEvaluateExtraction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self.captured: dict = {} + captured = self.captured + + def capturing_eval(response, answer, params): + captured["response"] = response + captured["answer"] = answer + captured["params"] = params + return {"is_correct": True, "feedback": "Captured."} + + commands.evaluation_function = capturing_eval + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = None + return super().tearDown() + + def test_math_submission_extracts_expression(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+1"}}, + }, + } + result = handler(event) + self.assertEqual(self.captured["response"], "x+1") + self.assertEqual(self.captured["answer"], "x+1") + self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + + def test_text_submission_extracts_text(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "TEXT", "content": {"text": "hello"}}, + "task": {"title": "T", "referenceSolution": {"text": "hello"}}, + }, + } + result = handler(event) + self.assertEqual(self.captured["response"], "hello") + self.assertEqual(self.captured["answer"], "hello") + self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + + def test_configuration_params_forwarded(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "configuration": {"params": {"strict_syntax": False}}, + }, + } + result = handler(event) + self.assertEqual(self.captured["params"], {"strict_syntax": False}) + self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + + def test_no_task_answer_is_none(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + }, + } + result = handler(event) + self.assertIsNone(self.captured["answer"]) + self.assertEqual(result[0]["awardedPoints"], True) # type: ignore + + +class TestMuEdPreviewHandlerFunction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + commands.preview_function = lambda response, params: { + "preview": {"latex": f"\\text{{{response}}}", "sympy": response} + } + commands.evaluation_function = lambda response, answer, params: { + "is_correct": True, "feedback": "Well done." + } + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.preview_function = None + commands.evaluation_function = None + return super().tearDown() + + def test_preview_returns_feedback_list(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertIsInstance(response, list) + self.assertEqual(len(response), 1) + + def test_preview_feedback_id_is_preSubmissionFeedback(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertNotIn("feedbackId", response[0]) # type: ignore + + def test_preview_contains_preSubmissionFeedback_field(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertIn("preSubmissionFeedback", response[0]) # type: ignore + + def test_preview_preSubmissionFeedback_has_latex_and_sympy(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + preview = response[0]["preSubmissionFeedback"] # type: ignore + self.assertIn("latex", preview) + self.assertIn("sympy", preview) + + def test_preview_missing_submission_returns_error(self): + event = { + "path": "/evaluate", + "body": { + "configuration": {"params": {}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertIn("error", response) + + def test_preview_bodyless_event_returns_error(self): + event = {"path": "/evaluate", "random": "metadata"} + + response = handler(event) + + self.assertIn("error", response) + self.assertEqual( + response["error"]["message"], # type: ignore + "No data supplied in request body.", + ) + + def test_preview_invalid_submission_type_returns_error(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "INVALID", "content": {}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertIn("error", response) + + def test_presubmission_disabled_runs_normal_evaluation(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": False}, + }, + } + + response = handler(event) + + self.assertIsInstance(response, list) + self.assertIn("awardedPoints", response[0]) # type: ignore + self.assertNotIn("preSubmissionFeedback", response[0]) # type: ignore + + +class TestMuEdPreviewExtraction(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + self.captured: dict = {} + captured = self.captured + + def capturing_preview(response, params): + captured["response"] = response + captured["params"] = params + return {"preview": {"latex": "captured", "sympy": str(response)}} + + commands.preview_function = capturing_preview + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.preview_function = None + return super().tearDown() + + def test_math_submission_extracts_expression(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + handler(event) + + self.assertEqual(self.captured["response"], "x+1") + + def test_text_submission_extracts_text(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "TEXT", "content": {"text": "hello"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + handler(event) + + self.assertEqual(self.captured["response"], "hello") + + def test_configuration_params_forwarded(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "configuration": {"params": {"strict_syntax": False, "is_latex": True}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + handler(event) + + self.assertEqual(self.captured["params"], {"strict_syntax": False, "is_latex": True}) + + def test_no_task_required(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "sin(x)"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertIsInstance(response, list) + self.assertEqual(self.captured["response"], "sin(x)") + + def test_preview_result_propagated(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "preSubmissionFeedback": {"enabled": True}, + }, + } + + response = handler(event) + + self.assertEqual(response[0]["preSubmissionFeedback"]["latex"], "captured") # type: ignore + self.assertEqual(response[0]["preSubmissionFeedback"]["sympy"], "x+1") # type: ignore + + +class TestMuEdMatchedCase(unittest.TestCase): + def setUp(self) -> None: + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + commands.evaluation_function = lambda response, answer, params: { + "is_correct": response == answer, + "feedback": "Default feedback.", + } + return super().setUp() + + def tearDown(self) -> None: + os.environ.pop("SCHEMA_DIR", None) + commands.evaluation_function = None + return super().tearDown() + + def test_matched_case_is_none_when_no_cases(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+2"}}, + }, + } + + response = handler(event) + + self.assertIsNone(response[0]["matchedCase"]) # type: ignore + + def test_matched_case_is_none_when_no_case_matches(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+2"}}, + "configuration": { + "params": { + "cases": [ + {"answer": "x+3", "feedback": "Try again."}, + ] + } + }, + }, + } + + response = handler(event) + + self.assertIsNone(response[0]["matchedCase"]) # type: ignore + + def test_matched_case_index_when_first_case_matches(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "100+100.1(j)"}}, + "task": { + "title": "Evaluation Task", + "referenceSolution": {"expression": "100+100.1j"}, + }, + "configuration": { + "params": { + "atol": 0, + "rtol": 0, + "strict_syntax": False, + "physical_quantity": False, + "elementary_functions": True, + "cases": [ + { + "answer": "100+100.1(j)", + "params": { + "atol": 0, + "rtol": 0, + "strict_syntax": False, + "physical_quantity": False, + "elementary_functions": True, + }, + "feedback": "Hello", + } + ], + } + }, + }, + } + + response = handler(event) + + self.assertEqual(response[0]["matchedCase"], 0) # type: ignore + self.assertEqual(response[0]["message"], "Hello") # type: ignore + + def test_matched_case_index_second_case(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+2"}}, + "configuration": { + "params": { + "cases": [ + {"answer": "x+3", "feedback": "Not case 0."}, + {"answer": "x+1", "feedback": "This is case 1."}, + {"answer": "x+4", "feedback": "Not case 2."}, + ] + } + }, + }, + } + + response = handler(event) + + self.assertEqual(response[0]["matchedCase"], 1) # type: ignore + self.assertEqual(response[0]["message"], "This is case 1.") # type: ignore + + def test_matched_case_is_none_when_correct(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+1"}}, + "configuration": { + "params": { + "cases": [ + {"answer": "x+1", "feedback": "Matched but not checked."}, + ] + } + }, + }, + } + + response = handler(event) + + self.assertEqual(response[0]["awardedPoints"], 1) # type: ignore + self.assertIsNone(response[0]["matchedCase"]) # type: ignore + + def test_matched_case_with_mark_override(self): + event = { + "path": "/evaluate", + "body": { + "submission": {"type": "MATH", "content": {"expression": "x+1"}}, + "task": {"title": "T", "referenceSolution": {"expression": "x+2"}}, + "configuration": { + "params": { + "cases": [ + {"answer": "x+1", "feedback": "Close enough!", "mark": 1}, + ] + } + }, + }, + } + + response = handler(event) + + self.assertEqual(response[0]["matchedCase"], 0) # type: ignore + self.assertEqual(response[0]["awardedPoints"], 1) # type: ignore + self.assertEqual(response[0]["message"], "Close enough!") # type: ignore + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/mued_requests_test.py b/tests/mued_requests_test.py new file mode 100644 index 0000000..23e30ff --- /dev/null +++ b/tests/mued_requests_test.py @@ -0,0 +1,87 @@ +import unittest + +from ..tools import validate +from ..tools.validate import MuEdReqBodyValidators, ValidationError + + +class TestMuEdRequestValidation(unittest.TestCase): + + def test_empty_request_body(self): + body = {} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertEqual( + e.exception.message, + "Failed to validate body against the evaluation schema.", + ) + + def test_missing_submission(self): + body = {"task": {"title": "test task"}} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertIn("submission", e.exception.error_thrown) # type: ignore + + def test_submission_missing_type(self): + body = {"submission": {"content": {"text": "hello"}}} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertIn("type", e.exception.error_thrown) # type: ignore + + def test_invalid_submission_type(self): + body = {"submission": {"type": "INVALID", "content": {}}} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + self.assertIn("INVALID", e.exception.error_thrown) # type: ignore + + def test_extra_fields_allowed(self): + # EvaluateRequest allows additional properties — unknown fields should not raise + body = {"submission": {"type": "TEXT", "content": {}}, "unknown_field": "value"} + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + def test_valid_minimal_request(self): + body = {"submission": {"type": "TEXT", "content": {}}} + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + def test_valid_request_with_task(self): + body = { + "submission": { + "type": "TEXT", + "content": {"text": "Explain polymorphism."}, + }, + "task": { + "title": "OOP Concepts", + "referenceSolution": {"text": "Polymorphism allows..."}, + }, + } + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + def test_valid_request_with_all_optional_fields(self): + body = { + "submission": { + "type": "TEXT", + "format": "plain", + "content": {"text": "hello"}, + }, + "task": { + "title": "Test Task", + "referenceSolution": {"text": "answer"}, + "learningObjectives": ["Understand polymorphism"], + }, + "criteria": [ + {"name": "Correctness", "context": "The solution is correct."} + ], + "configuration": {"key": "value"}, + } + validate.body(body, MuEdReqBodyValidators.EVALUATION) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/mued_responses_test.py b/tests/mued_responses_test.py new file mode 100644 index 0000000..73390fc --- /dev/null +++ b/tests/mued_responses_test.py @@ -0,0 +1,54 @@ +import unittest + +from ..tools import validate +from ..tools.validate import MuEdResBodyValidators, ValidationError + + +class TestMuEdResponseValidation(unittest.TestCase): + + def test_non_list_response(self): + body = {} + + with self.assertRaises(ValidationError) as e: + validate.body(body, MuEdResBodyValidators.EVALUATION) + + self.assertEqual( + e.exception.message, + "Failed to validate body against the evaluation schema.", + ) + + def test_valid_empty_list(self): + body = [] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_valid_minimal_feedback(self): + body = [{"message": "Good attempt."}] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_extra_fields_allowed(self): + # Feedback uses extra='allow' — unknown fields should not raise + body = [{"unknown_field": "value"}] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_valid_full_feedback(self): + body = [ + { + "title": "Correctness", + "message": "Your answer is correct.", + "suggestedAction": "Review the concept further.", + "criterion": {"name": "Correctness", "context": "Is the answer correct?"}, + "target": {"artefactType": "TEXT"}, + } + ] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + def test_multiple_feedback_items(self): + body = [ + {"message": "Well structured."}, + {"message": "Good use of examples."}, + ] + validate.body(body, MuEdResBodyValidators.EVALUATION) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/parse.py b/tests/parse_test.py similarity index 100% rename from tests/parse.py rename to tests/parse_test.py diff --git a/tests/requests.py b/tests/requests_test.py similarity index 92% rename from tests/requests.py rename to tests/requests_test.py index 818332f..ed41ad9 100755 --- a/tests/requests.py +++ b/tests/requests_test.py @@ -1,10 +1,20 @@ +import os import unittest +from pathlib import Path from ..tools import validate -from ..tools.validate import ReqBodyValidators, ValidationError +from ..tools.validate import LegacyReqBodyValidators as ReqBodyValidators, ValidationError + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") class TestRequestValidation(unittest.TestCase): + def setUp(self): + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + + def tearDown(self): + os.environ.pop("SCHEMA_DIR", None) + def test_empty_request_body(self): body = {} diff --git a/tests/responses.py b/tests/responses_test.py similarity index 94% rename from tests/responses.py rename to tests/responses_test.py index 010216e..805efda 100755 --- a/tests/responses.py +++ b/tests/responses_test.py @@ -1,10 +1,20 @@ +import os import unittest +from pathlib import Path from ..tools import validate -from ..tools.validate import ResBodyValidators, ValidationError +from ..tools.validate import LegacyResBodyValidators as ResBodyValidators, ValidationError + +_SCHEMAS_DIR = str(Path(__file__).parent.parent / "schemas") class TestResponseValidation(unittest.TestCase): + def setUp(self): + os.environ["SCHEMA_DIR"] = _SCHEMAS_DIR + + def tearDown(self): + os.environ.pop("SCHEMA_DIR", None) + def test_empty_response_body(self): body = {} diff --git a/tools/commands.py b/tools/commands.py index 3f4cc60..3fb9bca 100644 --- a/tools/commands.py +++ b/tools/commands.py @@ -1,17 +1,15 @@ import warnings -from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict, Union from evaluation_function_utils.errors import EvaluationException from . import healthcheck as health -from . import parse, validate from .utils import ( EvaluationFunctionType, JsonType, PreviewFunctionType, Response, ) -from .validate import ReqBodyValidators try: from ..evaluation import evaluation_function # type: ignore @@ -71,27 +69,18 @@ def healthcheck() -> Response: def preview( - event: JsonType, fnc: Optional[PreviewFunctionType] = None + body: JsonType, fnc: Optional[PreviewFunctionType] = None ) -> Response: """Run the preview command for the evaluation function. - Note: - The body of the event is validated against the preview schema - before running the preview function. - Args: - event (JsonType): The dictionary received by the gateway. Must - include a body field which may be a JSON string or a dictionary. + body (JsonType): The validated request body. fnc (Optional[PreviewFunctionType]): A function to override the current preview function (for testing). Defaults to None. Returns: Response: The result given the response and params in the body. """ - body = parse.body(event) - - validate.body(body, ReqBodyValidators.PREVIEW) - params = body.get("params", {}) fnc = fnc or preview_function @@ -100,42 +89,28 @@ def preview( return Response(command="preview", result=result) -def evaluate(event: JsonType) -> Response: - """Run the evaluation command for the evaluation function. - - Note: - The body of the event is validated against the eval schema - before running the evaluation function. - - If cases are included in the params, this function checks for - matching answers and returns the specified feedback. +def _run_evaluation(response: Any, answer: Any, params: Dict) -> Dict: + """Core evaluation logic shared by legacy and muEd command functions. Args: - event (JsonType): The dictionary received by the gateway. Must - include a body field which may be a JSON string or a dictionary. - fnc (Optional[EvaluationFunctionType]): A function to override the - current evaluation function (for testing). Defaults to None. + response (Any): The student's response. + answer (Any): The reference answer. + params (Dict): The evaluation parameters. Returns: - Response: The result given the response and params in the body. + Dict: The raw result from the evaluation function, with case + feedback applied if applicable. """ - body = parse.body(event) - validate.body(body, ReqBodyValidators.EVALUATION) - - params = body.get("params", {}) - if evaluation_function is None: raise EvaluationException("Evaluation function is not defined.") - result = evaluation_function(body["response"], body["answer"], params) + result = evaluation_function(response, answer, params) if result["is_correct"] is False and "cases" in params and len(params["cases"]) > 0: - match, warnings = get_case_feedback( - body["response"], params, params["cases"] - ) + match, case_warnings = get_case_feedback(response, params, params["cases"]) - if warnings: - result["warnings"] = warnings + if case_warnings: + result["warnings"] = case_warnings if match is not None: result["feedback"] = match["feedback"] @@ -146,9 +121,84 @@ def evaluate(event: JsonType) -> Response: if "mark" in match: result["is_correct"] = bool(int(match["mark"])) + return result + + +def evaluate(body: JsonType) -> Response: + """Run the evaluation command for the evaluation function (legacy format). + + Args: + body (JsonType): The validated request body. + + Returns: + Response: The result given the response and params in the body. + """ + params = body.get("params", {}) + result = _run_evaluation(body["response"], body["answer"], params) return Response(command="eval", result=result) +def _extract_muEd_submission(body: JsonType): + """Extract response, params, and content_key from a muEd request body.""" + submission = body["submission"] + sub_type = submission.get("type", "MATH") + _type_key = {"MATH": "expression", "TEXT": "text", "CODE": "code", "MODEL": "model"} + content_key = _type_key.get(sub_type, "value") + response = submission.get("content", {}).get(content_key) + params = body.get("configuration", {}).get("params", {}) + return response, params, content_key + + +def _run_muEd_preview(body: JsonType) -> List[Dict]: + response, params, _ = _extract_muEd_submission(body) + preview_result = preview_function(response, params) + return [{"preSubmissionFeedback": preview_result.get("preview", {})}] + + +def _run_muEd_evaluation(body: JsonType) -> List[Dict]: + response, params, content_key = _extract_muEd_submission(body) + + task = body.get("task") + if task: + ref = task.get("referenceSolution") or {} + answer = ref.get(content_key) + else: + answer = None + + result = _run_evaluation(response, answer, params) + + is_correct = result.get("is_correct", 0) + feedback_text = result.get("feedback", "") + + feedback_item: Dict = { + "message": feedback_text if isinstance(feedback_text, str) else str(feedback_text), + "awardedPoints": int(is_correct), + "matchedCase": result.get("matched_case"), + "responseLatex": result.get("response_latex"), + "responseSimplified": result.get("response_simplified"), + } + + if result.get("tags"): + feedback_item["tags"] = result["tags"] + + return [feedback_item] + + +def evaluate_muEd(body: JsonType) -> List[Dict]: + """Run the evaluation command for the evaluation function (muEd format). + + Args: + body (JsonType): The validated muEd request body. + + Returns: + List[Dict]: A list of Feedback items. + """ + pre_sub = body.get("preSubmissionFeedback") or {} + if pre_sub.get("enabled"): + return _run_muEd_preview(body) + return _run_muEd_evaluation(body) + + def get_case_feedback( response: Any, params: Dict, diff --git a/tools/smoke_tests.py b/tools/smoke_tests.py index 15e3406..1e0a659 100644 --- a/tools/smoke_tests.py +++ b/tools/smoke_tests.py @@ -64,36 +64,36 @@ def test_load_user_docs(self): self.assertTrue(result["isBase64Encoded"]) def test_load_eval_req_schema(self): - schema = validate.load_validator_from_url( - validate.ReqBodyValidators.EVALUATION + schema = validate.load_validator( + validate.LegacyReqBodyValidators.EVALUATION ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_preview_req_schema(self): - schema = validate.load_validator_from_url( - validate.ReqBodyValidators.PREVIEW + schema = validate.load_validator( + validate.LegacyReqBodyValidators.PREVIEW ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_eval_res_schema(self): - schema = validate.load_validator_from_url( - validate.ResBodyValidators.EVALUATION + schema = validate.load_validator( + validate.LegacyResBodyValidators.EVALUATION ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_preview_res_schema(self): - schema = validate.load_validator_from_url( - validate.ResBodyValidators.PREVIEW + schema = validate.load_validator( + validate.LegacyResBodyValidators.PREVIEW ) self.assertIsInstance(schema, jsonschema.Draft7Validator) def test_load_health_res_schema(self): - schema = validate.load_validator_from_url( - validate.ResBodyValidators.HEALTHCHECK + schema = validate.load_validator( + validate.LegacyResBodyValidators.HEALTHCHECK ) self.assertIsInstance(schema, jsonschema.Draft7Validator) diff --git a/tools/validate.py b/tools/validate.py index f82e2f7..6c039c6 100755 --- a/tools/validate.py +++ b/tools/validate.py @@ -1,15 +1,32 @@ import enum -import functools +import json import os from typing import Dict, List, TypedDict, Union import dotenv import jsonschema import jsonschema.exceptions -import json +import yaml + +from .utils import Response dotenv.load_dotenv() +_openapi_spec = None + +# EvaluateResponse is not a named schema in the OpenAPI spec — the evaluate +# endpoint returns an inline array of Feedback items. +_EVALUATE_RESPONSE_SCHEMA = { + "type": "array", + "items": { + "type": "object", + "properties": { + "feedbackId": {"type": "string"}, + }, + "additionalProperties": True, + }, +} + class SchemaErrorThrown(TypedDict): """Detail object returned in the error response for schema exceptions.""" @@ -34,44 +51,80 @@ def __init__( """Enumeration objects for picking which schema to validate against.""" -class ReqBodyValidators(enum.Enum): - """Enum for all request body validators.""" +class LegacyReqBodyValidators(enum.Enum): + """Enum for all legacy request body validators.""" + + ORIGINAL = "legacy/request.json" + EVALUATION = "legacy/request/eval.json" + PREVIEW = "legacy/request/preview.json" + + +class LegacyResBodyValidators(enum.Enum): + """Enum for all legacy response body validators.""" + + ORIGINAL = "legacy/responsev2.json" + EVALUATION = "legacy/response/eval.json" + PREVIEW = "legacy/response/preview.json" + HEALTHCHECK = "legacy/response/healthcheck.json" + + +class MuEdReqBodyValidators(enum.Enum): + """Enum for muEd request body validators.""" + + EVALUATION = "EvaluateRequest" + - ORIGINAL = "request.json" - EVALUATION = "request/eval.json" - PREVIEW = "request/preview.json" +class MuEdResBodyValidators(enum.Enum): + """Enum for muEd response body validators.""" + EVALUATION = "EvaluateResponse" + HEALTHCHECK = "EvaluateHealthResponse" -class ResBodyValidators(enum.Enum): - """Enum for all response body validators.""" - ORIGINAL = "responsev2.json" - EVALUATION = "response/eval.json" - PREVIEW = "response/preview.json" - HEALTHCHECK = "response/healthcheck.json" +BodyValidators = Union[ + LegacyReqBodyValidators, + LegacyResBodyValidators, + MuEdReqBodyValidators, + MuEdResBodyValidators, +] -BodyValidators = Union[ReqBodyValidators, ResBodyValidators] +def _get_openapi_spec() -> dict: + """Load and cache the muEd OpenAPI spec.""" + global _openapi_spec + if _openapi_spec is None: + schema_dir = os.getenv("SCHEMA_DIR") + if schema_dir is not None: + spec_path = os.path.join(schema_dir, "muEd", "openapi-v0_1_0.yml") + else: + spec_path = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "schemas", "muEd", "openapi-v0_1_0.yml", + ) + with open(spec_path) as f: + _openapi_spec = yaml.safe_load(f) + return _openapi_spec + def load_validator( - validator_enum: BodyValidators, + validator_enum: Union[LegacyReqBodyValidators, LegacyResBodyValidators], ) -> jsonschema.Draft7Validator: """Loads a json schema for body validations. Args: - validator_enum (BodyValidators): The validator enum name. + validator_enum (Union[LegacyReqBodyValidators, LegacyResBodyValidators]): The validator enum name. Raises: - ValueError: Raised if the schema repo URL cannot be found. + RuntimeError: Raised if the schema directory cannot be found. Returns: Draft7Validator: The validator to use. """ schema_dir = os.getenv("SCHEMA_DIR") if schema_dir is None: - raise RuntimeError("No schema path suplied.") + raise RuntimeError("No schema path supplied.") - schema_path = os.path.join(schema_dir,validator_enum.value) + schema_path = os.path.join(schema_dir, validator_enum.value) try: with open(schema_path, "r") as f: @@ -82,25 +135,17 @@ def load_validator( return jsonschema.Draft7Validator(schema) -def body(body: Dict, validator_enum: BodyValidators) -> None: - """Validate the body of a request using the request-respone-schemas. - - Args: - body (Dict): The body object to validate. - validator_enum (BodyValidators): The enum name of the validator to use. - - Raises: - ValidationError: If the validation fails, or the validator could not - be obtained. - """ +def _validate_legacy( + body: Union[Dict, Response], + validator_enum: Union[LegacyReqBodyValidators, LegacyResBodyValidators], +) -> None: try: validator = load_validator(validator_enum) validator.validate(body) - return except jsonschema.exceptions.ValidationError as e: - error_thrown = SchemaErrorThrown( + error_thrown: Union[str, SchemaErrorThrown] = SchemaErrorThrown( message=e.message, schema_path=list(e.absolute_schema_path), instance_path=list(e.absolute_path), @@ -113,3 +158,49 @@ def body(body: Dict, validator_enum: BodyValidators) -> None: f"{validator_enum.name.lower()} schema.", error_thrown, ) + + +def _validate_openapi( + body: Union[Dict, List, Response], + validator_enum: Union[MuEdReqBodyValidators, MuEdResBodyValidators], +) -> None: + spec = _get_openapi_spec() + resolver = jsonschema.RefResolver(base_uri="", referrer=spec) + + schema_name = validator_enum.value + if schema_name == "EvaluateResponse": + schema = _EVALUATE_RESPONSE_SCHEMA + else: + schema = spec["components"]["schemas"][schema_name] + + try: + jsonschema.validate(body, schema, resolver=resolver) + return + + except jsonschema.exceptions.ValidationError as e: + error_thrown: str = e.message + except Exception as e: + error_thrown = str(e) + + raise ValidationError( + "Failed to validate body against the " + f"{validator_enum.name.lower()} schema.", + error_thrown, + ) + + +def body(body: Union[Dict, List, Response], validator_enum: BodyValidators) -> None: + """Validate the body of a request using the request-response schemas. + + Args: + body (Union[Dict, List, Response]): The body object to validate. + validator_enum (BodyValidators): The enum name of the validator to use. + + Raises: + ValidationError: If the validation fails, or the validator could not + be obtained. + """ + if isinstance(validator_enum, (MuEdReqBodyValidators, MuEdResBodyValidators)): + _validate_openapi(body, validator_enum) + else: + _validate_legacy(body, validator_enum)