feat: test runner

Author: cherman23

src/app/(web)/crm/task-templates/[id]/edit/page.tsx

@@ -9,6 +9,7 @@ import TestCaseEditor from '@/lib/components/core/TestCaseEditor';
 import { Button } from '@/lib/components/ui/Button';
 import TaskEditorSidebar from '@/lib/components/core/TaskEditorSidebar';
 import Breadcrumbs from '@/lib/components/core/Breadcrumbs';
+import useTestRunner from '@/lib/hooks/useTestRunner';
 
 export default function TaskTemplateEditPage({ params }: { params: Promise<{ id: string }> }) {
     const { id } = use(params);
@@ -44,6 +45,8 @@ export default function TaskTemplateEditPage({ params }: { params: Promise<{ id:
         handleLanguageSelectionChange,
         generateStubsForLanguages,
     } = useTaskTemplateEditPage(id);
+    const { runAssessmentTests, runEditPageTests, setCode, setLanguage, error, loading, output } =
+        useTestRunner(id);
 
     if (isLoading) {
         return (
@@ -73,6 +76,7 @@ export default function TaskTemplateEditPage({ params }: { params: Promise<{ id:
                 >
                     {isSaving ? 'Saving...' : 'Save Changes'}
                 </Button>
+                <Button onClick={runEditPageTests}> Run Code </Button>
             </div>
 
             <div className="flex min-h-0 flex-1 overflow-hidden">

src/app/api/runner/[taskTemplateId]/route.ts

@@ -0,0 +1,41 @@
+import judge0Connector, {
+    type JudgeSubmissionRequestBody,
+} from '@/lib/connectors/judge0.connector';
+import { SubmissionSchema } from '@/lib/schemas/submission.schema';
+import TaskTemplateService from '@/lib/services/task-template.service';
+import { getSession } from '@/lib/utils/auth.utils';
+import { handleError } from '@/lib/utils/errors.utils';
+import { mapLanguageToJudge } from '@/lib/utils/language.utils';
+import { type NextRequest } from 'next/server';
+
+export async function POST(
+    request: NextRequest,
+    { params }: { params: Promise<{ taskTemplateId: string }> }
+) {
+    try {
+        const body = await request.json();
+        const session = await getSession();
+        const { taskTemplateId } = await params;
+        const parsed = SubmissionSchema.parse(body);
+        const languageId = mapLanguageToJudge(parsed.language);
+        const taskTemplate = await TaskTemplateService.getTaskTemplate(
+            taskTemplateId,
+            session.activeOrganizationId
+        );
+        const tests = [...taskTemplate.privateTestCases, ...taskTemplate.publicTestCases];
+        const formatted: JudgeSubmissionRequestBody[] = tests.map((test) => ({
+            source_code: parsed.code,
+            language_id: languageId,
+            stdin: test.input,
+            expected_output: test.output,
+        }));
+
+        const result = await judge0Connector.executeSubmissions(formatted);
+        return Response.json({
+            data: result,
+            status: 200,
+        });
+    } catch (err) {
+        return handleError(err);
+    }
+}

src/app/api/runner/route.ts

@@ -0,0 +1,37 @@
+import { handleError } from '@/lib/utils/errors.utils';
+import { assertRecruiterOrAbove } from '@/lib/utils/permissions.utils';
+import { type NextRequest } from 'next/server';
+import judge0Connector, {
+    type JudgeSubmissionRequestBody,
+} from '@/lib/connectors/judge0.connector';
+import { TestSubmissionSchema } from '@/lib/schemas/submission.schema';
+import { mapLanguageToJudge } from '@/lib/utils/language.utils';
+
+export async function POST(request: NextRequest) {
+    try {
+        await assertRecruiterOrAbove(request.headers);
+        const body = await request.json();
+        const parsed = TestSubmissionSchema.parse(body);
+        const languageId = mapLanguageToJudge(parsed.language);
+        const formatted: JudgeSubmissionRequestBody[] = parsed.tests.map((test) => ({
+            // it doesnt make sense to send the code multiple times so formatting happens on server
+            source_code: parsed.code,
+            language_id: languageId,
+            stdin: test.input,
+            expected_output: test.output,
+        }));
+
+        const result = await judge0Connector.executeSubmissions(formatted);
+
+        return Response.json(
+            {
+                data: result,
+            },
+            {
+                status: 200,
+            }
+        );
+    } catch (err) {
+        return handleError(err);
+    }
+}

src/lib/api/runner.ts

@@ -0,0 +1,46 @@
+import {
+    type SubmissionSchemaDTO,
+    type TestSubmissionSchemaDTO,
+} from '@/lib/schemas/submission.schema';
+import { type JudgeResultRequestBody } from '@/lib/connectors/judge0.connector';
+
+export async function runEditorSubmission(
+    submission: TestSubmissionSchemaDTO
+): Promise<JudgeResultRequestBody> {
+    const result = await fetch(`/api/runner`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(submission),
+    });
+
+    const json = await result.json();
+
+    if (!result.ok) {
+        throw new Error(json.message);
+    }
+
+    return json.data;
+}
+
+export async function runAssessmentSubmission(
+    taskTemplateId: string,
+    submission: SubmissionSchemaDTO
+): Promise<JudgeResultRequestBody> {
+    const result = await fetch(`/api/${taskTemplateId}/runner`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(submission),
+    });
+
+    const json = await result.json();
+
+    if (!result.ok) {
+        throw new Error(json.message);
+    }
+
+    return json.data;
+}

src/lib/connectors/judge0.connector.ts

@@ -10,10 +10,12 @@ export interface JudgeSubmissionRequestBody {
 
 export interface JudgeResultRequestBody {
     stdout: string;
-    status_id: number;
     language_id: number;
     stderr: string;
-    status: object;
+    status: {
+        id: number;
+        description: string;
+    };
     token: string;
 }
 
@@ -35,7 +37,6 @@ class Judge0Connector {
         if (!body || body.length === 0) {
             throw new BadRequestException('At least one submission is required');
         }
-
         const url = `${process.env.JUDGE_URL}/submissions/batch?fields=${this.fields.join(',')}`;
         const requestBody = { submissions: [...body] };
         const response = await fetch(url, {
@@ -71,7 +72,10 @@ class Judge0Connector {
             throw new InternalServerException(`Judge0 API error: ${jsonResponse.error}`);
         }
 
-        return jsonResponse;
+        if (!('submissions' in jsonResponse)) {
+            throw new InternalServerException(`Submissions Not Returned`); // REVIEWER LOOK AT THIS IN CASE I FORGOR
+        }
+        return jsonResponse['submissions'];
     }
 
     // create the provided submissions w/judge0 for running test cases
@@ -114,7 +118,7 @@ class Judge0Connector {
 
             results.forEach((submissionResult, index) => {
                 const isComplete =
-                    submissionResult.status_id !== 1 && submissionResult.status_id !== 2;
+                    submissionResult.status.id !== 1 && submissionResult.status.id !== 2;
                 if (isComplete) {
                     allResults.push(submissionResult);
                 } else {
@@ -146,7 +150,7 @@ class Judge0Connector {
         };
 
         allResults.forEach((submissionResult) => {
-            switch (submissionResult.status_id) {
+            switch (submissionResult.status.id) {
                 case 3: // Accepted
                     categorizedResults.accepted.push(submissionResult);
                     break;

src/lib/hooks/useTestRunner.ts

@@ -0,0 +1,53 @@
+import { useState } from 'react';
+import { runEditorSubmission, runAssessmentSubmission } from '@/lib/api/runner';
+import { type JudgeResultRequestBody } from '@/lib/connectors/judge0.connector';
+import { type ProgrammingLanguage } from '@/generated/prisma';
+
+export default function useTestRunner(taskTemplateId: string) {
+    const [error, setError] = useState<Error>();
+    const [loading, setLoading] = useState<boolean>(false);
+    const [code, setCode] = useState<string>('');
+    const [language, setLanguage] = useState<ProgrammingLanguage>('python');
+    const [output, setOutput] = useState<JudgeResultRequestBody>();
+
+    async function runEditPageTests() {
+        try {
+            setLoading(true);
+            const result = await runEditorSubmission({
+                code: 'x=input()\nprint(x)',
+                language,
+                tests: [{ input: '2', output: '2' }],
+            });
+            setOutput(result);
+        } catch (err) {
+            setError(err as Error);
+        } finally {
+            setLoading(false);
+        }
+    }
+
+    async function runAssessmentTests() {
+        try {
+            setLoading(true);
+            const result = await runAssessmentSubmission(taskTemplateId, {
+                code,
+                language,
+            });
+            setOutput(result);
+        } catch (err) {
+            setError(err as Error);
+        } finally {
+            setLoading(false);
+        }
+    }
+
+    return {
+        runAssessmentTests,
+        runEditPageTests,
+        setCode,
+        setLanguage,
+        error,
+        loading,
+        output,
+    };
+}

src/lib/schemas/submission.schema.ts

@@ -0,0 +1,18 @@
+import { ProgrammingLanguage } from '@/generated/prisma';
+import { testCaseSchema } from '@/lib/schemas/task-template.schema';
+import { z } from 'zod';
+
+export const SubmissionSchema = z.object({
+    code: z.string(),
+    language: z.enum(ProgrammingLanguage),
+    additonalTests: z.array(testCaseSchema).optional(), // May be useful for letting users create their own test cases in the future
+});
+
+export const TestSubmissionSchema = SubmissionSchema.omit({
+    additonalTests: true,
+}).extend({
+    tests: z.array(testCaseSchema),
+});
+
+export type SubmissionSchemaDTO = z.infer<typeof SubmissionSchema>;
+export type TestSubmissionSchemaDTO = z.infer<typeof TestSubmissionSchema>;

src/lib/utils/language.utils.ts

@@ -205,3 +205,19 @@ export function generateCodeStub(
             return `// Code stub for ${language} is not available.`;
     }
 }
+
+export function mapLanguageToJudge(language: string): number {
+    try {
+        switch (language) {
+            case 'python':
+                return 100;
+            case 'javascript':
+                return 102;
+            default:
+                return -1;
+        }
+    } catch {
+        // figure this out later
+        throw new Error('language not foudn');
+    }
+}

test_runner_reqs.txt

@@ -0,0 +1,31 @@
+
+Objects:
+    - TaskTemplate: Stores test cases
+        - TaskTemplateLanguage: stores solution stub
+    - Task: Stores candidate code
+
+Requirements for CRM
+    Object mutation
+        - Can mutate all parts of Task and Task Template object
+    Code running
+        - Uses solution from TaskTemplateLanguage as code text to run
+
+Requirements for OA:
+    Object mutation
+        - Update Task object to change code
+        - Can't update object to change assessmentId, taskTemplateId
+    Code running:
+        - uses candidateCode from task template language as code text to run
+
+Need to store current language in both
+
+Would it make sense to have two separate endpoints?
+
+One requires a POST to send user code
+
+One requires a GET because there is no user code, unless we are changing solution
+
+Both could be POST if we always send solution as well
+
+refactor use task hook 
+