[OA][Fullstack]: 270/assessment test runner

prisma/seed-data/task-template.seed.ts

@@ -47,15 +47,15 @@ export const taskTemplatesData = [
             paragraph('two_sum_c3', ['- -10^9 <= target <= 10^9']),
             paragraph('two_sum_c4', ['- Only one valid answer exists.']),
         ],
-        // stdin: first line is space-separated nums, second line is target
+        // stdin: line 0 = space-separated nums, line 1 = target
         // stdout: space-separated indices
         publicTestCases: [
-            { input: '2 7 11 15\n9', output: '0 1' },
-            { input: '3 2 4\n6', output: '1 2' },
+            { input: '2 7 11 15\\n9', output: '0 1' },
+            { input: '3 2 4\\n6', output: '1 2' },
         ],
         privateTestCases: [
-            { input: '3 3\n6', output: '0 1' },
-            { input: '1 5 3 7 9\n10', output: '1 3' },
+            { input: '3 3\\n6', output: '0 1' },
+            { input: '1 5 3 7 9\\n10', output: '2 3' },
         ],
         orgId: 'org_nextlab_001',
         taskType: 'Single Function',
@@ -95,7 +95,7 @@ export const taskTemplatesData = [
             paragraph('rs_c1', ['- 1 <= s.length <= 10^5']),
             paragraph('rs_c2', ['- s[i] is a printable ascii character.']),
         ],
-        // stdin: space-separated characters
+        // stdin: line 0 = space-separated characters
         // stdout: space-separated reversed characters
         publicTestCases: [{ input: 'h e l l o', output: 'o l l e h' }],
         privateTestCases: [
@@ -140,8 +140,8 @@ export const taskTemplatesData = [
             paragraph('vp_c1', ['- 1 <= s.length <= 2 * 10^5']),
             paragraph('vp_c2', ['- s consists only of printable ASCII characters.']),
         ],
-        // stdin: the string
-        // stdout: true or false
+        // stdin: line 0 = raw string
+        // stdout: true or false (lowercase)
         publicTestCases: [
             { input: 'A man, a plan, a canal: Panama', output: 'true' },
             { input: 'race a car', output: 'false' },
@@ -177,8 +177,8 @@ export const taskTemplatesData = [
                 'Write a solution (or reduce an existing one) so it has as few characters as possible.',
             ]),
         ],
-        // stdin: empty — no input needed
-        // stdout: fizzbuzz output from 1 to 100
+        // stdin: line 0 = integer n
+        // stdout: FizzBuzz string for n
         publicTestCases: [
             { input: '3', output: 'Fizz' },
             { input: '5', output: 'Buzz' },
@@ -237,7 +237,7 @@ export const taskTemplatesData = [
             paragraph('ms_c1', ['- 1 <= nums.length <= 10^5']),
             paragraph('ms_c2', ['- -10^4 <= nums[i] <= 10^4']),
         ],
-        // stdin: space-separated nums
+        // stdin: line 0 = space-separated nums
         // stdout: the maximum subarray sum
         publicTestCases: [
             { input: '-2 1 -3 4 -1 2 1 -5 4', output: '6' },
@@ -283,15 +283,15 @@ export const taskTemplatesData = [
             paragraph('bs_c2', ['- -10^4 < nums[i], target < 10^4']),
             paragraph('bs_c3', ['- nums is sorted in ascending order.']),
         ],
-        // stdin: first line is space-separated nums, second line is target
+        // stdin: line 0 = space-separated nums, line 1 = target
         // stdout: the index, or -1
         publicTestCases: [
-            { input: '-1 0 3 5 9 12\n9', output: '4' },
-            { input: '-1 0 3 5 9 12\n2', output: '-1' },
+            { input: '-1 0 3 5 9 12\\n9', output: '4' },
+            { input: '-1 0 3 5 9 12\\n2', output: '-1' },
         ],
         privateTestCases: [
-            { input: '5\n5', output: '0' },
-            { input: '2 5\n5', output: '1' },
+            { input: '5\\n5', output: '0' },
+            { input: '2 5\\n5', output: '1' },
         ],
         orgId: 'org_nextlab_001',
         taskType: 'Single Function',
@@ -326,8 +326,8 @@ export const taskTemplatesData = [
             paragraph('vp2_c1', ['- 1 <= s.length <= 10^4']),
             paragraph('vp2_c2', ["- s consists of parentheses only '()[]{}'."]),
         ],
-        // stdin: the bracket string
-        // stdout: true or false
+        // stdin: line 0 = raw bracket string
+        // stdout: true or false (lowercase)
         publicTestCases: [
             { input: '()', output: 'true' },
             { input: '()[]{}', output: 'true' },
@@ -377,16 +377,15 @@ export const taskTemplatesData = [
             paragraph('mts_c2', ['- -100 <= Node.val <= 100']),
             paragraph('mts_c3', ['- Both list1 and list2 are sorted in non-decreasing order.']),
         ],
-        // stdin: first line is space-separated list1, second line is space-separated list2
-        //        empty line represents an empty list
+        // stdin: line 0 = space-separated list1 (empty line = empty list), line 1 = space-separated list2
         // stdout: space-separated merged list, or empty string for empty result
         publicTestCases: [
-            { input: '1 2 4\n1 3 4', output: '1 1 2 3 4 4' },
-            { input: '\n', output: '' },
+            { input: '1 2 4\\n1 3 4', output: '1 1 2 3 4 4' },
+            { input: '\\n', output: '' },
         ],
         privateTestCases: [
-            { input: '\n0', output: '0' },
-            { input: '1\n2', output: '1 2' },
+            { input: '\\n0', output: '0' },
+            { input: '1\\n2', output: '1 2' },
         ],
         orgId: 'org_nextlab_001',
         taskType: 'Single Function',
@@ -421,7 +420,7 @@ export const taskTemplatesData = [
             paragraph('lcp_c2', ['- 0 <= strs[i].length <= 200']),
             paragraph('lcp_c3', ['- strs[i] consists of only lowercase English letters.']),
         ],
-        // stdin: space-separated strings
+        // stdin: line 0 = space-separated strings
         // stdout: the longest common prefix, or empty string
         publicTestCases: [
             { input: 'flower flow flight', output: 'fl' },

src/app/(web)/crm/task-templates/[id]/edit/page.tsx

@@ -10,6 +10,8 @@ import { Button } from '@/lib/components/ui/Button';
 import TaskEditorSidebar from '@/lib/components/core/TaskEditorSidebar';
 import Breadcrumbs from '@/lib/components/core/Breadcrumbs';
 import useTestRunner from '@/lib/hooks/useTestRunner';
+import { type TestCaseDTO } from '@/lib/schemas/task-template.schema';
+import { type ProgrammingLanguage } from '@/generated/prisma';
 
 export default function TaskTemplateEditPage({ params }: { params: Promise<{ id: string }> }) {
     const { id } = use(params);
@@ -46,10 +48,14 @@ export default function TaskTemplateEditPage({ params }: { params: Promise<{ id:
         generateStubsForLanguages,
         getEditorContent,
     } = useTaskTemplateEditPage(id);
-    const { runEditPageTests } = useTestRunner(
-        getEditorContent(),
-        languages ? languages[selectedLanguage].language : 'python' // UHHHH
-    );
+    const { runEditPageTests } = useTestRunner();
+
+    const handleRunTests = (tests: TestCaseDTO[]) => {
+        const code = getEditorContent();
+        const language = languages?.[selectedLanguage]?.language;
+        if (!language) return;
+        runEditPageTests(tests, code, language as ProgrammingLanguage);
+    };
 
     if (isLoading) {
         return (
@@ -139,7 +145,7 @@ export default function TaskTemplateEditPage({ params }: { params: Promise<{ id:
                             setPublicTestCases={setPublicTestCases}
                             privateTestCases={privateTestCases}
                             setPrivateTestCases={setPrivateTestCases}
-                            runTests={runEditPageTests}
+                            runTests={handleRunTests}
                             isSaving={isSaving}
                         />
                     </div>

src/app/(web)/crm/templates/page.tsx

@@ -41,6 +41,7 @@ import {
 } from '@/lib/components/ui/Modal';
 import { useAuth } from '@/lib/auth/auth-context';
 import { AssessmentTemplatePreview } from '@/lib/components/templates/AssessmentTemplatePreview';
+import { toast } from 'sonner';
 
 export default function TemplatesPage() {
     const [selectedTaskTemplate, setSelectedTaskTemplate] =
@@ -132,7 +133,7 @@ export default function TemplatesPage() {
             });
             router.push(`/crm/task-templates/${created.id}/edit`);
         } catch (err) {
-            console.error('Failed to create task template:', err);
+            toast.error(`Failed to create task template: ${err}`);
         } finally {
             setIsCreating(false);
         }

src/app/api/runner/[taskTemplateId]/route.ts

@@ -1,12 +1,12 @@
+import { type NextRequest } from 'next/server';
 import judge0Connector, {
+    formatJudgeResult,
     type JudgeSubmissionRequestBody,
 } from '@/lib/connectors/judge0.connector';
 import { SubmissionSchema } from '@/lib/schemas/submission.schema';
 import TaskTemplateService from '@/lib/services/task-template.service';
-// import { getSession } from '@/lib/utils/auth.utils';
 import { handleError } from '@/lib/utils/errors.utils';
 import { mapLanguageToJudge } from '@/lib/utils/language.utils';
-import { type NextRequest } from 'next/server';
 
 export async function POST(
     request: NextRequest,
@@ -19,20 +19,26 @@ export async function POST(
         const languageId = mapLanguageToJudge(parsed.language);
         const taskTemplate = await TaskTemplateService.getTaskTemplate(
             taskTemplateId,
-            'TODO: REPLACE THIS LATER LAITH WITH COOKIE'
+            'org_nextlab_001' // lmk if we want this to stay as the TODO
         );
-        const tests = [...taskTemplate.privateTestCases, ...taskTemplate.publicTestCases];
+        const tests = [...taskTemplate.publicTestCases, ...taskTemplate.privateTestCases];
         const formatted: JudgeSubmissionRequestBody[] = tests.map((test) => ({
             source_code: parsed.code,
             language_id: languageId,
-            stdin: test.input,
-            expected_output: test.output,
+            // NOTE(laith): we're replacing newline characters to actual newline characters to
+            // display on the client that newline characters indicate the next parameter
+            stdin: test.input.replace(/\\n/g, '\n'),
+            // NOTE(laith): print() and console.log() auto append a newline character, so it would
+            // double append, this removes it and the judge0Connector will normalize it for us
+            expected_output: test.output.endsWith('\n') ? test.output : `${test.output}\n`,
         }));
 
         const result = await judge0Connector.executeSubmissions(formatted);
+        const formattedResults = result.map(formatJudgeResult);
+
         return Response.json({
-            data: result,
-            status: 200,
+            data: formattedResults,
+            status: 201,
         });
     } catch (err) {
         return handleError(err);

src/app/api/runner/route.ts

@@ -2,6 +2,7 @@ import { handleError } from '@/lib/utils/errors.utils';
 import { assertRecruiterOrAbove } from '@/lib/utils/permissions.utils';
 import { type NextRequest } from 'next/server';
 import judge0Connector, {
+    formatJudgeResult,
     type JudgeSubmissionRequestBody,
 } from '@/lib/connectors/judge0.connector';
 import { TestSubmissionSchema } from '@/lib/schemas/submission.schema';
@@ -16,15 +17,20 @@ export async function POST(request: NextRequest) {
         const formatted: JudgeSubmissionRequestBody[] = parsed.tests.map((test) => ({
             source_code: parsed.code,
             language_id: languageId,
-            stdin: test.input,
-            expected_output: test.output,
+            // NOTE(laith): we're replacing newline characters to actual newline characters to
+            // display on the client that newline characters indicate the next parameter
+            stdin: test.input.replace(/\\n/g, '\n'),
+            // NOTE(laith): print() and console.log() auto append a newline character, so it would
+            // double append, this removes it and the judge0Connector will normalize it for us
+            expected_output: test.output.endsWith('\n') ? test.output : `${test.output}\n`,
         }));
 
         const result = await judge0Connector.executeSubmissions(formatted);
+        const formattedResults = result.map(formatJudgeResult);
 
         return Response.json({
-            data: result,
-            status: 200,
+            data: formattedResults,
+            status: 201,
         });
     } catch (err) {
         return handleError(err);

src/lib/api/runner.ts

@@ -2,11 +2,11 @@ import {
     type SubmissionSchemaDTO,
     type TestSubmissionSchemaDTO,
 } from '@/lib/schemas/submission.schema';
-import { type JudgeResultRequestBody } from '@/lib/connectors/judge0.connector';
+import type { CandidateTestResult } from '@/lib/types/candidate-assessment.types';
 
 export async function runEditorSubmission(
     submission: TestSubmissionSchemaDTO
-): Promise<JudgeResultRequestBody> {
+): Promise<CandidateTestResult[]> {
     const result = await fetch(`/api/runner`, {
         method: 'POST',
         headers: {
@@ -27,7 +27,7 @@ export async function runEditorSubmission(
 export async function runAssessmentSubmission(
     taskTemplateId: string,
     submission: SubmissionSchemaDTO
-): Promise<JudgeResultRequestBody> {
+): Promise<CandidateTestResult[]> {
     const result = await fetch(`/api/runner/${taskTemplateId}`, {
         method: 'POST',
         headers: {

src/lib/components/assessment-flow/AssessmentTestCasesPanel.tsx

@@ -28,7 +28,7 @@ export default function AssessmentTestCasesPanel({
 }: AssessmentTestCasesPanelProps) {
     const [isOpen, setIsOpen] = useState(false);
     const [panelHeight, setPanelHeight] = useState(PANEL_DEFAULT_HEIGHT);
-    const [selectedIndex, setSelectedIndex] = useState<number | null>(null);
+    const [selectedIndices, setSelectedIndices] = useState<Set<number>>(new Set());
     const dragRef = useRef<{ startY: number; startHeight: number } | null>(null);
 
     const passCount = results.filter((r) => r.status === 'passed').length;
@@ -40,7 +40,15 @@ export default function AssessmentTestCasesPanel({
     );
 
     function toggleSelected(i: number) {
-        setSelectedIndex((prev) => (prev === i ? null : i));
+        setSelectedIndices((prev) => {
+            const next = new Set(prev);
+            if (next.has(i)) {
+                next.delete(i);
+            } else {
+                next.add(i);
+            }
+            return next;
+        });
     }
 
     // this is the dragging handle for when the panel is open and we want to resize it
@@ -116,7 +124,10 @@ export default function AssessmentTestCasesPanel({
                 <div className="flex items-center gap-2">
                     <Button
                         variant="secondary"
-                        onClick={onRunTests}
+                        onClick={() => {
+                            setIsOpen(true);
+                            onRunTests();
+                        }}
                         className="flex h-9 items-center gap-2 rounded-lg border px-4 text-sm font-medium"
                     >
                         <Play className="size-4" />
@@ -142,7 +153,7 @@ export default function AssessmentTestCasesPanel({
                             index={i}
                             test={tc}
                             result={results[i] ?? { status: 'default' }}
-                            selected={selectedIndex === i}
+                            selected={selectedIndices.has(i)}
                             onSelect={() => toggleSelected(i)}
                         />
                     ))}

src/lib/components/core/TaskDetailsTab.tsx

@@ -7,6 +7,7 @@ import { Combobox } from '@/lib/components/ui/Combobox';
 import { createTag } from '@/lib/api/tags';
 import type { BlockNoteContent } from '@/lib/types/task-template.types';
 import type { TagDTO } from '@/lib/schemas/tag.schema';
+import { toast } from 'sonner';
 
 // BlockNote: https://www.blocknotejs.org/docs/nextjs
 const BlockNoteEditor = dynamic(() => import('@/lib/components/core/BlockNoteEditor'), {
@@ -58,7 +59,7 @@ export default function TaskDetailsTab({
             setAvailableTags((prev) => [...prev, newTag]);
             setTags((prev: TagDTO[]) => [...prev, newTag]);
         } catch (err) {
-            console.error('Failed to create tag:', err);
+            toast.error(`Failed to create tag: ${err}`);
         }
     };
 

src/lib/components/modal/InviteUsersModal.tsx

@@ -98,8 +98,7 @@ export default function InviteUsersModal({
                 toast.error('An error occurred... please try again');
             }
         } catch (err) {
-            console.error('Error inviting users:', err);
-            toast.error('An error occurred... please try again');
+            toast.error(`Error inviting users: ${err}`);
         } finally {
             setInviting(false);
         }

src/lib/connectors/judge0.connector.ts

@@ -1,5 +1,6 @@
 import { BadRequestException, InternalServerException } from '@/lib/utils/errors.utils';
 import { sleep } from '@/lib/utils/utils';
+import type { CandidateTestResult } from '@/lib/types/candidate-assessment.types';
 
 export interface JudgeSubmissionRequestBody {
     source_code: string;
@@ -19,6 +20,15 @@ export interface JudgeResultRequestBody {
     token: string;
 }
 
+export function formatJudgeResult(result: JudgeResultRequestBody): CandidateTestResult {
+    return {
+        stdout: result.stdout,
+        stderr: result.stderr,
+        description: result.status?.description,
+        statusId: result.status?.id,
+    };
+}
+
 class Judge0Connector {
     private headers: HeadersInit;
     private fields: string[];
@@ -193,6 +203,13 @@ class Judge0Connector {
     ): Promise<JudgeResultRequestBody[]> {
         const tokens = await this.registerSubmissions(submissions);
         const { allResults } = await this.waitForSubmissions(tokens, totalTimeout);
+        const tokenIndexMap = new Map(tokens.map((token, index) => [token, index]));
+
+        // Sort results by their original submission order (this assumes judge0 returns the tokens in the right order)\
+        // in my practice this is true but I am unsure
+        allResults.sort(
+            (a, b) => (tokenIndexMap.get(a.token) ?? 0) - (tokenIndexMap.get(b.token) ?? 0)
+        );
         return allResults;
     }
 }