Cache lab

benchmarks/courseexam_bench/data/exams_metadata.json

@@ -0,0 +1,26 @@
+{
+  "exams": [
+    {
+      "exam_id": "example_course_2024_midterm",
+      "test_paper_name": "Example Systems Course: 2024 Midterm Exam",
+      "course": "Example Systems Course",
+      "institution": "Example University",
+      "year": 2024,
+      "score_total": 59,
+      "score_max": 59.0,
+      "score_avg": 42.0,
+      "score_median": 43,
+      "score_standard_deviation": 9.0,
+      "num_questions": 10
+    },
+    {
+      "exam_id": "cs537_fall_2021_final",
+      "test_paper_name": "CS 537 Fall 2021 Final",
+      "course": "Operating Systems",
+      "institution": "University of Wisconsin-Madison",
+      "year": 2021,
+      "score_total": 60,
+      "num_questions": 59
+    }
+  ]
+}

benchmarks/courseexam_bench/data/reference_materials/raft_basics.md

@@ -0,0 +1,7 @@
+# Raft Consensus Algorithm - Quick Reference
+
+This reference sheet provides essential information about the Raft consensus algorithm for distributed systems.
+
+## Overview
+
+Raft is a consensus algorithm designed as an alternative to the Paxos family of algorithms. It was meant to be more understandable than Paxos by means of separation of logic, but it is also formally proven safe and offers some additional features.

benchmarks/courselab_bench/data/cmu_15-213/task_cache_lab/compose.yaml

@@ -0,0 +1,6 @@
+services:
+  default:
+    network_mode: host
+    image: ubuntu:20.04
+    command: sleep infinity
+    working_dir: /workspace

benchmarks/courselab_bench/data/cmu_15-213/task_cache_lab/config.json

@@ -0,0 +1,15 @@
+{
+  "instance_id": "cmu_15-213__cache_lab",
+  "course_id": "cmu_15-213",
+  "timeout_minutes": 60,
+  "tags": [
+    "cache",
+    "memory-hierarchy",
+    "c-programming",
+    "performance"
+  ],
+  "artifacts": [
+    "csim.c",
+    "trans.c"
+  ]
+}

benchmarks/courselab_bench/data/cmu_15-213/task_cache_lab/evaluate.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "=== Evaluating Cache Lab ==="
+cd /workspace
+
+if [ -f /tmp/checksums/protected.sha256 ]; then
+    echo "Verifying protected files"
+    sha256sum -c /tmp/checksums/protected.sha256
+else
+    echo "WARN: No protected checksums found; continuing"
+fi
+
+required=(
+  "csim.c"
+  "trans.c"
+  "cachelab.c"
+  "cachelab.h"
+  "test-csim"
+  "test-trans.c"
+  "tracegen.c"
+  "Makefile"
+  "traces/yi.trace"
+  "traces/yi2.trace"
+  "traces/dave.trace"
+  "traces/trans.trace"
+  "traces/long.trace"
+)
+for f in "${required[@]}"; do
+    if [ ! -e "$f" ]; then
+        echo "FAIL: Missing required file $f"
+        exit 1
+    fi
+done
+
+echo "Building"
+make clean >/dev/null 2>&1 || true
+if ! timeout 300 make >/tmp/cachelab_build.log 2>&1; then
+    echo "FAIL: make failed"
+    tail -n 50 /tmp/cachelab_build.log
+    exit 1
+fi
+
+chmod +x test-csim test-trans csim-ref driver.py || true
+
+echo "Running test-csim"
+csim_output=$(timeout 300 ./test-csim 2>&1) || {
+    echo "FAIL: test-csim failed to run"
+    echo "$csim_output"
+    exit 1
+}
+echo "$csim_output"
+csim_result=$(echo "$csim_output" | grep "TEST_CSIM_RESULTS" | tail -n1 | awk -F '=' '{print $2}')
+if ! echo "$csim_result" | grep -Eq "^[0-9]+$"; then
+    echo "FAIL: Could not parse TEST_CSIM_RESULTS (got '$csim_result')"
+    exit 1
+fi
+
+expected_csim=27
+if [ "$csim_result" -ne "$expected_csim" ]; then
+    echo "FAIL: Cache simulator score unexpected (got $csim_result, expected $expected_csim)"
+    exit 1
+fi
+
+run_trans_test() {
+    local M="$1"
+    local N="$2"
+    local label="$3"
+
+    echo "Running test-trans for $label (${M}x${N})"
+    local output
+    local status=0
+    output=$(timeout 300 ./test-trans -M "$M" -N "$N" 2>&1) || status=$?
+    echo "$output"
+    if [ "$status" -ne 0 ]; then
+        echo "FAIL: test-trans exited with status $status for $label"
+        exit 1
+    fi
+
+    local line
+    line=$(echo "$output" | grep "TEST_TRANS_RESULTS" | tail -n1 | awk -F '=' '{print $2}')
+    if [ -z "$line" ]; then
+        echo "FAIL: Could not parse TEST_TRANS_RESULTS for $label"
+        exit 1
+    fi
+
+    local correct="${line%%:*}"
+    local misses="${line##*:}"
+
+    if [ "$correct" != "1" ]; then
+        echo "FAIL: Transpose failed correctness for $label"
+        exit 1
+    fi
+
+    if [ "$misses" = "2147483647" ] || ! echo "$misses" | grep -Eq "^[0-9]+$"; then
+        echo "FAIL: Invalid miss count for $label ($misses)"
+        exit 1
+    fi
+
+    # Enforce performance budgets similar to the autograder scoring windows
+    local budget=0
+    case "$label" in
+        "32x32") budget=600 ;;
+        "64x64") budget=2000 ;;
+        "61x67") budget=3000 ;;
+    esac
+
+    if [ "$budget" -gt 0 ]; then
+        awk "BEGIN { if ($misses > $budget) exit 1; }" || {
+            echo "FAIL: Miss count $misses exceeds budget $budget for $label"
+            exit 1
+        }
+    fi
+}
+
+run_trans_test 32 32 "32x32"
+run_trans_test 64 64 "64x64"
+run_trans_test 61 67 "61x67"
+
+echo "PASS: Cache lab simulator and transpose verified"
+exit 0

benchmarks/courselab_bench/data/cmu_15-213/task_cache_lab/preprocess.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -euo pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+export DEBCONF_NONINTERACTIVE_SEEN=true
+
+echo "=== Setting up CMU 15-213 Cache Lab ==="
+cd /workspace
+
+APT_OPTS=("-o" "Acquire::Retries=3" "-o" "Acquire::http::Timeout=20")
+if ! apt-get "${APT_OPTS[@]}" update -y >/tmp/apt-update.log 2>&1; then
+    cat /tmp/apt-update.log
+    exit 1
+fi
+echo "APT update completed"
+# Install apt-utils early to silence debconf and keep stderr clean
+if ! apt-get "${APT_OPTS[@]}" install -y --no-install-recommends apt-utils ca-certificates >/tmp/apt-utils.log 2>&1; then
+    cat /tmp/apt-utils.log
+    exit 1
+fi
+dpkg -s apt-utils >/dev/null 2>&1 || {
+    echo "ERROR: apt-utils missing after install"
+    exit 1
+}
+
+# 2. Installing dependencies
+if ! apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \
+    build-essential \
+    make \
+    valgrind \
+    python3 \
+    >/tmp/apt-deps.log 2>&1; then
+    cat /tmp/apt-deps.log
+    exit 1
+fi
+
+# 3. Verify files
+required_files=("csim.c" "trans.c" "cachelab.c" "cachelab.h" "csim-ref" "test-csim" "test-trans.c" "tracegen.c" "driver.py" "Makefile" "traces")
+for f in "${required_files[@]}"; do
+    if [ ! -e "$f" ]; then
+        echo "ERROR: missing required file $f"
+        exit 1
+    fi
+    echo "  ✓ $f"
+done
+
+# 4. Authenticate protected files
+chmod +x csim-ref test-csim driver.py || true
+
+# Record checksums for protected infra (not student solution files csim.c/trans.c)
+mkdir -p /tmp/checksums
+CHECKSUM_FILE=/tmp/checksums/protected.sha256
+: > "$CHECKSUM_FILE"
+protected_list=("cachelab.c" "cachelab.h" "csim-ref" "test-csim" "test-trans.c" "tracegen.c" "driver.py" "Makefile")
+for f in "${protected_list[@]}"; do
+    if [ -e "$f" ]; then
+        sha256sum "$f" >> "$CHECKSUM_FILE"
+        echo "  Protected: $f"
+    fi
+done
+for f in traces/*; do
+    if [ -f "$f" ]; then
+        sha256sum "$f" >> "$CHECKSUM_FILE"
+        echo "  Protected: $f"
+    fi
+done
+
+echo "Setup complete"
+exit 0