{ "schema": "l2s1.rtx3060.public-aggregates.v1", "date": "2026-09-23", "exportDate": "2026-09-26", "dataset": "JevBench v1.1 public 231", "gpu": "NVIDIA GeForce RTX 3060", "vramMiB": 12288, "driver": "595.71.05", "context": 8192, "models": 20, "decisions": 4620, "itemsPerModel": 231, "tierCounts": { "easy": 48, "original": 72, "hard": 111 }, "policy": { "minTopProbability": 0.8, "minCandidateMass": 0.05 }, "latencyScope": "Rust serial decide_batch, batch size 1; model load and one warmup excluded", "fileSizeScope": "GGUF model file bytes; frozen download metadata or SHA-256-verified local file stat", "gpuMemoryScope": "Maximum sampled whole-device memory used, 200 ms sampling", "jevbenchRevision": "f79a1cab94ab9a5879383b7ef9ee1805b9dc2d84", "evaluatorSha256": "312ceabd7f82ed769d97eb5528a726de6ad00dd29ed9585791d5a466280e45a8", "totals": { "planned": 4620, "attempted": 4620, "scorable": 4620, "valid": 4620, "rawCorrect": 2924, "accepted": 3122, "acceptedCorrect": 2279, "acceptedWrong": 843, "abstained": 1498, "errors": 0 }, "rows": [ { "fileSizeBytes": 4482403488, "id": "Qwen3.5-4B-Q8_0", "model": "Qwen3.5 4B Q8_0", "modelFile": "Qwen3.5-4B-Q8_0.gguf", "rawAccuracy": 79.65367965367966, "coverage": 59.74025974025974, "acceptedAccuracy": 93.47826086956522, "correctAll": 55.84415584415584, "p50Ms": 86.875174, "p95Ms": 1137.298412, "peakGpuMiB": 5039.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 184, "accepted": 138, "acceptedCorrect": 129, "acceptedWrong": 9, "abstained": 93, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 5680522464, "id": "Qwen3.5-9B-Q4_K_M", "model": "Qwen3.5 9B Q4_K_M", "modelFile": "Qwen3.5-9B-Q4_K_M.gguf", "rawAccuracy": 77.92207792207793, "coverage": 72.2943722943723, "acceptedAccuracy": 92.21556886227546, "correctAll": 66.66666666666666, "p50Ms": 130.55342000000002, "p95Ms": 1697.1431205, "peakGpuMiB": 5637.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 180, "accepted": 167, "acceptedCorrect": 154, "acceptedWrong": 13, "abstained": 64, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 9527502048, "id": "Qwen3.5-9B-Q8_0", "model": "Qwen3.5 9B Q8_0", "modelFile": "Qwen3.5-9B-Q8_0.gguf", "rawAccuracy": 77.92207792207793, "coverage": 71.42857142857143, "acceptedAccuracy": 91.51515151515152, "correctAll": 65.36796536796537, "p50Ms": 124.679976, "p95Ms": 1613.7917155, "peakGpuMiB": 8817.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 180, "accepted": 165, "acceptedCorrect": 151, "acceptedWrong": 14, "abstained": 66, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 4977171584, "id": "gemma-4-E4B-it-Q4_K_M", "model": "Gemma 4 E4B it Q4_K_M", "modelFile": "gemma-4-E4B-it-Q4_K_M.gguf", "rawAccuracy": 77.48917748917748, "coverage": 83.98268398268398, "acceptedAccuracy": 84.5360824742268, "correctAll": 70.995670995671, "p50Ms": 86.620102, "p95Ms": 1186.4660155, "peakGpuMiB": 3989.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 179, "accepted": 194, "acceptedCorrect": 164, "acceptedWrong": 30, "abstained": 37, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 8192953472, "id": "gemma-4-E4B-it-Q8_0", "model": "Gemma 4 E4B it Q8_0", "modelFile": "gemma-4-E4B-it-Q8_0.gguf", "rawAccuracy": 76.62337662337663, "coverage": 83.98268398268398, "acceptedAccuracy": 83.50515463917526, "correctAll": 70.12987012987013, "p50Ms": 85.03775399999999, "p95Ms": 1146.213878, "peakGpuMiB": 6047.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 177, "accepted": 194, "acceptedCorrect": 162, "acceptedWrong": 32, "abstained": 37, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 2740937888, "id": "Qwen3.5-4B-Q4_K_M", "model": "Qwen3.5 4B Q4_K_M", "modelFile": "Qwen3.5-4B-Q4_K_M.gguf", "rawAccuracy": 76.19047619047619, "coverage": 60.60606060606061, "acceptedAccuracy": 92.85714285714286, "correctAll": 56.27705627705628, "p50Ms": 88.46956, "p95Ms": 1175.1838305, "peakGpuMiB": 3381.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 176, "accepted": 140, "acceptedCorrect": 130, "acceptedWrong": 10, "abstained": 91, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 7266070528, "id": "Qwen3.8-27B-UD-IQ2_XXS", "model": "Qwen3.8 27B UD-IQ2_XXS", "modelFile": "Qwen3.8-27B-UD-IQ2_XXS.gguf", "rawAccuracy": 73.16017316017316, "coverage": 63.63636363636363, "acceptedAccuracy": 87.75510204081633, "correctAll": 55.84415584415584, "p50Ms": 414.60704999999996, "p95Ms": 5460.217004, "peakGpuMiB": 7825.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 169, "accepted": 147, "acceptedCorrect": 129, "acceptedWrong": 18, "abstained": 84, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 8709519168, "id": "Qwen3-8B-Q8_0", "model": "Qwen3 8B Q8_0", "modelFile": "Qwen3-8B-Q8_0.gguf", "rawAccuracy": 71.42857142857143, "coverage": 93.5064935064935, "acceptedAccuracy": 74.07407407407408, "correctAll": 69.26406926406926, "p50Ms": 121.450278, "p95Ms": 1817.940254, "peakGpuMiB": 9347.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 165, "accepted": 216, "acceptedCorrect": 160, "acceptedWrong": 56, "abstained": 15, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 4967497152, "id": "gemma-4-E2B-it-Q8_0", "model": "Gemma 4 E2B it Q8_0", "modelFile": "gemma-4-E2B-it-Q8_0.gguf", "rawAccuracy": 67.96536796536796, "coverage": 90.47619047619048, "acceptedAccuracy": 72.24880382775119, "correctAll": 65.36796536796537, "p50Ms": 46.787102, "p95Ms": 701.11393, "peakGpuMiB": 3025.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 157, "accepted": 209, "acceptedCorrect": 151, "acceptedWrong": 58, "abstained": 22, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 4280405792, "id": "Qwen3-4B-Q8_0", "model": "Qwen3 4B Q8_0", "modelFile": "Qwen3-4B-Q8_0.gguf", "rawAccuracy": 65.80086580086581, "coverage": 87.44588744588745, "acceptedAccuracy": 68.8118811881188, "correctAll": 60.17316017316018, "p50Ms": 85.26035999999999, "p95Ms": 1349.2774075000002, "peakGpuMiB": 5751.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 152, "accepted": 202, "acceptedCorrect": 139, "acceptedWrong": 63, "abstained": 29, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 5198386720, "id": "Ministral-3-8B-Instruct-2512-Q4_K_M", "model": "Ministral-3 8B Instruct 2512 Q4_K_M", "modelFile": "Ministral-3-8B-Instruct-2512-Q4_K_M.gguf", "rawAccuracy": 65.36796536796537, "coverage": 59.74025974025974, "acceptedAccuracy": 83.33333333333334, "correctAll": 49.78354978354979, "p50Ms": 441.42554, "p95Ms": 2399.6961725, "peakGpuMiB": 6281.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 151, "accepted": 138, "acceptedCorrect": 115, "acceptedWrong": 23, "abstained": 93, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 11624759488, "id": "gpt-oss-20b-Q4_K_M-no-cuda-graphs", "model": "gpt-oss 20B Q4_K_M", "modelFile": "gpt-oss-20b-Q4_K_M.gguf", "rawAccuracy": 64.93506493506493, "coverage": 64.5021645021645, "acceptedAccuracy": 79.19463087248322, "correctAll": 51.082251082251084, "p50Ms": 181.931967, "p95Ms": 2262.284453, "peakGpuMiB": 11901.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 150, "accepted": 149, "acceptedCorrect": 118, "acceptedWrong": 31, "abstained": 82, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": { "GGML_CUDA_DISABLE_GRAPHS": "1" } } }, { "fileSizeBytes": 2012012800, "id": "Qwen3.5-2B-Q8_0", "model": "Qwen3.5 2B Q8_0", "modelFile": "Qwen3.5-2B-Q8_0.gguf", "rawAccuracy": 62.33766233766234, "coverage": 42.42424242424242, "acceptedAccuracy": 84.6938775510204, "correctAll": 35.93073593073593, "p50Ms": 40.992193, "p95Ms": 513.032822, "peakGpuMiB": 2465.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 144, "accepted": 98, "acceptedCorrect": 83, "acceptedWrong": 15, "abstained": 133, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 4130402176, "id": "gemma-3-4b-it-Q8_0", "model": "Gemma 3 4B it Q8_0", "modelFile": "gemma-3-4b-it-Q8_0.gguf", "rawAccuracy": 59.74025974025974, "coverage": 96.1038961038961, "acceptedAccuracy": 60.36036036036037, "correctAll": 58.00865800865801, "p50Ms": 74.710081, "p95Ms": 879.920001, "peakGpuMiB": 5477.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 138, "accepted": 222, "acceptedCorrect": 134, "acceptedWrong": 88, "abstained": 9, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 4084611040, "id": "Phi-4-mini-instruct.Q8_0", "model": "Phi-4-mini Instruct Q8_0", "modelFile": "Phi-4-mini-instruct.Q8_0.gguf", "rawAccuracy": 56.709956709956714, "coverage": 50.21645021645021, "acceptedAccuracy": 74.13793103448276, "correctAll": 37.22943722943723, "p50Ms": 70.499215, "p95Ms": 971.9538835, "peakGpuMiB": 5337.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 131, "accepted": 116, "acceptedCorrect": 86, "acceptedWrong": 30, "abstained": 115, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 811843840, "id": "Qwen3.5-0.8B-Q8_0", "model": "Qwen3.5 0.8B Q8_0", "modelFile": "Qwen3.5-0.8B-Q8_0.gguf", "rawAccuracy": 51.94805194805194, "coverage": 20.77922077922078, "acceptedAccuracy": 66.66666666666666, "correctAll": 13.852813852813853, "p50Ms": 27.927573000000002, "p95Ms": 350.530776, "peakGpuMiB": 1317.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 120, "accepted": 48, "acceptedCorrect": 32, "acceptedWrong": 16, "abstained": 183, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 3275575840, "id": "SmolLM3-3B-Q8_0", "model": "SmolLM3 3B Q8_0", "modelFile": "SmolLM3-3B-Q8_0.gguf", "rawAccuracy": 46.320346320346324, "coverage": 45.02164502164502, "acceptedAccuracy": 60.57692307692307, "correctAll": 27.27272727272727, "p50Ms": 71.471313, "p95Ms": 884.7550485, "peakGpuMiB": 4047.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 107, "accepted": 104, "acceptedCorrect": 63, "acceptedWrong": 41, "abstained": 127, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 3421898816, "id": "Llama-3.2-3B-Instruct-Q8_0", "model": "Llama-3.2 3B Instruct Q8_0", "modelFile": "Llama-3.2-3B-Instruct-Q8_0.gguf", "rawAccuracy": 43.722943722943725, "coverage": 36.36363636363637, "acceptedAccuracy": 45.23809523809524, "correctAll": 16.450216450216452, "p50Ms": 68.183239, "p95Ms": 884.752845, "peakGpuMiB": 4585.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 101, "accepted": 84, "acceptedCorrect": 38, "acceptedWrong": 46, "abstained": 147, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 1069306368, "id": "gemma-3-1b-it-Q8_0", "model": "Gemma 3 1B it Q8_0", "modelFile": "gemma-3-1b-it-Q8_0.gguf", "rawAccuracy": 38.961038961038966, "coverage": 87.01298701298701, "acceptedAccuracy": 39.800995024875625, "correctAll": 34.63203463203463, "p50Ms": 25.056749, "p95Ms": 309.2379475, "peakGpuMiB": 1663.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 90, "accepted": 201, "acceptedCorrect": 80, "acceptedWrong": 121, "abstained": 30, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } }, { "fileSizeBytes": 639446688, "id": "Qwen3-0.6B-Q8_0", "model": "Qwen3 0.6B Q8_0", "modelFile": "Qwen3-0.6B-Q8_0.gguf", "rawAccuracy": 31.601731601731604, "coverage": 82.25108225108225, "acceptedAccuracy": 32.10526315789474, "correctAll": 26.406926406926406, "p50Ms": 34.014483, "p95Ms": 445.11573999999996, "peakGpuMiB": 1857.0, "valid": 231, "errors": 0, "counts": { "planned": 231, "attempted": 231, "scorable": 231, "valid": 231, "rawCorrect": 73, "accepted": 190, "acceptedCorrect": 61, "acceptedWrong": 129, "abstained": 41, "errors": 0 }, "config": { "context": 8192, "batch": 256, "microbatch": 256, "threads": 4, "flashAttention": "off", "executionMode": "fresh", "promptLayout": "legacy", "runtimeEnvironmentOverrides": {} } } ] }