#!/usr/bin/env bash
# Determinism check for local LLM inference.
# Claim under test: with a pinned local backend, greedy decoding is bitwise
# reproducible; changing the compute backend can change the answer.
#
# Usage:  bash reproduce.sh
# Requires: macOS + Homebrew (or llama.cpp already on PATH), ~1.1 GB disk, network on first run.

set -uo pipefail
START=$SECONDS
DIR="${HOME}/det-test"
MODEL="${DIR}/qwen.gguf"
URL="https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/qwen2.5-1.5b-instruct-q4_k_m.gguf"
PROMPT='Sort these words alphabetically: quartz, quart, quarry, quill, quiet.'
NTOK=80
SEED=42
OUT="${DIR}/results-sort"

mkdir -p "$DIR" "$OUT"; cd "$DIR" || exit 1

echo "=============================================================="
echo " LOCAL INFERENCE DETERMINISM CHECK"
echo "=============================================================="

# ---------- 0. environment ----------
echo
echo "-- environment --"
sw_vers -productVersion 2>/dev/null | sed 's/^/macOS            /'
sysctl -n machdep.cpu.brand_string 2>/dev/null | sed 's/^/CPU              /'
echo "RAM              $(( $(sysctl -n hw.memsize) / 1073741824 )) GB"
echo "cores            $(sysctl -n hw.ncpu)"

# ---------- 1. tooling ----------
if ! command -v llama-cli >/dev/null 2>&1; then
  echo
  echo "-- installing llama.cpp (this dominates first-run time) --"
  brew install llama.cpp || { echo "install failed"; exit 1; }
fi
echo "llama.cpp        $(llama-cli --version 2>&1 </dev/null | grep -o 'version: [^ ]*' | head -1)"

# ---------- 2. model ----------
if [ ! -f "$MODEL" ]; then
  echo
  echo "-- downloading model (~1.1 GB) --"
  curl -L --progress-bar -o "$MODEL" "$URL" || { echo "download failed"; exit 1; }
fi
echo "model            $(basename "$MODEL")  $(du -h "$MODEL" | cut -f1)"
echo "model sha256     $(shasum -a 256 "$MODEL" | cut -c1-16)…"
SETUP_DONE=$SECONDS

# ---------- 3. run matrix ----------
gen () { awk '/^> /{f=1;next} /^\[ Prompt:/{f=0} f' "$1"; }

run () {
  local label="$1"; shift
  llama-cli -m "$MODEL" -p "$PROMPT" -n "$NTOK" --temp 0 -s "$SEED" \
            -st -no-cnv "$@" >"$OUT/raw_$label.txt" 2>/dev/null </dev/null
  gen "$OUT/raw_$label.txt" > "$OUT/gen_$label.txt"
  printf '  %-14s %s  %5s bytes\n' "$label" \
     "$(md5 -q "$OUT/gen_$label.txt" | cut -c1-12)" \
     "$(wc -c <"$OUT/gen_$label.txt" | tr -d ' ')"
}

echo
echo "-- A. same settings, repeated (Metal, 8 threads) --"
for i in 1 2 3; do run "repeat$i" -t 8 -ngl 99; done

echo
echo "-- B. vary thread count, Metal --"
for t in 2 4 8 12; do run "metal_t$t" -t "$t" -ngl 99; done

echo
echo "-- C. vary thread count, CPU only --"
for t in 2 4 8 12; do run "cpu_t$t" -t "$t" -ngl 0; done

echo
echo "-- D. vary backend, 8 threads --"
run "gpu_all"     -t 8 -ngl 99
run "gpu_partial" -t 8 -ngl 12
run "cpu_only"    -t 8 -ngl 0

# ---------- 4. verdict ----------
echo
echo "-- distinct outputs --"
for f in "$OUT"/gen_*.txt; do
  printf '%s %s\n' "$(md5 -q "$f")" "$(basename "$f" .txt | sed 's/^gen_//')"
done | sort | awk '{h[$1]=h[$1]" "$2} END{n=0; for(k in h){n++; printf "  group %d (%s…):%s\n", n, substr(k,1,8), h[k]} printf "\n  %d distinct output(s) across %d runs\n", n, NR}'

echo
echo "-- where the backends diverge --"
if diff -q "$OUT/gen_gpu_all.txt" "$OUT/gen_cpu_only.txt" >/dev/null 2>&1; then
  echo "  identical"
else
  diff "$OUT/gen_gpu_all.txt" "$OUT/gen_cpu_only.txt" | head -12
fi

echo
echo "=============================================================="
echo " setup   ${SETUP_DONE}s   (install + download, cached on reruns)"
echo " runs    $(( SECONDS - SETUP_DONE ))s   (14 generations)"
echo " total   ${SECONDS}s"
echo " outputs $OUT"
echo "=============================================================="