#!/usr/bin/env bash
# ai-review-scorecard.sh — score an AI PR reviewer on your own merged PRs.
# Usage: ./ai-review-scorecard.sh OWNER/REPO BOT_LOGIN 2026-09-01 2026-09-30
# Needs: gh (logged in) and jq. Writes scorecard/prs.json and scorecard/threads.csv.
set -euo pipefail
REPO=$1 BOT=$2 SINCE=$3 UNTIL=$4
OUT=${OUT:-scorecard}
mkdir -p "$OUT"

# 1. Every merged PR in the window, with reviews and inline review threads.
gh api graphql --paginate \
  -f q="repo:$REPO is:pr is:merged merged:$SINCE..$UNTIL" \
  -f query='
query($q: String!, $endCursor: String) {
  search(query: $q, type: ISSUE, first: 25, after: $endCursor) {
    pageInfo { hasNextPage endCursor }
    nodes { ... on PullRequest {
      number createdAt mergedAt author { login }
      reviews(first: 100) { nodes { author { login __typename } submittedAt } }
      reviewThreads(first: 100) { nodes {
        isResolved isOutdated path
        comments(first: 1) { nodes { author { login } url body } }
      } }
    } }
  }
}' | jq -s '[.[].data.search.nodes[]]' > "$OUT/prs.json"

# 2. One row per bot thread, with an empty label column for hand-scoring.
jq -r --arg bot "$BOT" '
  ["pr","state","path","url","excerpt","label"],
  (.[] as $pr | $pr.reviewThreads.nodes[]
   | select(.comments.nodes[0].author.login? == $bot)
   | [$pr.number,
      (if .isOutdated then "changed" elif .isResolved then "resolved_unchanged" else "untouched" end),
      .path, .comments.nodes[0].url,
      (.comments.nodes[0].body | gsub("\\s+"; " ") | .[0:140] | if test("^[=+@-]") then "\u0027" + . else . end), ""])
  | @csv' "$OUT/prs.json" > "$OUT/threads.csv"

# 3. The automatic half of the scorecard.
jq -r --arg bot "$BOT" '
  def hours($a; $b): ((($b | fromdateiso8601) - ($a | fromdateiso8601)) / 3600 * 10 | round) / 10;
  def median: sort | if length == 0 then "n/a" elif length % 2 == 1 then .[length / 2 | floor] else ((.[length / 2 - 1] + .[length / 2]) / 2 * 10 | round) / 10 end;
  def pct($n; $d): if $d == 0 then "n/a" else "\($n * 1000 / $d | round / 10)%" end;
  ([.[] | . as $pr | .reviewThreads.nodes[]
    | select(.comments.nodes[0].author.login? == $bot)]) as $t
  | ($t | map(select(.isOutdated)) | length) as $changed
  | ($t | map(select(.isResolved and (.isOutdated | not))) | length) as $resolved
  | ([.[] | select(any(.reviews.nodes[]; .author.login? == $bot))]) as $covered
  | [ ["Merged PRs", length],
      ["PRs the bot reviewed", "\($covered | length) (\(pct($covered | length; length)))"],
      ["Bot inline threads", ($t | length)],
      ["Comments per reviewed PR", (if ($covered | length) == 0 then "n/a" else (($t | length) * 10 / ($covered | length) | round / 10) end)],
      ["Changed after comment", "\($changed) (\(pct($changed; $t | length)))"],
      ["Resolved without a change", "\($resolved) (\(pct($resolved; $t | length)))"],
      ["Untouched", "\(($t | length) - $changed - $resolved) (\(pct(($t | length) - $changed - $resolved; $t | length)))"],
      ["Median hours to first bot review",
        ([$covered[] | . as $pr | [.reviews.nodes[] | select(.author.login? == $bot) | .submittedAt] | min
          | hours($pr.createdAt; .)] | median)],
      ["Median hours to first human review",
        ([.[] | . as $pr | [.reviews.nodes[] | select(.author.__typename? == "User" and .author.login != $pr.author.login?) | .submittedAt]
          | select(length > 0) | min | hours($pr.createdAt; .)] | median)],
      ["Median hours to merge", ([.[] | hours(.createdAt; .mergedAt)] | median)]
    ][] | "\(.[0]): \(.[1])"' "$OUT/prs.json"

echo "Label a sample in $OUT/threads.csv: real-fixed, real-wontfix, wrong, noise."