Leaderboard.

 1import numpy as np
 2
 3def build_unit_scores(y_pred, y_true, unit_ids, score_unit, lower_is_better=False):
 4    """Return s_{a,t,u} after collapsing examples into units."""
 5    y_pred, y_true, unit_ids = map(np.asarray, (y_pred, y_true, unit_ids))
 6    scores = []
 7    for unit in np.unique(unit_ids):
 8        idx = unit_ids == unit
 9        value = score_unit(y_pred[idx], y_true[idx])
10        scores.append(-value if lower_is_better else value)
11    return np.asarray(scores, dtype=float)
12
13def track_score(unit_scores):
14    """Compute S_{a,t}. All returned scores are higher-is-better."""
15    return float(np.mean(unit_scores))

 1import numpy as np
 2from confidence_intervals import get_bootstrap_indices, get_conf_int
 3from statsmodels.stats.multitest import multipletests
 4
 5def bootstrap_track(unit_scores, unit_ids=None, n_boot=10_000):
 6    teams = list(unit_scores)
 7    n_units = len(unit_scores[teams[0]])
 8    score_boot = {team: np.empty(n_boot) for team in teams}
 9    rank_boot = {team: np.empty(n_boot, dtype=int) for team in teams}
10    for b in range(n_boot):
11        idx = get_bootstrap_indices(n_units, conditions=unit_ids, random_state=b)
12        scores = {team: float(np.mean(np.asarray(vals)[idx])) for team, vals in unit_scores.items()}
13        for rank, team in enumerate(sorted(teams, key=scores.get, reverse=True), start=1):
14            score_boot[team][b] = scores[team]
15            rank_boot[team][b] = rank
16    return score_boot, rank_boot
17
18def pair_summary(score_boot, team_a, team_c, alpha=5):
19    delta = score_boot[team_a] - score_boot[team_c]
20    ci_low, ci_high = get_conf_int(delta, alpha=alpha)
21    p_boot = 2 * min(np.mean(delta <= 0), np.mean(delta >= 0))
22    return {"delta": float(np.mean(delta)), "ci95": (float(ci_low), float(ci_high)),
23            "p_boot": min(float(p_boot), 1.0), "indistinguishable": bool(ci_low <= 0 <= ci_high)}
24
25def add_holm(rows, alpha=0.05):
26    reject, p_holm, _, _ = multipletests([r["p_boot"] for r in rows], method="holm", alpha=alpha)
27    for row, adj_p, keep in zip(rows, p_holm, reject):
28        row.update(p_holm=float(adj_p), significant_after_holm=bool(keep))
29    return rows
30
31def rank_stability(rank_boot):
32    return {team: {"top1": float(np.mean(r <= 1)), "top3": float(np.mean(r <= 3)),
33                   "top5": float(np.mean(r <= 5))} for team, r in rank_boot.items()}

 1import math
 2
 3def ci_half_width(sigma_hat, n_eff, z=1.96):
 4    return z * sigma_hat / math.sqrt(n_eff)
 5
 6def required_n_eff(sigma_hat, nu_t, z=1.96):
 7    """Smallest independent hidden-test count satisfying the target half-width."""
 8    return math.ceil((z * sigma_hat / nu_t) ** 2)
 9
10def meets_resolution_target(sigma_hat, n_eff, nu_t):
11    return ci_half_width(sigma_hat, n_eff) <= nu_t

 1def rank_points(rank, n_submissions):
 2    return 1.0 if n_submissions == 1 else 1.0 - (rank - 1) / (n_submissions - 1)
 3
 4def submitted_track_score(points_by_track, submitted_tracks):
 5    return sum(points_by_track[t] for t in submitted_tracks) / len(submitted_tracks)
 6
 7def all_track_score(points_by_track, all_tracks=("img", "bci", "sleep", "emg")):
 8    """Missing tracks get zero points."""
 9    return sum(points_by_track.get(t, 0.0) for t in all_tracks) / len(all_tracks)