""" Tutorial 89 - Pull live soccer results from ESPN's public scoreboard API. ESPN runs a keyless, undocumented-but-open JSON API behind its scoreboard pages: site.api.espn.com. One GET with a dates= parameter returns real Premier League results - no signup, no token. This tutorial pulls a single day's scoreboard to learn the response shape, then pulls the complete 2025-26 season (all 380 matches) with one date-range call, surviving the endpoint's two real traps measured from this machine: it silently caps a response at 100 events unless you raise limit=, and it 403s requests that impersonate a browser while accepting an honest default User-Agent - the exact opposite of stats.nba.com. The flattened season is saved to epl_results_2025_26.csv behind a promote() guard that refuses to overwrite a complete dataset with a worse refresh, and the payoff is the season read: goals per match, the home/draw/away split, and the month-by-month shape. Pulls live from site.api.espn.com; every later step - and the bundled fallback - works offline from epl_results_2025_26.csv next to this script. Run: python downloads/89_pull_live_soccer_results_from_espn.py """ import os import time import matplotlib.pyplot as plt import pandas as pd import requests import sdt_common as sdt sdt.init("pull-live-soccer-results-from-espn") HERE = os.path.dirname(os.path.abspath(__file__)) CSV = os.path.join(HERE, "epl_results_2025_26.csv") BASE = "https://site.api.espn.com/apis/site/v2/sports/soccer/eng.1/scoreboard" def session_for_espn(): """Retry/backoff session, but with the honest default User-Agent. ESPN's edge (measured Aug 2026, from this machine) rejects the spoofed browser User-Agent sdt.polite_session() sends for NBA-style hosts - and accepts plain python-requests. Identify honestly; it is also politer. """ ses = sdt.polite_session() ses.headers["User-Agent"] = requests.utils.default_user_agent() return ses def tidy(events): """Flatten ESPN scoreboard events into one row per completed match.""" rows = [] for ev in events: if not ev["status"]["type"]["completed"]: continue comp = ev["competitions"][0] sides = {c["homeAway"]: c for c in comp["competitors"]} rows.append({ "date": ev["date"][:10], "home": sides["home"]["team"]["displayName"], "away": sides["away"]["team"]["displayName"], "home_goals": int(sides["home"]["score"]), "away_goals": int(sides["away"]["score"]), "venue": comp.get("venue", {}).get("fullName", ""), "attendance": int(comp.get("attendance") or 0), }) return pd.DataFrame(rows).sort_values(["date", "home"]).reset_index(drop=True) def promote(new_df, path): """Write new_df to path only if it is at least as complete as the file there. The validate-before-write guard: a refresh that comes back smaller than what you already have (an outage, a forgotten limit=, a half season) is refused, so the good dataset on disk is never clobbered by a worse one. """ have = len(pd.read_csv(path)) if os.path.exists(path) else 0 if len(new_df) < have: print(f"REFUSED: fresh pull has {len(new_df)} rows; " f"{os.path.basename(path)} already holds {have}. Keeping the file.") return False new_df.to_csv(path, index=False) print(f"promoted: {len(new_df)} rows -> {os.path.basename(path)} " f"(replaced {have})") return True # --- live pulls (three polite calls), with the bundled-CSV offline fallback --------- live_err = None try: ses = session_for_espn() day = ses.get(BASE, params={"dates": "20260822"}, timeout=30) day.raise_for_status() day = day.json() time.sleep(1.0) short = ses.get(BASE, params={"dates": "20250801-20260601"}, timeout=60) short.raise_for_status() short = short.json() time.sleep(1.0) full = ses.get(BASE, params={"dates": "20250801-20260601", "limit": 500}, timeout=60) full.raise_for_status() full = full.json() except Exception as e: # offline fallback: the bundled file is the same table live_err = f"{type(e).__name__}" day = short = full = None with sdt.snippet("probe"): if day is None: print(f"live probe skipped ({live_err}); the bundled CSV below still works") else: lg = day["leagues"][0] print(f'{lg["name"]} - {lg["season"]["displayName"]}') print(f'scoreboard for dates=20260822: {len(day["events"])} events\n') for ev in day["events"]: sides = {c["homeAway"]: c for c in ev["competitions"][0]["competitors"]} print(f' {ev["date"]} {sides["home"]["team"]["displayName"]:<24} ' f'{sides["home"]["score"]}-{sides["away"]["score"]} ' f'{sides["away"]["team"]["displayName"]:<24} ' f'({ev["status"]["type"]["detail"]})') with sdt.snippet("limit"): if full is None: print(f"live pull skipped ({live_err}); reading bundled epl_results_2025_26.csv") else: print(f'dates=20250801-20260601 -> {len(short["events"])} events (!)') print(f'dates=20250801-20260601&limit=500 -> {len(full["events"])} events') print("\nSame range, same HTTP 200. Without limit= the endpoint quietly") print("returns its default page of 100 - no error, no warning field.") print("A 380-match season pulled naively is a 100-match season.") if full is not None: df = tidy(full["events"]) src = "live pull" else: df = pd.read_csv(CSV) src = f"bundled CSV (live pull failed: {live_err})" with sdt.snippet("tidy"): seasons = (sorted({ev["season"]["slug"] for ev in full["events"]}) if full is not None else ["(offline: bundled file)"]) print(f"{len(df)} completed matches from {src}") print(f"teams: {df['home'].nunique()} " f"dates: {df['date'].min()} to {df['date'].max()}") print(f"season tag on every event: {seasons}\n") sdt.show_df(df, n=5) with sdt.snippet("guard"): promote(df, CSV) if full is not None: # feed the guard the exact bad refresh step 2 produced: the capped pull promote(tidy(short["events"]), CSV) # --- the season read --------------------------------------------------------------- df["total"] = df["home_goals"] + df["away_goals"] df["result"] = "draw" df.loc[df["home_goals"] > df["away_goals"], "result"] = "home win" df.loc[df["home_goals"] < df["away_goals"], "result"] = "away win" split = df["result"].value_counts().reindex(["home win", "draw", "away win"]) by_month = (df.assign(month=pd.to_datetime(df["date"]).dt.to_period("M")) .groupby("month")["total"].agg(["mean", "count"])) with sdt.snippet("season"): n = len(df) print(f"2025-26 Premier League, all {n} matches:") print(f" goals: {df['total'].sum()} total, {df['total'].mean():.2f} per match") for r, c in split.items(): print(f" {r:<9} {c:>3} ({100 * c / n:.1f}%)") big = df.loc[df["total"].idxmax()] print(f"\nhighest-scoring match: {big.home} {big.home_goals}-{big.away_goals} " f"{big.away} ({big.date})") crowd = df.loc[df["attendance"].idxmax()] print(f"biggest crowd: {crowd.attendance:,} - {crowd.home} v {crowd.away} " f"at {crowd.venue} ({crowd.date})") quiet = by_month["mean"].idxmin(); loud = by_month["mean"].idxmax() print(f"scoring by month: {by_month['mean'].min():.2f} per match in {quiet} " f"to {by_month['mean'].max():.2f} in {loud}") # ---- the exhibit ------------------------------------------------------------------ GREEN = sdt.sport_color("soccer") fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10.6, 4.6)) colors = [GREEN, "#C2B7A1", "#20242B"] bars = ax1.bar(split.index, split.values, color=colors, width=0.62) for b, c in zip(bars, split.values): ax1.text(b.get_x() + b.get_width() / 2, c + 4, f"{c}\n({100 * c / len(df):.0f}%)", ha="center", fontsize=9) ax1.set_ylim(0, split.max() * 1.22) ax1.set_ylabel("matches") ax1.set_title(f"Every 2025-26 result: home edge intact") labels = [m.strftime("%b") for m in by_month.index] ax2.plot(range(len(by_month)), by_month["mean"], color=GREEN, marker="o", lw=2) ax2.axhline(df["total"].mean(), color="#C2B7A1", lw=1, ls="--") ax2.annotate(f'season avg {df["total"].mean():.2f}', (0.1, df["total"].mean()), textcoords="offset points", xytext=(2, 5), fontsize=8, color="#6C7079") ax2.set_xticks(range(len(by_month)), labels) ax2.set_ylabel("goals per match") ax2.set_title("Scoring month by month, Aug 2025 - May 2026") fig.tight_layout() sdt.save_fig(fig, "season_shape", source="ESPN public scoreboard API (site.api.espn.com), " "2025-26 Premier League, all 380 matches", asof="August 2026") print("\nOne keyless endpoint, one guard, one tidy CSV - a season you can requery any time.")