Extend run stats DSL with statistical and temporal analysis

README.md

@@ -56,30 +56,29 @@ invalid drafts retain the displayed results. Guided controls and the language
 reference remain available below the editor. CodeMirror 5.65.20 and its completion
 addon are vendored locally under the MIT license; the editor needs no CDN access.
 
-Queries support up to six metrics, numeric comparisons and inclusive ranges,
-case-insensitive title search, `has`/`missing` sensor filters, daily/weekly/monthly
-grouping, and evidence sorting. Periods use your account timezone: rolling days,
-weeks, calendar months, or inclusive explicit dates. Comparisons cover an equally
-long preceding period. Pace filters accept values such as `5:30 /km`; duration
-accepts `h`, `min`, or `s`, and distance accepts `km` or `m`.
+The DSL supports statistical summaries and percentiles, calculated fields,
+AND/OR filters, run gaps and changes, rolling workload, running streaks, calendar
+and metric-bin groups, group filters/ranking, cumulative and rolling trends,
+scatter plots with correlation, and CSV exports. See the complete
+[DSL reference and examples](docs/STATS_DSL.md) for units and execution semantics.
+
+For your ten biggest gaps between recorded runs:
 
 ```text
 runs
-  during last 6 months
-  where distance between 8 and 20 km
-  and pace <= 5:30 /km
-  and has cadence
-  summarize pace, heart_rate, cadence
-  by month
-  compare previous period
-  sort by pace asc
+  during all time
+  summarize gap_since_previous_run, distance
+  by run
+  sort by gap_since_previous_run desc
+  take 10
+  compare none
 ```
 
-The explorer covers whole runs and uses persisted summaries without loading
-recording samples. Metric definitions and usable-run coverage appear with each
-result. Select a chart point or the evidence period picker to inspect contributing
-runs. Save named views in this browser, or bookmark the results URL; relative
-queries follow the calendar as new runs are imported.
+Gaps use complete owner-scoped history before filters and link to the previous
+run. The explorer uses persisted run summaries without loading recording samples.
+Cards show metric definitions and usable-run coverage. Select a chart point or
+group row to inspect contributing runs. Save named views in this browser, or
+bookmark the results URL; relative queries follow the calendar as runs are added.
 
 ### Bringing in more activities
 

app/analysis_engine.py

@@ -0,0 +1,572 @@
+"""Owner-scoped analysis over compact run records; no raw samples are loaded."""
+
+from collections import defaultdict, deque
+from datetime import UTC, datetime, timedelta
+from itertools import pairwise
+from types import SimpleNamespace
+from zoneinfo import ZoneInfo
+
+from .extensions import db
+from .models import Activity
+from .stats import METRICS, format_value, metric_value, period_bounds
+from .stats import aggregate as legacy_aggregate
+from .stats_analysis import describe, evaluate, expression_tree, statistic, valid
+from .stats_dsl import QueryError
+
+
+def catalog_for(q):
+    catalog = dict(METRICS)
+    for spec in q["derived"]:
+        catalog[spec["name"]] = (
+            spec["name"].replace("_", " ").capitalize(),
+            spec["unit"],
+            f"Per-run calculation: {spec['expression']}. Missing inputs and zero divisors produce missing data.",
+        )
+    return catalog
+
+
+def value(row, key):
+    return row.values.get(key)
+
+
+def summarize(rows, key, q):
+    spec = q["aggregates"].get(key)
+    if spec:
+        function, field = spec["function"], spec["field"]
+        if function == "count" and not field:
+            return len(rows), len(rows)
+        values = [value(row, field) for row in rows]
+        coverage = sum(valid(number) for number in values)
+        if function == "weighted_mean":
+            pairs = [
+                (value(row, field), row.distance_m if field == "pace" else row.duration_s)
+                for row in rows
+                if valid(value(row, field))
+            ]
+            weight = sum(w for _, w in pairs)
+            return (sum(v * w for v, w in pairs) / weight if weight > 0 else None), coverage
+        return statistic(values, function), coverage
+    if key in {"count", "calories"}:
+        numbers = [value(row, key) for row in rows if valid(value(row, key))]
+        return (sum(numbers) if numbers or key == "count" else None), len(numbers)
+    if key in {
+        "pace",
+        "heart_rate",
+        "distance",
+        "duration",
+        "elevation",
+        "cadence",
+        "power",
+        "ground_contact",
+        "stride_length",
+        "vertical_oscillation",
+    }:
+        return legacy_aggregate(rows, key)
+    numbers = [value(row, key) for row in rows if valid(value(row, key))]
+    return statistic(numbers, "mean"), len(numbers)
+
+
+def formatted(number, key, q):
+    if not valid(number):
+        return "—"
+    spec = q["aggregates"].get(key)
+    raw = spec["field"] if spec and spec["function"] not in {"count", "stddev", "sum"} else key
+    if raw == "pace":
+        return format_value(number, "pace")
+    if key == "count" or (spec and spec["function"] == "count"):
+        return f"{number:,.0f}"
+    return (
+        f"{number:,.2f}"
+        if key in {"stride_length", "gap_since_previous_run"}
+        or key in {s["name"] for s in q["derived"]}
+        else f"{number:,.1f}"
+    )
+
+
+def condition_matches(number, condition):
+    if not valid(number):
+        return False
+    threshold = condition["value"]
+    return {
+        ">": number > threshold,
+        ">=": number >= threshold,
+        "<": number < threshold,
+        "<=": number <= threshold,
+        "=": number == threshold,
+        "!=": number != threshold,
+    }[condition["op"]]
+
+
+def matches(row, conditions):
+    for condition in conditions:
+        op, field = condition["op"], condition["field"]
+        if op == "contains":
+            if condition["value"].casefold() not in row.title.casefold():
+                return False
+        elif op == "has":
+            if not valid(value(row, field)):
+                return False
+        elif op == "missing":
+            if valid(value(row, field)):
+                return False
+        elif not condition_matches(value(row, field), condition):
+            return False
+    return True
+
+
+def ordered(rows, ordering):
+    key, direction = ordering
+    if key == "date":
+        return sorted(rows, key=lambda row: (row.instant, row.id), reverse=direction == "desc")
+    available = [row for row in rows if valid(value(row, key))]
+    absent = [row for row in rows if not valid(value(row, key))]
+    return sorted(
+        available,
+        key=lambda row: (value(row, key), row.instant, row.id),
+        reverse=direction == "desc",
+    ) + ordered(absent, ["date", "desc"])
+
+
+def prepare(raw, zone, q, catalog):
+    rows, windows, totals = [], {7: deque(), 28: deque()}, {7: 0.0, 28: 0.0}
+    previous, last_day, streak = None, None, 0
+    trees = [
+        (spec["name"], expression_tree(spec["expression"], {**catalog})[0]) for spec in q["derived"]
+    ]
+    for record in raw:
+        row = SimpleNamespace(**record._mapping)
+        row.instant = (
+            row.started_at.replace(tzinfo=UTC)
+            if row.started_at.tzinfo is None
+            else row.started_at.astimezone(UTC)
+        )
+        row.local = row.instant.astimezone(zone)
+        row.day = row.local.date()
+        row.values = {key: metric_value(row, key) for key in METRICS}
+        row.values.update(
+            count=1,
+            speed=row.distance_m / row.duration_s * 3.6 if row.duration_s > 0 else None,
+            elevation_per_km=row.elevation_m / (row.distance_m / 1000)
+            if row.distance_m > 0
+            else None,
+            max_heart_rate=(row.analysis_summary or {}).get("max_hr")
+            or (row.metrics or {}).get("max_heart_rate"),
+            calories=(row.metrics or {}).get("total_calories"),
+            training_effect=(row.metrics or {}).get("total_training_effect"),
+        )
+        for field in (
+            "pace",
+            "heart_rate",
+            "max_heart_rate",
+            "cadence",
+            "power",
+            "ground_contact",
+            "stride_length",
+            "vertical_oscillation",
+        ):
+            if not valid(row.values[field]) or row.values[field] <= 0:
+                row.values[field] = None
+        row.previous = previous
+        row.values["gap_since_previous_run"] = (
+            (row.instant - previous.instant).total_seconds() / 86400 if previous else None
+        )
+        for key, field in (
+            ("distance_change", "distance"),
+            ("pace_change", "pace"),
+            ("heart_rate_change", "heart_rate"),
+        ):
+            now, before = row.values[field], previous.values[field] if previous else None
+            row.values[key] = now - before if valid(now) and valid(before) else None
+        for days in (7, 28):
+            window = windows[days]
+            while window and (row.instant - window[0].instant).total_seconds() >= days * 86400:
+                totals[days] -= window.popleft().distance_m / 1000
+            window.append(row)
+            totals[days] += row.distance_m / 1000
+            row.values[f"rolling_{days}d_distance"] = max(0.0, totals[days])
+            row.values[f"rolling_{days}d_count"] = len(window)
+        if row.day != last_day:
+            streak = streak + 1 if last_day and (row.day - last_day).days == 1 else 1
+            last_day = row.day
+        row.values["run_day_streak"] = streak
+        for name, tree in trees:
+            row.values[name] = evaluate(tree, row.values)
+        rows.append(row)
+        previous = row
+    return rows
+
+
+def explore(owner_id, timezone, q, page=1, selected_bucket=None, export=False, raw_group=False):
+    zone, catalog = ZoneInfo(timezone), catalog_for(q)
+    start, end, prior_start = period_bounds(q, timezone)
+    boundary = lambda day: datetime.combine(day, datetime.min.time(), zone).astimezone(UTC)
+    # Temporal columns use every earlier run, independently of query filters.
+    raw = db.session.execute(
+        db.select(
+            Activity.id,
+            Activity.title,
+            Activity.started_at,
+            Activity.distance_m,
+            Activity.duration_s,
+            Activity.elevation_m,
+            Activity.avg_hr,
+            Activity.analysis_summary,
+            Activity.metrics,
+        )
+        .where(
+            Activity.owner_id == owner_id,
+            Activity.sport == "running",
+            Activity.started_at < boundary(end),
+        )
+        .order_by(Activity.started_at, Activity.id)
+    ).all()
+    history = prepare(raw, zone, q, catalog)
+    if q["all_time"] and history:
+        start = history[0].day
+        prior_start = start
+
+    def included(row):
+        return (
+            row.distance_m >= q["minimum"] * 1000
+            and (q["maximum"] is None or row.distance_m <= q["maximum"] * 1000)
+            and (q["hr_min"] is None or valid(row.avg_hr) and row.avg_hr >= q["hr_min"])
+            and (q["hr_max"] is None or valid(row.avg_hr) and row.avg_hr <= q["hr_max"])
+            and matches(row, q["filters"])
+            and (not q["filter_groups"] or any(matches(row, group) for group in q["filter_groups"]))
+        )
+
+    current = ordered(
+        [row for row in history if start <= row.day < end and included(row)], q["sort"]
+    )
+    prior = ordered(
+        [
+            row
+            for row in history
+            if q["compare"] and prior_start <= row.day < start and included(row)
+        ],
+        q["sort"],
+    )
+    if q["take"]:
+        current, prior = current[: q["take"]], prior[: q["take"]]
+    keys = [
+        q["metric"],
+        *([] if q["secondary"] == "none" else [q["secondary"]]),
+        *q["extra_metrics"],
+    ]
+    keys = list(dict.fromkeys(keys))
+    group, width = q["group"], q["bin"]
+
+    def bucket(row, offset=timedelta()):
+        day = row.day + offset
+        if group == "day":
+            return day
+        if group == "week":
+            return day - timedelta(days=day.weekday())
+        if group == "month":
+            return day.replace(day=1)
+        if group == "year":
+            return day.replace(month=1, day=1)
+        if group == "weekday":
+            return row.day.weekday()
+        if group == "hour":
+            return row.local.hour
+        if group == "run":
+            return row.id
+        if group == "all":
+            return "All runs"
+        number = value(row, width["field"])
+        return math_floor(number / width["width"]) if valid(number) else None
+
+    def label(token):
+        if group in {"day", "week", "month", "year"}:
+            return token.strftime("%d %b %Y")
+        if group == "weekday":
+            return ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")[
+                token
+            ]
+        if group == "hour":
+            return f"{token:02d}:00"
+        if group == "run":
+            row = next(row for row in current if row.id == token)
+            return row.local.strftime("%d %b %Y %H:%M") + f" · #{row.id}"
+        if group == "bin":
+            unit = catalog[width["field"]][1]
+            return f"{token * width['width']:g}–{(token + 1) * width['width']:g} {unit}"
+        return token
+
+    current_groups, prior_groups = defaultdict(list), defaultdict(list)
+    for row in current:
+        token = bucket(row)
+        if token is not None:
+            current_groups[token].append(row)
+    for row in prior:
+        token = bucket(
+            row, start - prior_start if group in {"day", "week", "month", "year"} else timedelta()
+        )
+        if token is not None:
+            prior_groups[token].append(row)
+    if group in {"day", "week", "month", "year"}:
+        dates = set()
+        for n in range((end - start).days):
+            day = start + timedelta(days=n)
+            dates.add(
+                day
+                if group == "day"
+                else day - timedelta(days=day.weekday())
+                if group == "week"
+                else day.replace(day=1)
+                if group == "month"
+                else day.replace(month=1, day=1)
+            )
+        tokens = sorted(dates)
+    elif group == "weekday":
+        tokens = list(range(7))
+    elif group == "hour":
+        tokens = list(range(24))
+    elif group == "run":
+        tokens = [row.id for row in current]
+    elif group == "all":
+        tokens = ["All runs"]
+    else:
+        tokens = sorted(set(current_groups) | set(prior_groups))
+    if len(tokens) > 1000 and not q["limit"]:
+        raise QueryError(
+            "This produces more than 1,000 chart groups. Use a wider grouping, take, or limit."
+        )
+
+    def measure(rows, key):
+        return summarize(rows, key, q)[0]
+
+    if q["having"]:
+        tokens = [
+            token
+            for token in tokens
+            if all(
+                condition_matches(measure(current_groups[token], condition["field"]), condition)
+                for condition in q["having"]
+            )
+        ]
+    if q["rank"]:
+        key, direction = q["rank"]
+        with_data = [token for token in tokens if valid(measure(current_groups[token], key))]
+        without = [token for token in tokens if not valid(measure(current_groups[token], key))]
+        tokens = (
+            sorted(
+                with_data,
+                key=lambda token: measure(current_groups[token], key),
+                reverse=direction == "desc",
+            )
+            + without
+        )
+    if q["limit"]:
+        tokens = (
+            tokens[: q["limit"]]
+            if q["rank"] or group not in {"day", "week", "month", "year"}
+            else tokens[-q["limit"] :]
+        )
+    if len(tokens) > 1000:
+        raise QueryError("Use at most 1,000 displayed groups.")
+    if q["having"] or q["limit"] or group == "bin":
+        kept = {row.id for token in tokens for row in current_groups[token]}
+        current = [row for row in current if row.id in kept]
+        kept_prior = {row.id for token in tokens for row in prior_groups[token]}
+        prior = [row for row in prior if row.id in kept_prior]
+    descriptors = {key: describe(key, catalog, q["aggregates"]) for key in keys}
+    cards = []
+    for key in keys:
+        number, coverage = summarize(current, key, q)
+        before, _ = summarize(prior, key, q)
+        title, unit, definition = descriptors[key]
+        cards.append(
+            {
+                "key": key,
+                "label": title,
+                "unit": unit,
+                "definition": definition,
+                "value": formatted(number, key, q),
+                "coverage": coverage,
+                "previous": formatted(before, key, q),
+                "delta": f"{number - before:+.2f}" if valid(number) and valid(before) else None,
+            }
+        )
+    labels = [label(token) for token in tokens]
+    evidence_groups = {
+        text: current_groups[token] for token, text in zip(tokens, labels, strict=True)
+    }
+    raw_evidence_groups = dict(evidence_groups)
+    series, groups = [], []
+    for token, text in zip(tokens, labels, strict=True):
+        rows = current_groups[token]
+        groups.append(
+            {
+                "label": text,
+                "count": len(rows),
+                "values": [formatted(measure(rows, key), key, q) for key in keys],
+                "raw_values": [measure(rows, key) for key in keys],
+            }
+        )
+    for key in keys:
+
+        def measurements(mapping, key=key):
+            results = []
+            for index, token in enumerate(tokens):
+                contributing = mapping[token]
+                if q["trend"] == "cumulative":
+                    contributing = [
+                        row for earlier in tokens[: index + 1] for row in mapping[earlier]
+                    ]
+                elif q["trend"] and q["trend"].startswith("rolling "):
+                    size = int(q["trend"].split()[1])
+                    contributing = [
+                        row
+                        for earlier in tokens[max(0, index - size + 1) : index + 1]
+                        for row in mapping[earlier]
+                    ]
+                results.append(measure(contributing, key))
+                if mapping is current_groups and key == keys[0]:
+                    evidence_groups[label(token)] = contributing
+            if q["trend"] in {"change", "percent_change"}:
+                original = list(results)
+                results = [None] + [
+                    (now - before if q["trend"] == "change" else (now - before) / abs(before) * 100)
+                    if valid(now) and valid(before) and (q["trend"] == "change" or before != 0)
+                    else None
+                    for before, now in pairwise(original)
+                ]
+                if not original:
+                    results = []
+            return results
+
+        title, unit, _ = descriptors[key]
+        pace_axis = unit == "/km"
+        if q["trend"] == "percent_change":
+            unit, pace_axis = "%", False
+        elif q["trend"] == "change" and pace_axis:
+            unit, pace_axis = "sec/km", False
+        series.append(
+            {
+                "key": key,
+                "label": title,
+                "unit": unit,
+                "values": measurements(current_groups),
+                "previous": measurements(prior_groups) if q["compare"] and group != "run" else [],
+                "pace_axis": pace_axis,
+            }
+        )
+    chart = {
+        "labels": labels,
+        "series": series,
+        "type": q["chart"],
+        "group": group,
+        "trend": q["trend"],
+    }
+    correlation = None
+    if q["plot"]:
+        x, y = q["plot"]
+        points = lambda rows: [
+            {
+                "x": value(row, x),
+                "y": value(row, y),
+                "id": row.id,
+                "label": row.title + " · " + row.local.strftime("%d %b %Y"),
+            }
+            for row in rows
+            if valid(value(row, x)) and valid(value(row, y))
+        ]
+        plotted = points(current)
+        if len(plotted) >= 2:
+            xs, ys = [p["x"] for p in plotted], [p["y"] for p in plotted]
+            if len(set(xs)) > 1 and len(set(ys)) > 1:
+                import statistics
+
+                correlation = statistics.correlation(xs, ys)
+        chart["scatter"] = {
+            "x": catalog[x],
+            "y": catalog[y],
+            "current": plotted,
+            "previous": points(prior) if q["compare"] else [],
+            "correlation": correlation,
+        }
+    evidence = current
+    if selected_bucket is not None:
+        text = (
+            selected_bucket.strftime("%d %b %Y")
+            if hasattr(selected_bucket, "strftime")
+            else str(selected_bucket)
+        )
+        chosen_groups = raw_evidence_groups if raw_group else evidence_groups
+        if text not in chosen_groups:
+            raise QueryError("Choose a group from the current chart.")
+        evidence = ordered(chosen_groups[text], q["sort"])
+    pages = max(1, (len(evidence) + 24) // 25)
+    if not 1 <= page <= pages:
+        raise QueryError("That results page does not exist.")
+
+    def evidence_row(row):
+        inputs = [
+            q["aggregates"].get(key, {}).get("field")
+            or ("count" if key in q["aggregates"] else key)
+            for key in keys
+        ]
+        return {
+            "id": row.id,
+            "title": row.title,
+            "date": row.local.strftime("%d %b %Y %H:%M"),
+            "distance": f"{row.distance_m / 1000:.2f}",
+            "values": [formatted(value(row, field), field, q) for field in inputs],
+            "raw_values": [value(row, field) for field in inputs],
+            "previous_run": {
+                "id": row.previous.id,
+                "title": row.previous.title,
+                "date": row.previous.local.strftime("%d %b %Y %H:%M"),
+            }
+            if row.previous
+            else None,
+        }
+
+    return {
+        "cards": cards,
+        "chart": chart,
+        "groups": groups,
+        "evidence_columns": [
+            {"key": field, "label": catalog[field][0], "unit": catalog[field][1]}
+            for key in keys
+            for field in [
+                q["aggregates"].get(key, {}).get("field")
+                or ("count" if key in q["aggregates"] else key)
+            ]
+        ],
+        "count": len(current),
+        "prior_count": len(prior),
+        "distance": sum(row.distance_m for row in current) / 1000,
+        "rows": [
+            evidence_row(row)
+            for row in (evidence if export else evidence[(page - 1) * 25 : page * 25])
+        ],
+        "start": start,
+        "end": end - timedelta(days=1),
+        "previous_start": prior_start,
+        "previous_end": start - timedelta(days=1),
+        "page": page,
+        "pages": pages,
+        "evidence_count": len(evidence),
+        "show_previous": any(
+            key in {"gap_since_previous_run", "distance_change", "pace_change", "heart_rate_change"}
+            or any(
+                name in key
+                for name in (
+                    "gap_since_previous_run",
+                    "distance_change",
+                    "pace_change",
+                    "heart_rate_change",
+                )
+            )
+            for key in keys
+        ),
+    }
+
+
+def math_floor(value):
+    import math
+
+    return math.floor(value)

app/routes.py

@@ -1,4 +1,6 @@
+import csv
 import hashlib
+import io
 import re
 import secrets
 from datetime import UTC, datetime
@@ -7,6 +9,7 @@ from zoneinfo import available_timezones
 
 from flask import (
     Blueprint,
+    Response,
     abort,
     current_app,
     flash,
@@ -30,6 +33,8 @@ from .history import activity_page, latest_mapped_run, overview, records_for
 from .models import Activity, Friendship, GarminConnection, User
 from .services import activity_data, import_cache, import_upload, route_data
 from .stats import DEFAULT, METRICS, explore, parse_query, period_bounds, query_text
+from .stats_analysis import EXTRA_UNITS, FUNCTIONS, GROUPS
+from .stats_dsl import UNITS
 
 web = Blueprint("web", __name__)
 DUMMY_HASH = generate_password_hash("not-a-real-account-password")
@@ -214,6 +219,19 @@ def stats():
         results=results,
         error=error,
         submitted_query=request.args.get("query", ""),
+        language={
+            "functions": sorted(FUNCTIONS),
+            "groups": sorted(GROUPS),
+            "fields": {
+                key: {
+                    "label": info[0],
+                    "unit": info[1],
+                    "description": info[2],
+                    "units": list(({**UNITS, **EXTRA_UNITS}).get(key, {})),
+                }
+                for key, info in METRICS.items()
+            },
+        },
     ), 400 if error else 200
 
 
@@ -222,15 +240,14 @@ def stats():
 def stats_runs():
     try:
         query = parse_query(request.args)
-        bucket = (
-            datetime.strptime(request.args.get("bucket", ""), "%d %b %Y").replace(tzinfo=UTC).date()
-        )
+        bucket = request.args.get("bucket", "")
         results = explore(
             current_user.id,
             current_user.timezone,
             query,
             int(request.args.get("page", "1")),
             selected_bucket=bucket,
+            raw_group=request.args.get("scope") == "group",
         )
     except (ValueError, OverflowError) as exc:
         return jsonify(error=str(exc)), 400
@@ -252,13 +269,69 @@ def stats_validate():
     try:
         query = parse_query({"query": payload["query"]})
         period_bounds(query, current_user.timezone)
-        return jsonify(valid=True, canonical=query_text(query), diagnostics=[])
+        return jsonify(
+            valid=True, canonical=query_text(query), diagnostics=[], derived=query["derived"]
+        )
     except (ValueError, OverflowError) as exc:
         return jsonify(
             valid=False, diagnostics=[{"line": getattr(exc, "line", 1), "message": str(exc)}]
         )
 
 
+@web.get("/stats/export")
+@login_required
+@limiter.limit("20 per minute")
+def stats_export():
+    try:
+        query = parse_query(request.args)
+        result = explore(current_user.id, current_user.timezone, query, export=True)
+        kind = request.args.get("kind", "groups")
+        if kind not in {"runs", "groups"}:
+            raise ValueError("Export runs or groups.")
+    except (ValueError, OverflowError) as exc:
+        return jsonify(error=str(exc)), 400
+    output = io.StringIO(newline="")
+    writer = csv.writer(output)
+
+    def safe(value):
+        # Keep user-authored run titles inert when opened in a spreadsheet.
+        text = str(value)
+        return "'" + text if text.lstrip().startswith(("=", "+", "-", "@")) else text
+
+    columns = result["cards"] if kind == "groups" else result["evidence_columns"]
+    headers = [
+        f"{card['key']} ({'sec/km' if card['unit'] == '/km' else card['unit']})" for card in columns
+    ]
+    if kind == "groups":
+        writer.writerow(["Group", "Contributing runs", *headers])
+        for group in result["groups"]:
+            writer.writerow([safe(group["label"]), group["count"], *group["raw_values"]])
+    else:
+        writer.writerow(
+            ["Run", "Local date", "Distance (km)", "Previous run", "Previous run date", *headers]
+        )
+        for row in result["rows"]:
+            previous = row["previous_run"] or {}
+            writer.writerow(
+                [
+                    safe(row["title"]),
+                    row["date"],
+                    row["distance"],
+                    safe(previous.get("title", "")),
+                    previous.get("date", ""),
+                    *row["raw_values"],
+                ]
+            )
+    return Response(
+        output.getvalue(),
+        mimetype="text/csv",
+        headers={
+            "Content-Disposition": f'attachment; filename="stride-{kind}.csv"',
+            "Cache-Control": "private, no-store",
+        },
+    )
+
+
 @web.route("/friends", methods=["GET", "POST"])
 @login_required
 @limiter.limit("20 per hour", methods=["POST"])

app/static/stats-editor.js

@@ -7,7 +7,10 @@
   const validation = document.getElementById('query-validation');
   const diagnostics = document.getElementById('query-diagnostics');
   const run = document.getElementById('run-query');
-  const metrics = [...root.querySelectorAll('[data-metric]')].map(button => ({name:button.dataset.metric, unit:button.querySelector('span').textContent}));
+  const language = JSON.parse(document.getElementById('query-language').textContent);
+  const metrics = Object.entries(language.fields).map(([name, field]) => ({name, ...field}));
+  let derivedMetadata = {};
+  const availableMetrics = () => [...metrics, ...getValue().split('\n').map(line => line.match(/^\s*let ([a-z][a-z0-9_]*) = /)?.[1]).filter(name=>name && !names.has(name)).map(name => ({name, unit:derivedMetadata[name]?.unit || 'calculated', units:derivedMetadata[name] ? [derivedMetadata[name].unit] : []}))];
   const names = new Set(metrics.map(metric => metric.name));
   let editor = null;
   let timer = null;
@@ -24,6 +27,22 @@
     ['during last 12 weeks', 'Rolling period ending today'],
     ['during last 6 months', 'Calendar months'],
     ['during last 30 days', 'Rolling days'],
+    ['during all time', 'Your complete recorded running history'],
+    ['let efficiency = speed / heart_rate', 'Reusable per-run arithmetic; missing inputs stay missing'],
+    ['where gap_since_previous_run >= 7 days', 'Elapsed time since the previous recorded run'],
+    ['or distance >= 20 km', 'Alternative filter; AND binds more strongly than OR'],
+    ['summarize count(), median(pace), p90(heart_rate)', 'Counts, percentiles and statistical summaries'],
+    ['having count() >= 3 runs', 'Keep groups with at least three runs'],
+    ['rank by sum(distance) desc', 'Rank groups by a selected summary expression'],
+    ['take 10', 'Keep the first ten sorted runs before grouping'],
+    ['limit 10', 'Keep ten groups; most recent for time groups'],
+    ['trend rolling 4 buckets', 'Recalculate summaries over four consecutive buckets'],
+    ['trend cumulative', 'Recalculate using all earlier displayed buckets'],
+    ['trend change', 'Difference from the preceding bucket'],
+    ['trend percent_change', 'Percentage change from the preceding bucket'],
+    ['plot speed vs heart_rate', 'Run-level scatter plot and Pearson correlation'],
+    ['chart bar', 'Choose auto, line, bar or none'],
+    ['by distance bins 5 km', 'Group runs into half-open distance ranges'],
     ['during 2026-07-01 to 2026-09-30', 'Inclusive local-date range'],
     ['ending 2026-10-02', 'Optional end date for a rolling period'],
     ['where distance between 8 and 20 km', 'Inclusive distance range'],
@@ -46,12 +65,13 @@
       if (stream.match(/#.*/)) return 'comment';
       if (stream.match(/"(?:[^"\\]|\\.)*"/)) return 'string';
       if (stream.match(/\d{4}-\d{2}-\d{2}\b|\d+:\d{2}\b|(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?/)) return 'number';
-      if (stream.match(/>=|<=|!=|>|<|=/)) return 'operator';
+      if (stream.match(/>=|<=|!=|>|<|=|[+*\/-]/)) return 'operator';
       if (stream.match(/\/km\b/)) return 'atom';
-      if (stream.match(/[a-zA-Z_]+/)) {
+      if (stream.match(/[a-zA-Z_][a-zA-Z0-9_]*/)) {
         const word = stream.current();
-        if (names.has(word)) return 'variable-2';
-        if (/^(runs|during|last|ending|where|and|between|summarize|by|compare|previous|period|none|has|missing|title|contains|sort|asc|desc|to)$/.test(word)) return 'keyword';
+        if (names.has(word) || availableMetrics().some(metric=>metric.name===word)) return 'variable-2';
+        if (language.functions.includes(word)) return 'builtin';
+        if (/^(runs|during|last|ending|where|and|or|between|summarize|by|compare|previous|period|none|has|missing|title|contains|sort|asc|desc|to|let|having|rank|take|limit|trend|rolling|cumulative|change|percent_change|plot|vs|chart|bins|all|time)$/.test(word)) return 'keyword';
         if (/^(day|days|week|weeks|month|months|km|m|bpm|spm|W|ms|h|min|s|date)$/.test(word)) return 'atom';
         return null;
       }
@@ -64,29 +84,40 @@
       const before = line.slice(0,line.length-token.length);
       let list = [];
       let from = {line:cursor.line,ch:cursor.ch-token.length};
-      if (/^\s*summarize\s/.test(line) || /^\s*sort by\s+\w*$/.test(line) || /^\s*(where|and)\s+(?:(has|missing)\s+)?\w*$/.test(line)) {
+      if (/^\s*having (?:\w+|\w+\(\w*\))\s+$/.test(line)) {
+        from=cursor;list=['>= ','<= ','> ','< ','= ','!= '].map(text=>({text,detail:'Filter grouped summary'}));
+      } else if (/^\s*rank by (?:\w+|\w+\(\w*\))\s+\w*$/.test(line)) {
+        list=['asc','desc'].filter(text=>text.startsWith(token)).map(text=>({text,detail:'Group ranking order'}));
+      } else if (/^\s*chart\s+\w*$/.test(line)) {
+        list=['auto','line','bar','none'].filter(text=>text.startsWith(token)).map(text=>({text,detail:'Chart presentation'}));
+      } else if (/^\s*trend\s+[\w ]*$/.test(line)) {
+        from={line:cursor.line,ch:line.indexOf('trend')+6};const prefix=line.slice(from.ch);
+        list=['cumulative','rolling 4 buckets','change','percent_change'].filter(text=>text.startsWith(prefix)).map(text=>({text,detail:'Transform displayed time buckets'}));
+      } else if (/^\s*plot\s+\w+\s+$/.test(line)) {
+        from=cursor;list=[{text:'vs ',detail:'Second scatter axis'}];
+      } else if (/^\s*(summarize|having|rank by|let \w+ =)\s/.test(line) || /^\s*plot\s+(?:\w+ vs )?\w*$/.test(line) || /^\s*sort by\s+\w*$/.test(line) || /^\s*(where|and|or)\s+(?:(has|missing)\s+)?\w*$/.test(line)) {
         const used=/^\s*summarize\s/.test(line)?new Set(before.replace(/^\s*summarize\s+/,'').split(',').map(name=>name.trim())):new Set();
-        list = metrics.filter(metric=>metric.name.startsWith(token)&&!used.has(metric.name)).map(metric=>({text:metric.name, detail:`${metric.unit} · recorded run metric`}));
+        list = availableMetrics().filter(metric=>metric.name.startsWith(token)&&!used.has(metric.name)).map(metric=>({text:metric.name + (/\(\w*$/.test(line) && cm.getLine(cursor.line)[cursor.ch] !== ')' ? ')' : ''), detail:`${metric.unit} · run metric`}));
+        if (/^\s*(summarize|having|rank by)\s/.test(line) && !/\(\w*$/.test(line)) language.functions.filter(name=>name.startsWith(token)).forEach(name=>list.push({text:name==='count'?'count()':name+'(',detail:'Statistical function · available values only'}));
         if (/sort by/.test(line) && 'date'.startsWith(token)) list.push({text:'date',detail:'Local run date'});
-        if (/^\s*(where|and)\s+\w*$/.test(line)) ['has','missing','title'].forEach(word=>{if(word.startsWith(token))list.push({text:word,detail:'Filter recorded data'});});
-      } else if (/^\s*(where|and)\s+\w+\s*$/.test(line)) {
+        if (/^\s*(where|and|or)\s+\w*$/.test(line)) ['has','missing','title'].forEach(word=>{if(word.startsWith(token))list.push({text:word,detail:'Filter recorded data'});});
+      } else if (/^\s*(where|and|or)\s+\w+\s*$/.test(line)) {
         from = cursor;
         list = ['>= ', '<= ', '> ', '< ', '= ', '!= ', 'between '].map(text=>({text,detail:'Comparison operator'}));
       } else if (/^\s*by\s/.test(line)) {
-        list = ['day','week','month'].filter(text=>text.startsWith(token)).map(text=>({text,detail:'Group results'}));
+        list = [...language.groups, ...metrics.map(metric=>`${metric.name} bins 5 ${metric.units[0]}`)].filter(text=>text.startsWith(token)).map(text=>({text,detail:'Group results'}));
       } else if (/^\s*sort by\s+\w+\s+/.test(line)) {
         list = ['asc','desc'].filter(text=>text.startsWith(token)).map(text=>({text,detail:'Evidence order'}));
-      } else if (/^\s*(where|and)\s+\w+\s+(?:>=|<=|>|<|=|!=|between)\s*$/.test(line)) {
+      } else if (/^\s*(where|and|or)\s+\w+\s+(?:>=|<=|>|<|=|!=|between)\s*$/.test(line)) {
         const field=line.trim().split(/\s+/)[1];
-        const values={distance:'8 km',pace:'5:30 /km',duration:'45 min',heart_rate:'145 bpm',cadence:'175 spm',power:'250 W',elevation:'100 m',ground_contact:'250 ms',stride_length:'1 m'};
+        const values={distance:'8 km',pace:'5:30 /km',duration:'45 min',heart_rate:'145 bpm',cadence:'175 spm',power:'250 W',elevation:'100 m',ground_contact:'250 ms',stride_length:'1 m',gap_since_previous_run:'7 days',speed:'12 km/h',rolling_7d_distance:'30 km',run_day_streak:'3 days'};
         const between=/between\s*$/.test(line);
         const ranges={distance:'8 and 20 km',pace:'5:00 and 6:00 /km',duration:'30 and 60 min',heart_rate:'140 and 150 bpm',cadence:'170 and 185 spm',power:'200 and 300 W',elevation:'50 and 150 m',ground_contact:'230 and 270 ms',stride_length:'0.8 and 1.2 m'};
-        list=values[field]?[{text:between?ranges[field]:values[field],detail:'Example value · adjust to your question'}]:[];
+        const metric=availableMetrics().find(metric=>metric.name===field); const example=values[field] || (metric?.units.length ? `5 ${metric.units[0]}` : null); list=example?[{text:between?(ranges[field] || `5 and 10 ${metric.units[0]}`):example,detail:'Example value · adjust to your question'}]:[];
         from=cursor;
-      } else if (/^\s*(where|and)\s+\w+\s+(?:>=|<=|>|<|=|!=|between)\s/.test(line) && /\d\s+[^ ]*$/.test(line)) {
+      } else if (/^\s*(where|and|or)\s+\w+\s+(?:>=|<=|>|<|=|!=|between)\s/.test(line) && /\d\s+[^ ]*$/.test(line)) {
         const field = line.trim().split(/\s+/)[1];
-        const units = {duration:['h','min','s'],distance:['km','m'],pace:['/km']};
-        list = (units[field] || [metrics.find(metric=>metric.name===field)?.unit]).filter(Boolean).filter(text=>text.startsWith(token)).map(text=>({text,detail:'Unit'}));
+        list = (availableMetrics().find(metric=>metric.name===field)?.units || []).filter(text=>text.startsWith(token)).map(text=>({text,detail:'Unit'}));
       } else {
         const prefix=line.trimStart();
         from = {line:cursor.line,ch:0};
@@ -135,7 +166,7 @@
       const result=await response.json();
       if(number!==requestNumber||text!==getValue())return null;
       clearErrors();
-      validated={text,...result};run.disabled=!result.valid;
+      validated={text,...result};run.disabled=!result.valid; if(result.valid) derivedMetadata=Object.fromEntries((result.derived||[]).map(spec=>[spec.name,spec]));
       validation.className=result.valid?'query-valid':'query-invalid';
       validation.textContent=result.valid?(result.canonical.trim()===root.dataset.query.trim()?'Valid query':'Valid query · changes not run'):'Fix the highlighted clause';
       (result.diagnostics||[]).forEach(diagnostic=>{

app/static/stats.css

@@ -5,3 +5,7 @@
 #stats-studio .CodeMirror{height:auto;min-height:235px;border:1px solid #465054;border-radius:9px;background:#111719;color:#e0e5e6;font:15px/1.9 Consolas,"Cascadia Code",monospace;overflow:hidden}#stats-studio .CodeMirror-scroll{max-height:420px;min-height:235px}#stats-studio .CodeMirror-lines{padding:16px 0}#stats-studio .CodeMirror-gutters{background:#151c1f;border-right:1px solid #2c363b}#stats-studio .CodeMirror-linenumber{color:#76888e;padding:0 8px 0 6px}#stats-studio .query-errors{width:16px}#stats-studio .CodeMirror-cursor{border-left-color:#f5f2ed}#stats-studio .CodeMirror-selected{background:#36484e}#stats-studio .cm-keyword{color:#ffa777}#stats-studio .cm-variable-2{color:#a9d8ea}#stats-studio .cm-number{color:#d7c9f4}#stats-studio .cm-atom{color:#b9d895}#stats-studio .cm-string{color:#bbdca7}#stats-studio .cm-comment{color:#8b999e;font-style:italic}#stats-studio .cm-operator{color:#efce98}.CodeMirror-hints{background:#212b30;color:#e2e8e8;border:1px solid #4e6168;border-radius:8px;padding:5px;max-width:min(410px,calc(100vw - 28px));max-height:260px;box-shadow:0 12px 32px #0006;z-index:1200}.CodeMirror-hint{color:#e2e8e8;padding:8px 10px;display:flex;flex-direction:column;gap:2px;white-space:normal;font:13px/1.5 Consolas,monospace;border-radius:5px}.CodeMirror-hint small{color:#a6bac2;font:11px/1.5 "Segoe UI",sans-serif}.CodeMirror-hint-active{background:#3c4f54;color:#fff}.CodeMirror-hint-active small{color:#d0dfe4}
 @media(max-width:680px){.query-reference{grid-template-columns:1fr}.query-actions kbd{display:none}.query-shortcut{display:none}#stats-studio .CodeMirror{font-size:14px}.metric-reference{grid-template-columns:1fr 1fr}.query-actions .button{font-size:11px}}
 @media(max-width:900px){.studio-filters{grid-template-columns:repeat(2,minmax(0,1fr))}.studio-charts{grid-template-columns:1fr}}@media(max-width:680px){.question-panel{padding:18px}.studio-sentence{font-size:18px}.studio-sentence select{font-size:16px;padding:7px}.studio-filters{grid-template-columns:1fr 1fr;gap:12px}.studio-metrics{grid-template-columns:1fr}.stats-nav{padding:8px;font-size:12px}.app-header{gap:6px}.studio-charts{padding:16px}.studio-chart-help{padding:0 16px 18px}.studio-evidence .section-head{flex-wrap:wrap}.studio-row .field-help{max-width:100%}}
+
+.studio-groups,.studio-scatter{margin-top:24px}.studio-scatter .studio-chart{padding:0 26px 24px}.studio-groups .section-head,.studio-scatter .section-head{flex-wrap:wrap;gap:12px}.group-drill{background:none;color:var(--ink);padding:0;text-align:left;border-bottom:1px dotted var(--green);font:inherit;cursor:pointer}.group-drill:hover{color:var(--green)}.previous-run{display:block;color:var(--muted);margin-top:8px;font-size:11px}.previous-run a{font-size:11px}#stats-studio .cm-builtin{color:#e9bfec}.studio-groups .activity-table-wrap{max-height:480px;overflow:auto}.studio-groups th{position:sticky;top:0;background:var(--white);z-index:1}
+
+.scatter-wrap{height:320px;margin:0 26px 20px;min-width:0}@media(max-width:680px){.scatter-wrap{height:280px;margin:0 16px 18px}}

app/static/stats.js

@@ -43,11 +43,11 @@
   let pending = null;
   clear.onclick = () => { pending?.abort(); period.value = ''; tbody.replaceChildren(...original.map(row => row.cloneNode(true))); pagination.replaceChildren(...originalPagination.map(node => node.cloneNode(true))); clear.hidden = true; document.getElementById('evidence-heading').textContent = 'Contributing runs'; status.textContent = ''; };
   period.onchange = () => { if (period.value) loadBucket(period.value); else clear.click(); };
-  async function loadBucket(bucket, page = 1) {
+  async function loadBucket(bucket, page = 1, scope = 'trend') {
     pending?.abort();
     const controller = new AbortController(); pending = controller;
     const url = new URL('/stats/runs', location.origin);
-    url.search = new URLSearchParams({query:root.dataset.query,bucket,page});
+    url.search = new URLSearchParams({query:root.dataset.query,bucket,page,scope});
     status.textContent = 'Loading contributing runs…';
     try {
       const response = await fetch(url, {signal:controller.signal});
@@ -58,6 +58,12 @@
         const tr = document.createElement('tr'); const td = document.createElement('td');
         const a = document.createElement('a'); a.href = `/activities/${row.id}`; a.textContent = `${row.title} · ${row.date}`;
         td.append(a); tr.append(td);
+        if (root.dataset.showPrevious === 'true') {
+          const detail=document.createElement('small');detail.className='previous-run';
+          detail.append('Previous: ');
+          if(row.previous_run){const previous=document.createElement('a');previous.href=`/activities/${row.previous_run.id}`;previous.textContent=`${row.previous_run.date} · ${row.previous_run.title}`;detail.append(previous);}else detail.append('No earlier recorded run');
+          td.append(detail);
+        }
         [`${row.distance} km`, ...row.values].forEach(text => { const cell = document.createElement('td'); cell.textContent = text; tr.append(cell); });
         tbody.append(tr);
       });
@@ -66,23 +72,24 @@
       pagination.replaceChildren();
       const label = document.createElement('span'); label.textContent = `Page ${result.page} of ${result.pages} · selected period`; pagination.append(label);
       [[result.page > 1, 'Previous', result.page - 1], [result.page < result.pages, 'Next', result.page + 1]].forEach(([show, name, target]) => {
-        if (!show) return; const button = document.createElement('button'); button.type = 'button'; button.className = 'button secondary'; button.textContent = name; button.onclick = () => loadBucket(bucket, target); pagination.append(button);
+        if (!show) return; const button = document.createElement('button'); button.type = 'button'; button.className = 'button secondary'; button.textContent = name; button.onclick = () => loadBucket(bucket, target, scope); pagination.append(button);
       });
       clear.hidden = false; status.textContent = 'Selected period loaded.';
     } catch (error) { if (error.name !== 'AbortError') status.textContent = 'Could not load this period. Try again.'; }
   }
+  root.querySelectorAll('[data-bucket]').forEach(button=>button.onclick=()=>loadBucket(button.dataset.bucket,1,'group'));
   chartData.series.forEach((series, index) => {
     const canvas = document.getElementById(`stats-chart-${index}`);
     if (!canvas) return;
     new Chart(canvas, {
-      type: ['distance', 'duration', 'elevation'].includes(series.key) ? 'bar' : 'line',
+      type: chartData.type === 'auto' ? (['bin','weekday','hour','run','all'].includes(chartData.group)||['distance','duration','elevation','count'].includes(series.key)?'bar':'line') : chartData.type,
       data: {labels: chartData.labels, datasets: [
         {label:'Current', data:series.values, borderColor:'#f48459', backgroundColor:'#f4845940', borderWidth:2, pointRadius:3, tension:.2, spanGaps:false},
         ...(series.previous.length ? [{label:'Previous (aligned)', data:series.previous, borderColor:'#9ba4a5', backgroundColor:'#9ba4a526', borderDash:[5,5], borderWidth:2, pointRadius:2, spanGaps:false}] : [])
       ]},
       options: {responsive:true, maintainAspectRatio:false, animation:false,
-        plugins:{legend:{labels:{color:'#bcc4c5'}}, tooltip:{callbacks:{label:c => `${c.dataset.label}: ${c.parsed.y == null ? '—' : series.key === 'pace' ? pace(c.parsed.y) : c.parsed.y.toFixed(1)} ${series.unit}`}}},
-        scales:{x:{ticks:{color:'#9ba4a5',maxTicksLimit:5},grid:{display:false}},y:{title:{display:true,text:series.unit,color:'#9ba4a5'},ticks:{color:'#9ba4a5',callback:n=>series.key==='pace'?pace(n):new Intl.NumberFormat(undefined,{maximumFractionDigits:2}).format(n)},grid:{color:'#303638'}}},
+        plugins:{legend:{labels:{color:'#bcc4c5'}}, tooltip:{callbacks:{label:c => `${c.dataset.label}: ${c.parsed.y == null ? '—' : series.pace_axis ? pace(c.parsed.y) : c.parsed.y.toFixed(2)} ${series.unit}`}}},
+        scales:{x:{ticks:{color:'#9ba4a5',maxTicksLimit:5},grid:{display:false}},y:{title:{display:true,text:series.unit,color:'#9ba4a5'},ticks:{color:'#9ba4a5',callback:n=>series.pace_axis?pace(n):new Intl.NumberFormat(undefined,{maximumFractionDigits:2}).format(n)},grid:{color:'#303638'}}},
         onClick:(_, elements) => {
           const point = elements.find(e => e.datasetIndex === 0); if (!point) return;
           const bucket = chartData.labels[point.index];
@@ -91,4 +98,10 @@
       }
     });
   });
+  const scatter=chartData.scatter;
+  const scatterCanvas=document.getElementById('stats-scatter');
+  if(scatter&&scatterCanvas){
+    const numeric=(n,info)=>info[1]==='/km'?pace(n):new Intl.NumberFormat(undefined,{maximumFractionDigits:2}).format(n);
+    new Chart(scatterCanvas,{type:'scatter',data:{datasets:[{label:'Current',data:scatter.current,backgroundColor:'#f48459',pointRadius:4},...(scatter.previous.length?[{label:'Previous',data:scatter.previous,backgroundColor:'#9ba4a5',pointRadius:3}]:[])]},options:{responsive:true,maintainAspectRatio:false,animation:false,plugins:{legend:{labels:{color:'#bcc4c5'}},tooltip:{callbacks:{label:c=>`${c.raw.label}: ${numeric(c.raw.x,scatter.x)} ${scatter.x[1]} / ${numeric(c.raw.y,scatter.y)} ${scatter.y[1]}`}}},scales:{x:{title:{display:true,text:`${scatter.x[0]} (${scatter.x[1]})`,color:'#9ba4a5'},ticks:{color:'#9ba4a5',callback:n=>numeric(n,scatter.x)},grid:{color:'#303638'}},y:{title:{display:true,text:`${scatter.y[0]} (${scatter.y[1]})`,color:'#9ba4a5'},ticks:{color:'#9ba4a5',callback:n=>numeric(n,scatter.y)},grid:{color:'#303638'}}},onClick:(_,elements,chart)=>{const point=elements[0];if(point){const row=chart.data.datasets[point.datasetIndex].data[point.index];location.assign(`/activities/${row.id}`);}}}});
+  }
 })();

app/stats.py

@@ -8,9 +8,8 @@ from datetime import UTC, date, datetime, timedelta
 from zoneinfo import ZoneInfo
 
 from .analytics import pace_text
-from .extensions import db
-from .models import Activity
-from .stats_dsl import UNITS, QueryError, parse_dsl
+from .stats_analysis import EXTRA_METRICS, EXTRA_UNITS, parse_analysis
+from .stats_dsl import UNITS, QueryError
 
 METRICS = {
     "pace": ("Pace", "/km", "Total timer time divided by total distance."),
@@ -31,6 +30,8 @@ METRICS = {
         "Recorded sample averages weighted by run timer duration.",
     ),
 }
+METRICS.update(EXTRA_METRICS)
+
 DEFAULT = {
     "metric": "pace",
     "secondary": "heart_rate",
@@ -48,13 +49,25 @@ DEFAULT = {
     "extra_metrics": [],
     "filters": [],
     "sort": ["date", "desc"],
+    "all_time": False,
+    "bin": None,
+    "take": None,
+    "limit": None,
+    "rank": None,
+    "trend": None,
+    "plot": None,
+    "chart": "auto",
+    "having": [],
+    "derived": [],
+    "aggregates": {},
+    "filter_groups": [],
 }
 
 
 def parse_query(args):
     q = dict(DEFAULT)
     if "query" in args:
-        return parse_dsl(args["query"], METRICS, DEFAULT)
+        return parse_analysis(args["query"], METRICS, DEFAULT)
     for key in (
         "metric",
         "secondary",
@@ -98,7 +111,9 @@ def number_text(value):
 
 
 def query_text(q):
-    if q["start"]:
+    if q["all_time"]:
+        period = "during all time"
+    elif q["start"]:
         period = f"during {q['start']} to {q['end']}"
     elif q["months"]:
         period = f"during last {q['months']} months"
@@ -107,9 +122,12 @@ def query_text(q):
     else:
         period = f"during last {q['weeks']} weeks"
     lines = ["runs", "  " + period]
+    for spec in q["derived"]:
+        lines.append(f"  let {spec['name']} = {spec['expression']}")
     if q["end"] and not q["start"]:
         lines.append(f"  ending {q['end']}")
-    lines.append(f"  where distance >= {number_text(q['minimum'])} km")
+    if not q["filter_groups"] or q["minimum"]:
+        lines.append(f"  where distance >= {number_text(q['minimum'])} km")
     for key, clause, unit in (
         ("maximum", "distance <=", "km"),
         ("hr_min", "heart_rate >=", "bpm"),
@@ -117,33 +135,74 @@ def query_text(q):
     ):
         if q[key] is not None:
             lines.append(f"  and {clause} {number_text(q[key])} {unit}")
-    for condition in q["filters"]:
+    units = {**UNITS, **EXTRA_UNITS, **{spec["name"]: {spec["unit"]: 1} for spec in q["derived"]}}
+
+    def clause(condition, prefix="and"):
         field, op = condition["field"], condition["op"]
         if op in {"has", "missing"}:
-            lines.append(f"  and {op} {field}")
-        elif op == "contains":
+            return f"  {prefix} {op} {field}"
+        if op == "contains":
+            return f"  {prefix} title contains {json.dumps(condition['value'], ensure_ascii=False)}"
+        unit = next(iter(units[field]))
+        value = condition["value"] / units[field][unit]
+        formatted = number_text(value)
+        if unit == "/km" and value > 0 and float(value).is_integer():
+            formatted = pace_text(value)
+        elif field == "duration" and value * 60 == round(value * 60) and value < 1:
+            formatted, unit = str(round(value * 60)), "min"
+        return f"  {prefix} {field} {op} {formatted} {unit}"
+
+    for index, conditions in enumerate(q["filter_groups"]):
+        for position, condition in enumerate(conditions):
             lines.append(
-                f"  and title contains {json.dumps(condition['value'], ensure_ascii=False)}"
+                clause(
+                    condition,
+                    "or"
+                    if index > 0 and position == 0
+                    else "where"
+                    if index == 0 and position == 0
+                    else "and",
+                )
             )
-        else:
-            unit = next(iter(UNITS[field]))
-            value = condition["value"] / UNITS[field][unit]
-            formatted = number_text(value)
-            if field == "pace" and value > 0 and value.is_integer():
-                formatted = pace_text(value)
-            elif field == "duration" and value * 60 == round(value * 60) and value < 1:
-                formatted, unit = str(round(value * 60)), "min"
-            lines.append(f"  and {field} {op} {formatted} {unit}")
+    for condition in q["filters"]:
+        lines.append(clause(condition))
     metrics = ", ".join(query_metrics(q))
     lines.extend(
         [
             f"  summarize {metrics}",
-            f"  by {q['group']}",
+            (
+                f"  by {q['bin']['field']} bins {number_text(q['bin']['width'])} {next(iter(units[q['bin']['field']]))}"
+                if q["bin"]
+                else f"  by {q['group']}"
+            ),
             "  compare " + ("previous period" if q["compare"] else "none"),
         ]
     )
     if q["sort"] != ["date", "desc"]:
         lines.append(f"  sort by {q['sort'][0]} {q['sort'][1]}")
+    for condition in q["having"]:
+        key = condition["field"]
+        spec = q["aggregates"].get(key)
+        field = spec["field"] if spec else key
+        unit = (
+            "runs"
+            if spec and spec["function"] == "count"
+            else "sec/km"
+            if spec and field == "pace" and spec["function"] in {"sum", "stddev"}
+            else next(iter(units[field]))
+        )
+        lines.append(f"  having {key} {condition['op']} {number_text(condition['value'])} {unit}")
+    if q["rank"]:
+        lines.append(f"  rank by {q['rank'][0]} {q['rank'][1]}")
+    for name in ("take", "limit"):
+        if q[name]:
+            lines.append(f"  {name} {q[name]}")
+    if q["trend"]:
+        lines.append(f"  trend {q['trend']}")
+    if q["plot"]:
+        lines.append(f"  plot {q['plot'][0]} vs {q['plot'][1]}")
+    if q["chart"] != "auto":
+        lines.append(f"  chart {q['chart']}")
     return "\n".join(lines)
 
 
@@ -258,135 +317,7 @@ def period_bounds(q, timezone):
     return start, end, previous
 
 
-def explore(owner_id, timezone, q, page=1, selected_bucket=None):
-    zone = ZoneInfo(timezone)
-    start, end, previous = period_bounds(q, timezone)
-    end_day = end - timedelta(days=1)
-    boundary = lambda day: datetime.combine(day, datetime.min.time(), zone).astimezone(UTC)
-    filters = [
-        Activity.owner_id == owner_id,
-        Activity.sport == "running",
-        Activity.started_at >= boundary(previous if q["compare"] else start),
-        Activity.started_at < boundary(end),
-        Activity.distance_m >= q["minimum"] * 1000,
-    ]
-    for key, column, multiplier, comparison in (
-        ("maximum", Activity.distance_m, 1000, "le"),
-        ("hr_min", Activity.avg_hr, 1, "ge"),
-        ("hr_max", Activity.avg_hr, 1, "le"),
-    ):
-        if q[key] is not None:
-            filters.append(
-                column <= q[key] * multiplier if comparison == "le" else column >= q[key]
-            )
-    rows = db.session.execute(
-        db.select(
-            Activity.id,
-            Activity.title,
-            Activity.started_at,
-            Activity.distance_m,
-            Activity.duration_s,
-            Activity.elevation_m,
-            Activity.avg_hr,
-            Activity.analysis_summary,
-        )
-        .where(*filters)
-        .order_by(Activity.started_at.desc(), Activity.id.desc())
-    ).all()
-    localday = lambda r: (
-        (r.started_at.replace(tzinfo=UTC) if r.started_at.tzinfo is None else r.started_at)
-        .astimezone(zone)
-        .date()
-    )
-    rows = [r for r in rows if matches(r, q["filters"])]
-    current = [r for r in rows if localday(r) >= start]
-    prior = [r for r in rows if localday(r) < start]
-    keys = query_metrics(q)
-    cards = []
-    for key in keys:
-        value, coverage = aggregate(current, key)
-        before, _ = aggregate(prior, key)
-        cards.append(
-            {
-                "key": key,
-                "label": METRICS[key][0],
-                "unit": METRICS[key][1],
-                "definition": METRICS[key][2],
-                "value": format_value(value, key),
-                "coverage": coverage,
-                "previous": format_value(before, key),
-                "delta": f"{value - before:+.1f}"
-                if value is not None and before is not None
-                else None,
-            }
-        )
-    bucket = lambda day: (
-        day
-        if q["group"] == "day"
-        else day.replace(day=1)
-        if q["group"] == "month"
-        else day - timedelta(days=day.weekday())
-    )
-    dates = sorted({bucket(start + timedelta(days=n)) for n in range((end - start).days)})
-    series = []
-    for key in keys:
-        values, prior_values = [], []
-        for day in dates:
-            values.append(aggregate([r for r in current if bucket(localday(r)) == day], key)[0])
-            # Align prior runs by elapsed days into the equally long comparison period.
-            prior_values.append(
-                aggregate(
-                    [r for r in prior if bucket(localday(r) + (start - previous)) == day], key
-                )[0]
-            )
-        series.append(
-            {
-                "key": key,
-                "label": METRICS[key][0],
-                "unit": METRICS[key][1],
-                "values": values,
-                "previous": prior_values if q["compare"] else [],
-            }
-        )
-    evidence = current
-    if selected_bucket is not None:
-        if selected_bucket not in dates:
-            raise ValueError("Choose a period from the current chart.")
-        evidence = [r for r in current if bucket(localday(r)) == selected_bucket]
-    sort_key, direction = q["sort"]
-    if sort_key != "date":
-        available = [r for r in evidence if metric_value(r, sort_key) is not None]
-        unavailable = [r for r in evidence if metric_value(r, sort_key) is None]
-        evidence = (
-            sorted(available, key=lambda r: metric_value(r, sort_key), reverse=direction == "desc")
-            + unavailable
-        )
-    elif direction == "asc":
-        evidence = list(reversed(evidence))
-    pages = max(1, math.ceil(len(evidence) / 25))
-    if not 1 <= page <= pages:
-        raise ValueError("That results page does not exist.")
-    return {
-        "cards": cards,
-        "chart": {"labels": [d.strftime("%d %b %Y") for d in dates], "series": series},
-        "count": len(current),
-        "prior_count": len(prior),
-        "distance": sum(r.distance_m for r in current) / 1000,
-        "rows": [
-            {
-                "id": r.id,
-                "title": r.title,
-                "date": localday(r).strftime("%d %b %Y"),
-                "distance": f"{r.distance_m / 1000:.2f}",
-                "values": [format_value(metric_value(r, k), k) for k in keys],
-            }
-            for r in evidence[(page - 1) * 25 : page * 25]
-        ],
-        "start": start,
-        "end": end_day,
-        "previous_start": previous,
-        "previous_end": start - timedelta(days=1),
-        "page": page,
-        "pages": pages,
-        "evidence_count": len(evidence),
-    }
+def explore(owner_id, timezone, q, page=1, selected_bucket=None, export=False, raw_group=False):
+    from .analysis_engine import explore as analyze
+
+    return analyze(owner_id, timezone, q, page, selected_bucket, export, raw_group)

app/stats_analysis.py

@@ -0,0 +1,492 @@
+"""Typed analysis expressions and extensions for the run query language."""
+
+import ast
+import math
+import re
+import statistics
+
+from .stats_dsl import UNITS, QueryError, parse_dsl
+
+FUNCTIONS = {
+    "sum",
+    "mean",
+    "weighted_mean",
+    "median",
+    "min",
+    "max",
+    "stddev",
+    "count",
+    "p10",
+    "p25",
+    "p75",
+    "p90",
+    "p95",
+    "p99",
+}
+GROUPS = {"day", "week", "month", "year", "weekday", "hour", "run", "all"}
+EXTRA_METRICS = {
+    "count": ("Runs", "runs", "Number of runs."),
+    "speed": ("Speed", "km/h", "Recorded distance divided by timer duration."),
+    "gap_since_previous_run": (
+        "Gap since previous run",
+        "days",
+        "Elapsed start-to-start time since your previous run, including runs outside this query. This is not time since the previous run ended.",
+    ),
+    "distance_change": (
+        "Distance change",
+        "km",
+        "Distance minus the previous run's distance, before filtering.",
+    ),
+    "pace_change": (
+        "Pace change",
+        "sec/km",
+        "Pace minus the previous run's pace. Negative means faster.",
+    ),
+    "heart_rate_change": (
+        "Heart rate change",
+        "bpm",
+        "Average heart rate minus the previous run's average, before filtering.",
+    ),
+    "rolling_7d_distance": (
+        "7-day distance",
+        "km",
+        "Distance in the preceding 168 hours, including this run. Calculated before filters.",
+    ),
+    "rolling_28d_distance": (
+        "28-day distance",
+        "km",
+        "Distance in the preceding 672 hours, including this run. Calculated before filters.",
+    ),
+    "rolling_7d_count": (
+        "7-day run count",
+        "runs",
+        "Number of runs in the preceding 168 hours, including this run, before filters.",
+    ),
+    "rolling_28d_count": (
+        "28-day run count",
+        "runs",
+        "Number of runs in the preceding 672 hours, including this run, before filters.",
+    ),
+    "run_day_streak": (
+        "Consecutive running days",
+        "days",
+        "Consecutive local calendar days with a run, through this run's local date, before filters.",
+    ),
+    "elevation_per_km": (
+        "Ascent per kilometre",
+        "m/km",
+        "Recorded ascent divided by recorded distance.",
+    ),
+    "max_heart_rate": ("Maximum heart rate", "bpm", "Recorded maximum heart rate."),
+    "vertical_oscillation": (
+        "Vertical oscillation",
+        "cm",
+        "Recorded sample average, weighted by run timer duration.",
+    ),
+    "calories": ("Calories", "kcal", "Recorded session calories. Missing values are excluded."),
+    "training_effect": (
+        "Aerobic training effect",
+        "score",
+        "Recorded Garmin aerobic training effect; an unweighted mean across runs.",
+    ),
+}
+EXTRA_UNITS = {
+    "count": {"runs": 1},
+    "speed": {"km/h": 1, "m/s": 3.6},
+    "gap_since_previous_run": {"days": 1, "h": 1 / 24, "min": 1 / 1440},
+    "distance_change": {"km": 1, "m": 0.001},
+    "pace_change": {"sec/km": 1},
+    "heart_rate_change": {"bpm": 1},
+    "rolling_7d_distance": {"km": 1, "m": 0.001},
+    "rolling_28d_distance": {"km": 1, "m": 0.001},
+    "rolling_7d_count": {"runs": 1},
+    "rolling_28d_count": {"runs": 1},
+    "run_day_streak": {"days": 1},
+    "elevation_per_km": {"m/km": 1},
+    "max_heart_rate": {"bpm": 1},
+    "vertical_oscillation": {"cm": 1},
+    "calories": {"kcal": 1},
+    "training_effect": {"score": 1},
+}
+
+
+def valid(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def expression_tree(source, catalog, line=1):
+    if len(source) > 200:
+        raise QueryError("Keep each calculation under 200 characters.", line)
+    try:
+        tree = ast.parse(source, mode="eval")
+    except SyntaxError:
+        raise QueryError("Use metric names, numbers, parentheses and + - * /.", line) from None
+    if len(list(ast.walk(tree))) > 64:
+        raise QueryError("Simplify this calculation.", line)
+
+    def unit(node):
+        if isinstance(node, ast.Expression):
+            return unit(node.body)
+        if isinstance(node, ast.Name) and node.id in catalog:
+            # Pace values inside arithmetic are seconds per km, not a formatted string.
+            return (
+                "sec/km"
+                if node.id == "pace"
+                else "h"
+                if node.id == "duration"
+                else catalog[node.id][1]
+            )
+        if isinstance(node, ast.Constant) and valid(node.value) and abs(node.value) <= 1000000:
+            return "ratio"
+        if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
+            return unit(node.operand)
+        if isinstance(node, ast.BinOp) and isinstance(
+            node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div)
+        ):
+            left, right = unit(node.left), unit(node.right)
+            if isinstance(node.op, (ast.Add, ast.Sub)):
+                if left != right:
+                    raise QueryError(
+                        f"Cannot add/subtract {left} and {right}. Use matching units.", line
+                    )
+                return left
+            if isinstance(node.op, ast.Div) and left == right:
+                return "ratio"
+            if right == "ratio":
+                return left
+            if left == "ratio" and isinstance(node.op, ast.Mult):
+                return right
+            if isinstance(node.op, ast.Div) and (left, right) in {("km", "h"), ("m", "km")}:
+                return f"{left}/{right}"
+            return f"({left}{'*' if isinstance(node.op, ast.Mult) else '/'}{right})"
+        raise QueryError(
+            "Calculations allow existing metrics, numbers and + - * /. Calls, attributes and indexing are unavailable.",
+            line,
+        )
+
+    return tree, unit(tree)
+
+
+def evaluate(tree, values):
+    def visit(node):
+        if isinstance(node, ast.Expression):
+            return visit(node.body)
+        if isinstance(node, ast.Name):
+            return values.get(node.id)
+        if isinstance(node, ast.Constant):
+            return node.value
+        if isinstance(node, ast.UnaryOp):
+            value = visit(node.operand)
+            return None if not valid(value) else -value if isinstance(node.op, ast.USub) else value
+        left, right = visit(node.left), visit(node.right)
+        if not valid(left) or not valid(right):
+            return None
+        if isinstance(node.op, ast.Add):
+            result = left + right
+        elif isinstance(node.op, ast.Sub):
+            result = left - right
+        elif isinstance(node.op, ast.Mult):
+            result = left * right
+        elif right == 0:
+            return None
+        else:
+            result = left / right
+        return result if valid(result) else None
+
+    return visit(tree)
+
+
+def parse_analysis(text, metrics, defaults):
+    if not isinstance(text, str) or len(text) > 4000:
+        raise QueryError("Keep the query under 4,000 characters.")
+    catalog, units = dict(metrics), {**UNITS, **EXTRA_UNITS}
+    lines = text.splitlines()
+    derived, aggregates, seen, locations = [], {}, set(), {}
+    options = {
+        "all_time": False,
+        "bin": None,
+        "take": None,
+        "limit": None,
+        "rank": None,
+        "trend": None,
+        "plot": None,
+        "chart": "auto",
+        "having": [],
+    }
+    group = None
+    or_lines = any(re.match(r"\s*or\s", line) for line in lines)
+    filter_counts = []
+    aliases = {}
+
+    def once(key, line):
+        if key in seen:
+            raise QueryError(f"Repeated {key} clause.", line)
+        seen.add(key)
+        locations[key] = line
+
+    def aggregate(source, line):
+        source = source.strip()
+        match = re.fullmatch(r"(\w+)\s*\(\s*(\w*)\s*\)", source)
+        if not match:
+            if source not in catalog:
+                raise QueryError(f"Unknown metric '{source}'.", line)
+            return source
+        function, field = match.groups()
+        if function not in FUNCTIONS or (not field and function != "count"):
+            raise QueryError(
+                "Use sum, mean, weighted_mean, median, min, max, stddev, count or p10/p25/p75/p90/p95/p99.",
+                line,
+            )
+        if field and field not in catalog:
+            raise QueryError(f"Unknown metric '{field}'.", line)
+        key = f"{function}({field})"
+        aggregates[key] = {"function": function, "field": field}
+        return key
+
+    for index, source in enumerate(lines):
+        line, number = source.strip(), index + 1
+        if not line or line.startswith("#"):
+            continue
+        if match := re.fullmatch(r"let (\w+) = (.+)", line):
+            name, expression = match.groups()
+            if (
+                not re.fullmatch(r"[a-z][a-z0-9_]{0,39}", name)
+                or name.startswith("analysis_")
+                or name in catalog
+                or len(derived) >= 8
+            ):
+                raise QueryError(
+                    "Choose a new lowercase metric name; at most eight calculations.", number
+                )
+            _, unit = expression_tree(expression, catalog, number)
+            catalog[name] = (
+                name.replace("_", " ").capitalize(),
+                unit,
+                f"Calculated per run: {expression}. Missing inputs or division by zero give missing data.",
+            )
+            units[name] = {unit: 1}
+            derived.append({"name": name, "expression": expression, "unit": unit})
+            lines[index] = ""
+        elif line == "during all time":
+            once("all-time period", number)
+            options["all_time"] = True
+            lines[index] = "during last 12 weeks"
+        elif line.startswith("summarize "):
+            keys = [aggregate(part, number) for part in line[10:].split(",")]
+            if len(keys) != len(set(keys)):
+                raise QueryError("Choose different metrics or summary expressions.", number)
+            replacements = []
+            for key in keys:
+                if key in aggregates:
+                    alias = f"analysis_agg_{len(aliases)}"
+                    aliases[alias] = key
+                    catalog[alias] = (
+                        key,
+                        "runs"
+                        if aggregates[key]["function"] == "count"
+                        else catalog[aggregates[key]["field"]][1],
+                        "Aggregate",
+                    )
+                    units[alias] = {catalog[alias][1]: 1}
+                    replacements.append(alias)
+                else:
+                    replacements.append(key)
+            lines[index] = "summarize " + ", ".join(replacements)
+        elif match := re.fullmatch(r"by (\w+)(?: bins (\S+) (\S+))?", line):
+            once("by", number)
+            name, width, unit = match.groups()
+            if width:
+                if name not in catalog or name not in units or unit not in units[name]:
+                    raise QueryError("Choose a numeric metric and its unit for bins.", number)
+                try:
+                    size = float(width) * units[name][unit]
+                except ValueError:
+                    raise QueryError("Use a positive numeric bin width.", number) from None
+                if not valid(size) or size <= 0 or size > 1000000:
+                    raise QueryError("Use a positive finite bin width.", number)
+                options["bin"] = {"field": name, "width": size}
+                group = "bin"
+            else:
+                if name not in GROUPS:
+                    raise QueryError(
+                        "Group by day, week, month, year, weekday, hour, run, all, or metric bins.",
+                        number,
+                    )
+                group = name
+            lines[index] = "by week" if group not in {"day", "week", "month"} else line
+        elif match := re.fullmatch(r"(take|limit) (\d+)", line):
+            name, raw = match.groups()
+            once(name, number)
+            value = int(raw)
+            if not 1 <= value <= 1000:
+                raise QueryError("Use a limit from 1 to 1,000.", number)
+            options[name] = value
+            lines[index] = ""
+        elif match := re.fullmatch(r"rank by (.+) (asc|desc)", line):
+            once("rank", number)
+            options["rank"] = [aggregate(match[1], number), match[2]]
+            lines[index] = ""
+        elif match := re.fullmatch(r"having (.+?) (>=|<=|>|<|=|!=) (-?\S+)(?: (\S+))?", line):
+            key, op, raw, unit = match.groups()
+            key = aggregate(key, number)
+            function = aggregates.get(key, {}).get("function")
+            field = aggregates.get(key, {}).get("field", key)
+            accepted = (
+                {"runs": 1}
+                if function == "count"
+                else {"sec/km": 1}
+                if field == "pace" and function in {"sum", "stddev"}
+                else units[field]
+            )
+            unit = unit or next(iter(accepted))
+            if unit not in accepted:
+                raise QueryError(f"Use {' or '.join(accepted)} for {key}.", number)
+            try:
+                if field == "pace" and unit == "/km" and ":" in raw:
+                    minutes, seconds = raw.split(":")
+                    if not re.fullmatch(r"\d+:\d{2}", raw) or int(seconds) >= 60:
+                        raise ValueError
+                    value = int(minutes) * 60 + int(seconds)
+                else:
+                    value = float(raw) * accepted[unit]
+            except ValueError:
+                raise QueryError("Use a numeric threshold for having.", number) from None
+            if not valid(value) or abs(value) > 1000000:
+                raise QueryError("Use a finite threshold.", number)
+            if len(options["having"]) >= 10:
+                raise QueryError("Use at most ten having conditions.", number)
+            options["having"].append({"field": key, "op": op, "value": value})
+            lines[index] = ""
+        elif match := re.fullmatch(
+            r"trend (cumulative|change|percent_change|rolling (\d+) buckets?)", line
+        ):
+            once("trend", number)
+            options["trend"] = match[1]
+            if match[2] and not 1 <= int(match[2]) <= 52:
+                raise QueryError("Use 1–52 rolling buckets.", number)
+            lines[index] = ""
+        elif match := re.fullmatch(r"plot (\w+) vs (\w+)", line):
+            once("plot", number)
+            if any(name not in catalog for name in match.groups()):
+                raise QueryError("Choose two numeric run metrics for the scatter plot.", number)
+            options["plot"] = list(match.groups())
+            lines[index] = ""
+        elif match := re.fullmatch(r"chart (auto|line|bar|none)", line):
+            once("chart", number)
+            options["chart"] = match[1]
+            lines[index] = ""
+        elif re.match(r"(?:where|and|or) ", line):
+            if or_lines:
+                # Keep AND higher precedence than OR and prevent SQL bounds from swallowing alternatives.
+                rewritten = re.sub(
+                    r"^(where|and|or) (distance|heart_rate) ", r"\1 analysis_raw_\2 ", line
+                )
+                for field in ("distance", "heart_rate"):
+                    alias = "analysis_raw_" + field
+                    catalog[alias] = catalog[field]
+                    units[alias] = units[field]
+                lines[index] = re.sub(r"^or ", "and ", rewritten)
+                filter_counts.append(
+                    (
+                        line.startswith("or "),
+                        2 if re.match(r"^(where|and|or) \w+ between ", line) else 1,
+                    )
+                )
+    q = parse_dsl("\n".join(lines), catalog, defaults, units=units)
+    q.update(options)
+    q["derived"], q["aggregates"] = derived, aggregates
+    if group:
+        q["group"] = group
+    for key in ("metric", "secondary"):
+        q[key] = aliases.get(q[key], q[key])
+    q["extra_metrics"] = [aliases.get(key, key) for key in q["extra_metrics"]]
+    selected = [
+        q["metric"],
+        *([] if q["secondary"] == "none" else [q["secondary"]]),
+        *q["extra_metrics"],
+    ]
+    if len(set(selected)) != len(selected):
+        raise QueryError("Choose different summary expressions.")
+    if options["rank"] and options["rank"][0] not in selected:
+        raise QueryError("Include the ranking expression in summarize.", locations["rank"])
+    if options["trend"] and q["group"] not in {"day", "week", "month", "year"}:
+        raise QueryError("Trends require grouping by day, week, month or year.", locations["trend"])
+    if options["trend"] and options["rank"]:
+        raise QueryError(
+            "Ranked groups and time trends use different orderings. Use one per query.",
+            locations["trend"],
+        )
+    if options["plot"] and options["chart"] == "none":
+        raise QueryError("A scatter plot conflicts with chart none.", locations["chart"])
+    if options["all_time"] and (q["end"] or q["start"]):
+        raise QueryError("All time cannot also specify an end date.", locations["all-time period"])
+    if options["all_time"]:
+        if any(line.strip() == "compare previous period" for line in text.splitlines()):
+            raise QueryError(
+                "All time has no preceding period. Use compare none.", locations["all-time period"]
+            )
+        q["compare"] = False
+    q["filter_groups"] = []
+    if or_lines:
+        position, groups = 0, [[]]
+        for new_group, count in filter_counts:
+            if new_group:
+                if not groups[-1]:
+                    raise QueryError("Start an OR filter with where or and.")
+                groups.append([])
+            batch = q["filters"][position : position + count]
+            for condition in batch:
+                condition["field"] = condition["field"].removeprefix("analysis_raw_")
+            groups[-1].extend(batch)
+            position += count
+        q["filter_groups"], q["filters"] = groups, []
+    return q
+
+
+def describe(key, catalog, aggregates):
+    if key not in aggregates:
+        return catalog[key]
+    spec = aggregates[key]
+    function, field = spec["function"], spec["field"]
+    label = (
+        "Runs"
+        if not field
+        else f"{function.replace('_', ' ').capitalize()} · {catalog[field][0].lower()}"
+    )
+    unit = "runs" if function == "count" else catalog[field][1]
+    if field == "pace" and function in {"stddev", "sum"}:
+        unit = "sec/km"
+    definition = (
+        (
+            "Count of rows"
+            if not field
+            else f"{function} of available per-run {catalog[field][0].lower()} values"
+        )
+        + ". Percentiles use linear interpolation; stddev is population standard deviation. Missing values are excluded."
+    )
+    return label, unit, definition
+
+
+def statistic(values, function):
+    values = [value for value in values if valid(value)]
+    if function == "count":
+        return len(values)
+    if not values:
+        return None
+    if function == "sum":
+        return sum(values)
+    if function == "mean":
+        return statistics.fmean(values)
+    if function == "median":
+        return statistics.median(values)
+    if function == "min":
+        return min(values)
+    if function == "max":
+        return max(values)
+    if function == "stddev":
+        return statistics.pstdev(values)
+    percentile = int(function[1:]) / 100
+    ordered = sorted(values)
+    index = (len(ordered) - 1) * percentile
+    lower = math.floor(index)
+    return ordered[lower] + (ordered[math.ceil(index)] - ordered[lower]) * (index - lower)

app/stats_dsl.py

@@ -26,7 +26,8 @@ class QueryError(ValueError):
         self.line = line
 
 
-def parse_dsl(text, metrics, defaults):
+def parse_dsl(text, metrics, defaults, units=None):
+    units = units or UNITS
     if not isinstance(text, str) or len(text) > 4000:
         raise QueryError("Keep the query under 4,000 characters.")
     q = dict(defaults)
@@ -52,8 +53,8 @@ def parse_dsl(text, metrics, defaults):
             raise QueryError(f"Unknown metric '{key}'.{hint}", line)
 
     def amount(key, raw, unit, line):
-        if unit not in UNITS[key]:
-            raise QueryError(f"Use {' or '.join(UNITS[key])} for {key}.", line)
+        if unit not in units[key]:
+            raise QueryError(f"Use {' or '.join(units[key])} for {key}.", line)
         if key == "pace" and ":" in raw:
             minutes, seconds = raw.split(":")
             if not minutes.isdecimal() or not seconds.isdecimal() or int(seconds) >= 60:
@@ -64,9 +65,16 @@ def parse_dsl(text, metrics, defaults):
                 value = float(raw)
             except ValueError:
                 raise QueryError("Use a numeric value with its unit.", line) from None
-        if not math.isfinite(value) or not 0 <= value <= 1000000:
-            raise QueryError("Filter values must be finite and between 0 and 1,000,000.", line)
-        return value * UNITS[key][unit]
+        if not math.isfinite(value) or not -1000000 <= value <= 1000000:
+            raise QueryError(
+                "Filter values must be finite and between -1,000,000 and 1,000,000.", line
+            )
+        if (
+            key in {"distance", "heart_rate", "analysis_raw_distance", "analysis_raw_heart_rate"}
+            and value < 0
+        ):
+            raise QueryError("Distance and heart rate cannot be negative.", line)
+        return value * units[key][unit]
 
     def condition(key, op, value, line):
         legacy = {

app/templates/stats.html

@@ -2,11 +2,12 @@
 {% block title %}Stats studio · Stride{% endblock %}
 {% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='vendor/codemirror.css') }}"><link rel="stylesheet" href="{{ url_for('static', filename='vendor/show-hint.css') }}"><link rel="stylesheet" href="{{ url_for('static', filename='stats.css') }}">{% endblock %}
 {% block content %}
-<div id="stats-studio" data-owner="{{ current_user.id }}" data-query="{{ query_text }}" data-csrf="{{ csrf_token() }}" data-validate-url="{{ url_for('web.stats_validate') }}">
+<div id="stats-studio" data-owner="{{ current_user.id }}" data-show-previous="{{ 'true' if results and results.show_previous else 'false' }}" data-query="{{ query_text }}" data-csrf="{{ csrf_token() }}" data-validate-url="{{ url_for('web.stats_validate') }}">
 <div class="page-heading"><div><span class="eyebrow">YOUR RUNNING / STATS STUDIO</span><h1>Your runs. Your questions.</h1><p>Write a query, explore the pattern, and inspect the runs behind it.</p></div><button type="button" class="button secondary" id="save-view">☆ Save view</button></div>
 <div id="saved-views" class="studio-presets" aria-label="Saved views"></div>
+<script id="query-language" type="application/json">{{ language|tojson }}</script>
 <form method="get" id="explorer-form" class="panel question-panel" novalidate>
-<div class="studio-row"><div><span class="eyebrow">RUNNING QUERY</span><span class="query-caption">A small language for a deeper look.</span></div><div class="query-actions"><button type="button" class="button secondary" id="format-query">Format</button><button type="button" class="button secondary" id="complete-query">Suggest <kbd>Ctrl Space</kbd></button></div></div>
+<div class="studio-row"><div><span class="eyebrow">RUNNING QUERY</span><span class="query-caption">An analysis language for your running history.</span></div><div class="query-actions"><button type="button" class="button secondary" id="format-query">Format</button><button type="button" class="button secondary" id="complete-query">Suggest <kbd>Ctrl Space</kbd></button></div></div>
 <div id="query-editor"><label class="sr-only" for="query-input">Running stats query</label><textarea id="query-input" name="query" spellcheck="false" maxlength="4000" rows="9">{{ submitted_query if submitted_query else query_text }}</textarea></div>
 <div class="studio-row query-state"><span id="query-validation" role="status">Checking query...</span><span class="query-shortcut"><kbd>Ctrl / Cmd Enter</kbd> to run</span></div>
 <div id="query-diagnostics" aria-live="polite">{% if error %}<p class="studio-error" role="alert">{{ error }}</p>{% endif %}</div>
@@ -16,10 +17,16 @@ where distance between 8 and 20 km
 and pace &lt;= 5:30 /km
 and has cadence
 and title contains "River"
-sort by pace asc</pre><p class="field-help">Periods: days, weeks, calendar months, or YYYY-MM-DD to YYYY-MM-DD. Group by day, week or month. Comparisons: previous period or none. Operators: &gt;, &gt;=, &lt;, &lt;=, =, !=. Comments start with # on their own line.</p></div><div><h3>Metrics · select up to six</h3><div class="metric-reference">{% for key, info in metrics.items() %}<button type="button" class="metric-insert" data-metric="{{ key }}"><code>{{ key }}</code><span>{{ info[1] }}</span></button>{% endfor %}</div><p class="field-help">duration accepts h, min or s; distance accepts km or m. Use has / missing to select recorded sensor coverage.</p></div></div></details>
+sort by pace asc</pre><p class="field-help">Periods: rolling days/weeks/months, explicit dates, or all time. Groups: day, week, month, year, weekday, hour, run, all, or metric bins. Operators: &gt;, &gt;=, &lt;, &lt;=, =, !=. AND binds before OR. Comments start with # on their own line.</p><h3>Analysis operations</h3><pre>summarize count(), mean(distance), p90(pace)
+having count() &gt;= 3 runs
+rank by mean(distance) desc
+limit 10
+trend rolling 4 buckets
+let efficiency = speed / heart_rate
+plot pace vs heart_rate</pre><p class="field-help">Functions: sum, mean, weighted_mean, median, min, max, count, population stddev, and p10/p25/p75/p90/p95/p99. take limits ranked runs before grouping; limit limits groups. Trends also accept cumulative, change and percent_change. Arithmetic uses km, hours and seconds/km. Missing inputs and zero divisors give missing values.</p></div><div><h3>Metrics · select up to six</h3><div class="metric-reference">{% for key, info in metrics.items() %}<button type="button" class="metric-insert" data-metric="{{ key }}" title="{{ info[2] }}"><code>{{ key }}</code><span>{{ info[1] }}</span></button>{% endfor %}</div><p class="field-help">duration accepts h, min or s; distance accepts km or m. Use has / missing to select recorded sensor coverage.</p></div></div></details>
 <details id="guided-controls" class="query-help"><summary>Guided controls</summary><p class="field-help">Adjust the basic clauses without removing additional metrics and filters.</p>
-<div class="studio-sentence">Show <label class="sr-only" for="metric">Primary metric</label><select id="metric" name="metric">{% for key, info in metrics.items() %}<option value="{{ key }}" {{ 'selected' if query.metric == key }}>{{ info[0]|lower }}</option>{% endfor %}</select> and <label class="sr-only" for="secondary">Second metric</label><select id="secondary" name="secondary"><option value="none" {{ 'selected' if query.secondary == 'none' }}>no second metric</option>{% for key, info in metrics.items() %}<option value="{{ key }}" {{ 'selected' if query.secondary == key }}>{{ info[0]|lower }}</option>{% endfor %}</select><br>
-from runs during the last <label class="sr-only" for="weeks">Number of weeks</label><input id="weeks" name="weeks" type="number" min="1" max="104" value="{{ query.weeks }}" required> weeks, grouped by <label class="sr-only" for="group">Grouping</label><select id="group" name="group"><option value="day" {{ 'selected' if query.group == 'day' }}>day</option><option value="week" {{ 'selected' if query.group == 'week' }}>week</option><option value="month" {{ 'selected' if query.group == 'month' }}>month</option></select>.</div>
+<div class="studio-sentence">Show <label class="sr-only" for="metric">Primary metric</label><select id="metric" name="metric">{% if query.metric not in metrics %}<option value="{{ query.metric }}" selected>{{ query.metric }}</option>{% endif %}{% for key, info in metrics.items() %}<option value="{{ key }}" {{ 'selected' if query.metric == key }}>{{ info[0]|lower }}</option>{% endfor %}</select> and <label class="sr-only" for="secondary">Second metric</label><select id="secondary" name="secondary">{% if query.secondary not in metrics and query.secondary != 'none' %}<option value="{{ query.secondary }}" selected>{{ query.secondary }}</option>{% endif %}<option value="none" {{ 'selected' if query.secondary == 'none' }}>no second metric</option>{% for key, info in metrics.items() %}<option value="{{ key }}" {{ 'selected' if query.secondary == key }}>{{ info[0]|lower }}</option>{% endfor %}</select><br>
+{% if query.all_time or query.start or query.days or query.months %}Change the period to the last{% else %}from runs during the last{% endif %} <label class="sr-only" for="weeks">Number of weeks</label><input id="weeks" name="weeks" type="number" min="1" max="104" value="{{ query.weeks }}" required> weeks, grouped by <label class="sr-only" for="group">Grouping</label><select id="group" name="group">{% if query.group == 'bin' %}<option value="bin" selected disabled>{{ query.bin.field }} bins</option>{% endif %}{% for grouping in language.groups %}<option value="{{ grouping }}" {{ 'selected' if query.group == grouping }}>{{ grouping }}</option>{% endfor %}</select>.</div>
 <div class="studio-filters">
 <label>Distance at least · km<input name="minimum" type="number" min="0" max="1000" step="any" value="{{ query.minimum }}"></label>
 <label>Distance at most · km<input name="maximum" type="number" min="0" max="1000" step="any" value="{{ query.maximum if query.maximum is not none else '' }}" placeholder="Any"></label>
@@ -30,17 +37,19 @@ from runs during the last <label class="sr-only" for="weeks">Number of weeks</la
 </div>
 </details>
 </form>
-<div class="studio-presets" aria-label="Starting questions"><span class="subtle-label">START WITH</span><a href="{{ url_for('web.stats',metric='distance',secondary='duration') }}">Weekly volume</a><a href="{{ url_for('web.stats',metric='pace',secondary='heart_rate',minimum=10) }}">Long-run progression</a><a href="{{ url_for('web.stats',metric='pace',secondary='heart_rate',hr_min=140,hr_max=150) }}">Pace at similar heart rate</a><a href="{{ url_for('web.stats',metric='cadence',secondary='ground_contact') }}">Running form</a></div>
+<div class="studio-presets" aria-label="Starting questions"><span class="subtle-label">START WITH</span><a href="{{ url_for('web.stats',query='runs\n  during all time\n  summarize gap_since_previous_run, distance\n  by run\n  sort by gap_since_previous_run desc\n  take 10\n  compare none') }}">Biggest run gaps</a><a href="{{ url_for('web.stats',query='runs\n  during last 24 months\n  summarize count(), median(pace), sum(distance)\n  by weekday\n  having count() >= 3 runs\n  compare none') }}">Weekday patterns</a><a href="{{ url_for('web.stats',query='runs\n  during last 12 weeks\n  summarize distance, count()\n  by week\n  trend rolling 4 buckets\n  compare none') }}">Rolling volume</a><a href="{{ url_for('web.stats',metric='distance',secondary='duration') }}">Weekly volume</a><a href="{{ url_for('web.stats',metric='pace',secondary='heart_rate',minimum=10) }}">Long-run progression</a><a href="{{ url_for('web.stats',metric='pace',secondary='heart_rate',hr_min=140,hr_max=150) }}">Pace at similar heart rate</a><a href="{{ url_for('web.stats',metric='cadence',secondary='ground_contact') }}">Running form</a></div>
 {% if results %}
 <div class="studio-row result-heading"><div><h2>{{ results.count }} matching {{ 'run' if results.count == 1 else 'runs' }}</h2><p>{{ results.start }} – {{ results.end }} · {{ '%.1f'|format(results.distance) }} km{% if query.compare %}<br>Compared with {{ results.previous_start }} – {{ results.previous_end }} · {{ results.prior_count }} runs{% endif %}</p></div></div>
 {% if not results.count %}<div class="panel studio-empty"><h2>No runs match this question.</h2><p>Try a longer period or widen the distance and heart rate filters.</p></div>{% else %}
-<div class="studio-metrics">{% for card in results.cards %}<article class="stat-card"><span class="eyebrow">{{ card.label }}</span><div class="stat-value">{{ card.value }}<small>{{ card.unit }}</small></div>{% if query.compare %}<p>Previous: {{ card.previous }} {{ card.unit }}{% if card.delta %} · {{ card.delta }} {{ 'sec/km' if card.key == 'pace' else card.unit }}{% endif %}</p>{% endif %}<p>{{ card.coverage }} of {{ results.count }} runs with usable data</p><details><summary>How this is calculated</summary><p>{{ card.definition }}{% if card.key in ('cadence','power','ground_contact','stride_length') %} Coverage counts runs, not sensor recording time.{% endif %}</p></details></article>{% endfor %}</div>
-<section class="panel"><div class="section-head"><div><span class="eyebrow">TRENDS / {{ query.group|upper }}</span><h2>Your question over time</h2></div><span class="subtle-label">Solid: current · Dashed: previous</span></div><div class="studio-charts">{% for card in results.cards %}<div class="studio-chart"><h3>{{ card.label }} <small>{{ card.unit }}</small></h3><div><canvas id="stats-chart-{{ loop.index0 }}" role="img" aria-label="{{ card.label }} grouped by {{ query.group }}"></canvas></div></div>{% endfor %}</div><p class="studio-chart-help">Select a current-period point to see its contributing runs below. Missing data leaves a gap; comparison runs are aligned by elapsed days.</p></section>
+<div class="studio-metrics">{% for card in results.cards %}<article class="stat-card"><span class="eyebrow">{{ card.label }}</span><div class="stat-value">{{ card.value }}<small>{{ card.unit }}</small></div>{% if query.compare %}<p>Previous: {{ card.previous }} {{ card.unit }}{% if card.delta %} · {{ card.delta }} {{ 'sec/km' if card.unit == '/km' else card.unit }}{% endif %}</p>{% endif %}<p>{{ card.coverage }} of {{ results.count }} runs with usable data</p><details><summary>How this is calculated</summary><p>{{ card.definition }}{% if card.key in ('cadence','power','ground_contact','stride_length') %} Coverage counts runs, not sensor recording time.{% endif %}</p></details></article>{% endfor %}</div>
+{% if query.chart != 'none' %}<section class="panel"><div class="section-head"><div><span class="eyebrow">TRENDS / {{ query.group|upper }}</span><h2>{{ 'Your question over time' if query.group in ('day','week','month','year') else 'Your question by ' ~ query.group }}</h2></div><span class="subtle-label">Solid: current · Dashed: previous</span></div><div class="studio-charts">{% for series in results.chart.series %}<div class="studio-chart"><h3>{{ series.label }} <small>{{ series.unit }}</small></h3><div><canvas id="stats-chart-{{ loop.index0 }}" role="img" aria-label="{{ series.label }} grouped by {{ query.group }}"></canvas></div></div>{% endfor %}</div><p class="studio-chart-help">Select a current-period point to inspect its runs. {% if query.trend %}Trend: {{ query.trend }} · Summary cards and the grouped table show the untransformed values.{% else %}Missing data leaves a gap.{% endif %} Time comparisons align by elapsed days; categorical comparisons use the same categories.</p></section>{% endif %}
 {% endif %}
-<section class="panel studio-evidence"><div class="section-head"><div><span class="eyebrow">BEHIND THE NUMBERS</span><h2 id="evidence-heading">Contributing runs</h2></div><div class="studio-row"><label class="studio-period" for="evidence-period">Period <select id="evidence-period"><option value="">All results</option>{% for label in results.chart.labels %}<option>{{ label }}</option>{% endfor %}</select></label><button type="button" class="text-button" id="clear-selection" hidden>Show all results</button></div></div><div class="activity-table-wrap"><table class="activity-table"><thead><tr><th>RUN / LOCAL DATE</th><th>DISTANCE</th>{% for card in results.cards %}<th>{{ card.label|upper }} · {{ card.unit }}</th>{% endfor %}</tr></thead><tbody id="evidence-body">{% for row in results.rows %}<tr><td><a href="{{ url_for('web.activity',activity_id=row.id) }}"><strong>{{ row.title }}</strong><br><small>{{ row.date }}</small></a></td><td>{{ row.distance }} km</td>{% for value in row['values'] %}<td>{{ value }}</td>{% endfor %}</tr>{% else %}<tr><td colspan="4">No matching runs.</td></tr>{% endfor %}</tbody></table></div><div class="studio-pagination"><span>Page {{ results.page }} of {{ results.pages }} · up to 25 runs per page</span>{% if results.page > 1 %}<a class="button secondary" href="{{ url_for('web.stats',query=query_text,page=results.page-1) }}">Previous</a>{% endif %}{% if results.page < results.pages %}<a class="button secondary" href="{{ url_for('web.stats',query=query_text,page=results.page+1) }}">Next</a>{% endif %}</div></section>
+{% if results.chart.scatter %}<section class="panel studio-scatter"><div class="section-head"><div><span class="eyebrow">RELATIONSHIP / RUN AVERAGES</span><h2>{{ results.chart.scatter.x[0] }} vs {{ results.chart.scatter.y[0] }}</h2></div><span class="subtle-label">{% if results.chart.scatter.correlation is not none %}Pearson r = {{ '%.3f'|format(results.chart.scatter.correlation) }}{% else %}Correlation unavailable{% endif %}</span></div><div class="scatter-wrap"><canvas id="stats-scatter" aria-label="Run metric scatter plot" role="img"></canvas></div><p class="studio-chart-help">Each point is one run. Select a point to open that run. Correlation describes association; it does not establish cause.</p></section>{% endif %}
+<section class="panel studio-groups"><div class="section-head"><div><span class="eyebrow">ANALYSIS / GROUPED RESULTS</span><h2>{{ query.group|capitalize }} breakdown</h2></div><a class="button secondary" href="{{ url_for('web.stats_export',query=query_text,kind='groups') }}">Export CSV</a></div><div class="activity-table-wrap"><table class="activity-table"><thead><tr><th>GROUP</th><th>RUNS</th>{% for card in results.cards %}<th>{{ card.label|upper }} · {{ card.unit }}</th>{% endfor %}</tr></thead><tbody>{% for group in results.groups %}<tr><td><button type="button" class="group-drill" data-bucket="{{ group.label }}">{{ group.label }}</button></td><td>{{ group.count }}</td>{% for value in group['values'] %}<td>{{ value }}</td>{% endfor %}</tr>{% endfor %}</tbody></table></div></section>
+<section class="panel studio-evidence"><div class="section-head"><div><span class="eyebrow">BEHIND THE NUMBERS</span><h2 id="evidence-heading">Contributing runs</h2></div><div class="studio-row"><a class="button secondary" href="{{ url_for('web.stats_export',query=query_text,kind='runs') }}">Export runs</a><label class="studio-period" for="evidence-period">Period <select id="evidence-period"><option value="">All results</option>{% for label in results.chart.labels %}<option>{{ label }}</option>{% endfor %}</select></label><button type="button" class="text-button" id="clear-selection" hidden>Show all results</button></div></div><div class="activity-table-wrap"><table class="activity-table"><thead><tr><th>RUN / LOCAL DATE</th><th>DISTANCE</th>{% for card in results.evidence_columns %}<th>{{ card.label|upper }} · {{ card.unit }}</th>{% endfor %}</tr></thead><tbody id="evidence-body">{% for row in results.rows %}<tr><td><a href="{{ url_for('web.activity',activity_id=row.id) }}"><strong>{{ row.title }}</strong><br><small>{{ row.date }}</small></a>{% if results.show_previous %}<small class="previous-run">Previous: {% if row.previous_run %}<a href="{{ url_for('web.activity',activity_id=row.previous_run.id) }}">{{ row.previous_run.date }} · {{ row.previous_run.title }}</a>{% else %}No earlier recorded run{% endif %}</small>{% endif %}</td><td>{{ row.distance }} km</td>{% for value in row['values'] %}<td>{{ value }}</td>{% endfor %}</tr>{% else %}<tr><td colspan="{{ results.cards|length + 2 }}">No matching runs.</td></tr>{% endfor %}</tbody></table></div><div class="studio-pagination"><span>Page {{ results.page }} of {{ results.pages }} · up to 25 runs per page</span>{% if results.page > 1 %}<a class="button secondary" href="{{ url_for('web.stats',query=query_text,page=results.page-1) }}">Previous</a>{% endif %}{% if results.page < results.pages %}<a class="button secondary" href="{{ url_for('web.stats',query=query_text,page=results.page+1) }}">Next</a>{% endif %}</div></section>
 <script id="stats-data" type="application/json">{{ results.chart|tojson }}</script>
 {% endif %}
-<div id="studio-status" role="status" class="field-help"></div>
+<p class="analysis-semantics field-help">Gaps, rolling metrics and streaks use your complete history before filters. Arithmetic and filters apply per run; summaries apply per group. Contributing runs show the individual input values for each summary. Empty sums remain missing; count is zero. Group ranking/limits and having select the displayed cohort before chart trends.</p><div id="studio-status" role="status" class="field-help"></div>
 </div>
 {% endblock %}
 {% block scripts %}<script src="{{ url_for('static',filename='vendor/codemirror.js') }}"></script><script src="{{ url_for('static',filename='vendor/show-hint.js') }}"></script><script src="{{ url_for('static',filename='stats-editor.js') }}"></script><script src="{{ url_for('static',filename='vendor/chart.umd.js') }}"></script><script src="{{ url_for('static',filename='stats.js') }}"></script>{% endblock %}

docs/STATS_DSL.md

@@ -0,0 +1,193 @@
+# Run analysis DSL
+
+Write queries in **Stats**, use **Ctrl+Space** for completion and **Ctrl/Cmd+Enter**
+to execute. **Format** produces a canonical query. Live validation uses the same
+parser as execution. Comments occupy their own lines starting with `#`.
+
+## Largest gaps between runs
+
+```text
+runs
+  during all time
+  summarize gap_since_previous_run, distance
+  by run
+  sort by gap_since_previous_run desc
+  take 10
+  compare none
+```
+
+Each gap is actual elapsed start-to-start time, expressed in days. Its predecessor
+comes from your complete running history, even if it falls outside the period or
+fails the query's filters. The first recorded run has a missing gap. Each result
+links to the preceding run; this cannot reveal unrecorded activities.
+
+## Statistical analysis
+
+```text
+runs
+  during last 24 months
+  where distance >= 5 km
+  summarize count(), median(pace), p90(heart_rate), sum(distance)
+  by weekday
+  having count() >= 5 runs
+  rank by median(pace) asc
+  limit 4
+  compare none
+  plot speed vs heart_rate
+```
+
+Functions: `sum`, `mean`, `weighted_mean`, `median`, `min`, `max`, `stddev`,
+`count`, `p10`, `p25`, `p75`, `p90`, `p95`, `p99`. Use `count()` for all runs or
+`count(power)` for runs with that metric. Percentiles use linear interpolation
+at `(n-1)*p`; standard deviation is population standard deviation.
+
+`mean(pace)` gives each run equal weight. `weighted_mean(pace)` weights by
+recorded distance. Other weighted means use timer duration. Bare `pace` is total
+timer time / distance; bare heart rate and running dynamics are weighted by timer
+time. Bare distance, duration, ascent, calories and count are totals. Other bare
+metrics are unweighted means. Calculated-field bare summaries are means.
+
+Missing values are excluded, not replaced by zero. Empty sums and means remain
+missing; counts are zero. Nonpositive recorded pace and sensor values are missing.
+Each summary card shows usable-run coverage and its definition.
+
+## Metrics and units
+
+| Metric | Input unit(s) |
+|---|---|
+| `distance`, `distance_change`, `rolling_7d_distance`, `rolling_28d_distance` | `km`, `m` |
+| `duration` | `h`, `min`, `s` |
+| `pace` | `/km`, such as `5:30 /km` or `330 /km` |
+| `pace_change` | `sec/km`, signed; negative is faster |
+| `heart_rate`, `heart_rate_change`, `max_heart_rate` | `bpm` |
+| `elevation`, `stride_length` | `m` |
+| `cadence` | `spm` |
+| `power` | `W` |
+| `ground_contact` | `ms` |
+| `vertical_oscillation` | `cm` |
+| `speed` | `km/h`, `m/s` |
+| `gap_since_previous_run` | `days`, `h`, `min` |
+| `run_day_streak` | `days` |
+| `count`, `rolling_7d_count`, `rolling_28d_count` | `runs` |
+| `elevation_per_km` | `m/km` |
+| `calories` | `kcal` |
+| `training_effect` | `score` |
+
+Changes compare with the immediately preceding recorded run before filtering.
+Rolling 7/28-day metrics include this run and earlier runs less than 168/672 hours
+old; the exact boundary is excluded. Streaks count consecutive local calendar days
+with a run. Multiple runs on one day don't lengthen the streak.
+
+Pace standard deviations and sums use numeric `sec/km`, not pace formatting.
+Garmin calories and aerobic training effect remain missing when not recorded.
+
+## Calculated fields
+
+```text
+runs
+  during last 12 weeks
+  let efficiency = speed / heart_rate
+  let pace_ratio = pace / pace
+  where efficiency > 0.07 (km/h/bpm)
+  summarize mean(efficiency), median(pace), count()
+  by week
+  plot efficiency vs distance
+  compare none
+```
+
+Define fields before referring to them. Arithmetic supports `+`, `-`, `*`, `/`,
+parentheses, numeric constants, and earlier fields. Addition/subtraction requires
+matching units. Pace arithmetic uses seconds/km, duration uses hours; division of
+equal units gives `ratio`. Scalar multiplication keeps the field's unit. Combined
+units appear in validation and completion, e.g. `(km/h/bpm)` for efficiency.
+Missing inputs, zero divisors and nonfinite results yield missing values.
+Expressions cannot call functions, access attributes, index values or execute code.
+
+## Filters and periods
+
+```text
+runs
+  during 2026-07-01 to 2026-09-30
+  where distance between 8 and 20 km
+  and heart_rate < 150 bpm
+  or title contains "Race"
+  and has power
+  summarize median(pace), count()
+  by month
+  compare previous period
+```
+
+Numeric operators: `=`, `!=`, `<`, `<=`, `>`, `>=`, and inclusive `between`.
+`has` / `missing` test metric availability; title search is case-insensitive and
+uses a double-quoted string. AND binds more strongly than OR: the example selects
+8–20 km runs below 150 bpm **or** race-titled runs with power. Parenthesized boolean
+filters are not supported; use separate AND chains joined by OR.
+
+Periods: `during last N days/weeks/months`, inclusive explicit date ranges, or
+`during all time`. Rolling periods may use `ending YYYY-MM-DD`. Dates and groups
+use your account timezone. Supported relative ranges are 1–732 days, 1–104 weeks,
+1–24 calendar months; explicit ranges cover at most 732 days. All time covers the
+complete recorded history through today and requires `compare none`.
+Comparisons otherwise use an equal-length preceding period. Time series align
+previous dates by elapsed calendar days; categorical groups use matching categories.
+
+## Grouping, ranking and limits
+
+Group with `by day/week/month/year`, `by weekday`, `by hour`, `by run`, `by all`, or
+`by distance bins 5 km`. Weeks start Monday. Numeric bins are half-open ranges
+`[lower, upper)` anchored at zero; negative inputs use negative bins. Missing bin
+values are excluded. Time groups include empty calendar buckets, weekday/hour
+include all categories; numeric bins include observed bins.
+
+- `sort by FIELD asc/desc` orders runs; missing values sort last. `date` is allowed.
+- `take N` selects the first sorted runs **before** grouping, also for comparison.
+- `having SUMMARY OP VALUE UNIT` filters groups after aggregation. Multiple having
+  clauses combine with AND; a summary need not be displayed to filter on it.
+- `rank by SUMMARY asc/desc` orders groups; the ranking summary must be selected.
+- `limit N` limits groups; without ranking, time groups retain the most recent N,
+  categorical groups retain the first N in category order.
+
+The order is history-derived metrics → date/per-run filters → sort/take → groups
+→ having → rank/limit → displayed cohort → chart trends. Cards and contributing
+runs reflect the displayed cohort. Limits are 1–1,000 runs/groups, six displayed
+summaries, eight calculated fields, twenty numeric filter conditions and ten
+having clauses. Wider grouping or limits are required above 1,000 displayed groups.
+
+## Trends, relationships and export
+
+```text
+runs
+  during last 24 weeks
+  summarize distance, count(), weighted_mean(pace)
+  by week
+  trend rolling 4 buckets
+  compare none
+```
+
+Time groups support `trend cumulative`, `trend rolling N buckets` (1–52),
+`trend change`, and `trend percent_change`. Rolling/cumulative summaries pool
+contributing runs and recalculate the selected statistic; they do not average
+bucket means. These windows operate on displayed buckets after having/limits.
+Change compares consecutive displayed buckets; the first, missing inputs, and
+percentage division by zero remain missing. Percentage change uses the absolute
+previous value as denominator. Trends cannot also rank groups.
+
+Charts support `chart auto/line/bar/none`. `plot X vs Y` adds a run-level scatter
+plot and Pearson correlation using complete pairs. Constant axes or fewer than
+two pairs have no correlation. Select a scatter point to open its activity.
+
+Select a chart point, group row, or period picker to inspect contributing runs.
+For cumulative/rolling chart points, drill-through shows the pooled runs; group
+rows show only the original group's runs. The grouped
+results table and cards always retain untransformed statistics. Evidence shows
+individual input values, not an aggregate applied to one run.
+
+**Export CSV** downloads untransformed grouped results. **Export runs** downloads
+all runs in the displayed cohort, not only the evidence page. CSV contains raw
+numeric precision; pace is seconds/km, duration is hours, and missing values are
+blank. Run titles are escaped against spreadsheet formula interpretation. All
+queries, predecessor links, scatter points and exports remain owner-scoped.
+
+Analysis operates on persisted whole-run summaries; it does not load raw samples
+or routes. Saved views are stored in this browser; bookmark URLs to share a query
+with yourself across devices. Queries cannot join other users or raw sample data.

tests/test_stats_analysis.py

@@ -0,0 +1,268 @@
+import csv
+import io
+from datetime import UTC, datetime
+
+import pytest
+from test_app import login
+
+from app.extensions import db
+from app.models import Activity
+from app.stats import explore, parse_query, query_text
+from app.stats_analysis import statistic
+from app.stats_dsl import QueryError
+
+pytest_plugins = ("test_app",)
+
+
+def query(source):
+    return parse_query({"query": "runs\n" + source})
+
+
+def run(key, day, km=10, seconds=3000, owner=1, hr=150, hour=0, title=None):
+    return Activity(
+        owner_id=owner,
+        source_key=key,
+        title=title or key,
+        sport="running",
+        started_at=datetime(2026, 9, day, hour, tzinfo=UTC),
+        distance_m=km * 1000,
+        duration_s=seconds,
+        avg_hr=hr,
+    )
+
+
+def analyze(source):
+    return explore(1, "Australia/Sydney", query(source))
+
+
+@pytest.mark.parametrize(
+    "source",
+    [
+        "during all time\nsummarize gap_since_previous_run, distance\nby run\nsort by gap_since_previous_run desc\ntake 10",
+        "let efficiency = speed / heart_rate\nwhere efficiency > 0.05 (km/h/bpm)\nsummarize median(efficiency), p95(pace), count()\nby weekday\nhaving count() >= 3 runs\nrank by median(efficiency) desc\nlimit 4\ncompare none",
+        'where distance < 8 km\nand has heart_rate\nor title contains "between hills"\nand duration > 20 min\nsummarize sum(distance), mean(heart_rate)\nby distance bins 5 km\ncompare none',
+        "summarize median(pace), stddev(pace)\nby week\nhaving median(pace) < 5:30 /km\nhaving stddev(pace) < 60 sec/km\ntrend rolling 4 buckets\nplot speed vs heart_rate\nchart line",
+        "summarize sum(pace), mean(duration)\nby hour\nhaving sum(pace) > 300 sec/km\nchart none",
+    ],
+)
+def test_analysis_canonical_roundtrip(source):
+    parsed = query(source)
+    assert query(query_text(parsed).removeprefix("runs\n")) == parsed
+
+
+def test_predecessor_and_rolling_use_history_before_filters(app):
+    with app.app_context():
+        db.session.add_all(
+            [
+                run("first", 1),
+                run("intermediate", 8, km=1),
+                run("target", 9),
+                run("private", 8, owner=2),
+            ]
+        )
+        db.session.commit()
+        result = analyze(
+            "during 2026-09-09 to 2026-09-09\nwhere distance >= 5 km\n"
+            "summarize gap_since_previous_run, rolling_7d_distance, rolling_7d_count, run_day_streak\nby run\ncompare none"
+        )
+        assert result["count"] == 1
+        assert result["rows"][0]["raw_values"] == [1, 11, 2, 2]
+        assert result["rows"][0]["previous_run"]["title"] == "intermediate"
+        # The exact 168-hour boundary is excluded from the rolling window.
+        result = analyze(
+            "during 2026-09-08 to 2026-09-08\nsummarize rolling_7d_distance\nby run\ncompare none"
+        )
+        assert result["rows"][0]["raw_values"] == [1]
+
+
+def test_biggest_gaps_ranking_take_and_missing(app):
+    with app.app_context():
+        db.session.add_all([run("first", 1), run("second", 3), run("third", 20)])
+        db.session.commit()
+        result = analyze(
+            "during all time\nsummarize gap_since_previous_run, distance\nby run\nsort by gap_since_previous_run desc\ntake 2"
+        )
+        assert [row["title"] for row in result["rows"]] == ["third", "A quiet morning run"]
+        assert [row["raw_values"][0] for row in result["rows"]] == [17, 10]
+        assert result["count"] == 2
+        assert len(result["groups"]) == 2
+
+
+def test_explicit_statistics_weighting_missing_and_percentiles(app):
+    with app.app_context():
+        db.session.add_all([run("fast-long", 29, km=15, seconds=4500, hr=None)])
+        db.session.commit()
+        result = analyze(
+            "during 2026-09-29 to 2026-09-30\nsummarize mean(pace), weighted_mean(pace), p90(pace), stddev(pace), count(heart_rate), count()\nby all\ncompare none"
+        )
+        assert result["groups"][0]["raw_values"] == [330, 315, 354, 30, 1, 2]
+        assert result["cards"][3]["unit"] == "sec/km"
+        assert result["cards"][3]["value"] == "30.0"
+        assert statistic([None], "sum") is None
+        assert statistic([], "count") == 0
+        assert statistic([1, 2, 3, 4], "p25") == 1.75
+
+
+def test_arithmetic_units_nulls_and_correlation(app):
+    with app.app_context():
+        db.session.add_all(
+            [run("slow", 28, km=5, seconds=3600, hr=100), run("missing", 29, hr=None)]
+        )
+        db.session.commit()
+        result = analyze(
+            "during 2026-09-28 to 2026-09-30\nlet efficiency = speed / heart_rate\nlet empty = speed / (heart_rate - heart_rate)\nsummarize mean(efficiency), count(efficiency), count(empty)\nby all\nplot speed vs heart_rate\ncompare none"
+        )
+        assert result["groups"][0]["raw_values"][0] == pytest.approx((5 / 100 + 10 / 140) / 2)
+        assert result["groups"][0]["raw_values"][1:] == [2, 0]
+        assert result["chart"]["scatter"]["correlation"] == pytest.approx(1)
+        assert len(result["chart"]["scatter"]["current"]) == 2
+
+
+@pytest.mark.parametrize(
+    "clause",
+    [
+        'let bad = __import__("os")',
+        "let bad = distance.__class__",
+        "let bad = distance[0]",
+        "let bad = distance ** 2",
+        "let bad = distance + heart_rate",
+        "let analysis_agg_0 = distance",
+        "let bad = unknown / distance",
+        "take 1001",
+        "by distance bins 0 km",
+        "summarize distance\nrank by median(pace) desc",
+        "by weekday\ntrend cumulative",
+        "trend rolling 0 buckets",
+        "during all time\ncompare previous period",
+        "plot speed vs heart_rate\nchart none",
+    ],
+)
+def test_rejects_unsafe_or_incoherent_analysis(clause):
+    with pytest.raises(QueryError):
+        query(clause + ("\nsummarize pace" if "summarize" not in clause else ""))
+
+
+def test_or_precedence_bins_having_ranking_and_drill(app):
+    with app.app_context():
+        db.session.add_all(
+            [
+                run("short-missing", 28, km=2, hr=None),
+                run("long", 29, km=20, hr=170),
+                run("short", 29, km=2),
+            ]
+        )
+        db.session.commit()
+        result = analyze(
+            "during 2026-09-28 to 2026-09-30\nwhere distance < 3 km\nand has heart_rate\nor distance >= 20 km\nand heart_rate > 160 bpm\nsummarize count(), sum(distance)\nby distance bins 5 km\nhaving count() >= 1 runs\nrank by sum(distance) desc\nlimit 1\ncompare none"
+        )
+        assert result["count"] == 1
+        assert result["groups"][0]["label"] == "20–25 km"
+        assert result["rows"][0]["title"] == "long"
+        drilled = explore(
+            1,
+            "Australia/Sydney",
+            query("during 2026-09-28 to 2026-09-30\nsummarize count()\nby weekday\ncompare none"),
+            selected_bucket="Wednesday",
+        )
+        assert drilled["evidence_count"] == 1
+
+
+def test_rolling_trends_pool_observations_and_percent_change(app):
+    with app.app_context():
+        db.session.add_all(
+            [
+                run("a", 28, km=10, seconds=3000),
+                run("b", 28, km=5, seconds=2400),
+                run("c", 29, km=5, seconds=1500),
+            ]
+        )
+        db.session.commit()
+        base = (
+            "during 2026-09-28 to 2026-09-30\nsummarize mean(pace), count()\nby day\ncompare none\n"
+        )
+        result = analyze(base + "trend rolling 2 buckets")
+        assert result["chart"]["series"][0]["values"] == [390, 360, 330]
+        assert result["chart"]["series"][1]["values"] == [2, 3, 2]
+        result = analyze(base + "trend percent_change")
+        assert result["chart"]["series"][1]["values"] == [None, -50, 0]
+        assert result["chart"]["series"][0]["unit"] == "%"
+        result = analyze(base + "trend change")
+        assert result["chart"]["series"][0]["values"] == [None, -90, 60]
+        assert result["chart"]["series"][0]["unit"] == "sec/km"
+
+
+def test_csv_precision_private_scope_and_formula_titles(app):
+    with app.app_context():
+        db.session.add_all(
+            [
+                run("formula", 29, km=1.123456789, title='=HYPERLINK("bad")'),
+                run("secret", 29, owner=2, title="Private secret"),
+            ]
+        )
+        db.session.commit()
+    client = app.test_client()
+    assert client.get("/stats/export").status_code == 302
+    login(client)
+    source = "runs\nduring 2026-09-29 to 2026-09-30\nsummarize distance, pace\nby all\ncompare none"
+    response = client.get("/stats/export", query_string={"query": source, "kind": "runs"})
+    assert response.status_code == 200
+    records = list(csv.reader(io.StringIO(response.text)))
+    assert records[0][-1] == "pace (sec/km)"
+    assert "Private secret" not in response.text
+    formula = next(row for row in records if "HYPERLINK" in row[0])
+    assert formula[0].startswith("'=HYPERLINK")
+    assert float(formula[-2]) == pytest.approx(1.123456789)
+    assert "no-store" in response.headers["Cache-Control"]
+    assert client.get("/stats", query_string={"query": source}).status_code == 200
+
+
+def test_elapsed_gaps_across_dst_and_same_day_streaks(app):
+    with app.app_context():
+        first = run("before-dst", 1)
+        first.started_at = datetime(2026, 10, 2, 22, tzinfo=UTC)  # 08:00 local Oct 3
+        second = run("after-dst", 2)
+        second.started_at = datetime(2026, 10, 3, 21, tzinfo=UTC)  # 08:00 local Oct 4
+        duplicate = run("same-day", 3)
+        duplicate.started_at = datetime(2026, 10, 3, 21, tzinfo=UTC)
+        db.session.add_all([first, second, duplicate])
+        db.session.commit()
+        result = analyze(
+            "during 2026-10-03 to 2026-10-04\nsummarize gap_since_previous_run, run_day_streak\nby run\nsort by date asc\ncompare none"
+        )
+        assert result["rows"][1]["raw_values"] == [23 / 24, 2]
+        assert result["rows"][2]["raw_values"] == [0, 2]
+
+
+def test_trend_and_group_drill_distinguish_window_and_input_values(app):
+    with app.app_context():
+        db.session.add_all([run("earlier", 29, km=15, seconds=4500)])
+        db.session.commit()
+        source = "during 2026-09-29 to 2026-09-30\nsummarize stddev(pace), count(heart_rate)\nby day\ntrend cumulative\ncompare none"
+        parsed = query(source)
+        window = explore(1, "Australia/Sydney", parsed, selected_bucket="30 Sep 2026")
+        group = explore(
+            1, "Australia/Sydney", parsed, selected_bucket="30 Sep 2026", raw_group=True
+        )
+        assert window["evidence_count"] == 2
+        assert group["evidence_count"] == 1
+        assert group["rows"][0]["raw_values"] == [360, 140]
+        assert group["evidence_columns"][0]["key"] == "pace"
+
+
+def test_matching_calculated_units_and_spaced_aggregate():
+    source = "let measured_speed = distance / duration\nlet combined_speed = measured_speed + speed\nsummarize median( combined_speed )\nby all"
+    parsed = query(source)
+    assert parsed["derived"][0]["unit"] == "km/h"
+    assert parsed["metric"] == "median(combined_speed)"
+    assert query(query_text(parsed).removeprefix("runs\n")) == parsed
+
+
+def test_sensor_zero_missing_and_initial_gap_coverage(app):
+    with app.app_context():
+        first = run("first-ever", 1, hr=0)
+        db.session.add(first)
+        db.session.commit()
+        result = analyze(
+            "during 2026-09-01 to 2026-09-01\nsummarize count(gap_since_previous_run), count(heart_rate), count()\nby all\ncompare none"
+        )
+        assert result["groups"][0]["raw_values"] == [0, 0, 1]