Extend run stats DSL with statistical and temporal analysis
This commit is contained in:
Peter Stockings
@@ -0,0 +1,492 @@
|
||||
"""Typed analysis expressions and extensions for the run query language."""
|
||||
|
||||
import ast
|
||||
import math
|
||||
import re
|
||||
import statistics
|
||||
|
||||
from .stats_dsl import UNITS, QueryError, parse_dsl
|
||||
|
||||
FUNCTIONS = {
|
||||
"sum",
|
||||
"mean",
|
||||
"weighted_mean",
|
||||
"median",
|
||||
"min",
|
||||
"max",
|
||||
"stddev",
|
||||
"count",
|
||||
"p10",
|
||||
"p25",
|
||||
"p75",
|
||||
"p90",
|
||||
"p95",
|
||||
"p99",
|
||||
}
|
||||
GROUPS = {"day", "week", "month", "year", "weekday", "hour", "run", "all"}
|
||||
EXTRA_METRICS = {
|
||||
"count": ("Runs", "runs", "Number of runs."),
|
||||
"speed": ("Speed", "km/h", "Recorded distance divided by timer duration."),
|
||||
"gap_since_previous_run": (
|
||||
"Gap since previous run",
|
||||
"days",
|
||||
"Elapsed start-to-start time since your previous run, including runs outside this query. This is not time since the previous run ended.",
|
||||
),
|
||||
"distance_change": (
|
||||
"Distance change",
|
||||
"km",
|
||||
"Distance minus the previous run's distance, before filtering.",
|
||||
),
|
||||
"pace_change": (
|
||||
"Pace change",
|
||||
"sec/km",
|
||||
"Pace minus the previous run's pace. Negative means faster.",
|
||||
),
|
||||
"heart_rate_change": (
|
||||
"Heart rate change",
|
||||
"bpm",
|
||||
"Average heart rate minus the previous run's average, before filtering.",
|
||||
),
|
||||
"rolling_7d_distance": (
|
||||
"7-day distance",
|
||||
"km",
|
||||
"Distance in the preceding 168 hours, including this run. Calculated before filters.",
|
||||
),
|
||||
"rolling_28d_distance": (
|
||||
"28-day distance",
|
||||
"km",
|
||||
"Distance in the preceding 672 hours, including this run. Calculated before filters.",
|
||||
),
|
||||
"rolling_7d_count": (
|
||||
"7-day run count",
|
||||
"runs",
|
||||
"Number of runs in the preceding 168 hours, including this run, before filters.",
|
||||
),
|
||||
"rolling_28d_count": (
|
||||
"28-day run count",
|
||||
"runs",
|
||||
"Number of runs in the preceding 672 hours, including this run, before filters.",
|
||||
),
|
||||
"run_day_streak": (
|
||||
"Consecutive running days",
|
||||
"days",
|
||||
"Consecutive local calendar days with a run, through this run's local date, before filters.",
|
||||
),
|
||||
"elevation_per_km": (
|
||||
"Ascent per kilometre",
|
||||
"m/km",
|
||||
"Recorded ascent divided by recorded distance.",
|
||||
),
|
||||
"max_heart_rate": ("Maximum heart rate", "bpm", "Recorded maximum heart rate."),
|
||||
"vertical_oscillation": (
|
||||
"Vertical oscillation",
|
||||
"cm",
|
||||
"Recorded sample average, weighted by run timer duration.",
|
||||
),
|
||||
"calories": ("Calories", "kcal", "Recorded session calories. Missing values are excluded."),
|
||||
"training_effect": (
|
||||
"Aerobic training effect",
|
||||
"score",
|
||||
"Recorded Garmin aerobic training effect; an unweighted mean across runs.",
|
||||
),
|
||||
}
|
||||
EXTRA_UNITS = {
|
||||
"count": {"runs": 1},
|
||||
"speed": {"km/h": 1, "m/s": 3.6},
|
||||
"gap_since_previous_run": {"days": 1, "h": 1 / 24, "min": 1 / 1440},
|
||||
"distance_change": {"km": 1, "m": 0.001},
|
||||
"pace_change": {"sec/km": 1},
|
||||
"heart_rate_change": {"bpm": 1},
|
||||
"rolling_7d_distance": {"km": 1, "m": 0.001},
|
||||
"rolling_28d_distance": {"km": 1, "m": 0.001},
|
||||
"rolling_7d_count": {"runs": 1},
|
||||
"rolling_28d_count": {"runs": 1},
|
||||
"run_day_streak": {"days": 1},
|
||||
"elevation_per_km": {"m/km": 1},
|
||||
"max_heart_rate": {"bpm": 1},
|
||||
"vertical_oscillation": {"cm": 1},
|
||||
"calories": {"kcal": 1},
|
||||
"training_effect": {"score": 1},
|
||||
}
|
||||
|
||||
|
||||
def valid(value):
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
||||
|
||||
|
||||
def expression_tree(source, catalog, line=1):
|
||||
if len(source) > 200:
|
||||
raise QueryError("Keep each calculation under 200 characters.", line)
|
||||
try:
|
||||
tree = ast.parse(source, mode="eval")
|
||||
except SyntaxError:
|
||||
raise QueryError("Use metric names, numbers, parentheses and + - * /.", line) from None
|
||||
if len(list(ast.walk(tree))) > 64:
|
||||
raise QueryError("Simplify this calculation.", line)
|
||||
|
||||
def unit(node):
|
||||
if isinstance(node, ast.Expression):
|
||||
return unit(node.body)
|
||||
if isinstance(node, ast.Name) and node.id in catalog:
|
||||
# Pace values inside arithmetic are seconds per km, not a formatted string.
|
||||
return (
|
||||
"sec/km"
|
||||
if node.id == "pace"
|
||||
else "h"
|
||||
if node.id == "duration"
|
||||
else catalog[node.id][1]
|
||||
)
|
||||
if isinstance(node, ast.Constant) and valid(node.value) and abs(node.value) <= 1000000:
|
||||
return "ratio"
|
||||
if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
|
||||
return unit(node.operand)
|
||||
if isinstance(node, ast.BinOp) and isinstance(
|
||||
node.op, (ast.Add, ast.Sub, ast.Mult, ast.Div)
|
||||
):
|
||||
left, right = unit(node.left), unit(node.right)
|
||||
if isinstance(node.op, (ast.Add, ast.Sub)):
|
||||
if left != right:
|
||||
raise QueryError(
|
||||
f"Cannot add/subtract {left} and {right}. Use matching units.", line
|
||||
)
|
||||
return left
|
||||
if isinstance(node.op, ast.Div) and left == right:
|
||||
return "ratio"
|
||||
if right == "ratio":
|
||||
return left
|
||||
if left == "ratio" and isinstance(node.op, ast.Mult):
|
||||
return right
|
||||
if isinstance(node.op, ast.Div) and (left, right) in {("km", "h"), ("m", "km")}:
|
||||
return f"{left}/{right}"
|
||||
return f"({left}{'*' if isinstance(node.op, ast.Mult) else '/'}{right})"
|
||||
raise QueryError(
|
||||
"Calculations allow existing metrics, numbers and + - * /. Calls, attributes and indexing are unavailable.",
|
||||
line,
|
||||
)
|
||||
|
||||
return tree, unit(tree)
|
||||
|
||||
|
||||
def evaluate(tree, values):
|
||||
def visit(node):
|
||||
if isinstance(node, ast.Expression):
|
||||
return visit(node.body)
|
||||
if isinstance(node, ast.Name):
|
||||
return values.get(node.id)
|
||||
if isinstance(node, ast.Constant):
|
||||
return node.value
|
||||
if isinstance(node, ast.UnaryOp):
|
||||
value = visit(node.operand)
|
||||
return None if not valid(value) else -value if isinstance(node.op, ast.USub) else value
|
||||
left, right = visit(node.left), visit(node.right)
|
||||
if not valid(left) or not valid(right):
|
||||
return None
|
||||
if isinstance(node.op, ast.Add):
|
||||
result = left + right
|
||||
elif isinstance(node.op, ast.Sub):
|
||||
result = left - right
|
||||
elif isinstance(node.op, ast.Mult):
|
||||
result = left * right
|
||||
elif right == 0:
|
||||
return None
|
||||
else:
|
||||
result = left / right
|
||||
return result if valid(result) else None
|
||||
|
||||
return visit(tree)
|
||||
|
||||
|
||||
def parse_analysis(text, metrics, defaults):
|
||||
if not isinstance(text, str) or len(text) > 4000:
|
||||
raise QueryError("Keep the query under 4,000 characters.")
|
||||
catalog, units = dict(metrics), {**UNITS, **EXTRA_UNITS}
|
||||
lines = text.splitlines()
|
||||
derived, aggregates, seen, locations = [], {}, set(), {}
|
||||
options = {
|
||||
"all_time": False,
|
||||
"bin": None,
|
||||
"take": None,
|
||||
"limit": None,
|
||||
"rank": None,
|
||||
"trend": None,
|
||||
"plot": None,
|
||||
"chart": "auto",
|
||||
"having": [],
|
||||
}
|
||||
group = None
|
||||
or_lines = any(re.match(r"\s*or\s", line) for line in lines)
|
||||
filter_counts = []
|
||||
aliases = {}
|
||||
|
||||
def once(key, line):
|
||||
if key in seen:
|
||||
raise QueryError(f"Repeated {key} clause.", line)
|
||||
seen.add(key)
|
||||
locations[key] = line
|
||||
|
||||
def aggregate(source, line):
|
||||
source = source.strip()
|
||||
match = re.fullmatch(r"(\w+)\s*\(\s*(\w*)\s*\)", source)
|
||||
if not match:
|
||||
if source not in catalog:
|
||||
raise QueryError(f"Unknown metric '{source}'.", line)
|
||||
return source
|
||||
function, field = match.groups()
|
||||
if function not in FUNCTIONS or (not field and function != "count"):
|
||||
raise QueryError(
|
||||
"Use sum, mean, weighted_mean, median, min, max, stddev, count or p10/p25/p75/p90/p95/p99.",
|
||||
line,
|
||||
)
|
||||
if field and field not in catalog:
|
||||
raise QueryError(f"Unknown metric '{field}'.", line)
|
||||
key = f"{function}({field})"
|
||||
aggregates[key] = {"function": function, "field": field}
|
||||
return key
|
||||
|
||||
for index, source in enumerate(lines):
|
||||
line, number = source.strip(), index + 1
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if match := re.fullmatch(r"let (\w+) = (.+)", line):
|
||||
name, expression = match.groups()
|
||||
if (
|
||||
not re.fullmatch(r"[a-z][a-z0-9_]{0,39}", name)
|
||||
or name.startswith("analysis_")
|
||||
or name in catalog
|
||||
or len(derived) >= 8
|
||||
):
|
||||
raise QueryError(
|
||||
"Choose a new lowercase metric name; at most eight calculations.", number
|
||||
)
|
||||
_, unit = expression_tree(expression, catalog, number)
|
||||
catalog[name] = (
|
||||
name.replace("_", " ").capitalize(),
|
||||
unit,
|
||||
f"Calculated per run: {expression}. Missing inputs or division by zero give missing data.",
|
||||
)
|
||||
units[name] = {unit: 1}
|
||||
derived.append({"name": name, "expression": expression, "unit": unit})
|
||||
lines[index] = ""
|
||||
elif line == "during all time":
|
||||
once("all-time period", number)
|
||||
options["all_time"] = True
|
||||
lines[index] = "during last 12 weeks"
|
||||
elif line.startswith("summarize "):
|
||||
keys = [aggregate(part, number) for part in line[10:].split(",")]
|
||||
if len(keys) != len(set(keys)):
|
||||
raise QueryError("Choose different metrics or summary expressions.", number)
|
||||
replacements = []
|
||||
for key in keys:
|
||||
if key in aggregates:
|
||||
alias = f"analysis_agg_{len(aliases)}"
|
||||
aliases[alias] = key
|
||||
catalog[alias] = (
|
||||
key,
|
||||
"runs"
|
||||
if aggregates[key]["function"] == "count"
|
||||
else catalog[aggregates[key]["field"]][1],
|
||||
"Aggregate",
|
||||
)
|
||||
units[alias] = {catalog[alias][1]: 1}
|
||||
replacements.append(alias)
|
||||
else:
|
||||
replacements.append(key)
|
||||
lines[index] = "summarize " + ", ".join(replacements)
|
||||
elif match := re.fullmatch(r"by (\w+)(?: bins (\S+) (\S+))?", line):
|
||||
once("by", number)
|
||||
name, width, unit = match.groups()
|
||||
if width:
|
||||
if name not in catalog or name not in units or unit not in units[name]:
|
||||
raise QueryError("Choose a numeric metric and its unit for bins.", number)
|
||||
try:
|
||||
size = float(width) * units[name][unit]
|
||||
except ValueError:
|
||||
raise QueryError("Use a positive numeric bin width.", number) from None
|
||||
if not valid(size) or size <= 0 or size > 1000000:
|
||||
raise QueryError("Use a positive finite bin width.", number)
|
||||
options["bin"] = {"field": name, "width": size}
|
||||
group = "bin"
|
||||
else:
|
||||
if name not in GROUPS:
|
||||
raise QueryError(
|
||||
"Group by day, week, month, year, weekday, hour, run, all, or metric bins.",
|
||||
number,
|
||||
)
|
||||
group = name
|
||||
lines[index] = "by week" if group not in {"day", "week", "month"} else line
|
||||
elif match := re.fullmatch(r"(take|limit) (\d+)", line):
|
||||
name, raw = match.groups()
|
||||
once(name, number)
|
||||
value = int(raw)
|
||||
if not 1 <= value <= 1000:
|
||||
raise QueryError("Use a limit from 1 to 1,000.", number)
|
||||
options[name] = value
|
||||
lines[index] = ""
|
||||
elif match := re.fullmatch(r"rank by (.+) (asc|desc)", line):
|
||||
once("rank", number)
|
||||
options["rank"] = [aggregate(match[1], number), match[2]]
|
||||
lines[index] = ""
|
||||
elif match := re.fullmatch(r"having (.+?) (>=|<=|>|<|=|!=) (-?\S+)(?: (\S+))?", line):
|
||||
key, op, raw, unit = match.groups()
|
||||
key = aggregate(key, number)
|
||||
function = aggregates.get(key, {}).get("function")
|
||||
field = aggregates.get(key, {}).get("field", key)
|
||||
accepted = (
|
||||
{"runs": 1}
|
||||
if function == "count"
|
||||
else {"sec/km": 1}
|
||||
if field == "pace" and function in {"sum", "stddev"}
|
||||
else units[field]
|
||||
)
|
||||
unit = unit or next(iter(accepted))
|
||||
if unit not in accepted:
|
||||
raise QueryError(f"Use {' or '.join(accepted)} for {key}.", number)
|
||||
try:
|
||||
if field == "pace" and unit == "/km" and ":" in raw:
|
||||
minutes, seconds = raw.split(":")
|
||||
if not re.fullmatch(r"\d+:\d{2}", raw) or int(seconds) >= 60:
|
||||
raise ValueError
|
||||
value = int(minutes) * 60 + int(seconds)
|
||||
else:
|
||||
value = float(raw) * accepted[unit]
|
||||
except ValueError:
|
||||
raise QueryError("Use a numeric threshold for having.", number) from None
|
||||
if not valid(value) or abs(value) > 1000000:
|
||||
raise QueryError("Use a finite threshold.", number)
|
||||
if len(options["having"]) >= 10:
|
||||
raise QueryError("Use at most ten having conditions.", number)
|
||||
options["having"].append({"field": key, "op": op, "value": value})
|
||||
lines[index] = ""
|
||||
elif match := re.fullmatch(
|
||||
r"trend (cumulative|change|percent_change|rolling (\d+) buckets?)", line
|
||||
):
|
||||
once("trend", number)
|
||||
options["trend"] = match[1]
|
||||
if match[2] and not 1 <= int(match[2]) <= 52:
|
||||
raise QueryError("Use 1–52 rolling buckets.", number)
|
||||
lines[index] = ""
|
||||
elif match := re.fullmatch(r"plot (\w+) vs (\w+)", line):
|
||||
once("plot", number)
|
||||
if any(name not in catalog for name in match.groups()):
|
||||
raise QueryError("Choose two numeric run metrics for the scatter plot.", number)
|
||||
options["plot"] = list(match.groups())
|
||||
lines[index] = ""
|
||||
elif match := re.fullmatch(r"chart (auto|line|bar|none)", line):
|
||||
once("chart", number)
|
||||
options["chart"] = match[1]
|
||||
lines[index] = ""
|
||||
elif re.match(r"(?:where|and|or) ", line):
|
||||
if or_lines:
|
||||
# Keep AND higher precedence than OR and prevent SQL bounds from swallowing alternatives.
|
||||
rewritten = re.sub(
|
||||
r"^(where|and|or) (distance|heart_rate) ", r"\1 analysis_raw_\2 ", line
|
||||
)
|
||||
for field in ("distance", "heart_rate"):
|
||||
alias = "analysis_raw_" + field
|
||||
catalog[alias] = catalog[field]
|
||||
units[alias] = units[field]
|
||||
lines[index] = re.sub(r"^or ", "and ", rewritten)
|
||||
filter_counts.append(
|
||||
(
|
||||
line.startswith("or "),
|
||||
2 if re.match(r"^(where|and|or) \w+ between ", line) else 1,
|
||||
)
|
||||
)
|
||||
q = parse_dsl("\n".join(lines), catalog, defaults, units=units)
|
||||
q.update(options)
|
||||
q["derived"], q["aggregates"] = derived, aggregates
|
||||
if group:
|
||||
q["group"] = group
|
||||
for key in ("metric", "secondary"):
|
||||
q[key] = aliases.get(q[key], q[key])
|
||||
q["extra_metrics"] = [aliases.get(key, key) for key in q["extra_metrics"]]
|
||||
selected = [
|
||||
q["metric"],
|
||||
*([] if q["secondary"] == "none" else [q["secondary"]]),
|
||||
*q["extra_metrics"],
|
||||
]
|
||||
if len(set(selected)) != len(selected):
|
||||
raise QueryError("Choose different summary expressions.")
|
||||
if options["rank"] and options["rank"][0] not in selected:
|
||||
raise QueryError("Include the ranking expression in summarize.", locations["rank"])
|
||||
if options["trend"] and q["group"] not in {"day", "week", "month", "year"}:
|
||||
raise QueryError("Trends require grouping by day, week, month or year.", locations["trend"])
|
||||
if options["trend"] and options["rank"]:
|
||||
raise QueryError(
|
||||
"Ranked groups and time trends use different orderings. Use one per query.",
|
||||
locations["trend"],
|
||||
)
|
||||
if options["plot"] and options["chart"] == "none":
|
||||
raise QueryError("A scatter plot conflicts with chart none.", locations["chart"])
|
||||
if options["all_time"] and (q["end"] or q["start"]):
|
||||
raise QueryError("All time cannot also specify an end date.", locations["all-time period"])
|
||||
if options["all_time"]:
|
||||
if any(line.strip() == "compare previous period" for line in text.splitlines()):
|
||||
raise QueryError(
|
||||
"All time has no preceding period. Use compare none.", locations["all-time period"]
|
||||
)
|
||||
q["compare"] = False
|
||||
q["filter_groups"] = []
|
||||
if or_lines:
|
||||
position, groups = 0, [[]]
|
||||
for new_group, count in filter_counts:
|
||||
if new_group:
|
||||
if not groups[-1]:
|
||||
raise QueryError("Start an OR filter with where or and.")
|
||||
groups.append([])
|
||||
batch = q["filters"][position : position + count]
|
||||
for condition in batch:
|
||||
condition["field"] = condition["field"].removeprefix("analysis_raw_")
|
||||
groups[-1].extend(batch)
|
||||
position += count
|
||||
q["filter_groups"], q["filters"] = groups, []
|
||||
return q
|
||||
|
||||
|
||||
def describe(key, catalog, aggregates):
|
||||
if key not in aggregates:
|
||||
return catalog[key]
|
||||
spec = aggregates[key]
|
||||
function, field = spec["function"], spec["field"]
|
||||
label = (
|
||||
"Runs"
|
||||
if not field
|
||||
else f"{function.replace('_', ' ').capitalize()} · {catalog[field][0].lower()}"
|
||||
)
|
||||
unit = "runs" if function == "count" else catalog[field][1]
|
||||
if field == "pace" and function in {"stddev", "sum"}:
|
||||
unit = "sec/km"
|
||||
definition = (
|
||||
(
|
||||
"Count of rows"
|
||||
if not field
|
||||
else f"{function} of available per-run {catalog[field][0].lower()} values"
|
||||
)
|
||||
+ ". Percentiles use linear interpolation; stddev is population standard deviation. Missing values are excluded."
|
||||
)
|
||||
return label, unit, definition
|
||||
|
||||
|
||||
def statistic(values, function):
|
||||
values = [value for value in values if valid(value)]
|
||||
if function == "count":
|
||||
return len(values)
|
||||
if not values:
|
||||
return None
|
||||
if function == "sum":
|
||||
return sum(values)
|
||||
if function == "mean":
|
||||
return statistics.fmean(values)
|
||||
if function == "median":
|
||||
return statistics.median(values)
|
||||
if function == "min":
|
||||
return min(values)
|
||||
if function == "max":
|
||||
return max(values)
|
||||
if function == "stddev":
|
||||
return statistics.pstdev(values)
|
||||
percentile = int(function[1:]) / 100
|
||||
ordered = sorted(values)
|
||||
index = (len(ordered) - 1) * percentile
|
||||
lower = math.floor(index)
|
||||
return ordered[lower] + (ordered[math.ceil(index)] - ordered[lower]) * (index - lower)
|
||||
Reference in New Issue
Block a user