Paginate activity history and aggregate dashboard totals in SQL

This commit is contained in:
Peter Stockings
2026-10-02 14:36:35 +10:00
parent a300ea007d
commit 5830976e31
11 changed files with 386 additions and 65 deletions
+118
View File
@@ -0,0 +1,118 @@
"""Bounded activity lists, SQL totals and small streamed record comparisons."""
from datetime import UTC, datetime, timedelta
from itertools import pairwise
from zoneinfo import ZoneInfo
from flask import abort, request
from sqlalchemy.orm import joinedload
from .extensions import db
from .models import Activity
from .records import EFFORTS_VERSION, record_book
PAGE_SIZE = 25
def activity_page(filters, total=None):
value = request.args.get("page", "1")
if not value.isascii() or not value.isdecimal() or len(value) > 10 or int(value) < 1:
abort(404)
pagination = db.paginate(
db.select(Activity)
.where(*filters)
.options(joinedload(Activity.owner))
.order_by(Activity.started_at.desc(), Activity.id.desc()),
page=int(value),
per_page=PAGE_SIZE,
max_per_page=PAGE_SIZE,
count=total is None,
error_out=True,
)
if total is not None:
pagination.total = total
return pagination
def overview(filters, timezone):
totals = db.session.execute(
db.select(
db.func.count(Activity.id),
db.func.sum(Activity.distance_m),
db.func.sum(Activity.duration_s),
db.func.sum(Activity.elevation_m),
db.func.max(Activity.started_at),
).where(*filters)
).one()
count, distance, duration, elevation, latest = totals
distance, duration, elevation = distance or 0, duration or 0, elevation or 0
zone = ZoneInfo(timezone)
latest_day = (latest or datetime.now(UTC)).replace(tzinfo=UTC).astimezone(zone).date()
monday = latest_day - timedelta(days=latest_day.weekday())
weeks = [monday - timedelta(weeks=n) for n in reversed(range(8))]
# Convert each local week boundary independently, including daylight-saving changes.
boundaries = [
datetime.combine(day, datetime.min.time(), tzinfo=zone).astimezone(UTC)
for day in [*weeks, monday + timedelta(weeks=1)]
]
weekly = db.session.execute(
db.select(
*[
db.func.coalesce(
db.func.sum(
db.case(
(
db.and_(Activity.started_at >= start, Activity.started_at < end),
Activity.distance_m,
),
else_=0,
)
),
0,
)
for start, end in pairwise(boundaries)
]
).where(
*filters, Activity.started_at >= boundaries[0], Activity.started_at < boundaries[-1]
)
).one()
seconds = round(duration * 1000 / distance) if distance else None
return {
"distance": distance / 1000,
"duration": duration,
"count": count,
"elevation": elevation,
"pace": f"{seconds // 60}:{seconds % 60:02d}" if seconds is not None else "—",
"weeks": [day.strftime("%d %b") for day in weeks],
"weekly_km": [round(value / 1000, 2) for value in weekly],
}
def latest_mapped_run(filters):
return db.session.scalar(
db.select(Activity)
.where(*filters, Activity.sport == "running", Activity.has_route.is_(True))
.order_by(Activity.started_at.desc(), Activity.id.desc())
.limit(1)
)
def records_for(owner_id, run_ids):
history = db.session.execute(
db.select(
Activity.id,
Activity.title,
Activity.started_at,
Activity.sport,
Activity.efforts_version,
Activity.best_efforts,
)
.where(
Activity.owner_id == owner_id,
Activity.sport == "running",
Activity.efforts_version == EFFORTS_VERSION,
)
.order_by(Activity.started_at, Activity.id)
.execution_options(yield_per=1000)
)
return record_book(history, run_ids=run_ids, presorted=True)
+7 -1
View File
@@ -70,7 +70,13 @@ class Activity(db.Model):
samples = db.deferred(db.Column(db.JSON, nullable=False, default=list))
laps = db.deferred(db.Column(db.JSON, nullable=False, default=list))
created_at = db.Column(db.DateTime(timezone=True), default=lambda: datetime.now(UTC))
__table_args__ = (db.UniqueConstraint("owner_id", "source_key"),)
__table_args__ = (
db.UniqueConstraint("owner_id", "source_key"),
db.Index("ix_activity_owner_started_id", "owner_id", "started_at", "id"),
db.Index(
"ix_activity_owner_visibility_started_id", "owner_id", "visibility", "started_at", "id"
),
)
@property
def distance_km(self):
+7 -3
View File
@@ -85,14 +85,17 @@ def persist_efforts(activity):
activity.has_route = any(sample.get("point") for sample in activity.samples)
def record_book(activities):
def record_book(activities, run_ids=None, presorted=False):
"""Compare persisted efforts in chronology; never read samples here.
First efforts establish a record. Strict improvements earn historical PR
badges, which remain on the run after a later activity beats that record.
"""
records, runs = {}, {}
for activity in sorted(activities, key=lambda row: (row.started_at, row.id)):
history = (
activities if presorted else sorted(activities, key=lambda row: (row.started_at, row.id))
)
for activity in history:
efforts = (
[dict(effort) for effort in activity.best_efforts]
if activity.efforts_version == EFFORTS_VERSION and activity.sport == "running"
@@ -103,7 +106,8 @@ def record_book(activities):
effort["pr"] = previous is None or effort["seconds"] < previous["seconds"] - 1e-6
if effort["pr"]:
records[effort["label"]] = dict(effort, activity=activity)
runs[activity.id] = efforts
if run_ids is None or activity.id in run_ids:
runs[activity.id] = efforts
return {"records": [records[label] for label, _ in DISTANCES if label in records], "runs": runs}
+25 -58
View File
@@ -1,10 +1,9 @@
import hashlib
import re
import secrets
from collections import defaultdict
from datetime import UTC, datetime, timedelta
from datetime import UTC, datetime
from pathlib import Path
from zoneinfo import ZoneInfo, available_timezones
from zoneinfo import available_timezones
from flask import (
Blueprint,
@@ -27,8 +26,8 @@ from garmin_sync.service import SyncError, atomic_write
from .analytics import pace_text
from .display import stored_analysis
from .extensions import db, limiter
from .history import activity_page, latest_mapped_run, overview, records_for
from .models import Activity, Friendship, GarminConnection, User
from .records import record_book
from .services import activity_data, import_cache, import_upload, route_data
web = Blueprint("web", __name__)
@@ -75,36 +74,6 @@ def are_friends(first, second):
)
def latest_mapped_run(rows):
return next(
(row for row in rows if row.sport == "running" and row.has_route),
None,
)
def overview(rows, timezone="Australia/Sydney"):
total_distance = sum(r.distance_m for r in rows)
total_time = sum(r.duration_s for r in rows)
latest = max((r.started_at for r in rows), default=datetime.now(UTC))
latest_day = latest.replace(tzinfo=UTC).astimezone(ZoneInfo(timezone)).date()
monday = latest_day - timedelta(days=latest_day.weekday())
weeks = [monday - timedelta(weeks=n) for n in reversed(range(8))]
distances = defaultdict(float)
for row in rows:
day = row.started_at.replace(tzinfo=UTC).astimezone(ZoneInfo(timezone)).date()
distances[day - timedelta(days=day.weekday())] += row.distance_km
seconds = round(total_time / (total_distance / 1000)) if total_distance else None
return {
"distance": total_distance / 1000,
"duration": total_time,
"count": len(rows),
"elevation": sum(r.elevation_m for r in rows),
"pace": f"{seconds // 60}:{seconds % 60:02d}" if seconds is not None else "—",
"weeks": [w.strftime("%d %b") for w in weeks],
"weekly_km": [round(distances[w], 2) for w in weeks],
}
@web.get("/health")
def health():
db.session.execute(db.text("SELECT 1"))
@@ -208,17 +177,18 @@ def logout():
@web.get("/dashboard")
@login_required
def dashboard():
rows = db.session.scalars(
db.select(Activity).filter_by(owner_id=current_user.id).order_by(Activity.started_at.desc())
).all()
filters = [Activity.owner_id == current_user.id]
stats = overview(filters, current_user.timezone)
pagination = activity_page(filters, total=stats["count"])
return render_template(
"dashboard.html",
rows=rows,
stats=overview(rows, current_user.timezone),
rows=pagination.items,
pagination=pagination,
stats=stats,
profile=current_user,
personal=True,
record_book=record_book(rows),
mapped_run=latest_mapped_run(rows),
record_book=records_for(current_user.id, {row.id for row in pagination.items}),
mapped_run=latest_mapped_run(filters),
)
@@ -257,13 +227,12 @@ def friends():
for link in links
]
accepted_ids = [p["person"].id for p in people if p["link"].status == "accepted"]
rows = db.session.scalars(
db.select(Activity)
.where(Activity.owner_id.in_(accepted_ids), Activity.visibility == "friends")
.order_by(Activity.started_at.desc())
.limit(50)
).all()
return render_template("friends.html", people=people, rows=rows)
pagination = activity_page(
[Activity.owner_id.in_(accepted_ids), Activity.visibility == "friends"]
)
return render_template(
"friends.html", people=people, rows=pagination.items, pagination=pagination
)
@web.post("/friends/<int:friendship_id>/<action>")
@@ -290,18 +259,17 @@ def profile(username):
user = db.first_or_404(db.select(User).filter_by(username=username))
if user.id != current_user.id and not are_friends(current_user.id, user.id):
abort(404)
rows = db.session.scalars(
db.select(Activity)
.filter_by(owner_id=user.id, visibility="friends")
.order_by(Activity.started_at.desc())
).all()
filters = [Activity.owner_id == user.id, Activity.visibility == "friends"]
stats = overview(filters, user.timezone)
pagination = activity_page(filters, total=stats["count"])
return render_template(
"dashboard.html",
rows=rows,
stats=overview(rows, user.timezone),
rows=pagination.items,
pagination=pagination,
stats=stats,
profile=user,
personal=False,
mapped_run=latest_mapped_run(rows),
mapped_run=latest_mapped_run(filters),
)
@@ -311,8 +279,7 @@ def activity(activity_id):
analysis = stored_analysis(row)
efforts = []
if owner:
history = db.session.scalars(db.select(Activity).filter_by(owner_id=row.owner_id)).all()
efforts = record_book(history)["runs"].get(row.id, [])
efforts = records_for(row.owner_id, {row.id})["runs"].get(row.id, [])
return render_template(
"activity.html",
activity=row,
+4
View File
@@ -35,3 +35,7 @@ body{font-family:"Segoe UI",Arial,sans-serif}h1{font-weight:650;letter-spacing:-
.run-badges { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 5px; }
.run-badges:empty { display: none; }
.pr-badge { display: inline-block; border: 1px solid #886b34; border-radius: 4px; padding: 3px 6px; color: #f3cb78; background: #3a3020; font-size: 11px; font-weight: 600; }
.history-pagination { display:flex; align-items:center; justify-content:space-between; gap:16px; padding-top:20px; border-top:1px solid var(--line); }
.history-pagination > div { display:flex; align-items:center; gap:12px; }
@media(max-width:680px) { .history-pagination { flex-direction:column; align-items:flex-start; } .history-pagination > div { flex-wrap:wrap; } }
+9
View File
@@ -3,3 +3,12 @@
{% for row in rows %}<tr><td><a class="activity-name" href="{{ url_for('web.activity',activity_id=row.id) }}"><span class="sport-icon">{{ icon('run') }}</span><span><strong>{{ row.title }}</strong>{% if record_book is defined %}<span class="run-badges">{% for effort in record_book.runs.get(row.id, []) if effort.pr %}<span class="pr-badge" title="Personal record set on this run · {{ effort.seconds|duration }}">★ {{ effort.label }} PR</span>{% endfor %}</span>{% endif %}<small>{{ row.started_at|localdate(row.owner.timezone) }} <span>·</span> {{ row.sport|replace('_',' ')|title }}</small></span></a></td><td><strong>{{ '%.2f'|format(row.distance_km) }}</strong><small> km</small></td><td>{{ row.duration_s|duration }}</td><td>{{ row.pace }}<small> /km</small></td><td>{% if personal|default(false) %}<span class="pill {{ row.visibility }}">{{ row.visibility|title }}</span>{% else %}<a class="athlete-link" href="{{ url_for('web.profile',username=row.owner.username) }}">{{ row.owner.display_name }}</a>{% endif %}</td><td><a class="row-arrow" aria-label="View {{ row.title }}" href="{{ url_for('web.activity',activity_id=row.id) }}">{{ icon('arrow') }}</a></td></tr>{% endfor %}
</tbody></table></div>
{% else %}<div class="empty-state"><span class="empty-symbol">↗</span><h3>No activities yet.</h3><p>{{ 'Import a FIT file or load your Garmin activities to get started.' if personal|default(false) else 'No shared activities just yet. Check back after the next run.' }}</p>{% if personal|default(false) %}<a class="button primary" href="{{ url_for('web.settings') }}">Import your first run {{ icon('arrow') }}</a>{% endif %}</div>{% endif %}
{% if pagination is defined and pagination.pages > 1 %}
<nav class="history-pagination" aria-label="Activity history pages">
<span class="field-help">Showing {{ pagination.first }}–{{ pagination.last }} of {{ pagination.total }} activities</span>
<div>{% if pagination.has_prev %}<a class="button secondary" rel="prev" href="{{ url_for(request.endpoint, page=pagination.prev_num, **(request.view_args or {})) }}#history">← Newer</a>{% endif %}
<span class="field-help">Page {{ pagination.page }} of {{ pagination.pages }}</span>
{% if pagination.has_next %}<a class="button secondary" rel="next" href="{{ url_for(request.endpoint, page=pagination.next_num, **(request.view_args or {})) }}#history">Older →</a>{% endif %}</div>
</nav>
{% endif %}
+1 -1
View File
@@ -25,6 +25,6 @@
<p class="section-description">Fastest stretches within your runs, including stops. PR badges mark records set at the time of the run; your first effort establishes a record.</p>
<div class="records-grid">{% for record in record_book.records %}<a class="record-card" href="{{ url_for('web.activity', activity_id=record.activity.id) }}"><span>{{ record.label }}</span><strong>{{ record.seconds|duration }}</strong><small>{{ record.activity.started_at|localdate(profile.timezone) }} · {{ record.activity.title }}</small></a>{% else %}<p class="field-help">Import a run with recorded distance and timestamps to establish your first record.</p>{% endfor %}</div></section>
{% endif %}
<section class="panel activities-panel"><div class="section-head"><div><span class="eyebrow">SESSION HISTORY</span><h2>{{ 'Your activities' if personal else 'Shared activities' }}</h2></div><span class="subtle-label">{{ rows|length }} activities</span></div>{% include '_activities.html' %}</section>
<section class="panel activities-panel" id="history"><div class="section-head"><div><span class="eyebrow">SESSION HISTORY</span><h2>{{ 'Your activities' if personal else 'Shared activities' }}</h2></div><span class="subtle-label">{{ stats.count }} activities</span></div>{% include '_activities.html' %}</section>
{% endblock %}
{% block scripts %}<script src="{{ url_for('static',filename='vendor/leaflet.js') }}"></script><script src="{{ url_for('static',filename='vendor/chart.umd.js') }}"></script><script src="{{ url_for('static',filename='dashboard.js') }}"></script>{% endblock %}
+1 -1
View File
@@ -10,5 +10,5 @@
{% if entry.link.status == 'pending' and entry.link.requested_by != current_user.id %}<form method="post" action="{{ url_for('web.friendship_action', friendship_id=entry.link.id, action='accept') }}"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"><button class="button primary">Accept</button></form>{% endif %}
<form method="post" action="{{ url_for('web.friendship_action', friendship_id=entry.link.id, action='remove') }}"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}"><button class="text-button">{{ 'Remove' if entry.link.status == 'accepted' else 'Cancel' if entry.link.requested_by == current_user.id else 'Decline' }}</button></form></div></div>{% else %}<p class="muted">Your circle starts here. Add a friend using the email they registered with.</p>{% endfor %}
</section></div>
<section class="panel activities-panel"><div class="section-head"><div><span class="eyebrow">MILES FROM YOUR CIRCLE</span><h2>Friends’ activities</h2></div><span class="subtle-label">Shared with you</span></div>{% include '_activities.html' %}</section>
<section class="panel activities-panel" id="history"><div class="section-head"><div><span class="eyebrow">MILES FROM YOUR CIRCLE</span><h2>Friends’ activities</h2></div><span class="subtle-label">Shared with you</span></div>{% include '_activities.html' %}</section>
{% endblock %}
+27
View File
@@ -63,3 +63,30 @@ HTTP 200 and `status: ok`; Dokku web and sync worker startup checks succeeded.
The synthetic browser fixture verified dashboard replay, route colour switching
and charts. Its in-memory SQLite server uses one request thread to avoid concurrent
access to the shared SQLite connection.
## Paginated history
Dashboard, shared profiles and the friends feed show 25 activities per page,
ordered by start time descending with activity ID as the tie-breaker. Navigation
preserves profile URLs and returns to the history section. Invalid or empty
out-of-range pages return 404; an empty first page remains valid.
Dashboard and profile totals are SQL aggregates over the entire authorized
history. Eight weekly buckets use independent local-time Monday boundaries
converted to UTC, including daylight-saving changes. The latest GPS run is an
independent indexed query, so paging does not change the map. Friend/profile
filters apply equally to pagination, counts, totals and route selection.
Personal record comparisons stream only ID, title, timestamp, sport, calculation
version and stored efforts in batches of 1,000. They retain at most six current
record holders and effort details for the requested page (or one run's detail).
No full-history Activity ORM objects, raw samples, charts or laps are loaded.
Comparisons remain linear in the number of recorded runs; page rows and ORM
object counts are bounded. Owners are loaded with the page to avoid per-row
athlete queries. Composite indexes support owner/date and shared owner/date lists.
A disposable local SQLite benchmark with 10,000 synthetic activities returned
25 rows and 26 ORM objects on pages 1, 200 and 400. HTML remained about 30 KiB;
measured request times were 148, 122 and 97 ms respectively. These are synthetic
local measurements, not production latency guarantees. Browser checks with 55
synthetic activities verified pages 1–3, including the five-row final page.
@@ -0,0 +1,22 @@
"""Index athlete activity history and shared profile pagination."""
from alembic import op
revision = "09a7c863b521"
down_revision = "f08db2902a74"
branch_labels = None
depends_on = None
def upgrade():
op.create_index("ix_activity_owner_started_id", "activity", ["owner_id", "started_at", "id"])
op.create_index(
"ix_activity_owner_visibility_started_id",
"activity",
["owner_id", "visibility", "started_at", "id"],
)
def downgrade():
op.drop_index("ix_activity_owner_visibility_started_id", table_name="activity")
op.drop_index("ix_activity_owner_started_id", table_name="activity")
+165 -1
View File
@@ -62,9 +62,173 @@ def app(tmp_path):
def login(client, name="runner"):
return client.post(
response = client.post(
"/login", data={"email": f"{name}@example.test", "password": "a-long-test-password"}
)
assert response.status_code == 302, response.data.decode()
return response
def add_history(count, visibility="private"):
from datetime import timedelta
for index in range(count):
db.session.add(
Activity(
owner_id=1,
source_key=f"history:{index}",
title=f"History run {index:03d}",
sport="running",
started_at=datetime(2026, 9, 1, tzinfo=UTC) + timedelta(days=index),
distance_m=1000,
duration_s=300,
elevation_m=1,
visibility=visibility,
efforts_version=EFFORTS_VERSION,
best_efforts=[
{
"label": "1 km",
"distance": 1000,
"seconds": 250 if index == 0 else 300,
"start_m": 0,
"end_m": 1000,
}
],
samples=[],
laps=[],
)
)
db.session.commit()
def test_history_pages_keep_full_totals_records_and_latest_route(app):
import re
from flask import template_rendered
with app.app_context():
add_history(60)
contexts = []
def capture(_app, template, context, **_extra):
contexts.append(context)
template_rendered.connect(capture, app)
client = app.test_client()
login(client)
try:
seen = []
for page, size in ((1, 25), (2, 25), (3, 11)):
response = client.get(f"/dashboard?page={page}")
assert response.status_code == 200
context = contexts[-1]
assert len(context["rows"]) == size
assert context["stats"]["count"] == 61
assert context["stats"]["distance"] == 65
assert context["stats"]["duration"] == 19800
assert context["stats"]["elevation"] == 100
assert context["mapped_run"].id == 1
assert context["record_book"]["records"][0]["seconds"] == 250
assert set(context["record_book"]["runs"]).issubset({row.id for row in context["rows"]})
seen.extend(
int(i)
for i in re.findall(
rb'class="activity-name" href="/activities/(\d+)"', response.data
)
)
assert len(seen) == len(set(seen)) == 61
assert b"Showing 51" in response.data
assert b"1 km PR" in response.data
assert client.get("/dashboard?page=4").status_code == 404
for value in ("0", "-1", "nope", "999999999999999999999", "1.5"):
assert client.get(f"/dashboard?page={value}").status_code == 404
finally:
template_rendered.disconnect(capture, app)
def test_pagination_uses_id_to_break_equal_timestamp_ties(app):
with app.app_context():
add_history(50)
db.session.execute(db.update(Activity).values(started_at=datetime(2026, 1, 1, tzinfo=UTC)))
db.session.commit()
import re
client = app.test_client()
login(client)
ids = []
for page in (1, 2, 3):
ids.extend(
int(i)
for i in re.findall(
rb'class="activity-name" href="/activities/(\d+)"',
client.get(f"/dashboard?page={page}").data,
)
)
assert ids == list(reversed(range(1, 52)))
def test_profile_and_friends_pages_include_only_visible_history(app):
from flask import template_rendered
with app.app_context():
add_history(53, visibility="friends")
db.session.add(Friendship(low_id=1, high_id=2, requested_by=1, status="accepted"))
db.session.commit()
contexts = []
def capture(_app, template, context, **_extra):
contexts.append(context)
template_rendered.connect(capture, app)
client = app.test_client()
login(client, "other")
try:
for path in ("/u/runner", "/friends"):
for page, expected in ((1, 25), (2, 25), (3, 3)):
response = client.get(f"{path}?page={page}")
assert response.status_code == 200
assert len(contexts[-1]["rows"]) == expected
assert contexts[-1]["pagination"].total == 53
assert b"A quiet morning run" not in response.data
if path == "/u/runner":
assert contexts[-1]["stats"]["distance"] == 53
client.post("/friends/1/remove")
assert client.get("/u/runner?page=2").status_code == 404
assert client.get("/friends?page=2").status_code == 404
finally:
template_rendered.disconnect(capture, app)
def test_weekly_sql_totals_respect_sydney_dst_boundaries(app):
from app.history import overview
with app.app_context():
for index, (started, distance) in enumerate(
(
(datetime(2026, 10, 4, 12, 59, 59, tzinfo=UTC), 1000),
(datetime(2026, 10, 4, 13, 0, 0, tzinfo=UTC), 2000),
)
):
db.session.add(
Activity(
owner_id=2,
source_key=f"dst:{index}",
title="DST run",
started_at=started,
distance_m=distance,
duration_s=600,
)
)
db.session.commit()
stats = overview([Activity.owner_id == 2], "Australia/Sydney")
assert stats["weeks"][-2:] == ["28 Sep", "05 Oct"]
assert stats["weekly_km"][-2:] == [1, 2]
assert stats["distance"] == 3
assert stats["count"] == 2
empty = overview([Activity.owner_id == 999], "Australia/Sydney")
assert empty["distance"] == 0
assert empty["pace"] == "—"
assert empty["weekly_km"] == [0] * 8
def test_private_activity_authorization_for_pages_json_and_mutations(app):