Persist best efforts during import and add resumable backfill

app/__init__.py

@@ -67,6 +67,9 @@ def create_app(config=None):
     from .sync import register_cli
 
     register_cli(app)
+    from .records import register_cli as register_records_cli
+
+    register_records_cli(app)
 
     @login_manager.user_loader
     def load_user(user_id):

app/models.py

@@ -54,8 +54,11 @@ class Activity(db.Model):
     metrics = db.Column(db.JSON, nullable=False, default=dict, server_default="{}")
     visibility = db.Column(db.String(12), nullable=False, default="private")
     share_token = db.Column(db.String(64), unique=True)
-    samples = db.Column(db.JSON, nullable=False, default=list)
-    laps = db.Column(db.JSON, nullable=False, default=list)
+    best_efforts = db.Column(db.JSON, nullable=False, default=list, server_default="[]")
+    efforts_version = db.Column(db.Integer, nullable=False, default=0, server_default="0")
+    has_route = db.Column(db.Boolean, nullable=False, default=False, server_default=db.false())
+    samples = db.deferred(db.Column(db.JSON, nullable=False, default=list))
+    laps = db.deferred(db.Column(db.JSON, nullable=False, default=list))
     created_at = db.Column(db.DateTime(timezone=True), default=lambda: datetime.now(UTC))
     __table_args__ = (db.UniqueConstraint("owner_id", "source_key"),)
 

app/records.py

@@ -2,8 +2,11 @@
 
 from bisect import bisect_left, bisect_right
 
+import click
+
 from .analytics import numeric
 
+EFFORTS_VERSION = 1
 DISTANCES = (
     ("1 km", 1000),
     ("2 km", 2000),
@@ -75,15 +78,26 @@ def best_efforts(activity):
     return [best[label] for label, _ in DISTANCES if label in best]
 
 
+def persist_efforts(activity):
+    """Store derived data in the same transaction as the imported run."""
+    activity.best_efforts = best_efforts(activity)
+    activity.efforts_version = EFFORTS_VERSION
+    activity.has_route = any(sample.get("point") for sample in activity.samples)
+
+
 def record_book(activities):
-    """Recompute in activity chronology so backfills and removals stay correct.
+    """Compare persisted efforts in chronology; never read samples here.
 
     First efforts establish a record. Strict improvements earn historical PR
     badges, which remain on the run after a later activity beats that record.
     """
     records, runs = {}, {}
     for activity in sorted(activities, key=lambda row: (row.started_at, row.id)):
-        efforts = best_efforts(activity)
+        efforts = (
+            [dict(effort) for effort in activity.best_efforts]
+            if activity.efforts_version == EFFORTS_VERSION and activity.sport == "running"
+            else []
+        )
         for effort in efforts:
             previous = records.get(effort["label"])
             effort["pr"] = previous is None or effort["seconds"] < previous["seconds"] - 1e-6
@@ -91,3 +105,48 @@ def record_book(activities):
                 records[effort["label"]] = dict(effort, activity=activity)
         runs[activity.id] = efforts
     return {"records": [records[label] for label, _ in DISTANCES if label in records], "runs": runs}
+
+
+def register_cli(app):
+    @app.cli.command("backfill-efforts")
+    @click.option("--batch-size", default=25, type=click.IntRange(1, 1000), show_default=True)
+    @click.option("--owner-id", type=click.IntRange(1), help="Limit to one athlete.")
+    @click.option("--force", is_flag=True, help="Recalculate even current-version efforts.")
+    def backfill(batch_size, owner_id, force):
+        """Backfill persisted best efforts in bounded, resumable transactions."""
+        from sqlalchemy.orm import undefer
+
+        from .extensions import db
+        from .models import Activity
+
+        last_id, count = 0, 0
+        # A fixed upper bound prevents concurrent imports extending this run forever.
+        maximum = db.session.scalar(db.select(db.func.max(Activity.id))) or 0
+        while last_id < maximum:
+            query = db.select(Activity).where(Activity.id > last_id, Activity.id <= maximum)
+            if owner_id:
+                query = query.where(Activity.owner_id == owner_id)
+            if not force:
+                query = query.where(Activity.efforts_version != EFFORTS_VERSION)
+            rows = db.session.scalars(
+                query.options(undefer(Activity.samples))
+                .order_by(Activity.id)
+                .limit(batch_size)
+                .with_for_update()
+            ).all()
+            if not rows:
+                break
+            try:
+                for row in rows:
+                    persist_efforts(row)
+                last_id = rows[-1].id
+                db.session.commit()
+            except Exception as error:
+                db.session.rollback()
+                raise click.ClickException(
+                    "Backfill failed; completed batches are saved. Rerun to resume."
+                ) from error
+            count += len(rows)
+            db.session.expunge_all()
+            click.echo(f"Backfilled {count} activities (through ID {last_id}).")
+        click.echo(f"Complete: {count} activities backfilled; effort version {EFFORTS_VERSION}.")

app/routes.py

@@ -79,7 +79,7 @@ def latest_mapped_run(rows):
         (
             row
             for row in rows
-            if row.sport == "running" and any(sample.get("point") for sample in row.samples)
+            if row.sport == "running" and row.has_route
         ),
         None,
     )

app/services.py

@@ -8,6 +8,7 @@ from garmin_sync.service import SyncError, complete, decode_fit
 
 from .extensions import db
 from .models import Activity
+from .records import persist_efforts
 
 
 def number(value, default=0):
@@ -110,6 +111,7 @@ def activity_from_fit(owner_id, source_key, documents, metadata=None, refresh=Fa
     }
     if existing:
         existing.samples, existing.laps, existing.metrics = samples, laps, metrics
+        persist_efforts(existing)
         return existing
     row = Activity(
         owner_id=owner_id,
@@ -125,6 +127,7 @@ def activity_from_fit(owner_id, source_key, documents, metadata=None, refresh=Fa
         samples=samples,
         laps=laps,
     )
+    persist_efforts(row)
     db.session.add(row)
     return row