hermes
6bc9fa0e8d
Previously 3 failed attempts at 06:00 (e.g. llamaswap model cold/starting) permanently blocked generation until the next day. Now the retry loop pauses SLOW_RETRY_MINUTES (default 60) after the burst budget is spent, then resumes — one attempt per hour until jokes exist.
72 lines
2.5 KiB
Python
72 lines
2.5 KiB
Python
"""Daily joke generation job with retry/backoff and stale-fallback semantics."""
|
|
import logging
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from . import db
|
|
from .config import settings
|
|
from .llm_client import generate_jokes
|
|
from .wiki_client import get_featured_article
|
|
|
|
log = logging.getLogger("jokes.generator")
|
|
|
|
# In-process retry bookkeeping (resets on container restart; cold-start
|
|
# generation covers the restart case anyway).
|
|
_attempts_today: dict[str, int] = {}
|
|
_last_failure: dict[str, datetime] = {}
|
|
|
|
|
|
def today_local() -> date:
|
|
return datetime.now(ZoneInfo(settings.timezone)).date()
|
|
|
|
|
|
def _attempt_key(day: date) -> str:
|
|
return day.isoformat()
|
|
|
|
|
|
def generation_exhausted(day: date) -> bool:
|
|
"""True while in cooldown: max attempts hit and the last failure is
|
|
younger than SLOW_RETRY_MINUTES. After the cooldown passes, the daily
|
|
retry loop resumes at the slow interval — so a cold/down LLM endpoint
|
|
at 06:00 no longer kills the whole day."""
|
|
if _attempts_today.get(_attempt_key(day), 0) < settings.max_attempts:
|
|
return False
|
|
last = _last_failure.get(_attempt_key(day))
|
|
if last is None:
|
|
return True
|
|
age = datetime.now(timezone.utc) - last
|
|
return age < timedelta(minutes=settings.slow_retry_minutes)
|
|
|
|
|
|
def run_generation() -> bool:
|
|
"""Try to generate today's jokes. Returns True on success."""
|
|
day = today_local()
|
|
if db.get_day(day) is not None:
|
|
log.info("Jokes for %s already exist, skipping", day)
|
|
return True
|
|
|
|
if generation_exhausted(day):
|
|
# In cooldown after max attempts; will resume after slow_retry_minutes.
|
|
return False
|
|
|
|
_attempts_today[_attempt_key(day)] = _attempts_today.get(_attempt_key(day), 0) + 1
|
|
try:
|
|
article = get_featured_article(day)
|
|
log.info("Fetched featured article: %s", article.title)
|
|
jokes = generate_jokes(article.title, article.extract)
|
|
db.save_batch(day, article.title, article.url, jokes)
|
|
log.info("Saved %d jokes for %s", len(jokes), day)
|
|
return True
|
|
except Exception as exc: # noqa: BLE001 — job must never crash the scheduler
|
|
used = _attempts_today[_attempt_key(day)]
|
|
_last_failure[_attempt_key(day)] = datetime.now(timezone.utc)
|
|
log.error(
|
|
"Generation attempt %d for %s failed: %s (cooldown %d min before next try)",
|
|
used, day, exc, settings.slow_retry_minutes,
|
|
)
|
|
return False
|
|
|
|
|
|
def next_retry_delay() -> timedelta:
|
|
return timedelta(minutes=settings.retry_minutes)
|