fix: make consolidation API async with background task + increased timeout
Three fixes for consolidation reliability:
1. Fire-and-forget API: POST /memory/consolidate now launches consolidation
as an asyncio background task and returns immediately. The old approach
blocked until Cat's WS response, which could take 5+ minutes (LLM
extraction calls), exceeding both the WS timeout and browser fetch
timeout. Web UI now polls /memory/status to track completion.
2. Increased timeout: cat_client.trigger_consolidation() timeout raised
from 300s to 600s (configurable via parameter). Logs unexpected WS
message types for debugging.
3. Better logging: Consolidation log messages prefixed with 🌙 for
grep-friendliness. cat_client errors include exc_info=True for
traceback visibility. Web UI shows elapsed time while polling.
This commit is contained in:
@@ -1,5 +1,8 @@
|
||||
"""Cheshire Cat memory management routes."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Form
|
||||
from fastapi.responses import JSONResponse
|
||||
@@ -88,13 +91,68 @@ async def get_episodic_memories():
|
||||
|
||||
@router.post("/memory/consolidate")
|
||||
async def trigger_memory_consolidation():
|
||||
"""Manually trigger memory consolidation (sleep consolidation process)."""
|
||||
"""
|
||||
Trigger memory consolidation as a background task.
|
||||
|
||||
Returns immediately — the Web UI should poll /memory/status
|
||||
to see when consolidation completes and view the result.
|
||||
"""
|
||||
from utils.cat_client import cat_adapter
|
||||
logger.info("🌙 Manual memory consolidation triggered via API")
|
||||
result = await cat_adapter.trigger_consolidation()
|
||||
if result is None:
|
||||
return JSONResponse(status_code=500, content={"success": False, "error": "Consolidation failed or timed out"})
|
||||
return {"success": True, "result": result}
|
||||
from utils.consolidation_scheduler import get_consolidation_status
|
||||
|
||||
# Check if already running
|
||||
status = get_consolidation_status()
|
||||
if status.get('is_running'):
|
||||
return {"success": True, "message": "Consolidation is already running", "status": status}
|
||||
|
||||
logger.info("🌙 Manual memory consolidation triggered via API (background)...")
|
||||
|
||||
# Launch consolidation as a background task so the API returns immediately.
|
||||
# The result is tracked via consolidation_scheduler's _last_consolidation state.
|
||||
asyncio.create_task(_run_consolidation_background())
|
||||
|
||||
return {"success": True, "message": "Consolidation started in background. Check status via /memory/status"}
|
||||
|
||||
|
||||
async def _run_consolidation_background():
|
||||
"""
|
||||
Run consolidation as a background task, updating the scheduler state.
|
||||
This prevents the API from blocking for minutes.
|
||||
"""
|
||||
from utils.cat_client import cat_adapter
|
||||
from utils.consolidation_scheduler import _last_consolidation
|
||||
|
||||
_last_consolidation['is_running'] = True
|
||||
_last_consolidation['last_run'] = datetime.now().isoformat()
|
||||
_last_consolidation['total_runs'] += 1
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
# Wait briefly for Cat to be ready if it was just started
|
||||
if not await cat_adapter.health_check():
|
||||
_last_consolidation['last_error'] = 'Cat health check failed'
|
||||
_last_consolidation['is_running'] = False
|
||||
return
|
||||
|
||||
result = await cat_adapter.trigger_consolidation(timeout=600)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
if result:
|
||||
logger.info(f"🌙 Manual consolidation completed in {elapsed:.1f}s: {result[:200]}")
|
||||
_last_consolidation['last_result'] = result
|
||||
_last_consolidation['last_error'] = None
|
||||
_last_consolidation['successful_runs'] += 1
|
||||
else:
|
||||
logger.error(f"🌙 Manual consolidation returned no result after {elapsed:.1f}s")
|
||||
_last_consolidation['last_error'] = f'No result returned after {elapsed:.1f}s (timeout or connection error)'
|
||||
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start_time
|
||||
logger.error(f"🌙 Manual consolidation failed after {elapsed:.1f}s: {e}")
|
||||
_last_consolidation['last_error'] = str(e)
|
||||
|
||||
finally:
|
||||
_last_consolidation['is_running'] = False
|
||||
|
||||
|
||||
@router.post("/memory/delete")
|
||||
|
||||
Reference in New Issue
Block a user