Compare commits
4 Commits
9eb081efb1
...
cb4be35f13
| Author | SHA1 | Date | |
|---|---|---|---|
| cb4be35f13 | |||
| f3c4a8fe5a | |||
| 811bcc0a5d | |||
| e6e81885b3 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -71,6 +71,7 @@ temp_*
|
||||
*.jpeg
|
||||
*.gif
|
||||
!static/images/*
|
||||
!bot/static/miku-favicon.png
|
||||
|
||||
# Backups
|
||||
backups/
|
||||
|
||||
11
bot/bot.py
11
bot/bot.py
@@ -122,6 +122,17 @@ async def on_ready():
|
||||
else:
|
||||
logger.warning("OWNER_USER_ID not set, DM analysis feature disabled")
|
||||
|
||||
# Schedule nightly memory consolidation (runs at 4 AM UTC every day)
|
||||
from utils.consolidation_scheduler import run_nightly_consolidation
|
||||
globals.scheduler.add_job(
|
||||
run_nightly_consolidation,
|
||||
'cron',
|
||||
hour=4,
|
||||
minute=0,
|
||||
id='nightly_consolidation'
|
||||
)
|
||||
logger.info("🌙 Scheduled nightly memory consolidation at 4:00 AM UTC")
|
||||
|
||||
# Setup autonomous speaking (now handled by server manager)
|
||||
setup_autonomous_speaking()
|
||||
load_last_sent_tweets()
|
||||
|
||||
@@ -14,15 +14,17 @@ router = APIRouter()
|
||||
|
||||
@router.get("/memory/status")
|
||||
async def get_cat_memory_status():
|
||||
"""Get Cheshire Cat connection status and feature flag."""
|
||||
"""Get Cheshire Cat connection status, feature flag, and consolidation state."""
|
||||
from utils.cat_client import cat_adapter
|
||||
from utils.consolidation_scheduler import get_consolidation_status
|
||||
is_healthy = await cat_adapter.health_check()
|
||||
return {
|
||||
"enabled": globals.USE_CHESHIRE_CAT,
|
||||
"healthy": is_healthy,
|
||||
"url": globals.CHESHIRE_CAT_URL,
|
||||
"circuit_breaker_active": cat_adapter._is_circuit_broken(),
|
||||
"consecutive_failures": cat_adapter._consecutive_failures
|
||||
"consecutive_failures": cat_adapter._consecutive_failures,
|
||||
"consolidation": get_consolidation_status(),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
<meta http-equiv="Pragma" content="no-cache">
|
||||
<meta http-equiv="Expires" content="0">
|
||||
<title>Miku Control Panel</title>
|
||||
<link rel="icon" href="/static/miku-favicon.png" type="image/png">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/cropperjs/1.6.2/cropper.min.css">
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/cropperjs/1.6.2/cropper.min.js"></script>
|
||||
<link rel="stylesheet" href="/static/css/style.css?v=20260502">
|
||||
@@ -1033,6 +1034,7 @@
|
||||
<h4 style="margin: 0 0 0.5rem 0;">🌙 Memory Consolidation</h4>
|
||||
<p style="color: #aaa; font-size: 0.85rem; margin-bottom: 0.75rem;">
|
||||
Trigger the sleep consolidation process: analyzes episodic memories, extracts important facts, and removes trivial entries.
|
||||
<br><span style="color: #888;">⏰ Scheduled nightly at 4:00 AM UTC</span>
|
||||
</p>
|
||||
<div style="display: flex; gap: 0.5rem; align-items: center;">
|
||||
<button id="consolidate-btn" onclick="triggerConsolidation()" style="background: #5b3a8c; color: #fff; padding: 0.5rem 1rem; border: none; border-radius: 4px; cursor: pointer; font-weight: bold;">
|
||||
@@ -1040,6 +1042,7 @@
|
||||
</button>
|
||||
<span id="consolidation-status" style="color: #888; font-size: 0.85rem;"></span>
|
||||
</div>
|
||||
<div id="consolidation-schedule-info" style="margin-top: 0.5rem; color: #666; font-size: 0.78rem;"></div>
|
||||
<div id="consolidation-result" style="display: none; margin-top: 0.75rem; background: #111; border: 1px solid #333; border-radius: 4px; padding: 0.75rem; font-size: 0.85rem; color: #ccc; white-space: pre-wrap; max-height: 200px; overflow-y: auto;"></div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -39,6 +39,30 @@ async function refreshMemoryStats() {
|
||||
document.getElementById('stat-declarative-count').textContent = '—';
|
||||
document.getElementById('stat-procedural-count').textContent = '—';
|
||||
}
|
||||
|
||||
// Show consolidation schedule info if available
|
||||
const consInfo = document.getElementById('consolidation-schedule-info');
|
||||
if (consInfo && statusData.consolidation) {
|
||||
let infoHtml = '';
|
||||
const cons = statusData.consolidation;
|
||||
if (cons.last_run) {
|
||||
const lastRun = new Date(cons.last_run).toLocaleString();
|
||||
infoHtml += `🕐 Last run: ${lastRun}`;
|
||||
if (cons.last_error) {
|
||||
infoHtml += ` <span style="color: #ff6b6b;">❌ ${escapeHtml(cons.last_error)}</span>`;
|
||||
} else if (cons.last_result) {
|
||||
infoHtml += ` <span style="color: #6fdc6f;">✅</span>`;
|
||||
}
|
||||
infoHtml += `<br>`;
|
||||
} else {
|
||||
infoHtml += `🕐 Last run: never<br>`;
|
||||
}
|
||||
infoHtml += `📊 Runs: ${cons.successful_runs}/${cons.total_runs} successful`;
|
||||
if (cons.is_running) {
|
||||
infoHtml += ` <span style="color: #dcb06f;">⏳ (running now)</span>`;
|
||||
}
|
||||
consInfo.innerHTML = infoHtml;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error refreshing memory stats:', err);
|
||||
document.getElementById('cat-status-indicator').innerHTML = '<span style="color: #ff6b6b;">● Error checking status</span>';
|
||||
@@ -113,11 +137,15 @@ async function loadFacts() {
|
||||
data.facts.forEach((fact, i) => {
|
||||
const source = fact.metadata?.source || 'unknown';
|
||||
const when = fact.metadata?.when ? new Date(fact.metadata.when * 1000).toLocaleString() : 'unknown';
|
||||
const persona = fact.metadata?.persona || 'miku';
|
||||
const personaBadge = persona === 'evil_miku'
|
||||
? '<span style="background: #7a2a2a; color: #ff9999; font-size: 0.7rem; padding: 0.1rem 0.4rem; border-radius: 3px; margin-left: 0.3rem;">😈 Evil Miku</span>'
|
||||
: '<span style="background: #2a5a7a; color: #99ccff; font-size: 0.7rem; padding: 0.1rem 0.4rem; border-radius: 3px; margin-left: 0.3rem;">🎤 Miku</span>';
|
||||
const factDataJson = escapeJsonForAttribute(fact);
|
||||
html += `
|
||||
<div class="memory-item" style="background: #242424; padding: 0.6rem 0.8rem; margin-bottom: 0.4rem; border-radius: 4px; border-left: 3px solid #2a9955; display: flex; justify-content: space-between; align-items: flex-start;">
|
||||
<div style="flex: 1;">
|
||||
<div style="color: #ddd; font-size: 0.9rem;">${escapeHtml(fact.content)}</div>
|
||||
<div style="color: #ddd; font-size: 0.9rem;">${escapeHtml(fact.content)}${personaBadge}</div>
|
||||
<div style="color: #666; font-size: 0.75rem; margin-top: 0.3rem;">
|
||||
Source: ${escapeHtml(source)} · ${when}
|
||||
</div>
|
||||
@@ -155,11 +183,15 @@ async function loadEpisodicMemories() {
|
||||
data.memories.forEach((mem, i) => {
|
||||
const source = mem.metadata?.source || 'unknown';
|
||||
const when = mem.metadata?.when ? new Date(mem.metadata.when * 1000).toLocaleString() : 'unknown';
|
||||
const persona = mem.metadata?.persona || 'miku';
|
||||
const personaBadge = persona === 'evil_miku'
|
||||
? '<span style="background: #7a2a2a; color: #ff9999; font-size: 0.7rem; padding: 0.1rem 0.4rem; border-radius: 3px; margin-left: 0.3rem;">😈 Evil Miku</span>'
|
||||
: '<span style="background: #2a5a7a; color: #99ccff; font-size: 0.7rem; padding: 0.1rem 0.4rem; border-radius: 3px; margin-left: 0.3rem;">🎤 Miku</span>';
|
||||
const memDataJson = escapeJsonForAttribute(mem);
|
||||
html += `
|
||||
<div class="memory-item" style="background: #242424; padding: 0.6rem 0.8rem; margin-bottom: 0.4rem; border-radius: 4px; border-left: 3px solid #2a5599; display: flex; justify-content: space-between; align-items: flex-start;">
|
||||
<div style="flex: 1;">
|
||||
<div style="color: #ddd; font-size: 0.9rem;">${escapeHtml(mem.content)}</div>
|
||||
<div style="color: #ddd; font-size: 0.9rem;">${escapeHtml(mem.content)}${personaBadge}</div>
|
||||
<div style="color: #666; font-size: 0.75rem; margin-top: 0.3rem;">
|
||||
Source: ${escapeHtml(source)} · ${when}
|
||||
</div>
|
||||
|
||||
BIN
bot/static/miku-favicon.png
Normal file
BIN
bot/static/miku-favicon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 358 KiB |
@@ -4,6 +4,7 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>🎛️ System Settings - Logging Configuration</title>
|
||||
<link rel="icon" href="/static/miku-favicon.png" type="image/png">
|
||||
<style>
|
||||
body {
|
||||
margin: 0;
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>System Settings - Miku Bot</title>
|
||||
<link rel="icon" href="/static/miku-favicon.png" type="image/png">
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
|
||||
105
bot/utils/consolidation_scheduler.py
Normal file
105
bot/utils/consolidation_scheduler.py
Normal file
@@ -0,0 +1,105 @@
|
||||
# utils/consolidation_scheduler.py
|
||||
"""
|
||||
Nightly memory consolidation scheduler.
|
||||
|
||||
Runs the Cheshire Cat memory consolidation pipeline on a fixed schedule
|
||||
(4:00 AM UTC by default) to mimic human REM sleep consolidation:
|
||||
|
||||
1. Query all unconsolidated episodic memories from Qdrant
|
||||
2. Classify and delete trivial messages
|
||||
3. Mark kept memories as consolidated
|
||||
4. Extract declarative facts per user via LLM
|
||||
5. Store facts in the declarative memory collection
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import globals
|
||||
from utils.logger import get_logger
|
||||
|
||||
logger = get_logger('consolidation')
|
||||
|
||||
# Tracks the last consolidation run time and result
|
||||
_last_consolidation = {
|
||||
'last_run': None, # ISO timestamp of last run
|
||||
'last_result': None, # Result string from Cat
|
||||
'last_error': None, # Error message if last run failed
|
||||
'is_running': False, # Whether consolidation is currently in progress
|
||||
'total_runs': 0, # Total number of consolidation runs
|
||||
'successful_runs': 0, # Number of successful runs
|
||||
}
|
||||
|
||||
|
||||
def get_consolidation_status() -> dict:
|
||||
"""Return the current consolidation status for the Web UI."""
|
||||
return {
|
||||
'last_run': _last_consolidation['last_run'],
|
||||
'last_result': _last_consolidation['last_result'],
|
||||
'last_error': _last_consolidation['last_error'],
|
||||
'is_running': _last_consolidation['is_running'],
|
||||
'total_runs': _last_consolidation['total_runs'],
|
||||
'successful_runs': _last_consolidation['successful_runs'],
|
||||
}
|
||||
|
||||
|
||||
async def run_nightly_consolidation():
|
||||
"""
|
||||
Run the nightly memory consolidation process.
|
||||
|
||||
This is the entry point called by APScheduler. It:
|
||||
1. Checks if Cheshire Cat is enabled and healthy
|
||||
2. Skips if consolidation is already running
|
||||
3. Sends a 'consolidate now' message via WebSocket to Cat
|
||||
4. Logs the result and updates tracking state
|
||||
"""
|
||||
# Prevent overlapping runs
|
||||
if _last_consolidation['is_running']:
|
||||
logger.warning("🌙 Consolidation already running, skipping scheduled run")
|
||||
return
|
||||
|
||||
if not globals.USE_CHESHIRE_CAT:
|
||||
logger.info("🌙 Skipping scheduled consolidation: Cheshire Cat is disabled")
|
||||
return
|
||||
|
||||
_last_consolidation['is_running'] = True
|
||||
_last_consolidation['last_run'] = datetime.now().isoformat()
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
from utils.cat_client import cat_adapter
|
||||
|
||||
# Check Cat health before attempting
|
||||
if not await cat_adapter.health_check():
|
||||
logger.warning("🌙 Skipping scheduled consolidation: Cat is not healthy")
|
||||
_last_consolidation['last_error'] = 'Cat health check failed'
|
||||
_last_consolidation['is_running'] = False
|
||||
_last_consolidation['total_runs'] += 1
|
||||
return
|
||||
|
||||
logger.info("🌙 Starting nightly memory consolidation (scheduled)...")
|
||||
|
||||
# Trigger consolidation via WebSocket
|
||||
# This runs synchronously within the hook and can take several minutes
|
||||
result = await cat_adapter.trigger_consolidation()
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
if result:
|
||||
logger.info(f"🌙 Nightly consolidation completed in {elapsed:.1f}s: {result[:200]}")
|
||||
_last_consolidation['last_result'] = result
|
||||
_last_consolidation['last_error'] = None
|
||||
_last_consolidation['successful_runs'] += 1
|
||||
else:
|
||||
logger.error(f"🌙 Nightly consolidation returned no result after {elapsed:.1f}s")
|
||||
_last_consolidation['last_error'] = 'No result returned (timeout or connection error)'
|
||||
|
||||
except Exception as e:
|
||||
elapsed = time.time() - start_time
|
||||
logger.error(f"🌙 Nightly consolidation failed after {elapsed:.1f}s: {e}")
|
||||
_last_consolidation['last_error'] = str(e)
|
||||
|
||||
finally:
|
||||
_last_consolidation['total_runs'] += 1
|
||||
_last_consolidation['is_running'] = False
|
||||
@@ -52,6 +52,7 @@ COMPONENTS = {
|
||||
'mood': 'Mood system and state changes',
|
||||
'dm': 'Direct message handling',
|
||||
'scheduled': 'Scheduled tasks and cron jobs',
|
||||
'consolidation': 'Memory consolidation scheduler',
|
||||
'gpu': 'GPU routing and model management',
|
||||
'media': 'Media processing (audio, video, images)',
|
||||
'server': 'Server management and configuration',
|
||||
|
||||
@@ -97,8 +97,12 @@ def before_cat_stores_episodic_memory(doc, cat):
|
||||
if author_name:
|
||||
doc.metadata['author_name'] = author_name
|
||||
|
||||
# Tag with persona so Evil Miku and Normal Miku know whose memory this is
|
||||
evil_mode = cat.working_memory.get('evil_mode', False)
|
||||
doc.metadata['persona'] = 'evil_miku' if evil_mode else 'miku'
|
||||
|
||||
print(f"💾 [Discord Bridge] Storing memory (unconsolidated): {message[:50]}...")
|
||||
print(f" User: {cat.user_id}, Guild: {guild_id}, Author: {author_name}")
|
||||
print(f" User: {cat.user_id}, Guild: {guild_id}, Author: {author_name}, Persona: {doc.metadata['persona']}")
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
@@ -77,19 +77,34 @@ def agent_prompt_prefix(prefix, cat):
|
||||
)
|
||||
|
||||
if results:
|
||||
high_confidence_facts = [
|
||||
item[0].page_content
|
||||
for item in results
|
||||
if item[1] > 0.5
|
||||
]
|
||||
# Build list of (fact_text, persona) tuples from results
|
||||
high_confidence_facts = []
|
||||
seen_personas = set()
|
||||
for item in results:
|
||||
if item[1] > 0.5:
|
||||
doc = item[0]
|
||||
fact_text = doc.page_content
|
||||
fact_persona = doc.metadata.get('persona', 'miku')
|
||||
high_confidence_facts.append((fact_text, fact_persona))
|
||||
seen_personas.add(fact_persona)
|
||||
|
||||
if high_confidence_facts:
|
||||
# Determine which persona is currently active
|
||||
current_evil = cat.working_memory.get('evil_mode', False)
|
||||
current_persona = 'evil_miku' if current_evil else 'miku'
|
||||
|
||||
# Build the facts section with persona annotations
|
||||
facts_text = "\n\n## Personal Facts About the User:\n"
|
||||
for fact in high_confidence_facts:
|
||||
facts_text += f"- {fact}\n"
|
||||
for fact, fact_persona in high_confidence_facts:
|
||||
# Annotate facts that came from the OTHER persona
|
||||
if fact_persona != current_persona:
|
||||
source_label = "Evil Miku" if fact_persona == 'evil_miku' else "Miku"
|
||||
facts_text += f"- {fact} (learned as {source_label})\n"
|
||||
else:
|
||||
facts_text += f"- {fact}\n"
|
||||
facts_text += "\n(Use these facts when answering the user's question)\n"
|
||||
prefix += facts_text
|
||||
print(f"[Declarative] Injected {len(high_confidence_facts)} facts into prompt")
|
||||
print(f"[Declarative] Injected {len(high_confidence_facts)} facts into prompt (personas: {seen_personas}, current: {current_persona})")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Declarative] Error: {e}")
|
||||
@@ -142,11 +157,16 @@ def before_cat_sends_message(message, cat):
|
||||
miku_response = str(message)
|
||||
|
||||
if miku_response and len(miku_response.strip()) > 3:
|
||||
# Determine which persona is active so the memory is tagged correctly
|
||||
evil_mode = cat.working_memory.get('evil_mode', False)
|
||||
persona = 'evil_miku' if evil_mode else 'miku'
|
||||
|
||||
metadata = {
|
||||
'source': cat.user_id,
|
||||
'when': datetime.now().timestamp(),
|
||||
'stored_at': datetime.now().isoformat(),
|
||||
'speaker': 'miku',
|
||||
'persona': persona,
|
||||
'consolidated': False,
|
||||
'guild_id': cat.working_memory.get('guild_id', 'dm'),
|
||||
'channel_id': cat.working_memory.get('channel_id'),
|
||||
@@ -160,7 +180,7 @@ def before_cat_sends_message(message, cat):
|
||||
vector=vector,
|
||||
metadata=metadata
|
||||
)
|
||||
print(f"[Miku Memory] Stored response: {miku_response[:50]}...")
|
||||
print(f"[Miku Memory] Stored response ({persona}): {miku_response[:50]}...")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Miku Memory] Error storing response: {e}")
|
||||
@@ -211,7 +231,9 @@ def trigger_consolidation_sync(cat):
|
||||
to_delete = []
|
||||
to_mark_consolidated = []
|
||||
# Group user messages by source (user_id) for per-user fact extraction
|
||||
# Also track which persona was active for each user's messages
|
||||
user_messages_by_source = {}
|
||||
user_persona_by_source = {} # source -> set of personas seen
|
||||
|
||||
for point in memories:
|
||||
content = point.payload.get('page_content', '').strip()
|
||||
@@ -235,7 +257,11 @@ def trigger_consolidation_sync(cat):
|
||||
source = metadata.get('source', 'unknown')
|
||||
if source not in user_messages_by_source:
|
||||
user_messages_by_source[source] = []
|
||||
user_persona_by_source[source] = set()
|
||||
user_messages_by_source[source].append(point.id)
|
||||
# Track which persona was active when this message was stored
|
||||
msg_persona = metadata.get('persona', 'miku')
|
||||
user_persona_by_source[source].add(msg_persona)
|
||||
|
||||
# Delete trivial memories
|
||||
if to_delete:
|
||||
@@ -255,8 +281,11 @@ def trigger_consolidation_sync(cat):
|
||||
# Extract facts per user
|
||||
total_facts = 0
|
||||
for source_user_id, memory_ids in user_messages_by_source.items():
|
||||
print(f"[Consolidation] Extracting facts for user '{source_user_id}' from {len(memory_ids)} messages...")
|
||||
facts = extract_and_store_facts(client, memory_ids, cat, source_user_id)
|
||||
# Determine the dominant persona for this user's messages
|
||||
personas = user_persona_by_source.get(source_user_id, {'miku'})
|
||||
dominant_persona = 'evil_miku' if 'evil_miku' in personas else 'miku'
|
||||
print(f"[Consolidation] Extracting facts for user '{source_user_id}' from {len(memory_ids)} messages (persona: {dominant_persona})...")
|
||||
facts = extract_and_store_facts(client, memory_ids, cat, source_user_id, dominant_persona)
|
||||
total_facts += facts
|
||||
print(f"[Consolidation] Extracted {facts} facts for user '{source_user_id}'")
|
||||
|
||||
@@ -275,10 +304,10 @@ def trigger_consolidation_sync(cat):
|
||||
# FACT EXTRACTION
|
||||
# ===================================================================
|
||||
|
||||
def extract_and_store_facts(client, memory_ids, cat, user_id):
|
||||
def extract_and_store_facts(client, memory_ids, cat, user_id, persona='miku'):
|
||||
"""
|
||||
Extract declarative facts from user memories using LLM and store them.
|
||||
Facts are scoped to the specific user_id.
|
||||
Facts are scoped to the specific user_id and tagged with the source persona.
|
||||
Uses Cat's embedder to ensure vector compatibility.
|
||||
Deduplicates against existing facts before storing.
|
||||
"""
|
||||
@@ -300,8 +329,16 @@ def extract_and_store_facts(client, memory_ids, cat, user_id):
|
||||
for mem in batch
|
||||
])
|
||||
|
||||
extraction_prompt = f"""Analyze these user messages and extract ONLY factual personal information.
|
||||
# Add persona context to the extraction prompt so the LLM knows
|
||||
# which version of Miku was active during these conversations
|
||||
persona_context = ""
|
||||
if persona == 'evil_miku':
|
||||
persona_context = "\nNOTE: These messages were exchanged with Evil Miku (the cruel, sadistic alter-ego).\n"
|
||||
else:
|
||||
persona_context = "\nNOTE: These messages were exchanged with Normal Miku (the cheerful virtual idol).\n"
|
||||
|
||||
extraction_prompt = f"""Analyze these user messages and extract ONLY factual personal information.
|
||||
{persona_context}
|
||||
User messages:
|
||||
{conversation_context}
|
||||
|
||||
@@ -395,13 +432,14 @@ IMPORTANT:
|
||||
'when': datetime.now().timestamp(),
|
||||
'fact_type': fact_type,
|
||||
'fact_value': fact_value,
|
||||
'persona': persona,
|
||||
}
|
||||
}
|
||||
}]
|
||||
)
|
||||
|
||||
facts_stored += 1
|
||||
print(f"[Fact Stored] [{user_id}] {fact_text}")
|
||||
print(f"[Fact Stored] [{user_id}] ({persona}) {fact_text}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[LLM Extract] Error: {e}")
|
||||
|
||||
Reference in New Issue
Block a user