From f498a3e3081ffe2a8a26134d8eb49d32cd9f0441 Mon Sep 17 00:00:00 2001 From: Slipstream Date: Sat, 3 May 2025 15:33:40 -0600 Subject: [PATCH] a --- api_service/api_server.py | 71 ++++++++---- api_service/code_verifier_store.py | 83 +++++++++++++- discord_oauth.py | 56 +++++++-- gurt_dashboard/index.html | 36 ------ gurt_dashboard/script.js | 176 ----------------------------- gurt_dashboard/style.css | 79 ------------- 6 files changed, 177 insertions(+), 324 deletions(-) delete mode 100644 gurt_dashboard/index.html delete mode 100644 gurt_dashboard/script.js delete mode 100644 gurt_dashboard/style.css diff --git a/api_service/api_server.py b/api_service/api_server.py index 5d6243f..c9cc6bf 100644 --- a/api_service/api_server.py +++ b/api_service/api_server.py @@ -279,7 +279,7 @@ async def auth(code: str, state: str = None, code_verifier: str = None, request: # Log the request details for debugging print(f"Received OAuth callback with code: {code[:10]}...") print(f"State: {state}") - print(f"Code verifier provided: {code_verifier is not None}") + print(f"Code verifier provided directly in URL: {code_verifier is not None}") print(f"Request URL: {request.url if request else 'No request object'}") print(f"Configured redirect URI: {DISCORD_REDIRECT_URI}") @@ -312,20 +312,35 @@ async def auth(code: str, state: str = None, code_verifier: str = None, request: "redirect_uri": actual_redirect_uri, } - # Add code_verifier if provided directly + # First try to get the code verifier from the store using the state parameter + # This is the most reliable method since the code verifier should have been stored + # by the Discord bot before the user was redirected here + stored_code_verifier = None + if state: + stored_code_verifier = code_verifier_store.get_code_verifier(state) + if stored_code_verifier: + print(f"Found code_verifier in store for state {state}: {stored_code_verifier[:10]}...") + else: + print(f"No code_verifier found in store for state {state}, will check other sources") + + # If we have a code_verifier parameter directly in the URL, use that if code_verifier: data["code_verifier"] = code_verifier - print(f"Using provided code_verifier parameter: {code_verifier[:10]}...") + print(f"Using code_verifier from URL parameter: {code_verifier[:10]}...") + # Otherwise use the stored code verifier if available + elif stored_code_verifier: + data["code_verifier"] = stored_code_verifier + print(f"Using code_verifier from store: {stored_code_verifier[:10]}...") + # Remove the code verifier from the store after using it + code_verifier_store.remove_code_verifier(state) else: - # Try to get the code verifier from the store using the state parameter - stored_code_verifier = code_verifier_store.get_code_verifier(state) if state else None - if stored_code_verifier: - data["code_verifier"] = stored_code_verifier - print(f"Using code_verifier from store for state {state}: {stored_code_verifier[:10]}...") - # Remove the code verifier from the store after using it - code_verifier_store.remove_code_verifier(state) - else: - print(f"Warning: No code_verifier found for state {state}") + # If we still don't have a code verifier, log a warning + print(f"WARNING: No code_verifier found for state {state} - OAuth will likely fail") + # Return a more helpful error message + return { + "message": "Authentication failed", + "error": "Missing code_verifier. This is required for PKCE OAuth flow. Please ensure the code_verifier is properly sent to the API server." + } # Log the token exchange request for debugging print(f"Exchanging code for token with data: {data}") @@ -1097,20 +1112,36 @@ async def store_code_verifier(request: Request): if not state or not code_verifier: raise HTTPException(status_code=400, detail="Missing state or code_verifier") + # Store the code verifier code_verifier_store.store_code_verifier(state, code_verifier) + + # Clean up expired code verifiers + code_verifier_store.cleanup_expired() + + # Log success + print(f"Successfully stored code verifier for state {state}") return {"success": True, "message": "Code verifier stored successfully"} except Exception as e: - raise HTTPException(status_code=400, detail=f"Error storing code verifier: {str(e)}") + error_msg = f"Error storing code verifier: {str(e)}" + print(error_msg) + raise HTTPException(status_code=400, detail=error_msg) @api_app.get("/code_verifier/{state}") @discordapi_app.get("/code_verifier/{state}") -async def get_code_verifier(state: str): - """Get the code verifier for a state""" - code_verifier = code_verifier_store.get_code_verifier(state) - if not code_verifier: - raise HTTPException(status_code=404, detail="No code verifier found for this state") - - return {"code_verifier": code_verifier} +async def check_code_verifier(state: str): + """Check if a code verifier exists for a state""" + try: + code_verifier = code_verifier_store.get_code_verifier(state) + if code_verifier: + # Don't return the actual code verifier for security reasons + # Just confirm it exists + return {"exists": True, "message": "Code verifier exists for this state"} + else: + return {"exists": False, "message": "No code verifier found for this state"} + except Exception as e: + error_msg = f"Error checking code verifier: {str(e)}" + print(error_msg) + raise HTTPException(status_code=400, detail=error_msg) # ============= Token Endpoints ============= diff --git a/api_service/code_verifier_store.py b/api_service/code_verifier_store.py index 7c57b71..f5ffab9 100644 --- a/api_service/code_verifier_store.py +++ b/api_service/code_verifier_store.py @@ -2,24 +2,99 @@ Code verifier store for the API service. This module provides a simple in-memory store for code verifiers used in the OAuth flow. +It also includes a file-based backup to ensure code verifiers persist across restarts. """ -from typing import Dict, Optional +import os +import json +import time +from typing import Dict, Optional, Any # In-memory storage for code verifiers -code_verifiers: Dict[str, str] = {} +code_verifiers: Dict[str, Dict[str, Any]] = {} + +# File path for persistent storage +STORAGE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") +STORAGE_FILE = os.path.join(STORAGE_DIR, "code_verifiers.json") + +# Ensure the storage directory exists +os.makedirs(STORAGE_DIR, exist_ok=True) + +def _load_from_file() -> None: + """Load code verifiers from file.""" + try: + if os.path.exists(STORAGE_FILE): + with open(STORAGE_FILE, 'r') as f: + stored_data = json.load(f) + + # Filter out expired entries (older than 10 minutes) + current_time = time.time() + for state, data in stored_data.items(): + if data.get("timestamp", 0) + 600 > current_time: # 10 minutes = 600 seconds + code_verifiers[state] = data + + print(f"Loaded {len(code_verifiers)} valid code verifiers from file") + except Exception as e: + print(f"Error loading code verifiers from file: {e}") + +def _save_to_file() -> None: + """Save code verifiers to file.""" + try: + with open(STORAGE_FILE, 'w') as f: + json.dump(code_verifiers, f) + print(f"Saved {len(code_verifiers)} code verifiers to file") + except Exception as e: + print(f"Error saving code verifiers to file: {e}") + +# Load existing code verifiers on module import +_load_from_file() def store_code_verifier(state: str, code_verifier: str) -> None: """Store a code verifier for a state.""" - code_verifiers[state] = code_verifier + # Store with timestamp for expiration + code_verifiers[state] = { + "code_verifier": code_verifier, + "timestamp": time.time() + } print(f"Stored code verifier for state {state}: {code_verifier[:10]}...") + # Save to file for persistence + _save_to_file() + def get_code_verifier(state: str) -> Optional[str]: """Get the code verifier for a state.""" - return code_verifiers.get(state) + # Check if state exists and is not expired + data = code_verifiers.get(state) + if data: + # Check if expired (older than 10 minutes) + if data.get("timestamp", 0) + 600 > time.time(): + return data.get("code_verifier") + else: + # Remove expired entry + remove_code_verifier(state) + print(f"Code verifier for state {state} has expired") + return None def remove_code_verifier(state: str) -> None: """Remove a code verifier for a state.""" if state in code_verifiers: del code_verifiers[state] print(f"Removed code verifier for state {state}") + # Update the file + _save_to_file() + +def cleanup_expired() -> None: + """Remove all expired code verifiers.""" + current_time = time.time() + expired_states = [] + + for state, data in code_verifiers.items(): + if data.get("timestamp", 0) + 600 <= current_time: # 10 minutes = 600 seconds + expired_states.append(state) + + for state in expired_states: + del code_verifiers[state] + + if expired_states: + print(f"Cleaned up {len(expired_states)} expired code verifiers") + _save_to_file() diff --git a/discord_oauth.py b/discord_oauth.py index 2eb88cf..965aec0 100644 --- a/discord_oauth.py +++ b/discord_oauth.py @@ -135,15 +135,36 @@ async def send_code_verifier_to_api(state: str, code_verifier: str) -> bool: # Send the code verifier to the API service print(f"Sending code verifier for state {state} to API service: {url}") - async with session.post(url, json=data) as resp: - if resp.status != 200: - error_text = await resp.text() - print(f"Failed to send code verifier to API service: {error_text}") - return False - response_data = await resp.json() - print(f"Successfully sent code verifier to API service: {response_data}") - return True + # Try multiple times with increasing delays to ensure delivery + max_retries = 3 + for retry in range(max_retries): + try: + async with session.post(url, json=data) as resp: + if resp.status == 200: + response_data = await resp.json() + print(f"Successfully sent code verifier to API service: {response_data}") + return True + else: + error_text = await resp.text() + print(f"Failed to send code verifier to API service (attempt {retry+1}/{max_retries}): {error_text}") + if retry < max_retries - 1: + # Wait before retrying, with exponential backoff + wait_time = 2 ** retry + print(f"Retrying in {wait_time} seconds...") + await asyncio.sleep(wait_time) + else: + return False + except aiohttp.ClientError as ce: + print(f"Connection error when sending code verifier (attempt {retry+1}/{max_retries}): {ce}") + if retry < max_retries - 1: + wait_time = 2 ** retry + print(f"Retrying in {wait_time} seconds...") + await asyncio.sleep(wait_time) + else: + raise + + return False except Exception as e: print(f"Error sending code verifier to API service: {e}") traceback.print_exc() @@ -190,8 +211,25 @@ def get_auth_url(state: str, code_verifier: str) -> str: print(f"Stored code verifier for state {state}: {code_verifier[:10]}...") # If API OAuth is enabled, send the code verifier to the API service + # This is critical for the PKCE flow to work with the API server if API_OAUTH_ENABLED: - asyncio.create_task(send_code_verifier_to_api(state, code_verifier)) + # Use a synchronous call to ensure the code verifier is sent before proceeding + # This is important because the user might click the auth URL immediately + loop = asyncio.get_event_loop() + send_success = False + try: + send_success = loop.run_until_complete(send_code_verifier_to_api(state, code_verifier)) + except Exception as e: + print(f"Error in synchronous code verifier send: {e}") + # Fall back to async task if synchronous call fails + asyncio.create_task(send_code_verifier_to_api(state, code_verifier)) + + if not send_success: + print("Warning: Failed to send code verifier synchronously, falling back to async task") + # Try again asynchronously as a backup + asyncio.create_task(send_code_verifier_to_api(state, code_verifier)) + else: + print(f"Successfully sent code verifier for state {state} to API service") return auth_url diff --git a/gurt_dashboard/index.html b/gurt_dashboard/index.html deleted file mode 100644 index 118237d..0000000 --- a/gurt_dashboard/index.html +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - Gurt Stats Dashboard - - - -

Gurt Internal Stats

-

Last Updated: Never

-
-
-

Runtime

- -
-
-

Memory

- -
-
-

API Stats

- -
-
-

Tool Stats

- -
-
-

Config Overview

- -
-
- - - diff --git a/gurt_dashboard/script.js b/gurt_dashboard/script.js deleted file mode 100644 index e08eaed..0000000 --- a/gurt_dashboard/script.js +++ /dev/null @@ -1,176 +0,0 @@ -const API_ENDPOINT = '/api/gurt/stats'; // Relative path to the API endpoint -const REFRESH_INTERVAL = 15000; // Refresh every 15 seconds (in milliseconds) - -const lastUpdatedElement = document.getElementById('last-updated'); -const runtimeStatsContainer = document.getElementById('runtime-stats'); -const memoryStatsContainer = document.getElementById('memory-stats'); -const apiStatsContainer = document.getElementById('api-stats'); -const toolStatsContainer = document.getElementById('tool-stats'); -const configStatsContainer = document.getElementById('config-stats'); - -function formatTimestamp(unixTimestamp) { - if (!unixTimestamp || unixTimestamp === 0) return 'N/A'; - const date = new Date(unixTimestamp * 1000); - return date.toLocaleString(); // Adjust format as needed -} - -function createStatItem(label, value, isCode = false) { - const item = document.createElement('div'); - item.classList.add('stat-item'); - - const labelSpan = document.createElement('span'); - labelSpan.classList.add('stat-label'); - labelSpan.textContent = label + ':'; - item.appendChild(labelSpan); - - const valueSpan = document.createElement('span'); - valueSpan.classList.add('stat-value'); - if (isCode) { - const code = document.createElement('code'); - code.textContent = value; - valueSpan.appendChild(code); - } else { - valueSpan.textContent = value; - } - item.appendChild(valueSpan); - return item; -} - -function createListStatItem(label, items) { - const item = document.createElement('div'); - item.classList.add('stat-item'); - - const labelSpan = document.createElement('span'); - labelSpan.classList.add('stat-label'); - labelSpan.textContent = label + ':'; - item.appendChild(labelSpan); - - if (items && items.length > 0) { - const list = document.createElement('ul'); - list.classList.add('stat-list'); - items.forEach(content => { - const li = document.createElement('li'); - li.textContent = content; - list.appendChild(li); - }); - item.appendChild(list); - } else { - const valueSpan = document.createElement('span'); - valueSpan.classList.add('stat-value'); - valueSpan.textContent = 'None'; - item.appendChild(valueSpan); - } - return item; -} - -function renderStats(stats) { - // Clear previous stats - runtimeStatsContainer.innerHTML = '

Runtime

'; - memoryStatsContainer.innerHTML = '

Memory

'; - apiStatsContainer.innerHTML = '

API Stats

'; - toolStatsContainer.innerHTML = '

Tool Stats

'; - configStatsContainer.innerHTML = '

Config Overview

'; - - // Runtime Stats - const runtime = stats.runtime || {}; - runtimeStatsContainer.appendChild(createStatItem('Current Mood', runtime.current_mood || 'N/A')); - runtimeStatsContainer.appendChild(createStatItem('Mood Changed', formatTimestamp(runtime.last_mood_change_timestamp))); - runtimeStatsContainer.appendChild(createStatItem('Background Task Running', runtime.background_task_running ? 'Yes' : 'No')); - runtimeStatsContainer.appendChild(createStatItem('Needs JSON Reminder', runtime.needs_json_reminder ? 'Yes' : 'No')); - runtimeStatsContainer.appendChild(createStatItem('Last Evolution', formatTimestamp(runtime.last_evolution_update_timestamp))); - runtimeStatsContainer.appendChild(createStatItem('Active Topics Channels', runtime.active_topics_channels || 0)); - runtimeStatsContainer.appendChild(createStatItem('Conv History Channels', runtime.conversation_history_channels || 0)); - runtimeStatsContainer.appendChild(createStatItem('Thread History Threads', runtime.thread_history_threads || 0)); - runtimeStatsContainer.appendChild(createStatItem('User Relationships Pairs', runtime.user_relationships_pairs || 0)); - runtimeStatsContainer.appendChild(createStatItem('Cached Summaries', runtime.conversation_summaries_cached || 0)); - runtimeStatsContainer.appendChild(createStatItem('Cached Channel Topics', runtime.channel_topics_cached || 0)); - runtimeStatsContainer.appendChild(createStatItem('Global Msg Cache', runtime.message_cache_global_count || 0)); - runtimeStatsContainer.appendChild(createStatItem('Mention Msg Cache', runtime.message_cache_mentioned_count || 0)); - runtimeStatsContainer.appendChild(createStatItem('Active Convos', runtime.active_conversations_count || 0)); - runtimeStatsContainer.appendChild(createStatItem('Sentiment Channels', runtime.conversation_sentiment_channels || 0)); - runtimeStatsContainer.appendChild(createStatItem('Gurt Participation Topics', runtime.gurt_participation_topics_count || 0)); - runtimeStatsContainer.appendChild(createStatItem('Tracked Reactions', runtime.gurt_message_reactions_tracked || 0)); - - // Memory Stats - const memory = stats.memory || {}; - if (memory.error) { - const errorItem = document.createElement('div'); - errorItem.classList.add('stat-item', 'error'); - errorItem.textContent = `Error: ${memory.error}`; - memoryStatsContainer.appendChild(errorItem); - } else { - memoryStatsContainer.appendChild(createStatItem('User Facts', memory.user_facts_count || 0)); - memoryStatsContainer.appendChild(createStatItem('General Facts', memory.general_facts_count || 0)); - memoryStatsContainer.appendChild(createStatItem('Chroma Messages', memory.chromadb_message_collection_count || 'N/A')); - memoryStatsContainer.appendChild(createStatItem('Chroma Facts', memory.chromadb_fact_collection_count || 'N/A')); - - const personality = memory.personality_traits || {}; - const pItems = Object.entries(personality).map(([k, v]) => `${k}: ${v}`); - memoryStatsContainer.appendChild(createListStatItem('Personality Traits', pItems)); - - const interests = memory.top_interests || []; - const iItems = interests.map(([t, l]) => `${t}: ${l.toFixed(2)}`); - memoryStatsContainer.appendChild(createListStatItem('Top Interests', iItems)); - } - - // API Stats - const apiStats = stats.api_stats || {}; - if (Object.keys(apiStats).length === 0) { - apiStatsContainer.appendChild(createStatItem('No API calls recorded yet.', '')); - } else { - for (const [model, data] of Object.entries(apiStats)) { - const value = `Success: ${data.success || 0}, Failure: ${data.failure || 0}, Retries: ${data.retries || 0}, Avg Time: ${data.average_time_ms || 0} ms, Count: ${data.count || 0}`; - apiStatsContainer.appendChild(createStatItem(model, value, true)); - } - } - - - // Tool Stats - const toolStats = stats.tool_stats || {}; - if (Object.keys(toolStats).length === 0) { - toolStatsContainer.appendChild(createStatItem('No tool calls recorded yet.', '')); - } else { - for (const [tool, data] of Object.entries(toolStats)) { - const value = `Success: ${data.success || 0}, Failure: ${data.failure || 0}, Avg Time: ${data.average_time_ms || 0} ms, Count: ${data.count || 0}`; - toolStatsContainer.appendChild(createStatItem(tool, value, true)); - } - } - - // Config Stats - const config = stats.config || {}; - configStatsContainer.appendChild(createStatItem('Default Model', config.default_model || 'N/A', true)); - configStatsContainer.appendChild(createStatItem('Fallback Model', config.fallback_model || 'N/A', true)); - configStatsContainer.appendChild(createStatItem('Semantic Model', config.semantic_model_name || 'N/A', true)); - configStatsContainer.appendChild(createStatItem('Max User Facts', config.max_user_facts || 'N/A')); - configStatsContainer.appendChild(createStatItem('Max General Facts', config.max_general_facts || 'N/A')); - configStatsContainer.appendChild(createStatItem('Context Window', config.context_window_size || 'N/A')); - configStatsContainer.appendChild(createStatItem('API Key Set', config.api_key_set ? 'Yes' : 'No')); - configStatsContainer.appendChild(createStatItem('Tavily Key Set', config.tavily_api_key_set ? 'Yes' : 'No')); - configStatsContainer.appendChild(createStatItem('Piston URL Set', config.piston_api_url_set ? 'Yes' : 'No')); - -} - -async function fetchStats() { - try { - const response = await fetch(API_ENDPOINT); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - const stats = await response.json(); - renderStats(stats); - lastUpdatedElement.textContent = new Date().toLocaleTimeString(); - } catch (error) { - console.error('Error fetching stats:', error); - lastUpdatedElement.textContent = `Error fetching stats at ${new Date().toLocaleTimeString()}`; - // Optionally display an error message in the UI - runtimeStatsContainer.innerHTML = '

Runtime

Could not load stats.

'; - memoryStatsContainer.innerHTML = '

Memory

'; - apiStatsContainer.innerHTML = '

API Stats

'; - toolStatsContainer.innerHTML = '

Tool Stats

'; - configStatsContainer.innerHTML = '

Config Overview

'; - } -} - -// Initial fetch and set interval -fetchStats(); -setInterval(fetchStats, REFRESH_INTERVAL); diff --git a/gurt_dashboard/style.css b/gurt_dashboard/style.css deleted file mode 100644 index f9c39f0..0000000 --- a/gurt_dashboard/style.css +++ /dev/null @@ -1,79 +0,0 @@ -body { - font-family: sans-serif; - line-height: 1.6; - margin: 20px; - background-color: #f4f4f4; - color: #333; -} - -h1, h2 { - color: #333; - border-bottom: 1px solid #ccc; - padding-bottom: 5px; -} - -#stats-container { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); - gap: 20px; -} - -.stats-section { - background-color: #fff; - padding: 15px; - border-radius: 5px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); -} - -.stats-section h2 { - margin-top: 0; - font-size: 1.2em; -} - -.stat-item { - margin-bottom: 10px; - padding-bottom: 10px; - border-bottom: 1px dotted #eee; -} - -.stat-item:last-child { - border-bottom: none; - margin-bottom: 0; - padding-bottom: 0; -} - -.stat-label { - font-weight: bold; - display: block; - margin-bottom: 3px; -} - -.stat-value { - font-family: monospace; - word-wrap: break-word; -} - -.stat-value code { - background-color: #eee; - padding: 2px 4px; - border-radius: 3px; -} - -.stat-list { - list-style: none; - padding-left: 0; -} - -.stat-list li { - margin-bottom: 5px; -} - -.error { - color: red; - font-weight: bold; -} - -#last-updated { - font-style: italic; - color: #555; -}