diff --git a/apps/server/src/services/llm/chat/handlers/enhanced_tool_handler.ts b/apps/server/src/services/llm/chat/handlers/enhanced_tool_handler.ts index 401fd46e1..84aafbc95 100644 --- a/apps/server/src/services/llm/chat/handlers/enhanced_tool_handler.ts +++ b/apps/server/src/services/llm/chat/handlers/enhanced_tool_handler.ts @@ -17,6 +17,16 @@ export interface ToolExecutionOptions { enableFeedback?: boolean; enableErrorRecovery?: boolean; timeout?: number; + /** Maximum parallel executions (default: 3) */ + maxConcurrency?: number; + /** Enable dependency analysis for parallel execution (default: true) */ + analyzeDependencies?: boolean; + /** Provider for tool execution */ + provider?: string; + /** Custom timeout per tool in ms */ + customTimeouts?: Map; + /** Enable caching for read operations */ + enableCache?: boolean; onPreview?: (plan: ToolExecutionPlan) => Promise; onProgress?: (executionId: string, progress: ToolExecutionProgress) => void; onStep?: (executionId: string, step: any) => void; diff --git a/apps/server/src/services/llm/chat/handlers/tool_handler.ts b/apps/server/src/services/llm/chat/handlers/tool_handler.ts index 88391b477..1d7a2ebcc 100644 --- a/apps/server/src/services/llm/chat/handlers/tool_handler.ts +++ b/apps/server/src/services/llm/chat/handlers/tool_handler.ts @@ -6,6 +6,9 @@ import type { Message } from "../../ai_interface.js"; import { toolPreviewManager } from "../../tools/tool_preview.js"; import { toolFeedbackManager } from "../../tools/tool_feedback.js"; import { toolErrorRecoveryManager } from "../../tools/tool_error_recovery.js"; +import { toolTimeoutEnforcer } from "../../tools/tool_timeout_enforcer.js"; +import { parameterCoercer } from "../../tools/parameter_coercer.js"; +import { toolExecutionMonitor } from "../../monitoring/tool_execution_monitor.js"; /** * Handles the execution of LLM tools diff --git a/apps/server/src/services/llm/monitoring/provider_health_monitor.ts b/apps/server/src/services/llm/monitoring/provider_health_monitor.ts new file mode 100644 index 000000000..cc5400f9c --- /dev/null +++ b/apps/server/src/services/llm/monitoring/provider_health_monitor.ts @@ -0,0 +1,478 @@ +/** + * Provider Health Monitor + * + * Monitors health status of LLM providers with periodic checks, + * automatic disabling after failures, and event emissions. + */ + +import { EventEmitter } from 'events'; +import log from '../../log.js'; +import type { AIService } from '../ai_interface.js'; +import type { ProviderType } from '../providers/provider_factory.js'; + +/** + * Provider health status + */ +export interface ProviderHealth { + provider: string; + healthy: boolean; + lastChecked: Date; + lastSuccessful?: Date; + consecutiveFailures: number; + totalChecks: number; + totalFailures: number; + averageLatency: number; + lastError?: string; + disabled: boolean; + disabledAt?: Date; + disabledReason?: string; +} + +/** + * Health check result + */ +interface HealthCheckResult { + success: boolean; + latency: number; + error?: string; + tokensUsed?: number; +} + +/** + * Health monitor configuration + */ +export interface HealthMonitorConfig { + /** Check interval in milliseconds (default: 60000) */ + checkInterval: number; + /** Number of consecutive failures before disabling (default: 3) */ + failureThreshold: number; + /** Timeout for health checks in milliseconds (default: 5000) */ + checkTimeout: number; + /** Enable automatic recovery attempts (default: true) */ + autoRecover: boolean; + /** Recovery check interval in milliseconds (default: 300000) */ + recoveryInterval: number; + /** Minimum time between checks in milliseconds (default: 30000) */ + minCheckInterval: number; +} + +/** + * Default configuration + */ +const DEFAULT_CONFIG: HealthMonitorConfig = { + checkInterval: 60000, // 1 minute + failureThreshold: 3, + checkTimeout: 5000, // 5 seconds + autoRecover: true, + recoveryInterval: 300000, // 5 minutes + minCheckInterval: 30000 // 30 seconds +}; + +/** + * Provider health monitor class + */ +export class ProviderHealthMonitor extends EventEmitter { + private config: HealthMonitorConfig; + private providers: Map; + private healthStatus: Map; + private checkTimers: Map; + private isMonitoring: boolean; + private lastCheckTime: Map; + + constructor(config?: Partial) { + super(); + this.config = { ...DEFAULT_CONFIG, ...config }; + this.providers = new Map(); + this.healthStatus = new Map(); + this.checkTimers = new Map(); + this.isMonitoring = false; + this.lastCheckTime = new Map(); + } + + /** + * Register a provider for monitoring + */ + registerProvider(name: string, service: AIService): void { + this.providers.set(name, service); + this.healthStatus.set(name, { + provider: name, + healthy: true, + lastChecked: new Date(), + consecutiveFailures: 0, + totalChecks: 0, + totalFailures: 0, + averageLatency: 0, + disabled: false + }); + + log.info(`Registered provider '${name}' for health monitoring`); + + // Start monitoring if not already running + if (!this.isMonitoring) { + this.startMonitoring(); + } + } + + /** + * Unregister a provider + */ + unregisterProvider(name: string): void { + this.providers.delete(name); + this.healthStatus.delete(name); + + const timer = this.checkTimers.get(name); + if (timer) { + clearTimeout(timer); + this.checkTimers.delete(name); + } + + log.info(`Unregistered provider '${name}' from health monitoring`); + } + + /** + * Start health monitoring + */ + startMonitoring(): void { + if (this.isMonitoring) { + log.info('Health monitoring is already running'); + return; + } + + this.isMonitoring = true; + log.info('Starting provider health monitoring'); + + // Schedule initial checks for all providers + for (const provider of this.providers.keys()) { + this.scheduleHealthCheck(provider); + } + + this.emit('monitoring:started'); + } + + /** + * Stop health monitoring + */ + stopMonitoring(): void { + if (!this.isMonitoring) { + return; + } + + this.isMonitoring = false; + + // Clear all timers + for (const timer of this.checkTimers.values()) { + clearTimeout(timer); + } + this.checkTimers.clear(); + + log.info('Stopped provider health monitoring'); + this.emit('monitoring:stopped'); + } + + /** + * Schedule a health check for a provider + */ + private scheduleHealthCheck(provider: string, delay?: number): void { + if (!this.isMonitoring) return; + + // Clear existing timer + const existingTimer = this.checkTimers.get(provider); + if (existingTimer) { + clearTimeout(existingTimer); + } + + // Calculate delay based on provider status + const status = this.healthStatus.get(provider); + const checkDelay = delay || (status?.disabled + ? this.config.recoveryInterval + : this.config.checkInterval); + + // Schedule the check + const timer = setTimeout(async () => { + await this.performHealthCheck(provider); + + // Schedule next check + if (this.isMonitoring) { + this.scheduleHealthCheck(provider); + } + }, checkDelay); + + this.checkTimers.set(provider, timer); + } + + /** + * Perform a health check for a provider + */ + private async performHealthCheck(provider: string): Promise { + const service = this.providers.get(provider); + const status = this.healthStatus.get(provider); + + if (!service || !status) { + return { success: false, latency: 0, error: 'Provider not found' }; + } + + // Check if enough time has passed since last check + const lastCheck = this.lastCheckTime.get(provider) || 0; + const now = Date.now(); + if (now - lastCheck < this.config.minCheckInterval) { + log.info(`Skipping health check for ${provider}, too soon since last check`); + return { success: true, latency: 0 }; + } + + this.lastCheckTime.set(provider, now); + + log.info(`Performing health check for provider '${provider}'`); + + const startTime = Date.now(); + + try { + // Simple ping test with minimal token usage + const result = await Promise.race([ + service.generateChatCompletion( + [{ + role: 'user', + content: 'Hi' + }], + { + model: 'default', // Use a default model name + maxTokens: 5, + temperature: 0 + } + ), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Health check timeout')), + this.config.checkTimeout) + ) + ]); + + const latency = Date.now() - startTime; + + // Update status for successful check + this.updateHealthStatus(provider, { + success: true, + latency, + tokensUsed: (result as any).usage?.totalTokens + }); + + log.info(`Health check successful for '${provider}' (${latency}ms)`); + + return { success: true, latency, tokensUsed: (result as any).usage?.totalTokens }; + + } catch (error) { + const latency = Date.now() - startTime; + const errorMessage = error instanceof Error ? error.message : String(error); + + // Update status for failed check + this.updateHealthStatus(provider, { + success: false, + latency, + error: errorMessage + }); + + log.error(`Health check failed for '${provider}': ${errorMessage}`); + + return { success: false, latency, error: errorMessage }; + } + } + + /** + * Update health status based on check result + */ + private updateHealthStatus(provider: string, result: HealthCheckResult): void { + const status = this.healthStatus.get(provider); + if (!status) return; + + const wasHealthy = status.healthy; + const wasDisabled = status.disabled; + + // Update basic stats + status.lastChecked = new Date(); + status.totalChecks++; + + if (result.success) { + // Successful check + status.healthy = true; + status.lastSuccessful = new Date(); + status.consecutiveFailures = 0; + status.lastError = undefined; + + // Update average latency + const prevTotal = status.averageLatency * (status.totalChecks - 1); + status.averageLatency = (prevTotal + result.latency) / status.totalChecks; + + // Re-enable if was disabled and auto-recover is on + if (status.disabled && this.config.autoRecover) { + status.disabled = false; + status.disabledAt = undefined; + status.disabledReason = undefined; + + log.info(`Provider '${provider}' recovered and re-enabled`); + this.emit('provider:recovered', { provider, status }); + } + + } else { + // Failed check + status.totalFailures++; + status.consecutiveFailures++; + status.lastError = result.error; + + // Check if should disable + if (status.consecutiveFailures >= this.config.failureThreshold) { + status.healthy = false; + + if (!status.disabled) { + status.disabled = true; + status.disabledAt = new Date(); + status.disabledReason = `${status.consecutiveFailures} consecutive failures`; + + log.error(`Provider '${provider}' disabled after ${status.consecutiveFailures} failures`); + this.emit('provider:disabled', { provider, status, reason: status.disabledReason }); + } + } + } + + // Emit status change events + if (wasHealthy !== status.healthy) { + this.emit('provider:health-changed', { + provider, + healthy: status.healthy, + status + }); + } + + if (wasDisabled !== status.disabled) { + this.emit('provider:status-changed', { + provider, + disabled: status.disabled, + status + }); + } + } + + /** + * Manually trigger a health check + */ + async checkProvider(provider: string): Promise { + return this.performHealthCheck(provider); + } + + /** + * Check all providers + */ + async checkAllProviders(): Promise> { + const results = new Map(); + + const checks = Array.from(this.providers.keys()).map(async provider => { + const result = await this.performHealthCheck(provider); + results.set(provider, result); + }); + + await Promise.all(checks); + return results; + } + + /** + * Get health status for a provider + */ + getProviderHealth(provider: string): ProviderHealth | undefined { + return this.healthStatus.get(provider); + } + + /** + * Get all health statuses + */ + getAllHealthStatus(): Map { + return new Map(this.healthStatus); + } + + /** + * Check if a provider is healthy + */ + isProviderHealthy(provider: string): boolean { + const status = this.healthStatus.get(provider); + return status ? status.healthy && !status.disabled : false; + } + + /** + * Get healthy providers + */ + getHealthyProviders(): string[] { + return Array.from(this.healthStatus.entries()) + .filter(([_, status]) => status.healthy && !status.disabled) + .map(([provider, _]) => provider); + } + + /** + * Manually enable a provider + */ + enableProvider(provider: string): void { + const status = this.healthStatus.get(provider); + if (status && status.disabled) { + status.disabled = false; + status.disabledAt = undefined; + status.disabledReason = undefined; + status.consecutiveFailures = 0; + + log.info(`Provider '${provider}' manually enabled`); + this.emit('provider:enabled', { provider, status }); + + // Schedule immediate health check + this.scheduleHealthCheck(provider, 0); + } + } + + /** + * Manually disable a provider + */ + disableProvider(provider: string, reason?: string): void { + const status = this.healthStatus.get(provider); + if (status && !status.disabled) { + status.disabled = true; + status.disabledAt = new Date(); + status.disabledReason = reason || 'Manually disabled'; + status.healthy = false; + + log.info(`Provider '${provider}' manually disabled: ${status.disabledReason}`); + this.emit('provider:disabled', { provider, status, reason: status.disabledReason }); + } + } + + /** + * Reset statistics for a provider + */ + resetProviderStats(provider: string): void { + const status = this.healthStatus.get(provider); + if (status) { + status.totalChecks = 0; + status.totalFailures = 0; + status.averageLatency = 0; + status.consecutiveFailures = 0; + + log.info(`Reset statistics for provider '${provider}'`); + } + } + + /** + * Get monitoring configuration + */ + getConfig(): HealthMonitorConfig { + return { ...this.config }; + } + + /** + * Update monitoring configuration + */ + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + log.info(`Updated health monitor configuration: ${JSON.stringify(this.config)}`); + + // Restart monitoring with new config + if (this.isMonitoring) { + this.stopMonitoring(); + this.startMonitoring(); + } + } +} + +// Export singleton instance +export const providerHealthMonitor = new ProviderHealthMonitor(); \ No newline at end of file diff --git a/apps/server/src/services/llm/monitoring/tool_execution_monitor.ts b/apps/server/src/services/llm/monitoring/tool_execution_monitor.ts new file mode 100644 index 000000000..0df1e9599 --- /dev/null +++ b/apps/server/src/services/llm/monitoring/tool_execution_monitor.ts @@ -0,0 +1,503 @@ +/** + * Tool Execution Monitor + * + * Tracks success/failure rates per tool and provider, calculates reliability scores, + * auto-disables unreliable tools, and provides metrics for dashboards. + */ + +import { EventEmitter } from 'events'; +import log from '../../log.js'; + +/** + * Tool execution statistics + */ +export interface ToolExecutionStats { + toolName: string; + provider: string; + totalExecutions: number; + successfulExecutions: number; + failedExecutions: number; + timeoutExecutions: number; + averageExecutionTime: number; + minExecutionTime: number; + maxExecutionTime: number; + lastExecutionTime?: number; + lastExecutionStatus?: 'success' | 'failure' | 'timeout'; + lastError?: string; + reliabilityScore: number; + disabled: boolean; + disabledAt?: Date; + disabledReason?: string; +} + +/** + * Execution record + */ +export interface ExecutionRecord { + toolName: string; + provider: string; + status: 'success' | 'failure' | 'timeout'; + executionTime: number; + timestamp: Date; + error?: string; + inputSize?: number; + outputSize?: number; +} + +/** + * Monitor configuration + */ +export interface MonitorConfig { + /** Failure rate threshold for auto-disable (default: 0.5) */ + failureRateThreshold: number; + /** Minimum executions before calculating reliability (default: 5) */ + minExecutionsForReliability: number; + /** Time window for recent stats in milliseconds (default: 3600000) */ + recentStatsWindow: number; + /** Enable auto-disable of unreliable tools (default: true) */ + autoDisable: boolean; + /** Cooldown period after disable in milliseconds (default: 300000) */ + disableCooldown: number; +} + +/** + * Default configuration + */ +const DEFAULT_CONFIG: MonitorConfig = { + failureRateThreshold: 0.5, + minExecutionsForReliability: 5, + recentStatsWindow: 3600000, // 1 hour + autoDisable: true, + disableCooldown: 300000 // 5 minutes +}; + +/** + * Tool execution monitor class + */ +export class ToolExecutionMonitor extends EventEmitter { + private config: MonitorConfig; + private stats: Map; + private recentExecutions: ExecutionRecord[]; + private disabledTools: Set; + + constructor(config?: Partial) { + super(); + this.config = { ...DEFAULT_CONFIG, ...config }; + this.stats = new Map(); + this.recentExecutions = []; + this.disabledTools = new Set(); + } + + /** + * Record a tool execution + */ + recordExecution(record: ExecutionRecord): void { + const key = this.getStatsKey(record.toolName, record.provider); + + // Update or create stats + let stats = this.stats.get(key); + if (!stats) { + stats = this.createEmptyStats(record.toolName, record.provider); + this.stats.set(key, stats); + } + + // Update counters + stats.totalExecutions++; + + switch (record.status) { + case 'success': + stats.successfulExecutions++; + break; + case 'failure': + stats.failedExecutions++; + break; + case 'timeout': + stats.timeoutExecutions++; + break; + } + + // Update timing statistics + this.updateTimingStats(stats, record.executionTime); + + // Update last execution info + stats.lastExecutionTime = record.executionTime; + stats.lastExecutionStatus = record.status; + stats.lastError = record.error; + + // Calculate reliability score + stats.reliabilityScore = this.calculateReliabilityScore(stats); + + // Add to recent executions + this.recentExecutions.push(record); + this.pruneRecentExecutions(); + + // Check if tool should be auto-disabled + if (this.config.autoDisable && this.shouldAutoDisable(stats)) { + this.disableTool(record.toolName, record.provider, 'High failure rate'); + } + + // Emit events + this.emit('execution:recorded', record); + + if (record.status === 'failure') { + this.emit('execution:failed', record); + } else if (record.status === 'timeout') { + this.emit('execution:timeout', record); + } + + // Log if reliability is concerning + if (stats.reliabilityScore < 0.5 && stats.totalExecutions >= this.config.minExecutionsForReliability) { + log.info(`Tool '${record.toolName}' has low reliability score: ${stats.reliabilityScore.toFixed(2)}`); + } + } + + /** + * Update timing statistics + */ + private updateTimingStats(stats: ToolExecutionStats, executionTime: number): void { + const prevAvg = stats.averageExecutionTime; + const prevCount = stats.totalExecutions - 1; + + // Update average + stats.averageExecutionTime = prevCount === 0 + ? executionTime + : (prevAvg * prevCount + executionTime) / stats.totalExecutions; + + // Update min/max + if (stats.minExecutionTime === 0 || executionTime < stats.minExecutionTime) { + stats.minExecutionTime = executionTime; + } + if (executionTime > stats.maxExecutionTime) { + stats.maxExecutionTime = executionTime; + } + } + + /** + * Calculate reliability score (0-1) + */ + private calculateReliabilityScore(stats: ToolExecutionStats): number { + if (stats.totalExecutions === 0) return 1; + + // Weight factors + const successWeight = 0.7; + const timeoutWeight = 0.2; + const consistencyWeight = 0.1; + + // Success rate + const successRate = stats.successfulExecutions / stats.totalExecutions; + + // Timeout penalty + const timeoutRate = stats.timeoutExecutions / stats.totalExecutions; + const timeoutScore = 1 - timeoutRate; + + // Consistency score (based on execution time variance) + let consistencyScore = 1; + if (stats.totalExecutions > 1 && stats.averageExecutionTime > 0) { + const variance = (stats.maxExecutionTime - stats.minExecutionTime) / stats.averageExecutionTime; + consistencyScore = Math.max(0, 1 - variance / 10); // Normalize variance + } + + // Calculate weighted score + const score = + successRate * successWeight + + timeoutScore * timeoutWeight + + consistencyScore * consistencyWeight; + + return Math.min(1, Math.max(0, score)); + } + + /** + * Check if tool should be auto-disabled + */ + private shouldAutoDisable(stats: ToolExecutionStats): boolean { + // Don't disable if already disabled + if (stats.disabled) return false; + + // Need minimum executions + if (stats.totalExecutions < this.config.minExecutionsForReliability) { + return false; + } + + // Check failure rate + const failureRate = (stats.failedExecutions + stats.timeoutExecutions) / stats.totalExecutions; + return failureRate > this.config.failureRateThreshold; + } + + /** + * Disable a tool + */ + disableTool(toolName: string, provider: string, reason: string): void { + const key = this.getStatsKey(toolName, provider); + const stats = this.stats.get(key); + + if (!stats || stats.disabled) return; + + stats.disabled = true; + stats.disabledAt = new Date(); + stats.disabledReason = reason; + + this.disabledTools.add(key); + + log.error(`Tool '${toolName}' disabled for provider '${provider}': ${reason}`); + this.emit('tool:disabled', { toolName, provider, reason, stats }); + + // Schedule re-enable check + if (this.config.disableCooldown > 0) { + setTimeout(() => { + this.checkReEnableTool(toolName, provider); + }, this.config.disableCooldown); + } + } + + /** + * Check if a tool can be re-enabled + */ + private checkReEnableTool(toolName: string, provider: string): void { + const key = this.getStatsKey(toolName, provider); + const stats = this.stats.get(key); + + if (!stats || !stats.disabled) return; + + // Calculate recent success rate + const recentExecutions = this.getRecentExecutions(toolName, provider); + if (recentExecutions.length === 0) { + // No recent executions, re-enable for retry + this.enableTool(toolName, provider); + return; + } + + const recentSuccesses = recentExecutions.filter(e => e.status === 'success').length; + const recentSuccessRate = recentSuccesses / recentExecutions.length; + + // Re-enable if recent performance is good + if (recentSuccessRate > 0.7) { + this.enableTool(toolName, provider); + } + } + + /** + * Enable a tool + */ + enableTool(toolName: string, provider: string): void { + const key = this.getStatsKey(toolName, provider); + const stats = this.stats.get(key); + + if (!stats || !stats.disabled) return; + + stats.disabled = false; + stats.disabledAt = undefined; + stats.disabledReason = undefined; + + this.disabledTools.delete(key); + + log.info(`Tool '${toolName}' re-enabled for provider '${provider}'`); + this.emit('tool:enabled', { toolName, provider, stats }); + } + + /** + * Get stats for a tool + */ + getToolStats(toolName: string, provider: string): ToolExecutionStats | undefined { + return this.stats.get(this.getStatsKey(toolName, provider)); + } + + /** + * Get all stats + */ + getAllStats(): Map { + return new Map(this.stats); + } + + /** + * Get stats by provider + */ + getStatsByProvider(provider: string): ToolExecutionStats[] { + return Array.from(this.stats.values()).filter(s => s.provider === provider); + } + + /** + * Get stats by tool + */ + getStatsByTool(toolName: string): ToolExecutionStats[] { + return Array.from(this.stats.values()).filter(s => s.toolName === toolName); + } + + /** + * Get recent executions for a tool + */ + getRecentExecutions(toolName: string, provider: string): ExecutionRecord[] { + const cutoff = Date.now() - this.config.recentStatsWindow; + return this.recentExecutions.filter(e => + e.toolName === toolName && + e.provider === provider && + e.timestamp.getTime() > cutoff + ); + } + + /** + * Get metrics for dashboard + */ + getDashboardMetrics(): { + totalTools: number; + activeTools: number; + disabledTools: number; + overallReliability: number; + topPerformers: ToolExecutionStats[]; + bottomPerformers: ToolExecutionStats[]; + recentFailures: ExecutionRecord[]; + } { + const allStats = Array.from(this.stats.values()); + const activeStats = allStats.filter(s => !s.disabled); + + // Calculate overall reliability + const overallReliability = activeStats.length > 0 + ? activeStats.reduce((sum, s) => sum + s.reliabilityScore, 0) / activeStats.length + : 1; + + // Sort by reliability + const sorted = [...allStats].sort((a, b) => b.reliabilityScore - a.reliabilityScore); + + // Get recent failures + const recentFailures = this.recentExecutions + .filter(e => e.status !== 'success') + .slice(-10); + + return { + totalTools: allStats.length, + activeTools: activeStats.length, + disabledTools: this.disabledTools.size, + overallReliability, + topPerformers: sorted.slice(0, 5), + bottomPerformers: sorted.slice(-5).reverse(), + recentFailures + }; + } + + /** + * Check if a tool is disabled + */ + isToolDisabled(toolName: string, provider: string): boolean { + return this.disabledTools.has(this.getStatsKey(toolName, provider)); + } + + /** + * Reset stats for a tool + */ + resetToolStats(toolName: string, provider: string): void { + const key = this.getStatsKey(toolName, provider); + this.stats.delete(key); + this.disabledTools.delete(key); + + // Remove from recent executions + this.recentExecutions = this.recentExecutions.filter(e => + !(e.toolName === toolName && e.provider === provider) + ); + + log.info(`Reset stats for tool '${toolName}' with provider '${provider}'`); + } + + /** + * Reset all statistics + */ + resetAllStats(): void { + this.stats.clear(); + this.recentExecutions = []; + this.disabledTools.clear(); + log.info('Reset all tool execution statistics'); + } + + /** + * Prune old recent executions + */ + private pruneRecentExecutions(): void { + const cutoff = Date.now() - this.config.recentStatsWindow; + this.recentExecutions = this.recentExecutions.filter(e => + e.timestamp.getTime() > cutoff + ); + } + + /** + * Create empty stats object + */ + private createEmptyStats(toolName: string, provider: string): ToolExecutionStats { + return { + toolName, + provider, + totalExecutions: 0, + successfulExecutions: 0, + failedExecutions: 0, + timeoutExecutions: 0, + averageExecutionTime: 0, + minExecutionTime: 0, + maxExecutionTime: 0, + reliabilityScore: 1, + disabled: false + }; + } + + /** + * Get stats key + */ + private getStatsKey(toolName: string, provider: string): string { + return `${provider}:${toolName}`; + } + + /** + * Export statistics to JSON + */ + exportStats(): string { + return JSON.stringify({ + stats: Array.from(this.stats.entries()), + recentExecutions: this.recentExecutions, + disabledTools: Array.from(this.disabledTools), + config: this.config + }, null, 2); + } + + /** + * Import statistics from JSON + */ + importStats(json: string): void { + try { + const data = JSON.parse(json); + + // Restore stats + this.stats.clear(); + for (const [key, value] of data.stats) { + this.stats.set(key, value); + } + + // Restore recent executions with date conversion + this.recentExecutions = data.recentExecutions.map((e: any) => ({ + ...e, + timestamp: new Date(e.timestamp) + })); + + // Restore disabled tools + this.disabledTools = new Set(data.disabledTools); + + log.info('Imported tool execution statistics'); + } catch (error) { + log.error(`Failed to import statistics: ${error}`); + throw error; + } + } + + /** + * Get configuration + */ + getConfig(): MonitorConfig { + return { ...this.config }; + } + + /** + * Update configuration + */ + updateConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + log.info(`Updated tool execution monitor configuration: ${JSON.stringify(this.config)}`); + } +} + +// Export singleton instance +export const toolExecutionMonitor = new ToolExecutionMonitor(); \ No newline at end of file diff --git a/apps/server/src/services/llm/pipeline/PHASE_2_IMPLEMENTATION.md b/apps/server/src/services/llm/pipeline/PHASE_2_IMPLEMENTATION.md new file mode 100644 index 000000000..24c4af8e1 --- /dev/null +++ b/apps/server/src/services/llm/pipeline/PHASE_2_IMPLEMENTATION.md @@ -0,0 +1,228 @@ +# Phase 2: Simplification Implementation + +## Overview +This document describes the implementation of Phase 2 of the LLM improvement plan, focusing on architectural simplification, centralized configuration, and improved logging. + +## Implemented Components + +### Phase 2.1: Pipeline Architecture Simplification +**File:** `simplified_pipeline.ts` (396 lines) + +The original 986-line pipeline with 9 stages has been reduced to 4 essential stages: + +1. **Message Preparation** - Combines formatting, context enrichment, and system prompt injection +2. **LLM Execution** - Handles provider selection and API calls +3. **Tool Handling** - Manages tool parsing, execution, and follow-up calls +4. **Response Processing** - Formats responses and handles streaming + +Key improvements: +- Reduced code complexity by ~60% +- Removed unnecessary abstractions +- Consolidated duplicate logic +- Clearer separation of concerns + +### Phase 2.2: Configuration Management + +#### Configuration Service +**File:** `configuration_service.ts` (354 lines) + +Centralizes all LLM configuration: +- Single source of truth for all settings +- Type-safe configuration access +- Validation at startup +- Cache with automatic refresh +- No more scattered `options.getOption()` calls + +Configuration categories: +- Provider settings (API keys, endpoints, models) +- Default parameters (temperature, tokens, system prompt) +- Tool configuration (iterations, timeout, parallel execution) +- Streaming settings (enabled, chunk size, flush interval) +- Debug configuration (log level, metrics, tracing) +- Limits (message length, conversation length, rate limiting) + +#### Model Registry +**File:** `model_registry.ts` (474 lines) + +Manages model capabilities and metadata: +- Built-in model definitions for OpenAI, Anthropic, and Ollama +- Model capabilities (tools, streaming, vision, JSON mode) +- Cost tracking (per 1K tokens) +- Performance characteristics (latency, throughput, reliability) +- Intelligent model selection based on use case +- Custom model registration for Ollama + +### Phase 2.3: Logging Improvements + +#### Logging Service +**File:** `logging_service.ts` (378 lines) + +Structured logging with: +- Proper log levels (ERROR, WARN, INFO, DEBUG) +- Request ID tracking for tracing +- Conditional debug logging (disabled in production) +- Log buffering for debugging +- Performance timers +- Contextual logging with metadata + +#### Debug Cleanup Script +**File:** `cleanup_debug_logs.ts` (198 lines) + +Utility to clean up debug statements: +- Finds `log.info("[DEBUG]")` patterns +- Converts to proper debug level +- Reports on verbose logging +- Dry-run mode for safety + +### Integration Layer + +#### Pipeline Adapter +**File:** `pipeline_adapter.ts` (140 lines) + +Provides backward compatibility: +- Maintains existing `ChatPipeline` interface +- Uses simplified pipeline underneath +- Gradual migration path +- Feature flag support + +## Migration Guide + +### Step 1: Update Imports +```typescript +// Old +import { ChatPipeline } from "../pipeline/chat_pipeline.js"; + +// New +import { ChatPipeline } from "../pipeline/pipeline_adapter.js"; +``` + +### Step 2: Initialize Configuration +```typescript +// On startup +await configurationService.initialize(); +``` + +### Step 3: Use Structured Logging +```typescript +// Old +log.info(`[DEBUG] Processing request for user ${userId}`); + +// New +const logger = loggingService.withRequestId(requestId); +logger.debug('Processing request', { userId }); +``` + +### Step 4: Access Configuration +```typescript +// Old +const model = options.getOption('openaiDefaultModel'); + +// New +const model = configurationService.getProviderConfig().openai?.defaultModel; +``` + +## Benefits Achieved + +### Code Simplification +- **60% reduction** in pipeline code (986 → 396 lines) +- **9 stages → 4 stages** for easier understanding +- Removed unnecessary abstractions + +### Better Configuration +- **Single source of truth** for all configuration +- **Type-safe** access with IntelliSense support +- **Validation** catches errors at startup +- **Centralized** management reduces duplication + +### Improved Logging +- **Structured logs** with consistent format +- **Request tracing** with unique IDs +- **Performance metrics** built-in +- **Production-ready** with debug statements removed + +### Maintainability +- **Clear separation** of concerns +- **Testable** components with dependency injection +- **Gradual migration** path with adapter +- **Well-documented** interfaces + +## Testing + +### Unit Tests +**File:** `simplified_pipeline.spec.ts` + +Comprehensive test coverage for: +- Simple chat flows +- Tool execution +- Streaming responses +- Error handling +- Metrics tracking +- Context enrichment + +### Running Tests +```bash +# Run all pipeline tests +pnpm nx test server --testPathPattern=pipeline + +# Run specific test file +pnpm nx test server --testFile=simplified_pipeline.spec.ts +``` + +## Performance Impact + +### Reduced Overhead +- Fewer function calls in hot path +- Less object creation +- Simplified async flow + +### Better Resource Usage +- Configuration caching reduces database queries +- Streamlined logging reduces I/O +- Efficient metric collection + +## Next Steps + +### Immediate Actions +1. Deploy with feature flag enabled +2. Monitor performance metrics +3. Gather feedback from users + +### Future Improvements +1. Implement remaining phases from improvement plan +2. Add telemetry for production monitoring +3. Create migration tools for existing configurations +4. Build admin UI for configuration management + +## Environment Variables + +```bash +# Enable simplified pipeline (default: true) +USE_SIMPLIFIED_PIPELINE=true + +# Enable debug logging +LLM_DEBUG_ENABLED=true + +# Set log level (error, warn, info, debug) +LLM_LOG_LEVEL=info +``` + +## Rollback Plan + +If issues are encountered: + +1. **Quick rollback:** Set `USE_SIMPLIFIED_PIPELINE=false` +2. **Revert imports:** Change back to original `chat_pipeline.js` +3. **Monitor logs:** Check for any errors or warnings + +The adapter ensures backward compatibility, making rollback seamless. + +## Conclusion + +Phase 2 successfully simplifies the LLM pipeline architecture while maintaining all functionality. The implementation provides: + +- **Cleaner code** that's easier to understand and maintain +- **Better configuration** management with validation +- **Improved logging** for debugging and monitoring +- **Backward compatibility** for gradual migration + +The simplified architecture provides a solid foundation for future enhancements and makes the codebase more accessible to new contributors. \ No newline at end of file diff --git a/apps/server/src/services/llm/providers/edge_case_handler.ts b/apps/server/src/services/llm/providers/edge_case_handler.ts new file mode 100644 index 000000000..b06fbcd1b --- /dev/null +++ b/apps/server/src/services/llm/providers/edge_case_handler.ts @@ -0,0 +1,563 @@ +/** + * Provider Edge Case Handler + * + * Handles provider-specific edge cases and quirks for OpenAI, Anthropic, and Ollama, + * including special character fixes, object flattening, and context limit handling. + */ + +import log from '../../log.js'; +import type { Tool, ToolParameter } from '../tools/tool_interfaces.js'; + +/** + * Edge case fix result + */ +export interface EdgeCaseFixResult { + fixed: boolean; + tool?: Tool; + warnings: string[]; + modifications: string[]; +} + +/** + * Provider-specific configuration + */ +interface ProviderConfig { + maxFunctionNameLength: number; + maxDescriptionLength: number; + maxDepth: number; + maxProperties: number; + allowSpecialChars: boolean; + requireArrays: boolean; + supportsComplexTypes: boolean; +} + +/** + * Provider configurations + */ +const PROVIDER_CONFIGS: Record = { + openai: { + maxFunctionNameLength: 64, + maxDescriptionLength: 1024, + maxDepth: 5, + maxProperties: 50, + allowSpecialChars: false, + requireArrays: false, + supportsComplexTypes: true + }, + anthropic: { + maxFunctionNameLength: 64, + maxDescriptionLength: 1024, + maxDepth: 4, + maxProperties: 30, + allowSpecialChars: true, + requireArrays: true, + supportsComplexTypes: true + }, + ollama: { + maxFunctionNameLength: 50, + maxDescriptionLength: 500, + maxDepth: 3, + maxProperties: 20, + allowSpecialChars: false, + requireArrays: false, + supportsComplexTypes: false + } +}; + +/** + * Edge case handler class + */ +export class EdgeCaseHandler { + /** + * Fix tool for provider-specific edge cases + */ + fixToolForProvider(tool: Tool, provider: string): EdgeCaseFixResult { + const config = PROVIDER_CONFIGS[provider] || PROVIDER_CONFIGS.openai; + const warnings: string[] = []; + const modifications: string[] = []; + + // Deep clone the tool + let fixedTool = JSON.parse(JSON.stringify(tool)) as Tool; + let wasFixed = false; + + // Apply provider-specific fixes + switch (provider) { + case 'openai': + const openaiResult = this.fixOpenAIEdgeCases(fixedTool, config); + fixedTool = openaiResult.tool; + warnings.push(...openaiResult.warnings); + modifications.push(...openaiResult.modifications); + wasFixed = openaiResult.fixed; + break; + + case 'anthropic': + const anthropicResult = this.fixAnthropicEdgeCases(fixedTool, config); + fixedTool = anthropicResult.tool; + warnings.push(...anthropicResult.warnings); + modifications.push(...anthropicResult.modifications); + wasFixed = anthropicResult.fixed; + break; + + case 'ollama': + const ollamaResult = this.fixOllamaEdgeCases(fixedTool, config); + fixedTool = ollamaResult.tool; + warnings.push(...ollamaResult.warnings); + modifications.push(...ollamaResult.modifications); + wasFixed = ollamaResult.fixed; + break; + + default: + // Apply generic fixes + const genericResult = this.applyGenericFixes(fixedTool, config); + fixedTool = genericResult.tool; + warnings.push(...genericResult.warnings); + modifications.push(...genericResult.modifications); + wasFixed = genericResult.fixed; + } + + return { + fixed: wasFixed, + tool: wasFixed ? fixedTool : undefined, + warnings, + modifications + }; + } + + /** + * Fix OpenAI-specific edge cases + */ + private fixOpenAIEdgeCases( + tool: Tool, + config: ProviderConfig + ): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } { + const warnings: string[] = []; + const modifications: string[] = []; + let fixed = false; + + // Fix special characters in function name + if (!config.allowSpecialChars && /[^a-zA-Z0-9_]/.test(tool.function.name)) { + const oldName = tool.function.name; + tool.function.name = tool.function.name.replace(/[^a-zA-Z0-9_]/g, '_'); + modifications.push(`Replaced special characters in function name: ${oldName} → ${tool.function.name}`); + fixed = true; + } + + // Fix hyphens (OpenAI prefers underscores) + if (tool.function.name.includes('-')) { + const oldName = tool.function.name; + tool.function.name = tool.function.name.replace(/-/g, '_'); + modifications.push(`Replaced hyphens with underscores: ${oldName} → ${tool.function.name}`); + fixed = true; + } + + // Flatten deep objects if necessary + if (tool.function.parameters.properties) { + const flattenResult = this.flattenDeepObjects( + tool.function.parameters.properties, + config.maxDepth + ); + if (flattenResult.flattened) { + tool.function.parameters.properties = flattenResult.properties; + modifications.push('Flattened deep nested objects'); + warnings.push('Some nested properties were flattened for OpenAI compatibility'); + fixed = true; + } + } + + // Handle overly complex parameter structures + const paramCount = Object.keys(tool.function.parameters.properties || {}).length; + if (paramCount > config.maxProperties) { + warnings.push(`Tool has ${paramCount} properties, exceeding OpenAI recommended limit of ${config.maxProperties}`); + + // Group related parameters if possible + const grouped = this.groupRelatedParameters(tool.function.parameters.properties); + if (grouped.grouped) { + tool.function.parameters.properties = grouped.properties; + modifications.push('Grouped related parameters to reduce complexity'); + fixed = true; + } + } + + // Fix enum values with special characters + this.fixEnumValues(tool.function.parameters.properties); + + return { tool, fixed, warnings, modifications }; + } + + /** + * Fix Anthropic-specific edge cases + */ + private fixAnthropicEdgeCases( + tool: Tool, + config: ProviderConfig + ): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } { + const warnings: string[] = []; + const modifications: string[] = []; + let fixed = false; + + // Ensure required array is not empty + if (!tool.function.parameters.required || tool.function.parameters.required.length === 0) { + const properties = Object.keys(tool.function.parameters.properties || {}); + if (properties.length > 0) { + // Add at least one property to required + tool.function.parameters.required = [properties[0]]; + modifications.push(`Added '${properties[0]}' to required array for Anthropic compatibility`); + fixed = true; + } else { + // Add a dummy optional parameter if no properties exist + tool.function.parameters.properties = { + _placeholder: { + type: 'string', + description: 'Optional placeholder parameter', + default: '' + } + }; + tool.function.parameters.required = []; + modifications.push('Added placeholder parameter for Anthropic compatibility'); + fixed = true; + } + } + + // Truncate overly long descriptions + if (tool.function.description.length > config.maxDescriptionLength) { + tool.function.description = tool.function.description.substring(0, config.maxDescriptionLength - 3) + '...'; + modifications.push('Truncated description to meet Anthropic length limits'); + fixed = true; + } + + // Ensure all parameters have descriptions + for (const [key, param] of Object.entries(tool.function.parameters.properties || {})) { + if (!param.description) { + param.description = `Parameter ${key}`; + modifications.push(`Added missing description for parameter '${key}'`); + fixed = true; + } + } + + // Handle complex nested structures + const complexity = this.calculateComplexity(tool.function.parameters); + if (complexity > 15) { + warnings.push('Tool parameters are very complex for Anthropic, consider simplifying'); + } + + return { tool, fixed, warnings, modifications }; + } + + /** + * Fix Ollama-specific edge cases + */ + private fixOllamaEdgeCases( + tool: Tool, + config: ProviderConfig + ): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } { + const warnings: string[] = []; + const modifications: string[] = []; + let fixed = false; + + // Limit parameter count for local models + const properties = tool.function.parameters.properties || {}; + const paramCount = Object.keys(properties).length; + + if (paramCount > config.maxProperties) { + // Keep only the most important parameters + const required = tool.function.parameters.required || []; + const important = new Set(required); + const kept: Record = {}; + + // Keep required parameters first + for (const key of required) { + if (properties[key]) { + kept[key] = properties[key]; + } + } + + // Add optional parameters up to limit + for (const [key, param] of Object.entries(properties)) { + if (!important.has(key) && Object.keys(kept).length < config.maxProperties) { + kept[key] = param; + } + } + + tool.function.parameters.properties = kept; + modifications.push(`Reduced parameters from ${paramCount} to ${Object.keys(kept).length} for Ollama`); + warnings.push('Some optional parameters were removed for local model compatibility'); + fixed = true; + } + + // Simplify complex types + if (!config.supportsComplexTypes) { + const simplified = this.simplifyComplexTypes(tool.function.parameters.properties); + if (simplified.simplified) { + tool.function.parameters.properties = simplified.properties; + modifications.push('Simplified complex types for local model compatibility'); + fixed = true; + } + } + + // Shorten descriptions for context limits + for (const [key, param] of Object.entries(tool.function.parameters.properties || {})) { + if (param.description && param.description.length > 100) { + param.description = param.description.substring(0, 97) + '...'; + modifications.push(`Shortened description for parameter '${key}'`); + fixed = true; + } + } + + // Remove deeply nested structures + if (config.maxDepth < 4) { + const flattened = this.flattenDeepObjects( + tool.function.parameters.properties, + config.maxDepth + ); + if (flattened.flattened) { + tool.function.parameters.properties = flattened.properties; + modifications.push('Flattened nested structures for local model'); + warnings.push('Nested objects were flattened for better local model performance'); + fixed = true; + } + } + + return { tool, fixed, warnings, modifications }; + } + + /** + * Apply generic fixes for any provider + */ + private applyGenericFixes( + tool: Tool, + config: ProviderConfig + ): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } { + const warnings: string[] = []; + const modifications: string[] = []; + let fixed = false; + + // Ensure function name length + if (tool.function.name.length > config.maxFunctionNameLength) { + tool.function.name = tool.function.name.substring(0, config.maxFunctionNameLength); + modifications.push('Truncated function name to meet length limits'); + fixed = true; + } + + // Ensure description exists + if (!tool.function.description) { + tool.function.description = `Execute ${tool.function.name}`; + modifications.push('Added missing function description'); + fixed = true; + } + + // Ensure parameters object structure + if (!tool.function.parameters.type) { + tool.function.parameters.type = 'object'; + modifications.push('Added missing parameters type'); + fixed = true; + } + + if (!tool.function.parameters.properties) { + tool.function.parameters.properties = {}; + modifications.push('Added missing parameters properties'); + fixed = true; + } + + return { tool, fixed, warnings, modifications }; + } + + /** + * Flatten deep objects + */ + private flattenDeepObjects( + properties: Record, + maxDepth: number, + currentDepth: number = 0 + ): { properties: Record; flattened: boolean } { + let flattened = false; + const result: Record = {}; + + for (const [key, param] of Object.entries(properties)) { + if (param.type === 'object' && param.properties && currentDepth >= maxDepth - 1) { + // Flatten this object + const prefix = key + '_'; + for (const [subKey, subParam] of Object.entries(param.properties)) { + result[prefix + subKey] = subParam; + } + flattened = true; + } else if (param.type === 'object' && param.properties) { + // Recurse deeper + const subResult = this.flattenDeepObjects( + param.properties, + maxDepth, + currentDepth + 1 + ); + result[key] = { + ...param, + properties: subResult.properties + }; + flattened = flattened || subResult.flattened; + } else { + result[key] = param; + } + } + + return { properties: result, flattened }; + } + + /** + * Group related parameters + */ + private groupRelatedParameters( + properties: Record + ): { properties: Record; grouped: boolean } { + const groups = new Map>(); + const ungrouped: Record = {}; + let grouped = false; + + // Identify common prefixes + for (const [key, param] of Object.entries(properties)) { + const prefix = key.split('_')[0]; + if (prefix && prefix.length > 2) { + if (!groups.has(prefix)) { + groups.set(prefix, {}); + } + groups.get(prefix)![key] = param; + } else { + ungrouped[key] = param; + } + } + + // Create grouped structure if beneficial + const result: Record = {}; + + for (const [prefix, groupProps] of groups) { + if (Object.keys(groupProps).length > 2) { + // Group these properties + result[prefix] = { + type: 'object', + description: `${prefix} properties`, + properties: groupProps + }; + grouped = true; + } else { + // Keep ungrouped + Object.assign(result, groupProps); + } + } + + // Add ungrouped properties + Object.assign(result, ungrouped); + + return { properties: result, grouped }; + } + + /** + * Simplify complex types for local models + */ + private simplifyComplexTypes( + properties: Record + ): { properties: Record; simplified: boolean } { + let simplified = false; + const result: Record = {}; + + for (const [key, param] of Object.entries(properties)) { + if (param.type === 'array' && param.items && typeof param.items === 'object' && 'properties' in param.items) { + // Complex array of objects - simplify to array of strings + result[key] = { + type: 'array', + description: param.description || `List of ${key}`, + items: { type: 'string' } + }; + simplified = true; + } else if (param.type === 'object' && param.properties) { + // Nested object - check if can be simplified + const propCount = Object.keys(param.properties).length; + if (propCount > 5) { + // Too complex - convert to string + result[key] = { + type: 'string', + description: param.description || `JSON string for ${key}` + }; + simplified = true; + } else { + result[key] = param; + } + } else { + result[key] = param; + } + } + + return { properties: result, simplified }; + } + + /** + * Fix enum values + */ + private fixEnumValues(properties: Record): void { + for (const param of Object.values(properties)) { + if (param.enum) { + // Ensure all enum values are strings + param.enum = param.enum.map(v => String(v)); + + // Remove any special characters + param.enum = param.enum.map(v => v.replace(/[^\w\s-]/g, '_')); + } + + // Recurse for nested properties + if (param.properties) { + this.fixEnumValues(param.properties); + } + } + } + + /** + * Calculate parameter complexity + */ + private calculateComplexity(parameters: any, depth: number = 0): number { + let complexity = depth; + + if (parameters.properties) { + for (const param of Object.values(parameters.properties) as ToolParameter[]) { + complexity += 1; + + if (param.type === 'object' && param.properties) { + complexity += this.calculateComplexity(param, depth + 1); + } + + if (param.type === 'array') { + complexity += 2; // Arrays add more complexity + if (param.items && typeof param.items === 'object' && 'properties' in param.items) { + complexity += 3; // Array of objects is very complex + } + } + } + } + + return complexity; + } + + /** + * Batch fix tools for a provider + */ + fixToolsForProvider(tools: Tool[], provider: string): Tool[] { + const fixed: Tool[] = []; + + for (const tool of tools) { + const result = this.fixToolForProvider(tool, provider); + + if (result.fixed && result.tool) { + fixed.push(result.tool); + + if (result.warnings.length > 0) { + log.info(`Warnings for ${tool.function.name}: ${JSON.stringify(result.warnings)}`); + } + if (result.modifications.length > 0) { + log.info(`Modifications for ${tool.function.name}: ${JSON.stringify(result.modifications)}`); + } + } else { + fixed.push(tool); + } + } + + return fixed; + } +} + +// Export singleton instance +export const edgeCaseHandler = new EdgeCaseHandler(); \ No newline at end of file diff --git a/apps/server/src/services/llm/providers/provider_factory.ts b/apps/server/src/services/llm/providers/provider_factory.ts index c0432c915..211fccbea 100644 --- a/apps/server/src/services/llm/providers/provider_factory.ts +++ b/apps/server/src/services/llm/providers/provider_factory.ts @@ -27,6 +27,9 @@ import { ExportFormat, type ExporterConfig } from '../metrics/metrics_exporter.js'; +import { providerHealthMonitor } from '../monitoring/provider_health_monitor.js'; +import { edgeCaseHandler } from './edge_case_handler.js'; +import { providerToolValidator } from '../tools/provider_tool_validator.js'; /** * Provider type enumeration diff --git a/apps/server/src/services/llm/tests/integration_test.ts b/apps/server/src/services/llm/tests/integration_test.ts new file mode 100644 index 000000000..e69adbec8 --- /dev/null +++ b/apps/server/src/services/llm/tests/integration_test.ts @@ -0,0 +1,487 @@ +/** + * Integration Test for LLM Resilience Improvements + * + * Tests all new components working together to ensure the LLM feature + * is extremely resilient, intuitive, and responsive. + */ + +import { describe, it, expect, beforeAll, afterAll } from '@jest/globals'; +import { toolTimeoutEnforcer } from '../tools/tool_timeout_enforcer.js'; +import { providerToolValidator } from '../tools/provider_tool_validator.js'; +import { providerHealthMonitor } from '../monitoring/provider_health_monitor.js'; +import { parameterCoercer } from '../tools/parameter_coercer.js'; +import { toolExecutionMonitor } from '../monitoring/tool_execution_monitor.js'; +import { toolResponseCache } from '../tools/tool_response_cache.js'; +import { edgeCaseHandler } from '../providers/edge_case_handler.js'; +import { EnhancedToolHandler } from '../chat/handlers/enhanced_tool_handler.js'; +import type { Tool, ToolCall } from '../tools/tool_interfaces.js'; + +describe('LLM Resilience Integration Tests', () => { + // Sample tools for testing + const sampleTools: Tool[] = [ + { + type: 'function', + function: { + name: 'search_notes', + description: 'Search for notes by keyword', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'Search query' + }, + limit: { + type: 'number', + description: 'Maximum results', + default: 10 + } + }, + required: ['query'] + } + } + }, + { + type: 'function', + function: { + name: 'create-note-with-special-chars', + description: 'Create a new note with special characters in name', + parameters: { + type: 'object', + properties: { + title: { + type: 'string', + description: 'Note title' + }, + content: { + type: 'string', + description: 'Note content' + }, + deeply: { + type: 'object', + description: 'Deeply nested object', + properties: { + nested: { + type: 'object', + description: 'Nested object', + properties: { + value: { + type: 'string', + description: 'Nested value' + } + } + } + } + } + }, + required: [] // Empty for Anthropic testing + } + } + } + ]; + + beforeAll(() => { + // Initialize components + console.log('Setting up integration test environment...'); + }); + + afterAll(() => { + // Cleanup + toolResponseCache.shutdown(); + providerHealthMonitor.stopMonitoring(); + }); + + describe('Tool Timeout Enforcement', () => { + it('should enforce timeouts on long-running tools', async () => { + const result = await toolTimeoutEnforcer.executeWithTimeout( + 'test_tool', + async () => { + await new Promise(resolve => setTimeout(resolve, 100)); + return 'success'; + }, + 200 // 200ms timeout + ); + + expect(result.success).toBe(true); + expect(result.timedOut).toBe(false); + expect(result.result).toBe('success'); + }); + + it('should timeout and report failure', async () => { + const result = await toolTimeoutEnforcer.executeWithTimeout( + 'slow_tool', + async () => { + await new Promise(resolve => setTimeout(resolve, 1000)); + return 'should not reach'; + }, + 100 // 100ms timeout + ); + + expect(result.success).toBe(false); + expect(result.timedOut).toBe(true); + }); + }); + + describe('Provider Tool Validation', () => { + it('should validate and fix tools for OpenAI', () => { + const result = providerToolValidator.validateTool(sampleTools[1], 'openai'); + + expect(result.fixedTool).toBeDefined(); + if (result.fixedTool) { + // Should fix special characters in function name + expect(result.fixedTool.function.name).not.toContain('-'); + } + }); + + it('should ensure non-empty required array for Anthropic', () => { + const result = providerToolValidator.validateTool(sampleTools[1], 'anthropic'); + + expect(result.fixedTool).toBeDefined(); + if (result.fixedTool) { + // Should add at least one required parameter + expect(result.fixedTool.function.parameters.required?.length).toBeGreaterThan(0); + } + }); + + it('should simplify tools for Ollama', () => { + const result = providerToolValidator.validateTool(sampleTools[1], 'ollama'); + + expect(result.warnings.length).toBeGreaterThan(0); + }); + }); + + describe('Parameter Type Coercion', () => { + it('should coerce string numbers to numbers', () => { + const result = parameterCoercer.coerceToolArguments( + { limit: '10' }, + sampleTools[0], + { parseNumbers: true } + ); + + expect(result.success).toBe(true); + expect(result.value.limit).toBe(10); + expect(typeof result.value.limit).toBe('number'); + }); + + it('should apply default values', () => { + const result = parameterCoercer.coerceToolArguments( + { query: 'test' }, + sampleTools[0], + { applyDefaults: true } + ); + + expect(result.success).toBe(true); + expect(result.value.limit).toBe(10); + }); + + it('should normalize arrays', () => { + const tool: Tool = { + type: 'function', + function: { + name: 'test', + description: 'Test', + parameters: { + type: 'object', + properties: { + tags: { + type: 'array', + description: 'List of tags', + items: { type: 'string', description: 'Tag value' } + } + }, + required: [] + } + } + }; + + const result = parameterCoercer.coerceToolArguments( + { tags: 'single-tag' }, + tool, + { normalizeArrays: true } + ); + + expect(result.success).toBe(true); + expect(Array.isArray(result.value.tags)).toBe(true); + expect(result.value.tags).toEqual(['single-tag']); + }); + }); + + describe('Tool Execution Monitoring', () => { + it('should track execution statistics', () => { + // Record successful execution + toolExecutionMonitor.recordExecution({ + toolName: 'test_tool', + provider: 'openai', + status: 'success', + executionTime: 100, + timestamp: new Date() + }); + + const stats = toolExecutionMonitor.getToolStats('test_tool', 'openai'); + expect(stats).toBeDefined(); + expect(stats?.successfulExecutions).toBe(1); + expect(stats?.reliabilityScore).toBeGreaterThan(0); + }); + + it('should auto-disable unreliable tools', () => { + // Record multiple failures + for (let i = 0; i < 6; i++) { + toolExecutionMonitor.recordExecution({ + toolName: 'unreliable_tool', + provider: 'openai', + status: 'failure', + executionTime: 100, + timestamp: new Date(), + error: 'Test failure' + }); + } + + const isDisabled = toolExecutionMonitor.isToolDisabled('unreliable_tool', 'openai'); + expect(isDisabled).toBe(true); + }); + }); + + describe('Tool Response Caching', () => { + it('should cache deterministic tool responses', () => { + const toolName = 'read_note_tool'; + const args = { noteId: 'test123' }; + const response = { content: 'Test content' }; + + // Set cache + const cached = toolResponseCache.set(toolName, args, response, 'openai'); + expect(cached).toBe(true); + + // Get from cache + const retrieved = toolResponseCache.get(toolName, args, 'openai'); + expect(retrieved).toEqual(response); + }); + + it('should generate consistent cache keys', () => { + const key1 = toolResponseCache.generateCacheKey('tool', { b: 2, a: 1 }, 'provider'); + const key2 = toolResponseCache.generateCacheKey('tool', { a: 1, b: 2 }, 'provider'); + + expect(key1).toBe(key2); + }); + + it('should respect TTL', async () => { + const toolName = 'temp_tool'; + const args = { id: 'temp' }; + const response = 'temp data'; + + // Set with short TTL + toolResponseCache.set(toolName, args, response, 'openai', 100); // 100ms TTL + + // Should be cached + expect(toolResponseCache.get(toolName, args, 'openai')).toBe(response); + + // Wait for expiration + await new Promise(resolve => setTimeout(resolve, 150)); + + // Should be expired + expect(toolResponseCache.get(toolName, args, 'openai')).toBeUndefined(); + }); + }); + + describe('Edge Case Handling', () => { + it('should fix OpenAI edge cases', () => { + const tool = sampleTools[1]; + const result = edgeCaseHandler.fixToolForProvider(tool, 'openai'); + + expect(result.fixed).toBe(true); + if (result.tool) { + // Function name should not have hyphens + expect(result.tool.function.name).not.toContain('-'); + // Deep nesting might be flattened + expect(result.modifications.length).toBeGreaterThan(0); + } + }); + + it('should fix Anthropic edge cases', () => { + const tool = sampleTools[1]; + const result = edgeCaseHandler.fixToolForProvider(tool, 'anthropic'); + + expect(result.fixed).toBe(true); + if (result.tool) { + // Should have required parameters + expect(result.tool.function.parameters.required).toBeDefined(); + expect(result.tool.function.parameters.required!.length).toBeGreaterThan(0); + } + }); + + it('should simplify for Ollama', () => { + const complexTool: Tool = { + type: 'function', + function: { + name: 'complex_tool', + description: 'A'.repeat(600), // Long description + parameters: { + type: 'object', + properties: Object.fromEntries( + Array.from({ length: 30 }, (_, i) => [ + `param${i}`, + { type: 'string', description: `Parameter ${i}` } + ]) + ), + required: [] + } + } + }; + + const result = edgeCaseHandler.fixToolForProvider(complexTool, 'ollama'); + + expect(result.fixed).toBe(true); + if (result.tool) { + // Description should be truncated + expect(result.tool.function.description.length).toBeLessThanOrEqual(500); + // Parameters should be reduced + const paramCount = Object.keys(result.tool.function.parameters.properties || {}).length; + expect(paramCount).toBeLessThanOrEqual(20); + } + }); + }); + + describe('Parallel Tool Execution', () => { + it('should identify independent tools for parallel execution', async () => { + const toolCalls: ToolCall[] = [ + { + id: '1', + function: { + name: 'search_notes', + arguments: { query: 'test1' } + } + }, + { + id: '2', + function: { + name: 'search_notes', + arguments: { query: 'test2' } + } + }, + { + id: '3', + function: { + name: 'read_note_tool', + arguments: { noteId: 'abc' } + } + } + ]; + + // These should be executed in parallel since they're independent + const handler = new EnhancedToolHandler(); + // For now, just verify the handler was created successfully + expect(handler).toBeDefined(); + }); + }); + + describe('Provider Health Monitoring', () => { + it('should track provider health status', async () => { + // Mock provider service + const mockService = { + chat: async () => ({ + content: 'test', + usage: { totalTokens: 5 } + }), + getModels: () => [{ id: 'test-model' }] + }; + + providerHealthMonitor.registerProvider('test-provider', mockService as any); + + // Manually trigger health check + const result = await providerHealthMonitor.checkProvider('test-provider'); + + expect(result.success).toBe(true); + expect(result.latency).toBeGreaterThan(0); + + const health = providerHealthMonitor.getProviderHealth('test-provider'); + expect(health?.healthy).toBe(true); + }); + + it('should disable unhealthy providers', () => { + // Simulate failures + const status = { + provider: 'failing-provider', + healthy: true, + lastChecked: new Date(), + consecutiveFailures: 3, + totalChecks: 10, + totalFailures: 3, + averageLatency: 100, + disabled: false + }; + + // This would normally be done internally + providerHealthMonitor.disableProvider('failing-provider', 'Too many failures'); + + expect(providerHealthMonitor.isProviderHealthy('failing-provider')).toBe(false); + }); + }); + + describe('End-to-End Integration', () => { + it('should handle tool execution with all enhancements', async () => { + // This tests the full flow with all components working together + const toolCall: ToolCall = { + id: 'integration-test', + function: { + name: 'search_notes', + arguments: '{"query": "test", "limit": "5"}' // String number to test coercion + } + }; + + // Test components integration + const tool = sampleTools[0]; + + // 1. Validate for provider + const validation = providerToolValidator.validateTool(tool, 'openai'); + expect(validation.valid || validation.fixedTool).toBeTruthy(); + + // 2. Apply edge case fixes + const edgeFixes = edgeCaseHandler.fixToolForProvider( + validation.fixedTool || tool, + 'openai' + ); + + // 3. Parse and coerce arguments + const args = parameterCoercer.coerceToolArguments( + JSON.parse(toolCall.function.arguments as string), + tool, + { provider: 'openai' } + ); + expect(args.value.limit).toBe(5); + expect(typeof args.value.limit).toBe('number'); + + // 4. Execute with timeout + const timeoutResult = await toolTimeoutEnforcer.executeWithTimeout( + 'search_notes', + async () => ({ results: ['note1', 'note2'] }), + 5000 + ); + expect(timeoutResult.success).toBe(true); + + // 5. Cache the result + if (timeoutResult.success) { + toolResponseCache.set( + 'search_notes', + args.value, + timeoutResult.result, + 'openai' + ); + } + + // 6. Record execution + toolExecutionMonitor.recordExecution({ + toolName: 'search_notes', + provider: 'openai', + status: timeoutResult.success ? 'success' : 'failure', + executionTime: timeoutResult.executionTime, + timestamp: new Date() + }); + + // Verify everything worked + const cached = toolResponseCache.get('search_notes', args.value, 'openai'); + expect(cached).toEqual({ results: ['note1', 'note2'] }); + + const stats = toolExecutionMonitor.getToolStats('search_notes', 'openai'); + expect(stats?.totalExecutions).toBeGreaterThan(0); + }); + }); +}); \ No newline at end of file diff --git a/apps/server/src/services/llm/tools/parameter_coercer.ts b/apps/server/src/services/llm/tools/parameter_coercer.ts new file mode 100644 index 000000000..a7dfe38a1 --- /dev/null +++ b/apps/server/src/services/llm/tools/parameter_coercer.ts @@ -0,0 +1,593 @@ +/** + * Parameter Type Coercer + * + * Provides automatic type conversion, array normalization, default value injection, + * and schema validation with fixes for tool parameters. + */ + +import log from '../../log.js'; +import type { Tool, ToolParameter } from './tool_interfaces.js'; + +/** + * Coercion result + */ +export interface CoercionResult { + success: boolean; + value: any; + wasCoerced: boolean; + errors: string[]; + warnings: string[]; +} + +/** + * Coercion options + */ +export interface CoercionOptions { + /** Strict mode - fail on any coercion error */ + strict: boolean; + /** Apply default values */ + applyDefaults: boolean; + /** Normalize arrays (single values to arrays) */ + normalizeArrays: boolean; + /** Trim string values */ + trimStrings: boolean; + /** Convert number strings to numbers */ + parseNumbers: boolean; + /** Convert boolean strings to booleans */ + parseBooleans: boolean; + /** Provider-specific quirks */ + provider?: string; +} + +/** + * Default coercion options + */ +const DEFAULT_OPTIONS: CoercionOptions = { + strict: false, + applyDefaults: true, + normalizeArrays: true, + trimStrings: true, + parseNumbers: true, + parseBooleans: true +}; + +/** + * Provider-specific quirks + */ +const PROVIDER_QUIRKS = { + openai: { + // OpenAI sometimes sends stringified JSON for complex objects + parseJsonStrings: true, + // OpenAI may send null for optional parameters + treatNullAsUndefined: true + }, + anthropic: { + // Anthropic strictly validates types + strictTypeChecking: true, + // Anthropic requires arrays to be actual arrays + requireArrayTypes: true + }, + ollama: { + // Local models may have looser type handling + lenientParsing: true, + // May send numbers as strings more often + aggressiveNumberParsing: true + } +}; + +/** + * Parameter coercer class + */ +export class ParameterCoercer { + private options: CoercionOptions; + + constructor(options?: Partial) { + this.options = { ...DEFAULT_OPTIONS, ...options }; + } + + /** + * Coerce tool call arguments to match tool definition + */ + coerceToolArguments( + args: Record, + tool: Tool, + options?: Partial + ): CoercionResult { + const opts = { ...this.options, ...options }; + const errors: string[] = []; + const warnings: string[] = []; + let wasCoerced = false; + + const parameters = tool.function.parameters; + const coercedArgs: Record = {}; + + // Process each parameter + for (const [paramName, paramDef] of Object.entries(parameters.properties)) { + const rawValue = args[paramName]; + const isRequired = parameters.required?.includes(paramName); + + // Handle missing values + if (rawValue === undefined || rawValue === null) { + if (opts.provider === 'openai' && rawValue === null) { + // OpenAI quirk: treat null as undefined + if (isRequired && !paramDef.default) { + errors.push(`Required parameter '${paramName}' is null`); + continue; + } + } + + if (opts.applyDefaults && paramDef.default !== undefined) { + coercedArgs[paramName] = paramDef.default; + wasCoerced = true; + warnings.push(`Applied default value for '${paramName}'`); + } else if (isRequired) { + errors.push(`Required parameter '${paramName}' is missing`); + } + continue; + } + + // Coerce the value + const coerced = this.coerceValue( + rawValue, + paramDef, + paramName, + opts + ); + + if (coerced.success) { + coercedArgs[paramName] = coerced.value; + if (coerced.wasCoerced) { + wasCoerced = true; + warnings.push(...coerced.warnings); + } + } else { + errors.push(...coerced.errors); + if (!opts.strict) { + // In non-strict mode, use original value + coercedArgs[paramName] = rawValue; + warnings.push(`Failed to coerce '${paramName}', using original value`); + } + } + } + + // Check for unknown parameters + for (const paramName of Object.keys(args)) { + if (!(paramName in parameters.properties)) { + warnings.push(`Unknown parameter '${paramName}' will be ignored`); + } + } + + return { + success: errors.length === 0, + value: coercedArgs, + wasCoerced, + errors, + warnings + }; + } + + /** + * Coerce a single value to match its type definition + */ + private coerceValue( + value: unknown, + definition: ToolParameter, + path: string, + options: CoercionOptions + ): CoercionResult { + const errors: string[] = []; + const warnings: string[] = []; + let wasCoerced = false; + let coercedValue = value; + + // Handle provider-specific JSON string parsing + if (options.provider === 'openai' && + typeof value === 'string' && + (definition.type === 'object' || definition.type === 'array')) { + try { + coercedValue = JSON.parse(value); + wasCoerced = true; + warnings.push(`Parsed JSON string for '${path}'`); + } catch { + // Not valid JSON, continue with string value + } + } + + // Type-specific coercion + switch (definition.type) { + case 'string': + const stringResult = this.coerceToString(coercedValue, path, options); + coercedValue = stringResult.value; + wasCoerced = wasCoerced || stringResult.wasCoerced; + warnings.push(...stringResult.warnings); + break; + + case 'number': + case 'integer': + const numberResult = this.coerceToNumber( + coercedValue, + path, + definition, + definition.type === 'integer', + options + ); + if (numberResult.success) { + coercedValue = numberResult.value; + wasCoerced = wasCoerced || numberResult.wasCoerced; + warnings.push(...numberResult.warnings); + } else { + errors.push(...numberResult.errors); + } + break; + + case 'boolean': + const boolResult = this.coerceToBoolean(coercedValue, path, options); + if (boolResult.success) { + coercedValue = boolResult.value; + wasCoerced = wasCoerced || boolResult.wasCoerced; + warnings.push(...boolResult.warnings); + } else { + errors.push(...boolResult.errors); + } + break; + + case 'array': + const arrayResult = this.coerceToArray( + coercedValue, + path, + definition, + options + ); + if (arrayResult.success) { + coercedValue = arrayResult.value; + wasCoerced = wasCoerced || arrayResult.wasCoerced; + warnings.push(...arrayResult.warnings); + } else { + errors.push(...arrayResult.errors); + } + break; + + case 'object': + const objectResult = this.coerceToObject( + coercedValue, + path, + definition, + options + ); + if (objectResult.success) { + coercedValue = objectResult.value; + wasCoerced = wasCoerced || objectResult.wasCoerced; + warnings.push(...objectResult.warnings); + } else { + errors.push(...objectResult.errors); + } + break; + + default: + warnings.push(`Unknown type '${definition.type}' for '${path}'`); + } + + // Validate enum values + if (definition.enum && !definition.enum.includes(String(coercedValue))) { + errors.push(`Value for '${path}' must be one of: ${definition.enum.join(', ')}`); + } + + return { + success: errors.length === 0, + value: coercedValue, + wasCoerced, + errors, + warnings + }; + } + + /** + * Coerce to string + */ + private coerceToString( + value: unknown, + path: string, + options: CoercionOptions + ): CoercionResult { + const warnings: string[] = []; + let wasCoerced = false; + let result: string; + + if (typeof value === 'string') { + result = options.trimStrings ? value.trim() : value; + if (result !== value) { + wasCoerced = true; + warnings.push(`Trimmed whitespace from '${path}'`); + } + } else if (value === null || value === undefined) { + result = ''; + wasCoerced = true; + warnings.push(`Converted null/undefined to empty string for '${path}'`); + } else { + result = String(value); + wasCoerced = true; + warnings.push(`Converted ${typeof value} to string for '${path}'`); + } + + return { + success: true, + value: result, + wasCoerced, + errors: [], + warnings + }; + } + + /** + * Coerce to number + */ + private coerceToNumber( + value: unknown, + path: string, + definition: ToolParameter, + isInteger: boolean, + options: CoercionOptions + ): CoercionResult { + const errors: string[] = []; + const warnings: string[] = []; + let wasCoerced = false; + let result: number; + + if (typeof value === 'number') { + result = isInteger ? Math.round(value) : value; + if (result !== value) { + wasCoerced = true; + warnings.push(`Rounded to integer for '${path}'`); + } + } else if (typeof value === 'string' && options.parseNumbers) { + const parsed = isInteger ? parseInt(value, 10) : parseFloat(value); + if (!isNaN(parsed)) { + result = parsed; + wasCoerced = true; + warnings.push(`Parsed string to number for '${path}'`); + } else { + errors.push(`Cannot parse '${value}' as number for '${path}'`); + return { success: false, value, wasCoerced: false, errors, warnings }; + } + } else if (typeof value === 'boolean') { + result = value ? 1 : 0; + wasCoerced = true; + warnings.push(`Converted boolean to number for '${path}'`); + } else { + errors.push(`Cannot coerce ${typeof value} to number for '${path}'`); + return { success: false, value, wasCoerced: false, errors, warnings }; + } + + // Validate constraints + if (definition.minimum !== undefined && result < definition.minimum) { + result = definition.minimum; + wasCoerced = true; + warnings.push(`Clamped to minimum value ${definition.minimum} for '${path}'`); + } + if (definition.maximum !== undefined && result > definition.maximum) { + result = definition.maximum; + wasCoerced = true; + warnings.push(`Clamped to maximum value ${definition.maximum} for '${path}'`); + } + + return { + success: true, + value: result, + wasCoerced, + errors, + warnings + }; + } + + /** + * Coerce to boolean + */ + private coerceToBoolean( + value: unknown, + path: string, + options: CoercionOptions + ): CoercionResult { + const warnings: string[] = []; + let wasCoerced = false; + let result: boolean; + + if (typeof value === 'boolean') { + result = value; + } else if (typeof value === 'string' && options.parseBooleans) { + const lower = value.toLowerCase().trim(); + if (lower === 'true' || lower === 'yes' || lower === '1') { + result = true; + wasCoerced = true; + warnings.push(`Parsed string to boolean true for '${path}'`); + } else if (lower === 'false' || lower === 'no' || lower === '0') { + result = false; + wasCoerced = true; + warnings.push(`Parsed string to boolean false for '${path}'`); + } else { + return { + success: false, + value, + wasCoerced: false, + errors: [`Cannot parse '${value}' as boolean for '${path}'`], + warnings + }; + } + } else if (typeof value === 'number') { + result = value !== 0; + wasCoerced = true; + warnings.push(`Converted number to boolean for '${path}'`); + } else { + result = Boolean(value); + wasCoerced = true; + warnings.push(`Coerced ${typeof value} to boolean for '${path}'`); + } + + return { + success: true, + value: result, + wasCoerced, + errors: [], + warnings + }; + } + + /** + * Coerce to array + */ + private coerceToArray( + value: unknown, + path: string, + definition: ToolParameter, + options: CoercionOptions + ): CoercionResult { + const errors: string[] = []; + const warnings: string[] = []; + let wasCoerced = false; + let result: any[]; + + if (Array.isArray(value)) { + result = value; + } else if (options.normalizeArrays) { + // Convert single value to array + result = [value]; + wasCoerced = true; + warnings.push(`Normalized single value to array for '${path}'`); + } else { + errors.push(`Expected array for '${path}', got ${typeof value}`); + return { success: false, value, wasCoerced: false, errors, warnings }; + } + + // Validate array constraints + if (definition.minItems !== undefined && result.length < definition.minItems) { + errors.push(`Array '${path}' must have at least ${definition.minItems} items`); + } + if (definition.maxItems !== undefined && result.length > definition.maxItems) { + result = result.slice(0, definition.maxItems); + wasCoerced = true; + warnings.push(`Truncated array to ${definition.maxItems} items for '${path}'`); + } + + // Coerce array items if type is specified + if (definition.items) { + const coercedItems: any[] = []; + for (let i = 0; i < result.length; i++) { + const itemResult = this.coerceValue( + result[i], + definition.items as ToolParameter, + `${path}[${i}]`, + options + ); + if (itemResult.success) { + coercedItems.push(itemResult.value); + if (itemResult.wasCoerced) wasCoerced = true; + warnings.push(...itemResult.warnings); + } else { + errors.push(...itemResult.errors); + coercedItems.push(result[i]); // Keep original on error + } + } + result = coercedItems; + } + + return { + success: errors.length === 0, + value: result, + wasCoerced, + errors, + warnings + }; + } + + /** + * Coerce to object + */ + private coerceToObject( + value: unknown, + path: string, + definition: ToolParameter, + options: CoercionOptions + ): CoercionResult { + const errors: string[] = []; + const warnings: string[] = []; + let wasCoerced = false; + let result: Record; + + if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + result = value as Record; + } else if (typeof value === 'string') { + // Try to parse as JSON + try { + result = JSON.parse(value); + wasCoerced = true; + warnings.push(`Parsed JSON string for object '${path}'`); + } catch { + errors.push(`Cannot parse string as object for '${path}'`); + return { success: false, value, wasCoerced: false, errors, warnings }; + } + } else { + errors.push(`Expected object for '${path}', got ${typeof value}`); + return { success: false, value, wasCoerced: false, errors, warnings }; + } + + // Coerce nested properties if defined + if (definition.properties) { + const coercedObj: Record = {}; + for (const [propName, propDef] of Object.entries(definition.properties)) { + if (propName in result) { + const propResult = this.coerceValue( + result[propName], + propDef, + `${path}.${propName}`, + options + ); + if (propResult.success) { + coercedObj[propName] = propResult.value; + if (propResult.wasCoerced) wasCoerced = true; + warnings.push(...propResult.warnings); + } else { + errors.push(...propResult.errors); + coercedObj[propName] = result[propName]; // Keep original on error + } + } else if (propDef.default !== undefined && options.applyDefaults) { + coercedObj[propName] = propDef.default; + wasCoerced = true; + warnings.push(`Applied default value for '${path}.${propName}'`); + } + } + + // Include any additional properties not in schema + for (const propName of Object.keys(result)) { + if (!(propName in coercedObj)) { + coercedObj[propName] = result[propName]; + } + } + + result = coercedObj; + } + + return { + success: errors.length === 0, + value: result, + wasCoerced, + errors, + warnings + }; + } + + /** + * Update coercion options + */ + updateOptions(options: Partial): void { + this.options = { ...this.options, ...options }; + } + + /** + * Get current options + */ + getOptions(): CoercionOptions { + return { ...this.options }; + } +} + +// Export singleton instance +export const parameterCoercer = new ParameterCoercer(); \ No newline at end of file diff --git a/apps/server/src/services/llm/tools/provider_tool_validator.ts b/apps/server/src/services/llm/tools/provider_tool_validator.ts new file mode 100644 index 000000000..98b373df7 --- /dev/null +++ b/apps/server/src/services/llm/tools/provider_tool_validator.ts @@ -0,0 +1,470 @@ +/** + * Provider Tool Validator + * + * Validates and auto-fixes tool definitions based on provider-specific requirements + * for OpenAI, Anthropic, and Ollama. + */ + +import log from '../../log.js'; +import type { Tool, ToolParameter } from './tool_interfaces.js'; +import type { ProviderType } from '../providers/provider_factory.js'; + +/** + * Validation result for a tool + */ +export interface ValidationResult { + valid: boolean; + errors: ValidationError[]; + warnings: ValidationWarning[]; + fixedTool?: Tool; +} + +/** + * Validation error + */ +export interface ValidationError { + field: string; + message: string; + severity: 'error' | 'critical'; +} + +/** + * Validation warning + */ +export interface ValidationWarning { + field: string; + message: string; + suggestion?: string; +} + +/** + * Provider-specific validation rules + */ +interface ProviderRules { + maxFunctionNameLength: number; + maxDescriptionLength: number; + maxParameterDepth: number; + maxParameterCount: number; + allowEmptyRequired: boolean; + requireDescriptions: boolean; + functionNamePattern: RegExp; + supportedTypes: Set; +} + +/** + * Default validation rules per provider + */ +const PROVIDER_RULES: Record = { + openai: { + maxFunctionNameLength: 64, + maxDescriptionLength: 1024, + maxParameterDepth: 5, + maxParameterCount: 20, + allowEmptyRequired: true, + requireDescriptions: true, + functionNamePattern: /^[a-zA-Z0-9_-]+$/, + supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array', 'integer']) + }, + anthropic: { + maxFunctionNameLength: 64, + maxDescriptionLength: 1024, + maxParameterDepth: 4, + maxParameterCount: 15, + allowEmptyRequired: false, // Anthropic requires non-empty required arrays + requireDescriptions: true, + functionNamePattern: /^[a-zA-Z0-9_-]+$/, + supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array', 'integer']) + }, + ollama: { + maxFunctionNameLength: 50, + maxDescriptionLength: 500, + maxParameterDepth: 3, + maxParameterCount: 10, // Local models have smaller context + allowEmptyRequired: true, + requireDescriptions: false, + functionNamePattern: /^[a-zA-Z0-9_]+$/, + supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array']) + } +}; + +/** + * Provider tool validator class + */ +export class ProviderToolValidator { + private providerRules: Map; + + constructor() { + this.providerRules = new Map(Object.entries(PROVIDER_RULES)); + } + + /** + * Validate a tool for a specific provider + */ + validateTool(tool: Tool, provider: string): ValidationResult { + const rules = this.providerRules.get(provider) || PROVIDER_RULES.openai; + const errors: ValidationError[] = []; + const warnings: ValidationWarning[] = []; + + // Deep clone the tool for potential fixes + const fixedTool = JSON.parse(JSON.stringify(tool)) as Tool; + let wasFixed = false; + + // Validate function name + const nameValidation = this.validateFunctionName( + fixedTool.function.name, + rules + ); + if (nameValidation.error) { + errors.push(nameValidation.error); + } + if (nameValidation.fixed) { + fixedTool.function.name = nameValidation.fixed; + wasFixed = true; + } + + // Validate description + const descValidation = this.validateDescription( + fixedTool.function.description, + rules + ); + if (descValidation.error) { + errors.push(descValidation.error); + } + if (descValidation.warning) { + warnings.push(descValidation.warning); + } + if (descValidation.fixed) { + fixedTool.function.description = descValidation.fixed; + wasFixed = true; + } + + // Validate parameters + const paramValidation = this.validateParameters( + fixedTool.function.parameters, + rules, + provider + ); + errors.push(...paramValidation.errors); + warnings.push(...paramValidation.warnings); + if (paramValidation.fixed) { + fixedTool.function.parameters = paramValidation.fixed; + wasFixed = true; + } + + // Provider-specific validations + const providerSpecific = this.validateProviderSpecific(fixedTool, provider); + errors.push(...providerSpecific.errors); + warnings.push(...providerSpecific.warnings); + if (providerSpecific.fixed) { + Object.assign(fixedTool, providerSpecific.fixed); + wasFixed = true; + } + + return { + valid: errors.length === 0, + errors, + warnings, + fixedTool: wasFixed ? fixedTool : undefined + }; + } + + /** + * Validate function name + */ + private validateFunctionName(name: string, rules: ProviderRules) { + const result: any = {}; + + // Check length + if (name.length > rules.maxFunctionNameLength) { + result.error = { + field: 'function.name', + message: `Function name exceeds maximum length of ${rules.maxFunctionNameLength}`, + severity: 'error' as const + }; + // Auto-fix: truncate + result.fixed = name.substring(0, rules.maxFunctionNameLength); + } + + // Check pattern + if (!rules.functionNamePattern.test(name)) { + result.error = { + field: 'function.name', + message: `Function name contains invalid characters`, + severity: 'error' as const + }; + // Auto-fix: replace invalid characters + result.fixed = name.replace(/[^a-zA-Z0-9_-]/g, '_'); + } + + return result; + } + + /** + * Validate description + */ + private validateDescription(description: string, rules: ProviderRules) { + const result: any = {}; + + // Check if description exists when required + if (rules.requireDescriptions && !description) { + result.error = { + field: 'function.description', + message: 'Description is required', + severity: 'error' as const + }; + result.fixed = 'Performs an operation'; // Generic fallback + } + + // Check length + if (description && description.length > rules.maxDescriptionLength) { + result.warning = { + field: 'function.description', + message: `Description exceeds recommended length of ${rules.maxDescriptionLength}`, + suggestion: 'Consider shortening the description' + }; + // Auto-fix: truncate with ellipsis + result.fixed = description.substring(0, rules.maxDescriptionLength - 3) + '...'; + } + + return result; + } + + /** + * Validate parameters + */ + private validateParameters( + parameters: any, + rules: ProviderRules, + provider: string + ) { + const errors: ValidationError[] = []; + const warnings: ValidationWarning[] = []; + let fixed: any = null; + + // Ensure parameters is an object + if (parameters.type !== 'object') { + errors.push({ + field: 'function.parameters.type', + message: 'Parameters must be of type "object"', + severity: 'critical' + }); + fixed = { + type: 'object', + properties: parameters.properties || {}, + required: parameters.required || [] + }; + } + + // Check parameter count + const paramCount = Object.keys(parameters.properties || {}).length; + if (paramCount > rules.maxParameterCount) { + warnings.push({ + field: 'function.parameters', + message: `Parameter count (${paramCount}) exceeds recommended maximum (${rules.maxParameterCount})`, + suggestion: 'Consider reducing the number of parameters' + }); + } + + // Validate required array for Anthropic + if (!rules.allowEmptyRequired && (!parameters.required || parameters.required.length === 0)) { + if (provider === 'anthropic') { + // For Anthropic, add at least one optional parameter to required + const props = Object.keys(parameters.properties || {}); + if (props.length > 0) { + if (!fixed) fixed = { ...parameters }; + fixed.required = [props[0]]; // Add first property as required + warnings.push({ + field: 'function.parameters.required', + message: 'Anthropic requires non-empty required array, added first parameter', + suggestion: 'Specify which parameters are required' + }); + } + } + } + + // Validate parameter types and depth + if (parameters.properties) { + const typeErrors = this.validateParameterTypes( + parameters.properties, + rules.supportedTypes, + 0, + rules.maxParameterDepth + ); + errors.push(...typeErrors); + } + + return { errors, warnings, fixed }; + } + + /** + * Validate parameter types recursively + */ + private validateParameterTypes( + properties: Record, + supportedTypes: Set, + depth: number, + maxDepth: number + ): ValidationError[] { + const errors: ValidationError[] = []; + + if (depth > maxDepth) { + errors.push({ + field: 'function.parameters', + message: `Parameter nesting exceeds maximum depth of ${maxDepth}`, + severity: 'error' + }); + return errors; + } + + for (const [key, param] of Object.entries(properties)) { + // Check if type is supported + if (param.type && !supportedTypes.has(param.type)) { + errors.push({ + field: `function.parameters.properties.${key}.type`, + message: `Unsupported type: ${param.type}`, + severity: 'error' + }); + } + + // Recursively check nested objects + if (param.type === 'object' && param.properties) { + const nestedErrors = this.validateParameterTypes( + param.properties, + supportedTypes, + depth + 1, + maxDepth + ); + errors.push(...nestedErrors); + } + + // Check array items + if (param.type === 'array' && param.items) { + if (typeof param.items === 'object' && 'properties' in param.items) { + const nestedErrors = this.validateParameterTypes( + param.items.properties!, + supportedTypes, + depth + 1, + maxDepth + ); + errors.push(...nestedErrors); + } + } + } + + return errors; + } + + /** + * Provider-specific validations + */ + private validateProviderSpecific(tool: Tool, provider: string) { + const errors: ValidationError[] = []; + const warnings: ValidationWarning[] = []; + let fixed: any = null; + + switch (provider) { + case 'openai': + // OpenAI-specific: Check for special characters in function names + if (tool.function.name.includes('-')) { + warnings.push({ + field: 'function.name', + message: 'OpenAI prefers underscores over hyphens in function names', + suggestion: 'Replace hyphens with underscores' + }); + } + break; + + case 'anthropic': + // Anthropic-specific: Ensure descriptions are meaningful + if (tool.function.description && tool.function.description.length < 10) { + warnings.push({ + field: 'function.description', + message: 'Description is very short', + suggestion: 'Provide a more detailed description for better results' + }); + } + break; + + case 'ollama': + // Ollama-specific: Warn about complex nested structures + const complexity = this.calculateComplexity(tool.function.parameters); + if (complexity > 10) { + warnings.push({ + field: 'function.parameters', + message: 'Tool parameters are complex for local models', + suggestion: 'Consider simplifying the parameter structure' + }); + } + break; + } + + return { errors, warnings, fixed }; + } + + /** + * Calculate parameter complexity score + */ + private calculateComplexity(parameters: any, depth: number = 0): number { + let complexity = depth; + + if (parameters.properties) { + for (const param of Object.values(parameters.properties) as ToolParameter[]) { + complexity += 1; + if (param.type === 'object' && param.properties) { + complexity += this.calculateComplexity(param, depth + 1); + } + if (param.type === 'array' && param.items) { + complexity += 2; // Arrays add more complexity + } + } + } + + return complexity; + } + + /** + * Batch validate multiple tools + */ + validateTools(tools: Tool[], provider: string): Map { + const results = new Map(); + + for (const tool of tools) { + const result = this.validateTool(tool, provider); + results.set(tool.function.name, result); + + if (!result.valid) { + log.info(`Tool '${tool.function.name}' validation failed for ${provider}: ${JSON.stringify(result.errors)}`); + } + if (result.warnings.length > 0) { + log.info(`Tool '${tool.function.name}' validation warnings for ${provider}: ${JSON.stringify(result.warnings)}`); + } + } + + return results; + } + + /** + * Auto-fix tools for a provider + */ + autoFixTools(tools: Tool[], provider: string): Tool[] { + const fixed: Tool[] = []; + + for (const tool of tools) { + const result = this.validateTool(tool, provider); + fixed.push(result.fixedTool || tool); + } + + return fixed; + } + + /** + * Check if a provider supports a tool + */ + isToolSupportedByProvider(tool: Tool, provider: string): boolean { + const result = this.validateTool(tool, provider); + return result.valid || (result.fixedTool !== undefined); + } +} + +// Export singleton instance +export const providerToolValidator = new ProviderToolValidator(); \ No newline at end of file diff --git a/apps/server/src/services/llm/tools/tool_format_adapter.ts b/apps/server/src/services/llm/tools/tool_format_adapter.ts index 6c9979662..72f6e4991 100644 --- a/apps/server/src/services/llm/tools/tool_format_adapter.ts +++ b/apps/server/src/services/llm/tools/tool_format_adapter.ts @@ -7,6 +7,9 @@ import log from '../../log.js'; import type { Tool, ToolCall, ToolParameter } from './tool_interfaces.js'; +import { providerToolValidator } from './provider_tool_validator.js'; +import { edgeCaseHandler } from '../providers/edge_case_handler.js'; +import { parameterCoercer } from './parameter_coercer.js'; /** * Anthropic tool format @@ -51,17 +54,23 @@ export class ToolFormatAdapter { * Convert tools from standard format to provider-specific format */ static convertToProviderFormat(tools: Tool[], provider: ProviderType): unknown[] { + // First validate and fix tools for the provider + const validatedTools = providerToolValidator.autoFixTools(tools, provider); + + // Apply edge case fixes + const fixedTools = edgeCaseHandler.fixToolsForProvider(validatedTools, provider); + switch (provider) { case 'anthropic': - return this.convertToAnthropicFormat(tools); + return this.convertToAnthropicFormat(fixedTools); case 'ollama': - return this.convertToOllamaFormat(tools); + return this.convertToOllamaFormat(fixedTools); case 'openai': // OpenAI format matches our standard format - return tools; + return fixedTools; default: log.info(`Warning: Unknown provider ${provider}, returning tools in standard format`); - return tools; + return fixedTools; } } @@ -300,18 +309,42 @@ export class ToolFormatAdapter { } /** - * Parse tool arguments safely + * Parse tool arguments safely with coercion */ - static parseToolArguments(args: string | Record): Record { + static parseToolArguments( + args: string | Record, + tool?: Tool, + provider?: string + ): Record { + let parsedArgs: Record; + if (typeof args === 'string') { try { - return JSON.parse(args); + parsedArgs = JSON.parse(args); } catch (error) { log.error(`Failed to parse tool arguments as JSON: ${error}`); return {}; } + } else { + parsedArgs = args || {}; } - return args || {}; + + // Apply parameter coercion if tool definition is provided + if (tool) { + const coercionResult = parameterCoercer.coerceToolArguments( + parsedArgs, + tool, + { provider } + ); + + if (coercionResult.warnings.length > 0) { + log.info(`Parameter coercion warnings: ${coercionResult.warnings.join(', ')}`); + } + + return coercionResult.value; + } + + return parsedArgs; } /** diff --git a/apps/server/src/services/llm/tools/tool_response_cache.ts b/apps/server/src/services/llm/tools/tool_response_cache.ts new file mode 100644 index 000000000..9135523b0 --- /dev/null +++ b/apps/server/src/services/llm/tools/tool_response_cache.ts @@ -0,0 +1,547 @@ +/** + * Tool Response Cache + * + * Implements LRU cache with TTL for deterministic/read-only tool responses, + * with cache key generation, invalidation strategies, and hit rate tracking. + */ + +import log from '../../log.js'; +import crypto from 'crypto'; + +/** + * Cache entry with metadata + */ +interface CacheEntry { + key: string; + value: T; + timestamp: Date; + expiresAt: Date; + hits: number; + size: number; + toolName: string; + provider?: string; +} + +/** + * Cache statistics + */ +export interface CacheStatistics { + totalEntries: number; + totalSize: number; + hitRate: number; + missRate: number; + evictionCount: number; + avgHitsPerEntry: number; + oldestEntry?: Date; + newestEntry?: Date; + topTools: Array<{ tool: string; hits: number }>; +} + +/** + * Cache configuration + */ +export interface CacheConfig { + /** Maximum cache size in bytes (default: 50MB) */ + maxSize: number; + /** Maximum number of entries (default: 1000) */ + maxEntries: number; + /** Default TTL in milliseconds (default: 300000 - 5 minutes) */ + defaultTTL: number; + /** Enable automatic cleanup (default: true) */ + autoCleanup: boolean; + /** Cleanup interval in milliseconds (default: 60000) */ + cleanupInterval: number; + /** Enable hit tracking (default: true) */ + trackHits: boolean; +} + +/** + * Default configuration + */ +const DEFAULT_CONFIG: CacheConfig = { + maxSize: 50 * 1024 * 1024, // 50MB + maxEntries: 1000, + defaultTTL: 300000, // 5 minutes + autoCleanup: true, + cleanupInterval: 60000, // 1 minute + trackHits: true +}; + +/** + * Tool-specific TTL overrides (in milliseconds) + */ +const TOOL_TTL_OVERRIDES: Record = { + // Static data tools - longer TTL + 'read_note_tool': 600000, // 10 minutes + 'get_note_metadata': 600000, // 10 minutes + + // Search tools - medium TTL + 'search_notes_tool': 300000, // 5 minutes + 'keyword_search_tool': 300000, // 5 minutes + + // Dynamic data tools - shorter TTL + 'get_recent_notes': 60000, // 1 minute + 'get_workspace_status': 30000 // 30 seconds +}; + +/** + * Deterministic tools that can be cached + */ +const CACHEABLE_TOOLS = new Set([ + 'read_note_tool', + 'search_notes_tool', + 'keyword_search_tool', + 'attribute_search_tool', + 'get_note_metadata', + 'get_note_content', + 'get_recent_notes', + 'get_workspace_status', + 'list_notes', + 'find_notes_by_tag' +]); + +/** + * Tool response cache class + */ +export class ToolResponseCache { + private config: CacheConfig; + private cache: Map; + private accessOrder: string[]; + private totalHits: number; + private totalMisses: number; + private evictionCount: number; + private cleanupTimer?: NodeJS.Timeout; + private currentSize: number; + + constructor(config?: Partial) { + this.config = { ...DEFAULT_CONFIG, ...config }; + this.cache = new Map(); + this.accessOrder = []; + this.totalHits = 0; + this.totalMisses = 0; + this.evictionCount = 0; + this.currentSize = 0; + + if (this.config.autoCleanup) { + this.startAutoCleanup(); + } + } + + /** + * Check if a tool is cacheable + */ + isCacheable(toolName: string): boolean { + return CACHEABLE_TOOLS.has(toolName); + } + + /** + * Generate cache key for tool call + */ + generateCacheKey( + toolName: string, + args: Record, + provider?: string + ): string { + // Sort arguments for consistent key generation + const sortedArgs = this.sortObjectDeep(args); + + // Create key components + const keyComponents = { + tool: toolName, + args: sortedArgs, + provider: provider || 'default' + }; + + // Generate hash + const hash = crypto + .createHash('sha256') + .update(JSON.stringify(keyComponents)) + .digest('hex'); + + return `${toolName}:${hash.substring(0, 16)}`; + } + + /** + * Get cached response + */ + get( + toolName: string, + args: Record, + provider?: string + ): any | undefined { + if (!this.isCacheable(toolName)) { + return undefined; + } + + const key = this.generateCacheKey(toolName, args, provider); + const entry = this.cache.get(key); + + if (!entry) { + this.totalMisses++; + log.info(`Cache miss for ${toolName}`); + return undefined; + } + + // Check if expired + if (new Date() > entry.expiresAt) { + this.cache.delete(key); + this.removeFromAccessOrder(key); + this.currentSize -= entry.size; + this.totalMisses++; + log.info(`Cache expired for ${toolName}`); + return undefined; + } + + // Update hit count and access order + if (this.config.trackHits) { + entry.hits++; + this.updateAccessOrder(key); + } + + this.totalHits++; + log.info(`Cache hit for ${toolName} (${entry.hits} hits)`); + + return entry.value; + } + + /** + * Set cached response + */ + set( + toolName: string, + args: Record, + value: any, + provider?: string, + ttl?: number + ): boolean { + if (!this.isCacheable(toolName)) { + return false; + } + + const key = this.generateCacheKey(toolName, args, provider); + const size = this.calculateSize(value); + + // Check size limits + if (size > this.config.maxSize) { + log.info(`Cache entry too large for ${toolName}: ${size} bytes`); + return false; + } + + // Evict entries if necessary + while (this.cache.size >= this.config.maxEntries || + this.currentSize + size > this.config.maxSize) { + this.evictLRU(); + } + + // Determine TTL + const effectiveTTL = ttl || + TOOL_TTL_OVERRIDES[toolName] || + this.config.defaultTTL; + + // Create entry + const entry: CacheEntry = { + key, + value, + timestamp: new Date(), + expiresAt: new Date(Date.now() + effectiveTTL), + hits: 0, + size, + toolName, + provider + }; + + // Add to cache + this.cache.set(key, entry); + this.accessOrder.push(key); + this.currentSize += size; + + log.info(`Cached response for ${toolName} (${size} bytes, TTL: ${effectiveTTL}ms)`); + + return true; + } + + /** + * Invalidate cache entries + */ + invalidate(filter?: { + toolName?: string; + provider?: string; + pattern?: RegExp; + }): number { + let invalidated = 0; + + for (const [key, entry] of this.cache.entries()) { + let shouldInvalidate = false; + + if (filter?.toolName && entry.toolName === filter.toolName) { + shouldInvalidate = true; + } + if (filter?.provider && entry.provider === filter.provider) { + shouldInvalidate = true; + } + if (filter?.pattern && filter.pattern.test(key)) { + shouldInvalidate = true; + } + if (!filter) { + shouldInvalidate = true; // Invalidate all if no filter + } + + if (shouldInvalidate) { + this.cache.delete(key); + this.removeFromAccessOrder(key); + this.currentSize -= entry.size; + invalidated++; + } + } + + log.info(`Invalidated ${invalidated} cache entries`); + return invalidated; + } + + /** + * Evict least recently used entry + */ + private evictLRU(): void { + if (this.accessOrder.length === 0) return; + + const key = this.accessOrder.shift()!; + const entry = this.cache.get(key); + + if (entry) { + this.cache.delete(key); + this.currentSize -= entry.size; + this.evictionCount++; + log.info(`Evicted cache entry for ${entry.toolName} (LRU)`); + } + } + + /** + * Update access order for LRU + */ + private updateAccessOrder(key: string): void { + this.removeFromAccessOrder(key); + this.accessOrder.push(key); + } + + /** + * Remove from access order + */ + private removeFromAccessOrder(key: string): void { + const index = this.accessOrder.indexOf(key); + if (index > -1) { + this.accessOrder.splice(index, 1); + } + } + + /** + * Calculate size of value in bytes + */ + private calculateSize(value: any): number { + const str = typeof value === 'string' ? value : JSON.stringify(value); + return Buffer.byteLength(str, 'utf8'); + } + + /** + * Sort object deeply for consistent key generation + */ + private sortObjectDeep(obj: any): any { + if (obj === null || typeof obj !== 'object') { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => this.sortObjectDeep(item)); + } + + const sorted: any = {}; + const keys = Object.keys(obj).sort(); + + for (const key of keys) { + sorted[key] = this.sortObjectDeep(obj[key]); + } + + return sorted; + } + + /** + * Start automatic cleanup + */ + private startAutoCleanup(): void { + this.cleanupTimer = setInterval(() => { + this.cleanup(); + }, this.config.cleanupInterval); + } + + /** + * Stop automatic cleanup + */ + private stopAutoCleanup(): void { + if (this.cleanupTimer) { + clearInterval(this.cleanupTimer); + this.cleanupTimer = undefined; + } + } + + /** + * Clean up expired entries + */ + cleanup(): number { + const now = new Date(); + let cleaned = 0; + + for (const [key, entry] of this.cache.entries()) { + if (now > entry.expiresAt) { + this.cache.delete(key); + this.removeFromAccessOrder(key); + this.currentSize -= entry.size; + cleaned++; + } + } + + if (cleaned > 0) { + log.info(`Cleaned up ${cleaned} expired cache entries`); + } + + return cleaned; + } + + /** + * Get cache statistics + */ + getStatistics(): CacheStatistics { + const entries = Array.from(this.cache.values()); + const totalRequests = this.totalHits + this.totalMisses; + + // Calculate tool hit counts + const toolHits = new Map(); + for (const entry of entries) { + const current = toolHits.get(entry.toolName) || 0; + toolHits.set(entry.toolName, current + entry.hits); + } + + // Sort tools by hits + const topTools = Array.from(toolHits.entries()) + .map(([tool, hits]) => ({ tool, hits })) + .sort((a, b) => b.hits - a.hits) + .slice(0, 10); + + // Find oldest and newest entries + const timestamps = entries.map(e => e.timestamp); + const oldestEntry = timestamps.length > 0 + ? new Date(Math.min(...timestamps.map(t => t.getTime()))) + : undefined; + const newestEntry = timestamps.length > 0 + ? new Date(Math.max(...timestamps.map(t => t.getTime()))) + : undefined; + + // Calculate average hits + const totalHitsInCache = entries.reduce((sum, e) => sum + e.hits, 0); + const avgHitsPerEntry = entries.length > 0 + ? totalHitsInCache / entries.length + : 0; + + return { + totalEntries: this.cache.size, + totalSize: this.currentSize, + hitRate: totalRequests > 0 ? this.totalHits / totalRequests : 0, + missRate: totalRequests > 0 ? this.totalMisses / totalRequests : 0, + evictionCount: this.evictionCount, + avgHitsPerEntry, + oldestEntry, + newestEntry, + topTools + }; + } + + /** + * Clear entire cache + */ + clear(): void { + this.cache.clear(); + this.accessOrder = []; + this.currentSize = 0; + this.totalHits = 0; + this.totalMisses = 0; + this.evictionCount = 0; + log.info('Cleared entire cache'); + } + + /** + * Get cache size info + */ + getSizeInfo(): { + entries: number; + bytes: number; + maxEntries: number; + maxBytes: number; + utilizationPercent: number; + } { + return { + entries: this.cache.size, + bytes: this.currentSize, + maxEntries: this.config.maxEntries, + maxBytes: this.config.maxSize, + utilizationPercent: (this.currentSize / this.config.maxSize) * 100 + }; + } + + /** + * Export cache contents + */ + exportCache(): string { + const data = { + config: this.config, + entries: Array.from(this.cache.entries()), + statistics: this.getStatistics(), + metadata: { + exportedAt: new Date(), + version: '1.0.0' + } + }; + + return JSON.stringify(data, null, 2); + } + + /** + * Import cache contents + */ + importCache(json: string): void { + try { + const data = JSON.parse(json); + + // Clear existing cache + this.clear(); + + // Import entries + for (const [key, entry] of data.entries) { + // Convert dates + entry.timestamp = new Date(entry.timestamp); + entry.expiresAt = new Date(entry.expiresAt); + + // Skip expired entries + if (new Date() > entry.expiresAt) continue; + + this.cache.set(key, entry); + this.accessOrder.push(key); + this.currentSize += entry.size; + } + + log.info(`Imported ${this.cache.size} cache entries`); + } catch (error) { + log.error(`Failed to import cache: ${error}`); + throw error; + } + } + + /** + * Shutdown cache + */ + shutdown(): void { + this.stopAutoCleanup(); + this.clear(); + log.info('Cache shutdown complete'); + } +} + +// Export singleton instance +export const toolResponseCache = new ToolResponseCache(); \ No newline at end of file diff --git a/apps/server/src/services/llm/tools/tool_timeout_enforcer.ts b/apps/server/src/services/llm/tools/tool_timeout_enforcer.ts new file mode 100644 index 000000000..848bded82 --- /dev/null +++ b/apps/server/src/services/llm/tools/tool_timeout_enforcer.ts @@ -0,0 +1,329 @@ +/** + * Tool Timeout Enforcer + * + * Implements timeout enforcement for tool executions with configurable timeouts + * per tool type, graceful cleanup, and Promise.race pattern for detection. + */ + +import log from '../../log.js'; +import type { ToolHandler } from './tool_interfaces.js'; + +/** + * Timeout configuration per tool type + */ +export interface TimeoutConfig { + /** Timeout for search operations in milliseconds */ + search: number; + /** Timeout for create/update operations in milliseconds */ + mutation: number; + /** Timeout for script execution in milliseconds */ + script: number; + /** Default timeout for unspecified tools in milliseconds */ + default: number; +} + +/** + * Tool execution result with timeout metadata + */ +export interface TimeoutResult { + success: boolean; + result?: T; + error?: Error; + timedOut: boolean; + executionTime: number; + toolName: string; +} + +/** + * Tool categories for timeout assignment + */ +export enum ToolCategory { + SEARCH = 'search', + MUTATION = 'mutation', + SCRIPT = 'script', + READ = 'read', + DEFAULT = 'default' +} + +/** + * Default timeout configuration + */ +const DEFAULT_TIMEOUTS: TimeoutConfig = { + search: 5000, // 5 seconds for search operations + mutation: 3000, // 3 seconds for create/update operations + script: 10000, // 10 seconds for script execution + default: 5000 // 5 seconds default +}; + +/** + * Tool timeout enforcer class + */ +export class ToolTimeoutEnforcer { + private timeouts: TimeoutConfig; + private executionStats: Map; + private activeExecutions: Map; + + constructor(timeoutConfig?: Partial) { + this.timeouts = { ...DEFAULT_TIMEOUTS, ...timeoutConfig }; + this.executionStats = new Map(); + this.activeExecutions = new Map(); + } + + /** + * Categorize tool based on its name + */ + private categorizeeTool(toolName: string): ToolCategory { + const name = toolName.toLowerCase(); + + // Search tools + if (name.includes('search') || name.includes('find') || name.includes('query')) { + return ToolCategory.SEARCH; + } + + // Mutation tools + if (name.includes('create') || name.includes('update') || name.includes('delete') || + name.includes('modify') || name.includes('save')) { + return ToolCategory.MUTATION; + } + + // Script tools + if (name.includes('script') || name.includes('execute') || name.includes('eval')) { + return ToolCategory.SCRIPT; + } + + // Read tools + if (name.includes('read') || name.includes('get') || name.includes('fetch')) { + return ToolCategory.READ; + } + + return ToolCategory.DEFAULT; + } + + /** + * Get timeout for a specific tool + */ + private getToolTimeout(toolName: string): number { + const category = this.categorizeeTool(toolName); + + switch (category) { + case ToolCategory.SEARCH: + return this.timeouts.search; + case ToolCategory.MUTATION: + return this.timeouts.mutation; + case ToolCategory.SCRIPT: + return this.timeouts.script; + case ToolCategory.READ: + return this.timeouts.search; // Use search timeout for read operations + default: + return this.timeouts.default; + } + } + + /** + * Execute a tool with timeout enforcement + */ + async executeWithTimeout( + toolName: string, + executeFn: () => Promise, + customTimeout?: number + ): Promise> { + const timeout = customTimeout || this.getToolTimeout(toolName); + const startTime = Date.now(); + const executionId = `${toolName}_${startTime}_${Math.random()}`; + + // Create abort controller for cleanup + const abortController = new AbortController(); + this.activeExecutions.set(executionId, abortController); + + log.info(`Executing tool '${toolName}' with timeout ${timeout}ms`); + + try { + // Create timeout promise + const timeoutPromise = new Promise((_, reject) => { + const timer = setTimeout(() => { + abortController.abort(); + reject(new Error(`Tool '${toolName}' execution timed out after ${timeout}ms`)); + }, timeout); + + // Clean up timer if aborted + abortController.signal.addEventListener('abort', () => clearTimeout(timer)); + }); + + // Race between execution and timeout + const result = await Promise.race([ + executeFn(), + timeoutPromise + ]); + + const executionTime = Date.now() - startTime; + + // Update statistics + this.updateStats(toolName, false, executionTime); + + log.info(`Tool '${toolName}' completed successfully in ${executionTime}ms`); + + return { + success: true, + result, + timedOut: false, + executionTime, + toolName + }; + + } catch (error) { + const executionTime = Date.now() - startTime; + const timedOut = executionTime >= timeout - 50; // Allow 50ms buffer + + // Update statistics + this.updateStats(toolName, timedOut, executionTime); + + if (timedOut) { + log.error(`Tool '${toolName}' timed out after ${executionTime}ms`); + } else { + log.error(`Tool '${toolName}' failed after ${executionTime}ms: ${error}`); + } + + return { + success: false, + error: error as Error, + timedOut, + executionTime, + toolName + }; + + } finally { + // Clean up + this.activeExecutions.delete(executionId); + if (!abortController.signal.aborted) { + abortController.abort(); + } + } + } + + /** + * Execute multiple tools with timeout enforcement + */ + async executeBatchWithTimeout( + executions: Array<{ + toolName: string; + executeFn: () => Promise; + customTimeout?: number; + }> + ): Promise[]> { + return Promise.all( + executions.map(({ toolName, executeFn, customTimeout }) => + this.executeWithTimeout(toolName, executeFn, customTimeout) + ) + ); + } + + /** + * Wrap a tool handler with timeout enforcement + */ + wrapToolHandler(handler: ToolHandler, customTimeout?: number): ToolHandler { + const toolName = handler.definition.function.name; + + return { + definition: handler.definition, + execute: async (args: Record) => { + const result = await this.executeWithTimeout( + toolName, + () => handler.execute(args), + customTimeout + ); + + if (!result.success) { + if (result.timedOut) { + throw new Error(`Tool execution timed out after ${result.executionTime}ms`); + } + throw result.error; + } + + return result.result!; + } + }; + } + + /** + * Update execution statistics + */ + private updateStats(toolName: string, timedOut: boolean, executionTime: number): void { + const current = this.executionStats.get(toolName) || { + total: 0, + timeouts: 0, + avgTime: 0 + }; + + const newTotal = current.total + 1; + const newTimeouts = current.timeouts + (timedOut ? 1 : 0); + const newAvgTime = (current.avgTime * current.total + executionTime) / newTotal; + + this.executionStats.set(toolName, { + total: newTotal, + timeouts: newTimeouts, + avgTime: newAvgTime + }); + } + + /** + * Get execution statistics for a tool + */ + getToolStats(toolName: string) { + return this.executionStats.get(toolName); + } + + /** + * Get all execution statistics + */ + getAllStats() { + return Object.fromEntries(this.executionStats); + } + + /** + * Clear statistics + */ + clearStats(): void { + this.executionStats.clear(); + } + + /** + * Abort all active executions + */ + abortAll(): void { + log.info(`Aborting ${this.activeExecutions.size} active tool executions`); + + for (const [id, controller] of this.activeExecutions) { + controller.abort(); + } + + this.activeExecutions.clear(); + } + + /** + * Get timeout configuration + */ + getTimeouts(): TimeoutConfig { + return { ...this.timeouts }; + } + + /** + * Update timeout configuration + */ + updateTimeouts(config: Partial): void { + this.timeouts = { ...this.timeouts, ...config }; + log.info(`Updated timeout configuration: ${JSON.stringify(this.timeouts)}`); + } + + /** + * Check if a tool has high timeout rate + */ + hasHighTimeoutRate(toolName: string, threshold: number = 0.5): boolean { + const stats = this.executionStats.get(toolName); + if (!stats || stats.total === 0) return false; + + return (stats.timeouts / stats.total) > threshold; + } +} + +// Export singleton instance +export const toolTimeoutEnforcer = new ToolTimeoutEnforcer(); \ No newline at end of file