feat(llm): try to coerce the LLM some more for tool calling

This commit is contained in:
perfectra1n
2025-08-09 14:19:30 -07:00
parent d38ca72e08
commit ac415c1007
13 changed files with 4255 additions and 8 deletions

View File

@@ -17,6 +17,16 @@ export interface ToolExecutionOptions {
enableFeedback?: boolean;
enableErrorRecovery?: boolean;
timeout?: number;
/** Maximum parallel executions (default: 3) */
maxConcurrency?: number;
/** Enable dependency analysis for parallel execution (default: true) */
analyzeDependencies?: boolean;
/** Provider for tool execution */
provider?: string;
/** Custom timeout per tool in ms */
customTimeouts?: Map<string, number>;
/** Enable caching for read operations */
enableCache?: boolean;
onPreview?: (plan: ToolExecutionPlan) => Promise<ToolApproval>;
onProgress?: (executionId: string, progress: ToolExecutionProgress) => void;
onStep?: (executionId: string, step: any) => void;

View File

@@ -6,6 +6,9 @@ import type { Message } from "../../ai_interface.js";
import { toolPreviewManager } from "../../tools/tool_preview.js";
import { toolFeedbackManager } from "../../tools/tool_feedback.js";
import { toolErrorRecoveryManager } from "../../tools/tool_error_recovery.js";
import { toolTimeoutEnforcer } from "../../tools/tool_timeout_enforcer.js";
import { parameterCoercer } from "../../tools/parameter_coercer.js";
import { toolExecutionMonitor } from "../../monitoring/tool_execution_monitor.js";
/**
* Handles the execution of LLM tools

View File

@@ -0,0 +1,478 @@
/**
* Provider Health Monitor
*
* Monitors health status of LLM providers with periodic checks,
* automatic disabling after failures, and event emissions.
*/
import { EventEmitter } from 'events';
import log from '../../log.js';
import type { AIService } from '../ai_interface.js';
import type { ProviderType } from '../providers/provider_factory.js';
/**
* Provider health status
*/
export interface ProviderHealth {
provider: string;
healthy: boolean;
lastChecked: Date;
lastSuccessful?: Date;
consecutiveFailures: number;
totalChecks: number;
totalFailures: number;
averageLatency: number;
lastError?: string;
disabled: boolean;
disabledAt?: Date;
disabledReason?: string;
}
/**
* Health check result
*/
interface HealthCheckResult {
success: boolean;
latency: number;
error?: string;
tokensUsed?: number;
}
/**
* Health monitor configuration
*/
export interface HealthMonitorConfig {
/** Check interval in milliseconds (default: 60000) */
checkInterval: number;
/** Number of consecutive failures before disabling (default: 3) */
failureThreshold: number;
/** Timeout for health checks in milliseconds (default: 5000) */
checkTimeout: number;
/** Enable automatic recovery attempts (default: true) */
autoRecover: boolean;
/** Recovery check interval in milliseconds (default: 300000) */
recoveryInterval: number;
/** Minimum time between checks in milliseconds (default: 30000) */
minCheckInterval: number;
}
/**
* Default configuration
*/
const DEFAULT_CONFIG: HealthMonitorConfig = {
checkInterval: 60000, // 1 minute
failureThreshold: 3,
checkTimeout: 5000, // 5 seconds
autoRecover: true,
recoveryInterval: 300000, // 5 minutes
minCheckInterval: 30000 // 30 seconds
};
/**
* Provider health monitor class
*/
export class ProviderHealthMonitor extends EventEmitter {
private config: HealthMonitorConfig;
private providers: Map<string, AIService>;
private healthStatus: Map<string, ProviderHealth>;
private checkTimers: Map<string, NodeJS.Timeout>;
private isMonitoring: boolean;
private lastCheckTime: Map<string, number>;
constructor(config?: Partial<HealthMonitorConfig>) {
super();
this.config = { ...DEFAULT_CONFIG, ...config };
this.providers = new Map();
this.healthStatus = new Map();
this.checkTimers = new Map();
this.isMonitoring = false;
this.lastCheckTime = new Map();
}
/**
* Register a provider for monitoring
*/
registerProvider(name: string, service: AIService): void {
this.providers.set(name, service);
this.healthStatus.set(name, {
provider: name,
healthy: true,
lastChecked: new Date(),
consecutiveFailures: 0,
totalChecks: 0,
totalFailures: 0,
averageLatency: 0,
disabled: false
});
log.info(`Registered provider '${name}' for health monitoring`);
// Start monitoring if not already running
if (!this.isMonitoring) {
this.startMonitoring();
}
}
/**
* Unregister a provider
*/
unregisterProvider(name: string): void {
this.providers.delete(name);
this.healthStatus.delete(name);
const timer = this.checkTimers.get(name);
if (timer) {
clearTimeout(timer);
this.checkTimers.delete(name);
}
log.info(`Unregistered provider '${name}' from health monitoring`);
}
/**
* Start health monitoring
*/
startMonitoring(): void {
if (this.isMonitoring) {
log.info('Health monitoring is already running');
return;
}
this.isMonitoring = true;
log.info('Starting provider health monitoring');
// Schedule initial checks for all providers
for (const provider of this.providers.keys()) {
this.scheduleHealthCheck(provider);
}
this.emit('monitoring:started');
}
/**
* Stop health monitoring
*/
stopMonitoring(): void {
if (!this.isMonitoring) {
return;
}
this.isMonitoring = false;
// Clear all timers
for (const timer of this.checkTimers.values()) {
clearTimeout(timer);
}
this.checkTimers.clear();
log.info('Stopped provider health monitoring');
this.emit('monitoring:stopped');
}
/**
* Schedule a health check for a provider
*/
private scheduleHealthCheck(provider: string, delay?: number): void {
if (!this.isMonitoring) return;
// Clear existing timer
const existingTimer = this.checkTimers.get(provider);
if (existingTimer) {
clearTimeout(existingTimer);
}
// Calculate delay based on provider status
const status = this.healthStatus.get(provider);
const checkDelay = delay || (status?.disabled
? this.config.recoveryInterval
: this.config.checkInterval);
// Schedule the check
const timer = setTimeout(async () => {
await this.performHealthCheck(provider);
// Schedule next check
if (this.isMonitoring) {
this.scheduleHealthCheck(provider);
}
}, checkDelay);
this.checkTimers.set(provider, timer);
}
/**
* Perform a health check for a provider
*/
private async performHealthCheck(provider: string): Promise<HealthCheckResult> {
const service = this.providers.get(provider);
const status = this.healthStatus.get(provider);
if (!service || !status) {
return { success: false, latency: 0, error: 'Provider not found' };
}
// Check if enough time has passed since last check
const lastCheck = this.lastCheckTime.get(provider) || 0;
const now = Date.now();
if (now - lastCheck < this.config.minCheckInterval) {
log.info(`Skipping health check for ${provider}, too soon since last check`);
return { success: true, latency: 0 };
}
this.lastCheckTime.set(provider, now);
log.info(`Performing health check for provider '${provider}'`);
const startTime = Date.now();
try {
// Simple ping test with minimal token usage
const result = await Promise.race([
service.generateChatCompletion(
[{
role: 'user',
content: 'Hi'
}],
{
model: 'default', // Use a default model name
maxTokens: 5,
temperature: 0
}
),
new Promise((_, reject) =>
setTimeout(() => reject(new Error('Health check timeout')),
this.config.checkTimeout)
)
]);
const latency = Date.now() - startTime;
// Update status for successful check
this.updateHealthStatus(provider, {
success: true,
latency,
tokensUsed: (result as any).usage?.totalTokens
});
log.info(`Health check successful for '${provider}' (${latency}ms)`);
return { success: true, latency, tokensUsed: (result as any).usage?.totalTokens };
} catch (error) {
const latency = Date.now() - startTime;
const errorMessage = error instanceof Error ? error.message : String(error);
// Update status for failed check
this.updateHealthStatus(provider, {
success: false,
latency,
error: errorMessage
});
log.error(`Health check failed for '${provider}': ${errorMessage}`);
return { success: false, latency, error: errorMessage };
}
}
/**
* Update health status based on check result
*/
private updateHealthStatus(provider: string, result: HealthCheckResult): void {
const status = this.healthStatus.get(provider);
if (!status) return;
const wasHealthy = status.healthy;
const wasDisabled = status.disabled;
// Update basic stats
status.lastChecked = new Date();
status.totalChecks++;
if (result.success) {
// Successful check
status.healthy = true;
status.lastSuccessful = new Date();
status.consecutiveFailures = 0;
status.lastError = undefined;
// Update average latency
const prevTotal = status.averageLatency * (status.totalChecks - 1);
status.averageLatency = (prevTotal + result.latency) / status.totalChecks;
// Re-enable if was disabled and auto-recover is on
if (status.disabled && this.config.autoRecover) {
status.disabled = false;
status.disabledAt = undefined;
status.disabledReason = undefined;
log.info(`Provider '${provider}' recovered and re-enabled`);
this.emit('provider:recovered', { provider, status });
}
} else {
// Failed check
status.totalFailures++;
status.consecutiveFailures++;
status.lastError = result.error;
// Check if should disable
if (status.consecutiveFailures >= this.config.failureThreshold) {
status.healthy = false;
if (!status.disabled) {
status.disabled = true;
status.disabledAt = new Date();
status.disabledReason = `${status.consecutiveFailures} consecutive failures`;
log.error(`Provider '${provider}' disabled after ${status.consecutiveFailures} failures`);
this.emit('provider:disabled', { provider, status, reason: status.disabledReason });
}
}
}
// Emit status change events
if (wasHealthy !== status.healthy) {
this.emit('provider:health-changed', {
provider,
healthy: status.healthy,
status
});
}
if (wasDisabled !== status.disabled) {
this.emit('provider:status-changed', {
provider,
disabled: status.disabled,
status
});
}
}
/**
* Manually trigger a health check
*/
async checkProvider(provider: string): Promise<HealthCheckResult> {
return this.performHealthCheck(provider);
}
/**
* Check all providers
*/
async checkAllProviders(): Promise<Map<string, HealthCheckResult>> {
const results = new Map<string, HealthCheckResult>();
const checks = Array.from(this.providers.keys()).map(async provider => {
const result = await this.performHealthCheck(provider);
results.set(provider, result);
});
await Promise.all(checks);
return results;
}
/**
* Get health status for a provider
*/
getProviderHealth(provider: string): ProviderHealth | undefined {
return this.healthStatus.get(provider);
}
/**
* Get all health statuses
*/
getAllHealthStatus(): Map<string, ProviderHealth> {
return new Map(this.healthStatus);
}
/**
* Check if a provider is healthy
*/
isProviderHealthy(provider: string): boolean {
const status = this.healthStatus.get(provider);
return status ? status.healthy && !status.disabled : false;
}
/**
* Get healthy providers
*/
getHealthyProviders(): string[] {
return Array.from(this.healthStatus.entries())
.filter(([_, status]) => status.healthy && !status.disabled)
.map(([provider, _]) => provider);
}
/**
* Manually enable a provider
*/
enableProvider(provider: string): void {
const status = this.healthStatus.get(provider);
if (status && status.disabled) {
status.disabled = false;
status.disabledAt = undefined;
status.disabledReason = undefined;
status.consecutiveFailures = 0;
log.info(`Provider '${provider}' manually enabled`);
this.emit('provider:enabled', { provider, status });
// Schedule immediate health check
this.scheduleHealthCheck(provider, 0);
}
}
/**
* Manually disable a provider
*/
disableProvider(provider: string, reason?: string): void {
const status = this.healthStatus.get(provider);
if (status && !status.disabled) {
status.disabled = true;
status.disabledAt = new Date();
status.disabledReason = reason || 'Manually disabled';
status.healthy = false;
log.info(`Provider '${provider}' manually disabled: ${status.disabledReason}`);
this.emit('provider:disabled', { provider, status, reason: status.disabledReason });
}
}
/**
* Reset statistics for a provider
*/
resetProviderStats(provider: string): void {
const status = this.healthStatus.get(provider);
if (status) {
status.totalChecks = 0;
status.totalFailures = 0;
status.averageLatency = 0;
status.consecutiveFailures = 0;
log.info(`Reset statistics for provider '${provider}'`);
}
}
/**
* Get monitoring configuration
*/
getConfig(): HealthMonitorConfig {
return { ...this.config };
}
/**
* Update monitoring configuration
*/
updateConfig(config: Partial<HealthMonitorConfig>): void {
this.config = { ...this.config, ...config };
log.info(`Updated health monitor configuration: ${JSON.stringify(this.config)}`);
// Restart monitoring with new config
if (this.isMonitoring) {
this.stopMonitoring();
this.startMonitoring();
}
}
}
// Export singleton instance
export const providerHealthMonitor = new ProviderHealthMonitor();

View File

@@ -0,0 +1,503 @@
/**
* Tool Execution Monitor
*
* Tracks success/failure rates per tool and provider, calculates reliability scores,
* auto-disables unreliable tools, and provides metrics for dashboards.
*/
import { EventEmitter } from 'events';
import log from '../../log.js';
/**
* Tool execution statistics
*/
export interface ToolExecutionStats {
toolName: string;
provider: string;
totalExecutions: number;
successfulExecutions: number;
failedExecutions: number;
timeoutExecutions: number;
averageExecutionTime: number;
minExecutionTime: number;
maxExecutionTime: number;
lastExecutionTime?: number;
lastExecutionStatus?: 'success' | 'failure' | 'timeout';
lastError?: string;
reliabilityScore: number;
disabled: boolean;
disabledAt?: Date;
disabledReason?: string;
}
/**
* Execution record
*/
export interface ExecutionRecord {
toolName: string;
provider: string;
status: 'success' | 'failure' | 'timeout';
executionTime: number;
timestamp: Date;
error?: string;
inputSize?: number;
outputSize?: number;
}
/**
* Monitor configuration
*/
export interface MonitorConfig {
/** Failure rate threshold for auto-disable (default: 0.5) */
failureRateThreshold: number;
/** Minimum executions before calculating reliability (default: 5) */
minExecutionsForReliability: number;
/** Time window for recent stats in milliseconds (default: 3600000) */
recentStatsWindow: number;
/** Enable auto-disable of unreliable tools (default: true) */
autoDisable: boolean;
/** Cooldown period after disable in milliseconds (default: 300000) */
disableCooldown: number;
}
/**
* Default configuration
*/
const DEFAULT_CONFIG: MonitorConfig = {
failureRateThreshold: 0.5,
minExecutionsForReliability: 5,
recentStatsWindow: 3600000, // 1 hour
autoDisable: true,
disableCooldown: 300000 // 5 minutes
};
/**
* Tool execution monitor class
*/
export class ToolExecutionMonitor extends EventEmitter {
private config: MonitorConfig;
private stats: Map<string, ToolExecutionStats>;
private recentExecutions: ExecutionRecord[];
private disabledTools: Set<string>;
constructor(config?: Partial<MonitorConfig>) {
super();
this.config = { ...DEFAULT_CONFIG, ...config };
this.stats = new Map();
this.recentExecutions = [];
this.disabledTools = new Set();
}
/**
* Record a tool execution
*/
recordExecution(record: ExecutionRecord): void {
const key = this.getStatsKey(record.toolName, record.provider);
// Update or create stats
let stats = this.stats.get(key);
if (!stats) {
stats = this.createEmptyStats(record.toolName, record.provider);
this.stats.set(key, stats);
}
// Update counters
stats.totalExecutions++;
switch (record.status) {
case 'success':
stats.successfulExecutions++;
break;
case 'failure':
stats.failedExecutions++;
break;
case 'timeout':
stats.timeoutExecutions++;
break;
}
// Update timing statistics
this.updateTimingStats(stats, record.executionTime);
// Update last execution info
stats.lastExecutionTime = record.executionTime;
stats.lastExecutionStatus = record.status;
stats.lastError = record.error;
// Calculate reliability score
stats.reliabilityScore = this.calculateReliabilityScore(stats);
// Add to recent executions
this.recentExecutions.push(record);
this.pruneRecentExecutions();
// Check if tool should be auto-disabled
if (this.config.autoDisable && this.shouldAutoDisable(stats)) {
this.disableTool(record.toolName, record.provider, 'High failure rate');
}
// Emit events
this.emit('execution:recorded', record);
if (record.status === 'failure') {
this.emit('execution:failed', record);
} else if (record.status === 'timeout') {
this.emit('execution:timeout', record);
}
// Log if reliability is concerning
if (stats.reliabilityScore < 0.5 && stats.totalExecutions >= this.config.minExecutionsForReliability) {
log.info(`Tool '${record.toolName}' has low reliability score: ${stats.reliabilityScore.toFixed(2)}`);
}
}
/**
* Update timing statistics
*/
private updateTimingStats(stats: ToolExecutionStats, executionTime: number): void {
const prevAvg = stats.averageExecutionTime;
const prevCount = stats.totalExecutions - 1;
// Update average
stats.averageExecutionTime = prevCount === 0
? executionTime
: (prevAvg * prevCount + executionTime) / stats.totalExecutions;
// Update min/max
if (stats.minExecutionTime === 0 || executionTime < stats.minExecutionTime) {
stats.minExecutionTime = executionTime;
}
if (executionTime > stats.maxExecutionTime) {
stats.maxExecutionTime = executionTime;
}
}
/**
* Calculate reliability score (0-1)
*/
private calculateReliabilityScore(stats: ToolExecutionStats): number {
if (stats.totalExecutions === 0) return 1;
// Weight factors
const successWeight = 0.7;
const timeoutWeight = 0.2;
const consistencyWeight = 0.1;
// Success rate
const successRate = stats.successfulExecutions / stats.totalExecutions;
// Timeout penalty
const timeoutRate = stats.timeoutExecutions / stats.totalExecutions;
const timeoutScore = 1 - timeoutRate;
// Consistency score (based on execution time variance)
let consistencyScore = 1;
if (stats.totalExecutions > 1 && stats.averageExecutionTime > 0) {
const variance = (stats.maxExecutionTime - stats.minExecutionTime) / stats.averageExecutionTime;
consistencyScore = Math.max(0, 1 - variance / 10); // Normalize variance
}
// Calculate weighted score
const score =
successRate * successWeight +
timeoutScore * timeoutWeight +
consistencyScore * consistencyWeight;
return Math.min(1, Math.max(0, score));
}
/**
* Check if tool should be auto-disabled
*/
private shouldAutoDisable(stats: ToolExecutionStats): boolean {
// Don't disable if already disabled
if (stats.disabled) return false;
// Need minimum executions
if (stats.totalExecutions < this.config.minExecutionsForReliability) {
return false;
}
// Check failure rate
const failureRate = (stats.failedExecutions + stats.timeoutExecutions) / stats.totalExecutions;
return failureRate > this.config.failureRateThreshold;
}
/**
* Disable a tool
*/
disableTool(toolName: string, provider: string, reason: string): void {
const key = this.getStatsKey(toolName, provider);
const stats = this.stats.get(key);
if (!stats || stats.disabled) return;
stats.disabled = true;
stats.disabledAt = new Date();
stats.disabledReason = reason;
this.disabledTools.add(key);
log.error(`Tool '${toolName}' disabled for provider '${provider}': ${reason}`);
this.emit('tool:disabled', { toolName, provider, reason, stats });
// Schedule re-enable check
if (this.config.disableCooldown > 0) {
setTimeout(() => {
this.checkReEnableTool(toolName, provider);
}, this.config.disableCooldown);
}
}
/**
* Check if a tool can be re-enabled
*/
private checkReEnableTool(toolName: string, provider: string): void {
const key = this.getStatsKey(toolName, provider);
const stats = this.stats.get(key);
if (!stats || !stats.disabled) return;
// Calculate recent success rate
const recentExecutions = this.getRecentExecutions(toolName, provider);
if (recentExecutions.length === 0) {
// No recent executions, re-enable for retry
this.enableTool(toolName, provider);
return;
}
const recentSuccesses = recentExecutions.filter(e => e.status === 'success').length;
const recentSuccessRate = recentSuccesses / recentExecutions.length;
// Re-enable if recent performance is good
if (recentSuccessRate > 0.7) {
this.enableTool(toolName, provider);
}
}
/**
* Enable a tool
*/
enableTool(toolName: string, provider: string): void {
const key = this.getStatsKey(toolName, provider);
const stats = this.stats.get(key);
if (!stats || !stats.disabled) return;
stats.disabled = false;
stats.disabledAt = undefined;
stats.disabledReason = undefined;
this.disabledTools.delete(key);
log.info(`Tool '${toolName}' re-enabled for provider '${provider}'`);
this.emit('tool:enabled', { toolName, provider, stats });
}
/**
* Get stats for a tool
*/
getToolStats(toolName: string, provider: string): ToolExecutionStats | undefined {
return this.stats.get(this.getStatsKey(toolName, provider));
}
/**
* Get all stats
*/
getAllStats(): Map<string, ToolExecutionStats> {
return new Map(this.stats);
}
/**
* Get stats by provider
*/
getStatsByProvider(provider: string): ToolExecutionStats[] {
return Array.from(this.stats.values()).filter(s => s.provider === provider);
}
/**
* Get stats by tool
*/
getStatsByTool(toolName: string): ToolExecutionStats[] {
return Array.from(this.stats.values()).filter(s => s.toolName === toolName);
}
/**
* Get recent executions for a tool
*/
getRecentExecutions(toolName: string, provider: string): ExecutionRecord[] {
const cutoff = Date.now() - this.config.recentStatsWindow;
return this.recentExecutions.filter(e =>
e.toolName === toolName &&
e.provider === provider &&
e.timestamp.getTime() > cutoff
);
}
/**
* Get metrics for dashboard
*/
getDashboardMetrics(): {
totalTools: number;
activeTools: number;
disabledTools: number;
overallReliability: number;
topPerformers: ToolExecutionStats[];
bottomPerformers: ToolExecutionStats[];
recentFailures: ExecutionRecord[];
} {
const allStats = Array.from(this.stats.values());
const activeStats = allStats.filter(s => !s.disabled);
// Calculate overall reliability
const overallReliability = activeStats.length > 0
? activeStats.reduce((sum, s) => sum + s.reliabilityScore, 0) / activeStats.length
: 1;
// Sort by reliability
const sorted = [...allStats].sort((a, b) => b.reliabilityScore - a.reliabilityScore);
// Get recent failures
const recentFailures = this.recentExecutions
.filter(e => e.status !== 'success')
.slice(-10);
return {
totalTools: allStats.length,
activeTools: activeStats.length,
disabledTools: this.disabledTools.size,
overallReliability,
topPerformers: sorted.slice(0, 5),
bottomPerformers: sorted.slice(-5).reverse(),
recentFailures
};
}
/**
* Check if a tool is disabled
*/
isToolDisabled(toolName: string, provider: string): boolean {
return this.disabledTools.has(this.getStatsKey(toolName, provider));
}
/**
* Reset stats for a tool
*/
resetToolStats(toolName: string, provider: string): void {
const key = this.getStatsKey(toolName, provider);
this.stats.delete(key);
this.disabledTools.delete(key);
// Remove from recent executions
this.recentExecutions = this.recentExecutions.filter(e =>
!(e.toolName === toolName && e.provider === provider)
);
log.info(`Reset stats for tool '${toolName}' with provider '${provider}'`);
}
/**
* Reset all statistics
*/
resetAllStats(): void {
this.stats.clear();
this.recentExecutions = [];
this.disabledTools.clear();
log.info('Reset all tool execution statistics');
}
/**
* Prune old recent executions
*/
private pruneRecentExecutions(): void {
const cutoff = Date.now() - this.config.recentStatsWindow;
this.recentExecutions = this.recentExecutions.filter(e =>
e.timestamp.getTime() > cutoff
);
}
/**
* Create empty stats object
*/
private createEmptyStats(toolName: string, provider: string): ToolExecutionStats {
return {
toolName,
provider,
totalExecutions: 0,
successfulExecutions: 0,
failedExecutions: 0,
timeoutExecutions: 0,
averageExecutionTime: 0,
minExecutionTime: 0,
maxExecutionTime: 0,
reliabilityScore: 1,
disabled: false
};
}
/**
* Get stats key
*/
private getStatsKey(toolName: string, provider: string): string {
return `${provider}:${toolName}`;
}
/**
* Export statistics to JSON
*/
exportStats(): string {
return JSON.stringify({
stats: Array.from(this.stats.entries()),
recentExecutions: this.recentExecutions,
disabledTools: Array.from(this.disabledTools),
config: this.config
}, null, 2);
}
/**
* Import statistics from JSON
*/
importStats(json: string): void {
try {
const data = JSON.parse(json);
// Restore stats
this.stats.clear();
for (const [key, value] of data.stats) {
this.stats.set(key, value);
}
// Restore recent executions with date conversion
this.recentExecutions = data.recentExecutions.map((e: any) => ({
...e,
timestamp: new Date(e.timestamp)
}));
// Restore disabled tools
this.disabledTools = new Set(data.disabledTools);
log.info('Imported tool execution statistics');
} catch (error) {
log.error(`Failed to import statistics: ${error}`);
throw error;
}
}
/**
* Get configuration
*/
getConfig(): MonitorConfig {
return { ...this.config };
}
/**
* Update configuration
*/
updateConfig(config: Partial<MonitorConfig>): void {
this.config = { ...this.config, ...config };
log.info(`Updated tool execution monitor configuration: ${JSON.stringify(this.config)}`);
}
}
// Export singleton instance
export const toolExecutionMonitor = new ToolExecutionMonitor();

View File

@@ -0,0 +1,228 @@
# Phase 2: Simplification Implementation
## Overview
This document describes the implementation of Phase 2 of the LLM improvement plan, focusing on architectural simplification, centralized configuration, and improved logging.
## Implemented Components
### Phase 2.1: Pipeline Architecture Simplification
**File:** `simplified_pipeline.ts` (396 lines)
The original 986-line pipeline with 9 stages has been reduced to 4 essential stages:
1. **Message Preparation** - Combines formatting, context enrichment, and system prompt injection
2. **LLM Execution** - Handles provider selection and API calls
3. **Tool Handling** - Manages tool parsing, execution, and follow-up calls
4. **Response Processing** - Formats responses and handles streaming
Key improvements:
- Reduced code complexity by ~60%
- Removed unnecessary abstractions
- Consolidated duplicate logic
- Clearer separation of concerns
### Phase 2.2: Configuration Management
#### Configuration Service
**File:** `configuration_service.ts` (354 lines)
Centralizes all LLM configuration:
- Single source of truth for all settings
- Type-safe configuration access
- Validation at startup
- Cache with automatic refresh
- No more scattered `options.getOption()` calls
Configuration categories:
- Provider settings (API keys, endpoints, models)
- Default parameters (temperature, tokens, system prompt)
- Tool configuration (iterations, timeout, parallel execution)
- Streaming settings (enabled, chunk size, flush interval)
- Debug configuration (log level, metrics, tracing)
- Limits (message length, conversation length, rate limiting)
#### Model Registry
**File:** `model_registry.ts` (474 lines)
Manages model capabilities and metadata:
- Built-in model definitions for OpenAI, Anthropic, and Ollama
- Model capabilities (tools, streaming, vision, JSON mode)
- Cost tracking (per 1K tokens)
- Performance characteristics (latency, throughput, reliability)
- Intelligent model selection based on use case
- Custom model registration for Ollama
### Phase 2.3: Logging Improvements
#### Logging Service
**File:** `logging_service.ts` (378 lines)
Structured logging with:
- Proper log levels (ERROR, WARN, INFO, DEBUG)
- Request ID tracking for tracing
- Conditional debug logging (disabled in production)
- Log buffering for debugging
- Performance timers
- Contextual logging with metadata
#### Debug Cleanup Script
**File:** `cleanup_debug_logs.ts` (198 lines)
Utility to clean up debug statements:
- Finds `log.info("[DEBUG]")` patterns
- Converts to proper debug level
- Reports on verbose logging
- Dry-run mode for safety
### Integration Layer
#### Pipeline Adapter
**File:** `pipeline_adapter.ts` (140 lines)
Provides backward compatibility:
- Maintains existing `ChatPipeline` interface
- Uses simplified pipeline underneath
- Gradual migration path
- Feature flag support
## Migration Guide
### Step 1: Update Imports
```typescript
// Old
import { ChatPipeline } from "../pipeline/chat_pipeline.js";
// New
import { ChatPipeline } from "../pipeline/pipeline_adapter.js";
```
### Step 2: Initialize Configuration
```typescript
// On startup
await configurationService.initialize();
```
### Step 3: Use Structured Logging
```typescript
// Old
log.info(`[DEBUG] Processing request for user ${userId}`);
// New
const logger = loggingService.withRequestId(requestId);
logger.debug('Processing request', { userId });
```
### Step 4: Access Configuration
```typescript
// Old
const model = options.getOption('openaiDefaultModel');
// New
const model = configurationService.getProviderConfig().openai?.defaultModel;
```
## Benefits Achieved
### Code Simplification
- **60% reduction** in pipeline code (986 → 396 lines)
- **9 stages → 4 stages** for easier understanding
- Removed unnecessary abstractions
### Better Configuration
- **Single source of truth** for all configuration
- **Type-safe** access with IntelliSense support
- **Validation** catches errors at startup
- **Centralized** management reduces duplication
### Improved Logging
- **Structured logs** with consistent format
- **Request tracing** with unique IDs
- **Performance metrics** built-in
- **Production-ready** with debug statements removed
### Maintainability
- **Clear separation** of concerns
- **Testable** components with dependency injection
- **Gradual migration** path with adapter
- **Well-documented** interfaces
## Testing
### Unit Tests
**File:** `simplified_pipeline.spec.ts`
Comprehensive test coverage for:
- Simple chat flows
- Tool execution
- Streaming responses
- Error handling
- Metrics tracking
- Context enrichment
### Running Tests
```bash
# Run all pipeline tests
pnpm nx test server --testPathPattern=pipeline
# Run specific test file
pnpm nx test server --testFile=simplified_pipeline.spec.ts
```
## Performance Impact
### Reduced Overhead
- Fewer function calls in hot path
- Less object creation
- Simplified async flow
### Better Resource Usage
- Configuration caching reduces database queries
- Streamlined logging reduces I/O
- Efficient metric collection
## Next Steps
### Immediate Actions
1. Deploy with feature flag enabled
2. Monitor performance metrics
3. Gather feedback from users
### Future Improvements
1. Implement remaining phases from improvement plan
2. Add telemetry for production monitoring
3. Create migration tools for existing configurations
4. Build admin UI for configuration management
## Environment Variables
```bash
# Enable simplified pipeline (default: true)
USE_SIMPLIFIED_PIPELINE=true
# Enable debug logging
LLM_DEBUG_ENABLED=true
# Set log level (error, warn, info, debug)
LLM_LOG_LEVEL=info
```
## Rollback Plan
If issues are encountered:
1. **Quick rollback:** Set `USE_SIMPLIFIED_PIPELINE=false`
2. **Revert imports:** Change back to original `chat_pipeline.js`
3. **Monitor logs:** Check for any errors or warnings
The adapter ensures backward compatibility, making rollback seamless.
## Conclusion
Phase 2 successfully simplifies the LLM pipeline architecture while maintaining all functionality. The implementation provides:
- **Cleaner code** that's easier to understand and maintain
- **Better configuration** management with validation
- **Improved logging** for debugging and monitoring
- **Backward compatibility** for gradual migration
The simplified architecture provides a solid foundation for future enhancements and makes the codebase more accessible to new contributors.

View File

@@ -0,0 +1,563 @@
/**
* Provider Edge Case Handler
*
* Handles provider-specific edge cases and quirks for OpenAI, Anthropic, and Ollama,
* including special character fixes, object flattening, and context limit handling.
*/
import log from '../../log.js';
import type { Tool, ToolParameter } from '../tools/tool_interfaces.js';
/**
* Edge case fix result
*/
export interface EdgeCaseFixResult {
fixed: boolean;
tool?: Tool;
warnings: string[];
modifications: string[];
}
/**
* Provider-specific configuration
*/
interface ProviderConfig {
maxFunctionNameLength: number;
maxDescriptionLength: number;
maxDepth: number;
maxProperties: number;
allowSpecialChars: boolean;
requireArrays: boolean;
supportsComplexTypes: boolean;
}
/**
* Provider configurations
*/
const PROVIDER_CONFIGS: Record<string, ProviderConfig> = {
openai: {
maxFunctionNameLength: 64,
maxDescriptionLength: 1024,
maxDepth: 5,
maxProperties: 50,
allowSpecialChars: false,
requireArrays: false,
supportsComplexTypes: true
},
anthropic: {
maxFunctionNameLength: 64,
maxDescriptionLength: 1024,
maxDepth: 4,
maxProperties: 30,
allowSpecialChars: true,
requireArrays: true,
supportsComplexTypes: true
},
ollama: {
maxFunctionNameLength: 50,
maxDescriptionLength: 500,
maxDepth: 3,
maxProperties: 20,
allowSpecialChars: false,
requireArrays: false,
supportsComplexTypes: false
}
};
/**
* Edge case handler class
*/
export class EdgeCaseHandler {
/**
* Fix tool for provider-specific edge cases
*/
fixToolForProvider(tool: Tool, provider: string): EdgeCaseFixResult {
const config = PROVIDER_CONFIGS[provider] || PROVIDER_CONFIGS.openai;
const warnings: string[] = [];
const modifications: string[] = [];
// Deep clone the tool
let fixedTool = JSON.parse(JSON.stringify(tool)) as Tool;
let wasFixed = false;
// Apply provider-specific fixes
switch (provider) {
case 'openai':
const openaiResult = this.fixOpenAIEdgeCases(fixedTool, config);
fixedTool = openaiResult.tool;
warnings.push(...openaiResult.warnings);
modifications.push(...openaiResult.modifications);
wasFixed = openaiResult.fixed;
break;
case 'anthropic':
const anthropicResult = this.fixAnthropicEdgeCases(fixedTool, config);
fixedTool = anthropicResult.tool;
warnings.push(...anthropicResult.warnings);
modifications.push(...anthropicResult.modifications);
wasFixed = anthropicResult.fixed;
break;
case 'ollama':
const ollamaResult = this.fixOllamaEdgeCases(fixedTool, config);
fixedTool = ollamaResult.tool;
warnings.push(...ollamaResult.warnings);
modifications.push(...ollamaResult.modifications);
wasFixed = ollamaResult.fixed;
break;
default:
// Apply generic fixes
const genericResult = this.applyGenericFixes(fixedTool, config);
fixedTool = genericResult.tool;
warnings.push(...genericResult.warnings);
modifications.push(...genericResult.modifications);
wasFixed = genericResult.fixed;
}
return {
fixed: wasFixed,
tool: wasFixed ? fixedTool : undefined,
warnings,
modifications
};
}
/**
* Fix OpenAI-specific edge cases
*/
private fixOpenAIEdgeCases(
tool: Tool,
config: ProviderConfig
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
const warnings: string[] = [];
const modifications: string[] = [];
let fixed = false;
// Fix special characters in function name
if (!config.allowSpecialChars && /[^a-zA-Z0-9_]/.test(tool.function.name)) {
const oldName = tool.function.name;
tool.function.name = tool.function.name.replace(/[^a-zA-Z0-9_]/g, '_');
modifications.push(`Replaced special characters in function name: ${oldName}${tool.function.name}`);
fixed = true;
}
// Fix hyphens (OpenAI prefers underscores)
if (tool.function.name.includes('-')) {
const oldName = tool.function.name;
tool.function.name = tool.function.name.replace(/-/g, '_');
modifications.push(`Replaced hyphens with underscores: ${oldName}${tool.function.name}`);
fixed = true;
}
// Flatten deep objects if necessary
if (tool.function.parameters.properties) {
const flattenResult = this.flattenDeepObjects(
tool.function.parameters.properties,
config.maxDepth
);
if (flattenResult.flattened) {
tool.function.parameters.properties = flattenResult.properties;
modifications.push('Flattened deep nested objects');
warnings.push('Some nested properties were flattened for OpenAI compatibility');
fixed = true;
}
}
// Handle overly complex parameter structures
const paramCount = Object.keys(tool.function.parameters.properties || {}).length;
if (paramCount > config.maxProperties) {
warnings.push(`Tool has ${paramCount} properties, exceeding OpenAI recommended limit of ${config.maxProperties}`);
// Group related parameters if possible
const grouped = this.groupRelatedParameters(tool.function.parameters.properties);
if (grouped.grouped) {
tool.function.parameters.properties = grouped.properties;
modifications.push('Grouped related parameters to reduce complexity');
fixed = true;
}
}
// Fix enum values with special characters
this.fixEnumValues(tool.function.parameters.properties);
return { tool, fixed, warnings, modifications };
}
/**
* Fix Anthropic-specific edge cases
*/
private fixAnthropicEdgeCases(
tool: Tool,
config: ProviderConfig
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
const warnings: string[] = [];
const modifications: string[] = [];
let fixed = false;
// Ensure required array is not empty
if (!tool.function.parameters.required || tool.function.parameters.required.length === 0) {
const properties = Object.keys(tool.function.parameters.properties || {});
if (properties.length > 0) {
// Add at least one property to required
tool.function.parameters.required = [properties[0]];
modifications.push(`Added '${properties[0]}' to required array for Anthropic compatibility`);
fixed = true;
} else {
// Add a dummy optional parameter if no properties exist
tool.function.parameters.properties = {
_placeholder: {
type: 'string',
description: 'Optional placeholder parameter',
default: ''
}
};
tool.function.parameters.required = [];
modifications.push('Added placeholder parameter for Anthropic compatibility');
fixed = true;
}
}
// Truncate overly long descriptions
if (tool.function.description.length > config.maxDescriptionLength) {
tool.function.description = tool.function.description.substring(0, config.maxDescriptionLength - 3) + '...';
modifications.push('Truncated description to meet Anthropic length limits');
fixed = true;
}
// Ensure all parameters have descriptions
for (const [key, param] of Object.entries(tool.function.parameters.properties || {})) {
if (!param.description) {
param.description = `Parameter ${key}`;
modifications.push(`Added missing description for parameter '${key}'`);
fixed = true;
}
}
// Handle complex nested structures
const complexity = this.calculateComplexity(tool.function.parameters);
if (complexity > 15) {
warnings.push('Tool parameters are very complex for Anthropic, consider simplifying');
}
return { tool, fixed, warnings, modifications };
}
/**
* Fix Ollama-specific edge cases
*/
private fixOllamaEdgeCases(
tool: Tool,
config: ProviderConfig
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
const warnings: string[] = [];
const modifications: string[] = [];
let fixed = false;
// Limit parameter count for local models
const properties = tool.function.parameters.properties || {};
const paramCount = Object.keys(properties).length;
if (paramCount > config.maxProperties) {
// Keep only the most important parameters
const required = tool.function.parameters.required || [];
const important = new Set(required);
const kept: Record<string, ToolParameter> = {};
// Keep required parameters first
for (const key of required) {
if (properties[key]) {
kept[key] = properties[key];
}
}
// Add optional parameters up to limit
for (const [key, param] of Object.entries(properties)) {
if (!important.has(key) && Object.keys(kept).length < config.maxProperties) {
kept[key] = param;
}
}
tool.function.parameters.properties = kept;
modifications.push(`Reduced parameters from ${paramCount} to ${Object.keys(kept).length} for Ollama`);
warnings.push('Some optional parameters were removed for local model compatibility');
fixed = true;
}
// Simplify complex types
if (!config.supportsComplexTypes) {
const simplified = this.simplifyComplexTypes(tool.function.parameters.properties);
if (simplified.simplified) {
tool.function.parameters.properties = simplified.properties;
modifications.push('Simplified complex types for local model compatibility');
fixed = true;
}
}
// Shorten descriptions for context limits
for (const [key, param] of Object.entries(tool.function.parameters.properties || {})) {
if (param.description && param.description.length > 100) {
param.description = param.description.substring(0, 97) + '...';
modifications.push(`Shortened description for parameter '${key}'`);
fixed = true;
}
}
// Remove deeply nested structures
if (config.maxDepth < 4) {
const flattened = this.flattenDeepObjects(
tool.function.parameters.properties,
config.maxDepth
);
if (flattened.flattened) {
tool.function.parameters.properties = flattened.properties;
modifications.push('Flattened nested structures for local model');
warnings.push('Nested objects were flattened for better local model performance');
fixed = true;
}
}
return { tool, fixed, warnings, modifications };
}
/**
* Apply generic fixes for any provider
*/
private applyGenericFixes(
tool: Tool,
config: ProviderConfig
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
const warnings: string[] = [];
const modifications: string[] = [];
let fixed = false;
// Ensure function name length
if (tool.function.name.length > config.maxFunctionNameLength) {
tool.function.name = tool.function.name.substring(0, config.maxFunctionNameLength);
modifications.push('Truncated function name to meet length limits');
fixed = true;
}
// Ensure description exists
if (!tool.function.description) {
tool.function.description = `Execute ${tool.function.name}`;
modifications.push('Added missing function description');
fixed = true;
}
// Ensure parameters object structure
if (!tool.function.parameters.type) {
tool.function.parameters.type = 'object';
modifications.push('Added missing parameters type');
fixed = true;
}
if (!tool.function.parameters.properties) {
tool.function.parameters.properties = {};
modifications.push('Added missing parameters properties');
fixed = true;
}
return { tool, fixed, warnings, modifications };
}
/**
* Flatten deep objects
*/
private flattenDeepObjects(
properties: Record<string, ToolParameter>,
maxDepth: number,
currentDepth: number = 0
): { properties: Record<string, ToolParameter>; flattened: boolean } {
let flattened = false;
const result: Record<string, ToolParameter> = {};
for (const [key, param] of Object.entries(properties)) {
if (param.type === 'object' && param.properties && currentDepth >= maxDepth - 1) {
// Flatten this object
const prefix = key + '_';
for (const [subKey, subParam] of Object.entries(param.properties)) {
result[prefix + subKey] = subParam;
}
flattened = true;
} else if (param.type === 'object' && param.properties) {
// Recurse deeper
const subResult = this.flattenDeepObjects(
param.properties,
maxDepth,
currentDepth + 1
);
result[key] = {
...param,
properties: subResult.properties
};
flattened = flattened || subResult.flattened;
} else {
result[key] = param;
}
}
return { properties: result, flattened };
}
/**
* Group related parameters
*/
private groupRelatedParameters(
properties: Record<string, ToolParameter>
): { properties: Record<string, ToolParameter>; grouped: boolean } {
const groups = new Map<string, Record<string, ToolParameter>>();
const ungrouped: Record<string, ToolParameter> = {};
let grouped = false;
// Identify common prefixes
for (const [key, param] of Object.entries(properties)) {
const prefix = key.split('_')[0];
if (prefix && prefix.length > 2) {
if (!groups.has(prefix)) {
groups.set(prefix, {});
}
groups.get(prefix)![key] = param;
} else {
ungrouped[key] = param;
}
}
// Create grouped structure if beneficial
const result: Record<string, ToolParameter> = {};
for (const [prefix, groupProps] of groups) {
if (Object.keys(groupProps).length > 2) {
// Group these properties
result[prefix] = {
type: 'object',
description: `${prefix} properties`,
properties: groupProps
};
grouped = true;
} else {
// Keep ungrouped
Object.assign(result, groupProps);
}
}
// Add ungrouped properties
Object.assign(result, ungrouped);
return { properties: result, grouped };
}
/**
* Simplify complex types for local models
*/
private simplifyComplexTypes(
properties: Record<string, ToolParameter>
): { properties: Record<string, ToolParameter>; simplified: boolean } {
let simplified = false;
const result: Record<string, ToolParameter> = {};
for (const [key, param] of Object.entries(properties)) {
if (param.type === 'array' && param.items && typeof param.items === 'object' && 'properties' in param.items) {
// Complex array of objects - simplify to array of strings
result[key] = {
type: 'array',
description: param.description || `List of ${key}`,
items: { type: 'string' }
};
simplified = true;
} else if (param.type === 'object' && param.properties) {
// Nested object - check if can be simplified
const propCount = Object.keys(param.properties).length;
if (propCount > 5) {
// Too complex - convert to string
result[key] = {
type: 'string',
description: param.description || `JSON string for ${key}`
};
simplified = true;
} else {
result[key] = param;
}
} else {
result[key] = param;
}
}
return { properties: result, simplified };
}
/**
* Fix enum values
*/
private fixEnumValues(properties: Record<string, ToolParameter>): void {
for (const param of Object.values(properties)) {
if (param.enum) {
// Ensure all enum values are strings
param.enum = param.enum.map(v => String(v));
// Remove any special characters
param.enum = param.enum.map(v => v.replace(/[^\w\s-]/g, '_'));
}
// Recurse for nested properties
if (param.properties) {
this.fixEnumValues(param.properties);
}
}
}
/**
* Calculate parameter complexity
*/
private calculateComplexity(parameters: any, depth: number = 0): number {
let complexity = depth;
if (parameters.properties) {
for (const param of Object.values(parameters.properties) as ToolParameter[]) {
complexity += 1;
if (param.type === 'object' && param.properties) {
complexity += this.calculateComplexity(param, depth + 1);
}
if (param.type === 'array') {
complexity += 2; // Arrays add more complexity
if (param.items && typeof param.items === 'object' && 'properties' in param.items) {
complexity += 3; // Array of objects is very complex
}
}
}
}
return complexity;
}
/**
* Batch fix tools for a provider
*/
fixToolsForProvider(tools: Tool[], provider: string): Tool[] {
const fixed: Tool[] = [];
for (const tool of tools) {
const result = this.fixToolForProvider(tool, provider);
if (result.fixed && result.tool) {
fixed.push(result.tool);
if (result.warnings.length > 0) {
log.info(`Warnings for ${tool.function.name}: ${JSON.stringify(result.warnings)}`);
}
if (result.modifications.length > 0) {
log.info(`Modifications for ${tool.function.name}: ${JSON.stringify(result.modifications)}`);
}
} else {
fixed.push(tool);
}
}
return fixed;
}
}
// Export singleton instance
export const edgeCaseHandler = new EdgeCaseHandler();

View File

@@ -27,6 +27,9 @@ import {
ExportFormat,
type ExporterConfig
} from '../metrics/metrics_exporter.js';
import { providerHealthMonitor } from '../monitoring/provider_health_monitor.js';
import { edgeCaseHandler } from './edge_case_handler.js';
import { providerToolValidator } from '../tools/provider_tool_validator.js';
/**
* Provider type enumeration

View File

@@ -0,0 +1,487 @@
/**
* Integration Test for LLM Resilience Improvements
*
* Tests all new components working together to ensure the LLM feature
* is extremely resilient, intuitive, and responsive.
*/
import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
import { toolTimeoutEnforcer } from '../tools/tool_timeout_enforcer.js';
import { providerToolValidator } from '../tools/provider_tool_validator.js';
import { providerHealthMonitor } from '../monitoring/provider_health_monitor.js';
import { parameterCoercer } from '../tools/parameter_coercer.js';
import { toolExecutionMonitor } from '../monitoring/tool_execution_monitor.js';
import { toolResponseCache } from '../tools/tool_response_cache.js';
import { edgeCaseHandler } from '../providers/edge_case_handler.js';
import { EnhancedToolHandler } from '../chat/handlers/enhanced_tool_handler.js';
import type { Tool, ToolCall } from '../tools/tool_interfaces.js';
describe('LLM Resilience Integration Tests', () => {
// Sample tools for testing
const sampleTools: Tool[] = [
{
type: 'function',
function: {
name: 'search_notes',
description: 'Search for notes by keyword',
parameters: {
type: 'object',
properties: {
query: {
type: 'string',
description: 'Search query'
},
limit: {
type: 'number',
description: 'Maximum results',
default: 10
}
},
required: ['query']
}
}
},
{
type: 'function',
function: {
name: 'create-note-with-special-chars',
description: 'Create a new note with special characters in name',
parameters: {
type: 'object',
properties: {
title: {
type: 'string',
description: 'Note title'
},
content: {
type: 'string',
description: 'Note content'
},
deeply: {
type: 'object',
description: 'Deeply nested object',
properties: {
nested: {
type: 'object',
description: 'Nested object',
properties: {
value: {
type: 'string',
description: 'Nested value'
}
}
}
}
}
},
required: [] // Empty for Anthropic testing
}
}
}
];
beforeAll(() => {
// Initialize components
console.log('Setting up integration test environment...');
});
afterAll(() => {
// Cleanup
toolResponseCache.shutdown();
providerHealthMonitor.stopMonitoring();
});
describe('Tool Timeout Enforcement', () => {
it('should enforce timeouts on long-running tools', async () => {
const result = await toolTimeoutEnforcer.executeWithTimeout(
'test_tool',
async () => {
await new Promise(resolve => setTimeout(resolve, 100));
return 'success';
},
200 // 200ms timeout
);
expect(result.success).toBe(true);
expect(result.timedOut).toBe(false);
expect(result.result).toBe('success');
});
it('should timeout and report failure', async () => {
const result = await toolTimeoutEnforcer.executeWithTimeout(
'slow_tool',
async () => {
await new Promise(resolve => setTimeout(resolve, 1000));
return 'should not reach';
},
100 // 100ms timeout
);
expect(result.success).toBe(false);
expect(result.timedOut).toBe(true);
});
});
describe('Provider Tool Validation', () => {
it('should validate and fix tools for OpenAI', () => {
const result = providerToolValidator.validateTool(sampleTools[1], 'openai');
expect(result.fixedTool).toBeDefined();
if (result.fixedTool) {
// Should fix special characters in function name
expect(result.fixedTool.function.name).not.toContain('-');
}
});
it('should ensure non-empty required array for Anthropic', () => {
const result = providerToolValidator.validateTool(sampleTools[1], 'anthropic');
expect(result.fixedTool).toBeDefined();
if (result.fixedTool) {
// Should add at least one required parameter
expect(result.fixedTool.function.parameters.required?.length).toBeGreaterThan(0);
}
});
it('should simplify tools for Ollama', () => {
const result = providerToolValidator.validateTool(sampleTools[1], 'ollama');
expect(result.warnings.length).toBeGreaterThan(0);
});
});
describe('Parameter Type Coercion', () => {
it('should coerce string numbers to numbers', () => {
const result = parameterCoercer.coerceToolArguments(
{ limit: '10' },
sampleTools[0],
{ parseNumbers: true }
);
expect(result.success).toBe(true);
expect(result.value.limit).toBe(10);
expect(typeof result.value.limit).toBe('number');
});
it('should apply default values', () => {
const result = parameterCoercer.coerceToolArguments(
{ query: 'test' },
sampleTools[0],
{ applyDefaults: true }
);
expect(result.success).toBe(true);
expect(result.value.limit).toBe(10);
});
it('should normalize arrays', () => {
const tool: Tool = {
type: 'function',
function: {
name: 'test',
description: 'Test',
parameters: {
type: 'object',
properties: {
tags: {
type: 'array',
description: 'List of tags',
items: { type: 'string', description: 'Tag value' }
}
},
required: []
}
}
};
const result = parameterCoercer.coerceToolArguments(
{ tags: 'single-tag' },
tool,
{ normalizeArrays: true }
);
expect(result.success).toBe(true);
expect(Array.isArray(result.value.tags)).toBe(true);
expect(result.value.tags).toEqual(['single-tag']);
});
});
describe('Tool Execution Monitoring', () => {
it('should track execution statistics', () => {
// Record successful execution
toolExecutionMonitor.recordExecution({
toolName: 'test_tool',
provider: 'openai',
status: 'success',
executionTime: 100,
timestamp: new Date()
});
const stats = toolExecutionMonitor.getToolStats('test_tool', 'openai');
expect(stats).toBeDefined();
expect(stats?.successfulExecutions).toBe(1);
expect(stats?.reliabilityScore).toBeGreaterThan(0);
});
it('should auto-disable unreliable tools', () => {
// Record multiple failures
for (let i = 0; i < 6; i++) {
toolExecutionMonitor.recordExecution({
toolName: 'unreliable_tool',
provider: 'openai',
status: 'failure',
executionTime: 100,
timestamp: new Date(),
error: 'Test failure'
});
}
const isDisabled = toolExecutionMonitor.isToolDisabled('unreliable_tool', 'openai');
expect(isDisabled).toBe(true);
});
});
describe('Tool Response Caching', () => {
it('should cache deterministic tool responses', () => {
const toolName = 'read_note_tool';
const args = { noteId: 'test123' };
const response = { content: 'Test content' };
// Set cache
const cached = toolResponseCache.set(toolName, args, response, 'openai');
expect(cached).toBe(true);
// Get from cache
const retrieved = toolResponseCache.get(toolName, args, 'openai');
expect(retrieved).toEqual(response);
});
it('should generate consistent cache keys', () => {
const key1 = toolResponseCache.generateCacheKey('tool', { b: 2, a: 1 }, 'provider');
const key2 = toolResponseCache.generateCacheKey('tool', { a: 1, b: 2 }, 'provider');
expect(key1).toBe(key2);
});
it('should respect TTL', async () => {
const toolName = 'temp_tool';
const args = { id: 'temp' };
const response = 'temp data';
// Set with short TTL
toolResponseCache.set(toolName, args, response, 'openai', 100); // 100ms TTL
// Should be cached
expect(toolResponseCache.get(toolName, args, 'openai')).toBe(response);
// Wait for expiration
await new Promise(resolve => setTimeout(resolve, 150));
// Should be expired
expect(toolResponseCache.get(toolName, args, 'openai')).toBeUndefined();
});
});
describe('Edge Case Handling', () => {
it('should fix OpenAI edge cases', () => {
const tool = sampleTools[1];
const result = edgeCaseHandler.fixToolForProvider(tool, 'openai');
expect(result.fixed).toBe(true);
if (result.tool) {
// Function name should not have hyphens
expect(result.tool.function.name).not.toContain('-');
// Deep nesting might be flattened
expect(result.modifications.length).toBeGreaterThan(0);
}
});
it('should fix Anthropic edge cases', () => {
const tool = sampleTools[1];
const result = edgeCaseHandler.fixToolForProvider(tool, 'anthropic');
expect(result.fixed).toBe(true);
if (result.tool) {
// Should have required parameters
expect(result.tool.function.parameters.required).toBeDefined();
expect(result.tool.function.parameters.required!.length).toBeGreaterThan(0);
}
});
it('should simplify for Ollama', () => {
const complexTool: Tool = {
type: 'function',
function: {
name: 'complex_tool',
description: 'A'.repeat(600), // Long description
parameters: {
type: 'object',
properties: Object.fromEntries(
Array.from({ length: 30 }, (_, i) => [
`param${i}`,
{ type: 'string', description: `Parameter ${i}` }
])
),
required: []
}
}
};
const result = edgeCaseHandler.fixToolForProvider(complexTool, 'ollama');
expect(result.fixed).toBe(true);
if (result.tool) {
// Description should be truncated
expect(result.tool.function.description.length).toBeLessThanOrEqual(500);
// Parameters should be reduced
const paramCount = Object.keys(result.tool.function.parameters.properties || {}).length;
expect(paramCount).toBeLessThanOrEqual(20);
}
});
});
describe('Parallel Tool Execution', () => {
it('should identify independent tools for parallel execution', async () => {
const toolCalls: ToolCall[] = [
{
id: '1',
function: {
name: 'search_notes',
arguments: { query: 'test1' }
}
},
{
id: '2',
function: {
name: 'search_notes',
arguments: { query: 'test2' }
}
},
{
id: '3',
function: {
name: 'read_note_tool',
arguments: { noteId: 'abc' }
}
}
];
// These should be executed in parallel since they're independent
const handler = new EnhancedToolHandler();
// For now, just verify the handler was created successfully
expect(handler).toBeDefined();
});
});
describe('Provider Health Monitoring', () => {
it('should track provider health status', async () => {
// Mock provider service
const mockService = {
chat: async () => ({
content: 'test',
usage: { totalTokens: 5 }
}),
getModels: () => [{ id: 'test-model' }]
};
providerHealthMonitor.registerProvider('test-provider', mockService as any);
// Manually trigger health check
const result = await providerHealthMonitor.checkProvider('test-provider');
expect(result.success).toBe(true);
expect(result.latency).toBeGreaterThan(0);
const health = providerHealthMonitor.getProviderHealth('test-provider');
expect(health?.healthy).toBe(true);
});
it('should disable unhealthy providers', () => {
// Simulate failures
const status = {
provider: 'failing-provider',
healthy: true,
lastChecked: new Date(),
consecutiveFailures: 3,
totalChecks: 10,
totalFailures: 3,
averageLatency: 100,
disabled: false
};
// This would normally be done internally
providerHealthMonitor.disableProvider('failing-provider', 'Too many failures');
expect(providerHealthMonitor.isProviderHealthy('failing-provider')).toBe(false);
});
});
describe('End-to-End Integration', () => {
it('should handle tool execution with all enhancements', async () => {
// This tests the full flow with all components working together
const toolCall: ToolCall = {
id: 'integration-test',
function: {
name: 'search_notes',
arguments: '{"query": "test", "limit": "5"}' // String number to test coercion
}
};
// Test components integration
const tool = sampleTools[0];
// 1. Validate for provider
const validation = providerToolValidator.validateTool(tool, 'openai');
expect(validation.valid || validation.fixedTool).toBeTruthy();
// 2. Apply edge case fixes
const edgeFixes = edgeCaseHandler.fixToolForProvider(
validation.fixedTool || tool,
'openai'
);
// 3. Parse and coerce arguments
const args = parameterCoercer.coerceToolArguments(
JSON.parse(toolCall.function.arguments as string),
tool,
{ provider: 'openai' }
);
expect(args.value.limit).toBe(5);
expect(typeof args.value.limit).toBe('number');
// 4. Execute with timeout
const timeoutResult = await toolTimeoutEnforcer.executeWithTimeout(
'search_notes',
async () => ({ results: ['note1', 'note2'] }),
5000
);
expect(timeoutResult.success).toBe(true);
// 5. Cache the result
if (timeoutResult.success) {
toolResponseCache.set(
'search_notes',
args.value,
timeoutResult.result,
'openai'
);
}
// 6. Record execution
toolExecutionMonitor.recordExecution({
toolName: 'search_notes',
provider: 'openai',
status: timeoutResult.success ? 'success' : 'failure',
executionTime: timeoutResult.executionTime,
timestamp: new Date()
});
// Verify everything worked
const cached = toolResponseCache.get('search_notes', args.value, 'openai');
expect(cached).toEqual({ results: ['note1', 'note2'] });
const stats = toolExecutionMonitor.getToolStats('search_notes', 'openai');
expect(stats?.totalExecutions).toBeGreaterThan(0);
});
});
});

View File

@@ -0,0 +1,593 @@
/**
* Parameter Type Coercer
*
* Provides automatic type conversion, array normalization, default value injection,
* and schema validation with fixes for tool parameters.
*/
import log from '../../log.js';
import type { Tool, ToolParameter } from './tool_interfaces.js';
/**
* Coercion result
*/
export interface CoercionResult {
success: boolean;
value: any;
wasCoerced: boolean;
errors: string[];
warnings: string[];
}
/**
* Coercion options
*/
export interface CoercionOptions {
/** Strict mode - fail on any coercion error */
strict: boolean;
/** Apply default values */
applyDefaults: boolean;
/** Normalize arrays (single values to arrays) */
normalizeArrays: boolean;
/** Trim string values */
trimStrings: boolean;
/** Convert number strings to numbers */
parseNumbers: boolean;
/** Convert boolean strings to booleans */
parseBooleans: boolean;
/** Provider-specific quirks */
provider?: string;
}
/**
* Default coercion options
*/
const DEFAULT_OPTIONS: CoercionOptions = {
strict: false,
applyDefaults: true,
normalizeArrays: true,
trimStrings: true,
parseNumbers: true,
parseBooleans: true
};
/**
* Provider-specific quirks
*/
const PROVIDER_QUIRKS = {
openai: {
// OpenAI sometimes sends stringified JSON for complex objects
parseJsonStrings: true,
// OpenAI may send null for optional parameters
treatNullAsUndefined: true
},
anthropic: {
// Anthropic strictly validates types
strictTypeChecking: true,
// Anthropic requires arrays to be actual arrays
requireArrayTypes: true
},
ollama: {
// Local models may have looser type handling
lenientParsing: true,
// May send numbers as strings more often
aggressiveNumberParsing: true
}
};
/**
* Parameter coercer class
*/
export class ParameterCoercer {
private options: CoercionOptions;
constructor(options?: Partial<CoercionOptions>) {
this.options = { ...DEFAULT_OPTIONS, ...options };
}
/**
* Coerce tool call arguments to match tool definition
*/
coerceToolArguments(
args: Record<string, unknown>,
tool: Tool,
options?: Partial<CoercionOptions>
): CoercionResult {
const opts = { ...this.options, ...options };
const errors: string[] = [];
const warnings: string[] = [];
let wasCoerced = false;
const parameters = tool.function.parameters;
const coercedArgs: Record<string, any> = {};
// Process each parameter
for (const [paramName, paramDef] of Object.entries(parameters.properties)) {
const rawValue = args[paramName];
const isRequired = parameters.required?.includes(paramName);
// Handle missing values
if (rawValue === undefined || rawValue === null) {
if (opts.provider === 'openai' && rawValue === null) {
// OpenAI quirk: treat null as undefined
if (isRequired && !paramDef.default) {
errors.push(`Required parameter '${paramName}' is null`);
continue;
}
}
if (opts.applyDefaults && paramDef.default !== undefined) {
coercedArgs[paramName] = paramDef.default;
wasCoerced = true;
warnings.push(`Applied default value for '${paramName}'`);
} else if (isRequired) {
errors.push(`Required parameter '${paramName}' is missing`);
}
continue;
}
// Coerce the value
const coerced = this.coerceValue(
rawValue,
paramDef,
paramName,
opts
);
if (coerced.success) {
coercedArgs[paramName] = coerced.value;
if (coerced.wasCoerced) {
wasCoerced = true;
warnings.push(...coerced.warnings);
}
} else {
errors.push(...coerced.errors);
if (!opts.strict) {
// In non-strict mode, use original value
coercedArgs[paramName] = rawValue;
warnings.push(`Failed to coerce '${paramName}', using original value`);
}
}
}
// Check for unknown parameters
for (const paramName of Object.keys(args)) {
if (!(paramName in parameters.properties)) {
warnings.push(`Unknown parameter '${paramName}' will be ignored`);
}
}
return {
success: errors.length === 0,
value: coercedArgs,
wasCoerced,
errors,
warnings
};
}
/**
* Coerce a single value to match its type definition
*/
private coerceValue(
value: unknown,
definition: ToolParameter,
path: string,
options: CoercionOptions
): CoercionResult {
const errors: string[] = [];
const warnings: string[] = [];
let wasCoerced = false;
let coercedValue = value;
// Handle provider-specific JSON string parsing
if (options.provider === 'openai' &&
typeof value === 'string' &&
(definition.type === 'object' || definition.type === 'array')) {
try {
coercedValue = JSON.parse(value);
wasCoerced = true;
warnings.push(`Parsed JSON string for '${path}'`);
} catch {
// Not valid JSON, continue with string value
}
}
// Type-specific coercion
switch (definition.type) {
case 'string':
const stringResult = this.coerceToString(coercedValue, path, options);
coercedValue = stringResult.value;
wasCoerced = wasCoerced || stringResult.wasCoerced;
warnings.push(...stringResult.warnings);
break;
case 'number':
case 'integer':
const numberResult = this.coerceToNumber(
coercedValue,
path,
definition,
definition.type === 'integer',
options
);
if (numberResult.success) {
coercedValue = numberResult.value;
wasCoerced = wasCoerced || numberResult.wasCoerced;
warnings.push(...numberResult.warnings);
} else {
errors.push(...numberResult.errors);
}
break;
case 'boolean':
const boolResult = this.coerceToBoolean(coercedValue, path, options);
if (boolResult.success) {
coercedValue = boolResult.value;
wasCoerced = wasCoerced || boolResult.wasCoerced;
warnings.push(...boolResult.warnings);
} else {
errors.push(...boolResult.errors);
}
break;
case 'array':
const arrayResult = this.coerceToArray(
coercedValue,
path,
definition,
options
);
if (arrayResult.success) {
coercedValue = arrayResult.value;
wasCoerced = wasCoerced || arrayResult.wasCoerced;
warnings.push(...arrayResult.warnings);
} else {
errors.push(...arrayResult.errors);
}
break;
case 'object':
const objectResult = this.coerceToObject(
coercedValue,
path,
definition,
options
);
if (objectResult.success) {
coercedValue = objectResult.value;
wasCoerced = wasCoerced || objectResult.wasCoerced;
warnings.push(...objectResult.warnings);
} else {
errors.push(...objectResult.errors);
}
break;
default:
warnings.push(`Unknown type '${definition.type}' for '${path}'`);
}
// Validate enum values
if (definition.enum && !definition.enum.includes(String(coercedValue))) {
errors.push(`Value for '${path}' must be one of: ${definition.enum.join(', ')}`);
}
return {
success: errors.length === 0,
value: coercedValue,
wasCoerced,
errors,
warnings
};
}
/**
* Coerce to string
*/
private coerceToString(
value: unknown,
path: string,
options: CoercionOptions
): CoercionResult {
const warnings: string[] = [];
let wasCoerced = false;
let result: string;
if (typeof value === 'string') {
result = options.trimStrings ? value.trim() : value;
if (result !== value) {
wasCoerced = true;
warnings.push(`Trimmed whitespace from '${path}'`);
}
} else if (value === null || value === undefined) {
result = '';
wasCoerced = true;
warnings.push(`Converted null/undefined to empty string for '${path}'`);
} else {
result = String(value);
wasCoerced = true;
warnings.push(`Converted ${typeof value} to string for '${path}'`);
}
return {
success: true,
value: result,
wasCoerced,
errors: [],
warnings
};
}
/**
* Coerce to number
*/
private coerceToNumber(
value: unknown,
path: string,
definition: ToolParameter,
isInteger: boolean,
options: CoercionOptions
): CoercionResult {
const errors: string[] = [];
const warnings: string[] = [];
let wasCoerced = false;
let result: number;
if (typeof value === 'number') {
result = isInteger ? Math.round(value) : value;
if (result !== value) {
wasCoerced = true;
warnings.push(`Rounded to integer for '${path}'`);
}
} else if (typeof value === 'string' && options.parseNumbers) {
const parsed = isInteger ? parseInt(value, 10) : parseFloat(value);
if (!isNaN(parsed)) {
result = parsed;
wasCoerced = true;
warnings.push(`Parsed string to number for '${path}'`);
} else {
errors.push(`Cannot parse '${value}' as number for '${path}'`);
return { success: false, value, wasCoerced: false, errors, warnings };
}
} else if (typeof value === 'boolean') {
result = value ? 1 : 0;
wasCoerced = true;
warnings.push(`Converted boolean to number for '${path}'`);
} else {
errors.push(`Cannot coerce ${typeof value} to number for '${path}'`);
return { success: false, value, wasCoerced: false, errors, warnings };
}
// Validate constraints
if (definition.minimum !== undefined && result < definition.minimum) {
result = definition.minimum;
wasCoerced = true;
warnings.push(`Clamped to minimum value ${definition.minimum} for '${path}'`);
}
if (definition.maximum !== undefined && result > definition.maximum) {
result = definition.maximum;
wasCoerced = true;
warnings.push(`Clamped to maximum value ${definition.maximum} for '${path}'`);
}
return {
success: true,
value: result,
wasCoerced,
errors,
warnings
};
}
/**
* Coerce to boolean
*/
private coerceToBoolean(
value: unknown,
path: string,
options: CoercionOptions
): CoercionResult {
const warnings: string[] = [];
let wasCoerced = false;
let result: boolean;
if (typeof value === 'boolean') {
result = value;
} else if (typeof value === 'string' && options.parseBooleans) {
const lower = value.toLowerCase().trim();
if (lower === 'true' || lower === 'yes' || lower === '1') {
result = true;
wasCoerced = true;
warnings.push(`Parsed string to boolean true for '${path}'`);
} else if (lower === 'false' || lower === 'no' || lower === '0') {
result = false;
wasCoerced = true;
warnings.push(`Parsed string to boolean false for '${path}'`);
} else {
return {
success: false,
value,
wasCoerced: false,
errors: [`Cannot parse '${value}' as boolean for '${path}'`],
warnings
};
}
} else if (typeof value === 'number') {
result = value !== 0;
wasCoerced = true;
warnings.push(`Converted number to boolean for '${path}'`);
} else {
result = Boolean(value);
wasCoerced = true;
warnings.push(`Coerced ${typeof value} to boolean for '${path}'`);
}
return {
success: true,
value: result,
wasCoerced,
errors: [],
warnings
};
}
/**
* Coerce to array
*/
private coerceToArray(
value: unknown,
path: string,
definition: ToolParameter,
options: CoercionOptions
): CoercionResult {
const errors: string[] = [];
const warnings: string[] = [];
let wasCoerced = false;
let result: any[];
if (Array.isArray(value)) {
result = value;
} else if (options.normalizeArrays) {
// Convert single value to array
result = [value];
wasCoerced = true;
warnings.push(`Normalized single value to array for '${path}'`);
} else {
errors.push(`Expected array for '${path}', got ${typeof value}`);
return { success: false, value, wasCoerced: false, errors, warnings };
}
// Validate array constraints
if (definition.minItems !== undefined && result.length < definition.minItems) {
errors.push(`Array '${path}' must have at least ${definition.minItems} items`);
}
if (definition.maxItems !== undefined && result.length > definition.maxItems) {
result = result.slice(0, definition.maxItems);
wasCoerced = true;
warnings.push(`Truncated array to ${definition.maxItems} items for '${path}'`);
}
// Coerce array items if type is specified
if (definition.items) {
const coercedItems: any[] = [];
for (let i = 0; i < result.length; i++) {
const itemResult = this.coerceValue(
result[i],
definition.items as ToolParameter,
`${path}[${i}]`,
options
);
if (itemResult.success) {
coercedItems.push(itemResult.value);
if (itemResult.wasCoerced) wasCoerced = true;
warnings.push(...itemResult.warnings);
} else {
errors.push(...itemResult.errors);
coercedItems.push(result[i]); // Keep original on error
}
}
result = coercedItems;
}
return {
success: errors.length === 0,
value: result,
wasCoerced,
errors,
warnings
};
}
/**
* Coerce to object
*/
private coerceToObject(
value: unknown,
path: string,
definition: ToolParameter,
options: CoercionOptions
): CoercionResult {
const errors: string[] = [];
const warnings: string[] = [];
let wasCoerced = false;
let result: Record<string, any>;
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
result = value as Record<string, any>;
} else if (typeof value === 'string') {
// Try to parse as JSON
try {
result = JSON.parse(value);
wasCoerced = true;
warnings.push(`Parsed JSON string for object '${path}'`);
} catch {
errors.push(`Cannot parse string as object for '${path}'`);
return { success: false, value, wasCoerced: false, errors, warnings };
}
} else {
errors.push(`Expected object for '${path}', got ${typeof value}`);
return { success: false, value, wasCoerced: false, errors, warnings };
}
// Coerce nested properties if defined
if (definition.properties) {
const coercedObj: Record<string, any> = {};
for (const [propName, propDef] of Object.entries(definition.properties)) {
if (propName in result) {
const propResult = this.coerceValue(
result[propName],
propDef,
`${path}.${propName}`,
options
);
if (propResult.success) {
coercedObj[propName] = propResult.value;
if (propResult.wasCoerced) wasCoerced = true;
warnings.push(...propResult.warnings);
} else {
errors.push(...propResult.errors);
coercedObj[propName] = result[propName]; // Keep original on error
}
} else if (propDef.default !== undefined && options.applyDefaults) {
coercedObj[propName] = propDef.default;
wasCoerced = true;
warnings.push(`Applied default value for '${path}.${propName}'`);
}
}
// Include any additional properties not in schema
for (const propName of Object.keys(result)) {
if (!(propName in coercedObj)) {
coercedObj[propName] = result[propName];
}
}
result = coercedObj;
}
return {
success: errors.length === 0,
value: result,
wasCoerced,
errors,
warnings
};
}
/**
* Update coercion options
*/
updateOptions(options: Partial<CoercionOptions>): void {
this.options = { ...this.options, ...options };
}
/**
* Get current options
*/
getOptions(): CoercionOptions {
return { ...this.options };
}
}
// Export singleton instance
export const parameterCoercer = new ParameterCoercer();

View File

@@ -0,0 +1,470 @@
/**
* Provider Tool Validator
*
* Validates and auto-fixes tool definitions based on provider-specific requirements
* for OpenAI, Anthropic, and Ollama.
*/
import log from '../../log.js';
import type { Tool, ToolParameter } from './tool_interfaces.js';
import type { ProviderType } from '../providers/provider_factory.js';
/**
* Validation result for a tool
*/
export interface ValidationResult {
valid: boolean;
errors: ValidationError[];
warnings: ValidationWarning[];
fixedTool?: Tool;
}
/**
* Validation error
*/
export interface ValidationError {
field: string;
message: string;
severity: 'error' | 'critical';
}
/**
* Validation warning
*/
export interface ValidationWarning {
field: string;
message: string;
suggestion?: string;
}
/**
* Provider-specific validation rules
*/
interface ProviderRules {
maxFunctionNameLength: number;
maxDescriptionLength: number;
maxParameterDepth: number;
maxParameterCount: number;
allowEmptyRequired: boolean;
requireDescriptions: boolean;
functionNamePattern: RegExp;
supportedTypes: Set<string>;
}
/**
* Default validation rules per provider
*/
const PROVIDER_RULES: Record<string, ProviderRules> = {
openai: {
maxFunctionNameLength: 64,
maxDescriptionLength: 1024,
maxParameterDepth: 5,
maxParameterCount: 20,
allowEmptyRequired: true,
requireDescriptions: true,
functionNamePattern: /^[a-zA-Z0-9_-]+$/,
supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array', 'integer'])
},
anthropic: {
maxFunctionNameLength: 64,
maxDescriptionLength: 1024,
maxParameterDepth: 4,
maxParameterCount: 15,
allowEmptyRequired: false, // Anthropic requires non-empty required arrays
requireDescriptions: true,
functionNamePattern: /^[a-zA-Z0-9_-]+$/,
supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array', 'integer'])
},
ollama: {
maxFunctionNameLength: 50,
maxDescriptionLength: 500,
maxParameterDepth: 3,
maxParameterCount: 10, // Local models have smaller context
allowEmptyRequired: true,
requireDescriptions: false,
functionNamePattern: /^[a-zA-Z0-9_]+$/,
supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array'])
}
};
/**
* Provider tool validator class
*/
export class ProviderToolValidator {
private providerRules: Map<string, ProviderRules>;
constructor() {
this.providerRules = new Map(Object.entries(PROVIDER_RULES));
}
/**
* Validate a tool for a specific provider
*/
validateTool(tool: Tool, provider: string): ValidationResult {
const rules = this.providerRules.get(provider) || PROVIDER_RULES.openai;
const errors: ValidationError[] = [];
const warnings: ValidationWarning[] = [];
// Deep clone the tool for potential fixes
const fixedTool = JSON.parse(JSON.stringify(tool)) as Tool;
let wasFixed = false;
// Validate function name
const nameValidation = this.validateFunctionName(
fixedTool.function.name,
rules
);
if (nameValidation.error) {
errors.push(nameValidation.error);
}
if (nameValidation.fixed) {
fixedTool.function.name = nameValidation.fixed;
wasFixed = true;
}
// Validate description
const descValidation = this.validateDescription(
fixedTool.function.description,
rules
);
if (descValidation.error) {
errors.push(descValidation.error);
}
if (descValidation.warning) {
warnings.push(descValidation.warning);
}
if (descValidation.fixed) {
fixedTool.function.description = descValidation.fixed;
wasFixed = true;
}
// Validate parameters
const paramValidation = this.validateParameters(
fixedTool.function.parameters,
rules,
provider
);
errors.push(...paramValidation.errors);
warnings.push(...paramValidation.warnings);
if (paramValidation.fixed) {
fixedTool.function.parameters = paramValidation.fixed;
wasFixed = true;
}
// Provider-specific validations
const providerSpecific = this.validateProviderSpecific(fixedTool, provider);
errors.push(...providerSpecific.errors);
warnings.push(...providerSpecific.warnings);
if (providerSpecific.fixed) {
Object.assign(fixedTool, providerSpecific.fixed);
wasFixed = true;
}
return {
valid: errors.length === 0,
errors,
warnings,
fixedTool: wasFixed ? fixedTool : undefined
};
}
/**
* Validate function name
*/
private validateFunctionName(name: string, rules: ProviderRules) {
const result: any = {};
// Check length
if (name.length > rules.maxFunctionNameLength) {
result.error = {
field: 'function.name',
message: `Function name exceeds maximum length of ${rules.maxFunctionNameLength}`,
severity: 'error' as const
};
// Auto-fix: truncate
result.fixed = name.substring(0, rules.maxFunctionNameLength);
}
// Check pattern
if (!rules.functionNamePattern.test(name)) {
result.error = {
field: 'function.name',
message: `Function name contains invalid characters`,
severity: 'error' as const
};
// Auto-fix: replace invalid characters
result.fixed = name.replace(/[^a-zA-Z0-9_-]/g, '_');
}
return result;
}
/**
* Validate description
*/
private validateDescription(description: string, rules: ProviderRules) {
const result: any = {};
// Check if description exists when required
if (rules.requireDescriptions && !description) {
result.error = {
field: 'function.description',
message: 'Description is required',
severity: 'error' as const
};
result.fixed = 'Performs an operation'; // Generic fallback
}
// Check length
if (description && description.length > rules.maxDescriptionLength) {
result.warning = {
field: 'function.description',
message: `Description exceeds recommended length of ${rules.maxDescriptionLength}`,
suggestion: 'Consider shortening the description'
};
// Auto-fix: truncate with ellipsis
result.fixed = description.substring(0, rules.maxDescriptionLength - 3) + '...';
}
return result;
}
/**
* Validate parameters
*/
private validateParameters(
parameters: any,
rules: ProviderRules,
provider: string
) {
const errors: ValidationError[] = [];
const warnings: ValidationWarning[] = [];
let fixed: any = null;
// Ensure parameters is an object
if (parameters.type !== 'object') {
errors.push({
field: 'function.parameters.type',
message: 'Parameters must be of type "object"',
severity: 'critical'
});
fixed = {
type: 'object',
properties: parameters.properties || {},
required: parameters.required || []
};
}
// Check parameter count
const paramCount = Object.keys(parameters.properties || {}).length;
if (paramCount > rules.maxParameterCount) {
warnings.push({
field: 'function.parameters',
message: `Parameter count (${paramCount}) exceeds recommended maximum (${rules.maxParameterCount})`,
suggestion: 'Consider reducing the number of parameters'
});
}
// Validate required array for Anthropic
if (!rules.allowEmptyRequired && (!parameters.required || parameters.required.length === 0)) {
if (provider === 'anthropic') {
// For Anthropic, add at least one optional parameter to required
const props = Object.keys(parameters.properties || {});
if (props.length > 0) {
if (!fixed) fixed = { ...parameters };
fixed.required = [props[0]]; // Add first property as required
warnings.push({
field: 'function.parameters.required',
message: 'Anthropic requires non-empty required array, added first parameter',
suggestion: 'Specify which parameters are required'
});
}
}
}
// Validate parameter types and depth
if (parameters.properties) {
const typeErrors = this.validateParameterTypes(
parameters.properties,
rules.supportedTypes,
0,
rules.maxParameterDepth
);
errors.push(...typeErrors);
}
return { errors, warnings, fixed };
}
/**
* Validate parameter types recursively
*/
private validateParameterTypes(
properties: Record<string, ToolParameter>,
supportedTypes: Set<string>,
depth: number,
maxDepth: number
): ValidationError[] {
const errors: ValidationError[] = [];
if (depth > maxDepth) {
errors.push({
field: 'function.parameters',
message: `Parameter nesting exceeds maximum depth of ${maxDepth}`,
severity: 'error'
});
return errors;
}
for (const [key, param] of Object.entries(properties)) {
// Check if type is supported
if (param.type && !supportedTypes.has(param.type)) {
errors.push({
field: `function.parameters.properties.${key}.type`,
message: `Unsupported type: ${param.type}`,
severity: 'error'
});
}
// Recursively check nested objects
if (param.type === 'object' && param.properties) {
const nestedErrors = this.validateParameterTypes(
param.properties,
supportedTypes,
depth + 1,
maxDepth
);
errors.push(...nestedErrors);
}
// Check array items
if (param.type === 'array' && param.items) {
if (typeof param.items === 'object' && 'properties' in param.items) {
const nestedErrors = this.validateParameterTypes(
param.items.properties!,
supportedTypes,
depth + 1,
maxDepth
);
errors.push(...nestedErrors);
}
}
}
return errors;
}
/**
* Provider-specific validations
*/
private validateProviderSpecific(tool: Tool, provider: string) {
const errors: ValidationError[] = [];
const warnings: ValidationWarning[] = [];
let fixed: any = null;
switch (provider) {
case 'openai':
// OpenAI-specific: Check for special characters in function names
if (tool.function.name.includes('-')) {
warnings.push({
field: 'function.name',
message: 'OpenAI prefers underscores over hyphens in function names',
suggestion: 'Replace hyphens with underscores'
});
}
break;
case 'anthropic':
// Anthropic-specific: Ensure descriptions are meaningful
if (tool.function.description && tool.function.description.length < 10) {
warnings.push({
field: 'function.description',
message: 'Description is very short',
suggestion: 'Provide a more detailed description for better results'
});
}
break;
case 'ollama':
// Ollama-specific: Warn about complex nested structures
const complexity = this.calculateComplexity(tool.function.parameters);
if (complexity > 10) {
warnings.push({
field: 'function.parameters',
message: 'Tool parameters are complex for local models',
suggestion: 'Consider simplifying the parameter structure'
});
}
break;
}
return { errors, warnings, fixed };
}
/**
* Calculate parameter complexity score
*/
private calculateComplexity(parameters: any, depth: number = 0): number {
let complexity = depth;
if (parameters.properties) {
for (const param of Object.values(parameters.properties) as ToolParameter[]) {
complexity += 1;
if (param.type === 'object' && param.properties) {
complexity += this.calculateComplexity(param, depth + 1);
}
if (param.type === 'array' && param.items) {
complexity += 2; // Arrays add more complexity
}
}
}
return complexity;
}
/**
* Batch validate multiple tools
*/
validateTools(tools: Tool[], provider: string): Map<string, ValidationResult> {
const results = new Map<string, ValidationResult>();
for (const tool of tools) {
const result = this.validateTool(tool, provider);
results.set(tool.function.name, result);
if (!result.valid) {
log.info(`Tool '${tool.function.name}' validation failed for ${provider}: ${JSON.stringify(result.errors)}`);
}
if (result.warnings.length > 0) {
log.info(`Tool '${tool.function.name}' validation warnings for ${provider}: ${JSON.stringify(result.warnings)}`);
}
}
return results;
}
/**
* Auto-fix tools for a provider
*/
autoFixTools(tools: Tool[], provider: string): Tool[] {
const fixed: Tool[] = [];
for (const tool of tools) {
const result = this.validateTool(tool, provider);
fixed.push(result.fixedTool || tool);
}
return fixed;
}
/**
* Check if a provider supports a tool
*/
isToolSupportedByProvider(tool: Tool, provider: string): boolean {
const result = this.validateTool(tool, provider);
return result.valid || (result.fixedTool !== undefined);
}
}
// Export singleton instance
export const providerToolValidator = new ProviderToolValidator();

View File

@@ -7,6 +7,9 @@
import log from '../../log.js';
import type { Tool, ToolCall, ToolParameter } from './tool_interfaces.js';
import { providerToolValidator } from './provider_tool_validator.js';
import { edgeCaseHandler } from '../providers/edge_case_handler.js';
import { parameterCoercer } from './parameter_coercer.js';
/**
* Anthropic tool format
@@ -51,17 +54,23 @@ export class ToolFormatAdapter {
* Convert tools from standard format to provider-specific format
*/
static convertToProviderFormat(tools: Tool[], provider: ProviderType): unknown[] {
// First validate and fix tools for the provider
const validatedTools = providerToolValidator.autoFixTools(tools, provider);
// Apply edge case fixes
const fixedTools = edgeCaseHandler.fixToolsForProvider(validatedTools, provider);
switch (provider) {
case 'anthropic':
return this.convertToAnthropicFormat(tools);
return this.convertToAnthropicFormat(fixedTools);
case 'ollama':
return this.convertToOllamaFormat(tools);
return this.convertToOllamaFormat(fixedTools);
case 'openai':
// OpenAI format matches our standard format
return tools;
return fixedTools;
default:
log.info(`Warning: Unknown provider ${provider}, returning tools in standard format`);
return tools;
return fixedTools;
}
}
@@ -300,18 +309,42 @@ export class ToolFormatAdapter {
}
/**
* Parse tool arguments safely
* Parse tool arguments safely with coercion
*/
static parseToolArguments(args: string | Record<string, unknown>): Record<string, unknown> {
static parseToolArguments(
args: string | Record<string, unknown>,
tool?: Tool,
provider?: string
): Record<string, unknown> {
let parsedArgs: Record<string, unknown>;
if (typeof args === 'string') {
try {
return JSON.parse(args);
parsedArgs = JSON.parse(args);
} catch (error) {
log.error(`Failed to parse tool arguments as JSON: ${error}`);
return {};
}
} else {
parsedArgs = args || {};
}
return args || {};
// Apply parameter coercion if tool definition is provided
if (tool) {
const coercionResult = parameterCoercer.coerceToolArguments(
parsedArgs,
tool,
{ provider }
);
if (coercionResult.warnings.length > 0) {
log.info(`Parameter coercion warnings: ${coercionResult.warnings.join(', ')}`);
}
return coercionResult.value;
}
return parsedArgs;
}
/**

View File

@@ -0,0 +1,547 @@
/**
* Tool Response Cache
*
* Implements LRU cache with TTL for deterministic/read-only tool responses,
* with cache key generation, invalidation strategies, and hit rate tracking.
*/
import log from '../../log.js';
import crypto from 'crypto';
/**
* Cache entry with metadata
*/
interface CacheEntry<T = any> {
key: string;
value: T;
timestamp: Date;
expiresAt: Date;
hits: number;
size: number;
toolName: string;
provider?: string;
}
/**
* Cache statistics
*/
export interface CacheStatistics {
totalEntries: number;
totalSize: number;
hitRate: number;
missRate: number;
evictionCount: number;
avgHitsPerEntry: number;
oldestEntry?: Date;
newestEntry?: Date;
topTools: Array<{ tool: string; hits: number }>;
}
/**
* Cache configuration
*/
export interface CacheConfig {
/** Maximum cache size in bytes (default: 50MB) */
maxSize: number;
/** Maximum number of entries (default: 1000) */
maxEntries: number;
/** Default TTL in milliseconds (default: 300000 - 5 minutes) */
defaultTTL: number;
/** Enable automatic cleanup (default: true) */
autoCleanup: boolean;
/** Cleanup interval in milliseconds (default: 60000) */
cleanupInterval: number;
/** Enable hit tracking (default: true) */
trackHits: boolean;
}
/**
* Default configuration
*/
const DEFAULT_CONFIG: CacheConfig = {
maxSize: 50 * 1024 * 1024, // 50MB
maxEntries: 1000,
defaultTTL: 300000, // 5 minutes
autoCleanup: true,
cleanupInterval: 60000, // 1 minute
trackHits: true
};
/**
* Tool-specific TTL overrides (in milliseconds)
*/
const TOOL_TTL_OVERRIDES: Record<string, number> = {
// Static data tools - longer TTL
'read_note_tool': 600000, // 10 minutes
'get_note_metadata': 600000, // 10 minutes
// Search tools - medium TTL
'search_notes_tool': 300000, // 5 minutes
'keyword_search_tool': 300000, // 5 minutes
// Dynamic data tools - shorter TTL
'get_recent_notes': 60000, // 1 minute
'get_workspace_status': 30000 // 30 seconds
};
/**
* Deterministic tools that can be cached
*/
const CACHEABLE_TOOLS = new Set([
'read_note_tool',
'search_notes_tool',
'keyword_search_tool',
'attribute_search_tool',
'get_note_metadata',
'get_note_content',
'get_recent_notes',
'get_workspace_status',
'list_notes',
'find_notes_by_tag'
]);
/**
* Tool response cache class
*/
export class ToolResponseCache {
private config: CacheConfig;
private cache: Map<string, CacheEntry>;
private accessOrder: string[];
private totalHits: number;
private totalMisses: number;
private evictionCount: number;
private cleanupTimer?: NodeJS.Timeout;
private currentSize: number;
constructor(config?: Partial<CacheConfig>) {
this.config = { ...DEFAULT_CONFIG, ...config };
this.cache = new Map();
this.accessOrder = [];
this.totalHits = 0;
this.totalMisses = 0;
this.evictionCount = 0;
this.currentSize = 0;
if (this.config.autoCleanup) {
this.startAutoCleanup();
}
}
/**
* Check if a tool is cacheable
*/
isCacheable(toolName: string): boolean {
return CACHEABLE_TOOLS.has(toolName);
}
/**
* Generate cache key for tool call
*/
generateCacheKey(
toolName: string,
args: Record<string, any>,
provider?: string
): string {
// Sort arguments for consistent key generation
const sortedArgs = this.sortObjectDeep(args);
// Create key components
const keyComponents = {
tool: toolName,
args: sortedArgs,
provider: provider || 'default'
};
// Generate hash
const hash = crypto
.createHash('sha256')
.update(JSON.stringify(keyComponents))
.digest('hex');
return `${toolName}:${hash.substring(0, 16)}`;
}
/**
* Get cached response
*/
get(
toolName: string,
args: Record<string, any>,
provider?: string
): any | undefined {
if (!this.isCacheable(toolName)) {
return undefined;
}
const key = this.generateCacheKey(toolName, args, provider);
const entry = this.cache.get(key);
if (!entry) {
this.totalMisses++;
log.info(`Cache miss for ${toolName}`);
return undefined;
}
// Check if expired
if (new Date() > entry.expiresAt) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
this.currentSize -= entry.size;
this.totalMisses++;
log.info(`Cache expired for ${toolName}`);
return undefined;
}
// Update hit count and access order
if (this.config.trackHits) {
entry.hits++;
this.updateAccessOrder(key);
}
this.totalHits++;
log.info(`Cache hit for ${toolName} (${entry.hits} hits)`);
return entry.value;
}
/**
* Set cached response
*/
set(
toolName: string,
args: Record<string, any>,
value: any,
provider?: string,
ttl?: number
): boolean {
if (!this.isCacheable(toolName)) {
return false;
}
const key = this.generateCacheKey(toolName, args, provider);
const size = this.calculateSize(value);
// Check size limits
if (size > this.config.maxSize) {
log.info(`Cache entry too large for ${toolName}: ${size} bytes`);
return false;
}
// Evict entries if necessary
while (this.cache.size >= this.config.maxEntries ||
this.currentSize + size > this.config.maxSize) {
this.evictLRU();
}
// Determine TTL
const effectiveTTL = ttl ||
TOOL_TTL_OVERRIDES[toolName] ||
this.config.defaultTTL;
// Create entry
const entry: CacheEntry = {
key,
value,
timestamp: new Date(),
expiresAt: new Date(Date.now() + effectiveTTL),
hits: 0,
size,
toolName,
provider
};
// Add to cache
this.cache.set(key, entry);
this.accessOrder.push(key);
this.currentSize += size;
log.info(`Cached response for ${toolName} (${size} bytes, TTL: ${effectiveTTL}ms)`);
return true;
}
/**
* Invalidate cache entries
*/
invalidate(filter?: {
toolName?: string;
provider?: string;
pattern?: RegExp;
}): number {
let invalidated = 0;
for (const [key, entry] of this.cache.entries()) {
let shouldInvalidate = false;
if (filter?.toolName && entry.toolName === filter.toolName) {
shouldInvalidate = true;
}
if (filter?.provider && entry.provider === filter.provider) {
shouldInvalidate = true;
}
if (filter?.pattern && filter.pattern.test(key)) {
shouldInvalidate = true;
}
if (!filter) {
shouldInvalidate = true; // Invalidate all if no filter
}
if (shouldInvalidate) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
this.currentSize -= entry.size;
invalidated++;
}
}
log.info(`Invalidated ${invalidated} cache entries`);
return invalidated;
}
/**
* Evict least recently used entry
*/
private evictLRU(): void {
if (this.accessOrder.length === 0) return;
const key = this.accessOrder.shift()!;
const entry = this.cache.get(key);
if (entry) {
this.cache.delete(key);
this.currentSize -= entry.size;
this.evictionCount++;
log.info(`Evicted cache entry for ${entry.toolName} (LRU)`);
}
}
/**
* Update access order for LRU
*/
private updateAccessOrder(key: string): void {
this.removeFromAccessOrder(key);
this.accessOrder.push(key);
}
/**
* Remove from access order
*/
private removeFromAccessOrder(key: string): void {
const index = this.accessOrder.indexOf(key);
if (index > -1) {
this.accessOrder.splice(index, 1);
}
}
/**
* Calculate size of value in bytes
*/
private calculateSize(value: any): number {
const str = typeof value === 'string' ? value : JSON.stringify(value);
return Buffer.byteLength(str, 'utf8');
}
/**
* Sort object deeply for consistent key generation
*/
private sortObjectDeep(obj: any): any {
if (obj === null || typeof obj !== 'object') {
return obj;
}
if (Array.isArray(obj)) {
return obj.map(item => this.sortObjectDeep(item));
}
const sorted: any = {};
const keys = Object.keys(obj).sort();
for (const key of keys) {
sorted[key] = this.sortObjectDeep(obj[key]);
}
return sorted;
}
/**
* Start automatic cleanup
*/
private startAutoCleanup(): void {
this.cleanupTimer = setInterval(() => {
this.cleanup();
}, this.config.cleanupInterval);
}
/**
* Stop automatic cleanup
*/
private stopAutoCleanup(): void {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = undefined;
}
}
/**
* Clean up expired entries
*/
cleanup(): number {
const now = new Date();
let cleaned = 0;
for (const [key, entry] of this.cache.entries()) {
if (now > entry.expiresAt) {
this.cache.delete(key);
this.removeFromAccessOrder(key);
this.currentSize -= entry.size;
cleaned++;
}
}
if (cleaned > 0) {
log.info(`Cleaned up ${cleaned} expired cache entries`);
}
return cleaned;
}
/**
* Get cache statistics
*/
getStatistics(): CacheStatistics {
const entries = Array.from(this.cache.values());
const totalRequests = this.totalHits + this.totalMisses;
// Calculate tool hit counts
const toolHits = new Map<string, number>();
for (const entry of entries) {
const current = toolHits.get(entry.toolName) || 0;
toolHits.set(entry.toolName, current + entry.hits);
}
// Sort tools by hits
const topTools = Array.from(toolHits.entries())
.map(([tool, hits]) => ({ tool, hits }))
.sort((a, b) => b.hits - a.hits)
.slice(0, 10);
// Find oldest and newest entries
const timestamps = entries.map(e => e.timestamp);
const oldestEntry = timestamps.length > 0
? new Date(Math.min(...timestamps.map(t => t.getTime())))
: undefined;
const newestEntry = timestamps.length > 0
? new Date(Math.max(...timestamps.map(t => t.getTime())))
: undefined;
// Calculate average hits
const totalHitsInCache = entries.reduce((sum, e) => sum + e.hits, 0);
const avgHitsPerEntry = entries.length > 0
? totalHitsInCache / entries.length
: 0;
return {
totalEntries: this.cache.size,
totalSize: this.currentSize,
hitRate: totalRequests > 0 ? this.totalHits / totalRequests : 0,
missRate: totalRequests > 0 ? this.totalMisses / totalRequests : 0,
evictionCount: this.evictionCount,
avgHitsPerEntry,
oldestEntry,
newestEntry,
topTools
};
}
/**
* Clear entire cache
*/
clear(): void {
this.cache.clear();
this.accessOrder = [];
this.currentSize = 0;
this.totalHits = 0;
this.totalMisses = 0;
this.evictionCount = 0;
log.info('Cleared entire cache');
}
/**
* Get cache size info
*/
getSizeInfo(): {
entries: number;
bytes: number;
maxEntries: number;
maxBytes: number;
utilizationPercent: number;
} {
return {
entries: this.cache.size,
bytes: this.currentSize,
maxEntries: this.config.maxEntries,
maxBytes: this.config.maxSize,
utilizationPercent: (this.currentSize / this.config.maxSize) * 100
};
}
/**
* Export cache contents
*/
exportCache(): string {
const data = {
config: this.config,
entries: Array.from(this.cache.entries()),
statistics: this.getStatistics(),
metadata: {
exportedAt: new Date(),
version: '1.0.0'
}
};
return JSON.stringify(data, null, 2);
}
/**
* Import cache contents
*/
importCache(json: string): void {
try {
const data = JSON.parse(json);
// Clear existing cache
this.clear();
// Import entries
for (const [key, entry] of data.entries) {
// Convert dates
entry.timestamp = new Date(entry.timestamp);
entry.expiresAt = new Date(entry.expiresAt);
// Skip expired entries
if (new Date() > entry.expiresAt) continue;
this.cache.set(key, entry);
this.accessOrder.push(key);
this.currentSize += entry.size;
}
log.info(`Imported ${this.cache.size} cache entries`);
} catch (error) {
log.error(`Failed to import cache: ${error}`);
throw error;
}
}
/**
* Shutdown cache
*/
shutdown(): void {
this.stopAutoCleanup();
this.clear();
log.info('Cache shutdown complete');
}
}
// Export singleton instance
export const toolResponseCache = new ToolResponseCache();

View File

@@ -0,0 +1,329 @@
/**
* Tool Timeout Enforcer
*
* Implements timeout enforcement for tool executions with configurable timeouts
* per tool type, graceful cleanup, and Promise.race pattern for detection.
*/
import log from '../../log.js';
import type { ToolHandler } from './tool_interfaces.js';
/**
* Timeout configuration per tool type
*/
export interface TimeoutConfig {
/** Timeout for search operations in milliseconds */
search: number;
/** Timeout for create/update operations in milliseconds */
mutation: number;
/** Timeout for script execution in milliseconds */
script: number;
/** Default timeout for unspecified tools in milliseconds */
default: number;
}
/**
* Tool execution result with timeout metadata
*/
export interface TimeoutResult<T = any> {
success: boolean;
result?: T;
error?: Error;
timedOut: boolean;
executionTime: number;
toolName: string;
}
/**
* Tool categories for timeout assignment
*/
export enum ToolCategory {
SEARCH = 'search',
MUTATION = 'mutation',
SCRIPT = 'script',
READ = 'read',
DEFAULT = 'default'
}
/**
* Default timeout configuration
*/
const DEFAULT_TIMEOUTS: TimeoutConfig = {
search: 5000, // 5 seconds for search operations
mutation: 3000, // 3 seconds for create/update operations
script: 10000, // 10 seconds for script execution
default: 5000 // 5 seconds default
};
/**
* Tool timeout enforcer class
*/
export class ToolTimeoutEnforcer {
private timeouts: TimeoutConfig;
private executionStats: Map<string, { total: number; timeouts: number; avgTime: number }>;
private activeExecutions: Map<string, AbortController>;
constructor(timeoutConfig?: Partial<TimeoutConfig>) {
this.timeouts = { ...DEFAULT_TIMEOUTS, ...timeoutConfig };
this.executionStats = new Map();
this.activeExecutions = new Map();
}
/**
* Categorize tool based on its name
*/
private categorizeeTool(toolName: string): ToolCategory {
const name = toolName.toLowerCase();
// Search tools
if (name.includes('search') || name.includes('find') || name.includes('query')) {
return ToolCategory.SEARCH;
}
// Mutation tools
if (name.includes('create') || name.includes('update') || name.includes('delete') ||
name.includes('modify') || name.includes('save')) {
return ToolCategory.MUTATION;
}
// Script tools
if (name.includes('script') || name.includes('execute') || name.includes('eval')) {
return ToolCategory.SCRIPT;
}
// Read tools
if (name.includes('read') || name.includes('get') || name.includes('fetch')) {
return ToolCategory.READ;
}
return ToolCategory.DEFAULT;
}
/**
* Get timeout for a specific tool
*/
private getToolTimeout(toolName: string): number {
const category = this.categorizeeTool(toolName);
switch (category) {
case ToolCategory.SEARCH:
return this.timeouts.search;
case ToolCategory.MUTATION:
return this.timeouts.mutation;
case ToolCategory.SCRIPT:
return this.timeouts.script;
case ToolCategory.READ:
return this.timeouts.search; // Use search timeout for read operations
default:
return this.timeouts.default;
}
}
/**
* Execute a tool with timeout enforcement
*/
async executeWithTimeout<T>(
toolName: string,
executeFn: () => Promise<T>,
customTimeout?: number
): Promise<TimeoutResult<T>> {
const timeout = customTimeout || this.getToolTimeout(toolName);
const startTime = Date.now();
const executionId = `${toolName}_${startTime}_${Math.random()}`;
// Create abort controller for cleanup
const abortController = new AbortController();
this.activeExecutions.set(executionId, abortController);
log.info(`Executing tool '${toolName}' with timeout ${timeout}ms`);
try {
// Create timeout promise
const timeoutPromise = new Promise<never>((_, reject) => {
const timer = setTimeout(() => {
abortController.abort();
reject(new Error(`Tool '${toolName}' execution timed out after ${timeout}ms`));
}, timeout);
// Clean up timer if aborted
abortController.signal.addEventListener('abort', () => clearTimeout(timer));
});
// Race between execution and timeout
const result = await Promise.race([
executeFn(),
timeoutPromise
]);
const executionTime = Date.now() - startTime;
// Update statistics
this.updateStats(toolName, false, executionTime);
log.info(`Tool '${toolName}' completed successfully in ${executionTime}ms`);
return {
success: true,
result,
timedOut: false,
executionTime,
toolName
};
} catch (error) {
const executionTime = Date.now() - startTime;
const timedOut = executionTime >= timeout - 50; // Allow 50ms buffer
// Update statistics
this.updateStats(toolName, timedOut, executionTime);
if (timedOut) {
log.error(`Tool '${toolName}' timed out after ${executionTime}ms`);
} else {
log.error(`Tool '${toolName}' failed after ${executionTime}ms: ${error}`);
}
return {
success: false,
error: error as Error,
timedOut,
executionTime,
toolName
};
} finally {
// Clean up
this.activeExecutions.delete(executionId);
if (!abortController.signal.aborted) {
abortController.abort();
}
}
}
/**
* Execute multiple tools with timeout enforcement
*/
async executeBatchWithTimeout<T>(
executions: Array<{
toolName: string;
executeFn: () => Promise<T>;
customTimeout?: number;
}>
): Promise<TimeoutResult<T>[]> {
return Promise.all(
executions.map(({ toolName, executeFn, customTimeout }) =>
this.executeWithTimeout(toolName, executeFn, customTimeout)
)
);
}
/**
* Wrap a tool handler with timeout enforcement
*/
wrapToolHandler(handler: ToolHandler, customTimeout?: number): ToolHandler {
const toolName = handler.definition.function.name;
return {
definition: handler.definition,
execute: async (args: Record<string, unknown>) => {
const result = await this.executeWithTimeout(
toolName,
() => handler.execute(args),
customTimeout
);
if (!result.success) {
if (result.timedOut) {
throw new Error(`Tool execution timed out after ${result.executionTime}ms`);
}
throw result.error;
}
return result.result!;
}
};
}
/**
* Update execution statistics
*/
private updateStats(toolName: string, timedOut: boolean, executionTime: number): void {
const current = this.executionStats.get(toolName) || {
total: 0,
timeouts: 0,
avgTime: 0
};
const newTotal = current.total + 1;
const newTimeouts = current.timeouts + (timedOut ? 1 : 0);
const newAvgTime = (current.avgTime * current.total + executionTime) / newTotal;
this.executionStats.set(toolName, {
total: newTotal,
timeouts: newTimeouts,
avgTime: newAvgTime
});
}
/**
* Get execution statistics for a tool
*/
getToolStats(toolName: string) {
return this.executionStats.get(toolName);
}
/**
* Get all execution statistics
*/
getAllStats() {
return Object.fromEntries(this.executionStats);
}
/**
* Clear statistics
*/
clearStats(): void {
this.executionStats.clear();
}
/**
* Abort all active executions
*/
abortAll(): void {
log.info(`Aborting ${this.activeExecutions.size} active tool executions`);
for (const [id, controller] of this.activeExecutions) {
controller.abort();
}
this.activeExecutions.clear();
}
/**
* Get timeout configuration
*/
getTimeouts(): TimeoutConfig {
return { ...this.timeouts };
}
/**
* Update timeout configuration
*/
updateTimeouts(config: Partial<TimeoutConfig>): void {
this.timeouts = { ...this.timeouts, ...config };
log.info(`Updated timeout configuration: ${JSON.stringify(this.timeouts)}`);
}
/**
* Check if a tool has high timeout rate
*/
hasHighTimeoutRate(toolName: string, threshold: number = 0.5): boolean {
const stats = this.executionStats.get(toolName);
if (!stats || stats.total === 0) return false;
return (stats.timeouts / stats.total) > threshold;
}
}
// Export singleton instance
export const toolTimeoutEnforcer = new ToolTimeoutEnforcer();