mirror of
https://github.com/zadam/trilium.git
synced 2025-11-04 20:36:13 +01:00
feat(llm): try to coerce the LLM some more for tool calling
This commit is contained in:
@@ -17,6 +17,16 @@ export interface ToolExecutionOptions {
|
||||
enableFeedback?: boolean;
|
||||
enableErrorRecovery?: boolean;
|
||||
timeout?: number;
|
||||
/** Maximum parallel executions (default: 3) */
|
||||
maxConcurrency?: number;
|
||||
/** Enable dependency analysis for parallel execution (default: true) */
|
||||
analyzeDependencies?: boolean;
|
||||
/** Provider for tool execution */
|
||||
provider?: string;
|
||||
/** Custom timeout per tool in ms */
|
||||
customTimeouts?: Map<string, number>;
|
||||
/** Enable caching for read operations */
|
||||
enableCache?: boolean;
|
||||
onPreview?: (plan: ToolExecutionPlan) => Promise<ToolApproval>;
|
||||
onProgress?: (executionId: string, progress: ToolExecutionProgress) => void;
|
||||
onStep?: (executionId: string, step: any) => void;
|
||||
|
||||
@@ -6,6 +6,9 @@ import type { Message } from "../../ai_interface.js";
|
||||
import { toolPreviewManager } from "../../tools/tool_preview.js";
|
||||
import { toolFeedbackManager } from "../../tools/tool_feedback.js";
|
||||
import { toolErrorRecoveryManager } from "../../tools/tool_error_recovery.js";
|
||||
import { toolTimeoutEnforcer } from "../../tools/tool_timeout_enforcer.js";
|
||||
import { parameterCoercer } from "../../tools/parameter_coercer.js";
|
||||
import { toolExecutionMonitor } from "../../monitoring/tool_execution_monitor.js";
|
||||
|
||||
/**
|
||||
* Handles the execution of LLM tools
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
/**
|
||||
* Provider Health Monitor
|
||||
*
|
||||
* Monitors health status of LLM providers with periodic checks,
|
||||
* automatic disabling after failures, and event emissions.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import log from '../../log.js';
|
||||
import type { AIService } from '../ai_interface.js';
|
||||
import type { ProviderType } from '../providers/provider_factory.js';
|
||||
|
||||
/**
|
||||
* Provider health status
|
||||
*/
|
||||
export interface ProviderHealth {
|
||||
provider: string;
|
||||
healthy: boolean;
|
||||
lastChecked: Date;
|
||||
lastSuccessful?: Date;
|
||||
consecutiveFailures: number;
|
||||
totalChecks: number;
|
||||
totalFailures: number;
|
||||
averageLatency: number;
|
||||
lastError?: string;
|
||||
disabled: boolean;
|
||||
disabledAt?: Date;
|
||||
disabledReason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Health check result
|
||||
*/
|
||||
interface HealthCheckResult {
|
||||
success: boolean;
|
||||
latency: number;
|
||||
error?: string;
|
||||
tokensUsed?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Health monitor configuration
|
||||
*/
|
||||
export interface HealthMonitorConfig {
|
||||
/** Check interval in milliseconds (default: 60000) */
|
||||
checkInterval: number;
|
||||
/** Number of consecutive failures before disabling (default: 3) */
|
||||
failureThreshold: number;
|
||||
/** Timeout for health checks in milliseconds (default: 5000) */
|
||||
checkTimeout: number;
|
||||
/** Enable automatic recovery attempts (default: true) */
|
||||
autoRecover: boolean;
|
||||
/** Recovery check interval in milliseconds (default: 300000) */
|
||||
recoveryInterval: number;
|
||||
/** Minimum time between checks in milliseconds (default: 30000) */
|
||||
minCheckInterval: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration
|
||||
*/
|
||||
const DEFAULT_CONFIG: HealthMonitorConfig = {
|
||||
checkInterval: 60000, // 1 minute
|
||||
failureThreshold: 3,
|
||||
checkTimeout: 5000, // 5 seconds
|
||||
autoRecover: true,
|
||||
recoveryInterval: 300000, // 5 minutes
|
||||
minCheckInterval: 30000 // 30 seconds
|
||||
};
|
||||
|
||||
/**
|
||||
* Provider health monitor class
|
||||
*/
|
||||
export class ProviderHealthMonitor extends EventEmitter {
|
||||
private config: HealthMonitorConfig;
|
||||
private providers: Map<string, AIService>;
|
||||
private healthStatus: Map<string, ProviderHealth>;
|
||||
private checkTimers: Map<string, NodeJS.Timeout>;
|
||||
private isMonitoring: boolean;
|
||||
private lastCheckTime: Map<string, number>;
|
||||
|
||||
constructor(config?: Partial<HealthMonitorConfig>) {
|
||||
super();
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
this.providers = new Map();
|
||||
this.healthStatus = new Map();
|
||||
this.checkTimers = new Map();
|
||||
this.isMonitoring = false;
|
||||
this.lastCheckTime = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a provider for monitoring
|
||||
*/
|
||||
registerProvider(name: string, service: AIService): void {
|
||||
this.providers.set(name, service);
|
||||
this.healthStatus.set(name, {
|
||||
provider: name,
|
||||
healthy: true,
|
||||
lastChecked: new Date(),
|
||||
consecutiveFailures: 0,
|
||||
totalChecks: 0,
|
||||
totalFailures: 0,
|
||||
averageLatency: 0,
|
||||
disabled: false
|
||||
});
|
||||
|
||||
log.info(`Registered provider '${name}' for health monitoring`);
|
||||
|
||||
// Start monitoring if not already running
|
||||
if (!this.isMonitoring) {
|
||||
this.startMonitoring();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a provider
|
||||
*/
|
||||
unregisterProvider(name: string): void {
|
||||
this.providers.delete(name);
|
||||
this.healthStatus.delete(name);
|
||||
|
||||
const timer = this.checkTimers.get(name);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
this.checkTimers.delete(name);
|
||||
}
|
||||
|
||||
log.info(`Unregistered provider '${name}' from health monitoring`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health monitoring
|
||||
*/
|
||||
startMonitoring(): void {
|
||||
if (this.isMonitoring) {
|
||||
log.info('Health monitoring is already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isMonitoring = true;
|
||||
log.info('Starting provider health monitoring');
|
||||
|
||||
// Schedule initial checks for all providers
|
||||
for (const provider of this.providers.keys()) {
|
||||
this.scheduleHealthCheck(provider);
|
||||
}
|
||||
|
||||
this.emit('monitoring:started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop health monitoring
|
||||
*/
|
||||
stopMonitoring(): void {
|
||||
if (!this.isMonitoring) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isMonitoring = false;
|
||||
|
||||
// Clear all timers
|
||||
for (const timer of this.checkTimers.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.checkTimers.clear();
|
||||
|
||||
log.info('Stopped provider health monitoring');
|
||||
this.emit('monitoring:stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule a health check for a provider
|
||||
*/
|
||||
private scheduleHealthCheck(provider: string, delay?: number): void {
|
||||
if (!this.isMonitoring) return;
|
||||
|
||||
// Clear existing timer
|
||||
const existingTimer = this.checkTimers.get(provider);
|
||||
if (existingTimer) {
|
||||
clearTimeout(existingTimer);
|
||||
}
|
||||
|
||||
// Calculate delay based on provider status
|
||||
const status = this.healthStatus.get(provider);
|
||||
const checkDelay = delay || (status?.disabled
|
||||
? this.config.recoveryInterval
|
||||
: this.config.checkInterval);
|
||||
|
||||
// Schedule the check
|
||||
const timer = setTimeout(async () => {
|
||||
await this.performHealthCheck(provider);
|
||||
|
||||
// Schedule next check
|
||||
if (this.isMonitoring) {
|
||||
this.scheduleHealthCheck(provider);
|
||||
}
|
||||
}, checkDelay);
|
||||
|
||||
this.checkTimers.set(provider, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a health check for a provider
|
||||
*/
|
||||
private async performHealthCheck(provider: string): Promise<HealthCheckResult> {
|
||||
const service = this.providers.get(provider);
|
||||
const status = this.healthStatus.get(provider);
|
||||
|
||||
if (!service || !status) {
|
||||
return { success: false, latency: 0, error: 'Provider not found' };
|
||||
}
|
||||
|
||||
// Check if enough time has passed since last check
|
||||
const lastCheck = this.lastCheckTime.get(provider) || 0;
|
||||
const now = Date.now();
|
||||
if (now - lastCheck < this.config.minCheckInterval) {
|
||||
log.info(`Skipping health check for ${provider}, too soon since last check`);
|
||||
return { success: true, latency: 0 };
|
||||
}
|
||||
|
||||
this.lastCheckTime.set(provider, now);
|
||||
|
||||
log.info(`Performing health check for provider '${provider}'`);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Simple ping test with minimal token usage
|
||||
const result = await Promise.race([
|
||||
service.generateChatCompletion(
|
||||
[{
|
||||
role: 'user',
|
||||
content: 'Hi'
|
||||
}],
|
||||
{
|
||||
model: 'default', // Use a default model name
|
||||
maxTokens: 5,
|
||||
temperature: 0
|
||||
}
|
||||
),
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Health check timeout')),
|
||||
this.config.checkTimeout)
|
||||
)
|
||||
]);
|
||||
|
||||
const latency = Date.now() - startTime;
|
||||
|
||||
// Update status for successful check
|
||||
this.updateHealthStatus(provider, {
|
||||
success: true,
|
||||
latency,
|
||||
tokensUsed: (result as any).usage?.totalTokens
|
||||
});
|
||||
|
||||
log.info(`Health check successful for '${provider}' (${latency}ms)`);
|
||||
|
||||
return { success: true, latency, tokensUsed: (result as any).usage?.totalTokens };
|
||||
|
||||
} catch (error) {
|
||||
const latency = Date.now() - startTime;
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
|
||||
// Update status for failed check
|
||||
this.updateHealthStatus(provider, {
|
||||
success: false,
|
||||
latency,
|
||||
error: errorMessage
|
||||
});
|
||||
|
||||
log.error(`Health check failed for '${provider}': ${errorMessage}`);
|
||||
|
||||
return { success: false, latency, error: errorMessage };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update health status based on check result
|
||||
*/
|
||||
private updateHealthStatus(provider: string, result: HealthCheckResult): void {
|
||||
const status = this.healthStatus.get(provider);
|
||||
if (!status) return;
|
||||
|
||||
const wasHealthy = status.healthy;
|
||||
const wasDisabled = status.disabled;
|
||||
|
||||
// Update basic stats
|
||||
status.lastChecked = new Date();
|
||||
status.totalChecks++;
|
||||
|
||||
if (result.success) {
|
||||
// Successful check
|
||||
status.healthy = true;
|
||||
status.lastSuccessful = new Date();
|
||||
status.consecutiveFailures = 0;
|
||||
status.lastError = undefined;
|
||||
|
||||
// Update average latency
|
||||
const prevTotal = status.averageLatency * (status.totalChecks - 1);
|
||||
status.averageLatency = (prevTotal + result.latency) / status.totalChecks;
|
||||
|
||||
// Re-enable if was disabled and auto-recover is on
|
||||
if (status.disabled && this.config.autoRecover) {
|
||||
status.disabled = false;
|
||||
status.disabledAt = undefined;
|
||||
status.disabledReason = undefined;
|
||||
|
||||
log.info(`Provider '${provider}' recovered and re-enabled`);
|
||||
this.emit('provider:recovered', { provider, status });
|
||||
}
|
||||
|
||||
} else {
|
||||
// Failed check
|
||||
status.totalFailures++;
|
||||
status.consecutiveFailures++;
|
||||
status.lastError = result.error;
|
||||
|
||||
// Check if should disable
|
||||
if (status.consecutiveFailures >= this.config.failureThreshold) {
|
||||
status.healthy = false;
|
||||
|
||||
if (!status.disabled) {
|
||||
status.disabled = true;
|
||||
status.disabledAt = new Date();
|
||||
status.disabledReason = `${status.consecutiveFailures} consecutive failures`;
|
||||
|
||||
log.error(`Provider '${provider}' disabled after ${status.consecutiveFailures} failures`);
|
||||
this.emit('provider:disabled', { provider, status, reason: status.disabledReason });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit status change events
|
||||
if (wasHealthy !== status.healthy) {
|
||||
this.emit('provider:health-changed', {
|
||||
provider,
|
||||
healthy: status.healthy,
|
||||
status
|
||||
});
|
||||
}
|
||||
|
||||
if (wasDisabled !== status.disabled) {
|
||||
this.emit('provider:status-changed', {
|
||||
provider,
|
||||
disabled: status.disabled,
|
||||
status
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually trigger a health check
|
||||
*/
|
||||
async checkProvider(provider: string): Promise<HealthCheckResult> {
|
||||
return this.performHealthCheck(provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check all providers
|
||||
*/
|
||||
async checkAllProviders(): Promise<Map<string, HealthCheckResult>> {
|
||||
const results = new Map<string, HealthCheckResult>();
|
||||
|
||||
const checks = Array.from(this.providers.keys()).map(async provider => {
|
||||
const result = await this.performHealthCheck(provider);
|
||||
results.set(provider, result);
|
||||
});
|
||||
|
||||
await Promise.all(checks);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get health status for a provider
|
||||
*/
|
||||
getProviderHealth(provider: string): ProviderHealth | undefined {
|
||||
return this.healthStatus.get(provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all health statuses
|
||||
*/
|
||||
getAllHealthStatus(): Map<string, ProviderHealth> {
|
||||
return new Map(this.healthStatus);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a provider is healthy
|
||||
*/
|
||||
isProviderHealthy(provider: string): boolean {
|
||||
const status = this.healthStatus.get(provider);
|
||||
return status ? status.healthy && !status.disabled : false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get healthy providers
|
||||
*/
|
||||
getHealthyProviders(): string[] {
|
||||
return Array.from(this.healthStatus.entries())
|
||||
.filter(([_, status]) => status.healthy && !status.disabled)
|
||||
.map(([provider, _]) => provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually enable a provider
|
||||
*/
|
||||
enableProvider(provider: string): void {
|
||||
const status = this.healthStatus.get(provider);
|
||||
if (status && status.disabled) {
|
||||
status.disabled = false;
|
||||
status.disabledAt = undefined;
|
||||
status.disabledReason = undefined;
|
||||
status.consecutiveFailures = 0;
|
||||
|
||||
log.info(`Provider '${provider}' manually enabled`);
|
||||
this.emit('provider:enabled', { provider, status });
|
||||
|
||||
// Schedule immediate health check
|
||||
this.scheduleHealthCheck(provider, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually disable a provider
|
||||
*/
|
||||
disableProvider(provider: string, reason?: string): void {
|
||||
const status = this.healthStatus.get(provider);
|
||||
if (status && !status.disabled) {
|
||||
status.disabled = true;
|
||||
status.disabledAt = new Date();
|
||||
status.disabledReason = reason || 'Manually disabled';
|
||||
status.healthy = false;
|
||||
|
||||
log.info(`Provider '${provider}' manually disabled: ${status.disabledReason}`);
|
||||
this.emit('provider:disabled', { provider, status, reason: status.disabledReason });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset statistics for a provider
|
||||
*/
|
||||
resetProviderStats(provider: string): void {
|
||||
const status = this.healthStatus.get(provider);
|
||||
if (status) {
|
||||
status.totalChecks = 0;
|
||||
status.totalFailures = 0;
|
||||
status.averageLatency = 0;
|
||||
status.consecutiveFailures = 0;
|
||||
|
||||
log.info(`Reset statistics for provider '${provider}'`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get monitoring configuration
|
||||
*/
|
||||
getConfig(): HealthMonitorConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update monitoring configuration
|
||||
*/
|
||||
updateConfig(config: Partial<HealthMonitorConfig>): void {
|
||||
this.config = { ...this.config, ...config };
|
||||
log.info(`Updated health monitor configuration: ${JSON.stringify(this.config)}`);
|
||||
|
||||
// Restart monitoring with new config
|
||||
if (this.isMonitoring) {
|
||||
this.stopMonitoring();
|
||||
this.startMonitoring();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const providerHealthMonitor = new ProviderHealthMonitor();
|
||||
@@ -0,0 +1,503 @@
|
||||
/**
|
||||
* Tool Execution Monitor
|
||||
*
|
||||
* Tracks success/failure rates per tool and provider, calculates reliability scores,
|
||||
* auto-disables unreliable tools, and provides metrics for dashboards.
|
||||
*/
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import log from '../../log.js';
|
||||
|
||||
/**
|
||||
* Tool execution statistics
|
||||
*/
|
||||
export interface ToolExecutionStats {
|
||||
toolName: string;
|
||||
provider: string;
|
||||
totalExecutions: number;
|
||||
successfulExecutions: number;
|
||||
failedExecutions: number;
|
||||
timeoutExecutions: number;
|
||||
averageExecutionTime: number;
|
||||
minExecutionTime: number;
|
||||
maxExecutionTime: number;
|
||||
lastExecutionTime?: number;
|
||||
lastExecutionStatus?: 'success' | 'failure' | 'timeout';
|
||||
lastError?: string;
|
||||
reliabilityScore: number;
|
||||
disabled: boolean;
|
||||
disabledAt?: Date;
|
||||
disabledReason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execution record
|
||||
*/
|
||||
export interface ExecutionRecord {
|
||||
toolName: string;
|
||||
provider: string;
|
||||
status: 'success' | 'failure' | 'timeout';
|
||||
executionTime: number;
|
||||
timestamp: Date;
|
||||
error?: string;
|
||||
inputSize?: number;
|
||||
outputSize?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Monitor configuration
|
||||
*/
|
||||
export interface MonitorConfig {
|
||||
/** Failure rate threshold for auto-disable (default: 0.5) */
|
||||
failureRateThreshold: number;
|
||||
/** Minimum executions before calculating reliability (default: 5) */
|
||||
minExecutionsForReliability: number;
|
||||
/** Time window for recent stats in milliseconds (default: 3600000) */
|
||||
recentStatsWindow: number;
|
||||
/** Enable auto-disable of unreliable tools (default: true) */
|
||||
autoDisable: boolean;
|
||||
/** Cooldown period after disable in milliseconds (default: 300000) */
|
||||
disableCooldown: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration
|
||||
*/
|
||||
const DEFAULT_CONFIG: MonitorConfig = {
|
||||
failureRateThreshold: 0.5,
|
||||
minExecutionsForReliability: 5,
|
||||
recentStatsWindow: 3600000, // 1 hour
|
||||
autoDisable: true,
|
||||
disableCooldown: 300000 // 5 minutes
|
||||
};
|
||||
|
||||
/**
|
||||
* Tool execution monitor class
|
||||
*/
|
||||
export class ToolExecutionMonitor extends EventEmitter {
|
||||
private config: MonitorConfig;
|
||||
private stats: Map<string, ToolExecutionStats>;
|
||||
private recentExecutions: ExecutionRecord[];
|
||||
private disabledTools: Set<string>;
|
||||
|
||||
constructor(config?: Partial<MonitorConfig>) {
|
||||
super();
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
this.stats = new Map();
|
||||
this.recentExecutions = [];
|
||||
this.disabledTools = new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a tool execution
|
||||
*/
|
||||
recordExecution(record: ExecutionRecord): void {
|
||||
const key = this.getStatsKey(record.toolName, record.provider);
|
||||
|
||||
// Update or create stats
|
||||
let stats = this.stats.get(key);
|
||||
if (!stats) {
|
||||
stats = this.createEmptyStats(record.toolName, record.provider);
|
||||
this.stats.set(key, stats);
|
||||
}
|
||||
|
||||
// Update counters
|
||||
stats.totalExecutions++;
|
||||
|
||||
switch (record.status) {
|
||||
case 'success':
|
||||
stats.successfulExecutions++;
|
||||
break;
|
||||
case 'failure':
|
||||
stats.failedExecutions++;
|
||||
break;
|
||||
case 'timeout':
|
||||
stats.timeoutExecutions++;
|
||||
break;
|
||||
}
|
||||
|
||||
// Update timing statistics
|
||||
this.updateTimingStats(stats, record.executionTime);
|
||||
|
||||
// Update last execution info
|
||||
stats.lastExecutionTime = record.executionTime;
|
||||
stats.lastExecutionStatus = record.status;
|
||||
stats.lastError = record.error;
|
||||
|
||||
// Calculate reliability score
|
||||
stats.reliabilityScore = this.calculateReliabilityScore(stats);
|
||||
|
||||
// Add to recent executions
|
||||
this.recentExecutions.push(record);
|
||||
this.pruneRecentExecutions();
|
||||
|
||||
// Check if tool should be auto-disabled
|
||||
if (this.config.autoDisable && this.shouldAutoDisable(stats)) {
|
||||
this.disableTool(record.toolName, record.provider, 'High failure rate');
|
||||
}
|
||||
|
||||
// Emit events
|
||||
this.emit('execution:recorded', record);
|
||||
|
||||
if (record.status === 'failure') {
|
||||
this.emit('execution:failed', record);
|
||||
} else if (record.status === 'timeout') {
|
||||
this.emit('execution:timeout', record);
|
||||
}
|
||||
|
||||
// Log if reliability is concerning
|
||||
if (stats.reliabilityScore < 0.5 && stats.totalExecutions >= this.config.minExecutionsForReliability) {
|
||||
log.info(`Tool '${record.toolName}' has low reliability score: ${stats.reliabilityScore.toFixed(2)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update timing statistics
|
||||
*/
|
||||
private updateTimingStats(stats: ToolExecutionStats, executionTime: number): void {
|
||||
const prevAvg = stats.averageExecutionTime;
|
||||
const prevCount = stats.totalExecutions - 1;
|
||||
|
||||
// Update average
|
||||
stats.averageExecutionTime = prevCount === 0
|
||||
? executionTime
|
||||
: (prevAvg * prevCount + executionTime) / stats.totalExecutions;
|
||||
|
||||
// Update min/max
|
||||
if (stats.minExecutionTime === 0 || executionTime < stats.minExecutionTime) {
|
||||
stats.minExecutionTime = executionTime;
|
||||
}
|
||||
if (executionTime > stats.maxExecutionTime) {
|
||||
stats.maxExecutionTime = executionTime;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate reliability score (0-1)
|
||||
*/
|
||||
private calculateReliabilityScore(stats: ToolExecutionStats): number {
|
||||
if (stats.totalExecutions === 0) return 1;
|
||||
|
||||
// Weight factors
|
||||
const successWeight = 0.7;
|
||||
const timeoutWeight = 0.2;
|
||||
const consistencyWeight = 0.1;
|
||||
|
||||
// Success rate
|
||||
const successRate = stats.successfulExecutions / stats.totalExecutions;
|
||||
|
||||
// Timeout penalty
|
||||
const timeoutRate = stats.timeoutExecutions / stats.totalExecutions;
|
||||
const timeoutScore = 1 - timeoutRate;
|
||||
|
||||
// Consistency score (based on execution time variance)
|
||||
let consistencyScore = 1;
|
||||
if (stats.totalExecutions > 1 && stats.averageExecutionTime > 0) {
|
||||
const variance = (stats.maxExecutionTime - stats.minExecutionTime) / stats.averageExecutionTime;
|
||||
consistencyScore = Math.max(0, 1 - variance / 10); // Normalize variance
|
||||
}
|
||||
|
||||
// Calculate weighted score
|
||||
const score =
|
||||
successRate * successWeight +
|
||||
timeoutScore * timeoutWeight +
|
||||
consistencyScore * consistencyWeight;
|
||||
|
||||
return Math.min(1, Math.max(0, score));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if tool should be auto-disabled
|
||||
*/
|
||||
private shouldAutoDisable(stats: ToolExecutionStats): boolean {
|
||||
// Don't disable if already disabled
|
||||
if (stats.disabled) return false;
|
||||
|
||||
// Need minimum executions
|
||||
if (stats.totalExecutions < this.config.minExecutionsForReliability) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check failure rate
|
||||
const failureRate = (stats.failedExecutions + stats.timeoutExecutions) / stats.totalExecutions;
|
||||
return failureRate > this.config.failureRateThreshold;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable a tool
|
||||
*/
|
||||
disableTool(toolName: string, provider: string, reason: string): void {
|
||||
const key = this.getStatsKey(toolName, provider);
|
||||
const stats = this.stats.get(key);
|
||||
|
||||
if (!stats || stats.disabled) return;
|
||||
|
||||
stats.disabled = true;
|
||||
stats.disabledAt = new Date();
|
||||
stats.disabledReason = reason;
|
||||
|
||||
this.disabledTools.add(key);
|
||||
|
||||
log.error(`Tool '${toolName}' disabled for provider '${provider}': ${reason}`);
|
||||
this.emit('tool:disabled', { toolName, provider, reason, stats });
|
||||
|
||||
// Schedule re-enable check
|
||||
if (this.config.disableCooldown > 0) {
|
||||
setTimeout(() => {
|
||||
this.checkReEnableTool(toolName, provider);
|
||||
}, this.config.disableCooldown);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool can be re-enabled
|
||||
*/
|
||||
private checkReEnableTool(toolName: string, provider: string): void {
|
||||
const key = this.getStatsKey(toolName, provider);
|
||||
const stats = this.stats.get(key);
|
||||
|
||||
if (!stats || !stats.disabled) return;
|
||||
|
||||
// Calculate recent success rate
|
||||
const recentExecutions = this.getRecentExecutions(toolName, provider);
|
||||
if (recentExecutions.length === 0) {
|
||||
// No recent executions, re-enable for retry
|
||||
this.enableTool(toolName, provider);
|
||||
return;
|
||||
}
|
||||
|
||||
const recentSuccesses = recentExecutions.filter(e => e.status === 'success').length;
|
||||
const recentSuccessRate = recentSuccesses / recentExecutions.length;
|
||||
|
||||
// Re-enable if recent performance is good
|
||||
if (recentSuccessRate > 0.7) {
|
||||
this.enableTool(toolName, provider);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable a tool
|
||||
*/
|
||||
enableTool(toolName: string, provider: string): void {
|
||||
const key = this.getStatsKey(toolName, provider);
|
||||
const stats = this.stats.get(key);
|
||||
|
||||
if (!stats || !stats.disabled) return;
|
||||
|
||||
stats.disabled = false;
|
||||
stats.disabledAt = undefined;
|
||||
stats.disabledReason = undefined;
|
||||
|
||||
this.disabledTools.delete(key);
|
||||
|
||||
log.info(`Tool '${toolName}' re-enabled for provider '${provider}'`);
|
||||
this.emit('tool:enabled', { toolName, provider, stats });
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats for a tool
|
||||
*/
|
||||
getToolStats(toolName: string, provider: string): ToolExecutionStats | undefined {
|
||||
return this.stats.get(this.getStatsKey(toolName, provider));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all stats
|
||||
*/
|
||||
getAllStats(): Map<string, ToolExecutionStats> {
|
||||
return new Map(this.stats);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats by provider
|
||||
*/
|
||||
getStatsByProvider(provider: string): ToolExecutionStats[] {
|
||||
return Array.from(this.stats.values()).filter(s => s.provider === provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats by tool
|
||||
*/
|
||||
getStatsByTool(toolName: string): ToolExecutionStats[] {
|
||||
return Array.from(this.stats.values()).filter(s => s.toolName === toolName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent executions for a tool
|
||||
*/
|
||||
getRecentExecutions(toolName: string, provider: string): ExecutionRecord[] {
|
||||
const cutoff = Date.now() - this.config.recentStatsWindow;
|
||||
return this.recentExecutions.filter(e =>
|
||||
e.toolName === toolName &&
|
||||
e.provider === provider &&
|
||||
e.timestamp.getTime() > cutoff
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metrics for dashboard
|
||||
*/
|
||||
getDashboardMetrics(): {
|
||||
totalTools: number;
|
||||
activeTools: number;
|
||||
disabledTools: number;
|
||||
overallReliability: number;
|
||||
topPerformers: ToolExecutionStats[];
|
||||
bottomPerformers: ToolExecutionStats[];
|
||||
recentFailures: ExecutionRecord[];
|
||||
} {
|
||||
const allStats = Array.from(this.stats.values());
|
||||
const activeStats = allStats.filter(s => !s.disabled);
|
||||
|
||||
// Calculate overall reliability
|
||||
const overallReliability = activeStats.length > 0
|
||||
? activeStats.reduce((sum, s) => sum + s.reliabilityScore, 0) / activeStats.length
|
||||
: 1;
|
||||
|
||||
// Sort by reliability
|
||||
const sorted = [...allStats].sort((a, b) => b.reliabilityScore - a.reliabilityScore);
|
||||
|
||||
// Get recent failures
|
||||
const recentFailures = this.recentExecutions
|
||||
.filter(e => e.status !== 'success')
|
||||
.slice(-10);
|
||||
|
||||
return {
|
||||
totalTools: allStats.length,
|
||||
activeTools: activeStats.length,
|
||||
disabledTools: this.disabledTools.size,
|
||||
overallReliability,
|
||||
topPerformers: sorted.slice(0, 5),
|
||||
bottomPerformers: sorted.slice(-5).reverse(),
|
||||
recentFailures
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool is disabled
|
||||
*/
|
||||
isToolDisabled(toolName: string, provider: string): boolean {
|
||||
return this.disabledTools.has(this.getStatsKey(toolName, provider));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset stats for a tool
|
||||
*/
|
||||
resetToolStats(toolName: string, provider: string): void {
|
||||
const key = this.getStatsKey(toolName, provider);
|
||||
this.stats.delete(key);
|
||||
this.disabledTools.delete(key);
|
||||
|
||||
// Remove from recent executions
|
||||
this.recentExecutions = this.recentExecutions.filter(e =>
|
||||
!(e.toolName === toolName && e.provider === provider)
|
||||
);
|
||||
|
||||
log.info(`Reset stats for tool '${toolName}' with provider '${provider}'`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all statistics
|
||||
*/
|
||||
resetAllStats(): void {
|
||||
this.stats.clear();
|
||||
this.recentExecutions = [];
|
||||
this.disabledTools.clear();
|
||||
log.info('Reset all tool execution statistics');
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune old recent executions
|
||||
*/
|
||||
private pruneRecentExecutions(): void {
|
||||
const cutoff = Date.now() - this.config.recentStatsWindow;
|
||||
this.recentExecutions = this.recentExecutions.filter(e =>
|
||||
e.timestamp.getTime() > cutoff
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create empty stats object
|
||||
*/
|
||||
private createEmptyStats(toolName: string, provider: string): ToolExecutionStats {
|
||||
return {
|
||||
toolName,
|
||||
provider,
|
||||
totalExecutions: 0,
|
||||
successfulExecutions: 0,
|
||||
failedExecutions: 0,
|
||||
timeoutExecutions: 0,
|
||||
averageExecutionTime: 0,
|
||||
minExecutionTime: 0,
|
||||
maxExecutionTime: 0,
|
||||
reliabilityScore: 1,
|
||||
disabled: false
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get stats key
|
||||
*/
|
||||
private getStatsKey(toolName: string, provider: string): string {
|
||||
return `${provider}:${toolName}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export statistics to JSON
|
||||
*/
|
||||
exportStats(): string {
|
||||
return JSON.stringify({
|
||||
stats: Array.from(this.stats.entries()),
|
||||
recentExecutions: this.recentExecutions,
|
||||
disabledTools: Array.from(this.disabledTools),
|
||||
config: this.config
|
||||
}, null, 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Import statistics from JSON
|
||||
*/
|
||||
importStats(json: string): void {
|
||||
try {
|
||||
const data = JSON.parse(json);
|
||||
|
||||
// Restore stats
|
||||
this.stats.clear();
|
||||
for (const [key, value] of data.stats) {
|
||||
this.stats.set(key, value);
|
||||
}
|
||||
|
||||
// Restore recent executions with date conversion
|
||||
this.recentExecutions = data.recentExecutions.map((e: any) => ({
|
||||
...e,
|
||||
timestamp: new Date(e.timestamp)
|
||||
}));
|
||||
|
||||
// Restore disabled tools
|
||||
this.disabledTools = new Set(data.disabledTools);
|
||||
|
||||
log.info('Imported tool execution statistics');
|
||||
} catch (error) {
|
||||
log.error(`Failed to import statistics: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get configuration
|
||||
*/
|
||||
getConfig(): MonitorConfig {
|
||||
return { ...this.config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration
|
||||
*/
|
||||
updateConfig(config: Partial<MonitorConfig>): void {
|
||||
this.config = { ...this.config, ...config };
|
||||
log.info(`Updated tool execution monitor configuration: ${JSON.stringify(this.config)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const toolExecutionMonitor = new ToolExecutionMonitor();
|
||||
228
apps/server/src/services/llm/pipeline/PHASE_2_IMPLEMENTATION.md
Normal file
228
apps/server/src/services/llm/pipeline/PHASE_2_IMPLEMENTATION.md
Normal file
@@ -0,0 +1,228 @@
|
||||
# Phase 2: Simplification Implementation
|
||||
|
||||
## Overview
|
||||
This document describes the implementation of Phase 2 of the LLM improvement plan, focusing on architectural simplification, centralized configuration, and improved logging.
|
||||
|
||||
## Implemented Components
|
||||
|
||||
### Phase 2.1: Pipeline Architecture Simplification
|
||||
**File:** `simplified_pipeline.ts` (396 lines)
|
||||
|
||||
The original 986-line pipeline with 9 stages has been reduced to 4 essential stages:
|
||||
|
||||
1. **Message Preparation** - Combines formatting, context enrichment, and system prompt injection
|
||||
2. **LLM Execution** - Handles provider selection and API calls
|
||||
3. **Tool Handling** - Manages tool parsing, execution, and follow-up calls
|
||||
4. **Response Processing** - Formats responses and handles streaming
|
||||
|
||||
Key improvements:
|
||||
- Reduced code complexity by ~60%
|
||||
- Removed unnecessary abstractions
|
||||
- Consolidated duplicate logic
|
||||
- Clearer separation of concerns
|
||||
|
||||
### Phase 2.2: Configuration Management
|
||||
|
||||
#### Configuration Service
|
||||
**File:** `configuration_service.ts` (354 lines)
|
||||
|
||||
Centralizes all LLM configuration:
|
||||
- Single source of truth for all settings
|
||||
- Type-safe configuration access
|
||||
- Validation at startup
|
||||
- Cache with automatic refresh
|
||||
- No more scattered `options.getOption()` calls
|
||||
|
||||
Configuration categories:
|
||||
- Provider settings (API keys, endpoints, models)
|
||||
- Default parameters (temperature, tokens, system prompt)
|
||||
- Tool configuration (iterations, timeout, parallel execution)
|
||||
- Streaming settings (enabled, chunk size, flush interval)
|
||||
- Debug configuration (log level, metrics, tracing)
|
||||
- Limits (message length, conversation length, rate limiting)
|
||||
|
||||
#### Model Registry
|
||||
**File:** `model_registry.ts` (474 lines)
|
||||
|
||||
Manages model capabilities and metadata:
|
||||
- Built-in model definitions for OpenAI, Anthropic, and Ollama
|
||||
- Model capabilities (tools, streaming, vision, JSON mode)
|
||||
- Cost tracking (per 1K tokens)
|
||||
- Performance characteristics (latency, throughput, reliability)
|
||||
- Intelligent model selection based on use case
|
||||
- Custom model registration for Ollama
|
||||
|
||||
### Phase 2.3: Logging Improvements
|
||||
|
||||
#### Logging Service
|
||||
**File:** `logging_service.ts` (378 lines)
|
||||
|
||||
Structured logging with:
|
||||
- Proper log levels (ERROR, WARN, INFO, DEBUG)
|
||||
- Request ID tracking for tracing
|
||||
- Conditional debug logging (disabled in production)
|
||||
- Log buffering for debugging
|
||||
- Performance timers
|
||||
- Contextual logging with metadata
|
||||
|
||||
#### Debug Cleanup Script
|
||||
**File:** `cleanup_debug_logs.ts` (198 lines)
|
||||
|
||||
Utility to clean up debug statements:
|
||||
- Finds `log.info("[DEBUG]")` patterns
|
||||
- Converts to proper debug level
|
||||
- Reports on verbose logging
|
||||
- Dry-run mode for safety
|
||||
|
||||
### Integration Layer
|
||||
|
||||
#### Pipeline Adapter
|
||||
**File:** `pipeline_adapter.ts` (140 lines)
|
||||
|
||||
Provides backward compatibility:
|
||||
- Maintains existing `ChatPipeline` interface
|
||||
- Uses simplified pipeline underneath
|
||||
- Gradual migration path
|
||||
- Feature flag support
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### Step 1: Update Imports
|
||||
```typescript
|
||||
// Old
|
||||
import { ChatPipeline } from "../pipeline/chat_pipeline.js";
|
||||
|
||||
// New
|
||||
import { ChatPipeline } from "../pipeline/pipeline_adapter.js";
|
||||
```
|
||||
|
||||
### Step 2: Initialize Configuration
|
||||
```typescript
|
||||
// On startup
|
||||
await configurationService.initialize();
|
||||
```
|
||||
|
||||
### Step 3: Use Structured Logging
|
||||
```typescript
|
||||
// Old
|
||||
log.info(`[DEBUG] Processing request for user ${userId}`);
|
||||
|
||||
// New
|
||||
const logger = loggingService.withRequestId(requestId);
|
||||
logger.debug('Processing request', { userId });
|
||||
```
|
||||
|
||||
### Step 4: Access Configuration
|
||||
```typescript
|
||||
// Old
|
||||
const model = options.getOption('openaiDefaultModel');
|
||||
|
||||
// New
|
||||
const model = configurationService.getProviderConfig().openai?.defaultModel;
|
||||
```
|
||||
|
||||
## Benefits Achieved
|
||||
|
||||
### Code Simplification
|
||||
- **60% reduction** in pipeline code (986 → 396 lines)
|
||||
- **9 stages → 4 stages** for easier understanding
|
||||
- Removed unnecessary abstractions
|
||||
|
||||
### Better Configuration
|
||||
- **Single source of truth** for all configuration
|
||||
- **Type-safe** access with IntelliSense support
|
||||
- **Validation** catches errors at startup
|
||||
- **Centralized** management reduces duplication
|
||||
|
||||
### Improved Logging
|
||||
- **Structured logs** with consistent format
|
||||
- **Request tracing** with unique IDs
|
||||
- **Performance metrics** built-in
|
||||
- **Production-ready** with debug statements removed
|
||||
|
||||
### Maintainability
|
||||
- **Clear separation** of concerns
|
||||
- **Testable** components with dependency injection
|
||||
- **Gradual migration** path with adapter
|
||||
- **Well-documented** interfaces
|
||||
|
||||
## Testing
|
||||
|
||||
### Unit Tests
|
||||
**File:** `simplified_pipeline.spec.ts`
|
||||
|
||||
Comprehensive test coverage for:
|
||||
- Simple chat flows
|
||||
- Tool execution
|
||||
- Streaming responses
|
||||
- Error handling
|
||||
- Metrics tracking
|
||||
- Context enrichment
|
||||
|
||||
### Running Tests
|
||||
```bash
|
||||
# Run all pipeline tests
|
||||
pnpm nx test server --testPathPattern=pipeline
|
||||
|
||||
# Run specific test file
|
||||
pnpm nx test server --testFile=simplified_pipeline.spec.ts
|
||||
```
|
||||
|
||||
## Performance Impact
|
||||
|
||||
### Reduced Overhead
|
||||
- Fewer function calls in hot path
|
||||
- Less object creation
|
||||
- Simplified async flow
|
||||
|
||||
### Better Resource Usage
|
||||
- Configuration caching reduces database queries
|
||||
- Streamlined logging reduces I/O
|
||||
- Efficient metric collection
|
||||
|
||||
## Next Steps
|
||||
|
||||
### Immediate Actions
|
||||
1. Deploy with feature flag enabled
|
||||
2. Monitor performance metrics
|
||||
3. Gather feedback from users
|
||||
|
||||
### Future Improvements
|
||||
1. Implement remaining phases from improvement plan
|
||||
2. Add telemetry for production monitoring
|
||||
3. Create migration tools for existing configurations
|
||||
4. Build admin UI for configuration management
|
||||
|
||||
## Environment Variables
|
||||
|
||||
```bash
|
||||
# Enable simplified pipeline (default: true)
|
||||
USE_SIMPLIFIED_PIPELINE=true
|
||||
|
||||
# Enable debug logging
|
||||
LLM_DEBUG_ENABLED=true
|
||||
|
||||
# Set log level (error, warn, info, debug)
|
||||
LLM_LOG_LEVEL=info
|
||||
```
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
If issues are encountered:
|
||||
|
||||
1. **Quick rollback:** Set `USE_SIMPLIFIED_PIPELINE=false`
|
||||
2. **Revert imports:** Change back to original `chat_pipeline.js`
|
||||
3. **Monitor logs:** Check for any errors or warnings
|
||||
|
||||
The adapter ensures backward compatibility, making rollback seamless.
|
||||
|
||||
## Conclusion
|
||||
|
||||
Phase 2 successfully simplifies the LLM pipeline architecture while maintaining all functionality. The implementation provides:
|
||||
|
||||
- **Cleaner code** that's easier to understand and maintain
|
||||
- **Better configuration** management with validation
|
||||
- **Improved logging** for debugging and monitoring
|
||||
- **Backward compatibility** for gradual migration
|
||||
|
||||
The simplified architecture provides a solid foundation for future enhancements and makes the codebase more accessible to new contributors.
|
||||
563
apps/server/src/services/llm/providers/edge_case_handler.ts
Normal file
563
apps/server/src/services/llm/providers/edge_case_handler.ts
Normal file
@@ -0,0 +1,563 @@
|
||||
/**
|
||||
* Provider Edge Case Handler
|
||||
*
|
||||
* Handles provider-specific edge cases and quirks for OpenAI, Anthropic, and Ollama,
|
||||
* including special character fixes, object flattening, and context limit handling.
|
||||
*/
|
||||
|
||||
import log from '../../log.js';
|
||||
import type { Tool, ToolParameter } from '../tools/tool_interfaces.js';
|
||||
|
||||
/**
|
||||
* Edge case fix result
|
||||
*/
|
||||
export interface EdgeCaseFixResult {
|
||||
fixed: boolean;
|
||||
tool?: Tool;
|
||||
warnings: string[];
|
||||
modifications: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider-specific configuration
|
||||
*/
|
||||
interface ProviderConfig {
|
||||
maxFunctionNameLength: number;
|
||||
maxDescriptionLength: number;
|
||||
maxDepth: number;
|
||||
maxProperties: number;
|
||||
allowSpecialChars: boolean;
|
||||
requireArrays: boolean;
|
||||
supportsComplexTypes: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider configurations
|
||||
*/
|
||||
const PROVIDER_CONFIGS: Record<string, ProviderConfig> = {
|
||||
openai: {
|
||||
maxFunctionNameLength: 64,
|
||||
maxDescriptionLength: 1024,
|
||||
maxDepth: 5,
|
||||
maxProperties: 50,
|
||||
allowSpecialChars: false,
|
||||
requireArrays: false,
|
||||
supportsComplexTypes: true
|
||||
},
|
||||
anthropic: {
|
||||
maxFunctionNameLength: 64,
|
||||
maxDescriptionLength: 1024,
|
||||
maxDepth: 4,
|
||||
maxProperties: 30,
|
||||
allowSpecialChars: true,
|
||||
requireArrays: true,
|
||||
supportsComplexTypes: true
|
||||
},
|
||||
ollama: {
|
||||
maxFunctionNameLength: 50,
|
||||
maxDescriptionLength: 500,
|
||||
maxDepth: 3,
|
||||
maxProperties: 20,
|
||||
allowSpecialChars: false,
|
||||
requireArrays: false,
|
||||
supportsComplexTypes: false
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Edge case handler class
|
||||
*/
|
||||
export class EdgeCaseHandler {
|
||||
/**
|
||||
* Fix tool for provider-specific edge cases
|
||||
*/
|
||||
fixToolForProvider(tool: Tool, provider: string): EdgeCaseFixResult {
|
||||
const config = PROVIDER_CONFIGS[provider] || PROVIDER_CONFIGS.openai;
|
||||
const warnings: string[] = [];
|
||||
const modifications: string[] = [];
|
||||
|
||||
// Deep clone the tool
|
||||
let fixedTool = JSON.parse(JSON.stringify(tool)) as Tool;
|
||||
let wasFixed = false;
|
||||
|
||||
// Apply provider-specific fixes
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
const openaiResult = this.fixOpenAIEdgeCases(fixedTool, config);
|
||||
fixedTool = openaiResult.tool;
|
||||
warnings.push(...openaiResult.warnings);
|
||||
modifications.push(...openaiResult.modifications);
|
||||
wasFixed = openaiResult.fixed;
|
||||
break;
|
||||
|
||||
case 'anthropic':
|
||||
const anthropicResult = this.fixAnthropicEdgeCases(fixedTool, config);
|
||||
fixedTool = anthropicResult.tool;
|
||||
warnings.push(...anthropicResult.warnings);
|
||||
modifications.push(...anthropicResult.modifications);
|
||||
wasFixed = anthropicResult.fixed;
|
||||
break;
|
||||
|
||||
case 'ollama':
|
||||
const ollamaResult = this.fixOllamaEdgeCases(fixedTool, config);
|
||||
fixedTool = ollamaResult.tool;
|
||||
warnings.push(...ollamaResult.warnings);
|
||||
modifications.push(...ollamaResult.modifications);
|
||||
wasFixed = ollamaResult.fixed;
|
||||
break;
|
||||
|
||||
default:
|
||||
// Apply generic fixes
|
||||
const genericResult = this.applyGenericFixes(fixedTool, config);
|
||||
fixedTool = genericResult.tool;
|
||||
warnings.push(...genericResult.warnings);
|
||||
modifications.push(...genericResult.modifications);
|
||||
wasFixed = genericResult.fixed;
|
||||
}
|
||||
|
||||
return {
|
||||
fixed: wasFixed,
|
||||
tool: wasFixed ? fixedTool : undefined,
|
||||
warnings,
|
||||
modifications
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix OpenAI-specific edge cases
|
||||
*/
|
||||
private fixOpenAIEdgeCases(
|
||||
tool: Tool,
|
||||
config: ProviderConfig
|
||||
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
|
||||
const warnings: string[] = [];
|
||||
const modifications: string[] = [];
|
||||
let fixed = false;
|
||||
|
||||
// Fix special characters in function name
|
||||
if (!config.allowSpecialChars && /[^a-zA-Z0-9_]/.test(tool.function.name)) {
|
||||
const oldName = tool.function.name;
|
||||
tool.function.name = tool.function.name.replace(/[^a-zA-Z0-9_]/g, '_');
|
||||
modifications.push(`Replaced special characters in function name: ${oldName} → ${tool.function.name}`);
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
// Fix hyphens (OpenAI prefers underscores)
|
||||
if (tool.function.name.includes('-')) {
|
||||
const oldName = tool.function.name;
|
||||
tool.function.name = tool.function.name.replace(/-/g, '_');
|
||||
modifications.push(`Replaced hyphens with underscores: ${oldName} → ${tool.function.name}`);
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
// Flatten deep objects if necessary
|
||||
if (tool.function.parameters.properties) {
|
||||
const flattenResult = this.flattenDeepObjects(
|
||||
tool.function.parameters.properties,
|
||||
config.maxDepth
|
||||
);
|
||||
if (flattenResult.flattened) {
|
||||
tool.function.parameters.properties = flattenResult.properties;
|
||||
modifications.push('Flattened deep nested objects');
|
||||
warnings.push('Some nested properties were flattened for OpenAI compatibility');
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle overly complex parameter structures
|
||||
const paramCount = Object.keys(tool.function.parameters.properties || {}).length;
|
||||
if (paramCount > config.maxProperties) {
|
||||
warnings.push(`Tool has ${paramCount} properties, exceeding OpenAI recommended limit of ${config.maxProperties}`);
|
||||
|
||||
// Group related parameters if possible
|
||||
const grouped = this.groupRelatedParameters(tool.function.parameters.properties);
|
||||
if (grouped.grouped) {
|
||||
tool.function.parameters.properties = grouped.properties;
|
||||
modifications.push('Grouped related parameters to reduce complexity');
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Fix enum values with special characters
|
||||
this.fixEnumValues(tool.function.parameters.properties);
|
||||
|
||||
return { tool, fixed, warnings, modifications };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix Anthropic-specific edge cases
|
||||
*/
|
||||
private fixAnthropicEdgeCases(
|
||||
tool: Tool,
|
||||
config: ProviderConfig
|
||||
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
|
||||
const warnings: string[] = [];
|
||||
const modifications: string[] = [];
|
||||
let fixed = false;
|
||||
|
||||
// Ensure required array is not empty
|
||||
if (!tool.function.parameters.required || tool.function.parameters.required.length === 0) {
|
||||
const properties = Object.keys(tool.function.parameters.properties || {});
|
||||
if (properties.length > 0) {
|
||||
// Add at least one property to required
|
||||
tool.function.parameters.required = [properties[0]];
|
||||
modifications.push(`Added '${properties[0]}' to required array for Anthropic compatibility`);
|
||||
fixed = true;
|
||||
} else {
|
||||
// Add a dummy optional parameter if no properties exist
|
||||
tool.function.parameters.properties = {
|
||||
_placeholder: {
|
||||
type: 'string',
|
||||
description: 'Optional placeholder parameter',
|
||||
default: ''
|
||||
}
|
||||
};
|
||||
tool.function.parameters.required = [];
|
||||
modifications.push('Added placeholder parameter for Anthropic compatibility');
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate overly long descriptions
|
||||
if (tool.function.description.length > config.maxDescriptionLength) {
|
||||
tool.function.description = tool.function.description.substring(0, config.maxDescriptionLength - 3) + '...';
|
||||
modifications.push('Truncated description to meet Anthropic length limits');
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
// Ensure all parameters have descriptions
|
||||
for (const [key, param] of Object.entries(tool.function.parameters.properties || {})) {
|
||||
if (!param.description) {
|
||||
param.description = `Parameter ${key}`;
|
||||
modifications.push(`Added missing description for parameter '${key}'`);
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle complex nested structures
|
||||
const complexity = this.calculateComplexity(tool.function.parameters);
|
||||
if (complexity > 15) {
|
||||
warnings.push('Tool parameters are very complex for Anthropic, consider simplifying');
|
||||
}
|
||||
|
||||
return { tool, fixed, warnings, modifications };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix Ollama-specific edge cases
|
||||
*/
|
||||
private fixOllamaEdgeCases(
|
||||
tool: Tool,
|
||||
config: ProviderConfig
|
||||
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
|
||||
const warnings: string[] = [];
|
||||
const modifications: string[] = [];
|
||||
let fixed = false;
|
||||
|
||||
// Limit parameter count for local models
|
||||
const properties = tool.function.parameters.properties || {};
|
||||
const paramCount = Object.keys(properties).length;
|
||||
|
||||
if (paramCount > config.maxProperties) {
|
||||
// Keep only the most important parameters
|
||||
const required = tool.function.parameters.required || [];
|
||||
const important = new Set(required);
|
||||
const kept: Record<string, ToolParameter> = {};
|
||||
|
||||
// Keep required parameters first
|
||||
for (const key of required) {
|
||||
if (properties[key]) {
|
||||
kept[key] = properties[key];
|
||||
}
|
||||
}
|
||||
|
||||
// Add optional parameters up to limit
|
||||
for (const [key, param] of Object.entries(properties)) {
|
||||
if (!important.has(key) && Object.keys(kept).length < config.maxProperties) {
|
||||
kept[key] = param;
|
||||
}
|
||||
}
|
||||
|
||||
tool.function.parameters.properties = kept;
|
||||
modifications.push(`Reduced parameters from ${paramCount} to ${Object.keys(kept).length} for Ollama`);
|
||||
warnings.push('Some optional parameters were removed for local model compatibility');
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
// Simplify complex types
|
||||
if (!config.supportsComplexTypes) {
|
||||
const simplified = this.simplifyComplexTypes(tool.function.parameters.properties);
|
||||
if (simplified.simplified) {
|
||||
tool.function.parameters.properties = simplified.properties;
|
||||
modifications.push('Simplified complex types for local model compatibility');
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Shorten descriptions for context limits
|
||||
for (const [key, param] of Object.entries(tool.function.parameters.properties || {})) {
|
||||
if (param.description && param.description.length > 100) {
|
||||
param.description = param.description.substring(0, 97) + '...';
|
||||
modifications.push(`Shortened description for parameter '${key}'`);
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Remove deeply nested structures
|
||||
if (config.maxDepth < 4) {
|
||||
const flattened = this.flattenDeepObjects(
|
||||
tool.function.parameters.properties,
|
||||
config.maxDepth
|
||||
);
|
||||
if (flattened.flattened) {
|
||||
tool.function.parameters.properties = flattened.properties;
|
||||
modifications.push('Flattened nested structures for local model');
|
||||
warnings.push('Nested objects were flattened for better local model performance');
|
||||
fixed = true;
|
||||
}
|
||||
}
|
||||
|
||||
return { tool, fixed, warnings, modifications };
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply generic fixes for any provider
|
||||
*/
|
||||
private applyGenericFixes(
|
||||
tool: Tool,
|
||||
config: ProviderConfig
|
||||
): { tool: Tool; fixed: boolean; warnings: string[]; modifications: string[] } {
|
||||
const warnings: string[] = [];
|
||||
const modifications: string[] = [];
|
||||
let fixed = false;
|
||||
|
||||
// Ensure function name length
|
||||
if (tool.function.name.length > config.maxFunctionNameLength) {
|
||||
tool.function.name = tool.function.name.substring(0, config.maxFunctionNameLength);
|
||||
modifications.push('Truncated function name to meet length limits');
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
// Ensure description exists
|
||||
if (!tool.function.description) {
|
||||
tool.function.description = `Execute ${tool.function.name}`;
|
||||
modifications.push('Added missing function description');
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
// Ensure parameters object structure
|
||||
if (!tool.function.parameters.type) {
|
||||
tool.function.parameters.type = 'object';
|
||||
modifications.push('Added missing parameters type');
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
if (!tool.function.parameters.properties) {
|
||||
tool.function.parameters.properties = {};
|
||||
modifications.push('Added missing parameters properties');
|
||||
fixed = true;
|
||||
}
|
||||
|
||||
return { tool, fixed, warnings, modifications };
|
||||
}
|
||||
|
||||
/**
|
||||
* Flatten deep objects
|
||||
*/
|
||||
private flattenDeepObjects(
|
||||
properties: Record<string, ToolParameter>,
|
||||
maxDepth: number,
|
||||
currentDepth: number = 0
|
||||
): { properties: Record<string, ToolParameter>; flattened: boolean } {
|
||||
let flattened = false;
|
||||
const result: Record<string, ToolParameter> = {};
|
||||
|
||||
for (const [key, param] of Object.entries(properties)) {
|
||||
if (param.type === 'object' && param.properties && currentDepth >= maxDepth - 1) {
|
||||
// Flatten this object
|
||||
const prefix = key + '_';
|
||||
for (const [subKey, subParam] of Object.entries(param.properties)) {
|
||||
result[prefix + subKey] = subParam;
|
||||
}
|
||||
flattened = true;
|
||||
} else if (param.type === 'object' && param.properties) {
|
||||
// Recurse deeper
|
||||
const subResult = this.flattenDeepObjects(
|
||||
param.properties,
|
||||
maxDepth,
|
||||
currentDepth + 1
|
||||
);
|
||||
result[key] = {
|
||||
...param,
|
||||
properties: subResult.properties
|
||||
};
|
||||
flattened = flattened || subResult.flattened;
|
||||
} else {
|
||||
result[key] = param;
|
||||
}
|
||||
}
|
||||
|
||||
return { properties: result, flattened };
|
||||
}
|
||||
|
||||
/**
|
||||
* Group related parameters
|
||||
*/
|
||||
private groupRelatedParameters(
|
||||
properties: Record<string, ToolParameter>
|
||||
): { properties: Record<string, ToolParameter>; grouped: boolean } {
|
||||
const groups = new Map<string, Record<string, ToolParameter>>();
|
||||
const ungrouped: Record<string, ToolParameter> = {};
|
||||
let grouped = false;
|
||||
|
||||
// Identify common prefixes
|
||||
for (const [key, param] of Object.entries(properties)) {
|
||||
const prefix = key.split('_')[0];
|
||||
if (prefix && prefix.length > 2) {
|
||||
if (!groups.has(prefix)) {
|
||||
groups.set(prefix, {});
|
||||
}
|
||||
groups.get(prefix)![key] = param;
|
||||
} else {
|
||||
ungrouped[key] = param;
|
||||
}
|
||||
}
|
||||
|
||||
// Create grouped structure if beneficial
|
||||
const result: Record<string, ToolParameter> = {};
|
||||
|
||||
for (const [prefix, groupProps] of groups) {
|
||||
if (Object.keys(groupProps).length > 2) {
|
||||
// Group these properties
|
||||
result[prefix] = {
|
||||
type: 'object',
|
||||
description: `${prefix} properties`,
|
||||
properties: groupProps
|
||||
};
|
||||
grouped = true;
|
||||
} else {
|
||||
// Keep ungrouped
|
||||
Object.assign(result, groupProps);
|
||||
}
|
||||
}
|
||||
|
||||
// Add ungrouped properties
|
||||
Object.assign(result, ungrouped);
|
||||
|
||||
return { properties: result, grouped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify complex types for local models
|
||||
*/
|
||||
private simplifyComplexTypes(
|
||||
properties: Record<string, ToolParameter>
|
||||
): { properties: Record<string, ToolParameter>; simplified: boolean } {
|
||||
let simplified = false;
|
||||
const result: Record<string, ToolParameter> = {};
|
||||
|
||||
for (const [key, param] of Object.entries(properties)) {
|
||||
if (param.type === 'array' && param.items && typeof param.items === 'object' && 'properties' in param.items) {
|
||||
// Complex array of objects - simplify to array of strings
|
||||
result[key] = {
|
||||
type: 'array',
|
||||
description: param.description || `List of ${key}`,
|
||||
items: { type: 'string' }
|
||||
};
|
||||
simplified = true;
|
||||
} else if (param.type === 'object' && param.properties) {
|
||||
// Nested object - check if can be simplified
|
||||
const propCount = Object.keys(param.properties).length;
|
||||
if (propCount > 5) {
|
||||
// Too complex - convert to string
|
||||
result[key] = {
|
||||
type: 'string',
|
||||
description: param.description || `JSON string for ${key}`
|
||||
};
|
||||
simplified = true;
|
||||
} else {
|
||||
result[key] = param;
|
||||
}
|
||||
} else {
|
||||
result[key] = param;
|
||||
}
|
||||
}
|
||||
|
||||
return { properties: result, simplified };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix enum values
|
||||
*/
|
||||
private fixEnumValues(properties: Record<string, ToolParameter>): void {
|
||||
for (const param of Object.values(properties)) {
|
||||
if (param.enum) {
|
||||
// Ensure all enum values are strings
|
||||
param.enum = param.enum.map(v => String(v));
|
||||
|
||||
// Remove any special characters
|
||||
param.enum = param.enum.map(v => v.replace(/[^\w\s-]/g, '_'));
|
||||
}
|
||||
|
||||
// Recurse for nested properties
|
||||
if (param.properties) {
|
||||
this.fixEnumValues(param.properties);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate parameter complexity
|
||||
*/
|
||||
private calculateComplexity(parameters: any, depth: number = 0): number {
|
||||
let complexity = depth;
|
||||
|
||||
if (parameters.properties) {
|
||||
for (const param of Object.values(parameters.properties) as ToolParameter[]) {
|
||||
complexity += 1;
|
||||
|
||||
if (param.type === 'object' && param.properties) {
|
||||
complexity += this.calculateComplexity(param, depth + 1);
|
||||
}
|
||||
|
||||
if (param.type === 'array') {
|
||||
complexity += 2; // Arrays add more complexity
|
||||
if (param.items && typeof param.items === 'object' && 'properties' in param.items) {
|
||||
complexity += 3; // Array of objects is very complex
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return complexity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch fix tools for a provider
|
||||
*/
|
||||
fixToolsForProvider(tools: Tool[], provider: string): Tool[] {
|
||||
const fixed: Tool[] = [];
|
||||
|
||||
for (const tool of tools) {
|
||||
const result = this.fixToolForProvider(tool, provider);
|
||||
|
||||
if (result.fixed && result.tool) {
|
||||
fixed.push(result.tool);
|
||||
|
||||
if (result.warnings.length > 0) {
|
||||
log.info(`Warnings for ${tool.function.name}: ${JSON.stringify(result.warnings)}`);
|
||||
}
|
||||
if (result.modifications.length > 0) {
|
||||
log.info(`Modifications for ${tool.function.name}: ${JSON.stringify(result.modifications)}`);
|
||||
}
|
||||
} else {
|
||||
fixed.push(tool);
|
||||
}
|
||||
}
|
||||
|
||||
return fixed;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const edgeCaseHandler = new EdgeCaseHandler();
|
||||
@@ -27,6 +27,9 @@ import {
|
||||
ExportFormat,
|
||||
type ExporterConfig
|
||||
} from '../metrics/metrics_exporter.js';
|
||||
import { providerHealthMonitor } from '../monitoring/provider_health_monitor.js';
|
||||
import { edgeCaseHandler } from './edge_case_handler.js';
|
||||
import { providerToolValidator } from '../tools/provider_tool_validator.js';
|
||||
|
||||
/**
|
||||
* Provider type enumeration
|
||||
|
||||
487
apps/server/src/services/llm/tests/integration_test.ts
Normal file
487
apps/server/src/services/llm/tests/integration_test.ts
Normal file
@@ -0,0 +1,487 @@
|
||||
/**
|
||||
* Integration Test for LLM Resilience Improvements
|
||||
*
|
||||
* Tests all new components working together to ensure the LLM feature
|
||||
* is extremely resilient, intuitive, and responsive.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from '@jest/globals';
|
||||
import { toolTimeoutEnforcer } from '../tools/tool_timeout_enforcer.js';
|
||||
import { providerToolValidator } from '../tools/provider_tool_validator.js';
|
||||
import { providerHealthMonitor } from '../monitoring/provider_health_monitor.js';
|
||||
import { parameterCoercer } from '../tools/parameter_coercer.js';
|
||||
import { toolExecutionMonitor } from '../monitoring/tool_execution_monitor.js';
|
||||
import { toolResponseCache } from '../tools/tool_response_cache.js';
|
||||
import { edgeCaseHandler } from '../providers/edge_case_handler.js';
|
||||
import { EnhancedToolHandler } from '../chat/handlers/enhanced_tool_handler.js';
|
||||
import type { Tool, ToolCall } from '../tools/tool_interfaces.js';
|
||||
|
||||
describe('LLM Resilience Integration Tests', () => {
|
||||
// Sample tools for testing
|
||||
const sampleTools: Tool[] = [
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'search_notes',
|
||||
description: 'Search for notes by keyword',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
query: {
|
||||
type: 'string',
|
||||
description: 'Search query'
|
||||
},
|
||||
limit: {
|
||||
type: 'number',
|
||||
description: 'Maximum results',
|
||||
default: 10
|
||||
}
|
||||
},
|
||||
required: ['query']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'create-note-with-special-chars',
|
||||
description: 'Create a new note with special characters in name',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: {
|
||||
type: 'string',
|
||||
description: 'Note title'
|
||||
},
|
||||
content: {
|
||||
type: 'string',
|
||||
description: 'Note content'
|
||||
},
|
||||
deeply: {
|
||||
type: 'object',
|
||||
description: 'Deeply nested object',
|
||||
properties: {
|
||||
nested: {
|
||||
type: 'object',
|
||||
description: 'Nested object',
|
||||
properties: {
|
||||
value: {
|
||||
type: 'string',
|
||||
description: 'Nested value'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
required: [] // Empty for Anthropic testing
|
||||
}
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
beforeAll(() => {
|
||||
// Initialize components
|
||||
console.log('Setting up integration test environment...');
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
// Cleanup
|
||||
toolResponseCache.shutdown();
|
||||
providerHealthMonitor.stopMonitoring();
|
||||
});
|
||||
|
||||
describe('Tool Timeout Enforcement', () => {
|
||||
it('should enforce timeouts on long-running tools', async () => {
|
||||
const result = await toolTimeoutEnforcer.executeWithTimeout(
|
||||
'test_tool',
|
||||
async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
return 'success';
|
||||
},
|
||||
200 // 200ms timeout
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.timedOut).toBe(false);
|
||||
expect(result.result).toBe('success');
|
||||
});
|
||||
|
||||
it('should timeout and report failure', async () => {
|
||||
const result = await toolTimeoutEnforcer.executeWithTimeout(
|
||||
'slow_tool',
|
||||
async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
return 'should not reach';
|
||||
},
|
||||
100 // 100ms timeout
|
||||
);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.timedOut).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provider Tool Validation', () => {
|
||||
it('should validate and fix tools for OpenAI', () => {
|
||||
const result = providerToolValidator.validateTool(sampleTools[1], 'openai');
|
||||
|
||||
expect(result.fixedTool).toBeDefined();
|
||||
if (result.fixedTool) {
|
||||
// Should fix special characters in function name
|
||||
expect(result.fixedTool.function.name).not.toContain('-');
|
||||
}
|
||||
});
|
||||
|
||||
it('should ensure non-empty required array for Anthropic', () => {
|
||||
const result = providerToolValidator.validateTool(sampleTools[1], 'anthropic');
|
||||
|
||||
expect(result.fixedTool).toBeDefined();
|
||||
if (result.fixedTool) {
|
||||
// Should add at least one required parameter
|
||||
expect(result.fixedTool.function.parameters.required?.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should simplify tools for Ollama', () => {
|
||||
const result = providerToolValidator.validateTool(sampleTools[1], 'ollama');
|
||||
|
||||
expect(result.warnings.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Parameter Type Coercion', () => {
|
||||
it('should coerce string numbers to numbers', () => {
|
||||
const result = parameterCoercer.coerceToolArguments(
|
||||
{ limit: '10' },
|
||||
sampleTools[0],
|
||||
{ parseNumbers: true }
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.value.limit).toBe(10);
|
||||
expect(typeof result.value.limit).toBe('number');
|
||||
});
|
||||
|
||||
it('should apply default values', () => {
|
||||
const result = parameterCoercer.coerceToolArguments(
|
||||
{ query: 'test' },
|
||||
sampleTools[0],
|
||||
{ applyDefaults: true }
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.value.limit).toBe(10);
|
||||
});
|
||||
|
||||
it('should normalize arrays', () => {
|
||||
const tool: Tool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'test',
|
||||
description: 'Test',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
tags: {
|
||||
type: 'array',
|
||||
description: 'List of tags',
|
||||
items: { type: 'string', description: 'Tag value' }
|
||||
}
|
||||
},
|
||||
required: []
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = parameterCoercer.coerceToolArguments(
|
||||
{ tags: 'single-tag' },
|
||||
tool,
|
||||
{ normalizeArrays: true }
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(Array.isArray(result.value.tags)).toBe(true);
|
||||
expect(result.value.tags).toEqual(['single-tag']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Execution Monitoring', () => {
|
||||
it('should track execution statistics', () => {
|
||||
// Record successful execution
|
||||
toolExecutionMonitor.recordExecution({
|
||||
toolName: 'test_tool',
|
||||
provider: 'openai',
|
||||
status: 'success',
|
||||
executionTime: 100,
|
||||
timestamp: new Date()
|
||||
});
|
||||
|
||||
const stats = toolExecutionMonitor.getToolStats('test_tool', 'openai');
|
||||
expect(stats).toBeDefined();
|
||||
expect(stats?.successfulExecutions).toBe(1);
|
||||
expect(stats?.reliabilityScore).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('should auto-disable unreliable tools', () => {
|
||||
// Record multiple failures
|
||||
for (let i = 0; i < 6; i++) {
|
||||
toolExecutionMonitor.recordExecution({
|
||||
toolName: 'unreliable_tool',
|
||||
provider: 'openai',
|
||||
status: 'failure',
|
||||
executionTime: 100,
|
||||
timestamp: new Date(),
|
||||
error: 'Test failure'
|
||||
});
|
||||
}
|
||||
|
||||
const isDisabled = toolExecutionMonitor.isToolDisabled('unreliable_tool', 'openai');
|
||||
expect(isDisabled).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Response Caching', () => {
|
||||
it('should cache deterministic tool responses', () => {
|
||||
const toolName = 'read_note_tool';
|
||||
const args = { noteId: 'test123' };
|
||||
const response = { content: 'Test content' };
|
||||
|
||||
// Set cache
|
||||
const cached = toolResponseCache.set(toolName, args, response, 'openai');
|
||||
expect(cached).toBe(true);
|
||||
|
||||
// Get from cache
|
||||
const retrieved = toolResponseCache.get(toolName, args, 'openai');
|
||||
expect(retrieved).toEqual(response);
|
||||
});
|
||||
|
||||
it('should generate consistent cache keys', () => {
|
||||
const key1 = toolResponseCache.generateCacheKey('tool', { b: 2, a: 1 }, 'provider');
|
||||
const key2 = toolResponseCache.generateCacheKey('tool', { a: 1, b: 2 }, 'provider');
|
||||
|
||||
expect(key1).toBe(key2);
|
||||
});
|
||||
|
||||
it('should respect TTL', async () => {
|
||||
const toolName = 'temp_tool';
|
||||
const args = { id: 'temp' };
|
||||
const response = 'temp data';
|
||||
|
||||
// Set with short TTL
|
||||
toolResponseCache.set(toolName, args, response, 'openai', 100); // 100ms TTL
|
||||
|
||||
// Should be cached
|
||||
expect(toolResponseCache.get(toolName, args, 'openai')).toBe(response);
|
||||
|
||||
// Wait for expiration
|
||||
await new Promise(resolve => setTimeout(resolve, 150));
|
||||
|
||||
// Should be expired
|
||||
expect(toolResponseCache.get(toolName, args, 'openai')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Case Handling', () => {
|
||||
it('should fix OpenAI edge cases', () => {
|
||||
const tool = sampleTools[1];
|
||||
const result = edgeCaseHandler.fixToolForProvider(tool, 'openai');
|
||||
|
||||
expect(result.fixed).toBe(true);
|
||||
if (result.tool) {
|
||||
// Function name should not have hyphens
|
||||
expect(result.tool.function.name).not.toContain('-');
|
||||
// Deep nesting might be flattened
|
||||
expect(result.modifications.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should fix Anthropic edge cases', () => {
|
||||
const tool = sampleTools[1];
|
||||
const result = edgeCaseHandler.fixToolForProvider(tool, 'anthropic');
|
||||
|
||||
expect(result.fixed).toBe(true);
|
||||
if (result.tool) {
|
||||
// Should have required parameters
|
||||
expect(result.tool.function.parameters.required).toBeDefined();
|
||||
expect(result.tool.function.parameters.required!.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('should simplify for Ollama', () => {
|
||||
const complexTool: Tool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'complex_tool',
|
||||
description: 'A'.repeat(600), // Long description
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: Object.fromEntries(
|
||||
Array.from({ length: 30 }, (_, i) => [
|
||||
`param${i}`,
|
||||
{ type: 'string', description: `Parameter ${i}` }
|
||||
])
|
||||
),
|
||||
required: []
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = edgeCaseHandler.fixToolForProvider(complexTool, 'ollama');
|
||||
|
||||
expect(result.fixed).toBe(true);
|
||||
if (result.tool) {
|
||||
// Description should be truncated
|
||||
expect(result.tool.function.description.length).toBeLessThanOrEqual(500);
|
||||
// Parameters should be reduced
|
||||
const paramCount = Object.keys(result.tool.function.parameters.properties || {}).length;
|
||||
expect(paramCount).toBeLessThanOrEqual(20);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Parallel Tool Execution', () => {
|
||||
it('should identify independent tools for parallel execution', async () => {
|
||||
const toolCalls: ToolCall[] = [
|
||||
{
|
||||
id: '1',
|
||||
function: {
|
||||
name: 'search_notes',
|
||||
arguments: { query: 'test1' }
|
||||
}
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
function: {
|
||||
name: 'search_notes',
|
||||
arguments: { query: 'test2' }
|
||||
}
|
||||
},
|
||||
{
|
||||
id: '3',
|
||||
function: {
|
||||
name: 'read_note_tool',
|
||||
arguments: { noteId: 'abc' }
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
// These should be executed in parallel since they're independent
|
||||
const handler = new EnhancedToolHandler();
|
||||
// For now, just verify the handler was created successfully
|
||||
expect(handler).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provider Health Monitoring', () => {
|
||||
it('should track provider health status', async () => {
|
||||
// Mock provider service
|
||||
const mockService = {
|
||||
chat: async () => ({
|
||||
content: 'test',
|
||||
usage: { totalTokens: 5 }
|
||||
}),
|
||||
getModels: () => [{ id: 'test-model' }]
|
||||
};
|
||||
|
||||
providerHealthMonitor.registerProvider('test-provider', mockService as any);
|
||||
|
||||
// Manually trigger health check
|
||||
const result = await providerHealthMonitor.checkProvider('test-provider');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.latency).toBeGreaterThan(0);
|
||||
|
||||
const health = providerHealthMonitor.getProviderHealth('test-provider');
|
||||
expect(health?.healthy).toBe(true);
|
||||
});
|
||||
|
||||
it('should disable unhealthy providers', () => {
|
||||
// Simulate failures
|
||||
const status = {
|
||||
provider: 'failing-provider',
|
||||
healthy: true,
|
||||
lastChecked: new Date(),
|
||||
consecutiveFailures: 3,
|
||||
totalChecks: 10,
|
||||
totalFailures: 3,
|
||||
averageLatency: 100,
|
||||
disabled: false
|
||||
};
|
||||
|
||||
// This would normally be done internally
|
||||
providerHealthMonitor.disableProvider('failing-provider', 'Too many failures');
|
||||
|
||||
expect(providerHealthMonitor.isProviderHealthy('failing-provider')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('End-to-End Integration', () => {
|
||||
it('should handle tool execution with all enhancements', async () => {
|
||||
// This tests the full flow with all components working together
|
||||
const toolCall: ToolCall = {
|
||||
id: 'integration-test',
|
||||
function: {
|
||||
name: 'search_notes',
|
||||
arguments: '{"query": "test", "limit": "5"}' // String number to test coercion
|
||||
}
|
||||
};
|
||||
|
||||
// Test components integration
|
||||
const tool = sampleTools[0];
|
||||
|
||||
// 1. Validate for provider
|
||||
const validation = providerToolValidator.validateTool(tool, 'openai');
|
||||
expect(validation.valid || validation.fixedTool).toBeTruthy();
|
||||
|
||||
// 2. Apply edge case fixes
|
||||
const edgeFixes = edgeCaseHandler.fixToolForProvider(
|
||||
validation.fixedTool || tool,
|
||||
'openai'
|
||||
);
|
||||
|
||||
// 3. Parse and coerce arguments
|
||||
const args = parameterCoercer.coerceToolArguments(
|
||||
JSON.parse(toolCall.function.arguments as string),
|
||||
tool,
|
||||
{ provider: 'openai' }
|
||||
);
|
||||
expect(args.value.limit).toBe(5);
|
||||
expect(typeof args.value.limit).toBe('number');
|
||||
|
||||
// 4. Execute with timeout
|
||||
const timeoutResult = await toolTimeoutEnforcer.executeWithTimeout(
|
||||
'search_notes',
|
||||
async () => ({ results: ['note1', 'note2'] }),
|
||||
5000
|
||||
);
|
||||
expect(timeoutResult.success).toBe(true);
|
||||
|
||||
// 5. Cache the result
|
||||
if (timeoutResult.success) {
|
||||
toolResponseCache.set(
|
||||
'search_notes',
|
||||
args.value,
|
||||
timeoutResult.result,
|
||||
'openai'
|
||||
);
|
||||
}
|
||||
|
||||
// 6. Record execution
|
||||
toolExecutionMonitor.recordExecution({
|
||||
toolName: 'search_notes',
|
||||
provider: 'openai',
|
||||
status: timeoutResult.success ? 'success' : 'failure',
|
||||
executionTime: timeoutResult.executionTime,
|
||||
timestamp: new Date()
|
||||
});
|
||||
|
||||
// Verify everything worked
|
||||
const cached = toolResponseCache.get('search_notes', args.value, 'openai');
|
||||
expect(cached).toEqual({ results: ['note1', 'note2'] });
|
||||
|
||||
const stats = toolExecutionMonitor.getToolStats('search_notes', 'openai');
|
||||
expect(stats?.totalExecutions).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
593
apps/server/src/services/llm/tools/parameter_coercer.ts
Normal file
593
apps/server/src/services/llm/tools/parameter_coercer.ts
Normal file
@@ -0,0 +1,593 @@
|
||||
/**
|
||||
* Parameter Type Coercer
|
||||
*
|
||||
* Provides automatic type conversion, array normalization, default value injection,
|
||||
* and schema validation with fixes for tool parameters.
|
||||
*/
|
||||
|
||||
import log from '../../log.js';
|
||||
import type { Tool, ToolParameter } from './tool_interfaces.js';
|
||||
|
||||
/**
|
||||
* Coercion result
|
||||
*/
|
||||
export interface CoercionResult {
|
||||
success: boolean;
|
||||
value: any;
|
||||
wasCoerced: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Coercion options
|
||||
*/
|
||||
export interface CoercionOptions {
|
||||
/** Strict mode - fail on any coercion error */
|
||||
strict: boolean;
|
||||
/** Apply default values */
|
||||
applyDefaults: boolean;
|
||||
/** Normalize arrays (single values to arrays) */
|
||||
normalizeArrays: boolean;
|
||||
/** Trim string values */
|
||||
trimStrings: boolean;
|
||||
/** Convert number strings to numbers */
|
||||
parseNumbers: boolean;
|
||||
/** Convert boolean strings to booleans */
|
||||
parseBooleans: boolean;
|
||||
/** Provider-specific quirks */
|
||||
provider?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default coercion options
|
||||
*/
|
||||
const DEFAULT_OPTIONS: CoercionOptions = {
|
||||
strict: false,
|
||||
applyDefaults: true,
|
||||
normalizeArrays: true,
|
||||
trimStrings: true,
|
||||
parseNumbers: true,
|
||||
parseBooleans: true
|
||||
};
|
||||
|
||||
/**
|
||||
* Provider-specific quirks
|
||||
*/
|
||||
const PROVIDER_QUIRKS = {
|
||||
openai: {
|
||||
// OpenAI sometimes sends stringified JSON for complex objects
|
||||
parseJsonStrings: true,
|
||||
// OpenAI may send null for optional parameters
|
||||
treatNullAsUndefined: true
|
||||
},
|
||||
anthropic: {
|
||||
// Anthropic strictly validates types
|
||||
strictTypeChecking: true,
|
||||
// Anthropic requires arrays to be actual arrays
|
||||
requireArrayTypes: true
|
||||
},
|
||||
ollama: {
|
||||
// Local models may have looser type handling
|
||||
lenientParsing: true,
|
||||
// May send numbers as strings more often
|
||||
aggressiveNumberParsing: true
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Parameter coercer class
|
||||
*/
|
||||
export class ParameterCoercer {
|
||||
private options: CoercionOptions;
|
||||
|
||||
constructor(options?: Partial<CoercionOptions>) {
|
||||
this.options = { ...DEFAULT_OPTIONS, ...options };
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce tool call arguments to match tool definition
|
||||
*/
|
||||
coerceToolArguments(
|
||||
args: Record<string, unknown>,
|
||||
tool: Tool,
|
||||
options?: Partial<CoercionOptions>
|
||||
): CoercionResult {
|
||||
const opts = { ...this.options, ...options };
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
|
||||
const parameters = tool.function.parameters;
|
||||
const coercedArgs: Record<string, any> = {};
|
||||
|
||||
// Process each parameter
|
||||
for (const [paramName, paramDef] of Object.entries(parameters.properties)) {
|
||||
const rawValue = args[paramName];
|
||||
const isRequired = parameters.required?.includes(paramName);
|
||||
|
||||
// Handle missing values
|
||||
if (rawValue === undefined || rawValue === null) {
|
||||
if (opts.provider === 'openai' && rawValue === null) {
|
||||
// OpenAI quirk: treat null as undefined
|
||||
if (isRequired && !paramDef.default) {
|
||||
errors.push(`Required parameter '${paramName}' is null`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.applyDefaults && paramDef.default !== undefined) {
|
||||
coercedArgs[paramName] = paramDef.default;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Applied default value for '${paramName}'`);
|
||||
} else if (isRequired) {
|
||||
errors.push(`Required parameter '${paramName}' is missing`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Coerce the value
|
||||
const coerced = this.coerceValue(
|
||||
rawValue,
|
||||
paramDef,
|
||||
paramName,
|
||||
opts
|
||||
);
|
||||
|
||||
if (coerced.success) {
|
||||
coercedArgs[paramName] = coerced.value;
|
||||
if (coerced.wasCoerced) {
|
||||
wasCoerced = true;
|
||||
warnings.push(...coerced.warnings);
|
||||
}
|
||||
} else {
|
||||
errors.push(...coerced.errors);
|
||||
if (!opts.strict) {
|
||||
// In non-strict mode, use original value
|
||||
coercedArgs[paramName] = rawValue;
|
||||
warnings.push(`Failed to coerce '${paramName}', using original value`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for unknown parameters
|
||||
for (const paramName of Object.keys(args)) {
|
||||
if (!(paramName in parameters.properties)) {
|
||||
warnings.push(`Unknown parameter '${paramName}' will be ignored`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
success: errors.length === 0,
|
||||
value: coercedArgs,
|
||||
wasCoerced,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce a single value to match its type definition
|
||||
*/
|
||||
private coerceValue(
|
||||
value: unknown,
|
||||
definition: ToolParameter,
|
||||
path: string,
|
||||
options: CoercionOptions
|
||||
): CoercionResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
let coercedValue = value;
|
||||
|
||||
// Handle provider-specific JSON string parsing
|
||||
if (options.provider === 'openai' &&
|
||||
typeof value === 'string' &&
|
||||
(definition.type === 'object' || definition.type === 'array')) {
|
||||
try {
|
||||
coercedValue = JSON.parse(value);
|
||||
wasCoerced = true;
|
||||
warnings.push(`Parsed JSON string for '${path}'`);
|
||||
} catch {
|
||||
// Not valid JSON, continue with string value
|
||||
}
|
||||
}
|
||||
|
||||
// Type-specific coercion
|
||||
switch (definition.type) {
|
||||
case 'string':
|
||||
const stringResult = this.coerceToString(coercedValue, path, options);
|
||||
coercedValue = stringResult.value;
|
||||
wasCoerced = wasCoerced || stringResult.wasCoerced;
|
||||
warnings.push(...stringResult.warnings);
|
||||
break;
|
||||
|
||||
case 'number':
|
||||
case 'integer':
|
||||
const numberResult = this.coerceToNumber(
|
||||
coercedValue,
|
||||
path,
|
||||
definition,
|
||||
definition.type === 'integer',
|
||||
options
|
||||
);
|
||||
if (numberResult.success) {
|
||||
coercedValue = numberResult.value;
|
||||
wasCoerced = wasCoerced || numberResult.wasCoerced;
|
||||
warnings.push(...numberResult.warnings);
|
||||
} else {
|
||||
errors.push(...numberResult.errors);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'boolean':
|
||||
const boolResult = this.coerceToBoolean(coercedValue, path, options);
|
||||
if (boolResult.success) {
|
||||
coercedValue = boolResult.value;
|
||||
wasCoerced = wasCoerced || boolResult.wasCoerced;
|
||||
warnings.push(...boolResult.warnings);
|
||||
} else {
|
||||
errors.push(...boolResult.errors);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'array':
|
||||
const arrayResult = this.coerceToArray(
|
||||
coercedValue,
|
||||
path,
|
||||
definition,
|
||||
options
|
||||
);
|
||||
if (arrayResult.success) {
|
||||
coercedValue = arrayResult.value;
|
||||
wasCoerced = wasCoerced || arrayResult.wasCoerced;
|
||||
warnings.push(...arrayResult.warnings);
|
||||
} else {
|
||||
errors.push(...arrayResult.errors);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'object':
|
||||
const objectResult = this.coerceToObject(
|
||||
coercedValue,
|
||||
path,
|
||||
definition,
|
||||
options
|
||||
);
|
||||
if (objectResult.success) {
|
||||
coercedValue = objectResult.value;
|
||||
wasCoerced = wasCoerced || objectResult.wasCoerced;
|
||||
warnings.push(...objectResult.warnings);
|
||||
} else {
|
||||
errors.push(...objectResult.errors);
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
warnings.push(`Unknown type '${definition.type}' for '${path}'`);
|
||||
}
|
||||
|
||||
// Validate enum values
|
||||
if (definition.enum && !definition.enum.includes(String(coercedValue))) {
|
||||
errors.push(`Value for '${path}' must be one of: ${definition.enum.join(', ')}`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: errors.length === 0,
|
||||
value: coercedValue,
|
||||
wasCoerced,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce to string
|
||||
*/
|
||||
private coerceToString(
|
||||
value: unknown,
|
||||
path: string,
|
||||
options: CoercionOptions
|
||||
): CoercionResult {
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
let result: string;
|
||||
|
||||
if (typeof value === 'string') {
|
||||
result = options.trimStrings ? value.trim() : value;
|
||||
if (result !== value) {
|
||||
wasCoerced = true;
|
||||
warnings.push(`Trimmed whitespace from '${path}'`);
|
||||
}
|
||||
} else if (value === null || value === undefined) {
|
||||
result = '';
|
||||
wasCoerced = true;
|
||||
warnings.push(`Converted null/undefined to empty string for '${path}'`);
|
||||
} else {
|
||||
result = String(value);
|
||||
wasCoerced = true;
|
||||
warnings.push(`Converted ${typeof value} to string for '${path}'`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
value: result,
|
||||
wasCoerced,
|
||||
errors: [],
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce to number
|
||||
*/
|
||||
private coerceToNumber(
|
||||
value: unknown,
|
||||
path: string,
|
||||
definition: ToolParameter,
|
||||
isInteger: boolean,
|
||||
options: CoercionOptions
|
||||
): CoercionResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
let result: number;
|
||||
|
||||
if (typeof value === 'number') {
|
||||
result = isInteger ? Math.round(value) : value;
|
||||
if (result !== value) {
|
||||
wasCoerced = true;
|
||||
warnings.push(`Rounded to integer for '${path}'`);
|
||||
}
|
||||
} else if (typeof value === 'string' && options.parseNumbers) {
|
||||
const parsed = isInteger ? parseInt(value, 10) : parseFloat(value);
|
||||
if (!isNaN(parsed)) {
|
||||
result = parsed;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Parsed string to number for '${path}'`);
|
||||
} else {
|
||||
errors.push(`Cannot parse '${value}' as number for '${path}'`);
|
||||
return { success: false, value, wasCoerced: false, errors, warnings };
|
||||
}
|
||||
} else if (typeof value === 'boolean') {
|
||||
result = value ? 1 : 0;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Converted boolean to number for '${path}'`);
|
||||
} else {
|
||||
errors.push(`Cannot coerce ${typeof value} to number for '${path}'`);
|
||||
return { success: false, value, wasCoerced: false, errors, warnings };
|
||||
}
|
||||
|
||||
// Validate constraints
|
||||
if (definition.minimum !== undefined && result < definition.minimum) {
|
||||
result = definition.minimum;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Clamped to minimum value ${definition.minimum} for '${path}'`);
|
||||
}
|
||||
if (definition.maximum !== undefined && result > definition.maximum) {
|
||||
result = definition.maximum;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Clamped to maximum value ${definition.maximum} for '${path}'`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
value: result,
|
||||
wasCoerced,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce to boolean
|
||||
*/
|
||||
private coerceToBoolean(
|
||||
value: unknown,
|
||||
path: string,
|
||||
options: CoercionOptions
|
||||
): CoercionResult {
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
let result: boolean;
|
||||
|
||||
if (typeof value === 'boolean') {
|
||||
result = value;
|
||||
} else if (typeof value === 'string' && options.parseBooleans) {
|
||||
const lower = value.toLowerCase().trim();
|
||||
if (lower === 'true' || lower === 'yes' || lower === '1') {
|
||||
result = true;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Parsed string to boolean true for '${path}'`);
|
||||
} else if (lower === 'false' || lower === 'no' || lower === '0') {
|
||||
result = false;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Parsed string to boolean false for '${path}'`);
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
value,
|
||||
wasCoerced: false,
|
||||
errors: [`Cannot parse '${value}' as boolean for '${path}'`],
|
||||
warnings
|
||||
};
|
||||
}
|
||||
} else if (typeof value === 'number') {
|
||||
result = value !== 0;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Converted number to boolean for '${path}'`);
|
||||
} else {
|
||||
result = Boolean(value);
|
||||
wasCoerced = true;
|
||||
warnings.push(`Coerced ${typeof value} to boolean for '${path}'`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
value: result,
|
||||
wasCoerced,
|
||||
errors: [],
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce to array
|
||||
*/
|
||||
private coerceToArray(
|
||||
value: unknown,
|
||||
path: string,
|
||||
definition: ToolParameter,
|
||||
options: CoercionOptions
|
||||
): CoercionResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
let result: any[];
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
result = value;
|
||||
} else if (options.normalizeArrays) {
|
||||
// Convert single value to array
|
||||
result = [value];
|
||||
wasCoerced = true;
|
||||
warnings.push(`Normalized single value to array for '${path}'`);
|
||||
} else {
|
||||
errors.push(`Expected array for '${path}', got ${typeof value}`);
|
||||
return { success: false, value, wasCoerced: false, errors, warnings };
|
||||
}
|
||||
|
||||
// Validate array constraints
|
||||
if (definition.minItems !== undefined && result.length < definition.minItems) {
|
||||
errors.push(`Array '${path}' must have at least ${definition.minItems} items`);
|
||||
}
|
||||
if (definition.maxItems !== undefined && result.length > definition.maxItems) {
|
||||
result = result.slice(0, definition.maxItems);
|
||||
wasCoerced = true;
|
||||
warnings.push(`Truncated array to ${definition.maxItems} items for '${path}'`);
|
||||
}
|
||||
|
||||
// Coerce array items if type is specified
|
||||
if (definition.items) {
|
||||
const coercedItems: any[] = [];
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
const itemResult = this.coerceValue(
|
||||
result[i],
|
||||
definition.items as ToolParameter,
|
||||
`${path}[${i}]`,
|
||||
options
|
||||
);
|
||||
if (itemResult.success) {
|
||||
coercedItems.push(itemResult.value);
|
||||
if (itemResult.wasCoerced) wasCoerced = true;
|
||||
warnings.push(...itemResult.warnings);
|
||||
} else {
|
||||
errors.push(...itemResult.errors);
|
||||
coercedItems.push(result[i]); // Keep original on error
|
||||
}
|
||||
}
|
||||
result = coercedItems;
|
||||
}
|
||||
|
||||
return {
|
||||
success: errors.length === 0,
|
||||
value: result,
|
||||
wasCoerced,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce to object
|
||||
*/
|
||||
private coerceToObject(
|
||||
value: unknown,
|
||||
path: string,
|
||||
definition: ToolParameter,
|
||||
options: CoercionOptions
|
||||
): CoercionResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
let wasCoerced = false;
|
||||
let result: Record<string, any>;
|
||||
|
||||
if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
|
||||
result = value as Record<string, any>;
|
||||
} else if (typeof value === 'string') {
|
||||
// Try to parse as JSON
|
||||
try {
|
||||
result = JSON.parse(value);
|
||||
wasCoerced = true;
|
||||
warnings.push(`Parsed JSON string for object '${path}'`);
|
||||
} catch {
|
||||
errors.push(`Cannot parse string as object for '${path}'`);
|
||||
return { success: false, value, wasCoerced: false, errors, warnings };
|
||||
}
|
||||
} else {
|
||||
errors.push(`Expected object for '${path}', got ${typeof value}`);
|
||||
return { success: false, value, wasCoerced: false, errors, warnings };
|
||||
}
|
||||
|
||||
// Coerce nested properties if defined
|
||||
if (definition.properties) {
|
||||
const coercedObj: Record<string, any> = {};
|
||||
for (const [propName, propDef] of Object.entries(definition.properties)) {
|
||||
if (propName in result) {
|
||||
const propResult = this.coerceValue(
|
||||
result[propName],
|
||||
propDef,
|
||||
`${path}.${propName}`,
|
||||
options
|
||||
);
|
||||
if (propResult.success) {
|
||||
coercedObj[propName] = propResult.value;
|
||||
if (propResult.wasCoerced) wasCoerced = true;
|
||||
warnings.push(...propResult.warnings);
|
||||
} else {
|
||||
errors.push(...propResult.errors);
|
||||
coercedObj[propName] = result[propName]; // Keep original on error
|
||||
}
|
||||
} else if (propDef.default !== undefined && options.applyDefaults) {
|
||||
coercedObj[propName] = propDef.default;
|
||||
wasCoerced = true;
|
||||
warnings.push(`Applied default value for '${path}.${propName}'`);
|
||||
}
|
||||
}
|
||||
|
||||
// Include any additional properties not in schema
|
||||
for (const propName of Object.keys(result)) {
|
||||
if (!(propName in coercedObj)) {
|
||||
coercedObj[propName] = result[propName];
|
||||
}
|
||||
}
|
||||
|
||||
result = coercedObj;
|
||||
}
|
||||
|
||||
return {
|
||||
success: errors.length === 0,
|
||||
value: result,
|
||||
wasCoerced,
|
||||
errors,
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update coercion options
|
||||
*/
|
||||
updateOptions(options: Partial<CoercionOptions>): void {
|
||||
this.options = { ...this.options, ...options };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current options
|
||||
*/
|
||||
getOptions(): CoercionOptions {
|
||||
return { ...this.options };
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const parameterCoercer = new ParameterCoercer();
|
||||
470
apps/server/src/services/llm/tools/provider_tool_validator.ts
Normal file
470
apps/server/src/services/llm/tools/provider_tool_validator.ts
Normal file
@@ -0,0 +1,470 @@
|
||||
/**
|
||||
* Provider Tool Validator
|
||||
*
|
||||
* Validates and auto-fixes tool definitions based on provider-specific requirements
|
||||
* for OpenAI, Anthropic, and Ollama.
|
||||
*/
|
||||
|
||||
import log from '../../log.js';
|
||||
import type { Tool, ToolParameter } from './tool_interfaces.js';
|
||||
import type { ProviderType } from '../providers/provider_factory.js';
|
||||
|
||||
/**
|
||||
* Validation result for a tool
|
||||
*/
|
||||
export interface ValidationResult {
|
||||
valid: boolean;
|
||||
errors: ValidationError[];
|
||||
warnings: ValidationWarning[];
|
||||
fixedTool?: Tool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validation error
|
||||
*/
|
||||
export interface ValidationError {
|
||||
field: string;
|
||||
message: string;
|
||||
severity: 'error' | 'critical';
|
||||
}
|
||||
|
||||
/**
|
||||
* Validation warning
|
||||
*/
|
||||
export interface ValidationWarning {
|
||||
field: string;
|
||||
message: string;
|
||||
suggestion?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider-specific validation rules
|
||||
*/
|
||||
interface ProviderRules {
|
||||
maxFunctionNameLength: number;
|
||||
maxDescriptionLength: number;
|
||||
maxParameterDepth: number;
|
||||
maxParameterCount: number;
|
||||
allowEmptyRequired: boolean;
|
||||
requireDescriptions: boolean;
|
||||
functionNamePattern: RegExp;
|
||||
supportedTypes: Set<string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default validation rules per provider
|
||||
*/
|
||||
const PROVIDER_RULES: Record<string, ProviderRules> = {
|
||||
openai: {
|
||||
maxFunctionNameLength: 64,
|
||||
maxDescriptionLength: 1024,
|
||||
maxParameterDepth: 5,
|
||||
maxParameterCount: 20,
|
||||
allowEmptyRequired: true,
|
||||
requireDescriptions: true,
|
||||
functionNamePattern: /^[a-zA-Z0-9_-]+$/,
|
||||
supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array', 'integer'])
|
||||
},
|
||||
anthropic: {
|
||||
maxFunctionNameLength: 64,
|
||||
maxDescriptionLength: 1024,
|
||||
maxParameterDepth: 4,
|
||||
maxParameterCount: 15,
|
||||
allowEmptyRequired: false, // Anthropic requires non-empty required arrays
|
||||
requireDescriptions: true,
|
||||
functionNamePattern: /^[a-zA-Z0-9_-]+$/,
|
||||
supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array', 'integer'])
|
||||
},
|
||||
ollama: {
|
||||
maxFunctionNameLength: 50,
|
||||
maxDescriptionLength: 500,
|
||||
maxParameterDepth: 3,
|
||||
maxParameterCount: 10, // Local models have smaller context
|
||||
allowEmptyRequired: true,
|
||||
requireDescriptions: false,
|
||||
functionNamePattern: /^[a-zA-Z0-9_]+$/,
|
||||
supportedTypes: new Set(['string', 'number', 'boolean', 'object', 'array'])
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Provider tool validator class
|
||||
*/
|
||||
export class ProviderToolValidator {
|
||||
private providerRules: Map<string, ProviderRules>;
|
||||
|
||||
constructor() {
|
||||
this.providerRules = new Map(Object.entries(PROVIDER_RULES));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a tool for a specific provider
|
||||
*/
|
||||
validateTool(tool: Tool, provider: string): ValidationResult {
|
||||
const rules = this.providerRules.get(provider) || PROVIDER_RULES.openai;
|
||||
const errors: ValidationError[] = [];
|
||||
const warnings: ValidationWarning[] = [];
|
||||
|
||||
// Deep clone the tool for potential fixes
|
||||
const fixedTool = JSON.parse(JSON.stringify(tool)) as Tool;
|
||||
let wasFixed = false;
|
||||
|
||||
// Validate function name
|
||||
const nameValidation = this.validateFunctionName(
|
||||
fixedTool.function.name,
|
||||
rules
|
||||
);
|
||||
if (nameValidation.error) {
|
||||
errors.push(nameValidation.error);
|
||||
}
|
||||
if (nameValidation.fixed) {
|
||||
fixedTool.function.name = nameValidation.fixed;
|
||||
wasFixed = true;
|
||||
}
|
||||
|
||||
// Validate description
|
||||
const descValidation = this.validateDescription(
|
||||
fixedTool.function.description,
|
||||
rules
|
||||
);
|
||||
if (descValidation.error) {
|
||||
errors.push(descValidation.error);
|
||||
}
|
||||
if (descValidation.warning) {
|
||||
warnings.push(descValidation.warning);
|
||||
}
|
||||
if (descValidation.fixed) {
|
||||
fixedTool.function.description = descValidation.fixed;
|
||||
wasFixed = true;
|
||||
}
|
||||
|
||||
// Validate parameters
|
||||
const paramValidation = this.validateParameters(
|
||||
fixedTool.function.parameters,
|
||||
rules,
|
||||
provider
|
||||
);
|
||||
errors.push(...paramValidation.errors);
|
||||
warnings.push(...paramValidation.warnings);
|
||||
if (paramValidation.fixed) {
|
||||
fixedTool.function.parameters = paramValidation.fixed;
|
||||
wasFixed = true;
|
||||
}
|
||||
|
||||
// Provider-specific validations
|
||||
const providerSpecific = this.validateProviderSpecific(fixedTool, provider);
|
||||
errors.push(...providerSpecific.errors);
|
||||
warnings.push(...providerSpecific.warnings);
|
||||
if (providerSpecific.fixed) {
|
||||
Object.assign(fixedTool, providerSpecific.fixed);
|
||||
wasFixed = true;
|
||||
}
|
||||
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
errors,
|
||||
warnings,
|
||||
fixedTool: wasFixed ? fixedTool : undefined
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate function name
|
||||
*/
|
||||
private validateFunctionName(name: string, rules: ProviderRules) {
|
||||
const result: any = {};
|
||||
|
||||
// Check length
|
||||
if (name.length > rules.maxFunctionNameLength) {
|
||||
result.error = {
|
||||
field: 'function.name',
|
||||
message: `Function name exceeds maximum length of ${rules.maxFunctionNameLength}`,
|
||||
severity: 'error' as const
|
||||
};
|
||||
// Auto-fix: truncate
|
||||
result.fixed = name.substring(0, rules.maxFunctionNameLength);
|
||||
}
|
||||
|
||||
// Check pattern
|
||||
if (!rules.functionNamePattern.test(name)) {
|
||||
result.error = {
|
||||
field: 'function.name',
|
||||
message: `Function name contains invalid characters`,
|
||||
severity: 'error' as const
|
||||
};
|
||||
// Auto-fix: replace invalid characters
|
||||
result.fixed = name.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate description
|
||||
*/
|
||||
private validateDescription(description: string, rules: ProviderRules) {
|
||||
const result: any = {};
|
||||
|
||||
// Check if description exists when required
|
||||
if (rules.requireDescriptions && !description) {
|
||||
result.error = {
|
||||
field: 'function.description',
|
||||
message: 'Description is required',
|
||||
severity: 'error' as const
|
||||
};
|
||||
result.fixed = 'Performs an operation'; // Generic fallback
|
||||
}
|
||||
|
||||
// Check length
|
||||
if (description && description.length > rules.maxDescriptionLength) {
|
||||
result.warning = {
|
||||
field: 'function.description',
|
||||
message: `Description exceeds recommended length of ${rules.maxDescriptionLength}`,
|
||||
suggestion: 'Consider shortening the description'
|
||||
};
|
||||
// Auto-fix: truncate with ellipsis
|
||||
result.fixed = description.substring(0, rules.maxDescriptionLength - 3) + '...';
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate parameters
|
||||
*/
|
||||
private validateParameters(
|
||||
parameters: any,
|
||||
rules: ProviderRules,
|
||||
provider: string
|
||||
) {
|
||||
const errors: ValidationError[] = [];
|
||||
const warnings: ValidationWarning[] = [];
|
||||
let fixed: any = null;
|
||||
|
||||
// Ensure parameters is an object
|
||||
if (parameters.type !== 'object') {
|
||||
errors.push({
|
||||
field: 'function.parameters.type',
|
||||
message: 'Parameters must be of type "object"',
|
||||
severity: 'critical'
|
||||
});
|
||||
fixed = {
|
||||
type: 'object',
|
||||
properties: parameters.properties || {},
|
||||
required: parameters.required || []
|
||||
};
|
||||
}
|
||||
|
||||
// Check parameter count
|
||||
const paramCount = Object.keys(parameters.properties || {}).length;
|
||||
if (paramCount > rules.maxParameterCount) {
|
||||
warnings.push({
|
||||
field: 'function.parameters',
|
||||
message: `Parameter count (${paramCount}) exceeds recommended maximum (${rules.maxParameterCount})`,
|
||||
suggestion: 'Consider reducing the number of parameters'
|
||||
});
|
||||
}
|
||||
|
||||
// Validate required array for Anthropic
|
||||
if (!rules.allowEmptyRequired && (!parameters.required || parameters.required.length === 0)) {
|
||||
if (provider === 'anthropic') {
|
||||
// For Anthropic, add at least one optional parameter to required
|
||||
const props = Object.keys(parameters.properties || {});
|
||||
if (props.length > 0) {
|
||||
if (!fixed) fixed = { ...parameters };
|
||||
fixed.required = [props[0]]; // Add first property as required
|
||||
warnings.push({
|
||||
field: 'function.parameters.required',
|
||||
message: 'Anthropic requires non-empty required array, added first parameter',
|
||||
suggestion: 'Specify which parameters are required'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate parameter types and depth
|
||||
if (parameters.properties) {
|
||||
const typeErrors = this.validateParameterTypes(
|
||||
parameters.properties,
|
||||
rules.supportedTypes,
|
||||
0,
|
||||
rules.maxParameterDepth
|
||||
);
|
||||
errors.push(...typeErrors);
|
||||
}
|
||||
|
||||
return { errors, warnings, fixed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate parameter types recursively
|
||||
*/
|
||||
private validateParameterTypes(
|
||||
properties: Record<string, ToolParameter>,
|
||||
supportedTypes: Set<string>,
|
||||
depth: number,
|
||||
maxDepth: number
|
||||
): ValidationError[] {
|
||||
const errors: ValidationError[] = [];
|
||||
|
||||
if (depth > maxDepth) {
|
||||
errors.push({
|
||||
field: 'function.parameters',
|
||||
message: `Parameter nesting exceeds maximum depth of ${maxDepth}`,
|
||||
severity: 'error'
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
|
||||
for (const [key, param] of Object.entries(properties)) {
|
||||
// Check if type is supported
|
||||
if (param.type && !supportedTypes.has(param.type)) {
|
||||
errors.push({
|
||||
field: `function.parameters.properties.${key}.type`,
|
||||
message: `Unsupported type: ${param.type}`,
|
||||
severity: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
// Recursively check nested objects
|
||||
if (param.type === 'object' && param.properties) {
|
||||
const nestedErrors = this.validateParameterTypes(
|
||||
param.properties,
|
||||
supportedTypes,
|
||||
depth + 1,
|
||||
maxDepth
|
||||
);
|
||||
errors.push(...nestedErrors);
|
||||
}
|
||||
|
||||
// Check array items
|
||||
if (param.type === 'array' && param.items) {
|
||||
if (typeof param.items === 'object' && 'properties' in param.items) {
|
||||
const nestedErrors = this.validateParameterTypes(
|
||||
param.items.properties!,
|
||||
supportedTypes,
|
||||
depth + 1,
|
||||
maxDepth
|
||||
);
|
||||
errors.push(...nestedErrors);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider-specific validations
|
||||
*/
|
||||
private validateProviderSpecific(tool: Tool, provider: string) {
|
||||
const errors: ValidationError[] = [];
|
||||
const warnings: ValidationWarning[] = [];
|
||||
let fixed: any = null;
|
||||
|
||||
switch (provider) {
|
||||
case 'openai':
|
||||
// OpenAI-specific: Check for special characters in function names
|
||||
if (tool.function.name.includes('-')) {
|
||||
warnings.push({
|
||||
field: 'function.name',
|
||||
message: 'OpenAI prefers underscores over hyphens in function names',
|
||||
suggestion: 'Replace hyphens with underscores'
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'anthropic':
|
||||
// Anthropic-specific: Ensure descriptions are meaningful
|
||||
if (tool.function.description && tool.function.description.length < 10) {
|
||||
warnings.push({
|
||||
field: 'function.description',
|
||||
message: 'Description is very short',
|
||||
suggestion: 'Provide a more detailed description for better results'
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'ollama':
|
||||
// Ollama-specific: Warn about complex nested structures
|
||||
const complexity = this.calculateComplexity(tool.function.parameters);
|
||||
if (complexity > 10) {
|
||||
warnings.push({
|
||||
field: 'function.parameters',
|
||||
message: 'Tool parameters are complex for local models',
|
||||
suggestion: 'Consider simplifying the parameter structure'
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return { errors, warnings, fixed };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate parameter complexity score
|
||||
*/
|
||||
private calculateComplexity(parameters: any, depth: number = 0): number {
|
||||
let complexity = depth;
|
||||
|
||||
if (parameters.properties) {
|
||||
for (const param of Object.values(parameters.properties) as ToolParameter[]) {
|
||||
complexity += 1;
|
||||
if (param.type === 'object' && param.properties) {
|
||||
complexity += this.calculateComplexity(param, depth + 1);
|
||||
}
|
||||
if (param.type === 'array' && param.items) {
|
||||
complexity += 2; // Arrays add more complexity
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return complexity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch validate multiple tools
|
||||
*/
|
||||
validateTools(tools: Tool[], provider: string): Map<string, ValidationResult> {
|
||||
const results = new Map<string, ValidationResult>();
|
||||
|
||||
for (const tool of tools) {
|
||||
const result = this.validateTool(tool, provider);
|
||||
results.set(tool.function.name, result);
|
||||
|
||||
if (!result.valid) {
|
||||
log.info(`Tool '${tool.function.name}' validation failed for ${provider}: ${JSON.stringify(result.errors)}`);
|
||||
}
|
||||
if (result.warnings.length > 0) {
|
||||
log.info(`Tool '${tool.function.name}' validation warnings for ${provider}: ${JSON.stringify(result.warnings)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-fix tools for a provider
|
||||
*/
|
||||
autoFixTools(tools: Tool[], provider: string): Tool[] {
|
||||
const fixed: Tool[] = [];
|
||||
|
||||
for (const tool of tools) {
|
||||
const result = this.validateTool(tool, provider);
|
||||
fixed.push(result.fixedTool || tool);
|
||||
}
|
||||
|
||||
return fixed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a provider supports a tool
|
||||
*/
|
||||
isToolSupportedByProvider(tool: Tool, provider: string): boolean {
|
||||
const result = this.validateTool(tool, provider);
|
||||
return result.valid || (result.fixedTool !== undefined);
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const providerToolValidator = new ProviderToolValidator();
|
||||
@@ -7,6 +7,9 @@
|
||||
|
||||
import log from '../../log.js';
|
||||
import type { Tool, ToolCall, ToolParameter } from './tool_interfaces.js';
|
||||
import { providerToolValidator } from './provider_tool_validator.js';
|
||||
import { edgeCaseHandler } from '../providers/edge_case_handler.js';
|
||||
import { parameterCoercer } from './parameter_coercer.js';
|
||||
|
||||
/**
|
||||
* Anthropic tool format
|
||||
@@ -51,17 +54,23 @@ export class ToolFormatAdapter {
|
||||
* Convert tools from standard format to provider-specific format
|
||||
*/
|
||||
static convertToProviderFormat(tools: Tool[], provider: ProviderType): unknown[] {
|
||||
// First validate and fix tools for the provider
|
||||
const validatedTools = providerToolValidator.autoFixTools(tools, provider);
|
||||
|
||||
// Apply edge case fixes
|
||||
const fixedTools = edgeCaseHandler.fixToolsForProvider(validatedTools, provider);
|
||||
|
||||
switch (provider) {
|
||||
case 'anthropic':
|
||||
return this.convertToAnthropicFormat(tools);
|
||||
return this.convertToAnthropicFormat(fixedTools);
|
||||
case 'ollama':
|
||||
return this.convertToOllamaFormat(tools);
|
||||
return this.convertToOllamaFormat(fixedTools);
|
||||
case 'openai':
|
||||
// OpenAI format matches our standard format
|
||||
return tools;
|
||||
return fixedTools;
|
||||
default:
|
||||
log.info(`Warning: Unknown provider ${provider}, returning tools in standard format`);
|
||||
return tools;
|
||||
return fixedTools;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,18 +309,42 @@ export class ToolFormatAdapter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse tool arguments safely
|
||||
* Parse tool arguments safely with coercion
|
||||
*/
|
||||
static parseToolArguments(args: string | Record<string, unknown>): Record<string, unknown> {
|
||||
static parseToolArguments(
|
||||
args: string | Record<string, unknown>,
|
||||
tool?: Tool,
|
||||
provider?: string
|
||||
): Record<string, unknown> {
|
||||
let parsedArgs: Record<string, unknown>;
|
||||
|
||||
if (typeof args === 'string') {
|
||||
try {
|
||||
return JSON.parse(args);
|
||||
parsedArgs = JSON.parse(args);
|
||||
} catch (error) {
|
||||
log.error(`Failed to parse tool arguments as JSON: ${error}`);
|
||||
return {};
|
||||
}
|
||||
} else {
|
||||
parsedArgs = args || {};
|
||||
}
|
||||
return args || {};
|
||||
|
||||
// Apply parameter coercion if tool definition is provided
|
||||
if (tool) {
|
||||
const coercionResult = parameterCoercer.coerceToolArguments(
|
||||
parsedArgs,
|
||||
tool,
|
||||
{ provider }
|
||||
);
|
||||
|
||||
if (coercionResult.warnings.length > 0) {
|
||||
log.info(`Parameter coercion warnings: ${coercionResult.warnings.join(', ')}`);
|
||||
}
|
||||
|
||||
return coercionResult.value;
|
||||
}
|
||||
|
||||
return parsedArgs;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
547
apps/server/src/services/llm/tools/tool_response_cache.ts
Normal file
547
apps/server/src/services/llm/tools/tool_response_cache.ts
Normal file
@@ -0,0 +1,547 @@
|
||||
/**
|
||||
* Tool Response Cache
|
||||
*
|
||||
* Implements LRU cache with TTL for deterministic/read-only tool responses,
|
||||
* with cache key generation, invalidation strategies, and hit rate tracking.
|
||||
*/
|
||||
|
||||
import log from '../../log.js';
|
||||
import crypto from 'crypto';
|
||||
|
||||
/**
|
||||
* Cache entry with metadata
|
||||
*/
|
||||
interface CacheEntry<T = any> {
|
||||
key: string;
|
||||
value: T;
|
||||
timestamp: Date;
|
||||
expiresAt: Date;
|
||||
hits: number;
|
||||
size: number;
|
||||
toolName: string;
|
||||
provider?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache statistics
|
||||
*/
|
||||
export interface CacheStatistics {
|
||||
totalEntries: number;
|
||||
totalSize: number;
|
||||
hitRate: number;
|
||||
missRate: number;
|
||||
evictionCount: number;
|
||||
avgHitsPerEntry: number;
|
||||
oldestEntry?: Date;
|
||||
newestEntry?: Date;
|
||||
topTools: Array<{ tool: string; hits: number }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache configuration
|
||||
*/
|
||||
export interface CacheConfig {
|
||||
/** Maximum cache size in bytes (default: 50MB) */
|
||||
maxSize: number;
|
||||
/** Maximum number of entries (default: 1000) */
|
||||
maxEntries: number;
|
||||
/** Default TTL in milliseconds (default: 300000 - 5 minutes) */
|
||||
defaultTTL: number;
|
||||
/** Enable automatic cleanup (default: true) */
|
||||
autoCleanup: boolean;
|
||||
/** Cleanup interval in milliseconds (default: 60000) */
|
||||
cleanupInterval: number;
|
||||
/** Enable hit tracking (default: true) */
|
||||
trackHits: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration
|
||||
*/
|
||||
const DEFAULT_CONFIG: CacheConfig = {
|
||||
maxSize: 50 * 1024 * 1024, // 50MB
|
||||
maxEntries: 1000,
|
||||
defaultTTL: 300000, // 5 minutes
|
||||
autoCleanup: true,
|
||||
cleanupInterval: 60000, // 1 minute
|
||||
trackHits: true
|
||||
};
|
||||
|
||||
/**
|
||||
* Tool-specific TTL overrides (in milliseconds)
|
||||
*/
|
||||
const TOOL_TTL_OVERRIDES: Record<string, number> = {
|
||||
// Static data tools - longer TTL
|
||||
'read_note_tool': 600000, // 10 minutes
|
||||
'get_note_metadata': 600000, // 10 minutes
|
||||
|
||||
// Search tools - medium TTL
|
||||
'search_notes_tool': 300000, // 5 minutes
|
||||
'keyword_search_tool': 300000, // 5 minutes
|
||||
|
||||
// Dynamic data tools - shorter TTL
|
||||
'get_recent_notes': 60000, // 1 minute
|
||||
'get_workspace_status': 30000 // 30 seconds
|
||||
};
|
||||
|
||||
/**
|
||||
* Deterministic tools that can be cached
|
||||
*/
|
||||
const CACHEABLE_TOOLS = new Set([
|
||||
'read_note_tool',
|
||||
'search_notes_tool',
|
||||
'keyword_search_tool',
|
||||
'attribute_search_tool',
|
||||
'get_note_metadata',
|
||||
'get_note_content',
|
||||
'get_recent_notes',
|
||||
'get_workspace_status',
|
||||
'list_notes',
|
||||
'find_notes_by_tag'
|
||||
]);
|
||||
|
||||
/**
|
||||
* Tool response cache class
|
||||
*/
|
||||
export class ToolResponseCache {
|
||||
private config: CacheConfig;
|
||||
private cache: Map<string, CacheEntry>;
|
||||
private accessOrder: string[];
|
||||
private totalHits: number;
|
||||
private totalMisses: number;
|
||||
private evictionCount: number;
|
||||
private cleanupTimer?: NodeJS.Timeout;
|
||||
private currentSize: number;
|
||||
|
||||
constructor(config?: Partial<CacheConfig>) {
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
this.cache = new Map();
|
||||
this.accessOrder = [];
|
||||
this.totalHits = 0;
|
||||
this.totalMisses = 0;
|
||||
this.evictionCount = 0;
|
||||
this.currentSize = 0;
|
||||
|
||||
if (this.config.autoCleanup) {
|
||||
this.startAutoCleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool is cacheable
|
||||
*/
|
||||
isCacheable(toolName: string): boolean {
|
||||
return CACHEABLE_TOOLS.has(toolName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate cache key for tool call
|
||||
*/
|
||||
generateCacheKey(
|
||||
toolName: string,
|
||||
args: Record<string, any>,
|
||||
provider?: string
|
||||
): string {
|
||||
// Sort arguments for consistent key generation
|
||||
const sortedArgs = this.sortObjectDeep(args);
|
||||
|
||||
// Create key components
|
||||
const keyComponents = {
|
||||
tool: toolName,
|
||||
args: sortedArgs,
|
||||
provider: provider || 'default'
|
||||
};
|
||||
|
||||
// Generate hash
|
||||
const hash = crypto
|
||||
.createHash('sha256')
|
||||
.update(JSON.stringify(keyComponents))
|
||||
.digest('hex');
|
||||
|
||||
return `${toolName}:${hash.substring(0, 16)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached response
|
||||
*/
|
||||
get(
|
||||
toolName: string,
|
||||
args: Record<string, any>,
|
||||
provider?: string
|
||||
): any | undefined {
|
||||
if (!this.isCacheable(toolName)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const key = this.generateCacheKey(toolName, args, provider);
|
||||
const entry = this.cache.get(key);
|
||||
|
||||
if (!entry) {
|
||||
this.totalMisses++;
|
||||
log.info(`Cache miss for ${toolName}`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Check if expired
|
||||
if (new Date() > entry.expiresAt) {
|
||||
this.cache.delete(key);
|
||||
this.removeFromAccessOrder(key);
|
||||
this.currentSize -= entry.size;
|
||||
this.totalMisses++;
|
||||
log.info(`Cache expired for ${toolName}`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Update hit count and access order
|
||||
if (this.config.trackHits) {
|
||||
entry.hits++;
|
||||
this.updateAccessOrder(key);
|
||||
}
|
||||
|
||||
this.totalHits++;
|
||||
log.info(`Cache hit for ${toolName} (${entry.hits} hits)`);
|
||||
|
||||
return entry.value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set cached response
|
||||
*/
|
||||
set(
|
||||
toolName: string,
|
||||
args: Record<string, any>,
|
||||
value: any,
|
||||
provider?: string,
|
||||
ttl?: number
|
||||
): boolean {
|
||||
if (!this.isCacheable(toolName)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const key = this.generateCacheKey(toolName, args, provider);
|
||||
const size = this.calculateSize(value);
|
||||
|
||||
// Check size limits
|
||||
if (size > this.config.maxSize) {
|
||||
log.info(`Cache entry too large for ${toolName}: ${size} bytes`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Evict entries if necessary
|
||||
while (this.cache.size >= this.config.maxEntries ||
|
||||
this.currentSize + size > this.config.maxSize) {
|
||||
this.evictLRU();
|
||||
}
|
||||
|
||||
// Determine TTL
|
||||
const effectiveTTL = ttl ||
|
||||
TOOL_TTL_OVERRIDES[toolName] ||
|
||||
this.config.defaultTTL;
|
||||
|
||||
// Create entry
|
||||
const entry: CacheEntry = {
|
||||
key,
|
||||
value,
|
||||
timestamp: new Date(),
|
||||
expiresAt: new Date(Date.now() + effectiveTTL),
|
||||
hits: 0,
|
||||
size,
|
||||
toolName,
|
||||
provider
|
||||
};
|
||||
|
||||
// Add to cache
|
||||
this.cache.set(key, entry);
|
||||
this.accessOrder.push(key);
|
||||
this.currentSize += size;
|
||||
|
||||
log.info(`Cached response for ${toolName} (${size} bytes, TTL: ${effectiveTTL}ms)`);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cache entries
|
||||
*/
|
||||
invalidate(filter?: {
|
||||
toolName?: string;
|
||||
provider?: string;
|
||||
pattern?: RegExp;
|
||||
}): number {
|
||||
let invalidated = 0;
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
let shouldInvalidate = false;
|
||||
|
||||
if (filter?.toolName && entry.toolName === filter.toolName) {
|
||||
shouldInvalidate = true;
|
||||
}
|
||||
if (filter?.provider && entry.provider === filter.provider) {
|
||||
shouldInvalidate = true;
|
||||
}
|
||||
if (filter?.pattern && filter.pattern.test(key)) {
|
||||
shouldInvalidate = true;
|
||||
}
|
||||
if (!filter) {
|
||||
shouldInvalidate = true; // Invalidate all if no filter
|
||||
}
|
||||
|
||||
if (shouldInvalidate) {
|
||||
this.cache.delete(key);
|
||||
this.removeFromAccessOrder(key);
|
||||
this.currentSize -= entry.size;
|
||||
invalidated++;
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`Invalidated ${invalidated} cache entries`);
|
||||
return invalidated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict least recently used entry
|
||||
*/
|
||||
private evictLRU(): void {
|
||||
if (this.accessOrder.length === 0) return;
|
||||
|
||||
const key = this.accessOrder.shift()!;
|
||||
const entry = this.cache.get(key);
|
||||
|
||||
if (entry) {
|
||||
this.cache.delete(key);
|
||||
this.currentSize -= entry.size;
|
||||
this.evictionCount++;
|
||||
log.info(`Evicted cache entry for ${entry.toolName} (LRU)`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update access order for LRU
|
||||
*/
|
||||
private updateAccessOrder(key: string): void {
|
||||
this.removeFromAccessOrder(key);
|
||||
this.accessOrder.push(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove from access order
|
||||
*/
|
||||
private removeFromAccessOrder(key: string): void {
|
||||
const index = this.accessOrder.indexOf(key);
|
||||
if (index > -1) {
|
||||
this.accessOrder.splice(index, 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate size of value in bytes
|
||||
*/
|
||||
private calculateSize(value: any): number {
|
||||
const str = typeof value === 'string' ? value : JSON.stringify(value);
|
||||
return Buffer.byteLength(str, 'utf8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort object deeply for consistent key generation
|
||||
*/
|
||||
private sortObjectDeep(obj: any): any {
|
||||
if (obj === null || typeof obj !== 'object') {
|
||||
return obj;
|
||||
}
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(item => this.sortObjectDeep(item));
|
||||
}
|
||||
|
||||
const sorted: any = {};
|
||||
const keys = Object.keys(obj).sort();
|
||||
|
||||
for (const key of keys) {
|
||||
sorted[key] = this.sortObjectDeep(obj[key]);
|
||||
}
|
||||
|
||||
return sorted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start automatic cleanup
|
||||
*/
|
||||
private startAutoCleanup(): void {
|
||||
this.cleanupTimer = setInterval(() => {
|
||||
this.cleanup();
|
||||
}, this.config.cleanupInterval);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop automatic cleanup
|
||||
*/
|
||||
private stopAutoCleanup(): void {
|
||||
if (this.cleanupTimer) {
|
||||
clearInterval(this.cleanupTimer);
|
||||
this.cleanupTimer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up expired entries
|
||||
*/
|
||||
cleanup(): number {
|
||||
const now = new Date();
|
||||
let cleaned = 0;
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
if (now > entry.expiresAt) {
|
||||
this.cache.delete(key);
|
||||
this.removeFromAccessOrder(key);
|
||||
this.currentSize -= entry.size;
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
if (cleaned > 0) {
|
||||
log.info(`Cleaned up ${cleaned} expired cache entries`);
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStatistics(): CacheStatistics {
|
||||
const entries = Array.from(this.cache.values());
|
||||
const totalRequests = this.totalHits + this.totalMisses;
|
||||
|
||||
// Calculate tool hit counts
|
||||
const toolHits = new Map<string, number>();
|
||||
for (const entry of entries) {
|
||||
const current = toolHits.get(entry.toolName) || 0;
|
||||
toolHits.set(entry.toolName, current + entry.hits);
|
||||
}
|
||||
|
||||
// Sort tools by hits
|
||||
const topTools = Array.from(toolHits.entries())
|
||||
.map(([tool, hits]) => ({ tool, hits }))
|
||||
.sort((a, b) => b.hits - a.hits)
|
||||
.slice(0, 10);
|
||||
|
||||
// Find oldest and newest entries
|
||||
const timestamps = entries.map(e => e.timestamp);
|
||||
const oldestEntry = timestamps.length > 0
|
||||
? new Date(Math.min(...timestamps.map(t => t.getTime())))
|
||||
: undefined;
|
||||
const newestEntry = timestamps.length > 0
|
||||
? new Date(Math.max(...timestamps.map(t => t.getTime())))
|
||||
: undefined;
|
||||
|
||||
// Calculate average hits
|
||||
const totalHitsInCache = entries.reduce((sum, e) => sum + e.hits, 0);
|
||||
const avgHitsPerEntry = entries.length > 0
|
||||
? totalHitsInCache / entries.length
|
||||
: 0;
|
||||
|
||||
return {
|
||||
totalEntries: this.cache.size,
|
||||
totalSize: this.currentSize,
|
||||
hitRate: totalRequests > 0 ? this.totalHits / totalRequests : 0,
|
||||
missRate: totalRequests > 0 ? this.totalMisses / totalRequests : 0,
|
||||
evictionCount: this.evictionCount,
|
||||
avgHitsPerEntry,
|
||||
oldestEntry,
|
||||
newestEntry,
|
||||
topTools
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear entire cache
|
||||
*/
|
||||
clear(): void {
|
||||
this.cache.clear();
|
||||
this.accessOrder = [];
|
||||
this.currentSize = 0;
|
||||
this.totalHits = 0;
|
||||
this.totalMisses = 0;
|
||||
this.evictionCount = 0;
|
||||
log.info('Cleared entire cache');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache size info
|
||||
*/
|
||||
getSizeInfo(): {
|
||||
entries: number;
|
||||
bytes: number;
|
||||
maxEntries: number;
|
||||
maxBytes: number;
|
||||
utilizationPercent: number;
|
||||
} {
|
||||
return {
|
||||
entries: this.cache.size,
|
||||
bytes: this.currentSize,
|
||||
maxEntries: this.config.maxEntries,
|
||||
maxBytes: this.config.maxSize,
|
||||
utilizationPercent: (this.currentSize / this.config.maxSize) * 100
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Export cache contents
|
||||
*/
|
||||
exportCache(): string {
|
||||
const data = {
|
||||
config: this.config,
|
||||
entries: Array.from(this.cache.entries()),
|
||||
statistics: this.getStatistics(),
|
||||
metadata: {
|
||||
exportedAt: new Date(),
|
||||
version: '1.0.0'
|
||||
}
|
||||
};
|
||||
|
||||
return JSON.stringify(data, null, 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Import cache contents
|
||||
*/
|
||||
importCache(json: string): void {
|
||||
try {
|
||||
const data = JSON.parse(json);
|
||||
|
||||
// Clear existing cache
|
||||
this.clear();
|
||||
|
||||
// Import entries
|
||||
for (const [key, entry] of data.entries) {
|
||||
// Convert dates
|
||||
entry.timestamp = new Date(entry.timestamp);
|
||||
entry.expiresAt = new Date(entry.expiresAt);
|
||||
|
||||
// Skip expired entries
|
||||
if (new Date() > entry.expiresAt) continue;
|
||||
|
||||
this.cache.set(key, entry);
|
||||
this.accessOrder.push(key);
|
||||
this.currentSize += entry.size;
|
||||
}
|
||||
|
||||
log.info(`Imported ${this.cache.size} cache entries`);
|
||||
} catch (error) {
|
||||
log.error(`Failed to import cache: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown cache
|
||||
*/
|
||||
shutdown(): void {
|
||||
this.stopAutoCleanup();
|
||||
this.clear();
|
||||
log.info('Cache shutdown complete');
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const toolResponseCache = new ToolResponseCache();
|
||||
329
apps/server/src/services/llm/tools/tool_timeout_enforcer.ts
Normal file
329
apps/server/src/services/llm/tools/tool_timeout_enforcer.ts
Normal file
@@ -0,0 +1,329 @@
|
||||
/**
|
||||
* Tool Timeout Enforcer
|
||||
*
|
||||
* Implements timeout enforcement for tool executions with configurable timeouts
|
||||
* per tool type, graceful cleanup, and Promise.race pattern for detection.
|
||||
*/
|
||||
|
||||
import log from '../../log.js';
|
||||
import type { ToolHandler } from './tool_interfaces.js';
|
||||
|
||||
/**
|
||||
* Timeout configuration per tool type
|
||||
*/
|
||||
export interface TimeoutConfig {
|
||||
/** Timeout for search operations in milliseconds */
|
||||
search: number;
|
||||
/** Timeout for create/update operations in milliseconds */
|
||||
mutation: number;
|
||||
/** Timeout for script execution in milliseconds */
|
||||
script: number;
|
||||
/** Default timeout for unspecified tools in milliseconds */
|
||||
default: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tool execution result with timeout metadata
|
||||
*/
|
||||
export interface TimeoutResult<T = any> {
|
||||
success: boolean;
|
||||
result?: T;
|
||||
error?: Error;
|
||||
timedOut: boolean;
|
||||
executionTime: number;
|
||||
toolName: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tool categories for timeout assignment
|
||||
*/
|
||||
export enum ToolCategory {
|
||||
SEARCH = 'search',
|
||||
MUTATION = 'mutation',
|
||||
SCRIPT = 'script',
|
||||
READ = 'read',
|
||||
DEFAULT = 'default'
|
||||
}
|
||||
|
||||
/**
|
||||
* Default timeout configuration
|
||||
*/
|
||||
const DEFAULT_TIMEOUTS: TimeoutConfig = {
|
||||
search: 5000, // 5 seconds for search operations
|
||||
mutation: 3000, // 3 seconds for create/update operations
|
||||
script: 10000, // 10 seconds for script execution
|
||||
default: 5000 // 5 seconds default
|
||||
};
|
||||
|
||||
/**
|
||||
* Tool timeout enforcer class
|
||||
*/
|
||||
export class ToolTimeoutEnforcer {
|
||||
private timeouts: TimeoutConfig;
|
||||
private executionStats: Map<string, { total: number; timeouts: number; avgTime: number }>;
|
||||
private activeExecutions: Map<string, AbortController>;
|
||||
|
||||
constructor(timeoutConfig?: Partial<TimeoutConfig>) {
|
||||
this.timeouts = { ...DEFAULT_TIMEOUTS, ...timeoutConfig };
|
||||
this.executionStats = new Map();
|
||||
this.activeExecutions = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize tool based on its name
|
||||
*/
|
||||
private categorizeeTool(toolName: string): ToolCategory {
|
||||
const name = toolName.toLowerCase();
|
||||
|
||||
// Search tools
|
||||
if (name.includes('search') || name.includes('find') || name.includes('query')) {
|
||||
return ToolCategory.SEARCH;
|
||||
}
|
||||
|
||||
// Mutation tools
|
||||
if (name.includes('create') || name.includes('update') || name.includes('delete') ||
|
||||
name.includes('modify') || name.includes('save')) {
|
||||
return ToolCategory.MUTATION;
|
||||
}
|
||||
|
||||
// Script tools
|
||||
if (name.includes('script') || name.includes('execute') || name.includes('eval')) {
|
||||
return ToolCategory.SCRIPT;
|
||||
}
|
||||
|
||||
// Read tools
|
||||
if (name.includes('read') || name.includes('get') || name.includes('fetch')) {
|
||||
return ToolCategory.READ;
|
||||
}
|
||||
|
||||
return ToolCategory.DEFAULT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get timeout for a specific tool
|
||||
*/
|
||||
private getToolTimeout(toolName: string): number {
|
||||
const category = this.categorizeeTool(toolName);
|
||||
|
||||
switch (category) {
|
||||
case ToolCategory.SEARCH:
|
||||
return this.timeouts.search;
|
||||
case ToolCategory.MUTATION:
|
||||
return this.timeouts.mutation;
|
||||
case ToolCategory.SCRIPT:
|
||||
return this.timeouts.script;
|
||||
case ToolCategory.READ:
|
||||
return this.timeouts.search; // Use search timeout for read operations
|
||||
default:
|
||||
return this.timeouts.default;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a tool with timeout enforcement
|
||||
*/
|
||||
async executeWithTimeout<T>(
|
||||
toolName: string,
|
||||
executeFn: () => Promise<T>,
|
||||
customTimeout?: number
|
||||
): Promise<TimeoutResult<T>> {
|
||||
const timeout = customTimeout || this.getToolTimeout(toolName);
|
||||
const startTime = Date.now();
|
||||
const executionId = `${toolName}_${startTime}_${Math.random()}`;
|
||||
|
||||
// Create abort controller for cleanup
|
||||
const abortController = new AbortController();
|
||||
this.activeExecutions.set(executionId, abortController);
|
||||
|
||||
log.info(`Executing tool '${toolName}' with timeout ${timeout}ms`);
|
||||
|
||||
try {
|
||||
// Create timeout promise
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
abortController.abort();
|
||||
reject(new Error(`Tool '${toolName}' execution timed out after ${timeout}ms`));
|
||||
}, timeout);
|
||||
|
||||
// Clean up timer if aborted
|
||||
abortController.signal.addEventListener('abort', () => clearTimeout(timer));
|
||||
});
|
||||
|
||||
// Race between execution and timeout
|
||||
const result = await Promise.race([
|
||||
executeFn(),
|
||||
timeoutPromise
|
||||
]);
|
||||
|
||||
const executionTime = Date.now() - startTime;
|
||||
|
||||
// Update statistics
|
||||
this.updateStats(toolName, false, executionTime);
|
||||
|
||||
log.info(`Tool '${toolName}' completed successfully in ${executionTime}ms`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
result,
|
||||
timedOut: false,
|
||||
executionTime,
|
||||
toolName
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
const executionTime = Date.now() - startTime;
|
||||
const timedOut = executionTime >= timeout - 50; // Allow 50ms buffer
|
||||
|
||||
// Update statistics
|
||||
this.updateStats(toolName, timedOut, executionTime);
|
||||
|
||||
if (timedOut) {
|
||||
log.error(`Tool '${toolName}' timed out after ${executionTime}ms`);
|
||||
} else {
|
||||
log.error(`Tool '${toolName}' failed after ${executionTime}ms: ${error}`);
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error as Error,
|
||||
timedOut,
|
||||
executionTime,
|
||||
toolName
|
||||
};
|
||||
|
||||
} finally {
|
||||
// Clean up
|
||||
this.activeExecutions.delete(executionId);
|
||||
if (!abortController.signal.aborted) {
|
||||
abortController.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute multiple tools with timeout enforcement
|
||||
*/
|
||||
async executeBatchWithTimeout<T>(
|
||||
executions: Array<{
|
||||
toolName: string;
|
||||
executeFn: () => Promise<T>;
|
||||
customTimeout?: number;
|
||||
}>
|
||||
): Promise<TimeoutResult<T>[]> {
|
||||
return Promise.all(
|
||||
executions.map(({ toolName, executeFn, customTimeout }) =>
|
||||
this.executeWithTimeout(toolName, executeFn, customTimeout)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a tool handler with timeout enforcement
|
||||
*/
|
||||
wrapToolHandler(handler: ToolHandler, customTimeout?: number): ToolHandler {
|
||||
const toolName = handler.definition.function.name;
|
||||
|
||||
return {
|
||||
definition: handler.definition,
|
||||
execute: async (args: Record<string, unknown>) => {
|
||||
const result = await this.executeWithTimeout(
|
||||
toolName,
|
||||
() => handler.execute(args),
|
||||
customTimeout
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
if (result.timedOut) {
|
||||
throw new Error(`Tool execution timed out after ${result.executionTime}ms`);
|
||||
}
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
return result.result!;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update execution statistics
|
||||
*/
|
||||
private updateStats(toolName: string, timedOut: boolean, executionTime: number): void {
|
||||
const current = this.executionStats.get(toolName) || {
|
||||
total: 0,
|
||||
timeouts: 0,
|
||||
avgTime: 0
|
||||
};
|
||||
|
||||
const newTotal = current.total + 1;
|
||||
const newTimeouts = current.timeouts + (timedOut ? 1 : 0);
|
||||
const newAvgTime = (current.avgTime * current.total + executionTime) / newTotal;
|
||||
|
||||
this.executionStats.set(toolName, {
|
||||
total: newTotal,
|
||||
timeouts: newTimeouts,
|
||||
avgTime: newAvgTime
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get execution statistics for a tool
|
||||
*/
|
||||
getToolStats(toolName: string) {
|
||||
return this.executionStats.get(toolName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all execution statistics
|
||||
*/
|
||||
getAllStats() {
|
||||
return Object.fromEntries(this.executionStats);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear statistics
|
||||
*/
|
||||
clearStats(): void {
|
||||
this.executionStats.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort all active executions
|
||||
*/
|
||||
abortAll(): void {
|
||||
log.info(`Aborting ${this.activeExecutions.size} active tool executions`);
|
||||
|
||||
for (const [id, controller] of this.activeExecutions) {
|
||||
controller.abort();
|
||||
}
|
||||
|
||||
this.activeExecutions.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get timeout configuration
|
||||
*/
|
||||
getTimeouts(): TimeoutConfig {
|
||||
return { ...this.timeouts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update timeout configuration
|
||||
*/
|
||||
updateTimeouts(config: Partial<TimeoutConfig>): void {
|
||||
this.timeouts = { ...this.timeouts, ...config };
|
||||
log.info(`Updated timeout configuration: ${JSON.stringify(this.timeouts)}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool has high timeout rate
|
||||
*/
|
||||
hasHighTimeoutRate(toolName: string, threshold: number = 0.5): boolean {
|
||||
const stats = this.executionStats.get(toolName);
|
||||
if (!stats || stats.total === 0) return false;
|
||||
|
||||
return (stats.timeouts / stats.total) > threshold;
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const toolTimeoutEnforcer = new ToolTimeoutEnforcer();
|
||||
Reference in New Issue
Block a user