Dockery/Trilium
1
0
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-10-26 07:46:30 +01:00
Code Issues Packages Projects Releases Activity

feat(llm): try to stop some of the horrible memory management

Browse Source
This commit is contained in:
perfectra1n
2025-08-08 22:15:58 -07:00
parent 89fcfabd3c
commit 0d898385f6
6 changed files with 939 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
apps/client/src/widgets/llm_chat/communication.ts
View File

@@ -72,9 +72,14 @@ export async function setupStreamingResponse(
let timeoutId: number | null = null;
let initialTimeoutId: number | null = null;
let cleanupTimeoutId: number | null = null;
let heartbeatTimeoutId: number | null = null;
let receivedAnyMessage = false;
let eventListener: ((event: Event) => void) | null = null;
let lastMessageTimestamp = 0;
// Configuration for timeouts
const HEARTBEAT_TIMEOUT_MS = 30000; // 30 seconds between messages
const MAX_IDLE_TIME_MS = 60000; // 60 seconds max idle time
// Create a unique identifier for this response process
const responseId = `llm-stream-${Date.now()}-${Math.floor(Math.random() * 1000)}`;
@@ -107,12 +112,43 @@ export async function setupStreamingResponse(
}
})();
// Function to reset heartbeat timeout
const resetHeartbeatTimeout = () => {
if (heartbeatTimeoutId) {
window.clearTimeout(heartbeatTimeoutId);
}
heartbeatTimeoutId = window.setTimeout(() => {
const idleTime = Date.now() - lastMessageTimestamp;
console.warn(`[${responseId}] No message received for ${idleTime}ms`);
if (idleTime > MAX_IDLE_TIME_MS) {
console.error(`[${responseId}] Connection appears to be stalled (idle for ${idleTime}ms)`);
performCleanup();
reject(new Error('Connection lost: The AI service stopped responding. Please try again.'));
} else {
// Send a warning but continue waiting
console.warn(`[${responseId}] Connection may be slow, continuing to wait...`);
resetHeartbeatTimeout(); // Reset for another check
}
}, HEARTBEAT_TIMEOUT_MS);
};
// Function to safely perform cleanup
const performCleanup = () => {
// Clear all timeouts
if (cleanupTimeoutId) {
window.clearTimeout(cleanupTimeoutId);
cleanupTimeoutId = null;
}
if (heartbeatTimeoutId) {
window.clearTimeout(heartbeatTimeoutId);
heartbeatTimeoutId = null;
}
if (initialTimeoutId) {
window.clearTimeout(initialTimeoutId);
initialTimeoutId = null;
}
console.log(`[${responseId}] Performing final cleanup of event listener`);
cleanupEventListener(eventListener);
@@ -121,13 +157,15 @@ export async function setupStreamingResponse(
};
// Set initial timeout to catch cases where no message is received at all
// Increased timeout and better error messaging
const INITIAL_TIMEOUT_MS = 15000; // 15 seconds for initial response
initialTimeoutId = window.setTimeout(() => {
if (!receivedAnyMessage) {
console.error(`[${responseId}] No initial message received within timeout`);
console.error(`[${responseId}] No initial message received within ${INITIAL_TIMEOUT_MS}ms timeout`);
performCleanup();
reject(new Error('No response received from server'));
reject(new Error('Connection timeout: The AI service is taking longer than expected to respond. Please check your connection and try again.'));
}
}, 10000);
}, INITIAL_TIMEOUT_MS);
// Create a message handler for CustomEvents
eventListener = (event: Event) => {
@@ -161,6 +199,12 @@ export async function setupStreamingResponse(
window.clearTimeout(initialTimeoutId);
initialTimeoutId = null;
}
// Start heartbeat monitoring
resetHeartbeatTimeout();
} else {
// Reset heartbeat on each new message
resetHeartbeatTimeout();
}
// Handle error

309
apps/client/src/widgets/llm_chat/tool_execution_ui.ts Normal file
View File

@@ -0,0 +1,309 @@
/**
* Tool Execution UI Components
*
* This module provides enhanced UI components for displaying tool execution status,
* progress, and user-friendly error messages during LLM tool calls.
*/
import { t } from "../../services/i18n.js";
/**
* Tool execution status types
*/
export type ToolExecutionStatus = 'pending' | 'running' | 'success' | 'error' | 'cancelled';
/**
* Tool execution display data
*/
export interface ToolExecutionDisplay {
toolName: string;
displayName: string;
status: ToolExecutionStatus;
description?: string;
progress?: {
current: number;
total: number;
message?: string;
};
result?: string;
error?: string;
startTime?: number;
endTime?: number;
}
/**
* Map of tool names to user-friendly display names
*/
const TOOL_DISPLAY_NAMES: Record<string, string> = {
'search_notes': 'Searching Notes',
'get_note_content': 'Reading Note',
'create_note': 'Creating Note',
'update_note': 'Updating Note',
'execute_code': 'Running Code',
'web_search': 'Searching Web',
'get_note_attributes': 'Reading Note Properties',
'set_note_attribute': 'Setting Note Property',
'navigate_notes': 'Navigating Notes',
'query_decomposition': 'Analyzing Query',
'contextual_thinking': 'Processing Context'
};
/**
* Map of tool names to descriptions
*/
const TOOL_DESCRIPTIONS: Record<string, string> = {
'search_notes': 'Finding relevant notes in your knowledge base',
'get_note_content': 'Reading the content of a specific note',
'create_note': 'Creating a new note with the provided content',
'update_note': 'Updating an existing note',
'execute_code': 'Running code in a safe environment',
'web_search': 'Searching the web for current information',
'get_note_attributes': 'Reading note metadata and properties',
'set_note_attribute': 'Updating note metadata',
'navigate_notes': 'Exploring the note hierarchy',
'query_decomposition': 'Breaking down complex queries',
'contextual_thinking': 'Analyzing context for better understanding'
};
/**
* Create a tool execution indicator element
*/
export function createToolExecutionIndicator(toolName: string): HTMLElement {
const container = document.createElement('div');
container.className = 'tool-execution-indicator mb-2 p-2 border rounded bg-light';
container.dataset.toolName = toolName;
const displayName = TOOL_DISPLAY_NAMES[toolName] || toolName;
const description = TOOL_DESCRIPTIONS[toolName] || '';
container.innerHTML = `
<div class="d-flex align-items-center">
<div class="tool-status-icon me-2">
<div class="spinner-border spinner-border-sm text-primary" role="status">
<span class="visually-hidden">Loading...</span>
</div>
</div>
<div class="flex-grow-1">
<div class="tool-name fw-bold small">${displayName}</div>
${description ? `<div class="tool-description text-muted small">${description}</div>` : ''}
<div class="tool-progress" style="display: none;">
<div class="progress mt-1" style="height: 4px;">
<div class="progress-bar" role="progressbar" style="width: 0%"></div>
</div>
<div class="progress-message text-muted small mt-1"></div>
</div>
<div class="tool-result text-success small mt-1" style="display: none;"></div>
<div class="tool-error text-danger small mt-1" style="display: none;"></div>
</div>
<div class="tool-duration text-muted small ms-2" style="display: none;"></div>
</div>
`;
return container;
}
/**
* Update tool execution status
*/
export function updateToolExecutionStatus(
container: HTMLElement,
status: ToolExecutionStatus,
data?: {
progress?: { current: number; total: number; message?: string };
result?: string;
error?: string;
duration?: number;
}
): void {
const statusIcon = container.querySelector('.tool-status-icon');
const progressDiv = container.querySelector('.tool-progress') as HTMLElement;
const progressBar = container.querySelector('.progress-bar') as HTMLElement;
const progressMessage = container.querySelector('.progress-message') as HTMLElement;
const resultDiv = container.querySelector('.tool-result') as HTMLElement;
const errorDiv = container.querySelector('.tool-error') as HTMLElement;
const durationDiv = container.querySelector('.tool-duration') as HTMLElement;
if (!statusIcon) return;
// Update status icon
switch (status) {
case 'pending':
statusIcon.innerHTML = `
<div class="spinner-border spinner-border-sm text-secondary" role="status">
<span class="visually-hidden">Pending...</span>
</div>
`;
break;
case 'running':
statusIcon.innerHTML = `
<div class="spinner-border spinner-border-sm text-primary" role="status">
<span class="visually-hidden">Running...</span>
</div>
`;
break;
case 'success':
statusIcon.innerHTML = '<i class="bx bx-check-circle text-success fs-5"></i>';
container.classList.add('border-success', 'bg-success-subtle');
break;
case 'error':
statusIcon.innerHTML = '<i class="bx bx-error-circle text-danger fs-5"></i>';
container.classList.add('border-danger', 'bg-danger-subtle');
break;
case 'cancelled':
statusIcon.innerHTML = '<i class="bx bx-x-circle text-warning fs-5"></i>';
container.classList.add('border-warning', 'bg-warning-subtle');
break;
}
// Update progress if provided
if (data?.progress && progressDiv && progressBar && progressMessage) {
progressDiv.style.display = 'block';
const percentage = (data.progress.current / data.progress.total) * 100;
progressBar.style.width = `${percentage}%`;
if (data.progress.message) {
progressMessage.textContent = data.progress.message;
}
}
// Update result if provided
if (data?.result && resultDiv) {
resultDiv.style.display = 'block';
resultDiv.textContent = data.result;
}
// Update error if provided
if (data?.error && errorDiv) {
errorDiv.style.display = 'block';
errorDiv.textContent = formatErrorMessage(data.error);
}
// Update duration if provided
if (data?.duration && durationDiv) {
durationDiv.style.display = 'block';
durationDiv.textContent = formatDuration(data.duration);
}
}
/**
* Format error messages to be user-friendly
*/
function formatErrorMessage(error: string): string {
// Remove technical details and provide user-friendly messages
const errorMappings: Record<string, string> = {
'ECONNREFUSED': 'Connection refused. Please check if the service is running.',
'ETIMEDOUT': 'Request timed out. Please try again.',
'ENOTFOUND': 'Service not found. Please check your configuration.',
'401': 'Authentication failed. Please check your API credentials.',
'403': 'Access denied. Please check your permissions.',
'404': 'Resource not found.',
'429': 'Rate limit exceeded. Please wait a moment and try again.',
'500': 'Server error. Please try again later.',
'503': 'Service temporarily unavailable. Please try again later.'
};
for (const [key, message] of Object.entries(errorMappings)) {
if (error.includes(key)) {
return message;
}
}
// Generic error formatting
if (error.length > 100) {
return error.substring(0, 100) + '...';
}
return error;
}
/**
* Format duration in a human-readable way
*/
function formatDuration(milliseconds: number): string {
if (milliseconds < 1000) {
return `${milliseconds}ms`;
} else if (milliseconds < 60000) {
return `${(milliseconds / 1000).toFixed(1)}s`;
} else {
const minutes = Math.floor(milliseconds / 60000);
const seconds = Math.floor((milliseconds % 60000) / 1000);
return `${minutes}m ${seconds}s`;
}
}
/**
* Create a tool execution summary
*/
export function createToolExecutionSummary(executions: ToolExecutionDisplay[]): HTMLElement {
const container = document.createElement('div');
container.className = 'tool-execution-summary mt-2 p-2 border rounded bg-light small';
const successful = executions.filter(e => e.status === 'success').length;
const failed = executions.filter(e => e.status === 'error').length;
const total = executions.length;
const totalDuration = executions.reduce((sum, e) => {
if (e.startTime && e.endTime) {
return sum + (e.endTime - e.startTime);
}
return sum;
}, 0);
container.innerHTML = `
<div class="d-flex align-items-center justify-content-between">
<div>
<i class="bx bx-check-shield me-1"></i>
<span class="fw-bold">Tools Executed:</span>
<span class="badge bg-success ms-1">${successful} successful</span>
${failed > 0 ? `<span class="badge bg-danger ms-1">${failed} failed</span>` : ''}
<span class="badge bg-secondary ms-1">${total} total</span>
</div>
${totalDuration > 0 ? `
<div class="text-muted">
<i class="bx bx-time me-1"></i>
${formatDuration(totalDuration)}
</div>
` : ''}
</div>
`;
return container;
}
/**
* Create a loading indicator with custom message
*/
export function createLoadingIndicator(message: string = 'Processing...'): HTMLElement {
const container = document.createElement('div');
container.className = 'loading-indicator-enhanced d-flex align-items-center p-2';
container.innerHTML = `
<div class="spinner-grow spinner-grow-sm text-primary me-2" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span class="loading-message">${message}</span>
`;
return container;
}
/**
* Update loading indicator message
*/
export function updateLoadingMessage(container: HTMLElement, message: string): void {
const messageElement = container.querySelector('.loading-message');
if (messageElement) {
messageElement.textContent = message;
}
}
export default {
createToolExecutionIndicator,
updateToolExecutionStatus,
createToolExecutionSummary,
createLoadingIndicator,
updateLoadingMessage
};

190
apps/server/src/services/llm/ai_service_manager.ts
View File

@@ -39,10 +39,26 @@ interface NoteContext {
score?: number;
}
export class AIServiceManager implements IAIServiceManager {
private currentService: AIService | null = null;
private currentProvider: ServiceProviders | null = null;
// Service cache entry with TTL
interface ServiceCacheEntry {
service: AIService;
provider: ServiceProviders;
createdAt: number;
lastUsed: number;
}
// Disposable interface for proper resource cleanup
export interface Disposable {
dispose(): void | Promise<void>;
}
export class AIServiceManager implements IAIServiceManager, Disposable {
private serviceCache: Map<ServiceProviders, ServiceCacheEntry> = new Map();
private readonly SERVICE_TTL_MS = 5 * 60 * 1000; // 5 minutes TTL
private readonly CLEANUP_INTERVAL_MS = 60 * 1000; // Cleanup check every minute
private cleanupTimer: NodeJS.Timeout | null = null;
private initialized = false;
private disposed = false;
constructor() {
// Initialize tools immediately
@@ -50,7 +66,8 @@ export class AIServiceManager implements IAIServiceManager {
log.error(`Error initializing LLM tools during AIServiceManager construction: ${error.message || String(error)}`);
});
// Removed complex provider change listener - we'll read options fresh each time
// Start periodic cleanup of stale services
this.startCleanupTimer();
this.initialized = true;
}
@@ -372,34 +389,103 @@ export class AIServiceManager implements IAIServiceManager {
}
/**
* Clear the current provider (forces recreation on next access)
* Start the cleanup timer for removing stale services
*/
public clearCurrentProvider(): void {
this.currentService = null;
this.currentProvider = null;
log.info('Cleared current provider - will be recreated on next access');
private startCleanupTimer(): void {
if (this.cleanupTimer) return;
this.cleanupTimer = setInterval(() => {
this.cleanupStaleServices();
}, this.CLEANUP_INTERVAL_MS);
}
/**
* Get or create the current provider instance - only one instance total
* Stop the cleanup timer
*/
private stopCleanupTimer(): void {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = null;
}
}
/**
* Cleanup stale services that haven't been used recently
*/
private cleanupStaleServices(): void {
if (this.disposed) return;
const now = Date.now();
const staleProviders: ServiceProviders[] = [];
for (const [provider, entry] of this.serviceCache.entries()) {
if (now - entry.lastUsed > this.SERVICE_TTL_MS) {
staleProviders.push(provider);
}
}
for (const provider of staleProviders) {
this.disposeService(provider);
}
if (staleProviders.length > 0) {
log.info(`Cleaned up ${staleProviders.length} stale service(s): ${staleProviders.join(', ')}`);
}
}
/**
* Dispose a specific service
*/
private disposeService(provider: ServiceProviders): void {
const entry = this.serviceCache.get(provider);
if (entry) {
// If the service implements disposable, call dispose
if ('dispose' in entry.service && typeof (entry.service as any).dispose === 'function') {
try {
(entry.service as any).dispose();
} catch (error) {
log.error(`Error disposing ${provider} service: ${error}`);
}
}
this.serviceCache.delete(provider);
log.info(`Disposed ${provider} service`);
}
}
/**
* Clear all cached providers (forces recreation on next access)
*/
public clearCurrentProvider(): void {
// Clear all cached services
for (const provider of this.serviceCache.keys()) {
this.disposeService(provider);
}
log.info('Cleared all cached providers - will be recreated on next access');
}
/**
* Get or create a provider instance with proper caching and TTL
*/
private async getOrCreateChatProvider(providerName: ServiceProviders): Promise<AIService | null> {
// If provider type changed, clear the old one
if (this.currentProvider && this.currentProvider !== providerName) {
log.info(`Provider changed from ${this.currentProvider} to ${providerName}, clearing old service`);
this.currentService = null;
this.currentProvider = null;
if (this.disposed) {
throw new Error('AIServiceManager has been disposed');
}
// Return existing service if it matches and is available
if (this.currentService && this.currentProvider === providerName && this.currentService.isAvailable()) {
return this.currentService;
}
// Clear invalid service
if (this.currentService) {
this.currentService = null;
this.currentProvider = null;
// Check cache first
const cached = this.serviceCache.get(providerName);
if (cached && cached.service.isAvailable()) {
// Update last used time
cached.lastUsed = Date.now();
// Check if service is still within TTL
if (Date.now() - cached.createdAt <= this.SERVICE_TTL_MS) {
log.info(`Using cached ${providerName} service (age: ${Math.round((Date.now() - cached.createdAt) / 1000)}s)`);
return cached.service;
} else {
// Service is stale, dispose and recreate
log.info(`Cached ${providerName} service is stale, recreating`);
this.disposeService(providerName);
}
}
// Create new service for the requested provider
@@ -443,9 +529,14 @@ export class AIServiceManager implements IAIServiceManager {
}
if (service) {
// Cache the new service
this.currentService = service;
this.currentProvider = providerName;
// Cache the new service with metadata
const now = Date.now();
this.serviceCache.set(providerName, {
service,
provider: providerName,
createdAt: now,
lastUsed: now
});
log.info(`Created and cached new ${providerName} service`);
return service;
}
@@ -456,6 +547,26 @@ export class AIServiceManager implements IAIServiceManager {
return null;
}
/**
* Dispose of all resources and cleanup
*/
async dispose(): Promise<void> {
if (this.disposed) return;
log.info('Disposing AIServiceManager...');
this.disposed = true;
// Stop cleanup timer
this.stopCleanupTimer();
// Dispose all cached services
for (const provider of this.serviceCache.keys()) {
this.disposeService(provider);
}
log.info('AIServiceManager disposed successfully');
}
/**
* Initialize the AI Service using the new configuration system
*/
@@ -706,21 +817,40 @@ export class AIServiceManager implements IAIServiceManager {
}
// Don't create singleton immediately, use a lazy-loading pattern
// Singleton instance (lazy-loaded) - can be disposed and recreated
let instance: AIServiceManager | null = null;
/**
* Get the AIServiceManager instance (creates it if not already created)
* Get the AIServiceManager instance (creates it if not already created or disposed)
*/
function getInstance(): AIServiceManager {
if (!instance) {
if (!instance || (instance as any).disposed) {
instance = new AIServiceManager();
}
return instance;
}
/**
* Create a new AIServiceManager instance (for testing or isolated contexts)
*/
function createNewInstance(): AIServiceManager {
return new AIServiceManager();
}
/**
* Dispose the current singleton instance
*/
async function disposeInstance(): Promise<void> {
if (instance) {
await instance.dispose();
instance = null;
}
}
export default {
getInstance,
createNewInstance,
disposeInstance,
// Also export methods directly for convenience
isAnyServiceAvailable(): boolean {
return getInstance().isAnyServiceAvailable();

47
apps/server/src/services/llm/base_ai_service.ts
View File

@@ -1,9 +1,18 @@
import options from '../options.js';
import type { AIService, ChatCompletionOptions, ChatResponse, Message } from './ai_interface.js';
import { DEFAULT_SYSTEM_PROMPT } from './constants/llm_prompt_constants.js';
import log from '../log.js';
export abstract class BaseAIService implements AIService {
/**
* Disposable interface for proper resource cleanup
*/
export interface Disposable {
dispose(): void | Promise<void>;
}
export abstract class BaseAIService implements AIService, Disposable {
protected name: string;
protected disposed: boolean = false;
constructor(name: string) {
this.name = name;
@@ -12,6 +21,9 @@ export abstract class BaseAIService implements AIService {
abstract generateChatCompletion(messages: Message[], options?: ChatCompletionOptions): Promise<ChatResponse>;
isAvailable(): boolean {
if (this.disposed) {
return false;
}
return options.getOptionBool('aiEnabled'); // Base check if AI is enabled globally
}
@@ -23,4 +35,37 @@ export abstract class BaseAIService implements AIService {
// Use prompt from constants file if no custom prompt is provided
return customPrompt || DEFAULT_SYSTEM_PROMPT;
}
/**
* Dispose of any resources held by this service
* Override in subclasses to clean up specific resources
*/
async dispose(): Promise<void> {
if (this.disposed) {
return;
}
log.info(`Disposing ${this.name} service`);
this.disposed = true;
// Subclasses should override this to clean up their specific resources
await this.disposeResources();
}
/**
* Template method for subclasses to implement resource cleanup
*/
protected async disposeResources(): Promise<void> {
// Default implementation does nothing
// Subclasses should override to clean up their resources
}
/**
* Check if the service has been disposed
*/
protected checkDisposed(): void {
if (this.disposed) {
throw new Error(`${this.name} service has been disposed and cannot be used`);
}
}
}

161
apps/server/src/services/llm/providers/anthropic_service.ts
View File

@@ -7,7 +7,8 @@ import { getAnthropicOptions } from './providers.js';
import log from '../../log.js';
import Anthropic from '@anthropic-ai/sdk';
import { SEARCH_CONSTANTS } from '../constants/search_constants.js';
import type { ToolCall } from '../tools/tool_interfaces.js';
import type { ToolCall, Tool } from '../tools/tool_interfaces.js';
import { ToolFormatAdapter } from '../tools/tool_format_adapter.js';
interface AnthropicMessage extends Omit<Message, "content"> {
content: MessageContent[] | string;
@@ -34,6 +35,17 @@ export class AnthropicService extends BaseAIService {
return super.isAvailable() && !!options.getOption('anthropicApiKey');
}
/**
* Clean up resources when disposing
*/
protected async disposeResources(): Promise<void> {
if (this.client) {
// Clear the client reference
this.client = null;
log.info('Anthropic client disposed');
}
}
private getClient(apiKey: string, baseUrl: string, apiVersion?: string, betaVersion?: string): any {
if (!this.client) {
this.client = new Anthropic({
@@ -49,6 +61,9 @@ export class AnthropicService extends BaseAIService {
}
async generateChatCompletion(messages: Message[], opts: ChatCompletionOptions = {}): Promise<ChatResponse> {
// Check if service has been disposed
this.checkDisposed();
if (!this.isAvailable()) {
throw new Error('Anthropic service is not available. Check API key and AI settings.');
}
@@ -104,15 +119,18 @@ export class AnthropicService extends BaseAIService {
if (opts.tools && opts.tools.length > 0) {
log.info(`========== ANTHROPIC TOOL PROCESSING ==========`);
log.info(`Input tools count: ${opts.tools.length}`);
log.info(`Input tool names: ${opts.tools.map(t => t.function?.name || 'unnamed').join(', ')}`);
log.info(`Input tool names: ${opts.tools.map((t: any) => t.function?.name || 'unnamed').join(', ')}`);
// Convert OpenAI-style function tools to Anthropic format
const anthropicTools = this.convertToolsToAnthropicFormat(opts.tools);
// Use the new ToolFormatAdapter for consistent conversion
const anthropicTools = ToolFormatAdapter.convertToProviderFormat(
opts.tools as Tool[],
'anthropic'
);
if (anthropicTools.length > 0) {
requestParams.tools = anthropicTools;
log.info(`Successfully added ${anthropicTools.length} tools to Anthropic request`);
log.info(`Final tool names: ${anthropicTools.map(t => t.name).join(', ')}`);
log.info(`Final tool names: ${anthropicTools.map((t: any) => t.name).join(', ')}`);
} else {
log.error(`CRITICAL: Tool conversion failed - 0 tools converted from ${opts.tools.length} input tools`);
}
@@ -164,22 +182,11 @@ export class AnthropicService extends BaseAIService {
if (toolBlocks.length > 0) {
log.info(`[DEBUG] Found ${toolBlocks.length} tool-related blocks in response`);
toolCalls = toolBlocks.map((block: any) => {
if (block.type === 'tool_use') {
log.info(`[DEBUG] Processing tool_use block: ${JSON.stringify(block, null, 2)}`);
// Convert Anthropic tool_use format to standard format expected by our app
return {
id: block.id,
type: 'function', // Convert back to function type for internal use
function: {
name: block.name,
arguments: JSON.stringify(block.input || {})
}
};
}
return null;
}).filter(Boolean);
// Use ToolFormatAdapter to convert from Anthropic format
toolCalls = ToolFormatAdapter.convertToolCallsFromProvider(
toolBlocks,
'anthropic'
);
log.info(`Extracted ${toolCalls?.length} tool calls from Anthropic response`);
}
@@ -324,21 +331,12 @@ export class AnthropicService extends BaseAIService {
block => block.type === 'tool_use'
);
// Convert tool use blocks to our expected format
// Use ToolFormatAdapter to convert tool calls
if (toolUseBlocks.length > 0) {
toolCalls = toolUseBlocks.map(block => {
if (block.type === 'tool_use') {
return {
id: block.id,
type: 'function',
function: {
name: block.name,
arguments: JSON.stringify(block.input || {})
}
};
}
return null;
}).filter(Boolean);
toolCalls = ToolFormatAdapter.convertToolCallsFromProvider(
toolUseBlocks,
'anthropic'
);
// For any active tool calls, mark them as complete
for (const [toolId, toolCall] of activeToolCalls.entries()) {
@@ -525,96 +523,9 @@ export class AnthropicService extends BaseAIService {
return anthropicMessages;
}
/**
* Convert OpenAI-style function tools to Anthropic format
* OpenAI uses: { type: "function", function: { name, description, parameters } }
* Anthropic uses: { name, description, input_schema }
*/
private convertToolsToAnthropicFormat(tools: any[]): any[] {
if (!tools || tools.length === 0) {
return [];
}
log.info(`[TOOL DEBUG] Converting ${tools.length} tools to Anthropic format`);
// Filter out invalid tools
const validTools = tools.filter(tool => {
if (!tool || typeof tool !== 'object') {
log.error(`Invalid tool format (not an object)`);
return false;
}
// For function tools, validate required fields
if (tool.type === 'function') {
if (!tool.function || !tool.function.name) {
log.error(`Function tool missing required fields`);
return false;
}
}
return true;
});
if (validTools.length < tools.length) {
log.info(`Filtered out ${tools.length - validTools.length} invalid tools`);
}
// Convert tools to Anthropic format
const convertedTools = validTools.map((tool: any) => {
// Convert from OpenAI format to Anthropic format
if (tool.type === 'function' && tool.function) {
log.info(`[TOOL DEBUG] Converting function tool: ${tool.function.name}`);
// Check the parameters structure
if (tool.function.parameters) {
log.info(`[TOOL DEBUG] Parameters for ${tool.function.name}:`);
log.info(`[TOOL DEBUG] - Type: ${tool.function.parameters.type}`);
log.info(`[TOOL DEBUG] - Properties: ${JSON.stringify(tool.function.parameters.properties || {})}`);
log.info(`[TOOL DEBUG] - Required: ${JSON.stringify(tool.function.parameters.required || [])}`);
// Check if the required array is present and properly populated
if (!tool.function.parameters.required || !Array.isArray(tool.function.parameters.required)) {
log.error(`[TOOL DEBUG] WARNING: Tool ${tool.function.name} missing required array in parameters`);
} else if (tool.function.parameters.required.length === 0) {
log.error(`[TOOL DEBUG] WARNING: Tool ${tool.function.name} has empty required array - Anthropic may send empty inputs`);
}
} else {
log.error(`[TOOL DEBUG] WARNING: Tool ${tool.function.name} has no parameters defined`);
}
return {
name: tool.function.name,
description: tool.function.description || '',
input_schema: tool.function.parameters || {}
};
}
// Handle already converted Anthropic format (from our temporary fix)
if (tool.type === 'custom' && tool.custom) {
log.info(`[TOOL DEBUG] Converting custom tool: ${tool.custom.name}`);
return {
name: tool.custom.name,
description: tool.custom.description || '',
input_schema: tool.custom.parameters || {}
};
}
// If the tool is already in the correct Anthropic format
if (tool.name && (tool.input_schema || tool.parameters)) {
log.info(`[TOOL DEBUG] Tool already in Anthropic format: ${tool.name}`);
return {
name: tool.name,
description: tool.description || '',
input_schema: tool.input_schema || tool.parameters
};
}
log.error(`Unhandled tool format encountered`);
return null;
}).filter(Boolean); // Filter out any null values
return convertedTools;
}
// Tool conversion is now handled by ToolFormatAdapter
// The old convertToolsToAnthropicFormat method has been removed in favor of the centralized adapter
// This ensures consistent tool format conversion across all providers
/**
* Clear cached Anthropic client to force recreation with new settings

341
apps/server/src/services/llm/tools/tool_format_adapter.ts Normal file
View File

@@ -0,0 +1,341 @@
/**
* Tool Format Adapter
*
* This module provides standardized conversion between different LLM provider tool formats.
* It ensures consistent tool handling across OpenAI, Anthropic, Ollama, and other providers.
*/
import log from '../../log.js';
import type { Tool, ToolCall, ToolParameter } from './tool_interfaces.js';
/**
* Anthropic tool format
*/
export interface AnthropicTool {
name: string;
description: string;
input_schema: {
type: 'object';
properties: Record<string, unknown>;
required?: string[];
};
}
/**
* OpenAI tool format (already matches our standard Tool interface)
*/
export type OpenAITool = Tool;
/**
* Ollama tool format
*/
export interface OllamaTool {
type: 'function';
function: {
name: string;
description: string;
parameters: Record<string, unknown>;
};
}
/**
* Provider types
*/
export type ProviderType = 'openai' | 'anthropic' | 'ollama' | 'unknown';
/**
* Tool format adapter for converting between different provider formats
*/
export class ToolFormatAdapter {
/**
* Convert tools from standard format to provider-specific format
*/
static convertToProviderFormat(tools: Tool[], provider: ProviderType): unknown[] {
switch (provider) {
case 'anthropic':
return this.convertToAnthropicFormat(tools);
case 'ollama':
return this.convertToOllamaFormat(tools);
case 'openai':
// OpenAI format matches our standard format
return tools;
default:
log.warn(`Unknown provider ${provider}, returning tools in standard format`);
return tools;
}
}
/**
* Convert tools to Anthropic format
*/
static convertToAnthropicFormat(tools: Tool[]): AnthropicTool[] {
const converted: AnthropicTool[] = [];
for (const tool of tools) {
if (!this.validateTool(tool)) {
log.error(`Invalid tool skipped: ${JSON.stringify(tool)}`);
continue;
}
try {
const anthropicTool: AnthropicTool = {
name: tool.function.name,
description: tool.function.description || '',
input_schema: {
type: 'object',
properties: tool.function.parameters.properties || {},
required: tool.function.parameters.required || []
}
};
// Validate the converted tool
if (this.validateAnthropicTool(anthropicTool)) {
converted.push(anthropicTool);
log.info(`Successfully converted tool ${tool.function.name} to Anthropic format`);
} else {
log.error(`Failed to validate converted Anthropic tool: ${tool.function.name}`);
}
} catch (error) {
log.error(`Error converting tool ${tool.function.name} to Anthropic format: ${error}`);
}
}
return converted;
}
/**
* Convert tools to Ollama format
*/
static convertToOllamaFormat(tools: Tool[]): OllamaTool[] {
const converted: OllamaTool[] = [];
for (const tool of tools) {
if (!this.validateTool(tool)) {
log.error(`Invalid tool skipped: ${JSON.stringify(tool)}`);
continue;
}
try {
const ollamaTool: OllamaTool = {
type: 'function',
function: {
name: tool.function.name,
description: tool.function.description || '',
parameters: tool.function.parameters || {}
}
};
converted.push(ollamaTool);
log.info(`Successfully converted tool ${tool.function.name} to Ollama format`);
} catch (error) {
log.error(`Error converting tool ${tool.function.name} to Ollama format: ${error}`);
}
}
return converted;
}
/**
* Convert tool calls from provider format to standard format
*/
static convertToolCallsFromProvider(toolCalls: unknown[], provider: ProviderType): ToolCall[] {
switch (provider) {
case 'anthropic':
return this.convertAnthropicToolCalls(toolCalls);
case 'ollama':
return this.convertOllamaToolCalls(toolCalls);
case 'openai':
// OpenAI format matches our standard format
return toolCalls as ToolCall[];
default:
log.warn(`Unknown provider ${provider}, attempting standard conversion`);
return toolCalls as ToolCall[];
}
}
/**
* Convert Anthropic tool calls to standard format
*/
private static convertAnthropicToolCalls(toolCalls: unknown[]): ToolCall[] {
const converted: ToolCall[] = [];
for (const call of toolCalls) {
if (typeof call === 'object' && call !== null) {
const anthropicCall = call as any;
// Handle tool_use blocks from Anthropic
if (anthropicCall.type === 'tool_use') {
converted.push({
id: anthropicCall.id,
type: 'function',
function: {
name: anthropicCall.name,
arguments: typeof anthropicCall.input === 'string'
? anthropicCall.input
: JSON.stringify(anthropicCall.input || {})
}
});
}
// Handle already converted format
else if (anthropicCall.function) {
converted.push(anthropicCall as ToolCall);
}
}
}
return converted;
}
/**
* Convert Ollama tool calls to standard format
*/
private static convertOllamaToolCalls(toolCalls: unknown[]): ToolCall[] {
// Ollama typically uses a format similar to OpenAI
return toolCalls as ToolCall[];
}
/**
* Validate a standard tool definition
*/
static validateTool(tool: unknown): tool is Tool {
if (!tool || typeof tool !== 'object') {
return false;
}
const t = tool as any;
// Check required fields
if (t.type !== 'function') {
log.error(`Tool validation failed: type must be 'function', got '${t.type}'`);
return false;
}
if (!t.function || typeof t.function !== 'object') {
log.error('Tool validation failed: missing or invalid function object');
return false;
}
if (!t.function.name || typeof t.function.name !== 'string') {
log.error('Tool validation failed: missing or invalid function name');
return false;
}
if (!t.function.parameters || typeof t.function.parameters !== 'object') {
log.error(`Tool validation failed for ${t.function.name}: missing or invalid parameters`);
return false;
}
if (t.function.parameters.type !== 'object') {
log.error(`Tool validation failed for ${t.function.name}: parameters.type must be 'object'`);
return false;
}
// Validate required array if present
if (t.function.parameters.required && !Array.isArray(t.function.parameters.required)) {
log.error(`Tool validation failed for ${t.function.name}: parameters.required must be an array`);
return false;
}
return true;
}
/**
* Validate an Anthropic tool definition
*/
private static validateAnthropicTool(tool: AnthropicTool): boolean {
if (!tool.name || typeof tool.name !== 'string') {
log.error('Anthropic tool validation failed: missing or invalid name');
return false;
}
if (!tool.input_schema || typeof tool.input_schema !== 'object') {
log.error(`Anthropic tool validation failed for ${tool.name}: missing or invalid input_schema`);
return false;
}
if (tool.input_schema.type !== 'object') {
log.error(`Anthropic tool validation failed for ${tool.name}: input_schema.type must be 'object'`);
return false;
}
if (!tool.input_schema.properties || typeof tool.input_schema.properties !== 'object') {
log.error(`Anthropic tool validation failed for ${tool.name}: missing or invalid properties`);
return false;
}
// Warn if required array is missing or empty (Anthropic may send empty inputs)
if (!tool.input_schema.required || tool.input_schema.required.length === 0) {
log.warn(`Anthropic tool ${tool.name} has no required parameters - may receive empty inputs`);
}
return true;
}
/**
* Create a standardized error response for tool execution failures
*/
static createToolErrorResponse(toolName: string, error: unknown): string {
const errorMessage = error instanceof Error ? error.message : String(error);
return JSON.stringify({
error: true,
tool: toolName,
message: `Tool execution failed: ${errorMessage}`,
timestamp: new Date().toISOString()
});
}
/**
* Create a standardized success response for tool execution
*/
static createToolSuccessResponse(toolName: string, result: unknown): string {
if (typeof result === 'string') {
return result;
}
return JSON.stringify({
success: true,
tool: toolName,
result: result,
timestamp: new Date().toISOString()
});
}
/**
* Parse tool arguments safely
*/
static parseToolArguments(args: string | Record<string, unknown>): Record<string, unknown> {
if (typeof args === 'string') {
try {
return JSON.parse(args);
} catch (error) {
log.error(`Failed to parse tool arguments as JSON: ${error}`);
return {};
}
}
return args || {};
}
/**
* Detect provider type from tool format
*/
static detectProviderFromToolFormat(tool: unknown): ProviderType {
if (!tool || typeof tool !== 'object') {
return 'unknown';
}
const t = tool as any;
// Check for Anthropic format
if (t.name && t.input_schema) {
return 'anthropic';
}
// Check for OpenAI/standard format
if (t.type === 'function' && t.function) {
return 'openai';
}
return 'unknown';
}
}
export default ToolFormatAdapter;