Compare commits

...

9 Commits

Author SHA1 Message Date
Elian Doran
f6bc65471d fix(fs_sync): sync error on froca 2025-07-26 20:35:20 +03:00
Elian Doran
eb07d4b0ed fix(fs_sync): unique constraint failed 2025-07-26 20:33:54 +03:00
Elian Doran
2c096f3080 fix(fs_sync): new files from server not synced 2025-07-26 19:30:41 +03:00
Elian Doran
bac95c97e5 fix(fs_sync): missing autocomplete 2025-07-26 19:13:40 +03:00
Elian Doran
fe6daac979 fix(fs_sync): modal not showing 2025-07-26 19:05:52 +03:00
Elian Doran
770281214b fix(fs_sync): option not readable/writable by client 2025-07-26 18:56:48 +03:00
Elian Doran
15bd5aa4e4 fix(fs_sync): cls errors in router 2025-07-26 18:40:22 +03:00
Elian Doran
3da6838395 fix(fs_sync): modal shown immediately when entering advanced 2025-07-26 18:31:34 +03:00
Elian Doran
16cdd9e137 feat(fs_sync): draft implementation 2025-07-26 18:31:16 +03:00
23 changed files with 3798 additions and 5 deletions

View File

@@ -4,7 +4,7 @@ applyTo: '**'
// This file is automatically generated by Nx Console
You are in an nx workspace using Nx 21.3.5 and pnpm as the package manager.
You are in an nx workspace using Nx 21.3.7 and pnpm as the package manager.
You have access to the Nx MCP server and the tools it provides. Use them. Follow these guidelines in order to best help the user:

View File

@@ -35,7 +35,7 @@ async function processEntityChanges(entityChanges: EntityChange[]) {
loadResults.addOption(attributeEntity.name);
} else if (ec.entityName === "attachments") {
processAttachment(loadResults, ec);
} else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens") {
} else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "file_note_mappings" || ec.entityName === "file_system_mappings") {
// NOOP - these entities are handled at the backend level and don't require frontend processing
} else {
throw new Error(`Unknown entityName '${ec.entityName}'`);

View File

@@ -2201,3 +2201,189 @@ footer.file-footer button {
content: "\ec24";
transform: rotate(180deg);
}
/* File System Sync Modal Styles */
.mapping-modal {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
z-index: 1050;
display: flex;
align-items: center;
justify-content: center;
}
.mapping-modal .modal-backdrop {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.5);
z-index: 1051;
}
.mapping-modal .modal-content {
position: relative;
background: var(--main-background-color);
border: 1px solid var(--main-border-color);
border-radius: 5px;
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
width: 90%;
max-width: 600px;
max-height: 80vh;
overflow-y: auto;
z-index: 1052;
}
.mapping-modal .modal-header {
display: flex;
justify-content: between;
align-items: center;
padding: 1rem;
border-bottom: 1px solid var(--main-border-color);
}
.mapping-modal .modal-title {
margin: 0;
font-size: 1.25rem;
flex: 1;
}
.mapping-modal .modal-close {
background: none;
border: none;
font-size: 1.5rem;
cursor: pointer;
color: var(--muted-text-color);
padding: 0;
width: 30px;
height: 30px;
display: flex;
align-items: center;
justify-content: center;
}
.mapping-modal .modal-close:hover {
color: var(--main-text-color);
}
.mapping-modal .modal-body {
padding: 1rem;
}
.mapping-modal .modal-footer {
padding: 1rem;
border-top: 1px solid var(--main-border-color);
display: flex;
justify-content: flex-end;
gap: 0.5rem;
}
/* File System Sync Mapping Cards */
.mapping-item.card {
border: 1px solid var(--main-border-color);
border-radius: 5px;
transition: box-shadow 0.2s ease;
}
.mapping-item.card:hover {
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
}
.mapping-item .mapping-path {
font-family: monospace;
font-size: 0.9rem;
word-break: break-all;
}
.mapping-item .mapping-details {
font-size: 0.85rem;
margin-top: 0.25rem;
}
.mapping-item .mapping-status {
margin-top: 0.5rem;
display: flex;
align-items: center;
gap: 0.5rem;
}
.mapping-item .mapping-actions {
display: flex;
gap: 0.25rem;
}
.mapping-item .mapping-actions .btn {
padding: 0.25rem 0.5rem;
}
/* Status Badges */
.status-badge.badge {
font-size: 0.75rem;
padding: 0.25rem 0.5rem;
border-radius: 3px;
}
.status-badge.badge-success {
background-color: #28a745;
color: white;
}
.status-badge.badge-danger {
background-color: #dc3545;
color: white;
}
.status-badge.badge-secondary {
background-color: #6c757d;
color: white;
}
/* Path Validation Styles */
.path-validation-result {
margin-top: 0.5rem;
font-size: 0.875rem;
}
.path-validation-result .text-success {
color: #28a745;
}
.path-validation-result .text-warning {
color: #ffc107;
}
.path-validation-result .text-danger {
color: #dc3545;
}
/* Sync Status Section */
.sync-status-container {
margin: 1rem 0;
padding: 1rem;
background: var(--accented-background-color);
border-radius: 5px;
}
.sync-status-info .status-item,
.sync-status-info .active-mappings-count {
margin-bottom: 0.5rem;
}
/* Form Enhancements */
.mapping-form .form-group {
margin-bottom: 1rem;
}
.mapping-form .subtree-options {
margin-left: 1.5rem;
}
.mapping-form .help-block {
font-size: 0.875rem;
color: var(--muted-text-color);
margin-top: 0.25rem;
}

View File

@@ -27,6 +27,7 @@ import RevisionSnapshotsLimitOptions from "./options/other/revision_snapshots_li
import NetworkConnectionsOptions from "./options/other/network_connections.js";
import HtmlImportTagsOptions from "./options/other/html_import_tags.js";
import AdvancedSyncOptions from "./options/advanced/sync.js";
import FileSystemSyncOptions from "./options/advanced/file_system_sync.js";
import DatabaseIntegrityCheckOptions from "./options/advanced/database_integrity_check.js";
import VacuumDatabaseOptions from "./options/advanced/vacuum_database.js";
import DatabaseAnonymizationOptions from "./options/advanced/database_anonymization.js";
@@ -138,6 +139,7 @@ const CONTENT_WIDGETS: Record<OptionPages | "_backendLog", (typeof NoteContextAw
],
_optionsAdvanced: [
AdvancedSyncOptions,
FileSystemSyncOptions,
DatabaseIntegrityCheckOptions,
DatabaseAnonymizationOptions,
VacuumDatabaseOptions

View File

@@ -0,0 +1,659 @@
import OptionsWidget from "../options_widget.js";
import server from "../../../../services/server.js";
import toastService from "../../../../services/toast.js";
import noteAutocompleteService from "../../../../services/note_autocomplete.js";
import type { OptionMap } from "@triliumnext/commons";
import type { Suggestion } from "../../../../services/note_autocomplete.js";
interface FileSystemMapping {
mappingId: string;
noteId: string;
filePath: string;
syncDirection: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
isActive: boolean;
includeSubtree: boolean;
preserveHierarchy: boolean;
contentFormat: 'auto' | 'markdown' | 'html' | 'raw';
excludePatterns: string[] | null;
lastSyncTime: string | null;
syncErrors: string[] | null;
dateCreated: string;
dateModified: string;
}
interface SyncStatus {
enabled: boolean;
initialized: boolean;
status?: Record<string, any>;
}
// API Request/Response interfaces
interface PathValidationRequest {
filePath: string;
}
interface PathValidationResponse {
exists: boolean;
stats?: {
isDirectory: boolean;
size: number;
modified: string;
};
}
interface CreateMappingRequest {
noteId: string;
filePath: string;
syncDirection: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
contentFormat: 'auto' | 'markdown' | 'html' | 'raw';
includeSubtree: boolean;
preserveHierarchy: boolean;
excludePatterns: string[] | null;
}
interface UpdateMappingRequest extends CreateMappingRequest {}
interface SyncMappingResponse {
success: boolean;
message?: string;
}
interface ApiResponse {
success?: boolean;
message?: string;
}
const TPL = /*html*/`
<style>
.modal-hidden {
display: none !important;
}
.modal-visible {
display: flex !important;
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.5);
z-index: 1050;
align-items: center;
justify-content: center;
}
.modal-content {
background: white;
border-radius: 0.5rem;
max-width: 600px;
width: 90%;
max-height: 90%;
overflow-y: auto;
}
</style>
<div class="options-section">
<h4>File System Sync</h4>
<div class="form-group">
<label>
<input type="checkbox" class="file-sync-enabled-checkbox">
Enable file system synchronization
</label>
<div class="help-block">
Allows bidirectional synchronization between Trilium notes and files on your local file system.
</div>
</div>
<div class="file-sync-controls" style="display: none;">
<div class="alert alert-info">
<strong>Note:</strong> File system sync creates mappings between notes and files/directories.
Changes in either location will be synchronized automatically when enabled.
</div>
<div class="sync-status-container">
<h5>Sync Status</h5>
<div class="sync-status-info">
<div class="status-item">
<strong>Status:</strong> <span class="sync-status-text">Loading...</span>
</div>
<div class="active-mappings-count">
<strong>Active Mappings:</strong> <span class="mappings-count">0</span>
</div>
</div>
</div>
<div class="mappings-section">
<div class="d-flex justify-content-between align-items-center mb-3">
<h5>File System Mappings</h5>
<button class="btn btn-primary btn-sm create-mapping-button">
<i class="bx bx-plus"></i> Create Mapping
</button>
</div>
<div class="mappings-list">
<!-- Mappings will be populated here -->
</div>
</div>
<div class="sync-actions mt-3">
<button class="btn btn-secondary refresh-status-button">
<i class="bx bx-refresh"></i> Refresh Status
</button>
</div>
</div>
</div>
<!-- Create/Edit Mapping Modal -->
<div class="mapping-modal modal-hidden">
<div class="modal-backdrop"></div>
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">Create File System Mapping</h5>
<button type="button" class="modal-close" aria-label="Close">
<i class="bx bx-x"></i>
</button>
</div>
<div class="modal-body">
<form class="mapping-form">
<div class="form-group">
<label for="note-selector">Note:</label>
<div class="input-group">
<input type="text" id="note-selector" class="form-control note-selector"
placeholder="Search for a note...">
</div>
<div class="help-block">Select the note to map to the file system.</div>
</div>
<div class="form-group">
<label for="file-path">File/Directory Path:</label>
<div class="input-group">
<input type="text" id="file-path" class="form-control file-path-input"
placeholder="/path/to/file/or/directory">
<div class="input-group-append">
<button type="button" class="btn btn-secondary validate-path-button">
<i class="bx bx-search"></i> Validate
</button>
</div>
</div>
<div class="path-validation-result"></div>
</div>
<div class="form-group">
<label for="sync-direction">Sync Direction:</label>
<select id="sync-direction" class="form-control sync-direction-select">
<option value="bidirectional">Bidirectional (default)</option>
<option value="trilium_to_disk">Trilium → Disk only</option>
<option value="disk_to_trilium">Disk → Trilium only</option>
</select>
</div>
<div class="form-group">
<label for="content-format">Content Format:</label>
<select id="content-format" class="form-control content-format-select">
<option value="auto">Auto-detect (default)</option>
<option value="markdown">Markdown</option>
<option value="html">HTML</option>
<option value="raw">Raw/Binary</option>
</select>
</div>
<div class="form-group">
<label>
<input type="checkbox" class="include-subtree-checkbox">
Include subtree
</label>
<div class="help-block">Map entire note subtree to directory structure.</div>
</div>
<div class="form-group subtree-options" style="display: none;">
<label>
<input type="checkbox" class="preserve-hierarchy-checkbox" checked>
Preserve directory hierarchy
</label>
<div class="help-block">Create subdirectories matching note hierarchy.</div>
</div>
<div class="form-group">
<label for="exclude-patterns">Exclude Patterns (one per line):</label>
<textarea id="exclude-patterns" class="form-control exclude-patterns-textarea"
rows="3" placeholder="*.tmp&#10;node_modules&#10;.git"></textarea>
<div class="help-block">Files/directories matching these patterns will be ignored.</div>
</div>
</form>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary cancel-mapping-button">Cancel</button>
<button type="button" class="btn btn-primary save-mapping-button">Save Mapping</button>
</div>
</div>
</div>`;
const MAPPING_ITEM_TPL = /*html*/`
<div class="mapping-item card mb-2" data-mapping-id="">
<div class="card-body">
<div class="d-flex justify-content-between align-items-start">
<div class="mapping-info">
<div class="mapping-path">
<strong class="file-path"></strong>
</div>
<div class="mapping-details text-muted">
<span class="note-title"></span> •
<span class="sync-direction-text"></span> •
<span class="content-format-text"></span>
</div>
<div class="mapping-status">
<span class="status-badge"></span>
<span class="last-sync"></span>
</div>
</div>
<div class="mapping-actions">
<button class="btn btn-sm btn-secondary sync-mapping-button" title="Sync now">
<i class="bx bx-refresh"></i>
</button>
<button class="btn btn-sm btn-secondary edit-mapping-button" title="Edit">
<i class="bx bx-edit"></i>
</button>
<button class="btn btn-sm btn-danger delete-mapping-button" title="Delete">
<i class="bx bx-trash"></i>
</button>
</div>
</div>
<div class="sync-errors" style="display: none;">
<div class="alert alert-warning mt-2">
<strong>Sync Errors:</strong>
<ul class="error-list mb-0"></ul>
</div>
</div>
</div>
</div>`;
export default class FileSystemSyncOptions extends OptionsWidget {
private $fileSyncEnabledCheckbox!: JQuery<HTMLElement>;
private $fileSyncControls!: JQuery<HTMLElement>;
private $syncStatusText!: JQuery<HTMLElement>;
private $mappingsCount!: JQuery<HTMLElement>;
private $mappingsList!: JQuery<HTMLElement>;
private $createMappingButton!: JQuery<HTMLElement>;
private $refreshStatusButton!: JQuery<HTMLElement>;
// Modal elements
private $mappingModal!: JQuery<HTMLElement>;
private $modalTitle!: JQuery<HTMLElement>;
private $noteSelector!: JQuery<HTMLElement>;
private $filePathInput!: JQuery<HTMLElement>;
private $validatePathButton!: JQuery<HTMLElement>;
private $pathValidationResult!: JQuery<HTMLElement>;
private $syncDirectionSelect!: JQuery<HTMLElement>;
private $contentFormatSelect!: JQuery<HTMLElement>;
private $includeSubtreeCheckbox!: JQuery<HTMLElement>;
private $preserveHierarchyCheckbox!: JQuery<HTMLElement>;
private $subtreeOptions!: JQuery<HTMLElement>;
private $excludePatternsTextarea!: JQuery<HTMLElement>;
private $saveMappingButton!: JQuery<HTMLElement>;
private $cancelMappingButton!: JQuery<HTMLElement>;
private $modalClose!: JQuery<HTMLElement>;
private currentEditingMappingId: string | null = null;
private mappings: FileSystemMapping[] = [];
doRender() {
this.$widget = $(TPL);
this.initializeElements();
// Ensure modal is hidden on initialization
this.$mappingModal.addClass('modal-hidden').removeClass('modal-visible');
this.setupEventHandlers();
}
private initializeElements() {
this.$fileSyncEnabledCheckbox = this.$widget.find(".file-sync-enabled-checkbox");
this.$fileSyncControls = this.$widget.find(".file-sync-controls");
this.$syncStatusText = this.$widget.find(".sync-status-text");
this.$mappingsCount = this.$widget.find(".mappings-count");
this.$mappingsList = this.$widget.find(".mappings-list");
this.$createMappingButton = this.$widget.find(".create-mapping-button");
this.$refreshStatusButton = this.$widget.find(".refresh-status-button");
// Modal elements
this.$mappingModal = this.$widget.closest(".mapping-modal");
this.$modalTitle = this.$mappingModal.find(".modal-title");
this.$noteSelector = this.$mappingModal.find(".note-selector");
this.$filePathInput = this.$mappingModal.find(".file-path-input");
this.$validatePathButton = this.$mappingModal.find(".validate-path-button");
this.$pathValidationResult = this.$mappingModal.find(".path-validation-result");
this.$syncDirectionSelect = this.$mappingModal.find(".sync-direction-select");
this.$contentFormatSelect = this.$mappingModal.find(".content-format-select");
this.$includeSubtreeCheckbox = this.$mappingModal.find(".include-subtree-checkbox");
this.$preserveHierarchyCheckbox = this.$mappingModal.find(".preserve-hierarchy-checkbox");
this.$subtreeOptions = this.$mappingModal.find(".subtree-options");
this.$excludePatternsTextarea = this.$mappingModal.find(".exclude-patterns-textarea");
this.$saveMappingButton = this.$mappingModal.find(".save-mapping-button");
this.$cancelMappingButton = this.$mappingModal.find(".cancel-mapping-button");
this.$modalClose = this.$mappingModal.find(".modal-close");
}
private setupEventHandlers() {
this.$fileSyncEnabledCheckbox.on("change", async () => {
const isEnabled = this.$fileSyncEnabledCheckbox.prop("checked");
try {
if (isEnabled) {
await server.post<ApiResponse>("file-system-sync/enable");
} else {
await server.post<ApiResponse>("file-system-sync/disable");
}
this.toggleControls(isEnabled);
if (isEnabled) {
await this.refreshStatus();
}
toastService.showMessage(`File system sync ${isEnabled ? 'enabled' : 'disabled'}`);
} catch (error) {
toastService.showError(`Failed to ${isEnabled ? 'enable' : 'disable'} file system sync`);
// Revert checkbox state
this.$fileSyncEnabledCheckbox.prop("checked", !isEnabled);
}
});
this.$createMappingButton.on("click", () => {
this.showMappingModal();
});
this.$refreshStatusButton.on("click", () => {
this.refreshStatus();
});
this.$validatePathButton.on("click", () => {
this.validatePath();
});
this.$includeSubtreeCheckbox.on("change", () => {
const isChecked = this.$includeSubtreeCheckbox.prop("checked");
this.$subtreeOptions.toggle(isChecked);
});
// Modal handlers
this.$saveMappingButton.on("click", () => {
this.saveMapping();
});
this.$cancelMappingButton.on("click", () => {
this.hideMappingModal();
});
this.$modalClose.on("click", () => {
this.hideMappingModal();
});
this.$mappingModal.find(".modal-backdrop").on("click", () => {
this.hideMappingModal();
});
// Note selector autocomplete will be initialized in showMappingModal
}
private toggleControls(enabled: boolean) {
this.$fileSyncControls.toggle(enabled);
}
private async refreshStatus() {
try {
const status = await server.get<SyncStatus>("file-system-sync/status");
this.$syncStatusText.text(status.initialized ? "Active" : "Inactive");
if (status.initialized) {
await this.loadMappings();
}
} catch (error) {
this.$syncStatusText.text("Error");
toastService.showError("Failed to get sync status");
}
}
private async loadMappings() {
try {
this.mappings = await server.get<FileSystemMapping[]>("file-system-sync/mappings");
this.renderMappings();
this.$mappingsCount.text(this.mappings.length.toString());
} catch (error) {
toastService.showError("Failed to load mappings");
}
}
private renderMappings() {
this.$mappingsList.empty();
for (const mapping of this.mappings) {
const $item = $(MAPPING_ITEM_TPL);
$item.attr("data-mapping-id", mapping.mappingId);
$item.find(".file-path").text(mapping.filePath);
$item.find(".note-title").text(`Note: ${mapping.noteId}`); // TODO: Get actual note title
$item.find(".sync-direction-text").text(this.formatSyncDirection(mapping.syncDirection));
$item.find(".content-format-text").text(mapping.contentFormat);
// Status badge
const $statusBadge = $item.find(".status-badge");
if (mapping.syncErrors && mapping.syncErrors.length > 0) {
$statusBadge.addClass("badge badge-danger").text("Error");
const $errorsDiv = $item.find(".sync-errors");
const $errorList = $errorsDiv.find(".error-list");
mapping.syncErrors.forEach(error => {
$errorList.append(`<li>${error}</li>`);
});
$errorsDiv.show();
} else if (mapping.isActive) {
$statusBadge.addClass("badge badge-success").text("Active");
} else {
$statusBadge.addClass("badge badge-secondary").text("Inactive");
}
// Last sync time
if (mapping.lastSyncTime) {
const lastSync = new Date(mapping.lastSyncTime).toLocaleString();
$item.find(".last-sync").text(`Last sync: ${lastSync}`);
} else {
$item.find(".last-sync").text("Never synced");
}
// Action handlers
$item.find(".sync-mapping-button").on("click", () => {
this.syncMapping(mapping.mappingId);
});
$item.find(".edit-mapping-button").on("click", () => {
this.editMapping(mapping);
});
$item.find(".delete-mapping-button").on("click", () => {
this.deleteMapping(mapping.mappingId);
});
this.$mappingsList.append($item);
}
}
private formatSyncDirection(direction: string): string {
switch (direction) {
case 'bidirectional': return 'Bidirectional';
case 'trilium_to_disk': return 'Trilium → Disk';
case 'disk_to_trilium': return 'Disk → Trilium';
default: return direction;
}
}
private showMappingModal(mapping?: FileSystemMapping) {
this.currentEditingMappingId = mapping?.mappingId || null;
console.log("Showing mapping modal", this.currentEditingMappingId, this.$mappingModal);
if (mapping) {
this.$modalTitle.text("Edit File System Mapping");
this.populateMappingForm(mapping);
} else {
this.$modalTitle.text("Create File System Mapping");
this.clearMappingForm();
}
// Initialize note autocomplete
noteAutocompleteService.initNoteAutocomplete(this.$noteSelector, {
allowCreatingNotes: true,
});
// Handle note selection
this.$noteSelector.off("autocomplete:noteselected").on("autocomplete:noteselected", (event: JQuery.Event, suggestion: Suggestion) => {
// The note autocomplete service will automatically set the selected note path
// which we can retrieve using getSelectedNoteId()
});
this.$mappingModal.removeClass('modal-hidden').addClass('modal-visible');
}
private hideMappingModal() {
this.$mappingModal.removeClass('modal-visible').addClass('modal-hidden');
this.clearMappingForm();
this.currentEditingMappingId = null;
}
private async populateMappingForm(mapping: FileSystemMapping) {
// Set the note using the autocomplete service's setNote method
await this.$noteSelector.setNote(mapping.noteId);
this.$filePathInput.val(mapping.filePath);
this.$syncDirectionSelect.val(mapping.syncDirection);
this.$contentFormatSelect.val(mapping.contentFormat);
this.$includeSubtreeCheckbox.prop("checked", mapping.includeSubtree);
this.$preserveHierarchyCheckbox.prop("checked", mapping.preserveHierarchy);
this.$subtreeOptions.toggle(mapping.includeSubtree);
if (mapping.excludePatterns) {
this.$excludePatternsTextarea.val(mapping.excludePatterns.join('\n'));
}
}
private clearMappingForm() {
// Clear the note selector using autocomplete service
this.$noteSelector.val('').setSelectedNotePath('');
this.$filePathInput.val('');
this.$syncDirectionSelect.val('bidirectional');
this.$contentFormatSelect.val('auto');
this.$includeSubtreeCheckbox.prop("checked", false);
this.$preserveHierarchyCheckbox.prop("checked", true);
this.$subtreeOptions.hide();
this.$excludePatternsTextarea.val('');
this.$pathValidationResult.empty();
}
private async validatePath() {
const filePath = this.$filePathInput.val() as string;
if (!filePath) {
this.$pathValidationResult.html('<div class="text-danger">Please enter a file path</div>');
return;
}
try {
const result = await server.post<PathValidationResponse>("file-system-sync/validate-path", { filePath } as PathValidationRequest);
if (result.exists && result.stats) {
const type = result.stats.isDirectory ? 'directory' : 'file';
this.$pathValidationResult.html(
`<div class="text-success">✓ Valid ${type} (${result.stats.size} bytes, modified ${new Date(result.stats.modified).toLocaleString()})</div>`
);
} else {
this.$pathValidationResult.html('<div class="text-warning">⚠ Path does not exist</div>');
}
} catch (error) {
this.$pathValidationResult.html('<div class="text-danger">✗ Invalid path</div>');
}
}
private async saveMapping() {
const noteId = this.$noteSelector.getSelectedNoteId();
const filePath = this.$filePathInput.val() as string;
const syncDirection = this.$syncDirectionSelect.val() as string;
const contentFormat = this.$contentFormatSelect.val() as string;
const includeSubtree = this.$includeSubtreeCheckbox.prop("checked");
const preserveHierarchy = this.$preserveHierarchyCheckbox.prop("checked");
const excludePatternsText = this.$excludePatternsTextarea.val() as string;
// Validation
if (!noteId) {
toastService.showError("Please select a note");
return;
}
if (!filePath) {
toastService.showError("Please enter a file path");
return;
}
const excludePatterns = excludePatternsText.trim()
? excludePatternsText.split('\n').map(p => p.trim()).filter(p => p)
: null;
const mappingData: CreateMappingRequest = {
noteId,
filePath,
syncDirection: syncDirection as 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium',
contentFormat: contentFormat as 'auto' | 'markdown' | 'html' | 'raw',
includeSubtree,
preserveHierarchy,
excludePatterns
};
try {
if (this.currentEditingMappingId) {
await server.put<ApiResponse>(`file-system-sync/mappings/${this.currentEditingMappingId}`, mappingData as UpdateMappingRequest);
toastService.showMessage("Mapping updated successfully");
} else {
await server.post<ApiResponse>("file-system-sync/mappings", mappingData);
toastService.showMessage("Mapping created successfully");
}
this.hideMappingModal();
await this.loadMappings();
} catch (error) {
toastService.showError("Failed to save mapping");
}
}
private async syncMapping(mappingId: string) {
try {
const result = await server.post<SyncMappingResponse>(`file-system-sync/mappings/${mappingId}/sync`);
if (result.success) {
toastService.showMessage("Sync completed successfully");
} else {
toastService.showError(`Sync failed: ${result.message}`);
}
await this.loadMappings();
} catch (error) {
toastService.showError("Failed to trigger sync");
}
}
private editMapping(mapping: FileSystemMapping) {
this.showMappingModal(mapping);
}
private async deleteMapping(mappingId: string) {
if (!confirm("Are you sure you want to delete this mapping?")) {
return;
}
try {
await server.delete<ApiResponse>(`file-system-sync/mappings/${mappingId}`);
toastService.showMessage("Mapping deleted successfully");
await this.loadMappings();
} catch (error) {
toastService.showError("Failed to delete mapping");
}
}
async optionsLoaded(options: OptionMap) {
const isEnabled = options.fileSystemSyncEnabled === "true";
this.$fileSyncEnabledCheckbox.prop("checked", isEnabled);
this.toggleControls(isEnabled);
if (isEnabled) {
await this.refreshStatus();
}
}
}

View File

@@ -20,6 +20,7 @@ import log from "./services/log.js";
import "./services/handlers.js";
import "./becca/becca_loader.js";
import { RESOURCE_DIR } from "./services/resource_dir.js";
import fileSystemSyncInit from "./services/file_system_sync_init.js";
export default async function buildApp() {
const app = express();
@@ -32,6 +33,9 @@ export default async function buildApp() {
try {
log.info("Database initialized, LLM features available");
log.info("LLM features ready");
// Initialize file system sync after database is ready
await fileSystemSyncInit.init();
} catch (error) {
console.error("Error initializing LLM features:", error);
}
@@ -41,6 +45,9 @@ export default async function buildApp() {
if (sql_init.isDbInitialized()) {
try {
log.info("LLM features ready");
// Initialize file system sync if database is already ready
await fileSystemSyncInit.init();
} catch (error) {
console.error("Error initializing LLM features:", error);
}

View File

@@ -152,3 +152,56 @@ CREATE TABLE IF NOT EXISTS sessions (
data TEXT,
expires INTEGER
);
-- Table to store file system mappings for notes and subtrees
CREATE TABLE IF NOT EXISTS "file_system_mappings" (
"mappingId" TEXT NOT NULL PRIMARY KEY,
"noteId" TEXT NOT NULL,
"filePath" TEXT NOT NULL,
"syncDirection" TEXT NOT NULL DEFAULT 'bidirectional', -- 'bidirectional', 'trilium_to_disk', 'disk_to_trilium'
"isActive" INTEGER NOT NULL DEFAULT 1,
"includeSubtree" INTEGER NOT NULL DEFAULT 0,
"preserveHierarchy" INTEGER NOT NULL DEFAULT 1,
"contentFormat" TEXT NOT NULL DEFAULT 'auto', -- 'auto', 'markdown', 'html', 'raw'
"excludePatterns" TEXT DEFAULT NULL, -- JSON array of glob patterns to exclude
"lastSyncTime" TEXT DEFAULT NULL,
"syncErrors" TEXT DEFAULT NULL, -- JSON array of recent sync errors
"dateCreated" TEXT NOT NULL,
"dateModified" TEXT NOT NULL,
"utcDateCreated" TEXT NOT NULL,
"utcDateModified" TEXT NOT NULL
);
-- Table to track file to note mappings for efficient lookups
CREATE TABLE IF NOT EXISTS "file_note_mappings" (
"fileNoteId" TEXT NOT NULL PRIMARY KEY,
"mappingId" TEXT NOT NULL,
"noteId" TEXT NOT NULL,
"filePath" TEXT NOT NULL,
"fileHash" TEXT DEFAULT NULL,
"fileModifiedTime" TEXT DEFAULT NULL,
"lastSyncTime" TEXT DEFAULT NULL,
"syncStatus" TEXT NOT NULL DEFAULT 'synced', -- 'synced', 'pending', 'conflict', 'error'
"dateCreated" TEXT NOT NULL,
"dateModified" TEXT NOT NULL,
"utcDateCreated" TEXT NOT NULL,
"utcDateModified" TEXT NOT NULL,
FOREIGN KEY ("mappingId") REFERENCES "file_system_mappings" ("mappingId") ON DELETE CASCADE,
FOREIGN KEY ("noteId") REFERENCES "notes" ("noteId") ON DELETE CASCADE
);
-- Index for quick lookup by noteId
CREATE INDEX "IDX_file_system_mappings_noteId" ON "file_system_mappings" ("noteId");
-- Index for finding active mappings
CREATE INDEX "IDX_file_system_mappings_active" ON "file_system_mappings" ("isActive", "noteId");
-- Unique constraint to prevent duplicate mappings for same note
CREATE UNIQUE INDEX "IDX_file_system_mappings_note_unique" ON "file_system_mappings" ("noteId");
-- Index for quick lookup by file path
CREATE INDEX "IDX_file_note_mappings_filePath" ON "file_note_mappings" ("filePath");
-- Index for finding notes by mapping
CREATE INDEX "IDX_file_note_mappings_mapping" ON "file_note_mappings" ("mappingId", "noteId");
-- Index for finding pending syncs
CREATE INDEX "IDX_file_note_mappings_sync_status" ON "file_note_mappings" ("syncStatus", "mappingId");
-- Unique constraint for file path per mapping
CREATE UNIQUE INDEX "IDX_file_note_mappings_file_unique" ON "file_note_mappings" ("mappingId", "filePath");

View File

@@ -12,6 +12,8 @@ import type { AttachmentRow, BlobRow, RevisionRow } from "@triliumnext/commons";
import BBlob from "./entities/bblob.js";
import BRecentNote from "./entities/brecent_note.js";
import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js";
import type BFileSystemMapping from "./entities/bfile_system_mapping.js";
import type BFileNoteMapping from "./entities/bfile_note_mapping.js";
interface AttachmentOpts {
includeContentLength?: boolean;
@@ -32,6 +34,8 @@ export default class Becca {
attributeIndex!: Record<string, BAttribute[]>;
options!: Record<string, BOption>;
etapiTokens!: Record<string, BEtapiToken>;
fileSystemMappings!: Record<string, BFileSystemMapping>;
fileNoteMappings!: Record<string, BFileNoteMapping>;
allNoteSetCache: NoteSet | null;
@@ -48,6 +52,8 @@ export default class Becca {
this.attributeIndex = {};
this.options = {};
this.etapiTokens = {};
this.fileSystemMappings = {};
this.fileNoteMappings = {};
this.dirtyNoteSetCache();
@@ -213,6 +219,39 @@ export default class Becca {
return this.etapiTokens[etapiTokenId];
}
getFileSystemMapping(mappingId: string): BFileSystemMapping | null {
return this.fileSystemMappings[mappingId];
}
getFileSystemMappingOrThrow(mappingId: string): BFileSystemMapping {
const mapping = this.getFileSystemMapping(mappingId);
if (!mapping) {
throw new NotFoundError(`File system mapping '${mappingId}' has not been found.`);
}
return mapping;
}
getFileNoteMapping(fileNoteId: string): BFileNoteMapping | null {
return this.fileNoteMappings[fileNoteId];
}
getFileNoteMappingOrThrow(fileNoteId: string): BFileNoteMapping {
const mapping = this.getFileNoteMapping(fileNoteId);
if (!mapping) {
throw new NotFoundError(`File note mapping '${fileNoteId}' has not been found.`);
}
return mapping;
}
getFileSystemMappingByNoteId(noteId: string): BFileSystemMapping | null {
for (const mapping of Object.values(this.fileSystemMappings)) {
if (mapping.noteId === noteId) {
return mapping;
}
}
return null;
}
getEntity<T extends AbstractBeccaEntity<T>>(entityName: string, entityId: string): AbstractBeccaEntity<T> | null {
if (!entityName || !entityId) {
return null;
@@ -222,6 +261,10 @@ export default class Becca {
return this.getRevision(entityId);
} else if (entityName === "attachments") {
return this.getAttachment(entityId);
} else if (entityName === "file_system_mappings") {
return this.getFileSystemMapping(entityId);
} else if (entityName === "file_note_mappings") {
return this.getFileNoteMapping(entityId);
}
const camelCaseEntityName = entityName.toLowerCase().replace(/(_[a-z])/g, (group) => group.toUpperCase().replace("_", ""));

View File

@@ -9,9 +9,13 @@ import BBranch from "./entities/bbranch.js";
import BAttribute from "./entities/battribute.js";
import BOption from "./entities/boption.js";
import BEtapiToken from "./entities/betapi_token.js";
import BFileSystemMapping from "./entities/bfile_system_mapping.js";
import BFileNoteMapping from "./entities/bfile_note_mapping.js";
import cls from "../services/cls.js";
import entityConstructor from "../becca/entity_constructor.js";
import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";
import type { FileSystemMappingRow } from "./entities/bfile_system_mapping.js";
import type { FileNoteMappingRow } from "./entities/bfile_note_mapping.js";
import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js";
import ws from "../services/ws.js";
@@ -64,6 +68,14 @@ function load() {
new BEtapiToken(row);
}
for (const row of sql.getRows<FileSystemMappingRow>(/*sql*/`SELECT mappingId, noteId, filePath, syncDirection, isActive, includeSubtree, preserveHierarchy, contentFormat, excludePatterns, lastSyncTime, syncErrors, dateCreated, dateModified, utcDateCreated, utcDateModified FROM file_system_mappings`)) {
new BFileSystemMapping(row);
}
for (const row of sql.getRows<FileNoteMappingRow>(/*sql*/`SELECT fileNoteId, mappingId, noteId, filePath, fileHash, fileModifiedTime, lastSyncTime, syncStatus, dateCreated, dateModified, utcDateCreated, utcDateModified FROM file_note_mappings`)) {
new BFileNoteMapping(row);
}
});
for (const noteId in becca.notes) {
@@ -86,7 +98,7 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_CHANGE_SYNCED], ({ entity
return;
}
if (["notes", "branches", "attributes", "etapi_tokens", "options"].includes(entityName)) {
if (["notes", "branches", "attributes", "etapi_tokens", "options", "file_system_mappings", "file_note_mappings"].includes(entityName)) {
const EntityClass = entityConstructor.getEntityFromEntityName(entityName);
const primaryKeyName = EntityClass.primaryKeyName;
@@ -144,6 +156,10 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_DELETED, eventService.ENT
attributeDeleted(entityId);
} else if (entityName === "etapi_tokens") {
etapiTokenDeleted(entityId);
} else if (entityName === "file_system_mappings") {
fileSystemMappingDeleted(entityId);
} else if (entityName === "file_note_mappings") {
fileNoteMappingDeleted(entityId);
}
});
@@ -279,6 +295,14 @@ function etapiTokenDeleted(etapiTokenId: string) {
delete becca.etapiTokens[etapiTokenId];
}
function fileSystemMappingDeleted(mappingId: string) {
delete becca.fileSystemMappings[mappingId];
}
function fileNoteMappingDeleted(fileNoteId: string) {
delete becca.fileNoteMappings[fileNoteId];
}
eventService.subscribeBeccaLoader(eventService.ENTER_PROTECTED_SESSION, () => {
try {

View File

@@ -0,0 +1,233 @@
"use strict";
import AbstractBeccaEntity from "./abstract_becca_entity.js";
import dateUtils from "../../services/date_utils.js";
import { newEntityId } from "../../services/utils.js";
export interface FileNoteMappingRow {
fileNoteId?: string;
mappingId: string;
noteId: string;
filePath: string;
fileHash?: string | null;
fileModifiedTime?: string | null;
lastSyncTime?: string | null;
syncStatus?: 'synced' | 'pending' | 'conflict' | 'error';
dateCreated?: string;
dateModified?: string;
utcDateCreated?: string;
utcDateModified?: string;
}
/**
* FileNoteMapping represents the mapping between a specific file and a specific note
* This is used for tracking sync status and file metadata
*/
class BFileNoteMapping extends AbstractBeccaEntity<BFileNoteMapping> {
static get entityName() {
return "file_note_mappings";
}
static get primaryKeyName() {
return "fileNoteId";
}
static get hashedProperties() {
return ["fileNoteId", "mappingId", "noteId", "filePath", "fileHash", "syncStatus"];
}
fileNoteId!: string;
mappingId!: string;
noteId!: string;
filePath!: string;
fileHash?: string | null;
fileModifiedTime?: string | null;
lastSyncTime?: string | null;
syncStatus!: 'synced' | 'pending' | 'conflict' | 'error';
constructor(row?: FileNoteMappingRow) {
super();
if (!row) {
return;
}
this.updateFromRow(row);
this.init();
}
updateFromRow(row: FileNoteMappingRow) {
this.update([
row.fileNoteId,
row.mappingId,
row.noteId,
row.filePath,
row.fileHash,
row.fileModifiedTime,
row.lastSyncTime,
row.syncStatus || 'synced',
row.dateCreated,
row.dateModified,
row.utcDateCreated,
row.utcDateModified
]);
}
update([
fileNoteId,
mappingId,
noteId,
filePath,
fileHash,
fileModifiedTime,
lastSyncTime,
syncStatus,
dateCreated,
dateModified,
utcDateCreated,
utcDateModified
]: any) {
this.fileNoteId = fileNoteId;
this.mappingId = mappingId;
this.noteId = noteId;
this.filePath = filePath;
this.fileHash = fileHash;
this.fileModifiedTime = fileModifiedTime;
this.lastSyncTime = lastSyncTime;
this.syncStatus = syncStatus || 'synced';
this.dateCreated = dateCreated;
this.dateModified = dateModified;
this.utcDateCreated = utcDateCreated;
this.utcDateModified = utcDateModified;
return this;
}
override init() {
if (this.fileNoteId) {
this.becca.fileNoteMappings = this.becca.fileNoteMappings || {};
this.becca.fileNoteMappings[this.fileNoteId] = this;
}
}
get note() {
return this.becca.notes[this.noteId];
}
get mapping() {
return this.becca.fileSystemMappings?.[this.mappingId];
}
getNote() {
const note = this.becca.getNote(this.noteId);
if (!note) {
throw new Error(`Note '${this.noteId}' for file note mapping '${this.fileNoteId}' does not exist.`);
}
return note;
}
getMapping() {
const mapping = this.mapping;
if (!mapping) {
throw new Error(`File system mapping '${this.mappingId}' for file note mapping '${this.fileNoteId}' does not exist.`);
}
return mapping;
}
/**
* Mark this mapping as needing sync
*/
markPending() {
this.syncStatus = 'pending';
this.save();
}
/**
* Mark this mapping as having a conflict
*/
markConflict() {
this.syncStatus = 'conflict';
this.save();
}
/**
* Mark this mapping as having an error
*/
markError() {
this.syncStatus = 'error';
this.save();
}
/**
* Mark this mapping as synced and update sync time
*/
markSynced(fileHash?: string, fileModifiedTime?: string) {
this.syncStatus = 'synced';
this.lastSyncTime = dateUtils.utcNowDateTime();
if (fileHash !== undefined) {
this.fileHash = fileHash;
}
if (fileModifiedTime !== undefined) {
this.fileModifiedTime = fileModifiedTime;
}
this.save();
}
/**
* Check if the file has been modified since last sync
*/
hasFileChanged(currentFileHash: string, currentModifiedTime: string): boolean {
return this.fileHash !== currentFileHash || this.fileModifiedTime !== currentModifiedTime;
}
/**
* Check if the note has been modified since last sync
*/
hasNoteChanged(): boolean {
const note = this.note;
if (!note) return false;
if (!this.lastSyncTime) return true;
return (note.utcDateModified ?? note.dateModified ?? note.utcDateCreated) > this.lastSyncTime;
}
override beforeSaving() {
super.beforeSaving();
if (!this.fileNoteId) {
this.fileNoteId = newEntityId();
}
if (!this.dateCreated) {
this.dateCreated = dateUtils.localNowDateTime();
}
if (!this.utcDateCreated) {
this.utcDateCreated = dateUtils.utcNowDateTime();
}
this.dateModified = dateUtils.localNowDateTime();
this.utcDateModified = dateUtils.utcNowDateTime();
}
getPojo(): FileNoteMappingRow {
return {
fileNoteId: this.fileNoteId,
mappingId: this.mappingId,
noteId: this.noteId,
filePath: this.filePath,
fileHash: this.fileHash,
fileModifiedTime: this.fileModifiedTime,
lastSyncTime: this.lastSyncTime,
syncStatus: this.syncStatus,
dateCreated: this.dateCreated,
dateModified: this.dateModified,
utcDateCreated: this.utcDateCreated,
utcDateModified: this.utcDateModified
};
}
}
export default BFileNoteMapping;

View File

@@ -0,0 +1,236 @@
"use strict";
import AbstractBeccaEntity from "./abstract_becca_entity.js";
import dateUtils from "../../services/date_utils.js";
import { newEntityId } from "../../services/utils.js";
export interface FileSystemMappingRow {
mappingId?: string;
noteId: string;
filePath: string;
syncDirection?: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
isActive?: number;
includeSubtree?: number;
preserveHierarchy?: number;
contentFormat?: 'auto' | 'markdown' | 'html' | 'raw';
excludePatterns?: string | null;
lastSyncTime?: string | null;
syncErrors?: string | null;
dateCreated?: string;
dateModified?: string;
utcDateCreated?: string;
utcDateModified?: string;
}
/**
* FileSystemMapping represents a mapping between a note/subtree and a file system path
*/
class BFileSystemMapping extends AbstractBeccaEntity<BFileSystemMapping> {
static get entityName() {
return "file_system_mappings";
}
static get primaryKeyName() {
return "mappingId";
}
static get hashedProperties() {
return ["mappingId", "noteId", "filePath", "syncDirection", "isActive", "includeSubtree", "preserveHierarchy", "contentFormat"];
}
mappingId!: string;
noteId!: string;
filePath!: string;
syncDirection!: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
isActive!: boolean;
includeSubtree!: boolean;
preserveHierarchy!: boolean;
contentFormat!: 'auto' | 'markdown' | 'html' | 'raw';
excludePatterns?: (string | RegExp)[] | null;
lastSyncTime?: string | null;
syncErrors?: string[] | null;
constructor(row?: FileSystemMappingRow) {
super();
if (!row) {
return;
}
this.updateFromRow(row);
this.init();
}
updateFromRow(row: FileSystemMappingRow) {
this.update([
row.mappingId,
row.noteId,
row.filePath,
row.syncDirection || 'bidirectional',
row.isActive !== undefined ? row.isActive : 1,
row.includeSubtree !== undefined ? row.includeSubtree : 0,
row.preserveHierarchy !== undefined ? row.preserveHierarchy : 1,
row.contentFormat || 'auto',
row.excludePatterns,
row.lastSyncTime,
row.syncErrors,
row.dateCreated,
row.dateModified,
row.utcDateCreated,
row.utcDateModified
]);
}
update([
mappingId,
noteId,
filePath,
syncDirection,
isActive,
includeSubtree,
preserveHierarchy,
contentFormat,
excludePatterns,
lastSyncTime,
syncErrors,
dateCreated,
dateModified,
utcDateCreated,
utcDateModified
]: any) {
this.mappingId = mappingId;
this.noteId = noteId;
this.filePath = filePath;
this.syncDirection = syncDirection || 'bidirectional';
this.isActive = !!isActive;
this.includeSubtree = !!includeSubtree;
this.preserveHierarchy = !!preserveHierarchy;
this.contentFormat = contentFormat || 'auto';
// Parse JSON strings for arrays
try {
this.excludePatterns = excludePatterns ? JSON.parse(excludePatterns) : null;
} catch {
this.excludePatterns = null;
}
try {
this.syncErrors = syncErrors ? JSON.parse(syncErrors) : null;
} catch {
this.syncErrors = null;
}
this.lastSyncTime = lastSyncTime;
this.dateCreated = dateCreated;
this.dateModified = dateModified;
this.utcDateCreated = utcDateCreated;
this.utcDateModified = utcDateModified;
return this;
}
override init() {
if (this.mappingId) {
this.becca.fileSystemMappings = this.becca.fileSystemMappings || {};
this.becca.fileSystemMappings[this.mappingId] = this;
}
}
get note() {
return this.becca.notes[this.noteId];
}
getNote() {
const note = this.becca.getNote(this.noteId);
if (!note) {
throw new Error(`Note '${this.noteId}' for file system mapping '${this.mappingId}' does not exist.`);
}
return note;
}
/**
* Check if the mapping allows syncing from Trilium to disk
*/
get canSyncToDisk(): boolean {
return this.isActive && (this.syncDirection === 'bidirectional' || this.syncDirection === 'trilium_to_disk');
}
/**
* Check if the mapping allows syncing from disk to Trilium
*/
get canSyncFromDisk(): boolean {
return this.isActive && (this.syncDirection === 'bidirectional' || this.syncDirection === 'disk_to_trilium');
}
/**
* Add a sync error to the errors list
*/
addSyncError(error: string) {
if (!this.syncErrors) {
this.syncErrors = [];
}
this.syncErrors.push(error);
// Keep only the last 10 errors
if (this.syncErrors.length > 10) {
this.syncErrors = this.syncErrors.slice(-10);
}
this.save();
}
/**
* Clear all sync errors
*/
clearSyncErrors() {
this.syncErrors = null;
this.save();
}
/**
* Update the last sync time
*/
updateLastSyncTime() {
this.lastSyncTime = dateUtils.utcNowDateTime();
this.save();
}
override beforeSaving() {
super.beforeSaving();
if (!this.mappingId) {
this.mappingId = newEntityId();
}
if (!this.dateCreated) {
this.dateCreated = dateUtils.localNowDateTime();
}
if (!this.utcDateCreated) {
this.utcDateCreated = dateUtils.utcNowDateTime();
}
this.dateModified = dateUtils.localNowDateTime();
this.utcDateModified = dateUtils.utcNowDateTime();
}
getPojo(): FileSystemMappingRow {
return {
mappingId: this.mappingId,
noteId: this.noteId,
filePath: this.filePath,
syncDirection: this.syncDirection,
isActive: this.isActive ? 1 : 0,
includeSubtree: this.includeSubtree ? 1 : 0,
preserveHierarchy: this.preserveHierarchy ? 1 : 0,
contentFormat: this.contentFormat,
excludePatterns: this.excludePatterns ? JSON.stringify(this.excludePatterns) : null,
lastSyncTime: this.lastSyncTime,
syncErrors: this.syncErrors ? JSON.stringify(this.syncErrors) : null,
dateCreated: this.dateCreated,
dateModified: this.dateModified,
utcDateCreated: this.utcDateCreated,
utcDateModified: this.utcDateModified
};
}
}
export default BFileSystemMapping;

View File

@@ -9,6 +9,8 @@ import BNote from "./entities/bnote.js";
import BOption from "./entities/boption.js";
import BRecentNote from "./entities/brecent_note.js";
import BRevision from "./entities/brevision.js";
import BFileSystemMapping from "./entities/bfile_system_mapping.js";
import BFileNoteMapping from "./entities/bfile_note_mapping.js";
type EntityClass = new (row?: any) => AbstractBeccaEntity<any>;
@@ -21,7 +23,9 @@ const ENTITY_NAME_TO_ENTITY: Record<string, ConstructorData<any> & EntityClass>
notes: BNote,
options: BOption,
recent_notes: BRecentNote,
revisions: BRevision
revisions: BRevision,
file_system_mappings: BFileSystemMapping,
file_note_mappings: BFileNoteMapping
};
function getEntityFromEntityName(entityName: keyof typeof ENTITY_NAME_TO_ENTITY) {

View File

@@ -6,6 +6,64 @@
// Migrations should be kept in descending order, so the latest migration is first.
const MIGRATIONS: (SqlMigration | JsMigration)[] = [
// Add file system mapping support
{
version: 234,
sql: /*sql*/`
-- Table to store file system mappings for notes and subtrees
CREATE TABLE IF NOT EXISTS "file_system_mappings" (
"mappingId" TEXT NOT NULL PRIMARY KEY,
"noteId" TEXT NOT NULL,
"filePath" TEXT NOT NULL,
"syncDirection" TEXT NOT NULL DEFAULT 'bidirectional', -- 'bidirectional', 'trilium_to_disk', 'disk_to_trilium'
"isActive" INTEGER NOT NULL DEFAULT 1,
"includeSubtree" INTEGER NOT NULL DEFAULT 0,
"preserveHierarchy" INTEGER NOT NULL DEFAULT 1,
"contentFormat" TEXT NOT NULL DEFAULT 'auto', -- 'auto', 'markdown', 'html', 'raw'
"excludePatterns" TEXT DEFAULT NULL, -- JSON array of glob patterns to exclude
"lastSyncTime" TEXT DEFAULT NULL,
"syncErrors" TEXT DEFAULT NULL, -- JSON array of recent sync errors
"dateCreated" TEXT NOT NULL,
"dateModified" TEXT NOT NULL,
"utcDateCreated" TEXT NOT NULL,
"utcDateModified" TEXT NOT NULL
);
-- Index for quick lookup by noteId
CREATE INDEX "IDX_file_system_mappings_noteId" ON "file_system_mappings" ("noteId");
-- Index for finding active mappings
CREATE INDEX "IDX_file_system_mappings_active" ON "file_system_mappings" ("isActive", "noteId");
-- Unique constraint to prevent duplicate mappings for same note
CREATE UNIQUE INDEX "IDX_file_system_mappings_note_unique" ON "file_system_mappings" ("noteId");
-- Table to track file to note mappings for efficient lookups
CREATE TABLE IF NOT EXISTS "file_note_mappings" (
"fileNoteId" TEXT NOT NULL PRIMARY KEY,
"mappingId" TEXT NOT NULL,
"noteId" TEXT NOT NULL,
"filePath" TEXT NOT NULL,
"fileHash" TEXT DEFAULT NULL,
"fileModifiedTime" TEXT DEFAULT NULL,
"lastSyncTime" TEXT DEFAULT NULL,
"syncStatus" TEXT NOT NULL DEFAULT 'synced', -- 'synced', 'pending', 'conflict', 'error'
"dateCreated" TEXT NOT NULL,
"dateModified" TEXT NOT NULL,
"utcDateCreated" TEXT NOT NULL,
"utcDateModified" TEXT NOT NULL,
FOREIGN KEY ("mappingId") REFERENCES "file_system_mappings" ("mappingId") ON DELETE CASCADE,
FOREIGN KEY ("noteId") REFERENCES "notes" ("noteId") ON DELETE CASCADE
);
-- Index for quick lookup by file path
CREATE INDEX "IDX_file_note_mappings_filePath" ON "file_note_mappings" ("filePath");
-- Index for finding notes by mapping
CREATE INDEX "IDX_file_note_mappings_mapping" ON "file_note_mappings" ("mappingId", "noteId");
-- Index for finding pending syncs
CREATE INDEX "IDX_file_note_mappings_sync_status" ON "file_note_mappings" ("syncStatus", "mappingId");
-- Unique constraint for file path per mapping
CREATE UNIQUE INDEX "IDX_file_note_mappings_file_unique" ON "file_note_mappings" ("mappingId", "filePath");
`
},
// Migrate geo map to collection
{
version: 233,

View File

@@ -0,0 +1,297 @@
"use strict";
import becca from "../../becca/becca.js";
import BFileSystemMapping from "../../becca/entities/bfile_system_mapping.js";
import fileSystemSyncInit from "../../services/file_system_sync_init.js";
import log from "../../services/log.js";
import ValidationError from "../../errors/validation_error.js";
import fs from "fs-extra";
import path from "path";
import { router, asyncApiRoute, apiRoute } from "../route_api.js";
interface FileStat {
isFile: boolean;
isDirectory: boolean;
size: number;
modified: string;
}
// Get all file system mappings
apiRoute("get", "/mappings", () => {
const mappings = Object.values(becca.fileSystemMappings || {}).map(mapping => ({
mappingId: mapping.mappingId,
noteId: mapping.noteId,
filePath: mapping.filePath,
syncDirection: mapping.syncDirection,
isActive: mapping.isActive,
includeSubtree: mapping.includeSubtree,
preserveHierarchy: mapping.preserveHierarchy,
contentFormat: mapping.contentFormat,
excludePatterns: mapping.excludePatterns,
lastSyncTime: mapping.lastSyncTime,
syncErrors: mapping.syncErrors,
dateCreated: mapping.dateCreated,
dateModified: mapping.dateModified
}));
return mappings;
});
// Get a specific file system mapping
apiRoute("get", "/mappings/:mappingId", (req) => {
const { mappingId } = req.params;
const mapping = becca.fileSystemMappings[mappingId];
if (!mapping) {
return [404, { error: "Mapping not found" }];
}
return {
mappingId: mapping.mappingId,
noteId: mapping.noteId,
filePath: mapping.filePath,
syncDirection: mapping.syncDirection,
isActive: mapping.isActive,
includeSubtree: mapping.includeSubtree,
preserveHierarchy: mapping.preserveHierarchy,
contentFormat: mapping.contentFormat,
excludePatterns: mapping.excludePatterns,
lastSyncTime: mapping.lastSyncTime,
syncErrors: mapping.syncErrors,
dateCreated: mapping.dateCreated,
dateModified: mapping.dateModified
};
});
// Create a new file system mapping
asyncApiRoute("post", "/mappings", async (req) => {
const {
noteId,
filePath,
syncDirection = 'bidirectional',
isActive = true,
includeSubtree = false,
preserveHierarchy = true,
contentFormat = 'auto',
excludePatterns = null
} = req.body;
// Validate required fields
if (!noteId || !filePath) {
throw new ValidationError("noteId and filePath are required");
}
// Validate note exists
const note = becca.notes[noteId];
if (!note) {
throw new ValidationError(`Note ${noteId} not found`);
}
// Check if mapping already exists for this note
const existingMapping = becca.getFileSystemMappingByNoteId(noteId);
if (existingMapping) {
throw new ValidationError(`File system mapping already exists for note ${noteId}`);
}
// Validate file path exists
const normalizedPath = path.resolve(filePath);
if (!await fs.pathExists(normalizedPath)) {
throw new ValidationError(`File path does not exist: ${normalizedPath}`);
}
// Validate sync direction
const validDirections = ['bidirectional', 'trilium_to_disk', 'disk_to_trilium'];
if (!validDirections.includes(syncDirection)) {
throw new ValidationError(`Invalid sync direction. Must be one of: ${validDirections.join(', ')}`);
}
// Validate content format
const validFormats = ['auto', 'markdown', 'html', 'raw'];
if (!validFormats.includes(contentFormat)) {
throw new ValidationError(`Invalid content format. Must be one of: ${validFormats.join(', ')}`);
}
// Create the mapping
const mapping = new BFileSystemMapping({
noteId,
filePath: normalizedPath,
syncDirection,
isActive: isActive ? 1 : 0,
includeSubtree: includeSubtree ? 1 : 0,
preserveHierarchy: preserveHierarchy ? 1 : 0,
contentFormat,
excludePatterns: Array.isArray(excludePatterns) ? JSON.stringify(excludePatterns) : excludePatterns
}).save();
log.info(`Created file system mapping ${mapping.mappingId} for note ${noteId} -> ${normalizedPath}`);
return [201, {
mappingId: mapping.mappingId,
noteId: mapping.noteId,
filePath: mapping.filePath,
syncDirection: mapping.syncDirection,
isActive: mapping.isActive,
includeSubtree: mapping.includeSubtree,
preserveHierarchy: mapping.preserveHierarchy,
contentFormat: mapping.contentFormat,
excludePatterns: mapping.excludePatterns
}];
});
// Update a file system mapping
asyncApiRoute("put", "/mappings/:mappingId", async (req) => {
const { mappingId } = req.params;
const mapping = becca.fileSystemMappings[mappingId];
if (!mapping) {
return [404, { error: "Mapping not found" }];
}
const {
filePath,
syncDirection,
isActive,
includeSubtree,
preserveHierarchy,
contentFormat,
excludePatterns
} = req.body;
// Update fields if provided
if (filePath !== undefined) {
const normalizedPath = path.resolve(filePath);
if (!await fs.pathExists(normalizedPath)) {
throw new ValidationError(`File path does not exist: ${normalizedPath}`);
}
mapping.filePath = normalizedPath;
}
if (syncDirection !== undefined) {
const validDirections = ['bidirectional', 'trilium_to_disk', 'disk_to_trilium'];
if (!validDirections.includes(syncDirection)) {
throw new ValidationError(`Invalid sync direction. Must be one of: ${validDirections.join(', ')}`);
}
mapping.syncDirection = syncDirection;
}
if (isActive !== undefined) {
mapping.isActive = !!isActive;
}
if (includeSubtree !== undefined) {
mapping.includeSubtree = !!includeSubtree;
}
if (preserveHierarchy !== undefined) {
mapping.preserveHierarchy = !!preserveHierarchy;
}
if (contentFormat !== undefined) {
const validFormats = ['auto', 'markdown', 'html', 'raw'];
if (!validFormats.includes(contentFormat)) {
throw new ValidationError(`Invalid content format. Must be one of: ${validFormats.join(', ')}`);
}
mapping.contentFormat = contentFormat;
}
if (excludePatterns !== undefined) {
mapping.excludePatterns = Array.isArray(excludePatterns) ? excludePatterns : null;
}
mapping.save();
log.info(`Updated file system mapping ${mappingId}`);
return {
mappingId: mapping.mappingId,
noteId: mapping.noteId,
filePath: mapping.filePath,
syncDirection: mapping.syncDirection,
isActive: mapping.isActive,
includeSubtree: mapping.includeSubtree,
preserveHierarchy: mapping.preserveHierarchy,
contentFormat: mapping.contentFormat,
excludePatterns: mapping.excludePatterns
};
});
// Delete a file system mapping
apiRoute("delete", "/mappings/:mappingId", (req) => {
const { mappingId } = req.params;
const mapping = becca.fileSystemMappings[mappingId];
if (!mapping) {
return [404, { error: "Mapping not found" }];
}
mapping.markAsDeleted();
log.info(`Deleted file system mapping ${mappingId}`);
return { success: true };
});
// Trigger full sync for a mapping
asyncApiRoute("post", "/mappings/:mappingId/sync", async (req) => {
const { mappingId } = req.params;
if (!fileSystemSyncInit.isInitialized()) {
return [503, { error: "File system sync is not initialized" }];
}
const result = await fileSystemSyncInit.fullSync(mappingId);
if (result.success) {
return result;
} else {
return [400, result];
}
});
// Get sync status for all mappings
apiRoute("get", "/status", () => {
return fileSystemSyncInit.getStatus();
});
// Enable file system sync
asyncApiRoute("post", "/enable", async () => {
await fileSystemSyncInit.enable();
return { success: true, message: "File system sync enabled" };
});
// Disable file system sync
asyncApiRoute("post", "/disable", async () => {
await fileSystemSyncInit.disable();
return { success: true, message: "File system sync disabled" };
});
// Validate file path
asyncApiRoute("post", "/validate-path", async (req) => {
const { filePath } = req.body;
if (!filePath) {
throw new ValidationError("filePath is required");
}
const normalizedPath = path.resolve(filePath);
const exists = await fs.pathExists(normalizedPath);
let stats: FileStat | null = null;
if (exists) {
const fileStats = await fs.stat(normalizedPath);
stats = {
isFile: fileStats.isFile(),
isDirectory: fileStats.isDirectory(),
size: fileStats.size,
modified: fileStats.mtime.toISOString()
};
}
return {
path: normalizedPath,
exists,
stats
};
});
export default router;

View File

@@ -93,6 +93,7 @@ const ALLOWED_OPTIONS = new Set<OptionNames>([
"redirectBareDomain",
"showLoginInShareTheme",
"splitEditorOrientation",
"fileSystemSyncEnabled",
// AI/LLM integration options
"aiEnabled",

View File

@@ -59,6 +59,7 @@ import openaiRoute from "./api/openai.js";
import anthropicRoute from "./api/anthropic.js";
import llmRoute from "./api/llm.js";
import systemInfoRoute from "./api/system_info.js";
import fileSystemSyncRoute from "./api/file_system_sync.js";
import etapiAuthRoutes from "../etapi/auth.js";
import etapiAppInfoRoutes from "../etapi/app_info.js";
@@ -385,6 +386,9 @@ function register(app: express.Application) {
asyncApiRoute(GET, "/api/llm/providers/openai/models", openaiRoute.listModels);
asyncApiRoute(GET, "/api/llm/providers/anthropic/models", anthropicRoute.listModels);
// File system sync API
app.use("/api/file-system-sync", [auth.checkApiAuthOrElectron, csrfMiddleware], fileSystemSyncRoute);
// API Documentation
apiDocsRoute(app);

View File

@@ -3,7 +3,7 @@ import build from "./build.js";
import packageJson from "../../package.json" with { type: "json" };
import dataDir from "./data_dir.js";
const APP_DB_VERSION = 233;
const APP_DB_VERSION = 234;
const SYNC_VERSION = 36;
const CLIPPER_PROTOCOL_VERSION = "1.0";

View File

@@ -0,0 +1,464 @@
"use strict";
import path from "path";
import log from "./log.js";
import markdownExportService from "./export/markdown.js";
import markdownImportService from "./import/markdown.js";
import BNote from "../becca/entities/bnote.js";
import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js";
import utils from "./utils.js";
import { type NoteType } from "@triliumnext/commons";
export interface ConversionResult {
content: string | Buffer;
attributes?: Array<{ type: 'label' | 'relation'; name: string; value: string; isInheritable?: boolean }>;
mime?: string;
type?: NoteType;
}
export interface ConversionOptions {
preserveAttributes?: boolean;
includeFrontmatter?: boolean;
relativeImagePaths?: boolean;
}
/**
* Content converter for file system sync operations
* Handles conversion between Trilium note formats and file system formats
*/
class FileSystemContentConverter {
/**
* Convert note content to file format based on mapping configuration
*/
async noteToFile(note: BNote, mapping: BFileSystemMapping, filePath: string, options: ConversionOptions = {}): Promise<ConversionResult> {
const fileExt = path.extname(filePath).toLowerCase();
const contentFormat = mapping.contentFormat === 'auto' ? this.detectFormatFromExtension(fileExt) : mapping.contentFormat;
switch (contentFormat) {
case 'markdown':
return this.noteToMarkdown(note, options);
case 'html':
return this.noteToHtml(note, options);
case 'raw':
default:
return this.noteToRaw(note, options);
}
}
/**
* Convert file content to note format based on mapping configuration
*/
async fileToNote(fileContent: string | Buffer, mapping: BFileSystemMapping, filePath: string, options: ConversionOptions = {}): Promise<ConversionResult> {
const fileExt = path.extname(filePath).toLowerCase();
const contentFormat = mapping.contentFormat === 'auto' ? this.detectFormatFromExtension(fileExt) : mapping.contentFormat;
// Convert Buffer to string for text formats
const content = Buffer.isBuffer(fileContent) ? fileContent.toString('utf8') : fileContent;
switch (contentFormat) {
case 'markdown':
// Extract title from note for proper H1 deduplication
const note = mapping.note;
const title = note ? note.title : path.basename(filePath, path.extname(filePath));
return this.markdownToNote(content, options, title);
case 'html':
return this.htmlToNote(content, options);
case 'raw':
default:
return this.rawToNote(fileContent, fileExt, options);
}
}
/**
* Detect content format from file extension
*/
private detectFormatFromExtension(extension: string): 'markdown' | 'html' | 'raw' {
const markdownExts = ['.md', '.markdown', '.mdown', '.mkd'];
const htmlExts = ['.html', '.htm'];
if (markdownExts.includes(extension)) {
return 'markdown';
} else if (htmlExts.includes(extension)) {
return 'html';
} else {
return 'raw';
}
}
/**
* Convert note to Markdown format
*/
private async noteToMarkdown(note: BNote, options: ConversionOptions): Promise<ConversionResult> {
try {
let content = note.getContent() as string;
// Convert HTML content to Markdown
if (note.type === 'text' && note.mime === 'text/html') {
content = markdownExportService.toMarkdown(content);
}
// Add frontmatter with note attributes if requested
if (options.includeFrontmatter && options.preserveAttributes) {
const frontmatter = this.createFrontmatter(note);
if (frontmatter) {
content = `---\n${frontmatter}\n---\n\n${content}`;
}
}
return {
content,
mime: 'text/markdown',
type: 'text'
};
} catch (error) {
log.error(`Error converting note ${note.noteId} to Markdown: ${error}`);
throw error;
}
}
/**
* Convert note to HTML format
*/
private async noteToHtml(note: BNote, options: ConversionOptions): Promise<ConversionResult> {
let content = note.getContent() as string;
// If note is already HTML, just clean it up
if (note.type === 'text' && note.mime === 'text/html') {
// Could add HTML processing here if needed
} else if (note.type === 'code') {
// Wrap code content in pre/code tags
const language = this.getLanguageFromMime(note.mime);
content = `<pre><code class="language-${language}">${utils.escapeHtml(content)}</code></pre>`;
}
// Add HTML frontmatter as comments if requested
if (options.includeFrontmatter && options.preserveAttributes) {
const frontmatter = this.createFrontmatter(note);
if (frontmatter) {
content = `<!-- \n${frontmatter}\n-->\n\n${content}`;
}
}
return {
content,
mime: 'text/html',
type: 'text'
};
}
/**
* Convert note to raw format (preserve original content)
*/
private async noteToRaw(note: BNote, options: ConversionOptions): Promise<ConversionResult> {
const content = note.getContent();
return {
content,
mime: note.mime,
type: note.type
};
}
/**
* Convert Markdown content to note format
*/
private async markdownToNote(content: string, options: ConversionOptions, title: string = ''): Promise<ConversionResult> {
try {
let processedContent = content;
let attributes: ConversionResult['attributes'] = [];
// Extract frontmatter if present
if (options.preserveAttributes) {
const frontmatterResult = this.extractFrontmatter(content);
processedContent = frontmatterResult.content;
attributes = frontmatterResult.attributes;
}
// Convert Markdown to HTML using the correct method
// The title helps deduplicate <h1> tags with the note title
const htmlContent = markdownImportService.renderToHtml(processedContent, title);
return {
content: htmlContent,
attributes,
mime: 'text/html',
type: 'text'
};
} catch (error) {
log.error(`Error converting Markdown to note: ${error}`);
throw error;
}
}
/**
* Convert HTML content to note format
*/
private async htmlToNote(content: string, options: ConversionOptions): Promise<ConversionResult> {
let processedContent = content;
let attributes: ConversionResult['attributes'] = [];
// Extract HTML comment frontmatter if present
if (options.preserveAttributes) {
const frontmatterResult = this.extractHtmlFrontmatter(content);
processedContent = frontmatterResult.content;
attributes = frontmatterResult.attributes;
}
return {
content: processedContent,
attributes,
mime: 'text/html',
type: 'text'
};
}
/**
* Convert raw content to note format
*/
private async rawToNote(content: string | Buffer, extension: string, options: ConversionOptions): Promise<ConversionResult> {
// Determine note type and mime based on file extension
const { type, mime } = this.getTypeAndMimeFromExtension(extension);
return {
content,
mime,
type
};
}
/**
* Create YAML frontmatter from note attributes
*/
private createFrontmatter(note: BNote): string | null {
const attributes = note.getOwnedAttributes();
if (attributes.length === 0) {
return null;
}
const yamlLines: string[] = [];
yamlLines.push(`title: "${note.title.replace(/"/g, '\\"')}"`);
yamlLines.push(`noteId: "${note.noteId}"`);
yamlLines.push(`type: "${note.type}"`);
yamlLines.push(`mime: "${note.mime}"`);
const labels = attributes.filter(attr => attr.type === 'label');
const relations = attributes.filter(attr => attr.type === 'relation');
if (labels.length > 0) {
yamlLines.push('labels:');
for (const label of labels) {
const inheritable = label.isInheritable ? ' (inheritable)' : '';
yamlLines.push(` - name: "${label.name}"`);
yamlLines.push(` value: "${label.value.replace(/"/g, '\\"')}"`);
if (label.isInheritable) {
yamlLines.push(` inheritable: true`);
}
}
}
if (relations.length > 0) {
yamlLines.push('relations:');
for (const relation of relations) {
yamlLines.push(` - name: "${relation.name}"`);
yamlLines.push(` target: "${relation.value}"`);
if (relation.isInheritable) {
yamlLines.push(` inheritable: true`);
}
}
}
return yamlLines.join('\n');
}
/**
* Extract YAML frontmatter from Markdown content
*/
private extractFrontmatter(content: string): { content: string; attributes: ConversionResult['attributes'] } {
const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
const match = content.match(frontmatterRegex);
if (!match) {
return { content, attributes: [] };
}
const frontmatterYaml = match[1];
const mainContent = match[2];
try {
const attributes = this.parseFrontmatterYaml(frontmatterYaml);
return { content: mainContent, attributes };
} catch (error) {
log.info(`Error parsing frontmatter YAML: ${error}`);
return { content, attributes: [] };
}
}
/**
* Extract frontmatter from HTML comments
*/
private extractHtmlFrontmatter(content: string): { content: string; attributes: ConversionResult['attributes'] } {
const frontmatterRegex = /^<!--\s*\n([\s\S]*?)\n-->\s*\n([\s\S]*)$/;
const match = content.match(frontmatterRegex);
if (!match) {
return { content, attributes: [] };
}
const frontmatterYaml = match[1];
const mainContent = match[2];
try {
const attributes = this.parseFrontmatterYaml(frontmatterYaml);
return { content: mainContent, attributes };
} catch (error) {
log.info(`Error parsing HTML frontmatter YAML: ${error}`);
return { content, attributes: [] };
}
}
/**
* Parse YAML frontmatter into attributes (simplified YAML parser)
*/
private parseFrontmatterYaml(yaml: string): ConversionResult['attributes'] {
const attributes: ConversionResult['attributes'] = [];
const lines = yaml.split('\n');
let currentSection: 'labels' | 'relations' | null = null;
let currentItem: any = {};
for (const line of lines) {
const trimmed = line.trim();
if (trimmed === 'labels:') {
currentSection = 'labels';
continue;
} else if (trimmed === 'relations:') {
currentSection = 'relations';
continue;
} else if (trimmed.startsWith('- name:')) {
// Save previous item if exists
if (currentItem.name && currentSection) {
attributes.push({
type: currentSection === 'labels' ? 'label' : 'relation',
name: currentItem.name,
value: currentItem.value || currentItem.target || '',
isInheritable: currentItem.inheritable || false
});
}
currentItem = { name: this.extractQuotedValue(trimmed) };
} else if (trimmed.startsWith('name:')) {
currentItem.name = this.extractQuotedValue(trimmed);
} else if (trimmed.startsWith('value:')) {
currentItem.value = this.extractQuotedValue(trimmed);
} else if (trimmed.startsWith('target:')) {
currentItem.target = this.extractQuotedValue(trimmed);
} else if (trimmed.startsWith('inheritable:')) {
currentItem.inheritable = trimmed.includes('true');
}
}
// Save last item
if (currentItem.name && currentSection) {
attributes.push({
type: currentSection === 'labels' ? 'label' : 'relation',
name: currentItem.name,
value: currentItem.value || currentItem.target || '',
isInheritable: currentItem.inheritable || false
});
}
return attributes;
}
/**
* Extract quoted value from YAML line
*/
private extractQuotedValue(line: string): string {
const match = line.match(/:\s*"([^"]+)"/);
return match ? match[1].replace(/\\"/g, '"') : '';
}
/**
* Get language identifier from MIME type
*/
private getLanguageFromMime(mime: string): string {
const mimeToLang: Record<string, string> = {
'application/javascript': 'javascript',
'text/javascript': 'javascript',
'application/typescript': 'typescript',
'text/typescript': 'typescript',
'application/json': 'json',
'text/css': 'css',
'text/html': 'html',
'application/xml': 'xml',
'text/xml': 'xml',
'text/x-python': 'python',
'text/x-java': 'java',
'text/x-csharp': 'csharp',
'text/x-sql': 'sql',
'text/x-sh': 'bash',
'text/x-yaml': 'yaml'
};
return mimeToLang[mime] || 'text';
}
/**
* Get note type and MIME type from file extension
*/
private getTypeAndMimeFromExtension(extension: string): { type: NoteType; mime: string } {
const extToType: Record<string, { type: NoteType; mime: string }> = {
'.txt': { type: 'text', mime: 'text/plain' },
'.md': { type: 'text', mime: 'text/markdown' },
'.html': { type: 'text', mime: 'text/html' },
'.htm': { type: 'text', mime: 'text/html' },
'.js': { type: 'code', mime: 'application/javascript' },
'.ts': { type: 'code', mime: 'application/typescript' },
'.json': { type: 'code', mime: 'application/json' },
'.css': { type: 'code', mime: 'text/css' },
'.xml': { type: 'code', mime: 'application/xml' },
'.py': { type: 'code', mime: 'text/x-python' },
'.java': { type: 'code', mime: 'text/x-java' },
'.cs': { type: 'code', mime: 'text/x-csharp' },
'.sql': { type: 'code', mime: 'text/x-sql' },
'.sh': { type: 'code', mime: 'text/x-sh' },
'.yaml': { type: 'code', mime: 'text/x-yaml' },
'.yml': { type: 'code', mime: 'text/x-yaml' },
'.png': { type: 'image', mime: 'image/png' },
'.jpg': { type: 'image', mime: 'image/jpeg' },
'.jpeg': { type: 'image', mime: 'image/jpeg' },
'.gif': { type: 'image', mime: 'image/gif' },
'.svg': { type: 'image', mime: 'image/svg+xml' }
};
return extToType[extension] || { type: 'file', mime: 'application/octet-stream' };
}
/**
* Validate if a file type is supported for sync
*/
isSupportedFileType(filePath: string): boolean {
const extension = path.extname(filePath).toLowerCase();
const textExtensions = ['.txt', '.md', '.html', '.htm', '.js', '.ts', '.json', '.css', '.xml', '.py', '.java', '.cs', '.sql', '.sh', '.yaml', '.yml'];
const binaryExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf'];
return textExtensions.includes(extension) || binaryExtensions.includes(extension);
}
/**
* Check if file should be treated as binary
*/
isBinaryFile(filePath: string): boolean {
const extension = path.extname(filePath).toLowerCase();
const binaryExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf', '.doc', '.docx', '.zip', '.tar', '.gz'];
return binaryExtensions.includes(extension);
}
}
// Create singleton instance
const fileSystemContentConverter = new FileSystemContentConverter();
export default fileSystemContentConverter;

View File

@@ -0,0 +1,932 @@
"use strict";
import fs from "fs-extra";
import path from "path";
import crypto from "crypto";
import log from "./log.js";
import becca from "../becca/becca.js";
import BNote from "../becca/entities/bnote.js";
import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js";
import BFileNoteMapping from "../becca/entities/bfile_note_mapping.js";
import BAttribute from "../becca/entities/battribute.js";
import BBranch from "../becca/entities/bbranch.js";
import fileSystemContentConverter from "./file_system_content_converter.js";
import fileSystemWatcher from "./file_system_watcher.js";
import eventService from "./events.js";
import noteService from "./notes.js";
export interface SyncResult {
success: boolean;
message?: string;
conflicts?: ConflictInfo[];
}
export interface ConflictInfo {
type: 'content' | 'structure' | 'metadata';
filePath: string;
noteId: string;
fileModified: string;
noteModified: string;
description: string;
}
export interface SyncStats {
filesProcessed: number;
notesCreated: number;
notesUpdated: number;
filesCreated: number;
filesUpdated: number;
conflicts: number;
errors: number;
}
/**
* Bidirectional sync engine between Trilium notes and file system
*/
class FileSystemSync {
private isInitialized = false;
private syncInProgress = new Set<string>(); // Track ongoing syncs by mapping ID
constructor() {
this.setupEventHandlers();
}
/**
* Initialize the sync engine
*/
async init() {
if (this.isInitialized) {
return;
}
log.info('Initializing file system sync engine...');
// Initialize file system watcher
await fileSystemWatcher.init();
this.isInitialized = true;
log.info('File system sync engine initialized');
}
/**
* Shutdown the sync engine
*/
async shutdown() {
if (!this.isInitialized) {
return;
}
log.info('Shutting down file system sync engine...');
await fileSystemWatcher.shutdown();
this.isInitialized = false;
log.info('File system sync engine shutdown complete');
}
/**
* Setup event handlers for file changes and note changes
*/
private setupEventHandlers() {
// Handle file changes from watcher
eventService.subscribe('FILE_CHANGED', async ({ fileNoteMapping, mapping, fileContent, isNew }) => {
await this.handleFileChanged(fileNoteMapping, mapping, fileContent, isNew);
});
eventService.subscribe('FILE_DELETED', async ({ fileNoteMapping, mapping }) => {
await this.handleFileDeleted(fileNoteMapping, mapping);
});
// Handle note changes
eventService.subscribe(eventService.NOTE_CONTENT_CHANGE, async ({ entity: note }) => {
await this.handleNoteChanged(note as BNote);
});
eventService.subscribe(eventService.ENTITY_CHANGED, async ({ entityName, entity }) => {
if (entityName === 'notes') {
await this.handleNoteChanged(entity as BNote);
}
});
eventService.subscribe(eventService.ENTITY_DELETED, async ({ entityName, entityId }) => {
if (entityName === 'notes') {
await this.handleNoteDeleted(entityId);
}
});
}
/**
* Perform full sync for a specific mapping
*/
async fullSync(mappingId: string): Promise<SyncResult> {
const mapping = becca.fileSystemMappings[mappingId];
if (!mapping) {
return { success: false, message: `Mapping ${mappingId} not found` };
}
if (this.syncInProgress.has(mappingId)) {
return { success: false, message: 'Sync already in progress for this mapping' };
}
this.syncInProgress.add(mappingId);
const stats: SyncStats = {
filesProcessed: 0,
notesCreated: 0,
notesUpdated: 0,
filesCreated: 0,
filesUpdated: 0,
conflicts: 0,
errors: 0
};
try {
log.info(`Starting full sync for mapping ${mappingId}: ${mapping.filePath}`);
if (!await fs.pathExists(mapping.filePath)) {
throw new Error(`Path does not exist: ${mapping.filePath}`);
}
const pathStats = await fs.stat(mapping.filePath);
if (pathStats.isFile()) {
await this.syncSingleFile(mapping, mapping.filePath, stats);
} else if (pathStats.isDirectory()) {
await this.syncDirectory(mapping, mapping.filePath, stats);
}
// Reverse sync: export notes that don't have corresponding files
if (mapping.canSyncToDisk) {
await this.syncNotesToFiles(mapping, stats);
}
mapping.updateLastSyncTime();
mapping.clearSyncErrors();
log.info(`Full sync completed for mapping ${mappingId}. Stats: ${JSON.stringify(stats)}`);
return { success: true, message: `Sync completed successfully. ${stats.filesProcessed} files processed.` };
} catch (error) {
const errorMsg = `Full sync failed for mapping ${mappingId}: ${(error as Error).message}`;
log.error(errorMsg);
mapping.addSyncError(errorMsg);
stats.errors++;
return { success: false, message: errorMsg };
} finally {
this.syncInProgress.delete(mappingId);
}
}
/**
* Sync a single file
*/
private async syncSingleFile(mapping: BFileSystemMapping, filePath: string, stats: SyncStats) {
if (!fileSystemContentConverter.isSupportedFileType(filePath)) {
log.info(`DEBUG: Skipping unsupported file type: ${filePath}`);
return;
}
stats.filesProcessed++;
// Check if file note mapping exists
let fileNoteMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath);
if (fileNoteMapping) {
await this.syncExistingFile(mapping, fileNoteMapping, stats);
} else {
await this.syncNewFile(mapping, filePath, stats);
}
}
/**
* Sync a directory recursively
*/
private async syncDirectory(mapping: BFileSystemMapping, dirPath: string, stats: SyncStats) {
const entries = await fs.readdir(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
// Skip excluded patterns
if (this.isPathExcluded(fullPath, mapping)) {
continue;
}
if (entry.isFile()) {
await this.syncSingleFile(mapping, fullPath, stats);
} else if (entry.isDirectory() && mapping.includeSubtree) {
await this.syncDirectory(mapping, fullPath, stats);
}
}
}
/**
* Sync notes to files (reverse sync) - export notes that don't have corresponding files
*/
private async syncNotesToFiles(mapping: BFileSystemMapping, stats: SyncStats) {
const rootNote = mapping.getNote();
// Sync the root note itself if it's mapped to a file
const pathStats = await fs.stat(mapping.filePath);
if (pathStats.isFile()) {
await this.syncNoteToFile(mapping, rootNote, mapping.filePath, stats);
} else {
// Sync child notes in the subtree
await this.syncNoteSubtreeToFiles(mapping, rootNote, mapping.filePath, stats);
}
}
/**
* Sync a note subtree to files recursively
*/
private async syncNoteSubtreeToFiles(mapping: BFileSystemMapping, note: BNote, basePath: string, stats: SyncStats) {
for (const childBranch of note.children) {
const childNote = becca.notes[childBranch.noteId];
if (!childNote) continue;
// Skip system notes and other special notes
if (childNote.noteId.startsWith('_') || childNote.type === 'book') {
if (mapping.includeSubtree) {
// For book notes, recurse into children but don't create a file
await this.syncNoteSubtreeToFiles(mapping, childNote, basePath, stats);
}
continue;
}
// Generate file path for this note
const fileExtension = this.getFileExtensionForNote(childNote, mapping);
const fileName = this.sanitizeFileName(childNote.title) + fileExtension;
const filePath = path.join(basePath, fileName);
// Check if file already exists or has a mapping
const existingMapping = this.findFileNoteMappingByNote(mapping.mappingId, childNote.noteId);
if (!existingMapping && !await fs.pathExists(filePath)) {
// Note doesn't have a file mapping and file doesn't exist - create it
await this.syncNoteToFile(mapping, childNote, filePath, stats);
}
// Recurse into children if includeSubtree is enabled
if (mapping.includeSubtree && childNote.children.length > 0) {
const childDir = path.join(basePath, this.sanitizeFileName(childNote.title));
await fs.ensureDir(childDir);
await this.syncNoteSubtreeToFiles(mapping, childNote, childDir, stats);
}
}
}
/**
* Sync a single note to a file
*/
private async syncNoteToFile(mapping: BFileSystemMapping, note: BNote, filePath: string, stats: SyncStats) {
try {
// Convert note content to file format
const conversion = await fileSystemContentConverter.noteToFile(note, mapping, filePath, {
preserveAttributes: true,
includeFrontmatter: true
});
// Ensure directory exists
await fs.ensureDir(path.dirname(filePath));
// Write file
await fs.writeFile(filePath, conversion.content);
// Calculate file hash and get modification time
const fileStats = await fs.stat(filePath);
const fileHash = await this.calculateFileHash(filePath);
// Check if mapping already exists (safety check)
const existingMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath);
if (existingMapping) {
log.info(`File mapping already exists for ${filePath}, skipping creation`);
return;
}
// Create file note mapping
const fileNoteMapping = new BFileNoteMapping({
mappingId: mapping.mappingId,
noteId: note.noteId,
filePath,
fileHash,
fileModifiedTime: fileStats.mtime.toISOString(),
syncStatus: 'synced'
}).save();
stats.filesCreated++;
log.info(`Created file ${filePath} from note ${note.noteId}`);
} catch (error) {
log.error(`Error creating file from note ${note.noteId}: ${error}`);
mapping.addSyncError(`Error creating file from note ${note.noteId}: ${(error as Error).message}`);
stats.errors++;
}
}
/**
* Sync an existing file that has a note mapping
*/
private async syncExistingFile(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, stats: SyncStats) {
const filePath = fileNoteMapping.filePath;
if (!await fs.pathExists(filePath)) {
// File was deleted
if (mapping.canSyncFromDisk) {
await this.deleteNoteFromFileMapping(fileNoteMapping, stats);
}
return;
}
const fileStats = await fs.stat(filePath);
const fileHash = await this.calculateFileHash(filePath);
const fileModifiedTime = fileStats.mtime.toISOString();
const note = fileNoteMapping.note;
if (!note) {
log.info(`Note not found for file mapping: ${fileNoteMapping.noteId}`);
return;
}
const fileChanged = fileNoteMapping.hasFileChanged(fileHash, fileModifiedTime);
const noteChanged = fileNoteMapping.hasNoteChanged();
if (!fileChanged && !noteChanged) {
// No changes
return;
}
if (fileChanged && noteChanged) {
// Conflict - both changed
fileNoteMapping.markConflict();
stats.conflicts++;
log.info(`Conflict detected for ${filePath} - both file and note modified`);
return;
}
if (fileChanged && mapping.canSyncFromDisk) {
// Update note from file
await this.updateNoteFromFile(mapping, fileNoteMapping, fileHash, fileModifiedTime, stats);
} else if (noteChanged && mapping.canSyncToDisk) {
// Update file from note
await this.updateFileFromNote(mapping, fileNoteMapping, fileHash, fileModifiedTime, stats);
}
}
/**
* Sync a new file that doesn't have a note mapping
*/
private async syncNewFile(mapping: BFileSystemMapping, filePath: string, stats: SyncStats) {
if (!mapping.canSyncFromDisk) {
return;
}
try {
const fileStats = await fs.stat(filePath);
const fileHash = await this.calculateFileHash(filePath);
const fileModifiedTime = fileStats.mtime.toISOString();
// Create note from file
const note = await this.createNoteFromFile(mapping, filePath);
// Check if mapping already exists (safety check)
const existingMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath);
if (existingMapping) {
log.info(`File mapping already exists for ${filePath}, skipping creation`);
return;
}
// Create file note mapping
const fileNoteMapping = new BFileNoteMapping({
mappingId: mapping.mappingId,
noteId: note.noteId,
filePath,
fileHash,
fileModifiedTime,
syncStatus: 'synced'
}).save();
stats.notesCreated++;
log.info(`Created note ${note.noteId} from file ${filePath}`);
} catch (error) {
log.error(`Error creating note from file ${filePath}: ${error}`);
mapping.addSyncError(`Error creating note from file ${filePath}: ${(error as Error).message}`);
stats.errors++;
}
}
/**
* Create a new note from a file
*/
private async createNoteFromFile(mapping: BFileSystemMapping, filePath: string): Promise<BNote> {
const fileContent = await fs.readFile(filePath);
const fileName = path.basename(filePath, path.extname(filePath));
// Convert file content to note format
const conversion = await fileSystemContentConverter.fileToNote(fileContent, mapping, filePath, {
preserveAttributes: true,
includeFrontmatter: true
});
// Determine parent note
const parentNote = this.getParentNoteForFile(mapping, filePath);
// Create the note
const note = new BNote({
title: fileName,
type: conversion.type || 'text',
mime: conversion.mime || 'text/html'
}).save();
// Set content
note.setContent(conversion.content);
// Create branch
new BBranch({
noteId: note.noteId,
parentNoteId: parentNote.noteId
}).save();
// Add attributes from conversion
if (conversion.attributes) {
for (const attr of conversion.attributes) {
new BAttribute({
noteId: note.noteId,
type: attr.type,
name: attr.name,
value: attr.value,
isInheritable: attr.isInheritable || false
}).save();
}
}
return note;
}
/**
* Update note content from file
*/
private async updateNoteFromFile(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, fileHash: string, fileModifiedTime: string, stats: SyncStats) {
try {
const note = fileNoteMapping.getNote();
const fileContent = await fs.readFile(fileNoteMapping.filePath);
// Convert file content to note format
const conversion = await fileSystemContentConverter.fileToNote(fileContent, mapping, fileNoteMapping.filePath, {
preserveAttributes: true,
includeFrontmatter: true
});
// Update note content
note.setContent(conversion.content);
// Update note type/mime if they changed
if (conversion.type && conversion.type !== note.type) {
note.type = conversion.type as any;
note.save();
}
if (conversion.mime && conversion.mime !== note.mime) {
note.mime = conversion.mime;
note.save();
}
// Update attributes if needed
if (conversion.attributes) {
// Remove existing attributes that came from file
const existingAttrs = note.getOwnedAttributes();
for (const attr of existingAttrs) {
if (attr.name.startsWith('_fileSync_')) {
attr.markAsDeleted();
}
}
// Add new attributes
for (const attr of conversion.attributes) {
new BAttribute({
noteId: note.noteId,
type: attr.type,
name: attr.name,
value: attr.value,
isInheritable: attr.isInheritable || false
}).save();
}
}
fileNoteMapping.markSynced(fileHash, fileModifiedTime);
stats.notesUpdated++;
log.info(`DEBUG: Updated note ${note.noteId} from file ${fileNoteMapping.filePath}`);
} catch (error) {
log.error(`Error updating note from file ${fileNoteMapping.filePath}: ${error}`);
fileNoteMapping.markError();
mapping.addSyncError(`Error updating note from file: ${(error as Error).message}`);
stats.errors++;
}
}
/**
* Update file content from note
*/
private async updateFileFromNote(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, currentFileHash: string, currentModifiedTime: string, stats: SyncStats) {
try {
const note = fileNoteMapping.getNote();
// Convert note content to file format
const conversion = await fileSystemContentConverter.noteToFile(note, mapping, fileNoteMapping.filePath, {
preserveAttributes: true,
includeFrontmatter: true
});
// Ensure directory exists
await fs.ensureDir(path.dirname(fileNoteMapping.filePath));
// Write file
await fs.writeFile(fileNoteMapping.filePath, conversion.content);
// Update file note mapping with new file info
const newStats = await fs.stat(fileNoteMapping.filePath);
const newFileHash = await this.calculateFileHash(fileNoteMapping.filePath);
fileNoteMapping.markSynced(newFileHash, newStats.mtime.toISOString());
stats.filesUpdated++;
log.info(`DEBUG: Updated file ${fileNoteMapping.filePath} from note ${note.noteId}`);
} catch (error) {
log.error(`Error updating file from note ${fileNoteMapping.noteId}: ${error}`);
fileNoteMapping.markError();
mapping.addSyncError(`Error updating file from note: ${(error as Error).message}`);
stats.errors++;
}
}
/**
* Handle file change event from watcher
*/
private async handleFileChanged(fileNoteMapping: BFileNoteMapping, mapping: BFileSystemMapping, fileContent: Buffer, isNew: boolean) {
if (this.syncInProgress.has(mapping.mappingId)) {
return; // Skip if full sync in progress
}
const stats: SyncStats = {
filesProcessed: 1,
notesCreated: 0,
notesUpdated: 0,
filesCreated: 0,
filesUpdated: 0,
conflicts: 0,
errors: 0
};
if (isNew) {
await this.syncNewFile(mapping, fileNoteMapping.filePath, stats);
} else {
const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex');
const fileStats = await fs.stat(fileNoteMapping.filePath);
const fileModifiedTime = fileStats.mtime.toISOString();
await this.syncExistingFile(mapping, fileNoteMapping, stats);
}
}
/**
* Handle file deletion event from watcher
*/
private async handleFileDeleted(fileNoteMapping: BFileNoteMapping, mapping: BFileSystemMapping) {
if (this.syncInProgress.has(mapping.mappingId)) {
return; // Skip if full sync in progress
}
const stats: SyncStats = {
filesProcessed: 0,
notesCreated: 0,
notesUpdated: 0,
filesCreated: 0,
filesUpdated: 0,
conflicts: 0,
errors: 0
};
await this.deleteNoteFromFileMapping(fileNoteMapping, stats);
}
/**
* Handle note change event
*/
private async handleNoteChanged(note: BNote) {
// Find all file mappings for this note
const fileMappings = this.findFileNoteMappingsByNote(note.noteId);
for (const fileMapping of fileMappings) {
const mapping = fileMapping.mapping;
if (!mapping || !mapping.canSyncToDisk || this.syncInProgress.has(mapping.mappingId)) {
continue;
}
// Check if note was actually modified since last sync
if (!fileMapping.hasNoteChanged()) {
continue;
}
const stats: SyncStats = {
filesProcessed: 0,
notesCreated: 0,
notesUpdated: 0,
filesCreated: 0,
filesUpdated: 0,
conflicts: 0,
errors: 0
};
// Check for conflicts
if (await fs.pathExists(fileMapping.filePath)) {
const fileStats = await fs.stat(fileMapping.filePath);
const fileHash = await this.calculateFileHash(fileMapping.filePath);
const fileModifiedTime = fileStats.mtime.toISOString();
if (fileMapping.hasFileChanged(fileHash, fileModifiedTime)) {
// Conflict
fileMapping.markConflict();
log.info(`Conflict detected for note ${note.noteId} - both file and note modified`);
continue;
}
}
// Update file from note
const currentFileHash = await this.calculateFileHash(fileMapping.filePath);
const currentModifiedTime = (await fs.stat(fileMapping.filePath)).mtime.toISOString();
await this.updateFileFromNote(mapping, fileMapping, currentFileHash, currentModifiedTime, stats);
}
}
/**
* Handle note deletion event
*/
private async handleNoteDeleted(noteId: string) {
// Find all file mappings for this note
const fileMappings = this.findFileNoteMappingsByNote(noteId);
for (const fileMapping of fileMappings) {
const mapping = fileMapping.mapping;
if (!mapping || !mapping.canSyncToDisk || this.syncInProgress.has(mapping.mappingId)) {
continue;
}
try {
// Delete the file
if (await fs.pathExists(fileMapping.filePath)) {
await fs.remove(fileMapping.filePath);
log.info(`Deleted file ${fileMapping.filePath} for deleted note ${noteId}`);
}
// Delete the mapping
fileMapping.markAsDeleted();
} catch (error) {
log.error(`Error deleting file for note ${noteId}: ${error}`);
mapping.addSyncError(`Error deleting file: ${(error as Error).message}`);
}
}
}
/**
* Delete note when file is deleted
*/
private async deleteNoteFromFileMapping(fileNoteMapping: BFileNoteMapping, stats: SyncStats) {
try {
const note = fileNoteMapping.note;
if (note) {
note.deleteNote();
log.info(`Deleted note ${note.noteId} for deleted file ${fileNoteMapping.filePath}`);
}
// Delete the mapping
fileNoteMapping.markAsDeleted();
} catch (error) {
log.error(`Error deleting note for file ${fileNoteMapping.filePath}: ${error}`);
stats.errors++;
}
}
/**
* Get parent note for a file based on mapping configuration
*/
private getParentNoteForFile(mapping: BFileSystemMapping, filePath: string): BNote {
const mappedNote = mapping.getNote();
if (!mapping.preserveHierarchy || !mapping.includeSubtree) {
return mappedNote;
}
// Calculate relative path from mapping root
const relativePath = path.relative(mapping.filePath, path.dirname(filePath));
if (!relativePath || relativePath === '.') {
return mappedNote;
}
// Create directory structure as notes
const pathParts = relativePath.split(path.sep);
let currentParent = mappedNote;
for (const part of pathParts) {
if (!part) continue;
// Look for existing child note with this name
let childNote = currentParent.children.find(child => child.title === part);
if (!childNote) {
// Create new note for this directory
childNote = new BNote({
title: part,
type: 'text',
mime: 'text/html'
}).save();
childNote.setContent('<p>Directory note</p>');
// Create branch (notePosition will be auto-calculated)
new BBranch({
noteId: childNote.noteId,
parentNoteId: currentParent.noteId
}).save();
}
currentParent = childNote;
}
return currentParent;
}
/**
* Calculate SHA256 hash of a file
*/
private async calculateFileHash(filePath: string): Promise<string> {
const content = await fs.readFile(filePath);
return crypto.createHash('sha256').update(content).digest('hex');
}
/**
* Check if a path should be excluded based on mapping patterns
*/
private isPathExcluded(filePath: string, mapping: BFileSystemMapping): boolean {
if (!mapping.excludePatterns) {
return false;
}
const normalizedPath = path.normalize(filePath);
const basename = path.basename(normalizedPath);
for (const pattern of mapping.excludePatterns) {
if (typeof pattern === 'string') {
// Simple string matching
if (normalizedPath.includes(pattern) || basename.includes(pattern)) {
return true;
}
} else if (pattern instanceof RegExp) {
// Regex pattern
if (pattern.test(normalizedPath) || pattern.test(basename)) {
return true;
}
}
}
return false;
}
/**
* Find file note mapping by file path
*/
private findFileNoteMappingByPath(mappingId: string, filePath: string): BFileNoteMapping | null {
const normalizedPath = path.normalize(filePath);
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
if (mapping.mappingId === mappingId && path.normalize(mapping.filePath) === normalizedPath) {
return mapping;
}
}
return null;
}
/**
* Find all file note mappings for a note
*/
private findFileNoteMappingsByNote(noteId: string): BFileNoteMapping[] {
const mappings: BFileNoteMapping[] = [];
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
if (mapping.noteId === noteId) {
mappings.push(mapping);
}
}
return mappings;
}
/**
* Find file note mapping by note ID within a specific mapping
*/
private findFileNoteMappingByNote(mappingId: string, noteId: string): BFileNoteMapping | null {
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
if (mapping.mappingId === mappingId && mapping.noteId === noteId) {
return mapping;
}
}
return null;
}
/**
* Get appropriate file extension for a note based on its type and mapping configuration
*/
private getFileExtensionForNote(note: BNote, mapping: BFileSystemMapping): string {
const contentFormat = mapping.contentFormat;
if (contentFormat === 'markdown' || (contentFormat === 'auto' && note.type === 'text')) {
return '.md';
} else if (contentFormat === 'html' || (contentFormat === 'auto' && note.type === 'text' && note.mime === 'text/html')) {
return '.html';
} else if (note.type === 'code') {
// Map MIME types to file extensions
const mimeToExt: Record<string, string> = {
'application/javascript': '.js',
'text/javascript': '.js',
'application/typescript': '.ts',
'text/typescript': '.ts',
'application/json': '.json',
'text/css': '.css',
'text/x-python': '.py',
'text/x-java': '.java',
'text/x-csharp': '.cs',
'text/x-sql': '.sql',
'text/x-sh': '.sh',
'text/x-yaml': '.yaml',
'application/xml': '.xml',
'text/xml': '.xml'
};
return mimeToExt[note.mime] || '.txt';
} else if (note.type === 'image') {
const mimeToExt: Record<string, string> = {
'image/png': '.png',
'image/jpeg': '.jpg',
'image/gif': '.gif',
'image/svg+xml': '.svg'
};
return mimeToExt[note.mime] || '.png';
} else {
return '.txt';
}
}
/**
* Sanitize file name to be safe for file system
*/
private sanitizeFileName(fileName: string): string {
// Replace invalid characters with underscores
return fileName
.replace(/[<>:"/\\|?*]/g, '_')
.replace(/\s+/g, '_')
.replace(/_{2,}/g, '_')
.replace(/^_+|_+$/g, '')
.substring(0, 100); // Limit length
}
/**
* Get sync status for all mappings
*/
getSyncStatus() {
const status: Record<string, any> = {};
for (const mapping of Object.values(becca.fileSystemMappings || {})) {
const fileMappings = Object.values(becca.fileNoteMappings || {})
.filter(fm => fm.mappingId === mapping.mappingId);
const conflicts = fileMappings.filter(fm => fm.syncStatus === 'conflict').length;
const pending = fileMappings.filter(fm => fm.syncStatus === 'pending').length;
const errors = fileMappings.filter(fm => fm.syncStatus === 'error').length;
status[mapping.mappingId] = {
filePath: mapping.filePath,
isActive: mapping.isActive,
syncDirection: mapping.syncDirection,
fileCount: fileMappings.length,
conflicts,
pending,
errors,
lastSyncTime: mapping.lastSyncTime,
syncErrors: mapping.syncErrors,
isRunning: this.syncInProgress.has(mapping.mappingId)
};
}
return status;
}
}
// Create singleton instance
const fileSystemSync = new FileSystemSync();
export default fileSystemSync;

View File

@@ -0,0 +1,129 @@
"use strict";
import log from "./log.js";
import fileSystemSync from "./file_system_sync.js";
import eventService from "./events.js";
import optionService from "./options.js";
/**
* Initialization service for file system sync functionality
*/
class FileSystemSyncInit {
private initialized = false;
/**
* Initialize file system sync if enabled
*/
async init() {
if (this.initialized) {
return;
}
try {
// Check if file system sync is enabled
const isEnabled = optionService.getOption('fileSystemSyncEnabled') === 'true';
if (!isEnabled) {
log.info('File system sync is disabled');
return;
}
log.info('Initializing file system sync...');
// Initialize the sync engine
await fileSystemSync.init();
this.initialized = true;
log.info('File system sync initialized successfully');
} catch (error) {
log.error(`Failed to initialize file system sync: ${error}`);
throw error;
}
}
/**
* Shutdown file system sync
*/
async shutdown() {
if (!this.initialized) {
return;
}
try {
log.info('Shutting down file system sync...');
await fileSystemSync.shutdown();
this.initialized = false;
log.info('File system sync shutdown complete');
} catch (error) {
log.error(`Error shutting down file system sync: ${error}`);
}
}
/**
* Check if file system sync is initialized
*/
isInitialized(): boolean {
return this.initialized;
}
/**
* Get sync status
*/
getStatus() {
if (!this.initialized) {
return { enabled: false, initialized: false };
}
return {
enabled: true,
initialized: true,
status: fileSystemSync.getSyncStatus()
};
}
/**
* Enable file system sync
*/
async enable() {
optionService.setOption('fileSystemSyncEnabled', 'true');
if (!this.initialized) {
await this.init();
}
log.info('File system sync enabled');
}
/**
* Disable file system sync
*/
async disable() {
optionService.setOption('fileSystemSyncEnabled', 'false');
if (this.initialized) {
await this.shutdown();
}
log.info('File system sync disabled');
}
/**
* Perform full sync for a specific mapping
*/
async fullSync(mappingId: string) {
if (!this.initialized) {
throw new Error('File system sync is not initialized');
}
return await fileSystemSync.fullSync(mappingId);
}
}
// Create singleton instance
const fileSystemSyncInit = new FileSystemSyncInit();
export default fileSystemSyncInit;

View File

@@ -0,0 +1,457 @@
"use strict";
import chokidar from "chokidar";
import path from "path";
import fs from "fs-extra";
import crypto from "crypto";
import debounce from "debounce";
import log from "./log.js";
import becca from "../becca/becca.js";
import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js";
import BFileNoteMapping from "../becca/entities/bfile_note_mapping.js";
import eventService from "./events.js";
import { newEntityId } from "./utils.js";
import type { FSWatcher } from "chokidar";
interface WatchedMapping {
mapping: BFileSystemMapping;
watcher: FSWatcher;
}
interface FileChangeEvent {
type: 'add' | 'change' | 'unlink';
filePath: string;
mappingId: string;
stats?: fs.Stats;
}
class FileSystemWatcher {
private watchers: Map<string, WatchedMapping> = new Map();
private syncQueue: FileChangeEvent[] = [];
private isProcessing = false;
// Debounced sync to batch multiple file changes
private processSyncQueue = debounce(this._processSyncQueue.bind(this), 500);
constructor() {
// Subscribe to entity changes to watch for new/updated/deleted mappings
eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) => {
if (entityName === 'file_system_mappings') {
this.addWatcher(entity as BFileSystemMapping);
}
});
eventService.subscribe(eventService.ENTITY_CHANGED, ({ entityName, entity }) => {
if (entityName === 'file_system_mappings') {
this.updateWatcher(entity as BFileSystemMapping);
}
});
eventService.subscribe(eventService.ENTITY_DELETED, ({ entityName, entityId }) => {
if (entityName === 'file_system_mappings') {
this.removeWatcher(entityId);
}
});
}
/**
* Initialize the file system watcher by setting up watchers for all active mappings
*/
async init() {
log.info('Initializing file system watcher...');
try {
const mappings = Object.values(becca.fileSystemMappings || {});
for (const mapping of mappings) {
if (mapping.isActive && mapping.canSyncFromDisk) {
await this.addWatcher(mapping);
}
}
log.info(`File system watcher initialized with ${this.watchers.size} active mappings`);
} catch (error) {
log.error(`Failed to initialize file system watcher: ${error}`);
}
}
/**
* Shutdown all watchers
*/
async shutdown() {
log.info('Shutting down file system watcher...');
for (const [mappingId, { watcher }] of this.watchers) {
await watcher.close();
}
this.watchers.clear();
log.info('File system watcher shutdown complete');
}
/**
* Add a new file system watcher for a mapping
*/
private async addWatcher(mapping: BFileSystemMapping) {
if (this.watchers.has(mapping.mappingId)) {
await this.removeWatcher(mapping.mappingId);
}
if (!mapping.isActive || !mapping.canSyncFromDisk) {
return;
}
try {
// Check if the file path exists
if (!await fs.pathExists(mapping.filePath)) {
log.info(`File path does not exist for mapping ${mapping.mappingId}: ${mapping.filePath}`);
mapping.addSyncError(`File path does not exist: ${mapping.filePath}`);
return;
}
const stats = await fs.stat(mapping.filePath);
const watchPath = stats.isDirectory() ? mapping.filePath : path.dirname(mapping.filePath);
const watcher = chokidar.watch(watchPath, {
persistent: true,
ignoreInitial: true,
followSymlinks: false,
depth: mapping.includeSubtree ? undefined : 0,
ignored: this.buildIgnorePatterns(mapping)
});
watcher.on('add', (filePath, stats) => {
this.queueFileChange('add', filePath, mapping.mappingId, stats);
});
watcher.on('change', (filePath, stats) => {
this.queueFileChange('change', filePath, mapping.mappingId, stats);
});
watcher.on('unlink', (filePath) => {
this.queueFileChange('unlink', filePath, mapping.mappingId);
});
watcher.on('error', (error) => {
log.error(`File watcher error for mapping ${mapping.mappingId}: ${error}`);
if (error && typeof error === "object" && "message" in error && typeof error.message === 'string') {
mapping.addSyncError(`Watcher error: ${error.message}`);
}
});
watcher.on('ready', () => {
log.info(`File watcher ready for mapping ${mapping.mappingId}: ${mapping.filePath}`);
});
this.watchers.set(mapping.mappingId, { mapping, watcher });
} catch (error) {
log.error(`Failed to create file watcher for mapping ${mapping.mappingId}: ${error}`);
mapping.addSyncError(`Failed to create watcher: ${(error as Error).message}`);
}
}
/**
* Update an existing watcher (remove and re-add)
*/
private async updateWatcher(mapping: BFileSystemMapping) {
await this.addWatcher(mapping);
}
/**
* Remove a file system watcher
*/
private async removeWatcher(mappingId: string) {
const watchedMapping = this.watchers.get(mappingId);
if (watchedMapping) {
await watchedMapping.watcher.close();
this.watchers.delete(mappingId);
log.info(`Removed file watcher for mapping ${mappingId}`);
}
}
/**
* Build ignore patterns for chokidar based on mapping configuration
*/
private buildIgnorePatterns(mapping: BFileSystemMapping): (string | RegExp)[] {
const patterns: (string | RegExp)[] = [
// Always ignore common temp/system files
/^\./, // Hidden files
/\.tmp$/,
/\.temp$/,
/~$/, // Backup files
/\.swp$/, // Vim swap files
/\.DS_Store$/, // macOS
/Thumbs\.db$/ // Windows
];
// Add user-defined exclude patterns
if (mapping.excludePatterns) {
patterns.push(...mapping.excludePatterns);
}
return patterns;
}
/**
* Queue a file change event for processing
*/
private queueFileChange(type: 'add' | 'change' | 'unlink', filePath: string, mappingId: string, stats?: fs.Stats) {
this.syncQueue.push({
type,
filePath: path.normalize(filePath),
mappingId,
stats
});
// Trigger debounced processing
this.processSyncQueue();
}
/**
* Process the sync queue (called after debounce delay)
*/
private async _processSyncQueue() {
if (this.isProcessing || this.syncQueue.length === 0) {
return;
}
this.isProcessing = true;
const eventsToProcess = [...this.syncQueue];
this.syncQueue = [];
try {
// Group events by file path to handle multiple events for the same file
const eventMap = new Map<string, FileChangeEvent>();
for (const event of eventsToProcess) {
const key = `${event.mappingId}:${event.filePath}`;
eventMap.set(key, event); // Latest event wins
}
// Process each unique file change
for (const event of eventMap.values()) {
await this.processFileChange(event);
}
} catch (error) {
log.error(`Error processing file change queue: ${error}`);
} finally {
this.isProcessing = false;
// If more events were queued while processing, schedule another run
if (this.syncQueue.length > 0) {
this.processSyncQueue();
}
}
}
/**
* Process a single file change event
*/
private async processFileChange(event: FileChangeEvent) {
try {
const mapping = becca.fileSystemMappings[event.mappingId];
if (!mapping || !mapping.isActive || !mapping.canSyncFromDisk) {
return;
}
log.info(`DEBUG: Processing file ${event.type}: ${event.filePath} for mapping ${event.mappingId}`);
switch (event.type) {
case 'add':
case 'change':
await this.handleFileAddOrChange(event, mapping);
break;
case 'unlink':
await this.handleFileDelete(event, mapping);
break;
}
} catch (error) {
log.error(`Error processing file change for ${event.filePath}: ${error}`);
const mapping = becca.fileSystemMappings[event.mappingId];
if (mapping) {
mapping.addSyncError(`Error processing ${event.filePath}: ${(error as Error).message}`);
}
}
}
/**
* Handle file addition or modification
*/
private async handleFileAddOrChange(event: FileChangeEvent, mapping: BFileSystemMapping) {
if (!await fs.pathExists(event.filePath)) {
return; // File was deleted between queuing and processing
}
const stats = event.stats || await fs.stat(event.filePath);
if (stats.isDirectory()) {
return; // We only sync files, not directories
}
// Calculate file hash for change detection
const fileContent = await fs.readFile(event.filePath);
const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex');
const fileModifiedTime = stats.mtime.toISOString();
// Find existing file note mapping
let fileNoteMapping: BFileNoteMapping | null = null;
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
if (mapping.mappingId === event.mappingId && path.normalize(mapping.filePath) === path.normalize(event.filePath)) {
fileNoteMapping = mapping;
break;
}
}
// Check if file actually changed
if (fileNoteMapping && !fileNoteMapping.hasFileChanged(fileHash, fileModifiedTime)) {
return; // No actual change
}
if (fileNoteMapping) {
// Update existing mapping
if (fileNoteMapping.hasNoteChanged()) {
// Both file and note changed - mark as conflict
fileNoteMapping.markConflict();
log.info(`Conflict detected for ${event.filePath} - both file and note modified`);
return;
}
fileNoteMapping.markPending();
} else {
// Double-check if mapping exists before creating (race condition protection)
const existingCheck = Object.values(becca.fileNoteMappings || {}).find(m =>
m.mappingId === event.mappingId && path.normalize(m.filePath) === path.normalize(event.filePath)
);
if (existingCheck) {
log.info(`File mapping already exists for ${event.filePath}, using existing mapping`);
fileNoteMapping = existingCheck;
fileNoteMapping.markPending();
} else {
// Create new file note mapping
try {
fileNoteMapping = new BFileNoteMapping({
mappingId: event.mappingId,
noteId: '', // Will be determined by sync service
filePath: event.filePath,
fileHash,
fileModifiedTime,
syncStatus: 'pending'
}).save();
} catch (error: any) {
if (error.code === 'SQLITE_CONSTRAINT_UNIQUE') {
log.info(`File mapping constraint violation for ${event.filePath}, trying to find existing mapping`);
// Try to find the mapping again - it might have been created by another process
fileNoteMapping = Object.values(becca.fileNoteMappings || {}).find(m =>
m.mappingId === event.mappingId && path.normalize(m.filePath) === path.normalize(event.filePath)
) || null;
if (!fileNoteMapping) {
throw error; // Re-throw if we still can't find it
}
} else {
throw error;
}
}
}
}
// Emit event for sync service to handle
eventService.emit('FILE_CHANGED', {
fileNoteMapping,
mapping,
fileContent,
isNew: event.type === 'add'
});
}
/**
* Handle file deletion
*/
private async handleFileDelete(event: FileChangeEvent, mapping: BFileSystemMapping) {
// Find existing file note mapping
let fileNoteMapping: BFileNoteMapping | null = null;
for (const mappingObj of Object.values(becca.fileNoteMappings || {})) {
if (mappingObj.mappingId === event.mappingId && mappingObj.filePath === event.filePath) {
fileNoteMapping = mappingObj;
break;
}
}
if (fileNoteMapping) {
// Emit event for sync service to handle deletion
eventService.emit('FILE_DELETED', {
fileNoteMapping,
mapping
});
}
}
/**
* Get status of all watchers
*/
getWatcherStatus() {
const status: Record<string, any> = {};
for (const [mappingId, { mapping, watcher }] of this.watchers) {
status[mappingId] = {
filePath: mapping.filePath,
isActive: mapping.isActive,
watchedPaths: watcher.getWatched(),
syncDirection: mapping.syncDirection
};
}
return status;
}
/**
* Force a full sync for a specific mapping
*/
async forceSyncMapping(mappingId: string) {
const mapping = becca.fileSystemMappings[mappingId];
if (!mapping) {
throw new Error(`Mapping ${mappingId} not found`);
}
log.info(`Force syncing mapping ${mappingId}: ${mapping.filePath}`);
if (await fs.pathExists(mapping.filePath)) {
const stats = await fs.stat(mapping.filePath);
if (stats.isFile()) {
await this.queueFileChange('change', mapping.filePath, mappingId, stats);
} else if (stats.isDirectory() && mapping.includeSubtree) {
// Scan directory for files
await this.scanDirectoryForFiles(mapping.filePath, mapping);
}
}
}
/**
* Recursively scan directory for files and queue them for sync
*/
private async scanDirectoryForFiles(dirPath: string, mapping: BFileSystemMapping) {
try {
const entries = await fs.readdir(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
if (entry.isFile()) {
const stats = await fs.stat(fullPath);
this.queueFileChange('change', fullPath, mapping.mappingId, stats);
} else if (entry.isDirectory() && mapping.includeSubtree) {
await this.scanDirectoryForFiles(fullPath, mapping);
}
}
} catch (error) {
log.error(`Error scanning directory ${dirPath}: ${error}`);
mapping.addSyncError(`Error scanning directory: ${(error as Error).message}`);
}
}
}
// Create singleton instance
const fileSystemWatcher = new FileSystemWatcher();
export default fileSystemWatcher;

View File

@@ -211,6 +211,9 @@ const defaultOptions: DefaultOption[] = [
{ name: "aiTemperature", value: "0.7", isSynced: true },
{ name: "aiSystemPrompt", value: "", isSynced: true },
{ name: "aiSelectedProvider", value: "openai", isSynced: true },
// File system sync options
{ name: "fileSystemSyncEnabled", value: "false", isSynced: false },
];
/**

View File

@@ -123,6 +123,7 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions<KeyboardActi
/** Whether keyboard auto-completion for notes is triggered when typing `@` in text notes (attribute editing is not affected). */
textNoteCompletionEnabled: boolean;
backgroundEffects: boolean;
fileSystemSyncEnabled: boolean;
// Share settings
redirectBareDomain: boolean;