mirror of
https://github.com/zadam/trilium.git
synced 2025-10-26 07:46:30 +01:00
Compare commits
9 Commits
74a805056b
...
feature/fs
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f6bc65471d | ||
|
|
eb07d4b0ed | ||
|
|
2c096f3080 | ||
|
|
bac95c97e5 | ||
|
|
fe6daac979 | ||
|
|
770281214b | ||
|
|
15bd5aa4e4 | ||
|
|
3da6838395 | ||
|
|
16cdd9e137 |
2
.github/instructions/nx.instructions.md
vendored
2
.github/instructions/nx.instructions.md
vendored
@@ -4,7 +4,7 @@ applyTo: '**'
|
||||
|
||||
// This file is automatically generated by Nx Console
|
||||
|
||||
You are in an nx workspace using Nx 21.3.5 and pnpm as the package manager.
|
||||
You are in an nx workspace using Nx 21.3.7 and pnpm as the package manager.
|
||||
|
||||
You have access to the Nx MCP server and the tools it provides. Use them. Follow these guidelines in order to best help the user:
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ async function processEntityChanges(entityChanges: EntityChange[]) {
|
||||
loadResults.addOption(attributeEntity.name);
|
||||
} else if (ec.entityName === "attachments") {
|
||||
processAttachment(loadResults, ec);
|
||||
} else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens") {
|
||||
} else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "file_note_mappings" || ec.entityName === "file_system_mappings") {
|
||||
// NOOP - these entities are handled at the backend level and don't require frontend processing
|
||||
} else {
|
||||
throw new Error(`Unknown entityName '${ec.entityName}'`);
|
||||
|
||||
@@ -2201,3 +2201,189 @@ footer.file-footer button {
|
||||
content: "\ec24";
|
||||
transform: rotate(180deg);
|
||||
}
|
||||
|
||||
/* File System Sync Modal Styles */
|
||||
.mapping-modal {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 1050;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.mapping-modal .modal-backdrop {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
z-index: 1051;
|
||||
}
|
||||
|
||||
.mapping-modal .modal-content {
|
||||
position: relative;
|
||||
background: var(--main-background-color);
|
||||
border: 1px solid var(--main-border-color);
|
||||
border-radius: 5px;
|
||||
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
|
||||
width: 90%;
|
||||
max-width: 600px;
|
||||
max-height: 80vh;
|
||||
overflow-y: auto;
|
||||
z-index: 1052;
|
||||
}
|
||||
|
||||
.mapping-modal .modal-header {
|
||||
display: flex;
|
||||
justify-content: between;
|
||||
align-items: center;
|
||||
padding: 1rem;
|
||||
border-bottom: 1px solid var(--main-border-color);
|
||||
}
|
||||
|
||||
.mapping-modal .modal-title {
|
||||
margin: 0;
|
||||
font-size: 1.25rem;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.mapping-modal .modal-close {
|
||||
background: none;
|
||||
border: none;
|
||||
font-size: 1.5rem;
|
||||
cursor: pointer;
|
||||
color: var(--muted-text-color);
|
||||
padding: 0;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.mapping-modal .modal-close:hover {
|
||||
color: var(--main-text-color);
|
||||
}
|
||||
|
||||
.mapping-modal .modal-body {
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
.mapping-modal .modal-footer {
|
||||
padding: 1rem;
|
||||
border-top: 1px solid var(--main-border-color);
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
/* File System Sync Mapping Cards */
|
||||
.mapping-item.card {
|
||||
border: 1px solid var(--main-border-color);
|
||||
border-radius: 5px;
|
||||
transition: box-shadow 0.2s ease;
|
||||
}
|
||||
|
||||
.mapping-item.card:hover {
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.mapping-item .mapping-path {
|
||||
font-family: monospace;
|
||||
font-size: 0.9rem;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.mapping-item .mapping-details {
|
||||
font-size: 0.85rem;
|
||||
margin-top: 0.25rem;
|
||||
}
|
||||
|
||||
.mapping-item .mapping-status {
|
||||
margin-top: 0.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.mapping-item .mapping-actions {
|
||||
display: flex;
|
||||
gap: 0.25rem;
|
||||
}
|
||||
|
||||
.mapping-item .mapping-actions .btn {
|
||||
padding: 0.25rem 0.5rem;
|
||||
}
|
||||
|
||||
/* Status Badges */
|
||||
.status-badge.badge {
|
||||
font-size: 0.75rem;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.status-badge.badge-success {
|
||||
background-color: #28a745;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.status-badge.badge-danger {
|
||||
background-color: #dc3545;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.status-badge.badge-secondary {
|
||||
background-color: #6c757d;
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* Path Validation Styles */
|
||||
.path-validation-result {
|
||||
margin-top: 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.path-validation-result .text-success {
|
||||
color: #28a745;
|
||||
}
|
||||
|
||||
.path-validation-result .text-warning {
|
||||
color: #ffc107;
|
||||
}
|
||||
|
||||
.path-validation-result .text-danger {
|
||||
color: #dc3545;
|
||||
}
|
||||
|
||||
/* Sync Status Section */
|
||||
.sync-status-container {
|
||||
margin: 1rem 0;
|
||||
padding: 1rem;
|
||||
background: var(--accented-background-color);
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
.sync-status-info .status-item,
|
||||
.sync-status-info .active-mappings-count {
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
/* Form Enhancements */
|
||||
.mapping-form .form-group {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.mapping-form .subtree-options {
|
||||
margin-left: 1.5rem;
|
||||
}
|
||||
|
||||
.mapping-form .help-block {
|
||||
font-size: 0.875rem;
|
||||
color: var(--muted-text-color);
|
||||
margin-top: 0.25rem;
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import RevisionSnapshotsLimitOptions from "./options/other/revision_snapshots_li
|
||||
import NetworkConnectionsOptions from "./options/other/network_connections.js";
|
||||
import HtmlImportTagsOptions from "./options/other/html_import_tags.js";
|
||||
import AdvancedSyncOptions from "./options/advanced/sync.js";
|
||||
import FileSystemSyncOptions from "./options/advanced/file_system_sync.js";
|
||||
import DatabaseIntegrityCheckOptions from "./options/advanced/database_integrity_check.js";
|
||||
import VacuumDatabaseOptions from "./options/advanced/vacuum_database.js";
|
||||
import DatabaseAnonymizationOptions from "./options/advanced/database_anonymization.js";
|
||||
@@ -138,6 +139,7 @@ const CONTENT_WIDGETS: Record<OptionPages | "_backendLog", (typeof NoteContextAw
|
||||
],
|
||||
_optionsAdvanced: [
|
||||
AdvancedSyncOptions,
|
||||
FileSystemSyncOptions,
|
||||
DatabaseIntegrityCheckOptions,
|
||||
DatabaseAnonymizationOptions,
|
||||
VacuumDatabaseOptions
|
||||
|
||||
@@ -0,0 +1,659 @@
|
||||
import OptionsWidget from "../options_widget.js";
|
||||
import server from "../../../../services/server.js";
|
||||
import toastService from "../../../../services/toast.js";
|
||||
import noteAutocompleteService from "../../../../services/note_autocomplete.js";
|
||||
import type { OptionMap } from "@triliumnext/commons";
|
||||
import type { Suggestion } from "../../../../services/note_autocomplete.js";
|
||||
|
||||
interface FileSystemMapping {
|
||||
mappingId: string;
|
||||
noteId: string;
|
||||
filePath: string;
|
||||
syncDirection: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
|
||||
isActive: boolean;
|
||||
includeSubtree: boolean;
|
||||
preserveHierarchy: boolean;
|
||||
contentFormat: 'auto' | 'markdown' | 'html' | 'raw';
|
||||
excludePatterns: string[] | null;
|
||||
lastSyncTime: string | null;
|
||||
syncErrors: string[] | null;
|
||||
dateCreated: string;
|
||||
dateModified: string;
|
||||
}
|
||||
|
||||
interface SyncStatus {
|
||||
enabled: boolean;
|
||||
initialized: boolean;
|
||||
status?: Record<string, any>;
|
||||
}
|
||||
|
||||
// API Request/Response interfaces
|
||||
interface PathValidationRequest {
|
||||
filePath: string;
|
||||
}
|
||||
|
||||
interface PathValidationResponse {
|
||||
exists: boolean;
|
||||
stats?: {
|
||||
isDirectory: boolean;
|
||||
size: number;
|
||||
modified: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface CreateMappingRequest {
|
||||
noteId: string;
|
||||
filePath: string;
|
||||
syncDirection: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
|
||||
contentFormat: 'auto' | 'markdown' | 'html' | 'raw';
|
||||
includeSubtree: boolean;
|
||||
preserveHierarchy: boolean;
|
||||
excludePatterns: string[] | null;
|
||||
}
|
||||
|
||||
interface UpdateMappingRequest extends CreateMappingRequest {}
|
||||
|
||||
interface SyncMappingResponse {
|
||||
success: boolean;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
interface ApiResponse {
|
||||
success?: boolean;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const TPL = /*html*/`
|
||||
<style>
|
||||
.modal-hidden {
|
||||
display: none !important;
|
||||
}
|
||||
.modal-visible {
|
||||
display: flex !important;
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0, 0, 0, 0.5);
|
||||
z-index: 1050;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
.modal-content {
|
||||
background: white;
|
||||
border-radius: 0.5rem;
|
||||
max-width: 600px;
|
||||
width: 90%;
|
||||
max-height: 90%;
|
||||
overflow-y: auto;
|
||||
}
|
||||
</style>
|
||||
<div class="options-section">
|
||||
<h4>File System Sync</h4>
|
||||
|
||||
<div class="form-group">
|
||||
<label>
|
||||
<input type="checkbox" class="file-sync-enabled-checkbox">
|
||||
Enable file system synchronization
|
||||
</label>
|
||||
<div class="help-block">
|
||||
Allows bidirectional synchronization between Trilium notes and files on your local file system.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="file-sync-controls" style="display: none;">
|
||||
<div class="alert alert-info">
|
||||
<strong>Note:</strong> File system sync creates mappings between notes and files/directories.
|
||||
Changes in either location will be synchronized automatically when enabled.
|
||||
</div>
|
||||
|
||||
<div class="sync-status-container">
|
||||
<h5>Sync Status</h5>
|
||||
<div class="sync-status-info">
|
||||
<div class="status-item">
|
||||
<strong>Status:</strong> <span class="sync-status-text">Loading...</span>
|
||||
</div>
|
||||
<div class="active-mappings-count">
|
||||
<strong>Active Mappings:</strong> <span class="mappings-count">0</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mappings-section">
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h5>File System Mappings</h5>
|
||||
<button class="btn btn-primary btn-sm create-mapping-button">
|
||||
<i class="bx bx-plus"></i> Create Mapping
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="mappings-list">
|
||||
<!-- Mappings will be populated here -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="sync-actions mt-3">
|
||||
<button class="btn btn-secondary refresh-status-button">
|
||||
<i class="bx bx-refresh"></i> Refresh Status
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Create/Edit Mapping Modal -->
|
||||
<div class="mapping-modal modal-hidden">
|
||||
<div class="modal-backdrop"></div>
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Create File System Mapping</h5>
|
||||
<button type="button" class="modal-close" aria-label="Close">
|
||||
<i class="bx bx-x"></i>
|
||||
</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<form class="mapping-form">
|
||||
<div class="form-group">
|
||||
<label for="note-selector">Note:</label>
|
||||
<div class="input-group">
|
||||
<input type="text" id="note-selector" class="form-control note-selector"
|
||||
placeholder="Search for a note...">
|
||||
</div>
|
||||
<div class="help-block">Select the note to map to the file system.</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="file-path">File/Directory Path:</label>
|
||||
<div class="input-group">
|
||||
<input type="text" id="file-path" class="form-control file-path-input"
|
||||
placeholder="/path/to/file/or/directory">
|
||||
<div class="input-group-append">
|
||||
<button type="button" class="btn btn-secondary validate-path-button">
|
||||
<i class="bx bx-search"></i> Validate
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="path-validation-result"></div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="sync-direction">Sync Direction:</label>
|
||||
<select id="sync-direction" class="form-control sync-direction-select">
|
||||
<option value="bidirectional">Bidirectional (default)</option>
|
||||
<option value="trilium_to_disk">Trilium → Disk only</option>
|
||||
<option value="disk_to_trilium">Disk → Trilium only</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="content-format">Content Format:</label>
|
||||
<select id="content-format" class="form-control content-format-select">
|
||||
<option value="auto">Auto-detect (default)</option>
|
||||
<option value="markdown">Markdown</option>
|
||||
<option value="html">HTML</option>
|
||||
<option value="raw">Raw/Binary</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>
|
||||
<input type="checkbox" class="include-subtree-checkbox">
|
||||
Include subtree
|
||||
</label>
|
||||
<div class="help-block">Map entire note subtree to directory structure.</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group subtree-options" style="display: none;">
|
||||
<label>
|
||||
<input type="checkbox" class="preserve-hierarchy-checkbox" checked>
|
||||
Preserve directory hierarchy
|
||||
</label>
|
||||
<div class="help-block">Create subdirectories matching note hierarchy.</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="exclude-patterns">Exclude Patterns (one per line):</label>
|
||||
<textarea id="exclude-patterns" class="form-control exclude-patterns-textarea"
|
||||
rows="3" placeholder="*.tmp node_modules .git"></textarea>
|
||||
<div class="help-block">Files/directories matching these patterns will be ignored.</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary cancel-mapping-button">Cancel</button>
|
||||
<button type="button" class="btn btn-primary save-mapping-button">Save Mapping</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
const MAPPING_ITEM_TPL = /*html*/`
|
||||
<div class="mapping-item card mb-2" data-mapping-id="">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between align-items-start">
|
||||
<div class="mapping-info">
|
||||
<div class="mapping-path">
|
||||
<strong class="file-path"></strong>
|
||||
</div>
|
||||
<div class="mapping-details text-muted">
|
||||
<span class="note-title"></span> •
|
||||
<span class="sync-direction-text"></span> •
|
||||
<span class="content-format-text"></span>
|
||||
</div>
|
||||
<div class="mapping-status">
|
||||
<span class="status-badge"></span>
|
||||
<span class="last-sync"></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mapping-actions">
|
||||
<button class="btn btn-sm btn-secondary sync-mapping-button" title="Sync now">
|
||||
<i class="bx bx-refresh"></i>
|
||||
</button>
|
||||
<button class="btn btn-sm btn-secondary edit-mapping-button" title="Edit">
|
||||
<i class="bx bx-edit"></i>
|
||||
</button>
|
||||
<button class="btn btn-sm btn-danger delete-mapping-button" title="Delete">
|
||||
<i class="bx bx-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="sync-errors" style="display: none;">
|
||||
<div class="alert alert-warning mt-2">
|
||||
<strong>Sync Errors:</strong>
|
||||
<ul class="error-list mb-0"></ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
export default class FileSystemSyncOptions extends OptionsWidget {
|
||||
private $fileSyncEnabledCheckbox!: JQuery<HTMLElement>;
|
||||
private $fileSyncControls!: JQuery<HTMLElement>;
|
||||
private $syncStatusText!: JQuery<HTMLElement>;
|
||||
private $mappingsCount!: JQuery<HTMLElement>;
|
||||
private $mappingsList!: JQuery<HTMLElement>;
|
||||
private $createMappingButton!: JQuery<HTMLElement>;
|
||||
private $refreshStatusButton!: JQuery<HTMLElement>;
|
||||
|
||||
// Modal elements
|
||||
private $mappingModal!: JQuery<HTMLElement>;
|
||||
private $modalTitle!: JQuery<HTMLElement>;
|
||||
private $noteSelector!: JQuery<HTMLElement>;
|
||||
private $filePathInput!: JQuery<HTMLElement>;
|
||||
private $validatePathButton!: JQuery<HTMLElement>;
|
||||
private $pathValidationResult!: JQuery<HTMLElement>;
|
||||
private $syncDirectionSelect!: JQuery<HTMLElement>;
|
||||
private $contentFormatSelect!: JQuery<HTMLElement>;
|
||||
private $includeSubtreeCheckbox!: JQuery<HTMLElement>;
|
||||
private $preserveHierarchyCheckbox!: JQuery<HTMLElement>;
|
||||
private $subtreeOptions!: JQuery<HTMLElement>;
|
||||
private $excludePatternsTextarea!: JQuery<HTMLElement>;
|
||||
private $saveMappingButton!: JQuery<HTMLElement>;
|
||||
private $cancelMappingButton!: JQuery<HTMLElement>;
|
||||
private $modalClose!: JQuery<HTMLElement>;
|
||||
|
||||
private currentEditingMappingId: string | null = null;
|
||||
private mappings: FileSystemMapping[] = [];
|
||||
|
||||
doRender() {
|
||||
this.$widget = $(TPL);
|
||||
this.initializeElements();
|
||||
// Ensure modal is hidden on initialization
|
||||
this.$mappingModal.addClass('modal-hidden').removeClass('modal-visible');
|
||||
this.setupEventHandlers();
|
||||
}
|
||||
|
||||
private initializeElements() {
|
||||
this.$fileSyncEnabledCheckbox = this.$widget.find(".file-sync-enabled-checkbox");
|
||||
this.$fileSyncControls = this.$widget.find(".file-sync-controls");
|
||||
this.$syncStatusText = this.$widget.find(".sync-status-text");
|
||||
this.$mappingsCount = this.$widget.find(".mappings-count");
|
||||
this.$mappingsList = this.$widget.find(".mappings-list");
|
||||
this.$createMappingButton = this.$widget.find(".create-mapping-button");
|
||||
this.$refreshStatusButton = this.$widget.find(".refresh-status-button");
|
||||
|
||||
// Modal elements
|
||||
this.$mappingModal = this.$widget.closest(".mapping-modal");
|
||||
this.$modalTitle = this.$mappingModal.find(".modal-title");
|
||||
this.$noteSelector = this.$mappingModal.find(".note-selector");
|
||||
this.$filePathInput = this.$mappingModal.find(".file-path-input");
|
||||
this.$validatePathButton = this.$mappingModal.find(".validate-path-button");
|
||||
this.$pathValidationResult = this.$mappingModal.find(".path-validation-result");
|
||||
this.$syncDirectionSelect = this.$mappingModal.find(".sync-direction-select");
|
||||
this.$contentFormatSelect = this.$mappingModal.find(".content-format-select");
|
||||
this.$includeSubtreeCheckbox = this.$mappingModal.find(".include-subtree-checkbox");
|
||||
this.$preserveHierarchyCheckbox = this.$mappingModal.find(".preserve-hierarchy-checkbox");
|
||||
this.$subtreeOptions = this.$mappingModal.find(".subtree-options");
|
||||
this.$excludePatternsTextarea = this.$mappingModal.find(".exclude-patterns-textarea");
|
||||
this.$saveMappingButton = this.$mappingModal.find(".save-mapping-button");
|
||||
this.$cancelMappingButton = this.$mappingModal.find(".cancel-mapping-button");
|
||||
this.$modalClose = this.$mappingModal.find(".modal-close");
|
||||
}
|
||||
|
||||
private setupEventHandlers() {
|
||||
this.$fileSyncEnabledCheckbox.on("change", async () => {
|
||||
const isEnabled = this.$fileSyncEnabledCheckbox.prop("checked");
|
||||
|
||||
try {
|
||||
if (isEnabled) {
|
||||
await server.post<ApiResponse>("file-system-sync/enable");
|
||||
} else {
|
||||
await server.post<ApiResponse>("file-system-sync/disable");
|
||||
}
|
||||
|
||||
this.toggleControls(isEnabled);
|
||||
if (isEnabled) {
|
||||
await this.refreshStatus();
|
||||
}
|
||||
|
||||
toastService.showMessage(`File system sync ${isEnabled ? 'enabled' : 'disabled'}`);
|
||||
} catch (error) {
|
||||
toastService.showError(`Failed to ${isEnabled ? 'enable' : 'disable'} file system sync`);
|
||||
// Revert checkbox state
|
||||
this.$fileSyncEnabledCheckbox.prop("checked", !isEnabled);
|
||||
}
|
||||
});
|
||||
|
||||
this.$createMappingButton.on("click", () => {
|
||||
this.showMappingModal();
|
||||
});
|
||||
|
||||
this.$refreshStatusButton.on("click", () => {
|
||||
this.refreshStatus();
|
||||
});
|
||||
|
||||
this.$validatePathButton.on("click", () => {
|
||||
this.validatePath();
|
||||
});
|
||||
|
||||
this.$includeSubtreeCheckbox.on("change", () => {
|
||||
const isChecked = this.$includeSubtreeCheckbox.prop("checked");
|
||||
this.$subtreeOptions.toggle(isChecked);
|
||||
});
|
||||
|
||||
// Modal handlers
|
||||
this.$saveMappingButton.on("click", () => {
|
||||
this.saveMapping();
|
||||
});
|
||||
|
||||
this.$cancelMappingButton.on("click", () => {
|
||||
this.hideMappingModal();
|
||||
});
|
||||
|
||||
this.$modalClose.on("click", () => {
|
||||
this.hideMappingModal();
|
||||
});
|
||||
|
||||
this.$mappingModal.find(".modal-backdrop").on("click", () => {
|
||||
this.hideMappingModal();
|
||||
});
|
||||
|
||||
// Note selector autocomplete will be initialized in showMappingModal
|
||||
}
|
||||
|
||||
private toggleControls(enabled: boolean) {
|
||||
this.$fileSyncControls.toggle(enabled);
|
||||
}
|
||||
|
||||
private async refreshStatus() {
|
||||
try {
|
||||
const status = await server.get<SyncStatus>("file-system-sync/status");
|
||||
|
||||
this.$syncStatusText.text(status.initialized ? "Active" : "Inactive");
|
||||
|
||||
if (status.initialized) {
|
||||
await this.loadMappings();
|
||||
}
|
||||
} catch (error) {
|
||||
this.$syncStatusText.text("Error");
|
||||
toastService.showError("Failed to get sync status");
|
||||
}
|
||||
}
|
||||
|
||||
private async loadMappings() {
|
||||
try {
|
||||
this.mappings = await server.get<FileSystemMapping[]>("file-system-sync/mappings");
|
||||
this.renderMappings();
|
||||
this.$mappingsCount.text(this.mappings.length.toString());
|
||||
} catch (error) {
|
||||
toastService.showError("Failed to load mappings");
|
||||
}
|
||||
}
|
||||
|
||||
private renderMappings() {
|
||||
this.$mappingsList.empty();
|
||||
|
||||
for (const mapping of this.mappings) {
|
||||
const $item = $(MAPPING_ITEM_TPL);
|
||||
$item.attr("data-mapping-id", mapping.mappingId);
|
||||
|
||||
$item.find(".file-path").text(mapping.filePath);
|
||||
$item.find(".note-title").text(`Note: ${mapping.noteId}`); // TODO: Get actual note title
|
||||
$item.find(".sync-direction-text").text(this.formatSyncDirection(mapping.syncDirection));
|
||||
$item.find(".content-format-text").text(mapping.contentFormat);
|
||||
|
||||
// Status badge
|
||||
const $statusBadge = $item.find(".status-badge");
|
||||
if (mapping.syncErrors && mapping.syncErrors.length > 0) {
|
||||
$statusBadge.addClass("badge badge-danger").text("Error");
|
||||
const $errorsDiv = $item.find(".sync-errors");
|
||||
const $errorList = $errorsDiv.find(".error-list");
|
||||
mapping.syncErrors.forEach(error => {
|
||||
$errorList.append(`<li>${error}</li>`);
|
||||
});
|
||||
$errorsDiv.show();
|
||||
} else if (mapping.isActive) {
|
||||
$statusBadge.addClass("badge badge-success").text("Active");
|
||||
} else {
|
||||
$statusBadge.addClass("badge badge-secondary").text("Inactive");
|
||||
}
|
||||
|
||||
// Last sync time
|
||||
if (mapping.lastSyncTime) {
|
||||
const lastSync = new Date(mapping.lastSyncTime).toLocaleString();
|
||||
$item.find(".last-sync").text(`Last sync: ${lastSync}`);
|
||||
} else {
|
||||
$item.find(".last-sync").text("Never synced");
|
||||
}
|
||||
|
||||
// Action handlers
|
||||
$item.find(".sync-mapping-button").on("click", () => {
|
||||
this.syncMapping(mapping.mappingId);
|
||||
});
|
||||
|
||||
$item.find(".edit-mapping-button").on("click", () => {
|
||||
this.editMapping(mapping);
|
||||
});
|
||||
|
||||
$item.find(".delete-mapping-button").on("click", () => {
|
||||
this.deleteMapping(mapping.mappingId);
|
||||
});
|
||||
|
||||
this.$mappingsList.append($item);
|
||||
}
|
||||
}
|
||||
|
||||
private formatSyncDirection(direction: string): string {
|
||||
switch (direction) {
|
||||
case 'bidirectional': return 'Bidirectional';
|
||||
case 'trilium_to_disk': return 'Trilium → Disk';
|
||||
case 'disk_to_trilium': return 'Disk → Trilium';
|
||||
default: return direction;
|
||||
}
|
||||
}
|
||||
|
||||
private showMappingModal(mapping?: FileSystemMapping) {
|
||||
this.currentEditingMappingId = mapping?.mappingId || null;
|
||||
console.log("Showing mapping modal", this.currentEditingMappingId, this.$mappingModal);
|
||||
|
||||
if (mapping) {
|
||||
this.$modalTitle.text("Edit File System Mapping");
|
||||
this.populateMappingForm(mapping);
|
||||
} else {
|
||||
this.$modalTitle.text("Create File System Mapping");
|
||||
this.clearMappingForm();
|
||||
}
|
||||
|
||||
// Initialize note autocomplete
|
||||
noteAutocompleteService.initNoteAutocomplete(this.$noteSelector, {
|
||||
allowCreatingNotes: true,
|
||||
});
|
||||
|
||||
// Handle note selection
|
||||
this.$noteSelector.off("autocomplete:noteselected").on("autocomplete:noteselected", (event: JQuery.Event, suggestion: Suggestion) => {
|
||||
// The note autocomplete service will automatically set the selected note path
|
||||
// which we can retrieve using getSelectedNoteId()
|
||||
});
|
||||
|
||||
this.$mappingModal.removeClass('modal-hidden').addClass('modal-visible');
|
||||
}
|
||||
|
||||
private hideMappingModal() {
|
||||
this.$mappingModal.removeClass('modal-visible').addClass('modal-hidden');
|
||||
this.clearMappingForm();
|
||||
this.currentEditingMappingId = null;
|
||||
}
|
||||
|
||||
private async populateMappingForm(mapping: FileSystemMapping) {
|
||||
// Set the note using the autocomplete service's setNote method
|
||||
await this.$noteSelector.setNote(mapping.noteId);
|
||||
|
||||
this.$filePathInput.val(mapping.filePath);
|
||||
this.$syncDirectionSelect.val(mapping.syncDirection);
|
||||
this.$contentFormatSelect.val(mapping.contentFormat);
|
||||
this.$includeSubtreeCheckbox.prop("checked", mapping.includeSubtree);
|
||||
this.$preserveHierarchyCheckbox.prop("checked", mapping.preserveHierarchy);
|
||||
this.$subtreeOptions.toggle(mapping.includeSubtree);
|
||||
|
||||
if (mapping.excludePatterns) {
|
||||
this.$excludePatternsTextarea.val(mapping.excludePatterns.join('\n'));
|
||||
}
|
||||
}
|
||||
|
||||
private clearMappingForm() {
|
||||
// Clear the note selector using autocomplete service
|
||||
this.$noteSelector.val('').setSelectedNotePath('');
|
||||
this.$filePathInput.val('');
|
||||
this.$syncDirectionSelect.val('bidirectional');
|
||||
this.$contentFormatSelect.val('auto');
|
||||
this.$includeSubtreeCheckbox.prop("checked", false);
|
||||
this.$preserveHierarchyCheckbox.prop("checked", true);
|
||||
this.$subtreeOptions.hide();
|
||||
this.$excludePatternsTextarea.val('');
|
||||
this.$pathValidationResult.empty();
|
||||
}
|
||||
|
||||
private async validatePath() {
|
||||
const filePath = this.$filePathInput.val() as string;
|
||||
if (!filePath) {
|
||||
this.$pathValidationResult.html('<div class="text-danger">Please enter a file path</div>');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await server.post<PathValidationResponse>("file-system-sync/validate-path", { filePath } as PathValidationRequest);
|
||||
|
||||
if (result.exists && result.stats) {
|
||||
const type = result.stats.isDirectory ? 'directory' : 'file';
|
||||
this.$pathValidationResult.html(
|
||||
`<div class="text-success">✓ Valid ${type} (${result.stats.size} bytes, modified ${new Date(result.stats.modified).toLocaleString()})</div>`
|
||||
);
|
||||
} else {
|
||||
this.$pathValidationResult.html('<div class="text-warning">⚠ Path does not exist</div>');
|
||||
}
|
||||
} catch (error) {
|
||||
this.$pathValidationResult.html('<div class="text-danger">✗ Invalid path</div>');
|
||||
}
|
||||
}
|
||||
|
||||
private async saveMapping() {
|
||||
const noteId = this.$noteSelector.getSelectedNoteId();
|
||||
const filePath = this.$filePathInput.val() as string;
|
||||
const syncDirection = this.$syncDirectionSelect.val() as string;
|
||||
const contentFormat = this.$contentFormatSelect.val() as string;
|
||||
const includeSubtree = this.$includeSubtreeCheckbox.prop("checked");
|
||||
const preserveHierarchy = this.$preserveHierarchyCheckbox.prop("checked");
|
||||
const excludePatternsText = this.$excludePatternsTextarea.val() as string;
|
||||
|
||||
// Validation
|
||||
if (!noteId) {
|
||||
toastService.showError("Please select a note");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!filePath) {
|
||||
toastService.showError("Please enter a file path");
|
||||
return;
|
||||
}
|
||||
|
||||
const excludePatterns = excludePatternsText.trim()
|
||||
? excludePatternsText.split('\n').map(p => p.trim()).filter(p => p)
|
||||
: null;
|
||||
|
||||
const mappingData: CreateMappingRequest = {
|
||||
noteId,
|
||||
filePath,
|
||||
syncDirection: syncDirection as 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium',
|
||||
contentFormat: contentFormat as 'auto' | 'markdown' | 'html' | 'raw',
|
||||
includeSubtree,
|
||||
preserveHierarchy,
|
||||
excludePatterns
|
||||
};
|
||||
|
||||
try {
|
||||
if (this.currentEditingMappingId) {
|
||||
await server.put<ApiResponse>(`file-system-sync/mappings/${this.currentEditingMappingId}`, mappingData as UpdateMappingRequest);
|
||||
toastService.showMessage("Mapping updated successfully");
|
||||
} else {
|
||||
await server.post<ApiResponse>("file-system-sync/mappings", mappingData);
|
||||
toastService.showMessage("Mapping created successfully");
|
||||
}
|
||||
|
||||
this.hideMappingModal();
|
||||
await this.loadMappings();
|
||||
} catch (error) {
|
||||
toastService.showError("Failed to save mapping");
|
||||
}
|
||||
}
|
||||
|
||||
private async syncMapping(mappingId: string) {
|
||||
try {
|
||||
const result = await server.post<SyncMappingResponse>(`file-system-sync/mappings/${mappingId}/sync`);
|
||||
if (result.success) {
|
||||
toastService.showMessage("Sync completed successfully");
|
||||
} else {
|
||||
toastService.showError(`Sync failed: ${result.message}`);
|
||||
}
|
||||
await this.loadMappings();
|
||||
} catch (error) {
|
||||
toastService.showError("Failed to trigger sync");
|
||||
}
|
||||
}
|
||||
|
||||
private editMapping(mapping: FileSystemMapping) {
|
||||
this.showMappingModal(mapping);
|
||||
}
|
||||
|
||||
private async deleteMapping(mappingId: string) {
|
||||
if (!confirm("Are you sure you want to delete this mapping?")) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await server.delete<ApiResponse>(`file-system-sync/mappings/${mappingId}`);
|
||||
toastService.showMessage("Mapping deleted successfully");
|
||||
await this.loadMappings();
|
||||
} catch (error) {
|
||||
toastService.showError("Failed to delete mapping");
|
||||
}
|
||||
}
|
||||
|
||||
async optionsLoaded(options: OptionMap) {
|
||||
const isEnabled = options.fileSystemSyncEnabled === "true";
|
||||
this.$fileSyncEnabledCheckbox.prop("checked", isEnabled);
|
||||
this.toggleControls(isEnabled);
|
||||
|
||||
if (isEnabled) {
|
||||
await this.refreshStatus();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import log from "./services/log.js";
|
||||
import "./services/handlers.js";
|
||||
import "./becca/becca_loader.js";
|
||||
import { RESOURCE_DIR } from "./services/resource_dir.js";
|
||||
import fileSystemSyncInit from "./services/file_system_sync_init.js";
|
||||
|
||||
export default async function buildApp() {
|
||||
const app = express();
|
||||
@@ -32,6 +33,9 @@ export default async function buildApp() {
|
||||
try {
|
||||
log.info("Database initialized, LLM features available");
|
||||
log.info("LLM features ready");
|
||||
|
||||
// Initialize file system sync after database is ready
|
||||
await fileSystemSyncInit.init();
|
||||
} catch (error) {
|
||||
console.error("Error initializing LLM features:", error);
|
||||
}
|
||||
@@ -41,6 +45,9 @@ export default async function buildApp() {
|
||||
if (sql_init.isDbInitialized()) {
|
||||
try {
|
||||
log.info("LLM features ready");
|
||||
|
||||
// Initialize file system sync if database is already ready
|
||||
await fileSystemSyncInit.init();
|
||||
} catch (error) {
|
||||
console.error("Error initializing LLM features:", error);
|
||||
}
|
||||
|
||||
@@ -152,3 +152,56 @@ CREATE TABLE IF NOT EXISTS sessions (
|
||||
data TEXT,
|
||||
expires INTEGER
|
||||
);
|
||||
|
||||
-- Table to store file system mappings for notes and subtrees
|
||||
CREATE TABLE IF NOT EXISTS "file_system_mappings" (
|
||||
"mappingId" TEXT NOT NULL PRIMARY KEY,
|
||||
"noteId" TEXT NOT NULL,
|
||||
"filePath" TEXT NOT NULL,
|
||||
"syncDirection" TEXT NOT NULL DEFAULT 'bidirectional', -- 'bidirectional', 'trilium_to_disk', 'disk_to_trilium'
|
||||
"isActive" INTEGER NOT NULL DEFAULT 1,
|
||||
"includeSubtree" INTEGER NOT NULL DEFAULT 0,
|
||||
"preserveHierarchy" INTEGER NOT NULL DEFAULT 1,
|
||||
"contentFormat" TEXT NOT NULL DEFAULT 'auto', -- 'auto', 'markdown', 'html', 'raw'
|
||||
"excludePatterns" TEXT DEFAULT NULL, -- JSON array of glob patterns to exclude
|
||||
"lastSyncTime" TEXT DEFAULT NULL,
|
||||
"syncErrors" TEXT DEFAULT NULL, -- JSON array of recent sync errors
|
||||
"dateCreated" TEXT NOT NULL,
|
||||
"dateModified" TEXT NOT NULL,
|
||||
"utcDateCreated" TEXT NOT NULL,
|
||||
"utcDateModified" TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Table to track file to note mappings for efficient lookups
|
||||
CREATE TABLE IF NOT EXISTS "file_note_mappings" (
|
||||
"fileNoteId" TEXT NOT NULL PRIMARY KEY,
|
||||
"mappingId" TEXT NOT NULL,
|
||||
"noteId" TEXT NOT NULL,
|
||||
"filePath" TEXT NOT NULL,
|
||||
"fileHash" TEXT DEFAULT NULL,
|
||||
"fileModifiedTime" TEXT DEFAULT NULL,
|
||||
"lastSyncTime" TEXT DEFAULT NULL,
|
||||
"syncStatus" TEXT NOT NULL DEFAULT 'synced', -- 'synced', 'pending', 'conflict', 'error'
|
||||
"dateCreated" TEXT NOT NULL,
|
||||
"dateModified" TEXT NOT NULL,
|
||||
"utcDateCreated" TEXT NOT NULL,
|
||||
"utcDateModified" TEXT NOT NULL,
|
||||
FOREIGN KEY ("mappingId") REFERENCES "file_system_mappings" ("mappingId") ON DELETE CASCADE,
|
||||
FOREIGN KEY ("noteId") REFERENCES "notes" ("noteId") ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for quick lookup by noteId
|
||||
CREATE INDEX "IDX_file_system_mappings_noteId" ON "file_system_mappings" ("noteId");
|
||||
-- Index for finding active mappings
|
||||
CREATE INDEX "IDX_file_system_mappings_active" ON "file_system_mappings" ("isActive", "noteId");
|
||||
-- Unique constraint to prevent duplicate mappings for same note
|
||||
CREATE UNIQUE INDEX "IDX_file_system_mappings_note_unique" ON "file_system_mappings" ("noteId");
|
||||
|
||||
-- Index for quick lookup by file path
|
||||
CREATE INDEX "IDX_file_note_mappings_filePath" ON "file_note_mappings" ("filePath");
|
||||
-- Index for finding notes by mapping
|
||||
CREATE INDEX "IDX_file_note_mappings_mapping" ON "file_note_mappings" ("mappingId", "noteId");
|
||||
-- Index for finding pending syncs
|
||||
CREATE INDEX "IDX_file_note_mappings_sync_status" ON "file_note_mappings" ("syncStatus", "mappingId");
|
||||
-- Unique constraint for file path per mapping
|
||||
CREATE UNIQUE INDEX "IDX_file_note_mappings_file_unique" ON "file_note_mappings" ("mappingId", "filePath");
|
||||
|
||||
@@ -12,6 +12,8 @@ import type { AttachmentRow, BlobRow, RevisionRow } from "@triliumnext/commons";
|
||||
import BBlob from "./entities/bblob.js";
|
||||
import BRecentNote from "./entities/brecent_note.js";
|
||||
import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js";
|
||||
import type BFileSystemMapping from "./entities/bfile_system_mapping.js";
|
||||
import type BFileNoteMapping from "./entities/bfile_note_mapping.js";
|
||||
|
||||
interface AttachmentOpts {
|
||||
includeContentLength?: boolean;
|
||||
@@ -32,6 +34,8 @@ export default class Becca {
|
||||
attributeIndex!: Record<string, BAttribute[]>;
|
||||
options!: Record<string, BOption>;
|
||||
etapiTokens!: Record<string, BEtapiToken>;
|
||||
fileSystemMappings!: Record<string, BFileSystemMapping>;
|
||||
fileNoteMappings!: Record<string, BFileNoteMapping>;
|
||||
|
||||
allNoteSetCache: NoteSet | null;
|
||||
|
||||
@@ -48,6 +52,8 @@ export default class Becca {
|
||||
this.attributeIndex = {};
|
||||
this.options = {};
|
||||
this.etapiTokens = {};
|
||||
this.fileSystemMappings = {};
|
||||
this.fileNoteMappings = {};
|
||||
|
||||
this.dirtyNoteSetCache();
|
||||
|
||||
@@ -213,6 +219,39 @@ export default class Becca {
|
||||
return this.etapiTokens[etapiTokenId];
|
||||
}
|
||||
|
||||
getFileSystemMapping(mappingId: string): BFileSystemMapping | null {
|
||||
return this.fileSystemMappings[mappingId];
|
||||
}
|
||||
|
||||
getFileSystemMappingOrThrow(mappingId: string): BFileSystemMapping {
|
||||
const mapping = this.getFileSystemMapping(mappingId);
|
||||
if (!mapping) {
|
||||
throw new NotFoundError(`File system mapping '${mappingId}' has not been found.`);
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
getFileNoteMapping(fileNoteId: string): BFileNoteMapping | null {
|
||||
return this.fileNoteMappings[fileNoteId];
|
||||
}
|
||||
|
||||
getFileNoteMappingOrThrow(fileNoteId: string): BFileNoteMapping {
|
||||
const mapping = this.getFileNoteMapping(fileNoteId);
|
||||
if (!mapping) {
|
||||
throw new NotFoundError(`File note mapping '${fileNoteId}' has not been found.`);
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
getFileSystemMappingByNoteId(noteId: string): BFileSystemMapping | null {
|
||||
for (const mapping of Object.values(this.fileSystemMappings)) {
|
||||
if (mapping.noteId === noteId) {
|
||||
return mapping;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
getEntity<T extends AbstractBeccaEntity<T>>(entityName: string, entityId: string): AbstractBeccaEntity<T> | null {
|
||||
if (!entityName || !entityId) {
|
||||
return null;
|
||||
@@ -222,6 +261,10 @@ export default class Becca {
|
||||
return this.getRevision(entityId);
|
||||
} else if (entityName === "attachments") {
|
||||
return this.getAttachment(entityId);
|
||||
} else if (entityName === "file_system_mappings") {
|
||||
return this.getFileSystemMapping(entityId);
|
||||
} else if (entityName === "file_note_mappings") {
|
||||
return this.getFileNoteMapping(entityId);
|
||||
}
|
||||
|
||||
const camelCaseEntityName = entityName.toLowerCase().replace(/(_[a-z])/g, (group) => group.toUpperCase().replace("_", ""));
|
||||
|
||||
@@ -9,9 +9,13 @@ import BBranch from "./entities/bbranch.js";
|
||||
import BAttribute from "./entities/battribute.js";
|
||||
import BOption from "./entities/boption.js";
|
||||
import BEtapiToken from "./entities/betapi_token.js";
|
||||
import BFileSystemMapping from "./entities/bfile_system_mapping.js";
|
||||
import BFileNoteMapping from "./entities/bfile_note_mapping.js";
|
||||
import cls from "../services/cls.js";
|
||||
import entityConstructor from "../becca/entity_constructor.js";
|
||||
import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";
|
||||
import type { FileSystemMappingRow } from "./entities/bfile_system_mapping.js";
|
||||
import type { FileNoteMappingRow } from "./entities/bfile_note_mapping.js";
|
||||
import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js";
|
||||
import ws from "../services/ws.js";
|
||||
|
||||
@@ -64,6 +68,14 @@ function load() {
|
||||
new BEtapiToken(row);
|
||||
}
|
||||
|
||||
for (const row of sql.getRows<FileSystemMappingRow>(/*sql*/`SELECT mappingId, noteId, filePath, syncDirection, isActive, includeSubtree, preserveHierarchy, contentFormat, excludePatterns, lastSyncTime, syncErrors, dateCreated, dateModified, utcDateCreated, utcDateModified FROM file_system_mappings`)) {
|
||||
new BFileSystemMapping(row);
|
||||
}
|
||||
|
||||
for (const row of sql.getRows<FileNoteMappingRow>(/*sql*/`SELECT fileNoteId, mappingId, noteId, filePath, fileHash, fileModifiedTime, lastSyncTime, syncStatus, dateCreated, dateModified, utcDateCreated, utcDateModified FROM file_note_mappings`)) {
|
||||
new BFileNoteMapping(row);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
for (const noteId in becca.notes) {
|
||||
@@ -86,7 +98,7 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_CHANGE_SYNCED], ({ entity
|
||||
return;
|
||||
}
|
||||
|
||||
if (["notes", "branches", "attributes", "etapi_tokens", "options"].includes(entityName)) {
|
||||
if (["notes", "branches", "attributes", "etapi_tokens", "options", "file_system_mappings", "file_note_mappings"].includes(entityName)) {
|
||||
const EntityClass = entityConstructor.getEntityFromEntityName(entityName);
|
||||
const primaryKeyName = EntityClass.primaryKeyName;
|
||||
|
||||
@@ -144,6 +156,10 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_DELETED, eventService.ENT
|
||||
attributeDeleted(entityId);
|
||||
} else if (entityName === "etapi_tokens") {
|
||||
etapiTokenDeleted(entityId);
|
||||
} else if (entityName === "file_system_mappings") {
|
||||
fileSystemMappingDeleted(entityId);
|
||||
} else if (entityName === "file_note_mappings") {
|
||||
fileNoteMappingDeleted(entityId);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -279,6 +295,14 @@ function etapiTokenDeleted(etapiTokenId: string) {
|
||||
delete becca.etapiTokens[etapiTokenId];
|
||||
}
|
||||
|
||||
function fileSystemMappingDeleted(mappingId: string) {
|
||||
delete becca.fileSystemMappings[mappingId];
|
||||
}
|
||||
|
||||
function fileNoteMappingDeleted(fileNoteId: string) {
|
||||
delete becca.fileNoteMappings[fileNoteId];
|
||||
}
|
||||
|
||||
|
||||
eventService.subscribeBeccaLoader(eventService.ENTER_PROTECTED_SESSION, () => {
|
||||
try {
|
||||
|
||||
233
apps/server/src/becca/entities/bfile_note_mapping.ts
Normal file
233
apps/server/src/becca/entities/bfile_note_mapping.ts
Normal file
@@ -0,0 +1,233 @@
|
||||
"use strict";
|
||||
|
||||
import AbstractBeccaEntity from "./abstract_becca_entity.js";
|
||||
import dateUtils from "../../services/date_utils.js";
|
||||
import { newEntityId } from "../../services/utils.js";
|
||||
|
||||
export interface FileNoteMappingRow {
|
||||
fileNoteId?: string;
|
||||
mappingId: string;
|
||||
noteId: string;
|
||||
filePath: string;
|
||||
fileHash?: string | null;
|
||||
fileModifiedTime?: string | null;
|
||||
lastSyncTime?: string | null;
|
||||
syncStatus?: 'synced' | 'pending' | 'conflict' | 'error';
|
||||
dateCreated?: string;
|
||||
dateModified?: string;
|
||||
utcDateCreated?: string;
|
||||
utcDateModified?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* FileNoteMapping represents the mapping between a specific file and a specific note
|
||||
* This is used for tracking sync status and file metadata
|
||||
*/
|
||||
class BFileNoteMapping extends AbstractBeccaEntity<BFileNoteMapping> {
|
||||
static get entityName() {
|
||||
return "file_note_mappings";
|
||||
}
|
||||
static get primaryKeyName() {
|
||||
return "fileNoteId";
|
||||
}
|
||||
static get hashedProperties() {
|
||||
return ["fileNoteId", "mappingId", "noteId", "filePath", "fileHash", "syncStatus"];
|
||||
}
|
||||
|
||||
fileNoteId!: string;
|
||||
mappingId!: string;
|
||||
noteId!: string;
|
||||
filePath!: string;
|
||||
fileHash?: string | null;
|
||||
fileModifiedTime?: string | null;
|
||||
lastSyncTime?: string | null;
|
||||
syncStatus!: 'synced' | 'pending' | 'conflict' | 'error';
|
||||
|
||||
constructor(row?: FileNoteMappingRow) {
|
||||
super();
|
||||
|
||||
if (!row) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.updateFromRow(row);
|
||||
this.init();
|
||||
}
|
||||
|
||||
updateFromRow(row: FileNoteMappingRow) {
|
||||
this.update([
|
||||
row.fileNoteId,
|
||||
row.mappingId,
|
||||
row.noteId,
|
||||
row.filePath,
|
||||
row.fileHash,
|
||||
row.fileModifiedTime,
|
||||
row.lastSyncTime,
|
||||
row.syncStatus || 'synced',
|
||||
row.dateCreated,
|
||||
row.dateModified,
|
||||
row.utcDateCreated,
|
||||
row.utcDateModified
|
||||
]);
|
||||
}
|
||||
|
||||
update([
|
||||
fileNoteId,
|
||||
mappingId,
|
||||
noteId,
|
||||
filePath,
|
||||
fileHash,
|
||||
fileModifiedTime,
|
||||
lastSyncTime,
|
||||
syncStatus,
|
||||
dateCreated,
|
||||
dateModified,
|
||||
utcDateCreated,
|
||||
utcDateModified
|
||||
]: any) {
|
||||
this.fileNoteId = fileNoteId;
|
||||
this.mappingId = mappingId;
|
||||
this.noteId = noteId;
|
||||
this.filePath = filePath;
|
||||
this.fileHash = fileHash;
|
||||
this.fileModifiedTime = fileModifiedTime;
|
||||
this.lastSyncTime = lastSyncTime;
|
||||
this.syncStatus = syncStatus || 'synced';
|
||||
this.dateCreated = dateCreated;
|
||||
this.dateModified = dateModified;
|
||||
this.utcDateCreated = utcDateCreated;
|
||||
this.utcDateModified = utcDateModified;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
override init() {
|
||||
if (this.fileNoteId) {
|
||||
this.becca.fileNoteMappings = this.becca.fileNoteMappings || {};
|
||||
this.becca.fileNoteMappings[this.fileNoteId] = this;
|
||||
}
|
||||
}
|
||||
|
||||
get note() {
|
||||
return this.becca.notes[this.noteId];
|
||||
}
|
||||
|
||||
get mapping() {
|
||||
return this.becca.fileSystemMappings?.[this.mappingId];
|
||||
}
|
||||
|
||||
getNote() {
|
||||
const note = this.becca.getNote(this.noteId);
|
||||
if (!note) {
|
||||
throw new Error(`Note '${this.noteId}' for file note mapping '${this.fileNoteId}' does not exist.`);
|
||||
}
|
||||
return note;
|
||||
}
|
||||
|
||||
getMapping() {
|
||||
const mapping = this.mapping;
|
||||
if (!mapping) {
|
||||
throw new Error(`File system mapping '${this.mappingId}' for file note mapping '${this.fileNoteId}' does not exist.`);
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this mapping as needing sync
|
||||
*/
|
||||
markPending() {
|
||||
this.syncStatus = 'pending';
|
||||
this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this mapping as having a conflict
|
||||
*/
|
||||
markConflict() {
|
||||
this.syncStatus = 'conflict';
|
||||
this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this mapping as having an error
|
||||
*/
|
||||
markError() {
|
||||
this.syncStatus = 'error';
|
||||
this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this mapping as synced and update sync time
|
||||
*/
|
||||
markSynced(fileHash?: string, fileModifiedTime?: string) {
|
||||
this.syncStatus = 'synced';
|
||||
this.lastSyncTime = dateUtils.utcNowDateTime();
|
||||
|
||||
if (fileHash !== undefined) {
|
||||
this.fileHash = fileHash;
|
||||
}
|
||||
|
||||
if (fileModifiedTime !== undefined) {
|
||||
this.fileModifiedTime = fileModifiedTime;
|
||||
}
|
||||
|
||||
this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the file has been modified since last sync
|
||||
*/
|
||||
hasFileChanged(currentFileHash: string, currentModifiedTime: string): boolean {
|
||||
return this.fileHash !== currentFileHash || this.fileModifiedTime !== currentModifiedTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the note has been modified since last sync
|
||||
*/
|
||||
hasNoteChanged(): boolean {
|
||||
const note = this.note;
|
||||
if (!note) return false;
|
||||
|
||||
if (!this.lastSyncTime) return true;
|
||||
|
||||
return (note.utcDateModified ?? note.dateModified ?? note.utcDateCreated) > this.lastSyncTime;
|
||||
}
|
||||
|
||||
override beforeSaving() {
|
||||
super.beforeSaving();
|
||||
|
||||
if (!this.fileNoteId) {
|
||||
this.fileNoteId = newEntityId();
|
||||
}
|
||||
|
||||
if (!this.dateCreated) {
|
||||
this.dateCreated = dateUtils.localNowDateTime();
|
||||
}
|
||||
|
||||
if (!this.utcDateCreated) {
|
||||
this.utcDateCreated = dateUtils.utcNowDateTime();
|
||||
}
|
||||
|
||||
this.dateModified = dateUtils.localNowDateTime();
|
||||
this.utcDateModified = dateUtils.utcNowDateTime();
|
||||
}
|
||||
|
||||
getPojo(): FileNoteMappingRow {
|
||||
return {
|
||||
fileNoteId: this.fileNoteId,
|
||||
mappingId: this.mappingId,
|
||||
noteId: this.noteId,
|
||||
filePath: this.filePath,
|
||||
fileHash: this.fileHash,
|
||||
fileModifiedTime: this.fileModifiedTime,
|
||||
lastSyncTime: this.lastSyncTime,
|
||||
syncStatus: this.syncStatus,
|
||||
dateCreated: this.dateCreated,
|
||||
dateModified: this.dateModified,
|
||||
utcDateCreated: this.utcDateCreated,
|
||||
utcDateModified: this.utcDateModified
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export default BFileNoteMapping;
|
||||
236
apps/server/src/becca/entities/bfile_system_mapping.ts
Normal file
236
apps/server/src/becca/entities/bfile_system_mapping.ts
Normal file
@@ -0,0 +1,236 @@
|
||||
"use strict";
|
||||
|
||||
import AbstractBeccaEntity from "./abstract_becca_entity.js";
|
||||
import dateUtils from "../../services/date_utils.js";
|
||||
import { newEntityId } from "../../services/utils.js";
|
||||
|
||||
export interface FileSystemMappingRow {
|
||||
mappingId?: string;
|
||||
noteId: string;
|
||||
filePath: string;
|
||||
syncDirection?: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
|
||||
isActive?: number;
|
||||
includeSubtree?: number;
|
||||
preserveHierarchy?: number;
|
||||
contentFormat?: 'auto' | 'markdown' | 'html' | 'raw';
|
||||
excludePatterns?: string | null;
|
||||
lastSyncTime?: string | null;
|
||||
syncErrors?: string | null;
|
||||
dateCreated?: string;
|
||||
dateModified?: string;
|
||||
utcDateCreated?: string;
|
||||
utcDateModified?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* FileSystemMapping represents a mapping between a note/subtree and a file system path
|
||||
*/
|
||||
class BFileSystemMapping extends AbstractBeccaEntity<BFileSystemMapping> {
|
||||
static get entityName() {
|
||||
return "file_system_mappings";
|
||||
}
|
||||
static get primaryKeyName() {
|
||||
return "mappingId";
|
||||
}
|
||||
static get hashedProperties() {
|
||||
return ["mappingId", "noteId", "filePath", "syncDirection", "isActive", "includeSubtree", "preserveHierarchy", "contentFormat"];
|
||||
}
|
||||
|
||||
mappingId!: string;
|
||||
noteId!: string;
|
||||
filePath!: string;
|
||||
syncDirection!: 'bidirectional' | 'trilium_to_disk' | 'disk_to_trilium';
|
||||
isActive!: boolean;
|
||||
includeSubtree!: boolean;
|
||||
preserveHierarchy!: boolean;
|
||||
contentFormat!: 'auto' | 'markdown' | 'html' | 'raw';
|
||||
excludePatterns?: (string | RegExp)[] | null;
|
||||
lastSyncTime?: string | null;
|
||||
syncErrors?: string[] | null;
|
||||
|
||||
constructor(row?: FileSystemMappingRow) {
|
||||
super();
|
||||
|
||||
if (!row) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.updateFromRow(row);
|
||||
this.init();
|
||||
}
|
||||
|
||||
updateFromRow(row: FileSystemMappingRow) {
|
||||
this.update([
|
||||
row.mappingId,
|
||||
row.noteId,
|
||||
row.filePath,
|
||||
row.syncDirection || 'bidirectional',
|
||||
row.isActive !== undefined ? row.isActive : 1,
|
||||
row.includeSubtree !== undefined ? row.includeSubtree : 0,
|
||||
row.preserveHierarchy !== undefined ? row.preserveHierarchy : 1,
|
||||
row.contentFormat || 'auto',
|
||||
row.excludePatterns,
|
||||
row.lastSyncTime,
|
||||
row.syncErrors,
|
||||
row.dateCreated,
|
||||
row.dateModified,
|
||||
row.utcDateCreated,
|
||||
row.utcDateModified
|
||||
]);
|
||||
}
|
||||
|
||||
update([
|
||||
mappingId,
|
||||
noteId,
|
||||
filePath,
|
||||
syncDirection,
|
||||
isActive,
|
||||
includeSubtree,
|
||||
preserveHierarchy,
|
||||
contentFormat,
|
||||
excludePatterns,
|
||||
lastSyncTime,
|
||||
syncErrors,
|
||||
dateCreated,
|
||||
dateModified,
|
||||
utcDateCreated,
|
||||
utcDateModified
|
||||
]: any) {
|
||||
this.mappingId = mappingId;
|
||||
this.noteId = noteId;
|
||||
this.filePath = filePath;
|
||||
this.syncDirection = syncDirection || 'bidirectional';
|
||||
this.isActive = !!isActive;
|
||||
this.includeSubtree = !!includeSubtree;
|
||||
this.preserveHierarchy = !!preserveHierarchy;
|
||||
this.contentFormat = contentFormat || 'auto';
|
||||
|
||||
// Parse JSON strings for arrays
|
||||
try {
|
||||
this.excludePatterns = excludePatterns ? JSON.parse(excludePatterns) : null;
|
||||
} catch {
|
||||
this.excludePatterns = null;
|
||||
}
|
||||
|
||||
try {
|
||||
this.syncErrors = syncErrors ? JSON.parse(syncErrors) : null;
|
||||
} catch {
|
||||
this.syncErrors = null;
|
||||
}
|
||||
|
||||
this.lastSyncTime = lastSyncTime;
|
||||
this.dateCreated = dateCreated;
|
||||
this.dateModified = dateModified;
|
||||
this.utcDateCreated = utcDateCreated;
|
||||
this.utcDateModified = utcDateModified;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
override init() {
|
||||
if (this.mappingId) {
|
||||
this.becca.fileSystemMappings = this.becca.fileSystemMappings || {};
|
||||
this.becca.fileSystemMappings[this.mappingId] = this;
|
||||
}
|
||||
}
|
||||
|
||||
get note() {
|
||||
return this.becca.notes[this.noteId];
|
||||
}
|
||||
|
||||
getNote() {
|
||||
const note = this.becca.getNote(this.noteId);
|
||||
if (!note) {
|
||||
throw new Error(`Note '${this.noteId}' for file system mapping '${this.mappingId}' does not exist.`);
|
||||
}
|
||||
return note;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the mapping allows syncing from Trilium to disk
|
||||
*/
|
||||
get canSyncToDisk(): boolean {
|
||||
return this.isActive && (this.syncDirection === 'bidirectional' || this.syncDirection === 'trilium_to_disk');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the mapping allows syncing from disk to Trilium
|
||||
*/
|
||||
get canSyncFromDisk(): boolean {
|
||||
return this.isActive && (this.syncDirection === 'bidirectional' || this.syncDirection === 'disk_to_trilium');
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a sync error to the errors list
|
||||
*/
|
||||
addSyncError(error: string) {
|
||||
if (!this.syncErrors) {
|
||||
this.syncErrors = [];
|
||||
}
|
||||
this.syncErrors.push(error);
|
||||
|
||||
// Keep only the last 10 errors
|
||||
if (this.syncErrors.length > 10) {
|
||||
this.syncErrors = this.syncErrors.slice(-10);
|
||||
}
|
||||
|
||||
this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all sync errors
|
||||
*/
|
||||
clearSyncErrors() {
|
||||
this.syncErrors = null;
|
||||
this.save();
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the last sync time
|
||||
*/
|
||||
updateLastSyncTime() {
|
||||
this.lastSyncTime = dateUtils.utcNowDateTime();
|
||||
this.save();
|
||||
}
|
||||
|
||||
override beforeSaving() {
|
||||
super.beforeSaving();
|
||||
|
||||
if (!this.mappingId) {
|
||||
this.mappingId = newEntityId();
|
||||
}
|
||||
|
||||
if (!this.dateCreated) {
|
||||
this.dateCreated = dateUtils.localNowDateTime();
|
||||
}
|
||||
|
||||
if (!this.utcDateCreated) {
|
||||
this.utcDateCreated = dateUtils.utcNowDateTime();
|
||||
}
|
||||
|
||||
this.dateModified = dateUtils.localNowDateTime();
|
||||
this.utcDateModified = dateUtils.utcNowDateTime();
|
||||
}
|
||||
|
||||
getPojo(): FileSystemMappingRow {
|
||||
return {
|
||||
mappingId: this.mappingId,
|
||||
noteId: this.noteId,
|
||||
filePath: this.filePath,
|
||||
syncDirection: this.syncDirection,
|
||||
isActive: this.isActive ? 1 : 0,
|
||||
includeSubtree: this.includeSubtree ? 1 : 0,
|
||||
preserveHierarchy: this.preserveHierarchy ? 1 : 0,
|
||||
contentFormat: this.contentFormat,
|
||||
excludePatterns: this.excludePatterns ? JSON.stringify(this.excludePatterns) : null,
|
||||
lastSyncTime: this.lastSyncTime,
|
||||
syncErrors: this.syncErrors ? JSON.stringify(this.syncErrors) : null,
|
||||
dateCreated: this.dateCreated,
|
||||
dateModified: this.dateModified,
|
||||
utcDateCreated: this.utcDateCreated,
|
||||
utcDateModified: this.utcDateModified
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export default BFileSystemMapping;
|
||||
@@ -9,6 +9,8 @@ import BNote from "./entities/bnote.js";
|
||||
import BOption from "./entities/boption.js";
|
||||
import BRecentNote from "./entities/brecent_note.js";
|
||||
import BRevision from "./entities/brevision.js";
|
||||
import BFileSystemMapping from "./entities/bfile_system_mapping.js";
|
||||
import BFileNoteMapping from "./entities/bfile_note_mapping.js";
|
||||
|
||||
type EntityClass = new (row?: any) => AbstractBeccaEntity<any>;
|
||||
|
||||
@@ -21,7 +23,9 @@ const ENTITY_NAME_TO_ENTITY: Record<string, ConstructorData<any> & EntityClass>
|
||||
notes: BNote,
|
||||
options: BOption,
|
||||
recent_notes: BRecentNote,
|
||||
revisions: BRevision
|
||||
revisions: BRevision,
|
||||
file_system_mappings: BFileSystemMapping,
|
||||
file_note_mappings: BFileNoteMapping
|
||||
};
|
||||
|
||||
function getEntityFromEntityName(entityName: keyof typeof ENTITY_NAME_TO_ENTITY) {
|
||||
|
||||
@@ -6,6 +6,64 @@
|
||||
|
||||
// Migrations should be kept in descending order, so the latest migration is first.
|
||||
const MIGRATIONS: (SqlMigration | JsMigration)[] = [
|
||||
// Add file system mapping support
|
||||
{
|
||||
version: 234,
|
||||
sql: /*sql*/`
|
||||
-- Table to store file system mappings for notes and subtrees
|
||||
CREATE TABLE IF NOT EXISTS "file_system_mappings" (
|
||||
"mappingId" TEXT NOT NULL PRIMARY KEY,
|
||||
"noteId" TEXT NOT NULL,
|
||||
"filePath" TEXT NOT NULL,
|
||||
"syncDirection" TEXT NOT NULL DEFAULT 'bidirectional', -- 'bidirectional', 'trilium_to_disk', 'disk_to_trilium'
|
||||
"isActive" INTEGER NOT NULL DEFAULT 1,
|
||||
"includeSubtree" INTEGER NOT NULL DEFAULT 0,
|
||||
"preserveHierarchy" INTEGER NOT NULL DEFAULT 1,
|
||||
"contentFormat" TEXT NOT NULL DEFAULT 'auto', -- 'auto', 'markdown', 'html', 'raw'
|
||||
"excludePatterns" TEXT DEFAULT NULL, -- JSON array of glob patterns to exclude
|
||||
"lastSyncTime" TEXT DEFAULT NULL,
|
||||
"syncErrors" TEXT DEFAULT NULL, -- JSON array of recent sync errors
|
||||
"dateCreated" TEXT NOT NULL,
|
||||
"dateModified" TEXT NOT NULL,
|
||||
"utcDateCreated" TEXT NOT NULL,
|
||||
"utcDateModified" TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- Index for quick lookup by noteId
|
||||
CREATE INDEX "IDX_file_system_mappings_noteId" ON "file_system_mappings" ("noteId");
|
||||
-- Index for finding active mappings
|
||||
CREATE INDEX "IDX_file_system_mappings_active" ON "file_system_mappings" ("isActive", "noteId");
|
||||
-- Unique constraint to prevent duplicate mappings for same note
|
||||
CREATE UNIQUE INDEX "IDX_file_system_mappings_note_unique" ON "file_system_mappings" ("noteId");
|
||||
|
||||
-- Table to track file to note mappings for efficient lookups
|
||||
CREATE TABLE IF NOT EXISTS "file_note_mappings" (
|
||||
"fileNoteId" TEXT NOT NULL PRIMARY KEY,
|
||||
"mappingId" TEXT NOT NULL,
|
||||
"noteId" TEXT NOT NULL,
|
||||
"filePath" TEXT NOT NULL,
|
||||
"fileHash" TEXT DEFAULT NULL,
|
||||
"fileModifiedTime" TEXT DEFAULT NULL,
|
||||
"lastSyncTime" TEXT DEFAULT NULL,
|
||||
"syncStatus" TEXT NOT NULL DEFAULT 'synced', -- 'synced', 'pending', 'conflict', 'error'
|
||||
"dateCreated" TEXT NOT NULL,
|
||||
"dateModified" TEXT NOT NULL,
|
||||
"utcDateCreated" TEXT NOT NULL,
|
||||
"utcDateModified" TEXT NOT NULL,
|
||||
FOREIGN KEY ("mappingId") REFERENCES "file_system_mappings" ("mappingId") ON DELETE CASCADE,
|
||||
FOREIGN KEY ("noteId") REFERENCES "notes" ("noteId") ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for quick lookup by file path
|
||||
CREATE INDEX "IDX_file_note_mappings_filePath" ON "file_note_mappings" ("filePath");
|
||||
-- Index for finding notes by mapping
|
||||
CREATE INDEX "IDX_file_note_mappings_mapping" ON "file_note_mappings" ("mappingId", "noteId");
|
||||
-- Index for finding pending syncs
|
||||
CREATE INDEX "IDX_file_note_mappings_sync_status" ON "file_note_mappings" ("syncStatus", "mappingId");
|
||||
-- Unique constraint for file path per mapping
|
||||
CREATE UNIQUE INDEX "IDX_file_note_mappings_file_unique" ON "file_note_mappings" ("mappingId", "filePath");
|
||||
`
|
||||
},
|
||||
// Migrate geo map to collection
|
||||
{
|
||||
version: 233,
|
||||
|
||||
297
apps/server/src/routes/api/file_system_sync.ts
Normal file
297
apps/server/src/routes/api/file_system_sync.ts
Normal file
@@ -0,0 +1,297 @@
|
||||
"use strict";
|
||||
|
||||
import becca from "../../becca/becca.js";
|
||||
import BFileSystemMapping from "../../becca/entities/bfile_system_mapping.js";
|
||||
import fileSystemSyncInit from "../../services/file_system_sync_init.js";
|
||||
import log from "../../services/log.js";
|
||||
import ValidationError from "../../errors/validation_error.js";
|
||||
import fs from "fs-extra";
|
||||
import path from "path";
|
||||
import { router, asyncApiRoute, apiRoute } from "../route_api.js";
|
||||
|
||||
interface FileStat {
|
||||
isFile: boolean;
|
||||
isDirectory: boolean;
|
||||
size: number;
|
||||
modified: string;
|
||||
}
|
||||
|
||||
// Get all file system mappings
|
||||
apiRoute("get", "/mappings", () => {
|
||||
const mappings = Object.values(becca.fileSystemMappings || {}).map(mapping => ({
|
||||
mappingId: mapping.mappingId,
|
||||
noteId: mapping.noteId,
|
||||
filePath: mapping.filePath,
|
||||
syncDirection: mapping.syncDirection,
|
||||
isActive: mapping.isActive,
|
||||
includeSubtree: mapping.includeSubtree,
|
||||
preserveHierarchy: mapping.preserveHierarchy,
|
||||
contentFormat: mapping.contentFormat,
|
||||
excludePatterns: mapping.excludePatterns,
|
||||
lastSyncTime: mapping.lastSyncTime,
|
||||
syncErrors: mapping.syncErrors,
|
||||
dateCreated: mapping.dateCreated,
|
||||
dateModified: mapping.dateModified
|
||||
}));
|
||||
|
||||
return mappings;
|
||||
});
|
||||
|
||||
// Get a specific file system mapping
|
||||
apiRoute("get", "/mappings/:mappingId", (req) => {
|
||||
const { mappingId } = req.params;
|
||||
const mapping = becca.fileSystemMappings[mappingId];
|
||||
|
||||
if (!mapping) {
|
||||
return [404, { error: "Mapping not found" }];
|
||||
}
|
||||
|
||||
return {
|
||||
mappingId: mapping.mappingId,
|
||||
noteId: mapping.noteId,
|
||||
filePath: mapping.filePath,
|
||||
syncDirection: mapping.syncDirection,
|
||||
isActive: mapping.isActive,
|
||||
includeSubtree: mapping.includeSubtree,
|
||||
preserveHierarchy: mapping.preserveHierarchy,
|
||||
contentFormat: mapping.contentFormat,
|
||||
excludePatterns: mapping.excludePatterns,
|
||||
lastSyncTime: mapping.lastSyncTime,
|
||||
syncErrors: mapping.syncErrors,
|
||||
dateCreated: mapping.dateCreated,
|
||||
dateModified: mapping.dateModified
|
||||
};
|
||||
});
|
||||
|
||||
// Create a new file system mapping
|
||||
asyncApiRoute("post", "/mappings", async (req) => {
|
||||
const {
|
||||
noteId,
|
||||
filePath,
|
||||
syncDirection = 'bidirectional',
|
||||
isActive = true,
|
||||
includeSubtree = false,
|
||||
preserveHierarchy = true,
|
||||
contentFormat = 'auto',
|
||||
excludePatterns = null
|
||||
} = req.body;
|
||||
|
||||
// Validate required fields
|
||||
if (!noteId || !filePath) {
|
||||
throw new ValidationError("noteId and filePath are required");
|
||||
}
|
||||
|
||||
// Validate note exists
|
||||
const note = becca.notes[noteId];
|
||||
if (!note) {
|
||||
throw new ValidationError(`Note ${noteId} not found`);
|
||||
}
|
||||
|
||||
// Check if mapping already exists for this note
|
||||
const existingMapping = becca.getFileSystemMappingByNoteId(noteId);
|
||||
if (existingMapping) {
|
||||
throw new ValidationError(`File system mapping already exists for note ${noteId}`);
|
||||
}
|
||||
|
||||
// Validate file path exists
|
||||
const normalizedPath = path.resolve(filePath);
|
||||
if (!await fs.pathExists(normalizedPath)) {
|
||||
throw new ValidationError(`File path does not exist: ${normalizedPath}`);
|
||||
}
|
||||
|
||||
// Validate sync direction
|
||||
const validDirections = ['bidirectional', 'trilium_to_disk', 'disk_to_trilium'];
|
||||
if (!validDirections.includes(syncDirection)) {
|
||||
throw new ValidationError(`Invalid sync direction. Must be one of: ${validDirections.join(', ')}`);
|
||||
}
|
||||
|
||||
// Validate content format
|
||||
const validFormats = ['auto', 'markdown', 'html', 'raw'];
|
||||
if (!validFormats.includes(contentFormat)) {
|
||||
throw new ValidationError(`Invalid content format. Must be one of: ${validFormats.join(', ')}`);
|
||||
}
|
||||
|
||||
// Create the mapping
|
||||
const mapping = new BFileSystemMapping({
|
||||
noteId,
|
||||
filePath: normalizedPath,
|
||||
syncDirection,
|
||||
isActive: isActive ? 1 : 0,
|
||||
includeSubtree: includeSubtree ? 1 : 0,
|
||||
preserveHierarchy: preserveHierarchy ? 1 : 0,
|
||||
contentFormat,
|
||||
excludePatterns: Array.isArray(excludePatterns) ? JSON.stringify(excludePatterns) : excludePatterns
|
||||
}).save();
|
||||
|
||||
log.info(`Created file system mapping ${mapping.mappingId} for note ${noteId} -> ${normalizedPath}`);
|
||||
|
||||
return [201, {
|
||||
mappingId: mapping.mappingId,
|
||||
noteId: mapping.noteId,
|
||||
filePath: mapping.filePath,
|
||||
syncDirection: mapping.syncDirection,
|
||||
isActive: mapping.isActive,
|
||||
includeSubtree: mapping.includeSubtree,
|
||||
preserveHierarchy: mapping.preserveHierarchy,
|
||||
contentFormat: mapping.contentFormat,
|
||||
excludePatterns: mapping.excludePatterns
|
||||
}];
|
||||
});
|
||||
|
||||
// Update a file system mapping
|
||||
asyncApiRoute("put", "/mappings/:mappingId", async (req) => {
|
||||
const { mappingId } = req.params;
|
||||
const mapping = becca.fileSystemMappings[mappingId];
|
||||
|
||||
if (!mapping) {
|
||||
return [404, { error: "Mapping not found" }];
|
||||
}
|
||||
|
||||
const {
|
||||
filePath,
|
||||
syncDirection,
|
||||
isActive,
|
||||
includeSubtree,
|
||||
preserveHierarchy,
|
||||
contentFormat,
|
||||
excludePatterns
|
||||
} = req.body;
|
||||
|
||||
// Update fields if provided
|
||||
if (filePath !== undefined) {
|
||||
const normalizedPath = path.resolve(filePath);
|
||||
if (!await fs.pathExists(normalizedPath)) {
|
||||
throw new ValidationError(`File path does not exist: ${normalizedPath}`);
|
||||
}
|
||||
mapping.filePath = normalizedPath;
|
||||
}
|
||||
|
||||
if (syncDirection !== undefined) {
|
||||
const validDirections = ['bidirectional', 'trilium_to_disk', 'disk_to_trilium'];
|
||||
if (!validDirections.includes(syncDirection)) {
|
||||
throw new ValidationError(`Invalid sync direction. Must be one of: ${validDirections.join(', ')}`);
|
||||
}
|
||||
mapping.syncDirection = syncDirection;
|
||||
}
|
||||
|
||||
if (isActive !== undefined) {
|
||||
mapping.isActive = !!isActive;
|
||||
}
|
||||
|
||||
if (includeSubtree !== undefined) {
|
||||
mapping.includeSubtree = !!includeSubtree;
|
||||
}
|
||||
|
||||
if (preserveHierarchy !== undefined) {
|
||||
mapping.preserveHierarchy = !!preserveHierarchy;
|
||||
}
|
||||
|
||||
if (contentFormat !== undefined) {
|
||||
const validFormats = ['auto', 'markdown', 'html', 'raw'];
|
||||
if (!validFormats.includes(contentFormat)) {
|
||||
throw new ValidationError(`Invalid content format. Must be one of: ${validFormats.join(', ')}`);
|
||||
}
|
||||
mapping.contentFormat = contentFormat;
|
||||
}
|
||||
|
||||
if (excludePatterns !== undefined) {
|
||||
mapping.excludePatterns = Array.isArray(excludePatterns) ? excludePatterns : null;
|
||||
}
|
||||
|
||||
mapping.save();
|
||||
|
||||
log.info(`Updated file system mapping ${mappingId}`);
|
||||
|
||||
return {
|
||||
mappingId: mapping.mappingId,
|
||||
noteId: mapping.noteId,
|
||||
filePath: mapping.filePath,
|
||||
syncDirection: mapping.syncDirection,
|
||||
isActive: mapping.isActive,
|
||||
includeSubtree: mapping.includeSubtree,
|
||||
preserveHierarchy: mapping.preserveHierarchy,
|
||||
contentFormat: mapping.contentFormat,
|
||||
excludePatterns: mapping.excludePatterns
|
||||
};
|
||||
});
|
||||
|
||||
// Delete a file system mapping
|
||||
apiRoute("delete", "/mappings/:mappingId", (req) => {
|
||||
const { mappingId } = req.params;
|
||||
const mapping = becca.fileSystemMappings[mappingId];
|
||||
|
||||
if (!mapping) {
|
||||
return [404, { error: "Mapping not found" }];
|
||||
}
|
||||
|
||||
mapping.markAsDeleted();
|
||||
|
||||
log.info(`Deleted file system mapping ${mappingId}`);
|
||||
|
||||
return { success: true };
|
||||
});
|
||||
|
||||
// Trigger full sync for a mapping
|
||||
asyncApiRoute("post", "/mappings/:mappingId/sync", async (req) => {
|
||||
const { mappingId } = req.params;
|
||||
|
||||
if (!fileSystemSyncInit.isInitialized()) {
|
||||
return [503, { error: "File system sync is not initialized" }];
|
||||
}
|
||||
|
||||
const result = await fileSystemSyncInit.fullSync(mappingId);
|
||||
|
||||
if (result.success) {
|
||||
return result;
|
||||
} else {
|
||||
return [400, result];
|
||||
}
|
||||
});
|
||||
|
||||
// Get sync status for all mappings
|
||||
apiRoute("get", "/status", () => {
|
||||
return fileSystemSyncInit.getStatus();
|
||||
});
|
||||
|
||||
// Enable file system sync
|
||||
asyncApiRoute("post", "/enable", async () => {
|
||||
await fileSystemSyncInit.enable();
|
||||
return { success: true, message: "File system sync enabled" };
|
||||
});
|
||||
|
||||
// Disable file system sync
|
||||
asyncApiRoute("post", "/disable", async () => {
|
||||
await fileSystemSyncInit.disable();
|
||||
return { success: true, message: "File system sync disabled" };
|
||||
});
|
||||
|
||||
// Validate file path
|
||||
asyncApiRoute("post", "/validate-path", async (req) => {
|
||||
const { filePath } = req.body;
|
||||
|
||||
if (!filePath) {
|
||||
throw new ValidationError("filePath is required");
|
||||
}
|
||||
|
||||
const normalizedPath = path.resolve(filePath);
|
||||
const exists = await fs.pathExists(normalizedPath);
|
||||
|
||||
let stats: FileStat | null = null;
|
||||
if (exists) {
|
||||
const fileStats = await fs.stat(normalizedPath);
|
||||
stats = {
|
||||
isFile: fileStats.isFile(),
|
||||
isDirectory: fileStats.isDirectory(),
|
||||
size: fileStats.size,
|
||||
modified: fileStats.mtime.toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
path: normalizedPath,
|
||||
exists,
|
||||
stats
|
||||
};
|
||||
});
|
||||
|
||||
export default router;
|
||||
@@ -93,6 +93,7 @@ const ALLOWED_OPTIONS = new Set<OptionNames>([
|
||||
"redirectBareDomain",
|
||||
"showLoginInShareTheme",
|
||||
"splitEditorOrientation",
|
||||
"fileSystemSyncEnabled",
|
||||
|
||||
// AI/LLM integration options
|
||||
"aiEnabled",
|
||||
|
||||
@@ -59,6 +59,7 @@ import openaiRoute from "./api/openai.js";
|
||||
import anthropicRoute from "./api/anthropic.js";
|
||||
import llmRoute from "./api/llm.js";
|
||||
import systemInfoRoute from "./api/system_info.js";
|
||||
import fileSystemSyncRoute from "./api/file_system_sync.js";
|
||||
|
||||
import etapiAuthRoutes from "../etapi/auth.js";
|
||||
import etapiAppInfoRoutes from "../etapi/app_info.js";
|
||||
@@ -385,6 +386,9 @@ function register(app: express.Application) {
|
||||
asyncApiRoute(GET, "/api/llm/providers/openai/models", openaiRoute.listModels);
|
||||
asyncApiRoute(GET, "/api/llm/providers/anthropic/models", anthropicRoute.listModels);
|
||||
|
||||
// File system sync API
|
||||
app.use("/api/file-system-sync", [auth.checkApiAuthOrElectron, csrfMiddleware], fileSystemSyncRoute);
|
||||
|
||||
// API Documentation
|
||||
apiDocsRoute(app);
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import build from "./build.js";
|
||||
import packageJson from "../../package.json" with { type: "json" };
|
||||
import dataDir from "./data_dir.js";
|
||||
|
||||
const APP_DB_VERSION = 233;
|
||||
const APP_DB_VERSION = 234;
|
||||
const SYNC_VERSION = 36;
|
||||
const CLIPPER_PROTOCOL_VERSION = "1.0";
|
||||
|
||||
|
||||
464
apps/server/src/services/file_system_content_converter.ts
Normal file
464
apps/server/src/services/file_system_content_converter.ts
Normal file
@@ -0,0 +1,464 @@
|
||||
"use strict";
|
||||
|
||||
import path from "path";
|
||||
import log from "./log.js";
|
||||
import markdownExportService from "./export/markdown.js";
|
||||
import markdownImportService from "./import/markdown.js";
|
||||
import BNote from "../becca/entities/bnote.js";
|
||||
import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js";
|
||||
import utils from "./utils.js";
|
||||
import { type NoteType } from "@triliumnext/commons";
|
||||
|
||||
export interface ConversionResult {
|
||||
content: string | Buffer;
|
||||
attributes?: Array<{ type: 'label' | 'relation'; name: string; value: string; isInheritable?: boolean }>;
|
||||
mime?: string;
|
||||
type?: NoteType;
|
||||
}
|
||||
|
||||
export interface ConversionOptions {
|
||||
preserveAttributes?: boolean;
|
||||
includeFrontmatter?: boolean;
|
||||
relativeImagePaths?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Content converter for file system sync operations
|
||||
* Handles conversion between Trilium note formats and file system formats
|
||||
*/
|
||||
class FileSystemContentConverter {
|
||||
|
||||
/**
|
||||
* Convert note content to file format based on mapping configuration
|
||||
*/
|
||||
async noteToFile(note: BNote, mapping: BFileSystemMapping, filePath: string, options: ConversionOptions = {}): Promise<ConversionResult> {
|
||||
const fileExt = path.extname(filePath).toLowerCase();
|
||||
const contentFormat = mapping.contentFormat === 'auto' ? this.detectFormatFromExtension(fileExt) : mapping.contentFormat;
|
||||
|
||||
switch (contentFormat) {
|
||||
case 'markdown':
|
||||
return this.noteToMarkdown(note, options);
|
||||
case 'html':
|
||||
return this.noteToHtml(note, options);
|
||||
case 'raw':
|
||||
default:
|
||||
return this.noteToRaw(note, options);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert file content to note format based on mapping configuration
|
||||
*/
|
||||
async fileToNote(fileContent: string | Buffer, mapping: BFileSystemMapping, filePath: string, options: ConversionOptions = {}): Promise<ConversionResult> {
|
||||
const fileExt = path.extname(filePath).toLowerCase();
|
||||
const contentFormat = mapping.contentFormat === 'auto' ? this.detectFormatFromExtension(fileExt) : mapping.contentFormat;
|
||||
|
||||
// Convert Buffer to string for text formats
|
||||
const content = Buffer.isBuffer(fileContent) ? fileContent.toString('utf8') : fileContent;
|
||||
|
||||
switch (contentFormat) {
|
||||
case 'markdown':
|
||||
// Extract title from note for proper H1 deduplication
|
||||
const note = mapping.note;
|
||||
const title = note ? note.title : path.basename(filePath, path.extname(filePath));
|
||||
return this.markdownToNote(content, options, title);
|
||||
case 'html':
|
||||
return this.htmlToNote(content, options);
|
||||
case 'raw':
|
||||
default:
|
||||
return this.rawToNote(fileContent, fileExt, options);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect content format from file extension
|
||||
*/
|
||||
private detectFormatFromExtension(extension: string): 'markdown' | 'html' | 'raw' {
|
||||
const markdownExts = ['.md', '.markdown', '.mdown', '.mkd'];
|
||||
const htmlExts = ['.html', '.htm'];
|
||||
|
||||
if (markdownExts.includes(extension)) {
|
||||
return 'markdown';
|
||||
} else if (htmlExts.includes(extension)) {
|
||||
return 'html';
|
||||
} else {
|
||||
return 'raw';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert note to Markdown format
|
||||
*/
|
||||
private async noteToMarkdown(note: BNote, options: ConversionOptions): Promise<ConversionResult> {
|
||||
try {
|
||||
let content = note.getContent() as string;
|
||||
|
||||
// Convert HTML content to Markdown
|
||||
if (note.type === 'text' && note.mime === 'text/html') {
|
||||
content = markdownExportService.toMarkdown(content);
|
||||
}
|
||||
|
||||
// Add frontmatter with note attributes if requested
|
||||
if (options.includeFrontmatter && options.preserveAttributes) {
|
||||
const frontmatter = this.createFrontmatter(note);
|
||||
if (frontmatter) {
|
||||
content = `---\n${frontmatter}\n---\n\n${content}`;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
mime: 'text/markdown',
|
||||
type: 'text'
|
||||
};
|
||||
} catch (error) {
|
||||
log.error(`Error converting note ${note.noteId} to Markdown: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert note to HTML format
|
||||
*/
|
||||
private async noteToHtml(note: BNote, options: ConversionOptions): Promise<ConversionResult> {
|
||||
let content = note.getContent() as string;
|
||||
|
||||
// If note is already HTML, just clean it up
|
||||
if (note.type === 'text' && note.mime === 'text/html') {
|
||||
// Could add HTML processing here if needed
|
||||
} else if (note.type === 'code') {
|
||||
// Wrap code content in pre/code tags
|
||||
const language = this.getLanguageFromMime(note.mime);
|
||||
content = `<pre><code class="language-${language}">${utils.escapeHtml(content)}</code></pre>`;
|
||||
}
|
||||
|
||||
// Add HTML frontmatter as comments if requested
|
||||
if (options.includeFrontmatter && options.preserveAttributes) {
|
||||
const frontmatter = this.createFrontmatter(note);
|
||||
if (frontmatter) {
|
||||
content = `<!-- \n${frontmatter}\n-->\n\n${content}`;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
content,
|
||||
mime: 'text/html',
|
||||
type: 'text'
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert note to raw format (preserve original content)
|
||||
*/
|
||||
private async noteToRaw(note: BNote, options: ConversionOptions): Promise<ConversionResult> {
|
||||
const content = note.getContent();
|
||||
|
||||
return {
|
||||
content,
|
||||
mime: note.mime,
|
||||
type: note.type
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Markdown content to note format
|
||||
*/
|
||||
private async markdownToNote(content: string, options: ConversionOptions, title: string = ''): Promise<ConversionResult> {
|
||||
try {
|
||||
let processedContent = content;
|
||||
let attributes: ConversionResult['attributes'] = [];
|
||||
|
||||
// Extract frontmatter if present
|
||||
if (options.preserveAttributes) {
|
||||
const frontmatterResult = this.extractFrontmatter(content);
|
||||
processedContent = frontmatterResult.content;
|
||||
attributes = frontmatterResult.attributes;
|
||||
}
|
||||
|
||||
// Convert Markdown to HTML using the correct method
|
||||
// The title helps deduplicate <h1> tags with the note title
|
||||
const htmlContent = markdownImportService.renderToHtml(processedContent, title);
|
||||
|
||||
return {
|
||||
content: htmlContent,
|
||||
attributes,
|
||||
mime: 'text/html',
|
||||
type: 'text'
|
||||
};
|
||||
} catch (error) {
|
||||
log.error(`Error converting Markdown to note: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert HTML content to note format
|
||||
*/
|
||||
private async htmlToNote(content: string, options: ConversionOptions): Promise<ConversionResult> {
|
||||
let processedContent = content;
|
||||
let attributes: ConversionResult['attributes'] = [];
|
||||
|
||||
// Extract HTML comment frontmatter if present
|
||||
if (options.preserveAttributes) {
|
||||
const frontmatterResult = this.extractHtmlFrontmatter(content);
|
||||
processedContent = frontmatterResult.content;
|
||||
attributes = frontmatterResult.attributes;
|
||||
}
|
||||
|
||||
return {
|
||||
content: processedContent,
|
||||
attributes,
|
||||
mime: 'text/html',
|
||||
type: 'text'
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert raw content to note format
|
||||
*/
|
||||
private async rawToNote(content: string | Buffer, extension: string, options: ConversionOptions): Promise<ConversionResult> {
|
||||
// Determine note type and mime based on file extension
|
||||
const { type, mime } = this.getTypeAndMimeFromExtension(extension);
|
||||
|
||||
return {
|
||||
content,
|
||||
mime,
|
||||
type
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create YAML frontmatter from note attributes
|
||||
*/
|
||||
private createFrontmatter(note: BNote): string | null {
|
||||
const attributes = note.getOwnedAttributes();
|
||||
if (attributes.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const yamlLines: string[] = [];
|
||||
yamlLines.push(`title: "${note.title.replace(/"/g, '\\"')}"`);
|
||||
yamlLines.push(`noteId: "${note.noteId}"`);
|
||||
yamlLines.push(`type: "${note.type}"`);
|
||||
yamlLines.push(`mime: "${note.mime}"`);
|
||||
|
||||
const labels = attributes.filter(attr => attr.type === 'label');
|
||||
const relations = attributes.filter(attr => attr.type === 'relation');
|
||||
|
||||
if (labels.length > 0) {
|
||||
yamlLines.push('labels:');
|
||||
for (const label of labels) {
|
||||
const inheritable = label.isInheritable ? ' (inheritable)' : '';
|
||||
yamlLines.push(` - name: "${label.name}"`);
|
||||
yamlLines.push(` value: "${label.value.replace(/"/g, '\\"')}"`);
|
||||
if (label.isInheritable) {
|
||||
yamlLines.push(` inheritable: true`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (relations.length > 0) {
|
||||
yamlLines.push('relations:');
|
||||
for (const relation of relations) {
|
||||
yamlLines.push(` - name: "${relation.name}"`);
|
||||
yamlLines.push(` target: "${relation.value}"`);
|
||||
if (relation.isInheritable) {
|
||||
yamlLines.push(` inheritable: true`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yamlLines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract YAML frontmatter from Markdown content
|
||||
*/
|
||||
private extractFrontmatter(content: string): { content: string; attributes: ConversionResult['attributes'] } {
|
||||
const frontmatterRegex = /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/;
|
||||
const match = content.match(frontmatterRegex);
|
||||
|
||||
if (!match) {
|
||||
return { content, attributes: [] };
|
||||
}
|
||||
|
||||
const frontmatterYaml = match[1];
|
||||
const mainContent = match[2];
|
||||
|
||||
try {
|
||||
const attributes = this.parseFrontmatterYaml(frontmatterYaml);
|
||||
return { content: mainContent, attributes };
|
||||
} catch (error) {
|
||||
log.info(`Error parsing frontmatter YAML: ${error}`);
|
||||
return { content, attributes: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract frontmatter from HTML comments
|
||||
*/
|
||||
private extractHtmlFrontmatter(content: string): { content: string; attributes: ConversionResult['attributes'] } {
|
||||
const frontmatterRegex = /^<!--\s*\n([\s\S]*?)\n-->\s*\n([\s\S]*)$/;
|
||||
const match = content.match(frontmatterRegex);
|
||||
|
||||
if (!match) {
|
||||
return { content, attributes: [] };
|
||||
}
|
||||
|
||||
const frontmatterYaml = match[1];
|
||||
const mainContent = match[2];
|
||||
|
||||
try {
|
||||
const attributes = this.parseFrontmatterYaml(frontmatterYaml);
|
||||
return { content: mainContent, attributes };
|
||||
} catch (error) {
|
||||
log.info(`Error parsing HTML frontmatter YAML: ${error}`);
|
||||
return { content, attributes: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse YAML frontmatter into attributes (simplified YAML parser)
|
||||
*/
|
||||
private parseFrontmatterYaml(yaml: string): ConversionResult['attributes'] {
|
||||
const attributes: ConversionResult['attributes'] = [];
|
||||
const lines = yaml.split('\n');
|
||||
|
||||
let currentSection: 'labels' | 'relations' | null = null;
|
||||
let currentItem: any = {};
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
if (trimmed === 'labels:') {
|
||||
currentSection = 'labels';
|
||||
continue;
|
||||
} else if (trimmed === 'relations:') {
|
||||
currentSection = 'relations';
|
||||
continue;
|
||||
} else if (trimmed.startsWith('- name:')) {
|
||||
// Save previous item if exists
|
||||
if (currentItem.name && currentSection) {
|
||||
attributes.push({
|
||||
type: currentSection === 'labels' ? 'label' : 'relation',
|
||||
name: currentItem.name,
|
||||
value: currentItem.value || currentItem.target || '',
|
||||
isInheritable: currentItem.inheritable || false
|
||||
});
|
||||
}
|
||||
|
||||
currentItem = { name: this.extractQuotedValue(trimmed) };
|
||||
} else if (trimmed.startsWith('name:')) {
|
||||
currentItem.name = this.extractQuotedValue(trimmed);
|
||||
} else if (trimmed.startsWith('value:')) {
|
||||
currentItem.value = this.extractQuotedValue(trimmed);
|
||||
} else if (trimmed.startsWith('target:')) {
|
||||
currentItem.target = this.extractQuotedValue(trimmed);
|
||||
} else if (trimmed.startsWith('inheritable:')) {
|
||||
currentItem.inheritable = trimmed.includes('true');
|
||||
}
|
||||
}
|
||||
|
||||
// Save last item
|
||||
if (currentItem.name && currentSection) {
|
||||
attributes.push({
|
||||
type: currentSection === 'labels' ? 'label' : 'relation',
|
||||
name: currentItem.name,
|
||||
value: currentItem.value || currentItem.target || '',
|
||||
isInheritable: currentItem.inheritable || false
|
||||
});
|
||||
}
|
||||
|
||||
return attributes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract quoted value from YAML line
|
||||
*/
|
||||
private extractQuotedValue(line: string): string {
|
||||
const match = line.match(/:\s*"([^"]+)"/);
|
||||
return match ? match[1].replace(/\\"/g, '"') : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get language identifier from MIME type
|
||||
*/
|
||||
private getLanguageFromMime(mime: string): string {
|
||||
const mimeToLang: Record<string, string> = {
|
||||
'application/javascript': 'javascript',
|
||||
'text/javascript': 'javascript',
|
||||
'application/typescript': 'typescript',
|
||||
'text/typescript': 'typescript',
|
||||
'application/json': 'json',
|
||||
'text/css': 'css',
|
||||
'text/html': 'html',
|
||||
'application/xml': 'xml',
|
||||
'text/xml': 'xml',
|
||||
'text/x-python': 'python',
|
||||
'text/x-java': 'java',
|
||||
'text/x-csharp': 'csharp',
|
||||
'text/x-sql': 'sql',
|
||||
'text/x-sh': 'bash',
|
||||
'text/x-yaml': 'yaml'
|
||||
};
|
||||
|
||||
return mimeToLang[mime] || 'text';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get note type and MIME type from file extension
|
||||
*/
|
||||
private getTypeAndMimeFromExtension(extension: string): { type: NoteType; mime: string } {
|
||||
const extToType: Record<string, { type: NoteType; mime: string }> = {
|
||||
'.txt': { type: 'text', mime: 'text/plain' },
|
||||
'.md': { type: 'text', mime: 'text/markdown' },
|
||||
'.html': { type: 'text', mime: 'text/html' },
|
||||
'.htm': { type: 'text', mime: 'text/html' },
|
||||
'.js': { type: 'code', mime: 'application/javascript' },
|
||||
'.ts': { type: 'code', mime: 'application/typescript' },
|
||||
'.json': { type: 'code', mime: 'application/json' },
|
||||
'.css': { type: 'code', mime: 'text/css' },
|
||||
'.xml': { type: 'code', mime: 'application/xml' },
|
||||
'.py': { type: 'code', mime: 'text/x-python' },
|
||||
'.java': { type: 'code', mime: 'text/x-java' },
|
||||
'.cs': { type: 'code', mime: 'text/x-csharp' },
|
||||
'.sql': { type: 'code', mime: 'text/x-sql' },
|
||||
'.sh': { type: 'code', mime: 'text/x-sh' },
|
||||
'.yaml': { type: 'code', mime: 'text/x-yaml' },
|
||||
'.yml': { type: 'code', mime: 'text/x-yaml' },
|
||||
'.png': { type: 'image', mime: 'image/png' },
|
||||
'.jpg': { type: 'image', mime: 'image/jpeg' },
|
||||
'.jpeg': { type: 'image', mime: 'image/jpeg' },
|
||||
'.gif': { type: 'image', mime: 'image/gif' },
|
||||
'.svg': { type: 'image', mime: 'image/svg+xml' }
|
||||
};
|
||||
|
||||
return extToType[extension] || { type: 'file', mime: 'application/octet-stream' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate if a file type is supported for sync
|
||||
*/
|
||||
isSupportedFileType(filePath: string): boolean {
|
||||
const extension = path.extname(filePath).toLowerCase();
|
||||
const textExtensions = ['.txt', '.md', '.html', '.htm', '.js', '.ts', '.json', '.css', '.xml', '.py', '.java', '.cs', '.sql', '.sh', '.yaml', '.yml'];
|
||||
const binaryExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf'];
|
||||
|
||||
return textExtensions.includes(extension) || binaryExtensions.includes(extension);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file should be treated as binary
|
||||
*/
|
||||
isBinaryFile(filePath: string): boolean {
|
||||
const extension = path.extname(filePath).toLowerCase();
|
||||
const binaryExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.svg', '.pdf', '.doc', '.docx', '.zip', '.tar', '.gz'];
|
||||
|
||||
return binaryExtensions.includes(extension);
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
const fileSystemContentConverter = new FileSystemContentConverter();
|
||||
|
||||
export default fileSystemContentConverter;
|
||||
932
apps/server/src/services/file_system_sync.ts
Normal file
932
apps/server/src/services/file_system_sync.ts
Normal file
@@ -0,0 +1,932 @@
|
||||
"use strict";
|
||||
|
||||
import fs from "fs-extra";
|
||||
import path from "path";
|
||||
import crypto from "crypto";
|
||||
import log from "./log.js";
|
||||
import becca from "../becca/becca.js";
|
||||
import BNote from "../becca/entities/bnote.js";
|
||||
import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js";
|
||||
import BFileNoteMapping from "../becca/entities/bfile_note_mapping.js";
|
||||
import BAttribute from "../becca/entities/battribute.js";
|
||||
import BBranch from "../becca/entities/bbranch.js";
|
||||
import fileSystemContentConverter from "./file_system_content_converter.js";
|
||||
import fileSystemWatcher from "./file_system_watcher.js";
|
||||
import eventService from "./events.js";
|
||||
import noteService from "./notes.js";
|
||||
|
||||
export interface SyncResult {
|
||||
success: boolean;
|
||||
message?: string;
|
||||
conflicts?: ConflictInfo[];
|
||||
}
|
||||
|
||||
export interface ConflictInfo {
|
||||
type: 'content' | 'structure' | 'metadata';
|
||||
filePath: string;
|
||||
noteId: string;
|
||||
fileModified: string;
|
||||
noteModified: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface SyncStats {
|
||||
filesProcessed: number;
|
||||
notesCreated: number;
|
||||
notesUpdated: number;
|
||||
filesCreated: number;
|
||||
filesUpdated: number;
|
||||
conflicts: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bidirectional sync engine between Trilium notes and file system
|
||||
*/
|
||||
class FileSystemSync {
|
||||
private isInitialized = false;
|
||||
private syncInProgress = new Set<string>(); // Track ongoing syncs by mapping ID
|
||||
|
||||
constructor() {
|
||||
this.setupEventHandlers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the sync engine
|
||||
*/
|
||||
async init() {
|
||||
if (this.isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info('Initializing file system sync engine...');
|
||||
|
||||
// Initialize file system watcher
|
||||
await fileSystemWatcher.init();
|
||||
|
||||
this.isInitialized = true;
|
||||
log.info('File system sync engine initialized');
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown the sync engine
|
||||
*/
|
||||
async shutdown() {
|
||||
if (!this.isInitialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info('Shutting down file system sync engine...');
|
||||
|
||||
await fileSystemWatcher.shutdown();
|
||||
|
||||
this.isInitialized = false;
|
||||
log.info('File system sync engine shutdown complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup event handlers for file changes and note changes
|
||||
*/
|
||||
private setupEventHandlers() {
|
||||
// Handle file changes from watcher
|
||||
eventService.subscribe('FILE_CHANGED', async ({ fileNoteMapping, mapping, fileContent, isNew }) => {
|
||||
await this.handleFileChanged(fileNoteMapping, mapping, fileContent, isNew);
|
||||
});
|
||||
|
||||
eventService.subscribe('FILE_DELETED', async ({ fileNoteMapping, mapping }) => {
|
||||
await this.handleFileDeleted(fileNoteMapping, mapping);
|
||||
});
|
||||
|
||||
// Handle note changes
|
||||
eventService.subscribe(eventService.NOTE_CONTENT_CHANGE, async ({ entity: note }) => {
|
||||
await this.handleNoteChanged(note as BNote);
|
||||
});
|
||||
|
||||
eventService.subscribe(eventService.ENTITY_CHANGED, async ({ entityName, entity }) => {
|
||||
if (entityName === 'notes') {
|
||||
await this.handleNoteChanged(entity as BNote);
|
||||
}
|
||||
});
|
||||
|
||||
eventService.subscribe(eventService.ENTITY_DELETED, async ({ entityName, entityId }) => {
|
||||
if (entityName === 'notes') {
|
||||
await this.handleNoteDeleted(entityId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform full sync for a specific mapping
|
||||
*/
|
||||
async fullSync(mappingId: string): Promise<SyncResult> {
|
||||
const mapping = becca.fileSystemMappings[mappingId];
|
||||
if (!mapping) {
|
||||
return { success: false, message: `Mapping ${mappingId} not found` };
|
||||
}
|
||||
|
||||
if (this.syncInProgress.has(mappingId)) {
|
||||
return { success: false, message: 'Sync already in progress for this mapping' };
|
||||
}
|
||||
|
||||
this.syncInProgress.add(mappingId);
|
||||
const stats: SyncStats = {
|
||||
filesProcessed: 0,
|
||||
notesCreated: 0,
|
||||
notesUpdated: 0,
|
||||
filesCreated: 0,
|
||||
filesUpdated: 0,
|
||||
conflicts: 0,
|
||||
errors: 0
|
||||
};
|
||||
|
||||
try {
|
||||
log.info(`Starting full sync for mapping ${mappingId}: ${mapping.filePath}`);
|
||||
|
||||
if (!await fs.pathExists(mapping.filePath)) {
|
||||
throw new Error(`Path does not exist: ${mapping.filePath}`);
|
||||
}
|
||||
|
||||
const pathStats = await fs.stat(mapping.filePath);
|
||||
|
||||
if (pathStats.isFile()) {
|
||||
await this.syncSingleFile(mapping, mapping.filePath, stats);
|
||||
} else if (pathStats.isDirectory()) {
|
||||
await this.syncDirectory(mapping, mapping.filePath, stats);
|
||||
}
|
||||
|
||||
// Reverse sync: export notes that don't have corresponding files
|
||||
if (mapping.canSyncToDisk) {
|
||||
await this.syncNotesToFiles(mapping, stats);
|
||||
}
|
||||
|
||||
mapping.updateLastSyncTime();
|
||||
mapping.clearSyncErrors();
|
||||
|
||||
log.info(`Full sync completed for mapping ${mappingId}. Stats: ${JSON.stringify(stats)}`);
|
||||
return { success: true, message: `Sync completed successfully. ${stats.filesProcessed} files processed.` };
|
||||
|
||||
} catch (error) {
|
||||
const errorMsg = `Full sync failed for mapping ${mappingId}: ${(error as Error).message}`;
|
||||
log.error(errorMsg);
|
||||
mapping.addSyncError(errorMsg);
|
||||
stats.errors++;
|
||||
return { success: false, message: errorMsg };
|
||||
} finally {
|
||||
this.syncInProgress.delete(mappingId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a single file
|
||||
*/
|
||||
private async syncSingleFile(mapping: BFileSystemMapping, filePath: string, stats: SyncStats) {
|
||||
if (!fileSystemContentConverter.isSupportedFileType(filePath)) {
|
||||
log.info(`DEBUG: Skipping unsupported file type: ${filePath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
stats.filesProcessed++;
|
||||
|
||||
// Check if file note mapping exists
|
||||
let fileNoteMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath);
|
||||
|
||||
if (fileNoteMapping) {
|
||||
await this.syncExistingFile(mapping, fileNoteMapping, stats);
|
||||
} else {
|
||||
await this.syncNewFile(mapping, filePath, stats);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a directory recursively
|
||||
*/
|
||||
private async syncDirectory(mapping: BFileSystemMapping, dirPath: string, stats: SyncStats) {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
|
||||
// Skip excluded patterns
|
||||
if (this.isPathExcluded(fullPath, mapping)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.isFile()) {
|
||||
await this.syncSingleFile(mapping, fullPath, stats);
|
||||
} else if (entry.isDirectory() && mapping.includeSubtree) {
|
||||
await this.syncDirectory(mapping, fullPath, stats);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync notes to files (reverse sync) - export notes that don't have corresponding files
|
||||
*/
|
||||
private async syncNotesToFiles(mapping: BFileSystemMapping, stats: SyncStats) {
|
||||
const rootNote = mapping.getNote();
|
||||
|
||||
// Sync the root note itself if it's mapped to a file
|
||||
const pathStats = await fs.stat(mapping.filePath);
|
||||
if (pathStats.isFile()) {
|
||||
await this.syncNoteToFile(mapping, rootNote, mapping.filePath, stats);
|
||||
} else {
|
||||
// Sync child notes in the subtree
|
||||
await this.syncNoteSubtreeToFiles(mapping, rootNote, mapping.filePath, stats);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a note subtree to files recursively
|
||||
*/
|
||||
private async syncNoteSubtreeToFiles(mapping: BFileSystemMapping, note: BNote, basePath: string, stats: SyncStats) {
|
||||
for (const childBranch of note.children) {
|
||||
const childNote = becca.notes[childBranch.noteId];
|
||||
if (!childNote) continue;
|
||||
|
||||
// Skip system notes and other special notes
|
||||
if (childNote.noteId.startsWith('_') || childNote.type === 'book') {
|
||||
if (mapping.includeSubtree) {
|
||||
// For book notes, recurse into children but don't create a file
|
||||
await this.syncNoteSubtreeToFiles(mapping, childNote, basePath, stats);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Generate file path for this note
|
||||
const fileExtension = this.getFileExtensionForNote(childNote, mapping);
|
||||
const fileName = this.sanitizeFileName(childNote.title) + fileExtension;
|
||||
const filePath = path.join(basePath, fileName);
|
||||
|
||||
// Check if file already exists or has a mapping
|
||||
const existingMapping = this.findFileNoteMappingByNote(mapping.mappingId, childNote.noteId);
|
||||
|
||||
if (!existingMapping && !await fs.pathExists(filePath)) {
|
||||
// Note doesn't have a file mapping and file doesn't exist - create it
|
||||
await this.syncNoteToFile(mapping, childNote, filePath, stats);
|
||||
}
|
||||
|
||||
// Recurse into children if includeSubtree is enabled
|
||||
if (mapping.includeSubtree && childNote.children.length > 0) {
|
||||
const childDir = path.join(basePath, this.sanitizeFileName(childNote.title));
|
||||
await fs.ensureDir(childDir);
|
||||
await this.syncNoteSubtreeToFiles(mapping, childNote, childDir, stats);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a single note to a file
|
||||
*/
|
||||
private async syncNoteToFile(mapping: BFileSystemMapping, note: BNote, filePath: string, stats: SyncStats) {
|
||||
try {
|
||||
// Convert note content to file format
|
||||
const conversion = await fileSystemContentConverter.noteToFile(note, mapping, filePath, {
|
||||
preserveAttributes: true,
|
||||
includeFrontmatter: true
|
||||
});
|
||||
|
||||
// Ensure directory exists
|
||||
await fs.ensureDir(path.dirname(filePath));
|
||||
|
||||
// Write file
|
||||
await fs.writeFile(filePath, conversion.content);
|
||||
|
||||
// Calculate file hash and get modification time
|
||||
const fileStats = await fs.stat(filePath);
|
||||
const fileHash = await this.calculateFileHash(filePath);
|
||||
|
||||
// Check if mapping already exists (safety check)
|
||||
const existingMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath);
|
||||
if (existingMapping) {
|
||||
log.info(`File mapping already exists for ${filePath}, skipping creation`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create file note mapping
|
||||
const fileNoteMapping = new BFileNoteMapping({
|
||||
mappingId: mapping.mappingId,
|
||||
noteId: note.noteId,
|
||||
filePath,
|
||||
fileHash,
|
||||
fileModifiedTime: fileStats.mtime.toISOString(),
|
||||
syncStatus: 'synced'
|
||||
}).save();
|
||||
|
||||
stats.filesCreated++;
|
||||
log.info(`Created file ${filePath} from note ${note.noteId}`);
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error creating file from note ${note.noteId}: ${error}`);
|
||||
mapping.addSyncError(`Error creating file from note ${note.noteId}: ${(error as Error).message}`);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync an existing file that has a note mapping
|
||||
*/
|
||||
private async syncExistingFile(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, stats: SyncStats) {
|
||||
const filePath = fileNoteMapping.filePath;
|
||||
|
||||
if (!await fs.pathExists(filePath)) {
|
||||
// File was deleted
|
||||
if (mapping.canSyncFromDisk) {
|
||||
await this.deleteNoteFromFileMapping(fileNoteMapping, stats);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const fileStats = await fs.stat(filePath);
|
||||
const fileHash = await this.calculateFileHash(filePath);
|
||||
const fileModifiedTime = fileStats.mtime.toISOString();
|
||||
|
||||
const note = fileNoteMapping.note;
|
||||
if (!note) {
|
||||
log.info(`Note not found for file mapping: ${fileNoteMapping.noteId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const fileChanged = fileNoteMapping.hasFileChanged(fileHash, fileModifiedTime);
|
||||
const noteChanged = fileNoteMapping.hasNoteChanged();
|
||||
|
||||
if (!fileChanged && !noteChanged) {
|
||||
// No changes
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileChanged && noteChanged) {
|
||||
// Conflict - both changed
|
||||
fileNoteMapping.markConflict();
|
||||
stats.conflicts++;
|
||||
log.info(`Conflict detected for ${filePath} - both file and note modified`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileChanged && mapping.canSyncFromDisk) {
|
||||
// Update note from file
|
||||
await this.updateNoteFromFile(mapping, fileNoteMapping, fileHash, fileModifiedTime, stats);
|
||||
} else if (noteChanged && mapping.canSyncToDisk) {
|
||||
// Update file from note
|
||||
await this.updateFileFromNote(mapping, fileNoteMapping, fileHash, fileModifiedTime, stats);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync a new file that doesn't have a note mapping
|
||||
*/
|
||||
private async syncNewFile(mapping: BFileSystemMapping, filePath: string, stats: SyncStats) {
|
||||
if (!mapping.canSyncFromDisk) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const fileStats = await fs.stat(filePath);
|
||||
const fileHash = await this.calculateFileHash(filePath);
|
||||
const fileModifiedTime = fileStats.mtime.toISOString();
|
||||
|
||||
// Create note from file
|
||||
const note = await this.createNoteFromFile(mapping, filePath);
|
||||
|
||||
// Check if mapping already exists (safety check)
|
||||
const existingMapping = this.findFileNoteMappingByPath(mapping.mappingId, filePath);
|
||||
if (existingMapping) {
|
||||
log.info(`File mapping already exists for ${filePath}, skipping creation`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create file note mapping
|
||||
const fileNoteMapping = new BFileNoteMapping({
|
||||
mappingId: mapping.mappingId,
|
||||
noteId: note.noteId,
|
||||
filePath,
|
||||
fileHash,
|
||||
fileModifiedTime,
|
||||
syncStatus: 'synced'
|
||||
}).save();
|
||||
|
||||
stats.notesCreated++;
|
||||
log.info(`Created note ${note.noteId} from file ${filePath}`);
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error creating note from file ${filePath}: ${error}`);
|
||||
mapping.addSyncError(`Error creating note from file ${filePath}: ${(error as Error).message}`);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new note from a file
|
||||
*/
|
||||
private async createNoteFromFile(mapping: BFileSystemMapping, filePath: string): Promise<BNote> {
|
||||
const fileContent = await fs.readFile(filePath);
|
||||
const fileName = path.basename(filePath, path.extname(filePath));
|
||||
|
||||
// Convert file content to note format
|
||||
const conversion = await fileSystemContentConverter.fileToNote(fileContent, mapping, filePath, {
|
||||
preserveAttributes: true,
|
||||
includeFrontmatter: true
|
||||
});
|
||||
|
||||
// Determine parent note
|
||||
const parentNote = this.getParentNoteForFile(mapping, filePath);
|
||||
|
||||
// Create the note
|
||||
const note = new BNote({
|
||||
title: fileName,
|
||||
type: conversion.type || 'text',
|
||||
mime: conversion.mime || 'text/html'
|
||||
}).save();
|
||||
|
||||
// Set content
|
||||
note.setContent(conversion.content);
|
||||
|
||||
// Create branch
|
||||
new BBranch({
|
||||
noteId: note.noteId,
|
||||
parentNoteId: parentNote.noteId
|
||||
}).save();
|
||||
|
||||
// Add attributes from conversion
|
||||
if (conversion.attributes) {
|
||||
for (const attr of conversion.attributes) {
|
||||
new BAttribute({
|
||||
noteId: note.noteId,
|
||||
type: attr.type,
|
||||
name: attr.name,
|
||||
value: attr.value,
|
||||
isInheritable: attr.isInheritable || false
|
||||
}).save();
|
||||
}
|
||||
}
|
||||
|
||||
return note;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update note content from file
|
||||
*/
|
||||
private async updateNoteFromFile(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, fileHash: string, fileModifiedTime: string, stats: SyncStats) {
|
||||
try {
|
||||
const note = fileNoteMapping.getNote();
|
||||
const fileContent = await fs.readFile(fileNoteMapping.filePath);
|
||||
|
||||
// Convert file content to note format
|
||||
const conversion = await fileSystemContentConverter.fileToNote(fileContent, mapping, fileNoteMapping.filePath, {
|
||||
preserveAttributes: true,
|
||||
includeFrontmatter: true
|
||||
});
|
||||
|
||||
// Update note content
|
||||
note.setContent(conversion.content);
|
||||
|
||||
// Update note type/mime if they changed
|
||||
if (conversion.type && conversion.type !== note.type) {
|
||||
note.type = conversion.type as any;
|
||||
note.save();
|
||||
}
|
||||
if (conversion.mime && conversion.mime !== note.mime) {
|
||||
note.mime = conversion.mime;
|
||||
note.save();
|
||||
}
|
||||
|
||||
// Update attributes if needed
|
||||
if (conversion.attributes) {
|
||||
// Remove existing attributes that came from file
|
||||
const existingAttrs = note.getOwnedAttributes();
|
||||
for (const attr of existingAttrs) {
|
||||
if (attr.name.startsWith('_fileSync_')) {
|
||||
attr.markAsDeleted();
|
||||
}
|
||||
}
|
||||
|
||||
// Add new attributes
|
||||
for (const attr of conversion.attributes) {
|
||||
new BAttribute({
|
||||
noteId: note.noteId,
|
||||
type: attr.type,
|
||||
name: attr.name,
|
||||
value: attr.value,
|
||||
isInheritable: attr.isInheritable || false
|
||||
}).save();
|
||||
}
|
||||
}
|
||||
|
||||
fileNoteMapping.markSynced(fileHash, fileModifiedTime);
|
||||
stats.notesUpdated++;
|
||||
|
||||
log.info(`DEBUG: Updated note ${note.noteId} from file ${fileNoteMapping.filePath}`);
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error updating note from file ${fileNoteMapping.filePath}: ${error}`);
|
||||
fileNoteMapping.markError();
|
||||
mapping.addSyncError(`Error updating note from file: ${(error as Error).message}`);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update file content from note
|
||||
*/
|
||||
private async updateFileFromNote(mapping: BFileSystemMapping, fileNoteMapping: BFileNoteMapping, currentFileHash: string, currentModifiedTime: string, stats: SyncStats) {
|
||||
try {
|
||||
const note = fileNoteMapping.getNote();
|
||||
|
||||
// Convert note content to file format
|
||||
const conversion = await fileSystemContentConverter.noteToFile(note, mapping, fileNoteMapping.filePath, {
|
||||
preserveAttributes: true,
|
||||
includeFrontmatter: true
|
||||
});
|
||||
|
||||
// Ensure directory exists
|
||||
await fs.ensureDir(path.dirname(fileNoteMapping.filePath));
|
||||
|
||||
// Write file
|
||||
await fs.writeFile(fileNoteMapping.filePath, conversion.content);
|
||||
|
||||
// Update file note mapping with new file info
|
||||
const newStats = await fs.stat(fileNoteMapping.filePath);
|
||||
const newFileHash = await this.calculateFileHash(fileNoteMapping.filePath);
|
||||
|
||||
fileNoteMapping.markSynced(newFileHash, newStats.mtime.toISOString());
|
||||
stats.filesUpdated++;
|
||||
|
||||
log.info(`DEBUG: Updated file ${fileNoteMapping.filePath} from note ${note.noteId}`);
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error updating file from note ${fileNoteMapping.noteId}: ${error}`);
|
||||
fileNoteMapping.markError();
|
||||
mapping.addSyncError(`Error updating file from note: ${(error as Error).message}`);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file change event from watcher
|
||||
*/
|
||||
private async handleFileChanged(fileNoteMapping: BFileNoteMapping, mapping: BFileSystemMapping, fileContent: Buffer, isNew: boolean) {
|
||||
if (this.syncInProgress.has(mapping.mappingId)) {
|
||||
return; // Skip if full sync in progress
|
||||
}
|
||||
|
||||
const stats: SyncStats = {
|
||||
filesProcessed: 1,
|
||||
notesCreated: 0,
|
||||
notesUpdated: 0,
|
||||
filesCreated: 0,
|
||||
filesUpdated: 0,
|
||||
conflicts: 0,
|
||||
errors: 0
|
||||
};
|
||||
|
||||
if (isNew) {
|
||||
await this.syncNewFile(mapping, fileNoteMapping.filePath, stats);
|
||||
} else {
|
||||
const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex');
|
||||
const fileStats = await fs.stat(fileNoteMapping.filePath);
|
||||
const fileModifiedTime = fileStats.mtime.toISOString();
|
||||
|
||||
await this.syncExistingFile(mapping, fileNoteMapping, stats);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file deletion event from watcher
|
||||
*/
|
||||
private async handleFileDeleted(fileNoteMapping: BFileNoteMapping, mapping: BFileSystemMapping) {
|
||||
if (this.syncInProgress.has(mapping.mappingId)) {
|
||||
return; // Skip if full sync in progress
|
||||
}
|
||||
|
||||
const stats: SyncStats = {
|
||||
filesProcessed: 0,
|
||||
notesCreated: 0,
|
||||
notesUpdated: 0,
|
||||
filesCreated: 0,
|
||||
filesUpdated: 0,
|
||||
conflicts: 0,
|
||||
errors: 0
|
||||
};
|
||||
|
||||
await this.deleteNoteFromFileMapping(fileNoteMapping, stats);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle note change event
|
||||
*/
|
||||
private async handleNoteChanged(note: BNote) {
|
||||
// Find all file mappings for this note
|
||||
const fileMappings = this.findFileNoteMappingsByNote(note.noteId);
|
||||
|
||||
for (const fileMapping of fileMappings) {
|
||||
const mapping = fileMapping.mapping;
|
||||
if (!mapping || !mapping.canSyncToDisk || this.syncInProgress.has(mapping.mappingId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if note was actually modified since last sync
|
||||
if (!fileMapping.hasNoteChanged()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const stats: SyncStats = {
|
||||
filesProcessed: 0,
|
||||
notesCreated: 0,
|
||||
notesUpdated: 0,
|
||||
filesCreated: 0,
|
||||
filesUpdated: 0,
|
||||
conflicts: 0,
|
||||
errors: 0
|
||||
};
|
||||
|
||||
// Check for conflicts
|
||||
if (await fs.pathExists(fileMapping.filePath)) {
|
||||
const fileStats = await fs.stat(fileMapping.filePath);
|
||||
const fileHash = await this.calculateFileHash(fileMapping.filePath);
|
||||
const fileModifiedTime = fileStats.mtime.toISOString();
|
||||
|
||||
if (fileMapping.hasFileChanged(fileHash, fileModifiedTime)) {
|
||||
// Conflict
|
||||
fileMapping.markConflict();
|
||||
log.info(`Conflict detected for note ${note.noteId} - both file and note modified`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Update file from note
|
||||
const currentFileHash = await this.calculateFileHash(fileMapping.filePath);
|
||||
const currentModifiedTime = (await fs.stat(fileMapping.filePath)).mtime.toISOString();
|
||||
|
||||
await this.updateFileFromNote(mapping, fileMapping, currentFileHash, currentModifiedTime, stats);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle note deletion event
|
||||
*/
|
||||
private async handleNoteDeleted(noteId: string) {
|
||||
// Find all file mappings for this note
|
||||
const fileMappings = this.findFileNoteMappingsByNote(noteId);
|
||||
|
||||
for (const fileMapping of fileMappings) {
|
||||
const mapping = fileMapping.mapping;
|
||||
if (!mapping || !mapping.canSyncToDisk || this.syncInProgress.has(mapping.mappingId)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Delete the file
|
||||
if (await fs.pathExists(fileMapping.filePath)) {
|
||||
await fs.remove(fileMapping.filePath);
|
||||
log.info(`Deleted file ${fileMapping.filePath} for deleted note ${noteId}`);
|
||||
}
|
||||
|
||||
// Delete the mapping
|
||||
fileMapping.markAsDeleted();
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error deleting file for note ${noteId}: ${error}`);
|
||||
mapping.addSyncError(`Error deleting file: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete note when file is deleted
|
||||
*/
|
||||
private async deleteNoteFromFileMapping(fileNoteMapping: BFileNoteMapping, stats: SyncStats) {
|
||||
try {
|
||||
const note = fileNoteMapping.note;
|
||||
if (note) {
|
||||
note.deleteNote();
|
||||
log.info(`Deleted note ${note.noteId} for deleted file ${fileNoteMapping.filePath}`);
|
||||
}
|
||||
|
||||
// Delete the mapping
|
||||
fileNoteMapping.markAsDeleted();
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error deleting note for file ${fileNoteMapping.filePath}: ${error}`);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get parent note for a file based on mapping configuration
|
||||
*/
|
||||
private getParentNoteForFile(mapping: BFileSystemMapping, filePath: string): BNote {
|
||||
const mappedNote = mapping.getNote();
|
||||
|
||||
if (!mapping.preserveHierarchy || !mapping.includeSubtree) {
|
||||
return mappedNote;
|
||||
}
|
||||
|
||||
// Calculate relative path from mapping root
|
||||
const relativePath = path.relative(mapping.filePath, path.dirname(filePath));
|
||||
|
||||
if (!relativePath || relativePath === '.') {
|
||||
return mappedNote;
|
||||
}
|
||||
|
||||
// Create directory structure as notes
|
||||
const pathParts = relativePath.split(path.sep);
|
||||
let currentParent = mappedNote;
|
||||
|
||||
for (const part of pathParts) {
|
||||
if (!part) continue;
|
||||
|
||||
// Look for existing child note with this name
|
||||
let childNote = currentParent.children.find(child => child.title === part);
|
||||
|
||||
if (!childNote) {
|
||||
// Create new note for this directory
|
||||
childNote = new BNote({
|
||||
title: part,
|
||||
type: 'text',
|
||||
mime: 'text/html'
|
||||
}).save();
|
||||
|
||||
childNote.setContent('<p>Directory note</p>');
|
||||
|
||||
// Create branch (notePosition will be auto-calculated)
|
||||
new BBranch({
|
||||
noteId: childNote.noteId,
|
||||
parentNoteId: currentParent.noteId
|
||||
}).save();
|
||||
}
|
||||
|
||||
currentParent = childNote;
|
||||
}
|
||||
|
||||
return currentParent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate SHA256 hash of a file
|
||||
*/
|
||||
private async calculateFileHash(filePath: string): Promise<string> {
|
||||
const content = await fs.readFile(filePath);
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path should be excluded based on mapping patterns
|
||||
*/
|
||||
private isPathExcluded(filePath: string, mapping: BFileSystemMapping): boolean {
|
||||
if (!mapping.excludePatterns) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const normalizedPath = path.normalize(filePath);
|
||||
const basename = path.basename(normalizedPath);
|
||||
|
||||
for (const pattern of mapping.excludePatterns) {
|
||||
if (typeof pattern === 'string') {
|
||||
// Simple string matching
|
||||
if (normalizedPath.includes(pattern) || basename.includes(pattern)) {
|
||||
return true;
|
||||
}
|
||||
} else if (pattern instanceof RegExp) {
|
||||
// Regex pattern
|
||||
if (pattern.test(normalizedPath) || pattern.test(basename)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find file note mapping by file path
|
||||
*/
|
||||
private findFileNoteMappingByPath(mappingId: string, filePath: string): BFileNoteMapping | null {
|
||||
const normalizedPath = path.normalize(filePath);
|
||||
|
||||
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
|
||||
if (mapping.mappingId === mappingId && path.normalize(mapping.filePath) === normalizedPath) {
|
||||
return mapping;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find all file note mappings for a note
|
||||
*/
|
||||
private findFileNoteMappingsByNote(noteId: string): BFileNoteMapping[] {
|
||||
const mappings: BFileNoteMapping[] = [];
|
||||
|
||||
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
|
||||
if (mapping.noteId === noteId) {
|
||||
mappings.push(mapping);
|
||||
}
|
||||
}
|
||||
|
||||
return mappings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find file note mapping by note ID within a specific mapping
|
||||
*/
|
||||
private findFileNoteMappingByNote(mappingId: string, noteId: string): BFileNoteMapping | null {
|
||||
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
|
||||
if (mapping.mappingId === mappingId && mapping.noteId === noteId) {
|
||||
return mapping;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get appropriate file extension for a note based on its type and mapping configuration
|
||||
*/
|
||||
private getFileExtensionForNote(note: BNote, mapping: BFileSystemMapping): string {
|
||||
const contentFormat = mapping.contentFormat;
|
||||
|
||||
if (contentFormat === 'markdown' || (contentFormat === 'auto' && note.type === 'text')) {
|
||||
return '.md';
|
||||
} else if (contentFormat === 'html' || (contentFormat === 'auto' && note.type === 'text' && note.mime === 'text/html')) {
|
||||
return '.html';
|
||||
} else if (note.type === 'code') {
|
||||
// Map MIME types to file extensions
|
||||
const mimeToExt: Record<string, string> = {
|
||||
'application/javascript': '.js',
|
||||
'text/javascript': '.js',
|
||||
'application/typescript': '.ts',
|
||||
'text/typescript': '.ts',
|
||||
'application/json': '.json',
|
||||
'text/css': '.css',
|
||||
'text/x-python': '.py',
|
||||
'text/x-java': '.java',
|
||||
'text/x-csharp': '.cs',
|
||||
'text/x-sql': '.sql',
|
||||
'text/x-sh': '.sh',
|
||||
'text/x-yaml': '.yaml',
|
||||
'application/xml': '.xml',
|
||||
'text/xml': '.xml'
|
||||
};
|
||||
return mimeToExt[note.mime] || '.txt';
|
||||
} else if (note.type === 'image') {
|
||||
const mimeToExt: Record<string, string> = {
|
||||
'image/png': '.png',
|
||||
'image/jpeg': '.jpg',
|
||||
'image/gif': '.gif',
|
||||
'image/svg+xml': '.svg'
|
||||
};
|
||||
return mimeToExt[note.mime] || '.png';
|
||||
} else {
|
||||
return '.txt';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize file name to be safe for file system
|
||||
*/
|
||||
private sanitizeFileName(fileName: string): string {
|
||||
// Replace invalid characters with underscores
|
||||
return fileName
|
||||
.replace(/[<>:"/\\|?*]/g, '_')
|
||||
.replace(/\s+/g, '_')
|
||||
.replace(/_{2,}/g, '_')
|
||||
.replace(/^_+|_+$/g, '')
|
||||
.substring(0, 100); // Limit length
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sync status for all mappings
|
||||
*/
|
||||
getSyncStatus() {
|
||||
const status: Record<string, any> = {};
|
||||
|
||||
for (const mapping of Object.values(becca.fileSystemMappings || {})) {
|
||||
const fileMappings = Object.values(becca.fileNoteMappings || {})
|
||||
.filter(fm => fm.mappingId === mapping.mappingId);
|
||||
|
||||
const conflicts = fileMappings.filter(fm => fm.syncStatus === 'conflict').length;
|
||||
const pending = fileMappings.filter(fm => fm.syncStatus === 'pending').length;
|
||||
const errors = fileMappings.filter(fm => fm.syncStatus === 'error').length;
|
||||
|
||||
status[mapping.mappingId] = {
|
||||
filePath: mapping.filePath,
|
||||
isActive: mapping.isActive,
|
||||
syncDirection: mapping.syncDirection,
|
||||
fileCount: fileMappings.length,
|
||||
conflicts,
|
||||
pending,
|
||||
errors,
|
||||
lastSyncTime: mapping.lastSyncTime,
|
||||
syncErrors: mapping.syncErrors,
|
||||
isRunning: this.syncInProgress.has(mapping.mappingId)
|
||||
};
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
const fileSystemSync = new FileSystemSync();
|
||||
|
||||
export default fileSystemSync;
|
||||
129
apps/server/src/services/file_system_sync_init.ts
Normal file
129
apps/server/src/services/file_system_sync_init.ts
Normal file
@@ -0,0 +1,129 @@
|
||||
"use strict";
|
||||
|
||||
import log from "./log.js";
|
||||
import fileSystemSync from "./file_system_sync.js";
|
||||
import eventService from "./events.js";
|
||||
import optionService from "./options.js";
|
||||
|
||||
/**
|
||||
* Initialization service for file system sync functionality
|
||||
*/
|
||||
class FileSystemSyncInit {
|
||||
private initialized = false;
|
||||
|
||||
/**
|
||||
* Initialize file system sync if enabled
|
||||
*/
|
||||
async init() {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if file system sync is enabled
|
||||
const isEnabled = optionService.getOption('fileSystemSyncEnabled') === 'true';
|
||||
|
||||
if (!isEnabled) {
|
||||
log.info('File system sync is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
log.info('Initializing file system sync...');
|
||||
|
||||
// Initialize the sync engine
|
||||
await fileSystemSync.init();
|
||||
|
||||
this.initialized = true;
|
||||
log.info('File system sync initialized successfully');
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Failed to initialize file system sync: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown file system sync
|
||||
*/
|
||||
async shutdown() {
|
||||
if (!this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
log.info('Shutting down file system sync...');
|
||||
|
||||
await fileSystemSync.shutdown();
|
||||
|
||||
this.initialized = false;
|
||||
log.info('File system sync shutdown complete');
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error shutting down file system sync: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file system sync is initialized
|
||||
*/
|
||||
isInitialized(): boolean {
|
||||
return this.initialized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get sync status
|
||||
*/
|
||||
getStatus() {
|
||||
if (!this.initialized) {
|
||||
return { enabled: false, initialized: false };
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: true,
|
||||
initialized: true,
|
||||
status: fileSystemSync.getSyncStatus()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable file system sync
|
||||
*/
|
||||
async enable() {
|
||||
optionService.setOption('fileSystemSyncEnabled', 'true');
|
||||
|
||||
if (!this.initialized) {
|
||||
await this.init();
|
||||
}
|
||||
|
||||
log.info('File system sync enabled');
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable file system sync
|
||||
*/
|
||||
async disable() {
|
||||
optionService.setOption('fileSystemSyncEnabled', 'false');
|
||||
|
||||
if (this.initialized) {
|
||||
await this.shutdown();
|
||||
}
|
||||
|
||||
log.info('File system sync disabled');
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform full sync for a specific mapping
|
||||
*/
|
||||
async fullSync(mappingId: string) {
|
||||
if (!this.initialized) {
|
||||
throw new Error('File system sync is not initialized');
|
||||
}
|
||||
|
||||
return await fileSystemSync.fullSync(mappingId);
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
const fileSystemSyncInit = new FileSystemSyncInit();
|
||||
|
||||
export default fileSystemSyncInit;
|
||||
457
apps/server/src/services/file_system_watcher.ts
Normal file
457
apps/server/src/services/file_system_watcher.ts
Normal file
@@ -0,0 +1,457 @@
|
||||
"use strict";
|
||||
|
||||
import chokidar from "chokidar";
|
||||
import path from "path";
|
||||
import fs from "fs-extra";
|
||||
import crypto from "crypto";
|
||||
import debounce from "debounce";
|
||||
import log from "./log.js";
|
||||
import becca from "../becca/becca.js";
|
||||
import BFileSystemMapping from "../becca/entities/bfile_system_mapping.js";
|
||||
import BFileNoteMapping from "../becca/entities/bfile_note_mapping.js";
|
||||
import eventService from "./events.js";
|
||||
import { newEntityId } from "./utils.js";
|
||||
import type { FSWatcher } from "chokidar";
|
||||
|
||||
interface WatchedMapping {
|
||||
mapping: BFileSystemMapping;
|
||||
watcher: FSWatcher;
|
||||
}
|
||||
|
||||
interface FileChangeEvent {
|
||||
type: 'add' | 'change' | 'unlink';
|
||||
filePath: string;
|
||||
mappingId: string;
|
||||
stats?: fs.Stats;
|
||||
}
|
||||
|
||||
class FileSystemWatcher {
|
||||
private watchers: Map<string, WatchedMapping> = new Map();
|
||||
private syncQueue: FileChangeEvent[] = [];
|
||||
private isProcessing = false;
|
||||
|
||||
// Debounced sync to batch multiple file changes
|
||||
private processSyncQueue = debounce(this._processSyncQueue.bind(this), 500);
|
||||
|
||||
constructor() {
|
||||
// Subscribe to entity changes to watch for new/updated/deleted mappings
|
||||
eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) => {
|
||||
if (entityName === 'file_system_mappings') {
|
||||
this.addWatcher(entity as BFileSystemMapping);
|
||||
}
|
||||
});
|
||||
|
||||
eventService.subscribe(eventService.ENTITY_CHANGED, ({ entityName, entity }) => {
|
||||
if (entityName === 'file_system_mappings') {
|
||||
this.updateWatcher(entity as BFileSystemMapping);
|
||||
}
|
||||
});
|
||||
|
||||
eventService.subscribe(eventService.ENTITY_DELETED, ({ entityName, entityId }) => {
|
||||
if (entityName === 'file_system_mappings') {
|
||||
this.removeWatcher(entityId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the file system watcher by setting up watchers for all active mappings
|
||||
*/
|
||||
async init() {
|
||||
log.info('Initializing file system watcher...');
|
||||
|
||||
try {
|
||||
const mappings = Object.values(becca.fileSystemMappings || {});
|
||||
for (const mapping of mappings) {
|
||||
if (mapping.isActive && mapping.canSyncFromDisk) {
|
||||
await this.addWatcher(mapping);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`File system watcher initialized with ${this.watchers.size} active mappings`);
|
||||
} catch (error) {
|
||||
log.error(`Failed to initialize file system watcher: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shutdown all watchers
|
||||
*/
|
||||
async shutdown() {
|
||||
log.info('Shutting down file system watcher...');
|
||||
|
||||
for (const [mappingId, { watcher }] of this.watchers) {
|
||||
await watcher.close();
|
||||
}
|
||||
|
||||
this.watchers.clear();
|
||||
log.info('File system watcher shutdown complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new file system watcher for a mapping
|
||||
*/
|
||||
private async addWatcher(mapping: BFileSystemMapping) {
|
||||
if (this.watchers.has(mapping.mappingId)) {
|
||||
await this.removeWatcher(mapping.mappingId);
|
||||
}
|
||||
|
||||
if (!mapping.isActive || !mapping.canSyncFromDisk) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if the file path exists
|
||||
if (!await fs.pathExists(mapping.filePath)) {
|
||||
log.info(`File path does not exist for mapping ${mapping.mappingId}: ${mapping.filePath}`);
|
||||
mapping.addSyncError(`File path does not exist: ${mapping.filePath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const stats = await fs.stat(mapping.filePath);
|
||||
const watchPath = stats.isDirectory() ? mapping.filePath : path.dirname(mapping.filePath);
|
||||
|
||||
const watcher = chokidar.watch(watchPath, {
|
||||
persistent: true,
|
||||
ignoreInitial: true,
|
||||
followSymlinks: false,
|
||||
depth: mapping.includeSubtree ? undefined : 0,
|
||||
ignored: this.buildIgnorePatterns(mapping)
|
||||
});
|
||||
|
||||
watcher.on('add', (filePath, stats) => {
|
||||
this.queueFileChange('add', filePath, mapping.mappingId, stats);
|
||||
});
|
||||
|
||||
watcher.on('change', (filePath, stats) => {
|
||||
this.queueFileChange('change', filePath, mapping.mappingId, stats);
|
||||
});
|
||||
|
||||
watcher.on('unlink', (filePath) => {
|
||||
this.queueFileChange('unlink', filePath, mapping.mappingId);
|
||||
});
|
||||
|
||||
watcher.on('error', (error) => {
|
||||
log.error(`File watcher error for mapping ${mapping.mappingId}: ${error}`);
|
||||
if (error && typeof error === "object" && "message" in error && typeof error.message === 'string') {
|
||||
mapping.addSyncError(`Watcher error: ${error.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
watcher.on('ready', () => {
|
||||
log.info(`File watcher ready for mapping ${mapping.mappingId}: ${mapping.filePath}`);
|
||||
});
|
||||
|
||||
this.watchers.set(mapping.mappingId, { mapping, watcher });
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Failed to create file watcher for mapping ${mapping.mappingId}: ${error}`);
|
||||
mapping.addSyncError(`Failed to create watcher: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing watcher (remove and re-add)
|
||||
*/
|
||||
private async updateWatcher(mapping: BFileSystemMapping) {
|
||||
await this.addWatcher(mapping);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a file system watcher
|
||||
*/
|
||||
private async removeWatcher(mappingId: string) {
|
||||
const watchedMapping = this.watchers.get(mappingId);
|
||||
if (watchedMapping) {
|
||||
await watchedMapping.watcher.close();
|
||||
this.watchers.delete(mappingId);
|
||||
log.info(`Removed file watcher for mapping ${mappingId}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build ignore patterns for chokidar based on mapping configuration
|
||||
*/
|
||||
private buildIgnorePatterns(mapping: BFileSystemMapping): (string | RegExp)[] {
|
||||
const patterns: (string | RegExp)[] = [
|
||||
// Always ignore common temp/system files
|
||||
/^\./, // Hidden files
|
||||
/\.tmp$/,
|
||||
/\.temp$/,
|
||||
/~$/, // Backup files
|
||||
/\.swp$/, // Vim swap files
|
||||
/\.DS_Store$/, // macOS
|
||||
/Thumbs\.db$/ // Windows
|
||||
];
|
||||
|
||||
// Add user-defined exclude patterns
|
||||
if (mapping.excludePatterns) {
|
||||
patterns.push(...mapping.excludePatterns);
|
||||
}
|
||||
|
||||
return patterns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a file change event for processing
|
||||
*/
|
||||
private queueFileChange(type: 'add' | 'change' | 'unlink', filePath: string, mappingId: string, stats?: fs.Stats) {
|
||||
this.syncQueue.push({
|
||||
type,
|
||||
filePath: path.normalize(filePath),
|
||||
mappingId,
|
||||
stats
|
||||
});
|
||||
|
||||
// Trigger debounced processing
|
||||
this.processSyncQueue();
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the sync queue (called after debounce delay)
|
||||
*/
|
||||
private async _processSyncQueue() {
|
||||
if (this.isProcessing || this.syncQueue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isProcessing = true;
|
||||
const eventsToProcess = [...this.syncQueue];
|
||||
this.syncQueue = [];
|
||||
|
||||
try {
|
||||
// Group events by file path to handle multiple events for the same file
|
||||
const eventMap = new Map<string, FileChangeEvent>();
|
||||
|
||||
for (const event of eventsToProcess) {
|
||||
const key = `${event.mappingId}:${event.filePath}`;
|
||||
eventMap.set(key, event); // Latest event wins
|
||||
}
|
||||
|
||||
// Process each unique file change
|
||||
for (const event of eventMap.values()) {
|
||||
await this.processFileChange(event);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error processing file change queue: ${error}`);
|
||||
} finally {
|
||||
this.isProcessing = false;
|
||||
|
||||
// If more events were queued while processing, schedule another run
|
||||
if (this.syncQueue.length > 0) {
|
||||
this.processSyncQueue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single file change event
|
||||
*/
|
||||
private async processFileChange(event: FileChangeEvent) {
|
||||
try {
|
||||
const mapping = becca.fileSystemMappings[event.mappingId];
|
||||
if (!mapping || !mapping.isActive || !mapping.canSyncFromDisk) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(`DEBUG: Processing file ${event.type}: ${event.filePath} for mapping ${event.mappingId}`);
|
||||
|
||||
switch (event.type) {
|
||||
case 'add':
|
||||
case 'change':
|
||||
await this.handleFileAddOrChange(event, mapping);
|
||||
break;
|
||||
case 'unlink':
|
||||
await this.handleFileDelete(event, mapping);
|
||||
break;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
log.error(`Error processing file change for ${event.filePath}: ${error}`);
|
||||
const mapping = becca.fileSystemMappings[event.mappingId];
|
||||
if (mapping) {
|
||||
mapping.addSyncError(`Error processing ${event.filePath}: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file addition or modification
|
||||
*/
|
||||
private async handleFileAddOrChange(event: FileChangeEvent, mapping: BFileSystemMapping) {
|
||||
if (!await fs.pathExists(event.filePath)) {
|
||||
return; // File was deleted between queuing and processing
|
||||
}
|
||||
|
||||
const stats = event.stats || await fs.stat(event.filePath);
|
||||
if (stats.isDirectory()) {
|
||||
return; // We only sync files, not directories
|
||||
}
|
||||
|
||||
// Calculate file hash for change detection
|
||||
const fileContent = await fs.readFile(event.filePath);
|
||||
const fileHash = crypto.createHash('sha256').update(fileContent).digest('hex');
|
||||
const fileModifiedTime = stats.mtime.toISOString();
|
||||
|
||||
// Find existing file note mapping
|
||||
let fileNoteMapping: BFileNoteMapping | null = null;
|
||||
for (const mapping of Object.values(becca.fileNoteMappings || {})) {
|
||||
if (mapping.mappingId === event.mappingId && path.normalize(mapping.filePath) === path.normalize(event.filePath)) {
|
||||
fileNoteMapping = mapping;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if file actually changed
|
||||
if (fileNoteMapping && !fileNoteMapping.hasFileChanged(fileHash, fileModifiedTime)) {
|
||||
return; // No actual change
|
||||
}
|
||||
|
||||
if (fileNoteMapping) {
|
||||
// Update existing mapping
|
||||
if (fileNoteMapping.hasNoteChanged()) {
|
||||
// Both file and note changed - mark as conflict
|
||||
fileNoteMapping.markConflict();
|
||||
log.info(`Conflict detected for ${event.filePath} - both file and note modified`);
|
||||
return;
|
||||
}
|
||||
|
||||
fileNoteMapping.markPending();
|
||||
} else {
|
||||
// Double-check if mapping exists before creating (race condition protection)
|
||||
const existingCheck = Object.values(becca.fileNoteMappings || {}).find(m =>
|
||||
m.mappingId === event.mappingId && path.normalize(m.filePath) === path.normalize(event.filePath)
|
||||
);
|
||||
|
||||
if (existingCheck) {
|
||||
log.info(`File mapping already exists for ${event.filePath}, using existing mapping`);
|
||||
fileNoteMapping = existingCheck;
|
||||
fileNoteMapping.markPending();
|
||||
} else {
|
||||
// Create new file note mapping
|
||||
try {
|
||||
fileNoteMapping = new BFileNoteMapping({
|
||||
mappingId: event.mappingId,
|
||||
noteId: '', // Will be determined by sync service
|
||||
filePath: event.filePath,
|
||||
fileHash,
|
||||
fileModifiedTime,
|
||||
syncStatus: 'pending'
|
||||
}).save();
|
||||
} catch (error: any) {
|
||||
if (error.code === 'SQLITE_CONSTRAINT_UNIQUE') {
|
||||
log.info(`File mapping constraint violation for ${event.filePath}, trying to find existing mapping`);
|
||||
// Try to find the mapping again - it might have been created by another process
|
||||
fileNoteMapping = Object.values(becca.fileNoteMappings || {}).find(m =>
|
||||
m.mappingId === event.mappingId && path.normalize(m.filePath) === path.normalize(event.filePath)
|
||||
) || null;
|
||||
if (!fileNoteMapping) {
|
||||
throw error; // Re-throw if we still can't find it
|
||||
}
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit event for sync service to handle
|
||||
eventService.emit('FILE_CHANGED', {
|
||||
fileNoteMapping,
|
||||
mapping,
|
||||
fileContent,
|
||||
isNew: event.type === 'add'
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file deletion
|
||||
*/
|
||||
private async handleFileDelete(event: FileChangeEvent, mapping: BFileSystemMapping) {
|
||||
// Find existing file note mapping
|
||||
let fileNoteMapping: BFileNoteMapping | null = null;
|
||||
for (const mappingObj of Object.values(becca.fileNoteMappings || {})) {
|
||||
if (mappingObj.mappingId === event.mappingId && mappingObj.filePath === event.filePath) {
|
||||
fileNoteMapping = mappingObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (fileNoteMapping) {
|
||||
// Emit event for sync service to handle deletion
|
||||
eventService.emit('FILE_DELETED', {
|
||||
fileNoteMapping,
|
||||
mapping
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get status of all watchers
|
||||
*/
|
||||
getWatcherStatus() {
|
||||
const status: Record<string, any> = {};
|
||||
|
||||
for (const [mappingId, { mapping, watcher }] of this.watchers) {
|
||||
status[mappingId] = {
|
||||
filePath: mapping.filePath,
|
||||
isActive: mapping.isActive,
|
||||
watchedPaths: watcher.getWatched(),
|
||||
syncDirection: mapping.syncDirection
|
||||
};
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Force a full sync for a specific mapping
|
||||
*/
|
||||
async forceSyncMapping(mappingId: string) {
|
||||
const mapping = becca.fileSystemMappings[mappingId];
|
||||
if (!mapping) {
|
||||
throw new Error(`Mapping ${mappingId} not found`);
|
||||
}
|
||||
|
||||
log.info(`Force syncing mapping ${mappingId}: ${mapping.filePath}`);
|
||||
|
||||
if (await fs.pathExists(mapping.filePath)) {
|
||||
const stats = await fs.stat(mapping.filePath);
|
||||
if (stats.isFile()) {
|
||||
await this.queueFileChange('change', mapping.filePath, mappingId, stats);
|
||||
} else if (stats.isDirectory() && mapping.includeSubtree) {
|
||||
// Scan directory for files
|
||||
await this.scanDirectoryForFiles(mapping.filePath, mapping);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively scan directory for files and queue them for sync
|
||||
*/
|
||||
private async scanDirectoryForFiles(dirPath: string, mapping: BFileSystemMapping) {
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
|
||||
if (entry.isFile()) {
|
||||
const stats = await fs.stat(fullPath);
|
||||
this.queueFileChange('change', fullPath, mapping.mappingId, stats);
|
||||
} else if (entry.isDirectory() && mapping.includeSubtree) {
|
||||
await this.scanDirectoryForFiles(fullPath, mapping);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log.error(`Error scanning directory ${dirPath}: ${error}`);
|
||||
mapping.addSyncError(`Error scanning directory: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
const fileSystemWatcher = new FileSystemWatcher();
|
||||
|
||||
export default fileSystemWatcher;
|
||||
@@ -211,6 +211,9 @@ const defaultOptions: DefaultOption[] = [
|
||||
{ name: "aiTemperature", value: "0.7", isSynced: true },
|
||||
{ name: "aiSystemPrompt", value: "", isSynced: true },
|
||||
{ name: "aiSelectedProvider", value: "openai", isSynced: true },
|
||||
|
||||
// File system sync options
|
||||
{ name: "fileSystemSyncEnabled", value: "false", isSynced: false },
|
||||
];
|
||||
|
||||
/**
|
||||
|
||||
@@ -123,6 +123,7 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions<KeyboardActi
|
||||
/** Whether keyboard auto-completion for notes is triggered when typing `@` in text notes (attribute editing is not affected). */
|
||||
textNoteCompletionEnabled: boolean;
|
||||
backgroundEffects: boolean;
|
||||
fileSystemSyncEnabled: boolean;
|
||||
|
||||
// Share settings
|
||||
redirectBareDomain: boolean;
|
||||
|
||||
Reference in New Issue
Block a user