mirror of
https://github.com/zadam/trilium.git
synced 2025-11-06 13:26:01 +01:00
docs(dev): integrate rest of the documentation
This commit is contained in:
Elian Doran
168
docs/Developer Guide/!!!meta.json
vendored
168
docs/Developer Guide/!!!meta.json
vendored
@@ -172,64 +172,57 @@
|
||||
"children": [
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "2DJZgzpTJ078",
|
||||
"noteId": "dsMq2EIOMOBU",
|
||||
"notePath": [
|
||||
"jdjRLhLV3TtI",
|
||||
"MhwWMgxwDTZL",
|
||||
"2DJZgzpTJ078"
|
||||
"dsMq2EIOMOBU"
|
||||
],
|
||||
"title": "Client-server architecture",
|
||||
"title": "Frontend",
|
||||
"notePosition": 10,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"format": "markdown",
|
||||
"attachments": [],
|
||||
"dirFileName": "Client-server architecture",
|
||||
"children": [
|
||||
"attributes": [
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "dsMq2EIOMOBU",
|
||||
"notePath": [
|
||||
"jdjRLhLV3TtI",
|
||||
"MhwWMgxwDTZL",
|
||||
"2DJZgzpTJ078",
|
||||
"dsMq2EIOMOBU"
|
||||
],
|
||||
"title": "Frontend",
|
||||
"notePosition": 10,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Frontend.md",
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "tsswRlmHEnYW",
|
||||
"notePath": [
|
||||
"jdjRLhLV3TtI",
|
||||
"MhwWMgxwDTZL",
|
||||
"2DJZgzpTJ078",
|
||||
"tsswRlmHEnYW"
|
||||
],
|
||||
"title": "Backend",
|
||||
"notePosition": 20,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Backend.md",
|
||||
"attachments": []
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "frontend",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
}
|
||||
]
|
||||
],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Frontend.md",
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "tsswRlmHEnYW",
|
||||
"notePath": [
|
||||
"jdjRLhLV3TtI",
|
||||
"MhwWMgxwDTZL",
|
||||
"tsswRlmHEnYW"
|
||||
],
|
||||
"title": "Backend",
|
||||
"notePosition": 20,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "backend",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Backend.md",
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
"isClone": false,
|
||||
@@ -240,7 +233,7 @@
|
||||
"pRZhrVIGCbMu"
|
||||
],
|
||||
"title": "Database",
|
||||
"notePosition": 20,
|
||||
"notePosition": 40,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
@@ -785,15 +778,23 @@
|
||||
"MhwWMgxwDTZL",
|
||||
"Wxn82Em8B7U5"
|
||||
],
|
||||
"title": "API",
|
||||
"notePosition": 30,
|
||||
"title": "APIs",
|
||||
"notePosition": 50,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"attributes": [
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "api",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
"dataFileName": "API.md",
|
||||
"dataFileName": "APIs.md",
|
||||
"attachments": []
|
||||
},
|
||||
{
|
||||
@@ -805,7 +806,7 @@
|
||||
"Vk4zD1Iirarg"
|
||||
],
|
||||
"title": "Arhitecture Decision Records",
|
||||
"notePosition": 40,
|
||||
"notePosition": 60,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
@@ -817,6 +818,13 @@
|
||||
"value": "Jg7clqogFOyD",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
},
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "adr",
|
||||
"isInheritable": false,
|
||||
"position": 30
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
@@ -825,14 +833,14 @@
|
||||
},
|
||||
{
|
||||
"isClone": false,
|
||||
"noteId": "QW1MB7RZB5Gf",
|
||||
"noteId": "RHbKw3xiwk3S",
|
||||
"notePath": [
|
||||
"jdjRLhLV3TtI",
|
||||
"MhwWMgxwDTZL",
|
||||
"QW1MB7RZB5Gf"
|
||||
"RHbKw3xiwk3S"
|
||||
],
|
||||
"title": "Security Architecture",
|
||||
"notePosition": 50,
|
||||
"title": "Security",
|
||||
"notePosition": 80,
|
||||
"prefix": null,
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
@@ -841,13 +849,13 @@
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "security-architecture",
|
||||
"value": "security",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Security Architecture.md",
|
||||
"dataFileName": "Security.md",
|
||||
"attachments": []
|
||||
}
|
||||
]
|
||||
@@ -1153,6 +1161,13 @@
|
||||
"value": "bx bx-rocket",
|
||||
"isInheritable": false,
|
||||
"position": 30
|
||||
},
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "releasing",
|
||||
"isInheritable": false,
|
||||
"position": 40
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
@@ -1181,6 +1196,13 @@
|
||||
"value": "bx bxs-component",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
},
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "dependencies",
|
||||
"isInheritable": false,
|
||||
"position": 30
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
@@ -1527,6 +1549,13 @@
|
||||
"value": "bx bx-microchip",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
},
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "cache",
|
||||
"isInheritable": false,
|
||||
"position": 30
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
@@ -2001,7 +2030,15 @@
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"attributes": [
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "note-types",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
"attachments": [],
|
||||
"dirFileName": "Note Types",
|
||||
@@ -2547,6 +2584,7 @@
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Synchronisation.md",
|
||||
"attachments": [],
|
||||
"dirFileName": "Synchronisation",
|
||||
"children": [
|
||||
@@ -2794,7 +2832,15 @@
|
||||
"isExpanded": false,
|
||||
"type": "text",
|
||||
"mime": "text/html",
|
||||
"attributes": [],
|
||||
"attributes": [
|
||||
{
|
||||
"type": "label",
|
||||
"name": "shareAlias",
|
||||
"value": "unit-tests",
|
||||
"isInheritable": false,
|
||||
"position": 20
|
||||
}
|
||||
],
|
||||
"format": "markdown",
|
||||
"dataFileName": "Unit tests.md",
|
||||
"attachments": []
|
||||
|
||||
@@ -118,4 +118,85 @@ desktop → client → commons
|
||||
server → client → commons
|
||||
client → ckeditor5, codemirror, highlightjs
|
||||
ckeditor5 → ckeditor5-* plugins
|
||||
```
|
||||
```
|
||||
|
||||
## Security summary
|
||||
|
||||
### Encryption System
|
||||
|
||||
**Per-Note Encryption:**
|
||||
|
||||
* Notes can be individually protected
|
||||
* AES-128-CBC encryption for encrypted notes.
|
||||
* Separate protected session management
|
||||
|
||||
**Protected Session:**
|
||||
|
||||
* Time-limited access to protected notes
|
||||
* Automatic timeout
|
||||
* Re-authentication required
|
||||
* Frontend: `protected_session.ts`
|
||||
* Backend: `protected_session.ts`
|
||||
|
||||
### Authentication
|
||||
|
||||
**Password Auth:**
|
||||
|
||||
* PBKDF2 key derivation
|
||||
* Salt per installation
|
||||
* Hash verification
|
||||
|
||||
**OpenID Connect:**
|
||||
|
||||
* External identity provider support
|
||||
* OAuth 2.0 flow
|
||||
* Configurable providers
|
||||
|
||||
**TOTP (2FA):**
|
||||
|
||||
* Time-based one-time passwords
|
||||
* QR code setup
|
||||
* Backup codes
|
||||
|
||||
### Authorization
|
||||
|
||||
**Single-User Model:**
|
||||
|
||||
* Desktop: single user (owner)
|
||||
* Server: single user per installation
|
||||
|
||||
**Share Notes:**
|
||||
|
||||
* Public access without authentication
|
||||
* Separate Shaca cache
|
||||
* Read-only access
|
||||
|
||||
### CSRF Protection
|
||||
|
||||
**CSRF Tokens:**
|
||||
|
||||
* Required for state-changing operations
|
||||
* Token in header or cookie
|
||||
* Validation middleware
|
||||
|
||||
### Input Sanitization
|
||||
|
||||
**XSS Prevention:**
|
||||
|
||||
* DOMPurify for HTML sanitization
|
||||
* CKEditor content filtering
|
||||
* CSP headers
|
||||
|
||||
**SQL Injection:**
|
||||
|
||||
* Parameterized queries only
|
||||
* Better-sqlite3 prepared statements
|
||||
* No string concatenation in SQL
|
||||
|
||||
### Dependency Security
|
||||
|
||||
**Vulnerability Scanning:**
|
||||
|
||||
* Renovate bot for updates
|
||||
* npm audit integration
|
||||
* Override vulnerable sub-dependencies
|
||||
@@ -1,4 +1,4 @@
|
||||
# API
|
||||
# APIs
|
||||
### Internal API
|
||||
|
||||
**REST Endpoints** (`/api/*`)
|
||||
@@ -1,5 +1,5 @@
|
||||
# Database
|
||||
Trilium uses **SQLite** as its database engine, managed via `better-sqlite3`.
|
||||
Trilium uses **SQLite** (via `better-sqlite3`) as its embedded database engine, providing a reliable, file-based storage system that requires no separate database server. The database stores all notes, their relationships, metadata, and configuration.
|
||||
|
||||
Schema location: `apps/server/src/assets/db/schema.sql`
|
||||
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
# Security Architecture
|
||||
### Encryption System
|
||||
|
||||
**Per-Note Encryption:**
|
||||
|
||||
* Notes can be individually protected
|
||||
* AES-128-CBC encryption for encrypted notes.
|
||||
* Separate protected session management
|
||||
|
||||
**Protected Session:**
|
||||
|
||||
* Time-limited access to protected notes
|
||||
* Automatic timeout
|
||||
* Re-authentication required
|
||||
* Frontend: `protected_session.ts`
|
||||
* Backend: `protected_session.ts`
|
||||
|
||||
### Authentication
|
||||
|
||||
**Password Auth:**
|
||||
|
||||
* PBKDF2 key derivation
|
||||
* Salt per installation
|
||||
* Hash verification
|
||||
|
||||
**OpenID Connect:**
|
||||
|
||||
* External identity provider support
|
||||
* OAuth 2.0 flow
|
||||
* Configurable providers
|
||||
|
||||
**TOTP (2FA):**
|
||||
|
||||
* Time-based one-time passwords
|
||||
* QR code setup
|
||||
* Backup codes
|
||||
|
||||
### Authorization
|
||||
|
||||
**Single-User Model:**
|
||||
|
||||
* Desktop: single user (owner)
|
||||
* Server: single user per installation
|
||||
|
||||
**Share Notes:**
|
||||
|
||||
* Public access without authentication
|
||||
* Separate Shaca cache
|
||||
* Read-only access
|
||||
|
||||
### CSRF Protection
|
||||
|
||||
**CSRF Tokens:**
|
||||
|
||||
* Required for state-changing operations
|
||||
* Token in header or cookie
|
||||
* Validation middleware
|
||||
|
||||
### Input Sanitization
|
||||
|
||||
**XSS Prevention:**
|
||||
|
||||
* DOMPurify for HTML sanitization
|
||||
* CKEditor content filtering
|
||||
* CSP headers
|
||||
|
||||
**SQL Injection:**
|
||||
|
||||
* Parameterized queries only
|
||||
* Better-sqlite3 prepared statements
|
||||
* No string concatenation in SQL
|
||||
|
||||
### Dependency Security
|
||||
|
||||
**Vulnerability Scanning:**
|
||||
|
||||
* Renovate bot for updates
|
||||
* npm audit integration
|
||||
* Override vulnerable sub-dependencies
|
||||
464
docs/Developer Guide/Developer Guide/Architecture/Security.md
vendored
Normal file
464
docs/Developer Guide/Developer Guide/Architecture/Security.md
vendored
Normal file
@@ -0,0 +1,464 @@
|
||||
# Security
|
||||
Trilium implements a **defense-in-depth security model** with multiple layers of protection for user data. The security architecture covers authentication, authorization, encryption, input sanitization, and secure communication.
|
||||
|
||||
## Security Principles
|
||||
|
||||
1. **Data Privacy**: User data is protected at rest and in transit
|
||||
2. **Encryption**: Per-note encryption for sensitive content
|
||||
3. **Authentication**: Multiple authentication methods supported
|
||||
4. **Authorization**: Single-user model with granular protected sessions
|
||||
5. **Input Validation**: All user input sanitized
|
||||
6. **Secure Defaults**: Security features enabled by default
|
||||
7. **Transparency**: Open source allows security audits
|
||||
|
||||
## Threat Model
|
||||
|
||||
### Threats Considered
|
||||
|
||||
1. **Unauthorized Access**
|
||||
* Physical access to device
|
||||
* Network eavesdropping
|
||||
* Stolen credentials
|
||||
* Session hijacking
|
||||
2. **Data Exfiltration**
|
||||
* Malicious scripts
|
||||
* XSS attacks
|
||||
* SQL injection
|
||||
* CSRF attacks
|
||||
3. **Data Corruption**
|
||||
* Malicious modifications
|
||||
* Database tampering
|
||||
* Sync conflicts
|
||||
4. **Privacy Leaks**
|
||||
* Unencrypted backups
|
||||
* Search indexing
|
||||
* Temporary files
|
||||
* Memory dumps
|
||||
|
||||
### Out of Scope
|
||||
|
||||
* Nation-state attackers
|
||||
* Zero-day vulnerabilities in dependencies
|
||||
* Hardware vulnerabilities (Spectre, Meltdown)
|
||||
* Physical access with unlimited time
|
||||
* Quantum computing attacks
|
||||
|
||||
## Authentication
|
||||
|
||||
### Password Authentication
|
||||
|
||||
**Implementation:** `apps/server/src/services/password.ts`
|
||||
|
||||
### TOTP (Two-Factor Authentication)
|
||||
|
||||
**Implementation:** `apps/server/src/routes/api/login.ts`
|
||||
|
||||
### OpenID Connect
|
||||
|
||||
**Implementation:** `apps/server/src/routes/api/login.ts`
|
||||
|
||||
**Supported Providers:**
|
||||
|
||||
* Any OpenID Connect compatible provider
|
||||
* Google, GitHub, Auth0, etc.
|
||||
|
||||
**Flow:**
|
||||
|
||||
```typescript
|
||||
// 1. Redirect to provider
|
||||
GET /api/login/openid
|
||||
|
||||
// 2. Provider redirects back with code
|
||||
GET /api/login/openid/callback?code=...
|
||||
|
||||
// 3. Exchange code for tokens
|
||||
const tokens = await openidClient.callback(redirectUri, req.query)
|
||||
|
||||
// 4. Verify ID token
|
||||
const claims = tokens.claims()
|
||||
|
||||
// 5. Create session
|
||||
req.session.loggedIn = true
|
||||
```
|
||||
|
||||
### Session Management
|
||||
|
||||
**Session Storage:** SQLite database (sessions table)
|
||||
|
||||
**Session Configuration:**
|
||||
|
||||
```typescript
|
||||
app.use(session({
|
||||
secret: sessionSecret,
|
||||
resave: false,
|
||||
saveUninitialized: false,
|
||||
rolling: true,
|
||||
cookie: {
|
||||
maxAge: 7 * 24 * 60 * 60 * 1000, // 7 days
|
||||
httpOnly: true,
|
||||
secure: isHttps,
|
||||
sameSite: 'lax'
|
||||
},
|
||||
store: new SqliteStore({
|
||||
db: db,
|
||||
table: 'sessions'
|
||||
})
|
||||
}))
|
||||
```
|
||||
|
||||
**Session Invalidation:**
|
||||
|
||||
* Automatic timeout after inactivity
|
||||
* Manual logout clears session
|
||||
* Server restart invalidates all sessions (optional)
|
||||
|
||||
## Authorization
|
||||
|
||||
### Single-User Model
|
||||
|
||||
**Desktop:**
|
||||
|
||||
* Single user (owner of device)
|
||||
* No multi-user support
|
||||
* Full access to all notes
|
||||
|
||||
**Server:**
|
||||
|
||||
* Single user per installation
|
||||
* Authentication required for all operations
|
||||
* No user roles or permissions
|
||||
|
||||
### Protected Sessions
|
||||
|
||||
**Purpose:** Temporary access to encrypted (protected) notes
|
||||
|
||||
**Implementation:** `apps/server/src/services/protected_session.ts`
|
||||
|
||||
**Workflow:**
|
||||
|
||||
```typescript
|
||||
// 1. User enters password for protected notes
|
||||
POST /api/protected-session/enter
|
||||
Body: { password: "protected-password" }
|
||||
|
||||
// 2. Derive encryption key
|
||||
const protectedDataKey = deriveKey(password)
|
||||
|
||||
// 3. Verify password (decrypt known encrypted value)
|
||||
const decrypted = decrypt(testValue, protectedDataKey)
|
||||
if (decrypted === expectedValue) {
|
||||
// 4. Store in memory (not in session)
|
||||
protectedSessionHolder.setProtectedDataKey(protectedDataKey)
|
||||
|
||||
// 5. Set timeout
|
||||
setTimeout(() => {
|
||||
protectedSessionHolder.clearProtectedDataKey()
|
||||
}, timeout)
|
||||
}
|
||||
```
|
||||
|
||||
**Protected Session Timeout:**
|
||||
|
||||
* Default: 10 minutes (configurable)
|
||||
* Extends on activity
|
||||
* Cleared on browser close
|
||||
* Separate from main session
|
||||
|
||||
### API Authorization
|
||||
|
||||
**Internal API:**
|
||||
|
||||
* Requires authenticated session
|
||||
* CSRF token validation
|
||||
* Same-origin policy
|
||||
|
||||
**ETAPI (External API):**
|
||||
|
||||
* Token-based authentication
|
||||
* No session required
|
||||
* Rate limiting
|
||||
|
||||
## Encryption
|
||||
|
||||
### Note Encryption
|
||||
|
||||
**Encryption Algorithm:** AES-256-CBC
|
||||
|
||||
**Key Hierarchy:**
|
||||
|
||||
```
|
||||
User Password
|
||||
↓ (scrypt)
|
||||
Data Key (for protected notes)
|
||||
↓ (AES-128)
|
||||
Protected Note Content
|
||||
```
|
||||
|
||||
**Protected Note Metadata:**
|
||||
|
||||
* Content IS encrypted
|
||||
* Type and MIME are NOT encrypted
|
||||
* Attributes are NOT encrypted
|
||||
|
||||
### Data Key Management
|
||||
|
||||
**Key Rotation:**
|
||||
|
||||
* Not currently supported
|
||||
* Requires re-encrypting all protected notes
|
||||
|
||||
### Transport Encryption
|
||||
|
||||
**HTTPS:**
|
||||
|
||||
* Recommended for server installations
|
||||
* TLS 1.2+ only
|
||||
* Strong cipher suites preferred
|
||||
* Certificate validation enabled
|
||||
|
||||
**Desktop:**
|
||||
|
||||
* Local communication (no network)
|
||||
* No HTTPS required
|
||||
|
||||
### Backup Encryption
|
||||
|
||||
**Database Backups:**
|
||||
|
||||
* Protected notes remain encrypted in backup
|
||||
* Backup file should be protected separately
|
||||
* Consider encrypting backup storage location
|
||||
|
||||
## Input Sanitization
|
||||
|
||||
### XSS Prevention
|
||||
|
||||
* **HTML Sanitization**
|
||||
* **CKEditor Configuration:**
|
||||
|
||||
```
|
||||
// apps/client/src/widgets/type_widgets/text_type_widget.ts
|
||||
ClassicEditor.create(element, {
|
||||
// Restrict allowed content
|
||||
htmlSupport: {
|
||||
allow: [
|
||||
{ name: /./, attributes: true, classes: true, styles: true }
|
||||
],
|
||||
disallow: [
|
||||
{ name: 'script' },
|
||||
{ name: 'iframe', attributes: /^(?!src$).*/ }
|
||||
]
|
||||
}
|
||||
})
|
||||
```
|
||||
* Content Security Policy
|
||||
|
||||
### SQL Injection Prevention
|
||||
|
||||
**Parameterized Queries:**
|
||||
|
||||
```typescript
|
||||
const notes = sql.getRows(
|
||||
'SELECT * FROM notes WHERE title = ?',
|
||||
[userInput]
|
||||
)
|
||||
```
|
||||
|
||||
**ORM Usage:**
|
||||
|
||||
```typescript
|
||||
// Entity-based access prevents SQL injection
|
||||
const note = becca.getNote(noteId)
|
||||
note.title = userInput // Sanitized by entity
|
||||
note.save() // Parameterized query
|
||||
```
|
||||
|
||||
### CSRF Prevention
|
||||
|
||||
**CSRF Token Validation:**
|
||||
|
||||
Location: `apps/server/src/routes/csrf_protection.ts`
|
||||
|
||||
Stateless CSRF using [Double Submit Cookie Pattern](https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#double-submit-cookie) via [`csrf-csrf`](https://github.com/Psifi-Solutions/csrf-csrf).
|
||||
|
||||
### File Upload Validation
|
||||
|
||||
**Validation:**
|
||||
|
||||
```typescript
|
||||
// Validate file size
|
||||
const maxSize = 100 * 1024 * 1024 // 100 MB
|
||||
if (file.size > maxSize) {
|
||||
throw new Error('File too large')
|
||||
}
|
||||
```
|
||||
|
||||
## Network Security
|
||||
|
||||
### HTTPS Configuration
|
||||
|
||||
**Certificate Validation:**
|
||||
|
||||
* Require valid certificates in production
|
||||
* Self-signed certificates allowed for development
|
||||
* Certificate pinning not implemented
|
||||
|
||||
### Rate Limiting
|
||||
|
||||
**Login Rate Limiting:**
|
||||
|
||||
```typescript
|
||||
const loginLimiter = rateLimit({
|
||||
windowMs: 15 * 60 * 1000,
|
||||
max: 10, // 10 failed attempts
|
||||
skipSuccessfulRequests: true
|
||||
})
|
||||
|
||||
app.post('/api/login/password', loginLimiter, loginHandler)
|
||||
```
|
||||
|
||||
## Data Security
|
||||
|
||||
### Secure Data Deletion
|
||||
|
||||
**Soft Delete:**
|
||||
|
||||
```typescript
|
||||
// Mark as deleted (sync first)
|
||||
note.isDeleted = 1
|
||||
note.deleteId = generateUUID()
|
||||
note.save()
|
||||
|
||||
// Entity change tracked for sync
|
||||
addEntityChange('notes', noteId, note)
|
||||
```
|
||||
|
||||
**Hard Delete (Erase):**
|
||||
|
||||
```typescript
|
||||
// After sync completed
|
||||
sql.execute('DELETE FROM notes WHERE noteId = ?', [noteId])
|
||||
sql.execute('DELETE FROM branches WHERE noteId = ?', [noteId])
|
||||
sql.execute('DELETE FROM attributes WHERE noteId = ?', [noteId])
|
||||
|
||||
// Mark entity change as erased
|
||||
sql.execute('UPDATE entity_changes SET isErased = 1 WHERE entityId = ?', [noteId])
|
||||
```
|
||||
|
||||
**Blob Cleanup:**
|
||||
|
||||
```typescript
|
||||
// Find orphaned blobs (not referenced by any note/revision/attachment)
|
||||
const orphanedBlobs = sql.getRows(`
|
||||
SELECT blobId FROM blobs
|
||||
WHERE blobId NOT IN (SELECT blobId FROM notes WHERE blobId IS NOT NULL)
|
||||
AND blobId NOT IN (SELECT blobId FROM revisions WHERE blobId IS NOT NULL)
|
||||
AND blobId NOT IN (SELECT blobId FROM attachments WHERE blobId IS NOT NULL)
|
||||
`)
|
||||
|
||||
// Delete orphaned blobs
|
||||
for (const blob of orphanedBlobs) {
|
||||
sql.execute('DELETE FROM blobs WHERE blobId = ?', [blob.blobId])
|
||||
}
|
||||
```
|
||||
|
||||
### Memory Security
|
||||
|
||||
**Protected Data in Memory:**
|
||||
|
||||
* Protected data keys stored in memory only
|
||||
* Cleared on timeout
|
||||
* Not written to disk
|
||||
* Not in session storage
|
||||
|
||||
## Dependency Security
|
||||
|
||||
### Vulnerability Scanning
|
||||
|
||||
**Tools:**
|
||||
|
||||
* Renovate bot - Automatic dependency updates
|
||||
* `pnpm audit` - Check for known vulnerabilities
|
||||
* GitHub Dependabot alerts
|
||||
|
||||
**Process:**
|
||||
|
||||
```sh
|
||||
# Check for vulnerabilities
|
||||
npm audit
|
||||
|
||||
# Fix automatically
|
||||
npm audit fix
|
||||
|
||||
# Manual review for breaking changes
|
||||
npm audit fix --force
|
||||
```
|
||||
|
||||
### Dependency Pinning
|
||||
|
||||
**package.json:**
|
||||
|
||||
```
|
||||
{
|
||||
"dependencies": {
|
||||
"express": "4.18.2", // Exact version
|
||||
"better-sqlite3": "^9.2.2" // Compatible versions
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**pnpm Overrides:**
|
||||
|
||||
```
|
||||
{
|
||||
"pnpm": {
|
||||
"overrides": {
|
||||
"lodash@<4.17.21": ">=4.17.21", // Force minimum version
|
||||
"axios@<0.21.2": ">=0.21.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Patch Management
|
||||
|
||||
**pnpm Patches:**
|
||||
|
||||
```sh
|
||||
# Create patch
|
||||
pnpm patch @ckeditor/ckeditor5
|
||||
|
||||
# Edit files in temporary directory
|
||||
# ...
|
||||
|
||||
# Generate patch file
|
||||
pnpm patch-commit /tmp/ckeditor5-patch
|
||||
|
||||
# Patch applied automatically on install
|
||||
```
|
||||
|
||||
## Security Auditing
|
||||
|
||||
### Logs
|
||||
|
||||
**Security Events Logged:**
|
||||
|
||||
* Login attempts (success/failure)
|
||||
* Protected session access
|
||||
* Password changes
|
||||
* ETAPI token usage
|
||||
* Failed CSRF validations
|
||||
|
||||
**Log Location:**
|
||||
|
||||
* Desktop: Console output
|
||||
* Server: Log files or stdout
|
||||
|
||||
### Monitoring
|
||||
|
||||
**Metrics to Monitor:**
|
||||
|
||||
* Failed login attempts
|
||||
* API error rates
|
||||
* Unusual database changes
|
||||
* Large exports/imports
|
||||
484
docs/Developer Guide/Developer Guide/Concepts/Synchronisation.md
vendored
Normal file
484
docs/Developer Guide/Developer Guide/Concepts/Synchronisation.md
vendored
Normal file
@@ -0,0 +1,484 @@
|
||||
# Synchronisation
|
||||
Trilium implements a **bidirectional synchronization system** that allows users to sync their note databases across multiple devices (desktop clients and server instances). The sync protocol is designed to handle:
|
||||
|
||||
* Concurrent modifications across devices
|
||||
* Simple conflict resolution (without “merge conflict” indication).
|
||||
* Partial sync (only changed entities)
|
||||
* Protected note synchronization
|
||||
* Efficient bandwidth usage
|
||||
|
||||
## Sync Architecture
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
Desktop1[Desktop 1<br/>Client]
|
||||
Desktop2[Desktop 2<br/>Client]
|
||||
|
||||
subgraph SyncServer["Sync Server"]
|
||||
SyncService[Sync Service<br/>- Entity Change Management<br/>- Conflict Resolution<br/>- Version Tracking]
|
||||
SyncDB[(Database<br/>entity_changes)]
|
||||
end
|
||||
|
||||
Desktop1 <-->|WebSocket/HTTP| SyncService
|
||||
Desktop2 <-->|WebSocket/HTTP| SyncService
|
||||
SyncService --> SyncDB
|
||||
```
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### Entity Changes
|
||||
|
||||
Every modification to any entity (note, branch, attribute, etc.) creates an **entity change** record:
|
||||
|
||||
```
|
||||
entity_changes (
|
||||
id, -- Auto-increment ID
|
||||
entityName, -- 'notes', 'branches', 'attributes', etc.
|
||||
entityId, -- ID of the changed entity
|
||||
hash, -- Content hash for integrity
|
||||
isErased, -- If entity was erased (deleted permanently)
|
||||
changeId, -- Unique change identifier
|
||||
componentId, -- Unique component/widget identifier
|
||||
instanceId, -- Process instance identifier
|
||||
isSynced, -- Whether synced to server
|
||||
utcDateChanged -- When change occurred
|
||||
)
|
||||
```
|
||||
|
||||
**Key Properties:**
|
||||
|
||||
* **changeId**: Globally unique identifier (UUID) for the change
|
||||
* **componentId**: Unique identifier of the component/widget that generated to change (can be used to avoid refreshing the widget being edited).
|
||||
* **instanceId**: Unique per process (changes on restart)
|
||||
* **hash**: SHA-256 hash of entity data for integrity verification
|
||||
|
||||
### Sync Versions
|
||||
|
||||
Each Trilium installation tracks:
|
||||
|
||||
* **Local sync version**: Highest change ID seen locally
|
||||
* **Server sync version**: Highest change ID on server
|
||||
* **Entity versions**: Last sync version for each entity type
|
||||
|
||||
### Change Tracking
|
||||
|
||||
**When an entity is modified:**
|
||||
|
||||
```typescript
|
||||
// apps/server/src/services/entity_changes.ts
|
||||
function addEntityChange(entityName, entityId, entity) {
|
||||
const hash = calculateHash(entity)
|
||||
const changeId = generateUUID()
|
||||
|
||||
sql.insert('entity_changes', {
|
||||
entityName,
|
||||
entityId,
|
||||
hash,
|
||||
changeId,
|
||||
componentId: config.componentId,
|
||||
instanceId: config.instanceId,
|
||||
isSynced: 0,
|
||||
utcDateChanged: now()
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
**Entity modification triggers:**
|
||||
|
||||
* Note content update
|
||||
* Note metadata change
|
||||
* Branch creation/deletion/reorder
|
||||
* Attribute addition/removal
|
||||
* Options modification
|
||||
|
||||
## Sync Protocol
|
||||
|
||||
### Sync Handshake
|
||||
|
||||
**Step 1: Client Initiates Sync**
|
||||
|
||||
```typescript
|
||||
// Client sends current sync version
|
||||
POST /api/sync/check
|
||||
{
|
||||
"sourceId": "client-component-id",
|
||||
"maxChangeId": 12345
|
||||
}
|
||||
```
|
||||
|
||||
**Step 2: Server Responds with Status**
|
||||
|
||||
```typescript
|
||||
// Server checks for changes
|
||||
Response:
|
||||
{
|
||||
"entityChanges": 567, // Changes on server
|
||||
"maxChangeId": 12890, // Server's max change ID
|
||||
"outstandingPushCount": 23 // Client changes not yet synced
|
||||
}
|
||||
```
|
||||
|
||||
**Step 3: Decision**
|
||||
|
||||
* If `entityChanges > 0`: Pull changes from server
|
||||
* If `outstandingPushCount > 0`: Push changes to server
|
||||
* Both can happen in sequence
|
||||
|
||||
### Pull Sync (Server → Client)
|
||||
|
||||
**Client Requests Changes:**
|
||||
|
||||
```typescript
|
||||
POST /api/sync/pull
|
||||
{
|
||||
"sourceId": "client-component-id",
|
||||
"lastSyncedChangeId": 12345
|
||||
}
|
||||
```
|
||||
|
||||
**Server Responds:**
|
||||
|
||||
```typescript
|
||||
Response:
|
||||
{
|
||||
"notes": [
|
||||
{ noteId: "abc", title: "New Note", ... }
|
||||
],
|
||||
"branches": [...],
|
||||
"attributes": [...],
|
||||
"revisions": [...],
|
||||
"attachments": [...],
|
||||
"entityChanges": [
|
||||
{ entityName: "notes", entityId: "abc", changeId: "...", ... }
|
||||
],
|
||||
"maxChangeId": 12890
|
||||
}
|
||||
```
|
||||
|
||||
**Client Processing:**
|
||||
|
||||
1. Apply entity changes to local database
|
||||
2. Update Froca cache
|
||||
3. Update local sync version
|
||||
4. Trigger UI refresh
|
||||
|
||||
### Push Sync (Client → Server)
|
||||
|
||||
**Client Sends Changes:**
|
||||
|
||||
```typescript
|
||||
POST /api/sync/push
|
||||
{
|
||||
"sourceId": "client-component-id",
|
||||
"entities": [
|
||||
{
|
||||
"entity": {
|
||||
"noteId": "xyz",
|
||||
"title": "Modified Note",
|
||||
...
|
||||
},
|
||||
"entityChange": {
|
||||
"changeId": "change-uuid",
|
||||
"entityName": "notes",
|
||||
...
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Server Processing:**
|
||||
|
||||
1. Validate changes
|
||||
2. Check for conflicts
|
||||
3. Apply changes to database
|
||||
4. Update Becca cache
|
||||
5. Mark as synced
|
||||
6. Broadcast to other connected clients via WebSocket
|
||||
|
||||
**Conflict Detection:**
|
||||
|
||||
```typescript
|
||||
// Check if entity was modified on server since client's last sync
|
||||
const serverEntity = becca.getNote(noteId)
|
||||
const serverLastModified = serverEntity.utcDateModified
|
||||
|
||||
if (serverLastModified > clientSyncVersion) {
|
||||
// CONFLICT!
|
||||
resolveConflict(serverEntity, clientEntity)
|
||||
}
|
||||
```
|
||||
|
||||
## Conflict Resolution
|
||||
|
||||
### Conflict Types
|
||||
|
||||
**1\. Content Conflict**
|
||||
|
||||
* Both client and server modified same note content
|
||||
* **Resolution**: Last-write-wins based on `utcDateModified`
|
||||
|
||||
**2\. Structure Conflict**
|
||||
|
||||
* Branch moved/deleted on one side, modified on other
|
||||
* **Resolution**: Tombstone records, reconciliation
|
||||
|
||||
**3\. Attribute Conflict**
|
||||
|
||||
* Same attribute modified differently
|
||||
* **Resolution**: Last-write-wins
|
||||
|
||||
### Conflict Resolution Strategy
|
||||
|
||||
**Last-Write-Wins:**
|
||||
|
||||
```typescript
|
||||
if (clientEntity.utcDateModified > serverEntity.utcDateModified) {
|
||||
// Client wins, apply client changes
|
||||
applyClientChange(clientEntity)
|
||||
} else {
|
||||
// Server wins, reject client change
|
||||
// Client will pull server version on next sync
|
||||
}
|
||||
```
|
||||
|
||||
**Tombstone Records:**
|
||||
|
||||
* Deleted entities leave tombstone in `entity_changes`
|
||||
* Prevents re-sync of deleted items
|
||||
* `isErased = 1` for permanent deletions
|
||||
|
||||
### Protected Notes Sync
|
||||
|
||||
**Challenge:** Encrypted content can't be synced without password
|
||||
|
||||
**Solution:**
|
||||
|
||||
1. **Encrypted sync**: Content synced in encrypted form
|
||||
2. **Hash verification**: Integrity checked without decryption
|
||||
3. **Lazy decryption**: Only decrypt when accessed
|
||||
|
||||
## Sync States
|
||||
|
||||
### Connection States
|
||||
|
||||
* **Connected**: WebSocket connection active
|
||||
* **Disconnected**: No connection to sync server
|
||||
* **Syncing**: Actively transferring data
|
||||
* **Conflict**: Sync paused due to conflict
|
||||
|
||||
### Entity Sync States
|
||||
|
||||
Each entity can be in:
|
||||
|
||||
* **Synced**: In sync with server
|
||||
* **Pending**: Local changes not yet pushed
|
||||
* **Conflict**: Conflicting changes detected
|
||||
|
||||
### UI Indicators
|
||||
|
||||
```typescript
|
||||
// apps/client/src/widgets/sync_status.ts
|
||||
class SyncStatusWidget {
|
||||
showSyncStatus() {
|
||||
if (isConnected && allSynced) {
|
||||
showIcon('synced')
|
||||
} else if (isSyncing) {
|
||||
showIcon('syncing-spinner')
|
||||
} else {
|
||||
showIcon('not-synced')
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Optimizations
|
||||
|
||||
### Incremental Sync
|
||||
|
||||
Only entities changed since last sync are transferred:
|
||||
|
||||
```
|
||||
SELECT * FROM entity_changes
|
||||
WHERE id > :lastSyncedChangeId
|
||||
ORDER BY id ASC
|
||||
LIMIT 1000
|
||||
```
|
||||
|
||||
### Batch Processing
|
||||
|
||||
Changes sent in batches to reduce round trips:
|
||||
|
||||
```typescript
|
||||
const BATCH_SIZE = 1000
|
||||
const changes = getUnsyncedChanges(BATCH_SIZE)
|
||||
await syncBatch(changes)
|
||||
```
|
||||
|
||||
### Hash-Based Change Detection
|
||||
|
||||
```typescript
|
||||
// Only sync if hash differs
|
||||
const localHash = calculateHash(localEntity)
|
||||
const serverHash = getServerHash(entityId)
|
||||
|
||||
if (localHash !== serverHash) {
|
||||
syncEntity(localEntity)
|
||||
}
|
||||
```
|
||||
|
||||
### Compression
|
||||
|
||||
Large payloads compressed before transmission:
|
||||
|
||||
```typescript
|
||||
// Server sends compressed response
|
||||
res.setHeader('Content-Encoding', 'gzip')
|
||||
res.send(gzip(syncData))
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Network Errors
|
||||
|
||||
Reported to the user and the sync will be retried after the interval passes.
|
||||
|
||||
### Sync Integrity Checks
|
||||
|
||||
**Hash Verification:**
|
||||
|
||||
```typescript
|
||||
// Verify entity hash matches
|
||||
const calculatedHash = calculateHash(entity)
|
||||
const receivedHash = entityChange.hash
|
||||
|
||||
if (calculatedHash !== receivedHash) {
|
||||
throw new Error('Hash mismatch - data corruption detected')
|
||||
}
|
||||
```
|
||||
|
||||
**Consistency Checks:**
|
||||
|
||||
* Orphaned branches detection
|
||||
* Missing parent notes
|
||||
* Invalid entity references
|
||||
* Circular dependencies
|
||||
|
||||
## Sync Server Configuration
|
||||
|
||||
### Server Setup
|
||||
|
||||
**Required Options:**
|
||||
|
||||
```javascript
|
||||
{
|
||||
"syncServerHost": "https://sync.example.com",
|
||||
"syncServerTimeout": 60000,
|
||||
"syncProxy": "" // Optional HTTP proxy
|
||||
}
|
||||
```
|
||||
|
||||
**Authentication:**
|
||||
|
||||
* Username/password or
|
||||
* Sync token (generated on server)
|
||||
|
||||
## Sync API Endpoints
|
||||
|
||||
Located at: `apps/server/src/routes/api/sync.ts`
|
||||
|
||||
## WebSocket Sync Updates
|
||||
|
||||
Real-time sync via WebSocket:
|
||||
|
||||
```typescript
|
||||
// Server broadcasts change to all connected clients
|
||||
ws.broadcast('frontend-update', {
|
||||
lastSyncedPush,
|
||||
entityChanges
|
||||
})
|
||||
|
||||
// Client receives and processed the information.
|
||||
```
|
||||
|
||||
## Sync Scheduling
|
||||
|
||||
### Automatic Sync
|
||||
|
||||
**Desktop:**
|
||||
|
||||
* Sync on startup
|
||||
* Periodic sync (configurable interval, default: 60s)
|
||||
|
||||
**Server:**
|
||||
|
||||
* Sync on entity modification
|
||||
* WebSocket push to connected clients
|
||||
|
||||
### Manual Sync
|
||||
|
||||
User can trigger:
|
||||
|
||||
* Full sync
|
||||
* Sync now
|
||||
* Sync specific subtree
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**Sync stuck:**
|
||||
|
||||
```
|
||||
-- Reset sync state
|
||||
UPDATE entity_changes SET isSynced = 0;
|
||||
DELETE FROM options WHERE name LIKE 'sync%';
|
||||
```
|
||||
|
||||
**Hash mismatch:**
|
||||
|
||||
* Data corruption detected
|
||||
* Re-sync from backup
|
||||
* Check database integrity
|
||||
|
||||
**Conflict loop:**
|
||||
|
||||
* Manual intervention required
|
||||
* Export conflicting notes
|
||||
* Choose winning version
|
||||
* Re-sync
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Encrypted Sync
|
||||
|
||||
* Protected notes synced encrypted
|
||||
* No plain text over network
|
||||
* Server cannot read protected content
|
||||
|
||||
### Authentication
|
||||
|
||||
* Username/password over HTTPS only
|
||||
* Sync tokens for token-based auth
|
||||
* Session cookies with CSRF protection
|
||||
|
||||
### Authorization
|
||||
|
||||
* Users can only sync their own data
|
||||
* No cross-user sync support
|
||||
* Sync server validates ownership
|
||||
|
||||
## Performance Metrics
|
||||
|
||||
**Typical Sync Performance:**
|
||||
|
||||
* 1000 changes: ~2-5 seconds
|
||||
* 10000 changes: ~20-50 seconds
|
||||
* Initial full sync (100k notes): ~5-10 minutes
|
||||
|
||||
**Factors:**
|
||||
|
||||
* Network latency
|
||||
* Database size
|
||||
* Number of protected notes
|
||||
* Attachment sizes
|
||||
Reference in New Issue
Block a user