Merge branch 'perf/parallel-listing-upstream' into 'main'

perf(sync): list WebDAV folders several at a time

See merge request ArkHost/HelixNotes!17
This commit is contained in:
Yuri Karamian committed 2026-10-28 07:06:40 +00:00
commit a80425e084
6 files changed
+406 -17

No files matched your search

+26
View File
@@ -4103,6 +4103,28 @@ mod vault_identity_tests {
config.active_bookmark_id = None;
assert_eq!(active_vault_config(&config).unwrap().name, "Local");
}
#[test]
fn the_folder_request_setting_reaches_the_sync_and_older_configs_get_the_default() {
let mut config = AppConfig {
active_vault: Some("/v".to_string()),
vaults: vec![VaultConfig {
path: "/v".to_string(),
name: "v".to_string(),
sync_provider: Some("webdav".to_string()),
webdav_url: Some("https://x.test/dav".to_string()),
..Default::default()
}],
..Default::default()
};
assert_eq!(sync_config_from(&config).unwrap().listing_parallelism, 0);
config.vaults[0].sync_folder_requests = 3;
assert_eq!(sync_config_from(&config).unwrap().listing_parallelism, 3);
// A vault saved before the setting existed has no such field.
let older: VaultConfig = serde_json::from_str(r#"{"path":"/v","name":"v"}"#).unwrap();
assert_eq!(older.sync_folder_requests, 0);
}
}
fn sync_config_from(config: &AppConfig) -> Result<crate::sync::WebdavConfig, String> {
@@ -4119,6 +4141,7 @@ fn sync_config_from(config: &AppConfig) -> Result<crate::sync::WebdavConfig, Str
url,
username: v.webdav_username.clone().unwrap_or_default(),
password: v.webdav_password.clone().unwrap_or_default(),
listing_parallelism: v.sync_folder_requests as usize,
})
}
@@ -4133,6 +4156,7 @@ pub fn set_sync_settings(
sync_on_open: bool,
sync_on_change: bool,
sync_interval_minutes: u32,
sync_folder_requests: u32,
) -> Result<(), String> {
let mut config = state.config.lock().map_err(|e| e.to_string())?;
let active_index = active_vault_index(&config)?;
@@ -4144,6 +4168,8 @@ pub fn set_sync_settings(
v.sync_on_open = sync_on_open;
v.sync_on_change = sync_on_change;
v.sync_interval_minutes = sync_interval_minutes;
v.sync_folder_requests =
crate::sync::effective_listing_parallelism(sync_folder_requests as usize) as u32;
save_app_config(&config)?;
Ok(())
}
+359 -15
View File
@@ -22,12 +22,42 @@ use std::path::{Path, PathBuf};
use tauri::{Emitter, Manager};
use walkdir::WalkDir;
/// Folder listings requested at once unless the settings say otherwise. A handful hides most of
/// the round-trip time and stays polite to a small server.
pub const DEFAULT_LISTING_PARALLELISM: usize = 6;
/// The most a setting can ask for.
pub const MAX_LISTING_PARALLELISM: usize = 16;
/// What a stored setting means: `0` (never chosen) is the default, and nothing exceeds the
/// maximum. The one place these rules live.
pub fn effective_listing_parallelism(setting: usize) -> usize {
match setting {
0 => DEFAULT_LISTING_PARALLELISM,
n => n.min(MAX_LISTING_PARALLELISM),
}
}
/// A listing error that asking again cannot fix: a wrong login, a folder that is not there, or
/// a client error the server will answer the same way. Anything else, such as a dropped
/// connection, a timeout, 429 or a 5xx, may pass.
fn is_final_listing_error(error: &str) -> bool {
if error.starts_with("Authentication failed") || error.starts_with("WebDAV path not found") {
return true;
}
error
.strip_prefix("WebDAV PROPFIND failed: HTTP ")
.and_then(|code| code.trim().parse::<u16>().ok())
.is_some_and(|code| (400..500).contains(&code) && code != 408 && code != 429)
}
// ───────────────────────── config ─────────────────────────
pub struct WebdavConfig {
pub url: String,
pub username: String,
pub password: String,
/// Folders listed on the server at once: `0` for the default, `1` for one at a time.
pub listing_parallelism: usize,
}
#[derive(Default, Clone, Serialize)]
@@ -386,6 +416,7 @@ struct WebdavClient {
client: reqwest::blocking::Client,
user: String,
pass: String,
listing_parallelism: usize,
}
impl WebdavClient {
@@ -415,6 +446,7 @@ impl WebdavClient {
client,
user: cfg.username,
pass: cfg.password,
listing_parallelism: effective_listing_parallelism(cfg.listing_parallelism),
})
}
@@ -460,28 +492,85 @@ impl WebdavClient {
/// (Depth: infinity is disabled on some servers, so we walk instead).
fn list(&self) -> Result<HashMap<String, String>, String> {
let mut files = HashMap::new();
let mut visited: HashSet<String> = HashSet::new();
let mut queue: Vec<String> = vec![String::new()];
while let Some(dir) = queue.pop() {
if !visited.insert(dir.clone()) {
continue;
}
for e in self.propfind(&dir, "1")? {
if e.relpath == dir {
continue; // the collection itself
}
if e.is_dir {
if !visited.contains(&e.relpath) {
queue.push(e.relpath);
let mut visited: HashSet<String> = HashSet::from([String::new()]);
// One level of folders at a time: every folder of a level is listed at once.
let mut level: Vec<String> = vec![String::new()];
while !level.is_empty() {
let listings = self.list_folders(&level)?;
let mut next = Vec::new();
for (dir, entries) in level.iter().zip(listings) {
for e in entries {
if e.relpath == *dir {
continue; // the collection itself
}
if e.is_dir {
if visited.insert(e.relpath.clone()) {
next.push(e.relpath);
}
} else {
files.insert(e.relpath, e.etag);
}
} else {
files.insert(e.relpath, e.etag);
}
}
level = next;
}
Ok(files)
}
/// The contents of each folder, in the order given, with up to `listing_parallelism`
/// requests in flight. Once a request fails the others stop taking new folders. A folder
/// that failed, or was not reached, is then listed on its own, one after another, unless
/// its error is final: a server that dislikes concurrent requests costs time, not the sync,
/// while a wrong login or a missing folder is reported at once.
fn list_folders(&self, dirs: &[String]) -> Result<Vec<Vec<RemoteEntry>>, String> {
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
let workers = dirs.len().min(self.listing_parallelism);
if workers <= 1 {
return dirs.iter().map(|dir| self.propfind(dir, "1")).collect();
}
let next = AtomicUsize::new(0);
let failed = AtomicBool::new(false);
let mut listed: Vec<Option<Result<Vec<RemoteEntry>, String>>> =
dirs.iter().map(|_| None).collect();
std::thread::scope(|scope| {
let threads: Vec<_> = (0..workers)
.map(|_| {
scope.spawn(|| {
let mut done = Vec::new();
loop {
if failed.load(Ordering::Relaxed) {
break done;
}
let at = next.fetch_add(1, Ordering::Relaxed);
let Some(dir) = dirs.get(at) else {
break done;
};
let result = self.propfind(dir, "1");
if result.is_err() {
failed.store(true, Ordering::Relaxed);
}
done.push((at, result));
}
})
})
.collect();
for thread in threads {
for (at, result) in thread.join().expect("a listing thread does not panic") {
listed[at] = Some(result);
}
}
});
listed
.into_iter()
.zip(dirs)
.map(|(result, dir)| match result {
Some(Ok(entries)) => Ok(entries),
Some(Err(error)) if is_final_listing_error(&error) => Err(error),
Some(Err(_)) | None => self.propfind(dir, "1"),
})
.collect()
}
fn get(&self, relpath: &str, expected_etag: &str) -> Result<Vec<u8>, String> {
let mut request = self
.client
@@ -1100,6 +1189,253 @@ pub fn run_sync(
mod tests {
use super::*;
/// A WebDAV server for listing tests: `folders` folders below the root, each holding
/// `note.md`, every answer coming after `delay` like a server far away. `status` is given the
/// folder's path and how many requests were in progress as this one arrived (itself
/// included), and may answer with an error status instead of the listing. Returns the URL
/// and a count of the requests received.
fn listing_server(
folders: usize,
delay: std::time::Duration,
status: impl Fn(&str, usize) -> Option<u16> + Send + Sync + 'static,
) -> (String, std::sync::Arc<std::sync::atomic::AtomicUsize>) {
use std::io::{Read, Write};
use std::net::TcpListener;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}/vault", listener.local_addr().unwrap());
let status = Arc::new(status);
let requests = Arc::new(AtomicUsize::new(0));
let in_progress = Arc::new(AtomicUsize::new(0));
let counted = requests.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let mut stream = stream.unwrap();
let (status, requests, in_progress) =
(status.clone(), requests.clone(), in_progress.clone());
std::thread::spawn(move || {
// The whole request, body included: closing with part of it unread can
// reset the connection before the client has read the answer.
let mut request = Vec::new();
let mut chunk = [0; 4096];
let header_end = loop {
let len = stream.read(&mut chunk).unwrap();
assert!(len > 0);
request.extend_from_slice(&chunk[..len]);
if let Some(at) = request.windows(4).position(|part| part == b"\r\n\r\n") {
break at + 4;
}
};
let headers = String::from_utf8_lossy(&request[..header_end]).into_owned();
let content_length: usize = headers
.lines()
.find_map(|line| {
line.to_ascii_lowercase()
.strip_prefix("content-length:")
.and_then(|value| value.trim().parse().ok())
})
.unwrap_or(0);
while request.len() < header_end + content_length {
let len = stream.read(&mut chunk).unwrap();
assert!(len > 0);
request.extend_from_slice(&chunk[..len]);
}
let path = headers
.split_whitespace()
.nth(1)
.unwrap_or("/")
.trim_end_matches('/')
.to_string();
requests.fetch_add(1, Ordering::SeqCst);
let concurrent = in_progress.fetch_add(1, Ordering::SeqCst) + 1;
std::thread::sleep(delay);
in_progress.fetch_sub(1, Ordering::SeqCst);
let reply = match status(&path, concurrent) {
Some(code) => format!(
"HTTP/1.1 {code} Refused\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
),
None => {
let entry = |href: &str, collection: bool| {
format!(
"<d:response><d:href>{href}</d:href><d:propstat><d:prop>\
<d:getetag>\"e\"</d:getetag><d:resourcetype>{}</d:resourcetype>\
</d:prop></d:propstat></d:response>",
if collection { "<d:collection/>" } else { "" }
)
};
let mut body = String::new();
if path == "/vault" {
body += &entry("/vault/", true);
for n in 1..=folders {
body += &entry(&format!("/vault/f{n}/"), true);
}
} else {
body += &entry(&format!("{path}/"), true);
body += &entry(&format!("{path}/note.md"), false);
}
let body =
format!("<d:multistatus xmlns:d=\"DAV:\">{body}</d:multistatus>");
format!(
"HTTP/1.1 207 Multi-Status\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
)
}
};
stream.write_all(reply.as_bytes()).unwrap();
});
}
});
(url, counted)
}
fn listing_client(url: String, listing_parallelism: usize) -> WebdavClient {
WebdavClient::new(WebdavConfig {
url,
username: String::new(),
password: String::new(),
listing_parallelism,
})
.unwrap()
}
fn note_paths(folders: usize) -> Vec<String> {
let mut expected: Vec<String> = (1..=folders).map(|n| format!("f{n}/note.md")).collect();
expected.sort();
expected
}
fn listed_names(client: &WebdavClient) -> Vec<String> {
let mut found: Vec<_> = client.list().unwrap().into_keys().collect();
found.sort();
found
}
#[test]
fn folders_are_listed_at_once_by_default_and_the_result_is_complete() {
const FOLDERS: usize = 12;
const DELAY: std::time::Duration = std::time::Duration::from_millis(150);
let (url, _) = listing_server(FOLDERS, DELAY, |_, _| None);
let client = listing_client(url, 0);
let started = std::time::Instant::now();
let found = listed_names(&client);
let took = started.elapsed();
assert_eq!(found, note_paths(FOLDERS));
// One after another this takes (1 + FOLDERS) * DELAY = 1.95 s; the root and then two
// rounds of six take about 0.45 s.
assert!(
took < std::time::Duration::from_millis(1200),
"listing took {took:?}"
);
}
#[test]
fn a_setting_of_one_lists_one_folder_at_a_time_with_the_same_result() {
const FOLDERS: usize = 4;
const DELAY: std::time::Duration = std::time::Duration::from_millis(100);
let (url, _) = listing_server(FOLDERS, DELAY, |_, _| None);
let client = listing_client(url, 1);
let started = std::time::Instant::now();
let found = listed_names(&client);
let took = started.elapsed();
assert_eq!(found, note_paths(FOLDERS));
// The root and each folder, one after another.
assert!(
took >= DELAY * (FOLDERS as u32 + 1),
"listing took {took:?}"
);
}
#[test]
fn a_server_that_refuses_concurrent_requests_still_lists_everything() {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
const FOLDERS: usize = 4;
let refused = Arc::new(AtomicUsize::new(0));
let counted = refused.clone();
// 503 for a request that arrives while another is being served.
let (url, _) = listing_server(
FOLDERS,
std::time::Duration::from_millis(60),
move |_, concurrent| {
(concurrent > 1).then(|| {
counted.fetch_add(1, Ordering::SeqCst);
503
})
},
);
let client = listing_client(url, 4);
assert_eq!(listed_names(&client), note_paths(FOLDERS));
// The fallback really ran: with four requests at once, at least one was refused.
assert!(refused.load(Ordering::SeqCst) >= 1);
}
#[test]
fn a_final_error_stops_the_listing_at_once_without_asking_again() {
const FOLDERS: usize = 30;
// The first folder answers 403 every time.
let (url, requests) =
listing_server(FOLDERS, std::time::Duration::from_millis(100), |path, _| {
path.ends_with("/f1").then_some(403)
});
let client = listing_client(url, 6);
let error = client.list().unwrap_err();
assert!(error.contains("HTTP 403"), "{error}");
// The root, the six first folders, and no more: the rest were never asked for, and the
// refused one was not repeated.
let received = requests.load(std::sync::atomic::Ordering::SeqCst);
assert!(received < 14, "{received} requests for {FOLDERS} folders");
}
#[test]
fn only_errors_that_another_attempt_could_fix_are_retried() {
for final_error in [
"Authentication failed (check username / app password)",
"WebDAV path not found (check the URL)",
"WebDAV PROPFIND failed: HTTP 403",
"WebDAV PROPFIND failed: HTTP 404",
"WebDAV PROPFIND failed: HTTP 410",
] {
assert!(is_final_listing_error(final_error), "{final_error}");
}
for passing in [
"WebDAV PROPFIND failed: HTTP 429",
"WebDAV PROPFIND failed: HTTP 408",
"WebDAV PROPFIND failed: HTTP 500",
"WebDAV PROPFIND failed: HTTP 503",
"error sending request for url (http://x.test/dav): connection reset",
"operation timed out",
] {
assert!(!is_final_listing_error(passing), "{passing}");
}
}
#[test]
fn the_listing_setting_means_default_when_zero_and_never_exceeds_the_maximum() {
assert_eq!(
effective_listing_parallelism(0),
DEFAULT_LISTING_PARALLELISM
);
assert_eq!(effective_listing_parallelism(1), 1);
assert_eq!(effective_listing_parallelism(3), 3);
assert_eq!(
effective_listing_parallelism(MAX_LISTING_PARALLELISM),
MAX_LISTING_PARALLELISM
);
assert_eq!(effective_listing_parallelism(1000), MAX_LISTING_PARALLELISM);
let parallelism =
|setting| listing_client("http://x.test".into(), setting).listing_parallelism;
assert_eq!(parallelism(0), DEFAULT_LISTING_PARALLELISM);
assert_eq!(parallelism(1000), MAX_LISTING_PARALLELISM);
}
#[test]
fn a_conflict_copy_says_so_in_its_title() {
let raw =
@@ -1400,6 +1736,7 @@ mod tests {
url: "http://127.0.0.1:0/".to_string(),
username: String::new(),
password: String::new(),
listing_parallelism: 0,
})
.unwrap();
let mut changes = HashMap::new();
@@ -1446,6 +1783,7 @@ mod tests {
url: "http://127.0.0.1:0/".to_string(),
username: String::new(),
password: String::new(),
listing_parallelism: 0,
})
.unwrap();
let mut changes = HashMap::new();
@@ -1708,6 +2046,7 @@ mod tests {
url,
username: "alice".into(),
password: "secret".into(),
listing_parallelism: 0,
})
.unwrap(),
worker,
@@ -1742,24 +2081,28 @@ mod tests {
url: "HTTPS://Example.org:443/team/".into(),
username: "alice".into(),
password: "one".into(),
listing_parallelism: 0,
})
.unwrap();
let same = WebdavClient::new(WebdavConfig {
url: "https://example.org/team".into(),
username: "alice".into(),
password: "other".into(),
listing_parallelism: 0,
})
.unwrap();
let other = WebdavClient::new(WebdavConfig {
url: "https://example.org/team".into(),
username: "bob".into(),
password: "one".into(),
listing_parallelism: 0,
})
.unwrap();
let different_folder = WebdavClient::new(WebdavConfig {
url: "https://example.org/another".into(),
username: "alice".into(),
password: "one".into(),
listing_parallelism: 0,
})
.unwrap();
assert_eq!(first.identity(), same.identity());
@@ -2164,6 +2507,7 @@ mod tests {
url,
username: String::new(),
password: String::new(),
listing_parallelism: 0,
})
.unwrap();
client.client = reqwest::blocking::Client::builder()
+3
View File
@@ -102,6 +102,9 @@ pub struct VaultConfig {
pub sync_on_change: bool,
#[serde(default)]
pub sync_interval_minutes: u32,
/// Folders to list on the server at once; 0 means the default.
#[serde(default)]
pub sync_folder_requests: u32,
#[serde(default)]
pub last_sync_time: Option<String>,
#[serde(default)]
+2 -1
View File
@@ -553,8 +553,9 @@ export async function setSyncSettings(
syncOnOpen: boolean,
syncOnChange: boolean,
syncIntervalMinutes: number,
syncFolderRequests: number,
): Promise<void> {
return invoke("set_sync_settings", { provider, url, username, password, syncOnOpen, syncOnChange, syncIntervalMinutes });
return invoke("set_sync_settings", { provider, url, username, password, syncOnOpen, syncOnChange, syncIntervalMinutes, syncFolderRequests });
}
export async function testSyncConnection(): Promise<void> {
+15 -1
View File
@@ -474,6 +474,9 @@
let syncOnOpen = $state(activeVaultConfig($appConfig)?.sync_on_open ?? false);
let syncOnChange = $state(activeVaultConfig($appConfig)?.sync_on_change ?? false);
let syncIntervalMinutes = $state(activeVaultConfig($appConfig)?.sync_interval_minutes ?? 0);
// 0 means never chosen, which the sync treats as DEFAULT_LISTING_PARALLELISM (6) in sync.rs.
const defaultFolderRequests = 6;
let syncFolderRequests = $state(activeVaultConfig($appConfig)?.sync_folder_requests ?? 0);
// Re-seed the sync form when the active vault changes, so Settings reflects the current vault.
let lastSyncVault: string | null = null;
@@ -493,10 +496,11 @@
syncOnOpen = vc?.sync_on_open ?? false;
syncOnChange = vc?.sync_on_change ?? false;
syncIntervalMinutes = vc?.sync_interval_minutes ?? 0;
syncFolderRequests = vc?.sync_folder_requests ?? 0;
});
async function saveSyncSettings() {
await setSyncSettings(syncProvider, syncUrl || null, syncUsername || null, syncPassword || null, syncOnOpen, syncOnChange, syncIntervalMinutes);
await setSyncSettings(syncProvider, syncUrl || null, syncUsername || null, syncPassword || null, syncOnOpen, syncOnChange, syncIntervalMinutes, syncFolderRequests);
// Reflect into the active vault's config in the store so the top-bar button and auto-sync
// triggers update live, without an app restart.
if ($appConfig) {
@@ -513,6 +517,7 @@
sync_on_open: syncOnOpen,
sync_on_change: syncOnChange,
sync_interval_minutes: syncIntervalMinutes,
sync_folder_requests: syncFolderRequests,
} : vault),
};
}
@@ -2613,6 +2618,15 @@
<button class="toggle-switch" class:on={syncOnOpen} role="switch" aria-checked={syncOnOpen} aria-label="Sync when the vault opens" onclick={() => { syncOnOpen = !syncOnOpen; saveSyncSettings(); }}><span class="toggle-knob"></span></button>
</label>
</div>
<div class="settings-section">
<h3>Folder Request Parallelism</h3>
<div class="setting-options">
{#each [...new Set([1, 3, 6, 12, syncFolderRequests || defaultFolderRequests])].sort((a, b) => a - b) as opt}
<button class="option-btn" class:active={(syncFolderRequests || defaultFolderRequests) === opt} onclick={() => { syncFolderRequests = opt; saveSyncSettings(); }}>{opt}</button>
{/each}
</div>
<p class="setting-hint">How many folders are listed on the server at once. 6 suits most servers; choose 1 if yours rejects concurrent requests.</p>
</div>
{/if}
</div>
+1
View File
@@ -63,6 +63,7 @@ export interface VaultConfig {
sync_on_open?: boolean;
sync_on_change?: boolean;
sync_interval_minutes?: number;
sync_folder_requests?: number;
last_sync_time?: string | null;
last_backup_time?: string | null;
}