fix: read full frontmatter for note metadata and speed up title and graph scans

Note lists, mobile listings and tag counts read a fixed 1-2 KB prefix, so notes
with longer frontmatter lost their title and tags. Read the complete frontmatter
instead, stopping after the closing delimiter (256 KB cap if unterminated).

Scan note titles in parallel from the frontmatter only, and build graph nodes and
links in one parallel read per note without keeping note bodies in memory. The
Ctrl+K link dialog reuses the loaded wiki-link title cache.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Yuri KaramianandClaude Sonnet 5.5 committed 2026-10-07 20:39:21 +02:00
1 parent f07241dd3a
commit ac2c0aef94
3 files changed
+368 -137

No files matched your search

+252 -118
View File
@@ -1017,50 +1017,48 @@ pub async fn get_all_note_titles(
}
fn scan_all_note_titles(vault_path: &str) -> Result<Vec<NoteTitleEntry>, String> {
use rayon::prelude::*;
let vault = std::path::Path::new(vault_path);
let hn_dir = operations::helixnotes_dir(vault_path);
let mut entries = Vec::new();
for entry in walkdir::WalkDir::new(vault)
let paths: Vec<PathBuf> = walkdir::WalkDir::new(vault)
.into_iter()
// Cross-platform exclusion (string "/.helixnotes/" checks miss Windows
// backslash paths, which leaked .helixnotes/history snapshots in as dupes).
.filter_entry(|e| !operations::is_hidden(e.path()) && !e.path().starts_with(&hn_dir))
.filter_map(|e| e.ok())
{
let path = entry.path();
if !path.is_file() {
continue;
}
if path.extension().and_then(|e| e.to_str()) != Some("md") {
continue;
}
// Read frontmatter title, fallback to filename
let title = if let Ok(raw) = std::fs::read_to_string(path) {
crate::vault::frontmatter::extract_title(&raw).unwrap_or_else(|| {
path.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string()
})
} else {
path.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string()
};
.filter(|e| {
e.path().is_file() && e.path().extension().and_then(|x| x.to_str()) == Some("md")
})
.map(|e| e.into_path())
.collect();
// Store vault-relative path with forward slashes for cross-platform portability.
// Absolute paths (e.g. C:\Users\...) break when the vault is synced to another device.
let rel = path
.strip_prefix(vault)
.map(|r| r.to_string_lossy().replace('\\', "/").to_string())
.unwrap_or_else(|_| path.to_string_lossy().to_string());
// Only the frontmatter is needed, so read the note head in parallel instead of whole files.
Ok(paths
.par_iter()
.map(|path| {
// Read frontmatter title, fallback to filename
let title = operations::read_note_head(path)
.ok()
.and_then(|head| crate::vault::frontmatter::extract_title(&head))
.unwrap_or_else(|| {
path.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string()
});
entries.push(NoteTitleEntry { title, path: rel });
}
// Store vault-relative path with forward slashes for cross-platform portability.
// Absolute paths (e.g. C:\Users\...) break when the vault is synced to another device.
let rel = path
.strip_prefix(vault)
.map(|r| r.to_string_lossy().replace('\\', "/").to_string())
.unwrap_or_else(|_| path.to_string_lossy().to_string());
Ok(entries)
NoteTitleEntry { title, path: rel }
})
.collect())
}
#[tauri::command]
@@ -1096,65 +1094,109 @@ pub async fn get_graph_data(
.map_err(|error| error.to_string())?
}
/// Wiki-link targets in `body`, in order, with `|alias`, `#heading` and `^block` removed and
/// lowercased (inline scan, no regex).
fn extract_wiki_links(body: &str) -> Vec<String> {
let mut links = Vec::new();
let bytes = body.as_bytes();
let len = bytes.len();
let mut i = 0;
while i + 1 < len {
if bytes[i] == b'[' && bytes[i + 1] == b'[' {
i += 2;
let start = i;
while i + 1 < len && !(bytes[i] == b']' && bytes[i + 1] == b']') {
i += 1;
}
if i + 1 < len {
let link_raw = &body[start..i];
let link = link_raw.split('|').next().unwrap_or(link_raw);
let link = link.split('#').next().unwrap_or(link);
let link = link.split('^').next().unwrap_or(link);
links.push(link.trim().to_lowercase());
i += 2;
}
} else {
i += 1;
}
}
links
}
fn scan_graph_data(vault_path: &str) -> Result<crate::types::GraphData, String> {
use rayon::prelude::*;
use std::collections::{HashMap, HashSet};
let vault = std::path::Path::new(vault_path);
let hn_dir = operations::helixnotes_dir(vault_path);
// Collect every note path first. Cross-platform exclusion (string "/.helixnotes/" checks miss
// Windows backslash paths; is_hidden also covers .stversions/.stfolder/.trash/.git).
let candidates: Vec<PathBuf> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_entry(|e| !operations::is_hidden(e.path()) && !e.path().starts_with(&hn_dir))
.filter_map(|e| e.ok())
.filter(|entry| {
let path = entry.path();
path.is_file()
&& path.extension().and_then(|e| e.to_str()) == Some("md")
// Skip Syncthing conflict files
&& !path
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|name| name.contains(".sync-conflict-"))
})
.map(|entry| entry.into_path())
.collect();
// Deduplicate by canonical path (handles symlinks), keeping the first of each in walk order.
let canonical: Vec<String> = candidates
.par_iter()
.map(|path| {
path.canonicalize()
.unwrap_or_else(|_| path.clone())
.to_string_lossy()
.to_string()
})
.collect();
let mut seen_paths: HashSet<String> = HashSet::new();
let paths: Vec<&PathBuf> = candidates
.iter()
.zip(canonical)
.filter_map(|(path, canonical)| seen_paths.insert(canonical).then_some(path))
.collect();
// Read each note once, in parallel, and keep only its title and links so note bodies
// never accumulate in memory.
let scanned: Vec<(String, Vec<String>)> = paths
.par_iter()
.map(|path| {
let raw = std::fs::read_to_string(path).unwrap_or_default();
// Fast title extraction: scan for "title: " line in frontmatter without full YAML parse
let title = extract_title_fast(&raw).unwrap_or_else(|| {
path.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string()
});
(title, extract_wiki_links(&raw))
})
.collect();
// Pass 1: collect ALL notes as nodes (no title deduplication - every note must appear)
let mut graph_nodes = Vec::new();
let mut graph_nodes = Vec::with_capacity(paths.len());
// title → list of node indices (one title can map to multiple notes in different folders)
let mut title_to_idxs: HashMap<String, Vec<usize>> = HashMap::new();
// vault-relative path without extension → idx (for [[subfolder/note]] style links)
let mut relpath_to_idx: HashMap<String, usize> = HashMap::new();
// absolute path → idx (for fast active-note lookup)
let mut path_to_idx: HashMap<String, usize> = HashMap::new();
let mut seen_paths: HashSet<String> = HashSet::new();
let mut contents: Vec<String> = Vec::new();
let hn_dir = operations::helixnotes_dir(vault_path);
for entry in walkdir::WalkDir::new(vault)
.into_iter()
// Cross-platform exclusion (string "/.helixnotes/" checks miss Windows
// backslash paths; is_hidden also covers .stversions/.stfolder/.trash/.git).
.filter_entry(|e| !operations::is_hidden(e.path()) && !e.path().starts_with(&hn_dir))
.filter_map(|e| e.ok())
{
let path = entry.path();
if !path.is_file() {
continue;
}
let path_str = path.to_string_lossy().to_string();
if path.extension().and_then(|e| e.to_str()) != Some("md") {
continue;
}
// Skip Syncthing conflict files
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if name.contains(".sync-conflict-") {
continue;
}
}
// Deduplicate by canonical path (handles symlinks)
let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
let canonical_str = canonical.to_string_lossy().to_string();
if !seen_paths.insert(canonical_str) {
continue;
}
let raw = std::fs::read_to_string(path).unwrap_or_default();
// Fast title extraction: scan for "title: " line in frontmatter without full YAML parse
let title = extract_title_fast(&raw).unwrap_or_else(|| {
path.file_stem()
.unwrap_or_default()
.to_string_lossy()
.to_string()
});
let mut links_by_source: Vec<Vec<String>> = Vec::with_capacity(paths.len());
for (path, (title, links)) in paths.iter().zip(scanned) {
let idx = graph_nodes.len();
let title_lower = title.to_lowercase();
title_to_idxs.entry(title_lower).or_default().push(idx);
path_to_idx.insert(path_str.clone(), idx);
title_to_idxs
.entry(title.to_lowercase())
.or_default()
.push(idx);
// Also index by vault-relative path without extension (e.g. "subfolder/note name")
if let Ok(rel) = path.strip_prefix(vault) {
@@ -1169,12 +1211,12 @@ fn scan_graph_data(vault_path: &str) -> Result<crate::types::GraphData, String>
graph_nodes.push(crate::types::GraphNode {
title,
path: path_str,
path: path.to_string_lossy().to_string(),
});
contents.push(raw);
links_by_source.push(links);
}
// Pass 2: extract edges from wiki-links (inline scan, no regex)
// Pass 2: resolve wiki-links to edges
// edge_map: (src, tgt) → index in edges vec, used to detect reverse links
let mut edges: Vec<crate::types::GraphEdge> = Vec::new();
let mut edge_map: HashMap<(usize, usize), usize> = HashMap::new();
@@ -1203,47 +1245,25 @@ fn scan_graph_data(vault_path: &str) -> Result<crate::types::GraphData, String>
}
};
for (source_idx, body) in contents.iter().enumerate() {
let bytes = body.as_bytes();
let len = bytes.len();
let mut i = 0;
while i + 1 < len {
if bytes[i] == b'[' && bytes[i + 1] == b'[' {
i += 2;
let start = i;
while i + 1 < len && !(bytes[i] == b']' && bytes[i + 1] == b']') {
i += 1;
for (source_idx, links) in links_by_source.iter().enumerate() {
for link in links {
// 1. Try vault-relative path match (e.g. "arkhost/note name")
if let Some(&target_idx) = relpath_to_idx.get(link) {
add_edge(&mut edges, &mut edge_map, source_idx, target_idx);
} else if let Some(targets) = title_to_idxs.get(link) {
// 2. Title match - connect to all notes with this title
for &target_idx in targets {
add_edge(&mut edges, &mut edge_map, source_idx, target_idx);
}
if i + 1 < len {
let link_raw = &body[start..i];
// Strip |alias, #heading, ^block
let link = link_raw.split('|').next().unwrap_or(link_raw);
let link = link.split('#').next().unwrap_or(link);
let link = link.split('^').next().unwrap_or(link);
let link = link.trim().to_lowercase();
// 1. Try vault-relative path match (e.g. "arkhost/note name")
if let Some(&target_idx) = relpath_to_idx.get(&link) {
add_edge(&mut edges, &mut edge_map, source_idx, target_idx);
} else if let Some(targets) = title_to_idxs.get(&link) {
// 2. Title match - connect to all notes with this title
} else if link.contains('/') {
// 3. Path-based ref: try last segment as title fallback
if let Some(seg) = link.rsplit('/').next() {
if let Some(targets) = title_to_idxs.get(seg) {
for &target_idx in targets {
add_edge(&mut edges, &mut edge_map, source_idx, target_idx);
}
} else if link.contains('/') {
// 3. Path-based ref: try last segment as title fallback
if let Some(seg) = link.rsplit('/').next() {
if let Some(targets) = title_to_idxs.get(seg) {
for &target_idx in targets {
add_edge(&mut edges, &mut edge_map, source_idx, target_idx);
}
}
}
}
i += 2;
}
} else {
i += 1;
}
}
}
@@ -1386,6 +1406,120 @@ fn parse_note_tasks(note_path: &str, note_title: &str, body: &str) -> Vec<TaskIt
tasks
}
#[cfg(test)]
mod note_title_scan_tests {
use super::scan_all_note_titles;
use std::fs;
#[test]
fn titles_come_from_complete_frontmatter_with_filename_fallback_and_relative_paths() {
let parent = tempfile::tempdir().unwrap();
let vault = parent.path().join("vault");
fs::create_dir(&vault).unwrap();
let tags = (0..200)
.map(|index| format!("tag-{index}"))
.collect::<Vec<_>>()
.join(", ");
fs::create_dir_all(vault.join("Notebook/.helixnotes")).unwrap();
fs::create_dir_all(vault.join(".helixnotes/history")).unwrap();
fs::write(
vault.join("Notebook/Long.md"),
format!("---\ntags: [{tags}]\ntitle: Long Title\n---\n# Long Title\n"),
)
.unwrap();
fs::write(vault.join("Plain.md"), "# No frontmatter\n").unwrap();
fs::write(
vault.join(".helixnotes/history/Old.md"),
"---\ntitle: Old\n---\n",
)
.unwrap();
fs::write(vault.join("Other.txt"), "---\ntitle: Skip\n---\n").unwrap();
let mut titles = scan_all_note_titles(vault.to_str().unwrap()).unwrap();
titles.sort_by(|a, b| a.path.cmp(&b.path));
let found: Vec<_> = titles
.iter()
.map(|t| (t.path.as_str(), t.title.as_str()))
.collect();
assert_eq!(
found,
[("Notebook/Long.md", "Long Title"), ("Plain.md", "Plain")]
);
}
}
#[cfg(test)]
mod graph_scan_tests {
use super::scan_graph_data;
use std::fs;
#[test]
fn graph_links_resolve_by_path_and_title_and_mark_reverse_links() {
let parent = tempfile::tempdir().unwrap();
let vault = parent.path().join("vault");
fs::create_dir_all(vault.join("folder")).unwrap();
fs::create_dir_all(vault.join("other")).unwrap();
fs::create_dir_all(vault.join(".helixnotes")).unwrap();
let write = |path: &str, text: &str| fs::write(vault.join(path), text).unwrap();
write("A.md", "---\ntitle: \"Alpha\"\n---\n[[Beta]] [[folder/C|alias]] [[Twin#Head]] [[missing]] [[unterminated");
write(
"B.md",
"---\ntitle: Beta\n---\nback to [[alpha]] and [[Beta]] and [[nested/ghost/Beta]]",
);
write("folder/C.md", "---\ntitle: Gamma\n---\n[[A^block]]");
write("folder/Twin.md", "---\ntitle: Twin\n---\n");
write("other/Twin.md", "---\ntitle: Twin\n---\n");
write("A.sync-conflict-20260101.md", "[[Beta]]");
write(".helixnotes/Hidden.md", "[[Beta]]");
write("ignored.txt", "[[Beta]]");
let graph = scan_graph_data(vault.to_str().unwrap()).unwrap();
let name = |index: usize| {
let node = &graph.nodes[index];
let relative = node.path.strip_prefix(vault.to_str().unwrap()).unwrap();
format!(
"{}|{}",
relative.trim_start_matches(['/', '\\']).replace('\\', "/"),
node.title
)
};
let mut nodes: Vec<_> = (0..graph.nodes.len()).map(name).collect();
nodes.sort();
assert_eq!(
nodes,
[
"A.md|Alpha",
"B.md|Beta",
"folder/C.md|Gamma",
"folder/Twin.md|Twin",
"other/Twin.md|Twin"
]
);
let mut edges: Vec<_> = graph
.edges
.iter()
.map(|edge| {
let mut pair = [name(edge.source), name(edge.target)];
// Which side of a two-way link is stored first follows directory order.
if edge.bidirectional {
pair.sort();
}
format!("{}>{}:{}", pair[0], pair[1], edge.bidirectional)
})
.collect();
edges.sort();
assert_eq!(
edges,
[
"A.md|Alpha>B.md|Beta:true",
"A.md|Alpha>folder/C.md|Gamma:true",
"A.md|Alpha>folder/Twin.md|Twin:false",
"A.md|Alpha>other/Twin.md|Twin:false",
]
);
}
}
#[cfg(test)]
mod task_tests {
use super::scan_tasks;
+112 -18
View File
@@ -606,12 +606,7 @@ fn read_note_entry(path: &Path, vault_root: &Path) -> Result<NoteEntry, String>
/// Persisted creation dates survive atomic file replacement.
#[cfg(any(mobile, test))]
fn read_note_entry_metadata_only(path: &Path, vault_root: &Path) -> Result<NoteEntry, String> {
// Read first 2KB - enough for frontmatter with tags, title, pinned
let mut file = fs::File::open(path).map_err(|e| e.to_string())?;
let mut buf = vec![0u8; 2048];
let bytes_read = file.read(&mut buf).map_err(|e| e.to_string())?;
buf.truncate(bytes_read);
let raw = String::from_utf8_lossy(&buf);
let raw = read_note_head(path)?;
let filename = path
.file_name()
@@ -649,13 +644,54 @@ fn read_note_entry_metadata_only(path: &Path, vault_root: &Path) -> Result<NoteE
})
}
/// Fast version: reads only the first ~2KB of the file (enough for frontmatter + preview).
pub(crate) fn read_note_entry_fast(path: &Path, vault_root: &Path) -> Result<NoteEntry, String> {
const NOTE_HEAD_BLOCK: usize = 2048;
const NOTE_HEAD_LIMIT: usize = 256 * 1024;
/// Where the YAML frontmatter stands within a partially read note: `None` when the note has
/// none, `Some(None)` while the closing delimiter is still unread, else the offset after it.
fn frontmatter_state(head: &[u8]) -> Option<Option<usize>> {
let rest = head.strip_prefix(b"---")?;
if !matches!(rest.first(), None | Some(b'\n' | b'\r')) {
return None;
}
let mut offset = 3;
for line in head[3..].split_inclusive(|byte| *byte == b'\n') {
offset += line.len();
if line.ends_with(b"\n") && line.trim_ascii_end() == b"---" {
return Some(Some(offset));
}
}
Some(None)
}
/// Reads the start of a note: its whole frontmatter plus a block of body for previews, instead
/// of a fixed prefix that truncates long frontmatter and loses its title and tags.
pub(crate) fn read_note_head(path: &Path) -> Result<String, String> {
let mut file = fs::File::open(path).map_err(|e| e.to_string())?;
let mut buf = vec![0u8; 2048];
let bytes_read = file.read(&mut buf).map_err(|e| e.to_string())?;
buf.truncate(bytes_read);
let raw = String::from_utf8_lossy(&buf);
let mut head = Vec::with_capacity(NOTE_HEAD_BLOCK);
let mut target = NOTE_HEAD_BLOCK;
loop {
let wanted = target - head.len();
(&mut file)
.take(wanted as u64)
.read_to_end(&mut head)
.map_err(|e| e.to_string())?;
if head.len() < target || head.len() >= NOTE_HEAD_LIMIT {
break;
}
target = match frontmatter_state(&head) {
None => break,
Some(None) => (head.len() * 2).min(NOTE_HEAD_LIMIT),
Some(Some(end)) if head.len() >= end + NOTE_HEAD_BLOCK => break,
Some(Some(end)) => end + NOTE_HEAD_BLOCK,
};
}
Ok(String::from_utf8_lossy(&head).into_owned())
}
/// Fast version: reads the frontmatter and the start of the body (enough for a preview).
pub(crate) fn read_note_entry_fast(path: &Path, vault_root: &Path) -> Result<NoteEntry, String> {
let raw = read_note_head(path)?;
read_note_entry_from_str(&raw, path, vault_root)
}
@@ -1675,15 +1711,11 @@ pub fn get_all_tags(vault_path: &str) -> Result<Vec<(String, usize)>, String> {
.map(|e| e.path().to_path_buf())
.collect();
// Read frontmatter in parallel (only need tags, so partial read is fine)
// Read frontmatter in parallel (only need tags, so the body is not read)
let all_tags: Vec<Vec<String>> = md_files
.par_iter()
.filter_map(|path| {
let mut file = fs::File::open(path).ok()?;
let mut buf = vec![0u8; 1024]; // Tags are in frontmatter, 1KB is plenty
let n = file.read(&mut buf).ok()?;
buf.truncate(n);
let raw = String::from_utf8_lossy(&buf);
let raw = read_note_head(path).ok()?;
let filename = path.file_name()?.to_string_lossy().to_string();
let (meta, _) = frontmatter::parse_note(&raw, &filename);
Some(meta.tags)
@@ -3564,6 +3596,68 @@ mod tests {
assert!(!outside.join("Stashed").exists());
}
#[test]
fn header_reads_keep_metadata_when_frontmatter_exceeds_the_first_block() {
let vault = tempfile::tempdir().unwrap();
let tags: Vec<String> = (0..120)
.map(|index| format!("long-tag-name-{index}"))
.collect();
let raw = format!(
"---\nid: existing\ntitle: Long Frontmatter\ntags: [{}]\n---\n# Long Frontmatter\n\nBody.\n",
tags.join(", ")
);
assert!(raw.find("\n---\n#").unwrap() > 2048);
fs::write(vault.path().join("Long.md"), raw).unwrap();
let entry =
super::read_note_entry_fast(&vault.path().join("Long.md"), vault.path()).unwrap();
assert_eq!(entry.meta.title, "Long Frontmatter");
assert_eq!(entry.meta.tags, tags);
let counted = super::get_all_tags(vault.path().to_str().unwrap()).unwrap();
assert_eq!(counted.len(), tags.len());
}
#[test]
fn note_head_stops_after_frontmatter_and_one_body_block() {
let vault = tempfile::tempdir().unwrap();
let path = vault.path().join("Note.md");
let tags = (0..400)
.map(|i| format!("tag-{i}"))
.collect::<Vec<_>>()
.join(", ");
for newline in ["\n", "\r\n"] {
let body = "x".repeat(100_000);
let raw = format!("---{newline}tags: [{tags}]{newline}---{newline}{body}");
fs::write(&path, &raw).unwrap();
let head = super::read_note_head(&path).unwrap();
let end = raw.find(&format!("{newline}---{newline}")).unwrap() + newline.len() * 2 + 3;
assert!(head.len() >= end + 2048 && head.len() < end + 2 * 2048);
assert!(raw.starts_with(&head));
}
}
#[test]
fn note_head_without_frontmatter_reads_one_block_and_unterminated_is_capped() {
let vault = tempfile::tempdir().unwrap();
let path = vault.path().join("Note.md");
fs::write(&path, "# Title\n".to_string() + &"y".repeat(50_000)).unwrap();
assert_eq!(super::read_note_head(&path).unwrap().len(), 2048);
fs::write(
&path,
"---\ntitle: Open\n".to_string() + &"z\n".repeat(300_000),
)
.unwrap();
assert_eq!(super::read_note_head(&path).unwrap().len(), 256 * 1024);
fs::write(
&path,
"---not frontmatter\n".to_string() + &"y".repeat(50_000),
)
.unwrap();
assert_eq!(super::read_note_head(&path).unwrap().len(), 2048);
fs::write(&path, "---\n---\nShort").unwrap();
assert_eq!(super::read_note_head(&path).unwrap(), "---\n---\nShort");
}
#[test]
fn vault_state_save_replaces_file_without_truncating_open_reader() {
let vault = tempfile::tempdir().unwrap();