ArkHost/HelixNotes
1
0
Fork 0
mirror of https://gitlab.com/ArkHost/HelixNotes.git synced 2026-07-24 07:45:57 +02:00
Code Issues Projects Releases Wiki Activity

Robust Obsidian importer: frontmatter normalization, syntax conversion, deprecated property renaming, same-note heading links

Browse Source
This commit is contained in:
Yuri Karamian
2026-06-30 22:10:22 +02:00
parent 13fd70f944
commit 479c957698
7 changed files with 1051 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/commands.rs
+4 -441
View File
@@ -1276,6 +1276,9 @@ pub fn import_obsidian(app: AppHandle) -> Result<(), String> {
"success": true,
"files_converted": result.files_converted,
"links_converted": result.links_converted,
"frontmatter_normalized": result.frontmatter_normalized,
"syntax_converted": result.syntax_converted,
"attachments_moved": result.attachments_moved,
}),
);
}
@@ -1294,15 +1297,13 @@ pub fn import_obsidian(app: AppHandle) -> Result<(), String> {
}
fn do_import_obsidian(app: AppHandle, vault_path: &str) -> Result<ImportResult, String> {
// Pause file watcher during import
let state = app.state::<AppState>();
state
.importing
.store(true, std::sync::atomic::Ordering::Relaxed);
let result = do_import_obsidian_inner(vault_path);
let result = crate::vault::import::import(vault_path);
// Resume file watcher
state
.importing
.store(false, std::sync::atomic::Ordering::Relaxed);
@@ -1310,413 +1311,6 @@ fn do_import_obsidian(app: AppHandle, vault_path: &str) -> Result<ImportResult,
result
}
fn do_import_obsidian_inner(vault_path: &str) -> Result<ImportResult, String> {
let vault = std::path::Path::new(vault_path);
// Build file index: filename -> relative path from vault root
let mut file_index: std::collections::HashMap<String, String> =
std::collections::HashMap::new();
for entry in walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
{
let path = entry.path();
if !path.is_file() {
continue;
}
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if let Ok(rel) = path.strip_prefix(vault) {
file_index.insert(name.to_string(), rel.to_string_lossy().to_string());
}
}
}
let wiki_embed =
regex::Regex::new(r"!\[\[([^\]|]+?)(?:\|([^\]]*))?\]\]").map_err(|e| e.to_string())?;
let wiki_link =
regex::Regex::new(r"\[\[([^\]|]+?)(?:\|([^\]]*))?\]\]").map_err(|e| e.to_string())?;
let md_img = regex::Regex::new(r"!\[([^\]]*)\]\(([^)]+)\)").map_err(|e| e.to_string())?;
let md_link = regex::Regex::new(r"\[([^\]]+)\]\(([^)]+)\)").map_err(|e| e.to_string())?;
let mut files_converted: u64 = 0;
let mut links_converted: u64 = 0;
// Walk all markdown files
let md_files: Vec<_> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
let p = e.path();
p.is_file()
&& p.extension().and_then(|x| x.to_str()) == Some("md")
&& !p.to_string_lossy().contains("/.helixnotes/")
&& !p.to_string_lossy().contains("/.trash/")
})
.collect();
for entry in &md_files {
let path = entry.path();
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => continue,
};
let note_dir = path.parent().unwrap_or(std::path::Path::new(""));
let mut changed = false;
// Convert embeds first: ![[file.ext]] or ![[file.ext|alt]]
let after_embeds = wiki_embed
.replace_all(&content, |caps: &regex::Captures| {
changed = true;
links_converted += 1;
let file_ref = &caps[1];
let alt = caps.get(2).map(|m| m.as_str()).unwrap_or("");
let resolved = resolve_wiki_ref(&file_index, file_ref);
// Convert vault-root-relative path to note-relative
let abs_target = vault.join(&resolved);
let rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(resolved);
let encoded = rel.replace(' ', "%20");
// Use image syntax only for images/PDFs; regular link for other files
let ext = std::path::Path::new(file_ref)
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_lowercase();
let is_embeddable = matches!(
ext.as_str(),
"png" | "jpg" | "jpeg" | "gif" | "svg" | "webp" | "bmp" | "ico" | "pdf"
);
if is_embeddable {
format!("![{}]({})", alt, encoded)
} else {
let display = if alt.is_empty() {
file_ref.rsplit('/').next().unwrap_or(file_ref)
} else {
alt
};
format!("[{}]({})", display, encoded)
}
})
.to_string();
// Convert wiki links: [[note]] or [[note|display]]
let after_links = wiki_link
.replace_all(&after_embeds, |caps: &regex::Captures| {
changed = true;
links_converted += 1;
let note_ref = &caps[1];
let display = caps.get(2).map(|m| m.as_str()).unwrap_or(note_ref);
let resolved = resolve_wiki_ref(&file_index, note_ref);
// Convert vault-root-relative path to note-relative
let abs_target = vault.join(&resolved);
let rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(resolved);
let encoded = rel.replace(' ', "%20");
format!("[{}]({})", display, encoded)
})
.to_string();
// Fix bare-filename standard markdown references: ![alt](filename.ext) where filename has no path separator
let after_img_fix = md_img
.replace_all(&after_links, |caps: &regex::Captures| {
let alt = &caps[1];
let src = &caps[2];
let decoded = percent_decode(src);
// Skip URLs and absolute paths
if decoded.starts_with("http") || decoded.starts_with('/') {
return format!("![{}]({})", alt, src);
}
// Try to resolve bare filename via file index
if !decoded.contains('/') {
if let Some(rel) = file_index.get(&decoded) {
changed = true;
links_converted += 1;
let abs_target = vault.join(rel);
let note_rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(rel.clone());
let encoded = note_rel.replace(' ', "%20");
return format!("![{}]({})", alt, encoded);
}
}
// Fix vault-root-relative paths: if the path resolves from vault root, make it note-relative
if decoded.contains('/') {
let vault_abs = vault.join(&decoded);
if vault_abs.exists() {
let note_rel = pathdiff(
vault_abs.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(decoded.clone());
if note_rel != decoded {
changed = true;
links_converted += 1;
let encoded = note_rel.replace(' ', "%20");
return format!("![{}]({})", alt, encoded);
}
}
}
format!("![{}]({})", alt, src)
})
.to_string();
let after_link_fix = md_link
.replace_all(&after_img_fix, |caps: &regex::Captures| {
let display = &caps[1];
let href = &caps[2];
let decoded = percent_decode(href);
// Skip URLs and absolute paths
if decoded.starts_with("http") || decoded.starts_with('/') {
return format!("[{}]({})", display, href);
}
// Try to resolve bare filename via file index
if !decoded.contains('/') {
if let Some(rel) = file_index.get(&decoded) {
changed = true;
links_converted += 1;
let abs_target = vault.join(rel);
let note_rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(rel.clone());
let encoded = note_rel.replace(' ', "%20");
return format!("[{}]({})", display, encoded);
}
}
// Fix vault-root-relative paths
if decoded.contains('/') {
let vault_abs = vault.join(&decoded);
if vault_abs.exists() {
let note_rel = pathdiff(
vault_abs.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(decoded.clone());
if note_rel != decoded {
changed = true;
links_converted += 1;
let encoded = note_rel.replace(' ', "%20");
return format!("[{}]({})", display, encoded);
}
}
}
format!("[{}]({})", display, href)
})
.to_string();
if changed {
let _ = std::fs::write(path, after_link_fix);
files_converted += 1;
}
}
// ── Phase 2: Move all non-.md files into .helixnotes/attachments/ preserving structure ──
let attachments_dir = vault.join(".helixnotes").join("attachments");
let _ = std::fs::create_dir_all(&attachments_dir);
let attachment_files: Vec<_> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
let p = e.path();
if !p.is_file() {
return false;
}
if p.extension().and_then(|x| x.to_str()) == Some("md") {
return false;
}
let path_str = p.to_string_lossy();
!path_str.contains("/.helixnotes/")
&& !path_str.contains("/.obsidian/")
&& !path_str.contains("/.trash/")
&& !path_str.contains("/.stfolder")
&& !path_str.contains("/.claude/")
})
.collect();
// Map: old vault-relative path -> new vault-relative path (preserving structure)
let mut moved_files: std::collections::HashMap<String, String> =
std::collections::HashMap::new();
for entry in &attachment_files {
let src_path = entry.path();
let old_rel = match src_path.strip_prefix(vault) {
Ok(r) => r.to_string_lossy().to_string(),
Err(_) => continue,
};
// Preserve directory structure: ArkHost/Files/img.png -> .helixnotes/attachments/ArkHost/Files/img.png
let new_rel = format!(".helixnotes/attachments/{}", old_rel);
let dest_path = vault.join(&new_rel);
if let Some(parent) = dest_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
if std::fs::rename(src_path, &dest_path).is_ok()
|| std::fs::copy(src_path, &dest_path)
.and_then(|_| std::fs::remove_file(src_path))
.is_ok()
{
moved_files.insert(old_rel, new_rel);
}
}
// ── Phase 3: Rewrite references in markdown files to point to new locations ──
// Use regex to only replace inside ![...](...) and [...](...) patterns
if !moved_files.is_empty() {
let md_ref = regex::Regex::new(r"(!?\[[^\]]*\])\(([^)]+)\)").map_err(|e| e.to_string())?;
let md_files_pass2: Vec<_> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
let p = e.path();
p.is_file()
&& p.extension().and_then(|x| x.to_str()) == Some("md")
&& !p.to_string_lossy().contains("/.helixnotes/")
&& !p.to_string_lossy().contains("/.trash/")
})
.collect();
for entry in &md_files_pass2 {
let path = entry.path();
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => continue,
};
let note_dir = path.parent().unwrap_or(std::path::Path::new(""));
let mut changed_p3 = false;
let new_content = md_ref
.replace_all(&content, |caps: &regex::Captures| {
let prefix = &caps[1]; // ![alt] or [text]
let href = &caps[2];
let decoded_href = percent_decode(href);
// Skip URLs
if decoded_href.starts_with("http") || decoded_href.starts_with("data:") {
return format!("{}({})", prefix, href);
}
// Resolve the href to a vault-relative path (files already moved, can't use .exists())
// Try note-dir-relative first, then vault-root-relative
let abs_from_note = note_dir.join(&decoded_href);
let rel_from_note = abs_from_note
.strip_prefix(vault)
.ok()
.map(|r| r.to_string_lossy().to_string());
let rel_from_vault = decoded_href.clone();
// Check note-relative resolution first, then vault-root
let old_vault_rel = rel_from_note
.as_ref()
.filter(|r| moved_files.contains_key(r.as_str()))
.cloned()
.or_else(|| {
if moved_files.contains_key(&rel_from_vault) {
Some(rel_from_vault)
} else {
None
}
});
if let Some(old_rel) = old_vault_rel {
if let Some(new_rel) = moved_files.get(&old_rel) {
changed_p3 = true;
let new_abs = vault.join(new_rel);
let new_note_rel = pathdiff(
new_abs.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(new_rel.clone());
let encoded = new_note_rel.replace(' ', "%20");
return format!("{}({})", prefix, encoded);
}
}
format!("{}({})", prefix, href)
})
.to_string();
if changed_p3 {
let _ = std::fs::write(path, new_content);
}
}
}
// Clean up empty directories left behind after moving files
cleanup_empty_dirs(vault);
Ok(ImportResult {
files_converted,
links_converted,
})
}
/// Remove empty directories recursively (bottom-up), skipping .helixnotes and .obsidian
fn cleanup_empty_dirs(root: &std::path::Path) {
let mut dirs: Vec<_> = walkdir::WalkDir::new(root)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.map(|e| e.path().to_path_buf())
.collect();
// Sort by depth (deepest first) so we clean bottom-up
dirs.sort_by(|a, b| b.components().count().cmp(&a.components().count()));
for dir in dirs {
let dir_str = dir.to_string_lossy();
if dir_str.contains("/.helixnotes") || dir_str.contains("/.obsidian") || dir == root {
continue;
}
if let Ok(mut entries) = std::fs::read_dir(&dir) {
if entries.next().is_none() {
let _ = std::fs::remove_dir(&dir);
}
}
}
}
/// Compute a relative path from `base` to `target`
fn pathdiff(target: &str, base: &str) -> Result<String, ()> {
let target = std::path::Path::new(target);
let base = std::path::Path::new(base);
let target_components: Vec<_> = target.components().collect();
let base_components: Vec<_> = base.components().collect();
// Find common prefix length
let common = target_components
.iter()
.zip(base_components.iter())
.take_while(|(a, b)| a == b)
.count();
let mut result = std::path::PathBuf::new();
// Go up from base to common ancestor
for _ in common..base_components.len() {
result.push("..");
}
// Go down from common ancestor to target
for component in &target_components[common..] {
result.push(component);
}
Ok(result.to_string_lossy().to_string())
}
// ── Orphaned attachment cleanup ──
#[derive(serde::Serialize)]
@@ -1853,37 +1447,6 @@ fn percent_decode(s: &str) -> String {
result
}
fn resolve_wiki_ref(
file_index: &std::collections::HashMap<String, String>,
reference: &str,
) -> String {
let reference = reference.trim();
// Strip Obsidian heading anchors (#) and block references (^) before lookup
let reference = reference.split('#').next().unwrap_or(reference).trim();
let reference = reference.split('^').next().unwrap_or(reference).trim();
// Direct match by filename
if let Some(rel) = file_index.get(reference) {
return rel.clone();
}
// Try with .md extension
let with_md = format!("{}.md", reference);
if let Some(rel) = file_index.get(&with_md) {
return rel.clone();
}
// Try matching just the filename part (e.g. "folder/note" -> "note.md")
if let Some(name) = reference.rsplit('/').next() {
if let Some(rel) = file_index.get(name) {
return rel.clone();
}
let name_md = format!("{}.md", name);
if let Some(rel) = file_index.get(&name_md) {
return rel.clone();
}
}
// Fallback: return as-is
reference.to_string()
}
// ── Open file/URL with system default handler ──
fn xdg_open(arg: &str) -> Result<(), String> {
src-tauri/src/types.rs
+3
View File
@@ -393,6 +393,9 @@ pub struct FileEvent {
pub struct ImportResult {
pub files_converted: u64,
pub links_converted: u64,
pub frontmatter_normalized: u64,
pub syntax_converted: u64,
pub attachments_moved: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
src-tauri/src/vault/frontmatter.rs
+2 -2
View File
@@ -15,7 +15,7 @@ struct RawFrontmatter {
}
/// Try to parse a date string in multiple common formats.
fn parse_date_flexible(s: &str) -> Option<chrono::DateTime<Utc>> {
pub fn parse_date_flexible(s: &str) -> Option<chrono::DateTime<Utc>> {
let s = s.trim();
// RFC 3339 / ISO 8601 with timezone (e.g. 2024-01-15T10:30:00+00:00)
if let Ok(dt) = s.parse::<chrono::DateTime<Utc>>() {
@@ -191,7 +191,7 @@ pub fn merge_frontmatter(original_raw: &str, meta: &NoteMeta, body: &str) -> Str
format!("---\n{}---\n{}", yaml_str, body_with_heading)
}
fn filename_to_title(filename: &str) -> String {
pub fn filename_to_title(filename: &str) -> String {
let stem = filename.trim_end_matches(".md");
// Only replace dashes/underscores acting as word separators (between non-space chars).
// Keep dashes that are surrounded by spaces (e.g. "Title - Subtitle").
src-tauri/src/vault/import.rs
+1008
View File
File diff suppressed because it is too large Load Diff
src-tauri/src/vault/mod.rs
+1
View File
@@ -1,3 +1,4 @@
pub mod frontmatter;
pub mod import;
pub mod operations;
pub mod watcher;