ArkHost/HelixNotes
1
0
Fork 0
mirror of https://gitlab.com/ArkHost/HelixNotes.git synced 2026-07-24 07:45:57 +02:00
Code Issues Projects Releases Wiki Activity

Robust Obsidian importer: frontmatter normalization, syntax conversion, deprecated property renaming, same-note heading links

Browse Source
This commit is contained in:
Yuri Karamian
2026-06-30 22:10:22 +02:00
parent 13fd70f944
commit 479c957698
7 changed files with 1051 additions and 452 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/commands.rs
+4 -441
View File
@@ -1276,6 +1276,9 @@ pub fn import_obsidian(app: AppHandle) -> Result<(), String> {
"success": true, "success": true,
"files_converted": result.files_converted, "files_converted": result.files_converted,
"links_converted": result.links_converted, "links_converted": result.links_converted,
"frontmatter_normalized": result.frontmatter_normalized,
"syntax_converted": result.syntax_converted,
"attachments_moved": result.attachments_moved,
}), }),
); );
} }
@@ -1294,15 +1297,13 @@ pub fn import_obsidian(app: AppHandle) -> Result<(), String> {
} }
fn do_import_obsidian(app: AppHandle, vault_path: &str) -> Result<ImportResult, String> { fn do_import_obsidian(app: AppHandle, vault_path: &str) -> Result<ImportResult, String> {
// Pause file watcher during import
let state = app.state::<AppState>(); let state = app.state::<AppState>();
state state
.importing .importing
.store(true, std::sync::atomic::Ordering::Relaxed); .store(true, std::sync::atomic::Ordering::Relaxed);
let result = do_import_obsidian_inner(vault_path); let result = crate::vault::import::import(vault_path);
// Resume file watcher
state state
.importing .importing
.store(false, std::sync::atomic::Ordering::Relaxed); .store(false, std::sync::atomic::Ordering::Relaxed);
@@ -1310,413 +1311,6 @@ fn do_import_obsidian(app: AppHandle, vault_path: &str) -> Result<ImportResult,
result result
} }
fn do_import_obsidian_inner(vault_path: &str) -> Result<ImportResult, String> {
let vault = std::path::Path::new(vault_path);
// Build file index: filename -> relative path from vault root
let mut file_index: std::collections::HashMap<String, String> =
std::collections::HashMap::new();
for entry in walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
{
let path = entry.path();
if !path.is_file() {
continue;
}
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if let Ok(rel) = path.strip_prefix(vault) {
file_index.insert(name.to_string(), rel.to_string_lossy().to_string());
}
}
}
let wiki_embed =
regex::Regex::new(r"!\[\[([^\]|]+?)(?:\|([^\]]*))?\]\]").map_err(|e| e.to_string())?;
let wiki_link =
regex::Regex::new(r"\[\[([^\]|]+?)(?:\|([^\]]*))?\]\]").map_err(|e| e.to_string())?;
let md_img = regex::Regex::new(r"!\[([^\]]*)\]\(([^)]+)\)").map_err(|e| e.to_string())?;
let md_link = regex::Regex::new(r"\[([^\]]+)\]\(([^)]+)\)").map_err(|e| e.to_string())?;
let mut files_converted: u64 = 0;
let mut links_converted: u64 = 0;
// Walk all markdown files
let md_files: Vec<_> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
let p = e.path();
p.is_file()
&& p.extension().and_then(|x| x.to_str()) == Some("md")
&& !p.to_string_lossy().contains("/.helixnotes/")
&& !p.to_string_lossy().contains("/.trash/")
})
.collect();
for entry in &md_files {
let path = entry.path();
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => continue,
};
let note_dir = path.parent().unwrap_or(std::path::Path::new(""));
let mut changed = false;
// Convert embeds first: ![[file.ext]] or ![[file.ext|alt]]
let after_embeds = wiki_embed
.replace_all(&content, |caps: &regex::Captures| {
changed = true;
links_converted += 1;
let file_ref = &caps[1];
let alt = caps.get(2).map(|m| m.as_str()).unwrap_or("");
let resolved = resolve_wiki_ref(&file_index, file_ref);
// Convert vault-root-relative path to note-relative
let abs_target = vault.join(&resolved);
let rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(resolved);
let encoded = rel.replace(' ', "%20");
// Use image syntax only for images/PDFs; regular link for other files
let ext = std::path::Path::new(file_ref)
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_lowercase();
let is_embeddable = matches!(
ext.as_str(),
"png" | "jpg" | "jpeg" | "gif" | "svg" | "webp" | "bmp" | "ico" | "pdf"
);
if is_embeddable {
format!("![{}]({})", alt, encoded)
} else {
let display = if alt.is_empty() {
file_ref.rsplit('/').next().unwrap_or(file_ref)
} else {
alt
};
format!("[{}]({})", display, encoded)
}
})
.to_string();
// Convert wiki links: [[note]] or [[note|display]]
let after_links = wiki_link
.replace_all(&after_embeds, |caps: &regex::Captures| {
changed = true;
links_converted += 1;
let note_ref = &caps[1];
let display = caps.get(2).map(|m| m.as_str()).unwrap_or(note_ref);
let resolved = resolve_wiki_ref(&file_index, note_ref);
// Convert vault-root-relative path to note-relative
let abs_target = vault.join(&resolved);
let rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(resolved);
let encoded = rel.replace(' ', "%20");
format!("[{}]({})", display, encoded)
})
.to_string();
// Fix bare-filename standard markdown references: ![alt](filename.ext) where filename has no path separator
let after_img_fix = md_img
.replace_all(&after_links, |caps: &regex::Captures| {
let alt = &caps[1];
let src = &caps[2];
let decoded = percent_decode(src);
// Skip URLs and absolute paths
if decoded.starts_with("http") || decoded.starts_with('/') {
return format!("![{}]({})", alt, src);
}
// Try to resolve bare filename via file index
if !decoded.contains('/') {
if let Some(rel) = file_index.get(&decoded) {
changed = true;
links_converted += 1;
let abs_target = vault.join(rel);
let note_rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(rel.clone());
let encoded = note_rel.replace(' ', "%20");
return format!("![{}]({})", alt, encoded);
}
}
// Fix vault-root-relative paths: if the path resolves from vault root, make it note-relative
if decoded.contains('/') {
let vault_abs = vault.join(&decoded);
if vault_abs.exists() {
let note_rel = pathdiff(
vault_abs.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(decoded.clone());
if note_rel != decoded {
changed = true;
links_converted += 1;
let encoded = note_rel.replace(' ', "%20");
return format!("![{}]({})", alt, encoded);
}
}
}
format!("![{}]({})", alt, src)
})
.to_string();
let after_link_fix = md_link
.replace_all(&after_img_fix, |caps: &regex::Captures| {
let display = &caps[1];
let href = &caps[2];
let decoded = percent_decode(href);
// Skip URLs and absolute paths
if decoded.starts_with("http") || decoded.starts_with('/') {
return format!("[{}]({})", display, href);
}
// Try to resolve bare filename via file index
if !decoded.contains('/') {
if let Some(rel) = file_index.get(&decoded) {
changed = true;
links_converted += 1;
let abs_target = vault.join(rel);
let note_rel = pathdiff(
abs_target.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(rel.clone());
let encoded = note_rel.replace(' ', "%20");
return format!("[{}]({})", display, encoded);
}
}
// Fix vault-root-relative paths
if decoded.contains('/') {
let vault_abs = vault.join(&decoded);
if vault_abs.exists() {
let note_rel = pathdiff(
vault_abs.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(decoded.clone());
if note_rel != decoded {
changed = true;
links_converted += 1;
let encoded = note_rel.replace(' ', "%20");
return format!("[{}]({})", display, encoded);
}
}
}
format!("[{}]({})", display, href)
})
.to_string();
if changed {
let _ = std::fs::write(path, after_link_fix);
files_converted += 1;
}
}
// ── Phase 2: Move all non-.md files into .helixnotes/attachments/ preserving structure ──
let attachments_dir = vault.join(".helixnotes").join("attachments");
let _ = std::fs::create_dir_all(&attachments_dir);
let attachment_files: Vec<_> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
let p = e.path();
if !p.is_file() {
return false;
}
if p.extension().and_then(|x| x.to_str()) == Some("md") {
return false;
}
let path_str = p.to_string_lossy();
!path_str.contains("/.helixnotes/")
&& !path_str.contains("/.obsidian/")
&& !path_str.contains("/.trash/")
&& !path_str.contains("/.stfolder")
&& !path_str.contains("/.claude/")
})
.collect();
// Map: old vault-relative path -> new vault-relative path (preserving structure)
let mut moved_files: std::collections::HashMap<String, String> =
std::collections::HashMap::new();
for entry in &attachment_files {
let src_path = entry.path();
let old_rel = match src_path.strip_prefix(vault) {
Ok(r) => r.to_string_lossy().to_string(),
Err(_) => continue,
};
// Preserve directory structure: ArkHost/Files/img.png -> .helixnotes/attachments/ArkHost/Files/img.png
let new_rel = format!(".helixnotes/attachments/{}", old_rel);
let dest_path = vault.join(&new_rel);
if let Some(parent) = dest_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
if std::fs::rename(src_path, &dest_path).is_ok()
|| std::fs::copy(src_path, &dest_path)
.and_then(|_| std::fs::remove_file(src_path))
.is_ok()
{
moved_files.insert(old_rel, new_rel);
}
}
// ── Phase 3: Rewrite references in markdown files to point to new locations ──
// Use regex to only replace inside ![...](...) and [...](...) patterns
if !moved_files.is_empty() {
let md_ref = regex::Regex::new(r"(!?\[[^\]]*\])\(([^)]+)\)").map_err(|e| e.to_string())?;
let md_files_pass2: Vec<_> = walkdir::WalkDir::new(vault)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
let p = e.path();
p.is_file()
&& p.extension().and_then(|x| x.to_str()) == Some("md")
&& !p.to_string_lossy().contains("/.helixnotes/")
&& !p.to_string_lossy().contains("/.trash/")
})
.collect();
for entry in &md_files_pass2 {
let path = entry.path();
let content = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => continue,
};
let note_dir = path.parent().unwrap_or(std::path::Path::new(""));
let mut changed_p3 = false;
let new_content = md_ref
.replace_all(&content, |caps: &regex::Captures| {
let prefix = &caps[1]; // ![alt] or [text]
let href = &caps[2];
let decoded_href = percent_decode(href);
// Skip URLs
if decoded_href.starts_with("http") || decoded_href.starts_with("data:") {
return format!("{}({})", prefix, href);
}
// Resolve the href to a vault-relative path (files already moved, can't use .exists())
// Try note-dir-relative first, then vault-root-relative
let abs_from_note = note_dir.join(&decoded_href);
let rel_from_note = abs_from_note
.strip_prefix(vault)
.ok()
.map(|r| r.to_string_lossy().to_string());
let rel_from_vault = decoded_href.clone();
// Check note-relative resolution first, then vault-root
let old_vault_rel = rel_from_note
.as_ref()
.filter(|r| moved_files.contains_key(r.as_str()))
.cloned()
.or_else(|| {
if moved_files.contains_key(&rel_from_vault) {
Some(rel_from_vault)
} else {
None
}
});
if let Some(old_rel) = old_vault_rel {
if let Some(new_rel) = moved_files.get(&old_rel) {
changed_p3 = true;
let new_abs = vault.join(new_rel);
let new_note_rel = pathdiff(
new_abs.to_str().unwrap_or(""),
note_dir.to_str().unwrap_or(""),
)
.unwrap_or(new_rel.clone());
let encoded = new_note_rel.replace(' ', "%20");
return format!("{}({})", prefix, encoded);
}
}
format!("{}({})", prefix, href)
})
.to_string();
if changed_p3 {
let _ = std::fs::write(path, new_content);
}
}
}
// Clean up empty directories left behind after moving files
cleanup_empty_dirs(vault);
Ok(ImportResult {
files_converted,
links_converted,
})
}
/// Remove empty directories recursively (bottom-up), skipping .helixnotes and .obsidian
fn cleanup_empty_dirs(root: &std::path::Path) {
let mut dirs: Vec<_> = walkdir::WalkDir::new(root)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.map(|e| e.path().to_path_buf())
.collect();
// Sort by depth (deepest first) so we clean bottom-up
dirs.sort_by(|a, b| b.components().count().cmp(&a.components().count()));
for dir in dirs {
let dir_str = dir.to_string_lossy();
if dir_str.contains("/.helixnotes") || dir_str.contains("/.obsidian") || dir == root {
continue;
}
if let Ok(mut entries) = std::fs::read_dir(&dir) {
if entries.next().is_none() {
let _ = std::fs::remove_dir(&dir);
}
}
}
}
/// Compute a relative path from `base` to `target`
fn pathdiff(target: &str, base: &str) -> Result<String, ()> {
let target = std::path::Path::new(target);
let base = std::path::Path::new(base);
let target_components: Vec<_> = target.components().collect();
let base_components: Vec<_> = base.components().collect();
// Find common prefix length
let common = target_components
.iter()
.zip(base_components.iter())
.take_while(|(a, b)| a == b)
.count();
let mut result = std::path::PathBuf::new();
// Go up from base to common ancestor
for _ in common..base_components.len() {
result.push("..");
}
// Go down from common ancestor to target
for component in &target_components[common..] {
result.push(component);
}
Ok(result.to_string_lossy().to_string())
}
// ── Orphaned attachment cleanup ── // ── Orphaned attachment cleanup ──
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
@@ -1853,37 +1447,6 @@ fn percent_decode(s: &str) -> String {
result result
} }
fn resolve_wiki_ref(
file_index: &std::collections::HashMap<String, String>,
reference: &str,
) -> String {
let reference = reference.trim();
// Strip Obsidian heading anchors (#) and block references (^) before lookup
let reference = reference.split('#').next().unwrap_or(reference).trim();
let reference = reference.split('^').next().unwrap_or(reference).trim();
// Direct match by filename
if let Some(rel) = file_index.get(reference) {
return rel.clone();
}
// Try with .md extension
let with_md = format!("{}.md", reference);
if let Some(rel) = file_index.get(&with_md) {
return rel.clone();
}
// Try matching just the filename part (e.g. "folder/note" -> "note.md")
if let Some(name) = reference.rsplit('/').next() {
if let Some(rel) = file_index.get(name) {
return rel.clone();
}
let name_md = format!("{}.md", name);
if let Some(rel) = file_index.get(&name_md) {
return rel.clone();
}
}
// Fallback: return as-is
reference.to_string()
}
// ── Open file/URL with system default handler ── // ── Open file/URL with system default handler ──
fn xdg_open(arg: &str) -> Result<(), String> { fn xdg_open(arg: &str) -> Result<(), String> {
src-tauri/src/types.rs
+3
View File
@@ -393,6 +393,9 @@ pub struct FileEvent {
pub struct ImportResult { pub struct ImportResult {
pub files_converted: u64, pub files_converted: u64,
pub links_converted: u64, pub links_converted: u64,
pub frontmatter_normalized: u64,
pub syntax_converted: u64,
pub attachments_moved: u64,
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
src-tauri/src/vault/frontmatter.rs
+2 -2
View File
@@ -15,7 +15,7 @@ struct RawFrontmatter {
} }
/// Try to parse a date string in multiple common formats. /// Try to parse a date string in multiple common formats.
fn parse_date_flexible(s: &str) -> Option<chrono::DateTime<Utc>> { pub fn parse_date_flexible(s: &str) -> Option<chrono::DateTime<Utc>> {
let s = s.trim(); let s = s.trim();
// RFC 3339 / ISO 8601 with timezone (e.g. 2024-01-15T10:30:00+00:00) // RFC 3339 / ISO 8601 with timezone (e.g. 2024-01-15T10:30:00+00:00)
if let Ok(dt) = s.parse::<chrono::DateTime<Utc>>() { if let Ok(dt) = s.parse::<chrono::DateTime<Utc>>() {
@@ -191,7 +191,7 @@ pub fn merge_frontmatter(original_raw: &str, meta: &NoteMeta, body: &str) -> Str
format!("---\n{}---\n{}", yaml_str, body_with_heading) format!("---\n{}---\n{}", yaml_str, body_with_heading)
} }
fn filename_to_title(filename: &str) -> String { pub fn filename_to_title(filename: &str) -> String {
let stem = filename.trim_end_matches(".md"); let stem = filename.trim_end_matches(".md");
// Only replace dashes/underscores acting as word separators (between non-space chars). // Only replace dashes/underscores acting as word separators (between non-space chars).
// Keep dashes that are surrounded by spaces (e.g. "Title - Subtitle"). // Keep dashes that are surrounded by spaces (e.g. "Title - Subtitle").
src-tauri/src/vault/import.rs
+1008
View File
File diff suppressed because it is too large Load Diff
src-tauri/src/vault/mod.rs
+1
View File
@@ -1,3 +1,4 @@
pub mod frontmatter; pub mod frontmatter;
pub mod import;
pub mod operations; pub mod operations;
pub mod watcher; pub mod watcher;
src/lib/components/SettingsPanel.svelte
+26 -5
View File
@@ -739,10 +739,16 @@
importResult = null; importResult = null;
importError = null; importError = null;
const unlistenDone = await listen<{ success: boolean; files_converted?: number; links_converted?: number; error?: string }>('import-done', (event) => { const unlistenDone = await listen<{ success: boolean; files_converted?: number; links_converted?: number; frontmatter_normalized?: number; syntax_converted?: number; attachments_moved?: number; error?: string }>('import-done', (event) => {
const data = event.payload; const data = event.payload;
if (data.success) { if (data.success) {
importResult = { files_converted: data.files_converted ?? 0, links_converted: data.links_converted ?? 0 }; importResult = {
files_converted: data.files_converted ?? 0,
links_converted: data.links_converted ?? 0,
frontmatter_normalized: data.frontmatter_normalized ?? 0,
syntax_converted: data.syntax_converted ?? 0,
attachments_moved: data.attachments_moved ?? 0,
};
} else { } else {
importError = data.error ?? 'Import failed'; importError = data.error ?? 'Import failed';
} }
@@ -1807,7 +1813,7 @@
<div class="tab-content"> <div class="tab-content">
<div class="settings-section"> <div class="settings-section">
<h3>Import from Obsidian</h3> <h3>Import from Obsidian</h3>
<p class="import-desc">Convert Obsidian wiki-links (<code>![[image.png]]</code>, <code>[[note]]</code>) to standard markdown links across all notes in the current vault. This expects that you have opened an Obsidian vault directory directly as your HelixNotes vault.</p> <p class="import-desc">Fully converts an Obsidian vault to HelixNotes format: normalizes frontmatter (tags, IDs, dates, title headings), converts <code>[[wiki-links]]</code> to standard markdown links, converts <code>==highlights==</code> and <code>%%comments%%</code>, and relocates attachments to <code>.helixnotes/attachments/</code>. Open the Obsidian vault directory as your HelixNotes vault first.</p>
<div class="import-warn"> <div class="import-warn">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M10.29 3.86L1.82 18a2 2 0 001.71 3h16.94a2 2 0 001.71-3L13.71 3.86a2 2 0 00-3.42 0z" /><line x1="12" y1="9" x2="12" y2="13" /><line x1="12" y1="17" x2="12.01" y2="17" /></svg> <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M10.29 3.86L1.82 18a2 2 0 001.71 3h16.94a2 2 0 001.71-3L13.71 3.86a2 2 0 00-3.42 0z" /><line x1="12" y1="9" x2="12" y2="13" /><line x1="12" y1="17" x2="12.01" y2="17" /></svg>
<span>This modifies files in place. Make a backup before running!</span> <span>This modifies files in place. Make a backup before running!</span>
@@ -1829,7 +1835,16 @@
{#if importResult} {#if importResult}
<div class="import-result success"> <div class="import-result success">
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M22 11.08V12a10 10 0 11-5.93-9.14" /><polyline points="22 4 12 14.01 9 11.01" /></svg> <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M22 11.08V12a10 10 0 11-5.93-9.14" /><polyline points="22 4 12 14.01 9 11.01" /></svg>
<span>Converted {importResult.links_converted} links across {importResult.files_converted} files</span> <div class="import-stats">
<span>Import complete</span>
<ul>
{#if importResult.frontmatter_normalized}<li>Normalized frontmatter in {importResult.frontmatter_normalized} notes</li>{/if}
{#if importResult.links_converted}<li>Converted {importResult.links_converted} wiki-links to markdown</li>{/if}
{#if importResult.syntax_converted}<li>Converted Obsidian syntax (highlights, comments) in {importResult.syntax_converted} notes</li>{/if}
{#if importResult.attachments_moved}<li>Moved {importResult.attachments_moved} attachments to .helixnotes/attachments/</li>{/if}
{#if !importResult.frontmatter_normalized && !importResult.links_converted && !importResult.syntax_converted && !importResult.attachments_moved}<li>No changes needed</li>{/if}
</ul>
</div>
</div> </div>
{/if} {/if}
{#if importError} {#if importError}
@@ -2803,7 +2818,7 @@
.import-result { .import-result {
display: flex; display: flex;
align-items: center; align-items: flex-start;
gap: 8px; gap: 8px;
margin-top: 14px; margin-top: 14px;
padding: 10px 14px; padding: 10px 14px;
@@ -2816,6 +2831,12 @@
color: #059669; color: #059669;
} }
.import-stats ul {
margin: 4px 0 0 0;
padding-left: 16px;
line-height: 1.6;
}
.import-result.error { .import-result.error {
background: color-mix(in srgb, #e11d48 10%, transparent); background: color-mix(in srgb, #e11d48 10%, transparent);
color: #e11d48; color: #e11d48;
src/lib/types.ts
+3
View File
@@ -171,6 +171,9 @@ export interface FileEvent {
export interface ImportResult { export interface ImportResult {
files_converted: number; files_converted: number;
links_converted: number; links_converted: number;
frontmatter_normalized: number;
syntax_converted: number;
attachments_moved: number;
} }
export interface VaultStats { export interface VaultStats {