From 9fb9de4fd253896c308639b9f2ae8ff8b0a65fd0 Mon Sep 17 00:00:00 2001 From: Kim Junmo Date: Thu, 9 Apr 2026 07:59:22 +0900 Subject: [PATCH 1/2] fix: preserve original case in handelize() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blanket .toLowerCase() in handelize() drops filename casing, which breaks path resolution on case-sensitive filesystems (Linux). Files like README.md, CHANGELOG.md, and SKILL.md become unreachable when the index stores them as readme.md, changelog.md, skill.md. Since FTS5 already performs case-insensitive matching via the unicode61 tokenizer, lowercasing the stored path provides no search benefit — it only corrupts the metadata used to locate files on disk. Remove .toLowerCase() and update all affected test expectations. --- src/store.ts | 3 +-- test/cli.test.ts | 8 ++++---- test/store.helpers.unit.test.ts | 20 ++++++++++---------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/store.ts b/src/store.ts index ab4cbf4..2c26522 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1693,11 +1693,11 @@ export function getDocid(hash: string): string { /** * Handelize a filename to be more token-friendly. * - Convert triple underscore `___` to `/` (folder separator) - * - Convert to lowercase * - Replace sequences of non-word chars (except /) with single dash * - Remove leading/trailing dashes from path segments * - Preserve folder structure (a/b/c/d.md stays structured) * - Preserve file extension + * - Preserve original case (important for case-sensitive filesystems) */ /** Replace emoji/symbol codepoints with their hex representation (e.g. 🐘 → 1f418) */ function emojiToHex(str: string): string { @@ -1725,7 +1725,6 @@ export function handelize(path: string): string { const result = path .replace(/___/g, '/') // Triple underscore becomes folder separator - .toLowerCase() .split('/') .map((segment, idx, arr) => { const isLastSegment = idx === arr.length - 1; diff --git a/test/cli.test.ts b/test/cli.test.ts index c9c5644..f1acf8a 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -837,8 +837,8 @@ describe("CLI ls Command", () => { test("lists files in a collection", async () => { const { stdout, exitCode } = await runQmd(["ls", "fixtures"], { dbPath: localDbPath }); expect(exitCode).toBe(0); - // handelize converts to lowercase - expect(stdout).toContain("qmd://fixtures/readme.md"); + // handelize preserves original case + expect(stdout).toContain("qmd://fixtures/README.md"); expect(stdout).toContain("qmd://fixtures/notes/meeting.md"); }); @@ -847,8 +847,8 @@ describe("CLI ls Command", () => { expect(exitCode).toBe(0); expect(stdout).toContain("qmd://fixtures/notes/meeting.md"); expect(stdout).toContain("qmd://fixtures/notes/ideas.md"); - // Should not include files outside the prefix (handelize converts to lowercase) - expect(stdout).not.toContain("qmd://fixtures/readme.md"); + // Should not include files outside the prefix (case preserved) + expect(stdout).not.toContain("qmd://fixtures/README.md"); }); test("lists files with virtual path", async () => { diff --git a/test/store.helpers.unit.test.ts b/test/store.helpers.unit.test.ts index 99af680..9adefc9 100644 --- a/test/store.helpers.unit.test.ts +++ b/test/store.helpers.unit.test.ts @@ -119,14 +119,14 @@ describe("cleanupOrphanedVectors", () => { // ============================================================================= describe("handelize", () => { - test("converts to lowercase", () => { - expect(handelize("README.md")).toBe("readme.md"); - expect(handelize("MyFile.MD")).toBe("myfile.md"); + test("preserves original case", () => { + expect(handelize("README.md")).toBe("README.md"); + expect(handelize("MyFile.MD")).toBe("MyFile.MD"); }); test("preserves folder structure", () => { expect(handelize("a/b/c/d.md")).toBe("a/b/c/d.md"); - expect(handelize("docs/api/README.md")).toBe("docs/api/readme.md"); + expect(handelize("docs/api/README.md")).toBe("docs/api/README.md"); }); test("replaces non-word characters with dash", () => { @@ -156,7 +156,7 @@ describe("handelize", () => { test("handles complex real-world meeting notes", () => { const complexName = "Money Movement Licensing Review - 2025／11／19 10:25 EST - Notes by Gemini.md"; const result = handelize(complexName); - expect(result).toBe("money-movement-licensing-review-2025-11-19-10-25-est-notes-by-gemini.md"); + expect(result).toBe("Money-Movement-Licensing-Review-2025-11-19-10-25-EST-Notes-by-Gemini.md"); expect(result).not.toContain(" "); expect(result).not.toContain("／"); expect(result).not.toContain(":"); @@ -164,7 +164,7 @@ describe("handelize", () => { test("handles unicode characters", () => { expect(handelize("日本語.md")).toBe("日本語.md"); - expect(handelize("Зоны и проекты.md")).toBe("зоны-и-проекты.md"); + expect(handelize("Зоны и проекты.md")).toBe("Зоны-и-проекты.md"); expect(handelize("café-notes.md")).toBe("café-notes.md"); expect(handelize("naïve.md")).toBe("naïve.md"); expect(handelize("日本語-notes.md")).toBe("日本語-notes.md"); @@ -186,13 +186,13 @@ describe("handelize", () => { test("handles dates and times in filenames", () => { expect(handelize("meeting-2025-01-15.md")).toBe("meeting-2025-01-15.md"); expect(handelize("notes 2025/01/15.md")).toBe("notes-2025/01/15.md"); - expect(handelize("call_10:30_AM.md")).toBe("call-10-30-am.md"); + expect(handelize("call_10:30_AM.md")).toBe("call-10-30-AM.md"); }); test("handles special project naming patterns", () => { - expect(handelize("PROJECT_ABC_v2.0.md")).toBe("project-abc-v2-0.md"); - expect(handelize("[WIP] Feature Request.md")).toBe("wip-feature-request.md"); - expect(handelize("(DRAFT) Proposal v1.md")).toBe("draft-proposal-v1.md"); + expect(handelize("PROJECT_ABC_v2.0.md")).toBe("PROJECT-ABC-v2-0.md"); + expect(handelize("[WIP] Feature Request.md")).toBe("WIP-Feature-Request.md"); + expect(handelize("(DRAFT) Proposal v1.md")).toBe("DRAFT-Proposal-v1.md"); }); test("handles symbol-only route filenames", () => { From fee576bf98c181bc010c3478eea6a2cfaa209384 Mon Sep 17 00:00:00 2001 From: Kim Junmo Date: Thu, 9 Apr 2026 08:09:58 +0900 Subject: [PATCH 2/2] fix: migrate legacy lowercase paths on reindex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When qmd update runs against an index created before case-preservation, documents may exist under lowercase paths (e.g. "skill.md" for a file actually named "SKILL.md"). Add findOrMigrateLegacyDocument() that: - Falls back to a lowercase lookup when the canonical path is not found - Renames the document path in-place via UPDATE OR IGNORE - Manually rebuilds the FTS entry (FTS5 INSERT OR REPLACE does not reliably update existing rows via triggers) - Handles UNIQUE conflicts gracefully (returns null on conflict) Embeddings are keyed by content hash, so the rename preserves all existing vectors — no re-embedding required. Both the CLI indexer and the library reindexer share the same helper, eliminating the duplication that a previous review flagged. Includes integration tests for: successful migration, already-lowercase no-op, and UNIQUE conflict handling. --- CHANGELOG.md | 5 ++++ src/cli/qmd.ts | 5 ++-- src/store.ts | 55 +++++++++++++++++++++++++++++++++++- test/store.test.ts | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8feb5d..12b3b37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +- Fix: preserve original filename case in `handelize()`. The previous + `.toLowerCase()` call made indexed paths unreachable on case-sensitive + filesystems (Linux). `qmd update` automatically migrates legacy + lowercase paths without re-embedding. + ## [2.1.0] - 2026-04-05 Code files now chunk at function and class boundaries via tree-sitter, diff --git a/src/cli/qmd.ts b/src/cli/qmd.ts index a09ffb3..b533b40 100755 --- a/src/cli/qmd.ts +++ b/src/cli/qmd.ts @@ -45,6 +45,7 @@ import { insertContent, insertDocument, findActiveDocument, + findOrMigrateLegacyDocument, updateDocumentTitle, updateDocument, deactivateDocument, @@ -1567,8 +1568,8 @@ async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GLOB, coll const hash = await hashContent(content); const title = extractTitle(content, relativeFile); - // Check if document exists in this collection with this path - const existing = findActiveDocument(db, collectionName, path); + // Check if document exists (also migrates legacy lowercase paths) + const existing = findOrMigrateLegacyDocument(db, collectionName, path); if (existing) { if (existing.hash === hash) { diff --git a/src/store.ts b/src/store.ts index 2c26522..17882ca 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1135,6 +1135,7 @@ export type Store = { insertContent: (hash: string, content: string, createdAt: string) => void; insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => void; findActiveDocument: (collectionName: string, path: string) => { id: number; hash: string; title: string } | null; + findOrMigrateLegacyDocument: (collectionName: string, path: string) => { id: number; hash: string; title: string } | null; updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => void; updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => void; deactivateDocument: (collectionName: string, path: string) => void; @@ -1225,7 +1226,7 @@ export async function reindexCollection( const hash = await hashContent(content); const title = extractTitle(content, relativeFile); - const existing = findActiveDocument(db, collectionName, path); + const existing = findOrMigrateLegacyDocument(db, collectionName, path); if (existing) { if (existing.hash === hash) { @@ -1648,6 +1649,7 @@ export function createStore(dbPath?: string): Store { insertContent: (hash: string, content: string, createdAt: string) => insertContent(db, hash, content, createdAt), insertDocument: (collectionName: string, path: string, title: string, hash: string, createdAt: string, modifiedAt: string) => insertDocument(db, collectionName, path, title, hash, createdAt, modifiedAt), findActiveDocument: (collectionName: string, path: string) => findActiveDocument(db, collectionName, path), + findOrMigrateLegacyDocument: (collectionName: string, path: string) => findOrMigrateLegacyDocument(db, collectionName, path), updateDocumentTitle: (documentId: number, title: string, modifiedAt: string) => updateDocumentTitle(db, documentId, title, modifiedAt), updateDocument: (documentId: number, title: string, hash: string, modifiedAt: string) => updateDocument(db, documentId, title, hash, modifiedAt), deactivateDocument: (collectionName: string, path: string) => deactivateDocument(db, collectionName, path), @@ -2102,6 +2104,57 @@ export function findActiveDocument( return row ?? null; } +/** + * Find an active document, falling back to a legacy lowercase path. + * If found under the legacy path, renames it in-place and rebuilds the + * FTS entry. Embeddings are keyed by content hash, so the rename is + * safe — no re-embedding required. + * + * @internal Used by reindexCollection and indexFiles during qmd update. + * Returns null if the document does not exist under either path. + */ +export function findOrMigrateLegacyDocument( + db: Database, + collectionName: string, + path: string +): { id: number; hash: string; title: string } | null { + const existing = findActiveDocument(db, collectionName, path); + if (existing) return existing; + + const legacyPath = path.toLowerCase(); + if (legacyPath === path) return null; + + const legacy = findActiveDocument(db, collectionName, legacyPath); + if (!legacy) return null; + + // Wrap rename + FTS rebuild in a transaction for atomicity. + const migrate = db.transaction(() => { + // Use OR IGNORE so a UNIQUE conflict (e.g. both "readme.md" and + // "README.md" already exist) is a no-op rather than crashing. + const result = db.prepare( + `UPDATE OR IGNORE documents SET path = ? WHERE id = ? AND active = 1` + ).run(path, legacy.id); + + if (result.changes === 0) return false; + + // FTS5 does not reliably update via the documents_au trigger's + // INSERT OR REPLACE. Manually rebuild the FTS entry. + db.prepare(`DELETE FROM documents_fts WHERE rowid = ?`).run(legacy.id); + db.prepare(` + INSERT INTO documents_fts(rowid, filepath, title, body) + SELECT id, collection || '/' || path, title, + (SELECT doc FROM content WHERE hash = documents.hash) + FROM documents WHERE id = ? + `).run(legacy.id); + + return true; + }); + + if (!migrate()) return null; + + return findActiveDocument(db, collectionName, path); +} + /** * Update the title and modified_at timestamp for a document. */ diff --git a/test/store.test.ts b/test/store.test.ts index 47b481b..0104b21 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -3014,6 +3014,75 @@ describe("Content-Addressable Storage", () => { await cleanupTestDb(store); }); + + test("findOrMigrateLegacyDocument renames lowercase path to case-preserved", async () => { + const store = await createTestStore(); + const collectionName = await createTestCollection(); + const now = new Date().toISOString(); + + const content = "# My Skill"; + const hash = await hashContent(content); + store.insertContent(hash, content, now); + // Simulate legacy index: path stored as lowercase + store.insertDocument(collectionName, "skills/skill.md", "My Skill", hash, now, now); + + // Migration: look up case-preserved path, expect rename + const result = store.findOrMigrateLegacyDocument(collectionName, "skills/SKILL.md"); + expect(result).not.toBeNull(); + expect(result!.hash).toBe(hash); + + // Old lowercase path should no longer be findable + expect(store.findActiveDocument(collectionName, "skills/skill.md")).toBeNull(); + // New case-preserved path should be active + const migrated = store.findActiveDocument(collectionName, "skills/SKILL.md"); + expect(migrated).not.toBeNull(); + expect(migrated!.hash).toBe(hash); + + // FTS should reflect the new path (documents_au trigger) + const ftsRow = store.db.prepare( + `SELECT filepath FROM documents_fts WHERE rowid = ?` + ).get(result!.id) as { filepath: string } | undefined; + expect(ftsRow).toBeDefined(); + expect(ftsRow!.filepath).toContain("SKILL.md"); + + await cleanupTestDb(store); + }); + + test("findOrMigrateLegacyDocument returns null when path is already lowercase", async () => { + const store = await createTestStore(); + const collectionName = await createTestCollection(); + + // No document exists at all + const result = store.findOrMigrateLegacyDocument(collectionName, "readme.md"); + expect(result).toBeNull(); + + await cleanupTestDb(store); + }); + + test("findOrMigrateLegacyDocument returns existing doc when canonical path already present", async () => { + const store = await createTestStore(); + const collectionName = await createTestCollection(); + const now = new Date().toISOString(); + + const content = "# Content"; + const hash = await hashContent(content); + store.insertContent(hash, content, now); + // Both lowercase and case-preserved paths exist (edge case from prior partial migration) + store.insertDocument(collectionName, "readme.md", "Readme", hash, now, now); + store.insertDocument(collectionName, "README.md", "README", hash, now, now); + + // Should return the canonical-path document directly (fast path) + // The legacy "readme.md" row is untouched — no rename attempted. + const result = store.findOrMigrateLegacyDocument(collectionName, "README.md"); + expect(result).not.toBeNull(); + expect(result!.hash).toBe(hash); + + // Both rows still exist (legacy row not migrated, not deactivated here) + expect(store.findActiveDocument(collectionName, "readme.md")).not.toBeNull(); + expect(store.findActiveDocument(collectionName, "README.md")).not.toBeNull(); + + await cleanupTestDb(store); + }); }); // =============================================================================