From dcedfb52681537a72a10264c595b30ab0a043bef Mon Sep 17 00:00:00 2001 From: Tobi Lutke Date: Sun, 15 Feb 2026 17:15:47 -0400 Subject: [PATCH] feat: cross-runtime SQLite compat layer (bun:sqlite + better-sqlite3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/db.ts that dynamically imports bun:sqlite under Bun and better-sqlite3 under Node.js. Exports openDatabase(), loadSqliteVec(), and a shared Database interface. - sqlite-vec loading is now optional — FTS works without it, vector ops throw a clear error if unavailable - CI tests both runtimes: Node 22/23 via vitest, Bun via bun test - All 104 unit tests pass on both Node and Bun --- .github/workflows/ci.yml | 48 ++++++++++++++++++++++++++++++++++++- src/db.ts | 52 ++++++++++++++++++++++++++++++++++++++++ src/eval-bm25.test.ts | 2 +- src/models/eval.test.ts | 3 ++- src/models/mcp.test.ts | 12 +++++----- src/models/store.test.ts | 13 +++++----- src/qmd.ts | 3 ++- src/store.ts | 33 ++++++++++++------------- 8 files changed, 134 insertions(+), 32 deletions(-) create mode 100644 src/db.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85f3cd0..11b8ab9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,8 @@ on: branches: [main] jobs: - test: + test-node: + name: Node ${{ matrix.node-version }} (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -44,3 +45,48 @@ jobs: run: npx vitest run --reporter=verbose src/integration/*.test.ts env: CI: true + + test-bun: + name: Bun (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Install SQLite (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Install SQLite (macOS) + if: runner.os == 'macOS' + run: brew install sqlite + + - run: bun install + + - name: Unit tests + run: bun test --preload ./src/test-preload.ts src/*.test.ts + env: + DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib + LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu + + - name: Model tests + run: bun test --preload ./src/test-preload.ts src/models/*.test.ts + env: + CI: true + DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib + LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu + + - name: Integration tests + run: bun test --preload ./src/test-preload.ts src/integration/*.test.ts + env: + CI: true + DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib + LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 0000000..c5d0c2a --- /dev/null +++ b/src/db.ts @@ -0,0 +1,52 @@ +/** + * db.ts - Cross-runtime SQLite compatibility layer + * + * Provides a unified Database export that works under both Bun (bun:sqlite) + * and Node.js (better-sqlite3). The APIs are nearly identical — the main + * difference is the import path. + */ + +export const isBun = typeof globalThis.Bun !== "undefined"; + +let _Database: any; +let _sqliteVecLoad: (db: any) => void; + +if (isBun) { + _Database = (await import("bun:sqlite")).Database; + const { getLoadablePath } = await import("sqlite-vec"); + _sqliteVecLoad = (db: any) => db.loadExtension(getLoadablePath()); +} else { + _Database = (await import("better-sqlite3")).default; + const sqliteVec = await import("sqlite-vec"); + _sqliteVecLoad = (db: any) => sqliteVec.load(db); +} + +/** + * Open a SQLite database. Works with both bun:sqlite and better-sqlite3. + */ +export function openDatabase(path: string): Database { + return new _Database(path) as Database; +} + +/** + * Common subset of the Database interface used throughout QMD. + */ +export interface Database { + exec(sql: string): void; + prepare(sql: string): Statement; + loadExtension(path: string): void; + close(): void; +} + +export interface Statement { + run(...params: any[]): { changes: number; lastInsertRowid: number | bigint }; + get(...params: any[]): any; + all(...params: any[]): any[]; +} + +/** + * Load the sqlite-vec extension into a database. + */ +export function loadSqliteVec(db: Database): void { + _sqliteVecLoad(db); +} diff --git a/src/eval-bm25.test.ts b/src/eval-bm25.test.ts index 333ba33..b3fb2c2 100644 --- a/src/eval-bm25.test.ts +++ b/src/eval-bm25.test.ts @@ -8,7 +8,7 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest"; import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs"; import { join, dirname } from "path"; import { tmpdir } from "os"; -import Database from "better-sqlite3"; +import type { Database } from "./db.js"; import { createHash } from "crypto"; import { fileURLToPath } from "url"; diff --git a/src/models/eval.test.ts b/src/models/eval.test.ts index 88cd1e0..c01defe 100644 --- a/src/models/eval.test.ts +++ b/src/models/eval.test.ts @@ -14,7 +14,8 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest"; import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs"; import { join } from "path"; import { tmpdir } from "os"; -import Database from "better-sqlite3"; +import { openDatabase } from "../db.js"; +import type { Database } from "../db.js"; import { createHash } from "crypto"; import { fileURLToPath } from "url"; import { dirname } from "path"; diff --git a/src/models/mcp.test.ts b/src/models/mcp.test.ts index 2c0520d..5f515e2 100644 --- a/src/models/mcp.test.ts +++ b/src/models/mcp.test.ts @@ -6,8 +6,8 @@ */ import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from "vitest"; -import Database from "better-sqlite3"; -import * as sqliteVec from "sqlite-vec"; +import { openDatabase, loadSqliteVec } from "../db.js"; +import type { Database } from "../db.js"; import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { getDefaultLlamaCpp, disposeDefaultLlamaCpp } from "../llm"; @@ -31,7 +31,7 @@ afterAll(async () => { }); function initTestDatabase(db: Database): void { - sqliteVec.load(db); + loadSqliteVec(db); db.exec("PRAGMA journal_mode = WAL"); // Content-addressable storage - the source of truth for document content @@ -226,7 +226,7 @@ describe("MCP Server", () => { await writeFile(join(testConfigDir, "index.yml"), YAML.stringify(testConfig)); testDbPath = `/tmp/qmd-mcp-test-${Date.now()}.sqlite`; - testDb = new Database(testDbPath); + testDb = openDatabase(testDbPath); initTestDatabase(testDb); seedTestData(testDb); }); @@ -306,7 +306,7 @@ describe("MCP Server", () => { }); test("returns empty when no vector table exists", async () => { - const emptyDb = new Database(":memory:"); + const emptyDb = openDatabase(":memory:"); initTestDatabase(emptyDb); emptyDb.exec("DROP TABLE IF EXISTS vectors_vec"); @@ -880,7 +880,7 @@ describe("MCP HTTP Transport", () => { beforeAll(async () => { // Create isolated test database with seeded data httpTestDbPath = `/tmp/qmd-mcp-http-test-${Date.now()}.sqlite`; - const db = new Database(httpTestDbPath); + const db = openDatabase(httpTestDbPath); initTestDatabase(db); seedTestData(db); db.close(); diff --git a/src/models/store.test.ts b/src/models/store.test.ts index b838087..fbd5a56 100644 --- a/src/models/store.test.ts +++ b/src/models/store.test.ts @@ -7,8 +7,8 @@ */ import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from "vitest"; -import Database from "better-sqlite3"; -import * as sqliteVec from "sqlite-vec"; +import { openDatabase, loadSqliteVec } from "../db.js"; +import type { Database } from "../db.js"; import { unlink, mkdtemp, rmdir, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -431,7 +431,8 @@ describe("Store Creation", () => { test("createStore creates a new store with custom path", async () => { const store = await createTestStore(); expect(store.dbPath).toBe(testDbPath); - expect(store.db).toBeInstanceOf(Database); + expect(store.db).toBeDefined(); + expect(typeof store.db.exec).toBe("function"); await cleanupTestDb(store); }); @@ -461,7 +462,7 @@ describe("Store Creation", () => { }); test("verifySqliteVecLoaded throws when sqlite-vec is not loaded", () => { - const db = new Database(":memory:"); + const db = openDatabase(":memory:"); try { expect(() => verifySqliteVecLoaded(db)).toThrow("sqlite-vec extension is unavailable"); } finally { @@ -470,9 +471,9 @@ describe("Store Creation", () => { }); test("verifySqliteVecLoaded succeeds when sqlite-vec is loaded", () => { - const db = new Database(":memory:"); + const db = openDatabase(":memory:"); try { - sqliteVec.load(db); + loadSqliteVec(db); expect(() => verifySqliteVecLoaded(db)).not.toThrow(); } finally { db.close(); diff --git a/src/qmd.ts b/src/qmd.ts index 6c200dd..6c8c8f8 100755 --- a/src/qmd.ts +++ b/src/qmd.ts @@ -1,4 +1,5 @@ -import Database from "better-sqlite3"; +import { openDatabase } from "./db.js"; +import type { Database } from "./db.js"; import fastGlob from "fast-glob"; import { execSync, spawn as nodeSpawn } from "child_process"; import { fileURLToPath } from "url"; diff --git a/src/store.ts b/src/store.ts index b1bb4c7..de3acb3 100644 --- a/src/store.ts +++ b/src/store.ts @@ -11,11 +11,11 @@ * const store = createStore(); */ -import Database from "better-sqlite3"; +import { openDatabase, loadSqliteVec } from "./db.js"; +import type { Database } from "./db.js"; import picomatch from "picomatch"; import { createHash } from "crypto"; import { realpathSync, statSync, mkdirSync } from "node:fs"; -import * as sqliteVec from "sqlite-vec"; import { LlamaCpp, getDefaultLlamaCpp, @@ -618,22 +618,16 @@ export function verifySqliteVecLoaded(db: Database): void { } } +let _sqliteVecAvailable: boolean | null = null; + function initializeDatabase(db: Database): void { try { - sqliteVec.load(db); + loadSqliteVec(db); verifySqliteVecLoaded(db); - } catch (err) { - const message = getErrorMessage(err); - - if (message.includes("does not support dynamic extension loading")) { - throw createSqliteVecUnavailableError("SQLite build does not support dynamic extension loading"); - } - - if (message.includes("sqlite-vec extension is unavailable")) { - throw err; - } - - throw err; + _sqliteVecAvailable = true; + } catch { + // sqlite-vec is optional — vector search won't work but FTS is fine + _sqliteVecAvailable = false; } db.exec("PRAGMA journal_mode = WAL"); db.exec("PRAGMA foreign_keys = ON"); @@ -747,7 +741,14 @@ function initializeDatabase(db: Database): void { } +export function isSqliteVecAvailable(): boolean { + return _sqliteVecAvailable === true; +} + function ensureVecTableInternal(db: Database, dimensions: number): void { + if (!_sqliteVecAvailable) { + throw new Error("sqlite-vec is not available. Vector operations require a SQLite build with extension loading support."); + } const tableInfo = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get() as { sql: string } | null; if (tableInfo) { const match = tableInfo.sql.match(/float\[(\d+)\]/); @@ -845,7 +846,7 @@ export type Store = { */ export function createStore(dbPath?: string): Store { const resolvedPath = dbPath || getDefaultDbPath(); - const db = new Database(resolvedPath); + const db = openDatabase(resolvedPath); initializeDatabase(db); return {