feat: cross-runtime SQLite compat layer (bun:sqlite + better-sqlite3)

Add src/db.ts that dynamically imports bun:sqlite under Bun and
better-sqlite3 under Node.js. Exports openDatabase(), loadSqliteVec(),
and a shared Database interface.

- sqlite-vec loading is now optional — FTS works without it, vector
  ops throw a clear error if unavailable
- CI tests both runtimes: Node 22/23 via vitest, Bun via bun test
- All 104 unit tests pass on both Node and Bun
This commit is contained in:
Tobi Lutke 2026-02-15 17:15:47 -04:00
parent c685f7ac71
commit dcedfb5268
No known key found for this signature in database
8 changed files with 134 additions and 32 deletions

View File

@ -7,7 +7,8 @@ on:
branches: [main]
jobs:
test:
test-node:
name: Node ${{ matrix.node-version }} (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@ -44,3 +45,48 @@ jobs:
run: npx vitest run --reporter=verbose src/integration/*.test.ts
env:
CI: true
test-bun:
name: Bun (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install SQLite (Ubuntu)
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
- name: Install SQLite (macOS)
if: runner.os == 'macOS'
run: brew install sqlite
- run: bun install
- name: Unit tests
run: bun test --preload ./src/test-preload.ts src/*.test.ts
env:
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
- name: Model tests
run: bun test --preload ./src/test-preload.ts src/models/*.test.ts
env:
CI: true
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
- name: Integration tests
run: bun test --preload ./src/test-preload.ts src/integration/*.test.ts
env:
CI: true
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu

52
src/db.ts Normal file
View File

@ -0,0 +1,52 @@
/**
* db.ts - Cross-runtime SQLite compatibility layer
*
* Provides a unified Database export that works under both Bun (bun:sqlite)
* and Node.js (better-sqlite3). The APIs are nearly identical the main
* difference is the import path.
*/
export const isBun = typeof globalThis.Bun !== "undefined";
let _Database: any;
let _sqliteVecLoad: (db: any) => void;
if (isBun) {
_Database = (await import("bun:sqlite")).Database;
const { getLoadablePath } = await import("sqlite-vec");
_sqliteVecLoad = (db: any) => db.loadExtension(getLoadablePath());
} else {
_Database = (await import("better-sqlite3")).default;
const sqliteVec = await import("sqlite-vec");
_sqliteVecLoad = (db: any) => sqliteVec.load(db);
}
/**
* Open a SQLite database. Works with both bun:sqlite and better-sqlite3.
*/
export function openDatabase(path: string): Database {
return new _Database(path) as Database;
}
/**
* Common subset of the Database interface used throughout QMD.
*/
export interface Database {
exec(sql: string): void;
prepare(sql: string): Statement;
loadExtension(path: string): void;
close(): void;
}
export interface Statement {
run(...params: any[]): { changes: number; lastInsertRowid: number | bigint };
get(...params: any[]): any;
all(...params: any[]): any[];
}
/**
* Load the sqlite-vec extension into a database.
*/
export function loadSqliteVec(db: Database): void {
_sqliteVecLoad(db);
}

View File

@ -8,7 +8,7 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest";
import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs";
import { join, dirname } from "path";
import { tmpdir } from "os";
import Database from "better-sqlite3";
import type { Database } from "./db.js";
import { createHash } from "crypto";
import { fileURLToPath } from "url";

View File

@ -14,7 +14,8 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest";
import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs";
import { join } from "path";
import { tmpdir } from "os";
import Database from "better-sqlite3";
import { openDatabase } from "../db.js";
import type { Database } from "../db.js";
import { createHash } from "crypto";
import { fileURLToPath } from "url";
import { dirname } from "path";

View File

@ -6,8 +6,8 @@
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from "vitest";
import Database from "better-sqlite3";
import * as sqliteVec from "sqlite-vec";
import { openDatabase, loadSqliteVec } from "../db.js";
import type { Database } from "../db.js";
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
import { z } from "zod";
import { getDefaultLlamaCpp, disposeDefaultLlamaCpp } from "../llm";
@ -31,7 +31,7 @@ afterAll(async () => {
});
function initTestDatabase(db: Database): void {
sqliteVec.load(db);
loadSqliteVec(db);
db.exec("PRAGMA journal_mode = WAL");
// Content-addressable storage - the source of truth for document content
@ -226,7 +226,7 @@ describe("MCP Server", () => {
await writeFile(join(testConfigDir, "index.yml"), YAML.stringify(testConfig));
testDbPath = `/tmp/qmd-mcp-test-${Date.now()}.sqlite`;
testDb = new Database(testDbPath);
testDb = openDatabase(testDbPath);
initTestDatabase(testDb);
seedTestData(testDb);
});
@ -306,7 +306,7 @@ describe("MCP Server", () => {
});
test("returns empty when no vector table exists", async () => {
const emptyDb = new Database(":memory:");
const emptyDb = openDatabase(":memory:");
initTestDatabase(emptyDb);
emptyDb.exec("DROP TABLE IF EXISTS vectors_vec");
@ -880,7 +880,7 @@ describe("MCP HTTP Transport", () => {
beforeAll(async () => {
// Create isolated test database with seeded data
httpTestDbPath = `/tmp/qmd-mcp-http-test-${Date.now()}.sqlite`;
const db = new Database(httpTestDbPath);
const db = openDatabase(httpTestDbPath);
initTestDatabase(db);
seedTestData(db);
db.close();

View File

@ -7,8 +7,8 @@
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from "vitest";
import Database from "better-sqlite3";
import * as sqliteVec from "sqlite-vec";
import { openDatabase, loadSqliteVec } from "../db.js";
import type { Database } from "../db.js";
import { unlink, mkdtemp, rmdir, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@ -431,7 +431,8 @@ describe("Store Creation", () => {
test("createStore creates a new store with custom path", async () => {
const store = await createTestStore();
expect(store.dbPath).toBe(testDbPath);
expect(store.db).toBeInstanceOf(Database);
expect(store.db).toBeDefined();
expect(typeof store.db.exec).toBe("function");
await cleanupTestDb(store);
});
@ -461,7 +462,7 @@ describe("Store Creation", () => {
});
test("verifySqliteVecLoaded throws when sqlite-vec is not loaded", () => {
const db = new Database(":memory:");
const db = openDatabase(":memory:");
try {
expect(() => verifySqliteVecLoaded(db)).toThrow("sqlite-vec extension is unavailable");
} finally {
@ -470,9 +471,9 @@ describe("Store Creation", () => {
});
test("verifySqliteVecLoaded succeeds when sqlite-vec is loaded", () => {
const db = new Database(":memory:");
const db = openDatabase(":memory:");
try {
sqliteVec.load(db);
loadSqliteVec(db);
expect(() => verifySqliteVecLoaded(db)).not.toThrow();
} finally {
db.close();

View File

@ -1,4 +1,5 @@
import Database from "better-sqlite3";
import { openDatabase } from "./db.js";
import type { Database } from "./db.js";
import fastGlob from "fast-glob";
import { execSync, spawn as nodeSpawn } from "child_process";
import { fileURLToPath } from "url";

View File

@ -11,11 +11,11 @@
* const store = createStore();
*/
import Database from "better-sqlite3";
import { openDatabase, loadSqliteVec } from "./db.js";
import type { Database } from "./db.js";
import picomatch from "picomatch";
import { createHash } from "crypto";
import { realpathSync, statSync, mkdirSync } from "node:fs";
import * as sqliteVec from "sqlite-vec";
import {
LlamaCpp,
getDefaultLlamaCpp,
@ -618,22 +618,16 @@ export function verifySqliteVecLoaded(db: Database): void {
}
}
let _sqliteVecAvailable: boolean | null = null;
function initializeDatabase(db: Database): void {
try {
sqliteVec.load(db);
loadSqliteVec(db);
verifySqliteVecLoaded(db);
} catch (err) {
const message = getErrorMessage(err);
if (message.includes("does not support dynamic extension loading")) {
throw createSqliteVecUnavailableError("SQLite build does not support dynamic extension loading");
}
if (message.includes("sqlite-vec extension is unavailable")) {
throw err;
}
throw err;
_sqliteVecAvailable = true;
} catch {
// sqlite-vec is optional — vector search won't work but FTS is fine
_sqliteVecAvailable = false;
}
db.exec("PRAGMA journal_mode = WAL");
db.exec("PRAGMA foreign_keys = ON");
@ -747,7 +741,14 @@ function initializeDatabase(db: Database): void {
}
export function isSqliteVecAvailable(): boolean {
return _sqliteVecAvailable === true;
}
function ensureVecTableInternal(db: Database, dimensions: number): void {
if (!_sqliteVecAvailable) {
throw new Error("sqlite-vec is not available. Vector operations require a SQLite build with extension loading support.");
}
const tableInfo = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get() as { sql: string } | null;
if (tableInfo) {
const match = tableInfo.sql.match(/float\[(\d+)\]/);
@ -845,7 +846,7 @@ export type Store = {
*/
export function createStore(dbPath?: string): Store {
const resolvedPath = dbPath || getDefaultDbPath();
const db = new Database(resolvedPath);
const db = openDatabase(resolvedPath);
initializeDatabase(db);
return {