feat: cross-runtime SQLite compat layer (bun:sqlite + better-sqlite3)
Add src/db.ts that dynamically imports bun:sqlite under Bun and better-sqlite3 under Node.js. Exports openDatabase(), loadSqliteVec(), and a shared Database interface. - sqlite-vec loading is now optional — FTS works without it, vector ops throw a clear error if unavailable - CI tests both runtimes: Node 22/23 via vitest, Bun via bun test - All 104 unit tests pass on both Node and Bun
This commit is contained in:
parent
c685f7ac71
commit
dcedfb5268
48
.github/workflows/ci.yml
vendored
48
.github/workflows/ci.yml
vendored
@ -7,7 +7,8 @@ on:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
test-node:
|
||||
name: Node ${{ matrix.node-version }} (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@ -44,3 +45,48 @@ jobs:
|
||||
run: npx vitest run --reporter=verbose src/integration/*.test.ts
|
||||
env:
|
||||
CI: true
|
||||
|
||||
test-bun:
|
||||
name: Bun (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: latest
|
||||
|
||||
- name: Install SQLite (Ubuntu)
|
||||
if: runner.os == 'Linux'
|
||||
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
|
||||
|
||||
- name: Install SQLite (macOS)
|
||||
if: runner.os == 'macOS'
|
||||
run: brew install sqlite
|
||||
|
||||
- run: bun install
|
||||
|
||||
- name: Unit tests
|
||||
run: bun test --preload ./src/test-preload.ts src/*.test.ts
|
||||
env:
|
||||
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
|
||||
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
|
||||
|
||||
- name: Model tests
|
||||
run: bun test --preload ./src/test-preload.ts src/models/*.test.ts
|
||||
env:
|
||||
CI: true
|
||||
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
|
||||
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
|
||||
|
||||
- name: Integration tests
|
||||
run: bun test --preload ./src/test-preload.ts src/integration/*.test.ts
|
||||
env:
|
||||
CI: true
|
||||
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
|
||||
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
|
||||
|
||||
52
src/db.ts
Normal file
52
src/db.ts
Normal file
@ -0,0 +1,52 @@
|
||||
/**
|
||||
* db.ts - Cross-runtime SQLite compatibility layer
|
||||
*
|
||||
* Provides a unified Database export that works under both Bun (bun:sqlite)
|
||||
* and Node.js (better-sqlite3). The APIs are nearly identical — the main
|
||||
* difference is the import path.
|
||||
*/
|
||||
|
||||
export const isBun = typeof globalThis.Bun !== "undefined";
|
||||
|
||||
let _Database: any;
|
||||
let _sqliteVecLoad: (db: any) => void;
|
||||
|
||||
if (isBun) {
|
||||
_Database = (await import("bun:sqlite")).Database;
|
||||
const { getLoadablePath } = await import("sqlite-vec");
|
||||
_sqliteVecLoad = (db: any) => db.loadExtension(getLoadablePath());
|
||||
} else {
|
||||
_Database = (await import("better-sqlite3")).default;
|
||||
const sqliteVec = await import("sqlite-vec");
|
||||
_sqliteVecLoad = (db: any) => sqliteVec.load(db);
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a SQLite database. Works with both bun:sqlite and better-sqlite3.
|
||||
*/
|
||||
export function openDatabase(path: string): Database {
|
||||
return new _Database(path) as Database;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common subset of the Database interface used throughout QMD.
|
||||
*/
|
||||
export interface Database {
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): Statement;
|
||||
loadExtension(path: string): void;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface Statement {
|
||||
run(...params: any[]): { changes: number; lastInsertRowid: number | bigint };
|
||||
get(...params: any[]): any;
|
||||
all(...params: any[]): any[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the sqlite-vec extension into a database.
|
||||
*/
|
||||
export function loadSqliteVec(db: Database): void {
|
||||
_sqliteVecLoad(db);
|
||||
}
|
||||
@ -8,7 +8,7 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
||||
import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs";
|
||||
import { join, dirname } from "path";
|
||||
import { tmpdir } from "os";
|
||||
import Database from "better-sqlite3";
|
||||
import type { Database } from "./db.js";
|
||||
import { createHash } from "crypto";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
|
||||
@ -14,7 +14,8 @@ import { describe, test, expect, beforeAll, afterAll } from "vitest";
|
||||
import { mkdtempSync, rmSync, readFileSync, readdirSync } from "fs";
|
||||
import { join } from "path";
|
||||
import { tmpdir } from "os";
|
||||
import Database from "better-sqlite3";
|
||||
import { openDatabase } from "../db.js";
|
||||
import type { Database } from "../db.js";
|
||||
import { createHash } from "crypto";
|
||||
import { fileURLToPath } from "url";
|
||||
import { dirname } from "path";
|
||||
|
||||
@ -6,8 +6,8 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from "vitest";
|
||||
import Database from "better-sqlite3";
|
||||
import * as sqliteVec from "sqlite-vec";
|
||||
import { openDatabase, loadSqliteVec } from "../db.js";
|
||||
import type { Database } from "../db.js";
|
||||
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { z } from "zod";
|
||||
import { getDefaultLlamaCpp, disposeDefaultLlamaCpp } from "../llm";
|
||||
@ -31,7 +31,7 @@ afterAll(async () => {
|
||||
});
|
||||
|
||||
function initTestDatabase(db: Database): void {
|
||||
sqliteVec.load(db);
|
||||
loadSqliteVec(db);
|
||||
db.exec("PRAGMA journal_mode = WAL");
|
||||
|
||||
// Content-addressable storage - the source of truth for document content
|
||||
@ -226,7 +226,7 @@ describe("MCP Server", () => {
|
||||
await writeFile(join(testConfigDir, "index.yml"), YAML.stringify(testConfig));
|
||||
|
||||
testDbPath = `/tmp/qmd-mcp-test-${Date.now()}.sqlite`;
|
||||
testDb = new Database(testDbPath);
|
||||
testDb = openDatabase(testDbPath);
|
||||
initTestDatabase(testDb);
|
||||
seedTestData(testDb);
|
||||
});
|
||||
@ -306,7 +306,7 @@ describe("MCP Server", () => {
|
||||
});
|
||||
|
||||
test("returns empty when no vector table exists", async () => {
|
||||
const emptyDb = new Database(":memory:");
|
||||
const emptyDb = openDatabase(":memory:");
|
||||
initTestDatabase(emptyDb);
|
||||
emptyDb.exec("DROP TABLE IF EXISTS vectors_vec");
|
||||
|
||||
@ -880,7 +880,7 @@ describe("MCP HTTP Transport", () => {
|
||||
beforeAll(async () => {
|
||||
// Create isolated test database with seeded data
|
||||
httpTestDbPath = `/tmp/qmd-mcp-http-test-${Date.now()}.sqlite`;
|
||||
const db = new Database(httpTestDbPath);
|
||||
const db = openDatabase(httpTestDbPath);
|
||||
initTestDatabase(db);
|
||||
seedTestData(db);
|
||||
db.close();
|
||||
|
||||
@ -7,8 +7,8 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from "vitest";
|
||||
import Database from "better-sqlite3";
|
||||
import * as sqliteVec from "sqlite-vec";
|
||||
import { openDatabase, loadSqliteVec } from "../db.js";
|
||||
import type { Database } from "../db.js";
|
||||
import { unlink, mkdtemp, rmdir, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@ -431,7 +431,8 @@ describe("Store Creation", () => {
|
||||
test("createStore creates a new store with custom path", async () => {
|
||||
const store = await createTestStore();
|
||||
expect(store.dbPath).toBe(testDbPath);
|
||||
expect(store.db).toBeInstanceOf(Database);
|
||||
expect(store.db).toBeDefined();
|
||||
expect(typeof store.db.exec).toBe("function");
|
||||
await cleanupTestDb(store);
|
||||
});
|
||||
|
||||
@ -461,7 +462,7 @@ describe("Store Creation", () => {
|
||||
});
|
||||
|
||||
test("verifySqliteVecLoaded throws when sqlite-vec is not loaded", () => {
|
||||
const db = new Database(":memory:");
|
||||
const db = openDatabase(":memory:");
|
||||
try {
|
||||
expect(() => verifySqliteVecLoaded(db)).toThrow("sqlite-vec extension is unavailable");
|
||||
} finally {
|
||||
@ -470,9 +471,9 @@ describe("Store Creation", () => {
|
||||
});
|
||||
|
||||
test("verifySqliteVecLoaded succeeds when sqlite-vec is loaded", () => {
|
||||
const db = new Database(":memory:");
|
||||
const db = openDatabase(":memory:");
|
||||
try {
|
||||
sqliteVec.load(db);
|
||||
loadSqliteVec(db);
|
||||
expect(() => verifySqliteVecLoaded(db)).not.toThrow();
|
||||
} finally {
|
||||
db.close();
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import Database from "better-sqlite3";
|
||||
import { openDatabase } from "./db.js";
|
||||
import type { Database } from "./db.js";
|
||||
import fastGlob from "fast-glob";
|
||||
import { execSync, spawn as nodeSpawn } from "child_process";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
33
src/store.ts
33
src/store.ts
@ -11,11 +11,11 @@
|
||||
* const store = createStore();
|
||||
*/
|
||||
|
||||
import Database from "better-sqlite3";
|
||||
import { openDatabase, loadSqliteVec } from "./db.js";
|
||||
import type { Database } from "./db.js";
|
||||
import picomatch from "picomatch";
|
||||
import { createHash } from "crypto";
|
||||
import { realpathSync, statSync, mkdirSync } from "node:fs";
|
||||
import * as sqliteVec from "sqlite-vec";
|
||||
import {
|
||||
LlamaCpp,
|
||||
getDefaultLlamaCpp,
|
||||
@ -618,22 +618,16 @@ export function verifySqliteVecLoaded(db: Database): void {
|
||||
}
|
||||
}
|
||||
|
||||
let _sqliteVecAvailable: boolean | null = null;
|
||||
|
||||
function initializeDatabase(db: Database): void {
|
||||
try {
|
||||
sqliteVec.load(db);
|
||||
loadSqliteVec(db);
|
||||
verifySqliteVecLoaded(db);
|
||||
} catch (err) {
|
||||
const message = getErrorMessage(err);
|
||||
|
||||
if (message.includes("does not support dynamic extension loading")) {
|
||||
throw createSqliteVecUnavailableError("SQLite build does not support dynamic extension loading");
|
||||
}
|
||||
|
||||
if (message.includes("sqlite-vec extension is unavailable")) {
|
||||
throw err;
|
||||
}
|
||||
|
||||
throw err;
|
||||
_sqliteVecAvailable = true;
|
||||
} catch {
|
||||
// sqlite-vec is optional — vector search won't work but FTS is fine
|
||||
_sqliteVecAvailable = false;
|
||||
}
|
||||
db.exec("PRAGMA journal_mode = WAL");
|
||||
db.exec("PRAGMA foreign_keys = ON");
|
||||
@ -747,7 +741,14 @@ function initializeDatabase(db: Database): void {
|
||||
}
|
||||
|
||||
|
||||
export function isSqliteVecAvailable(): boolean {
|
||||
return _sqliteVecAvailable === true;
|
||||
}
|
||||
|
||||
function ensureVecTableInternal(db: Database, dimensions: number): void {
|
||||
if (!_sqliteVecAvailable) {
|
||||
throw new Error("sqlite-vec is not available. Vector operations require a SQLite build with extension loading support.");
|
||||
}
|
||||
const tableInfo = db.prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get() as { sql: string } | null;
|
||||
if (tableInfo) {
|
||||
const match = tableInfo.sql.match(/float\[(\d+)\]/);
|
||||
@ -845,7 +846,7 @@ export type Store = {
|
||||
*/
|
||||
export function createStore(dbPath?: string): Store {
|
||||
const resolvedPath = dbPath || getDefaultDbPath();
|
||||
const db = new Database(resolvedPath);
|
||||
const db = openDatabase(resolvedPath);
|
||||
initializeDatabase(db);
|
||||
|
||||
return {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user