core: migrate project table from JSON to structured columns for better query performance

pull/8586/head
Dax Raad 2026-01-15 13:42:05 -05:00
parent 9f96d8aa78
commit 2c234b8d62
13 changed files with 309 additions and 180 deletions

View File

@ -3,5 +3,5 @@ import { defineConfig } from "drizzle-kit"
export default defineConfig({
dialect: "sqlite",
schema: "./src/**/*.sql.ts",
out: "./drizzle",
out: "./migration",
})

View File

@ -1,20 +1,16 @@
CREATE TABLE `project` (
`id` text PRIMARY KEY NOT NULL,
`data` text NOT NULL
`worktree` text NOT NULL,
`vcs` text,
`name` text,
`icon_url` text,
`icon_color` text,
`time_created` integer NOT NULL,
`time_updated` integer NOT NULL,
`time_initialized` integer,
`sandboxes` text NOT NULL
);
--> statement-breakpoint
CREATE TABLE `session` (
`id` text PRIMARY KEY NOT NULL,
`project_id` text NOT NULL,
`parent_id` text,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
`data` text NOT NULL,
FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `session_project_idx` ON `session` (`project_id`);--> statement-breakpoint
CREATE INDEX `session_parent_idx` ON `session` (`parent_id`);--> statement-breakpoint
CREATE TABLE `message` (
`id` text PRIMARY KEY NOT NULL,
`session_id` text NOT NULL,
@ -46,6 +42,18 @@ CREATE TABLE `session_diff` (
FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE TABLE `session` (
`id` text PRIMARY KEY NOT NULL,
`project_id` text NOT NULL,
`parent_id` text,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL,
`data` text NOT NULL,
FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `session_project_idx` ON `session` (`project_id`);--> statement-breakpoint
CREATE INDEX `session_parent_idx` ON `session` (`parent_id`);--> statement-breakpoint
CREATE TABLE `todo` (
`session_id` text PRIMARY KEY NOT NULL,
`data` text NOT NULL,

View File

@ -1,7 +1,7 @@
{
"version": "6",
"dialect": "sqlite",
"id": "7255471a-8cff-422c-b0ef-419a2aa7d952",
"id": "f79c82ae-4de1-4a4c-a5f3-857bc3ee97f2",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"project": {
@ -14,8 +14,64 @@
"notNull": true,
"autoincrement": false
},
"data": {
"name": "data",
"worktree": {
"name": "worktree",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"vcs": {
"name": "vcs",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"icon_url": {
"name": "icon_url",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"icon_color": {
"name": "icon_color",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"time_created": {
"name": "time_created",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"time_updated": {
"name": "time_updated",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"time_initialized": {
"name": "time_initialized",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"sandboxes": {
"name": "sandboxes",
"type": "text",
"primaryKey": false,
"notNull": true,
@ -28,87 +84,6 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"session": {
"name": "session",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"project_id": {
"name": "project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"parent_id": {
"name": "parent_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"data": {
"name": "data",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"session_project_idx": {
"name": "session_project_idx",
"columns": [
"project_id"
],
"isUnique": false
},
"session_parent_idx": {
"name": "session_parent_idx",
"columns": [
"parent_id"
],
"isUnique": false
}
},
"foreignKeys": {
"session_project_id_project_id_fk": {
"name": "session_project_id_project_id_fk",
"tableFrom": "session",
"tableTo": "project",
"columnsFrom": [
"project_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"message": {
"name": "message",
"columns": {
@ -312,6 +287,87 @@
"uniqueConstraints": {},
"checkConstraints": {}
},
"session": {
"name": "session",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"project_id": {
"name": "project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"parent_id": {
"name": "parent_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"data": {
"name": "data",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"session_project_idx": {
"name": "session_project_idx",
"columns": [
"project_id"
],
"isUnique": false
},
"session_parent_idx": {
"name": "session_parent_idx",
"columns": [
"parent_id"
],
"isUnique": false
}
},
"foreignKeys": {
"session_project_id_project_id_fk": {
"name": "session_project_id_project_id_fk",
"tableFrom": "session",
"tableTo": "project",
"columnsFrom": [
"project_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"todo": {
"name": "todo",
"columns": {

View File

@ -5,8 +5,8 @@
{
"idx": 0,
"version": "6",
"when": 1768425777461,
"tag": "0000_initial",
"when": 1768501411495,
"tag": "0000_easy_albert_cleary",
"breakpoints": true
}
]

View File

@ -4,7 +4,7 @@ import { Glob } from "bun"
import path from "path"
import fs from "fs"
const migrationsDir = "./drizzle"
const migrationsDir = "./migration"
const outFile = "./src/storage/migrations.generated.ts"
if (!fs.existsSync(migrationsDir)) {

View File

@ -4,6 +4,7 @@ import { bootstrap } from "../bootstrap"
import { UI } from "../ui"
import { db } from "../../storage/db"
import { ProjectTable } from "../../project/project.sql"
import { Project } from "../../project/project"
import {
SessionTable,
MessageTable,
@ -55,7 +56,8 @@ const ExportCommand = cmd({
const projectDir = path.join(outDir, "project")
await fs.mkdir(projectDir, { recursive: true })
for (const row of db().select().from(ProjectTable).all()) {
await Bun.write(path.join(projectDir, `${row.id}.json`), JSON.stringify(row.data, null, 2))
const project = Project.fromRow(row)
await Bun.write(path.join(projectDir, `${row.id}.json`), JSON.stringify(project, null, 2))
stats.projects++
}

View File

@ -1,7 +1,14 @@
import { sqliteTable, text } from "drizzle-orm/sqlite-core"
import type { Project } from "./project"
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"
export const ProjectTable = sqliteTable("project", {
id: text("id").primaryKey(),
data: text("data", { mode: "json" }).notNull().$type<Project.Info>(),
worktree: text("worktree").notNull(),
vcs: text("vcs"),
name: text("name"),
icon_url: text("icon_url"),
icon_color: text("icon_color"),
time_created: integer("time_created").notNull(),
time_updated: integer("time_updated").notNull(),
time_initialized: integer("time_initialized"),
sandboxes: text("sandboxes", { mode: "json" }).notNull().$type<string[]>(),
})

View File

@ -53,6 +53,28 @@ export namespace Project {
Updated: BusEvent.define("project.updated", Info),
}
type Row = typeof ProjectTable.$inferSelect
export function fromRow(row: Row): Info {
const icon =
row.icon_url || row.icon_color
? { url: row.icon_url ?? undefined, color: row.icon_color ?? undefined }
: undefined
return {
id: row.id,
worktree: row.worktree,
vcs: row.vcs as Info["vcs"],
name: row.name ?? undefined,
icon,
time: {
created: row.time_created,
updated: row.time_updated,
initialized: row.time_initialized ?? undefined,
},
sandboxes: row.sandboxes,
}
}
export async function fromDirectory(directory: string) {
log.info("fromDirectory", { directory })
@ -179,9 +201,9 @@ export namespace Project {
})
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, id)).get()
let existing = row?.data
if (!existing) {
existing = {
const existing = await iife(async () => {
if (row) return fromRow(row)
const fresh: Info = {
id,
worktree,
vcs: vcs as Info["vcs"],
@ -194,10 +216,8 @@ export namespace Project {
if (id !== "global") {
await migrateFromGlobal(id, worktree)
}
}
// migrate old projects before sandboxes
if (!existing.sandboxes) existing.sandboxes = []
return fresh
})
if (Flag.OPENCODE_EXPERIMENTAL_ICON_DISCOVERY) discover(existing)
@ -212,11 +232,29 @@ export namespace Project {
}
if (sandbox !== result.worktree && !result.sandboxes.includes(sandbox)) result.sandboxes.push(sandbox)
result.sandboxes = result.sandboxes.filter((x) => existsSync(x))
db()
.insert(ProjectTable)
.values({ id, data: result })
.onConflictDoUpdate({ target: ProjectTable.id, set: { data: result } })
.run()
const insert = {
id: result.id,
worktree: result.worktree,
vcs: result.vcs,
name: result.name,
icon_url: result.icon?.url,
icon_color: result.icon?.color,
time_created: result.time.created,
time_updated: result.time.updated,
time_initialized: result.time.initialized,
sandboxes: result.sandboxes,
}
const update = {
worktree: result.worktree,
vcs: result.vcs,
name: result.name,
icon_url: result.icon?.url,
icon_color: result.icon?.color,
time_updated: result.time.updated,
time_initialized: result.time.initialized,
sandboxes: result.sandboxes,
}
db().insert(ProjectTable).values(insert).onConflictDoUpdate({ target: ProjectTable.id, set: update }).run()
GlobalBus.emit("event", {
payload: {
type: Event.Updated.type,
@ -283,10 +321,13 @@ export namespace Project {
}
export function setInitialized(projectID: string) {
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get()
if (!row) return
const data = { ...row.data, time: { ...row.data.time, initialized: Date.now() } }
db().update(ProjectTable).set({ data }).where(eq(ProjectTable.id, projectID)).run()
db()
.update(ProjectTable)
.set({
time_initialized: Date.now(),
})
.where(eq(ProjectTable.id, projectID))
.run()
}
export function list() {
@ -294,7 +335,7 @@ export namespace Project {
.select()
.from(ProjectTable)
.all()
.map((row) => row.data)
.map((row) => fromRow(row))
}
export const update = fn(
@ -305,24 +346,19 @@ export namespace Project {
commands: Info.shape.commands.optional(),
}),
async (input) => {
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, input.projectID)).get()
if (!row) throw new Error(`Project not found: ${input.projectID}`)
const data = { ...row.data }
if (input.name !== undefined) data.name = input.name
if (input.icon !== undefined) {
data.icon = { ...data.icon }
if (input.icon.url !== undefined) data.icon.url = input.icon.url
if (input.icon.override !== undefined) data.icon.override = input.icon.override || undefined
if (input.icon.color !== undefined) data.icon.color = input.icon.color
}
if (input.commands?.start !== undefined) {
const start = input.commands.start || undefined
data.commands = { ...(data.commands ?? {}) }
data.commands.start = start
if (!data.commands.start) data.commands = undefined
}
data.time.updated = Date.now()
db().update(ProjectTable).set({ data }).where(eq(ProjectTable.id, input.projectID)).run()
const result = db()
.update(ProjectTable)
.set({
name: input.name,
icon_url: input.icon?.url,
icon_color: input.icon?.color,
time_updated: Date.now(),
})
.where(eq(ProjectTable.id, input.projectID))
.returning()
.get()
if (!result) throw new Error(`Project not found: ${input.projectID}`)
const data = fromRow(result)
GlobalBus.emit("event", {
payload: {
type: Event.Updated.type,
@ -335,9 +371,10 @@ export namespace Project {
export async function sandboxes(projectID: string) {
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get()
if (!row?.data.sandboxes) return []
if (!row) return []
const data = fromRow(row)
const valid: string[] = []
for (const dir of row.data.sandboxes) {
for (const dir of data.sandboxes) {
const stat = await fs.stat(dir).catch(() => undefined)
if (stat?.isDirectory()) valid.push(dir)
}
@ -347,12 +384,16 @@ export namespace Project {
export async function addSandbox(projectID: string, directory: string) {
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get()
if (!row) throw new Error(`Project not found: ${projectID}`)
const data = { ...row.data }
const sandboxes = data.sandboxes ?? []
const sandboxes = row.sandboxes ?? []
if (!sandboxes.includes(directory)) sandboxes.push(directory)
data.sandboxes = sandboxes
data.time.updated = Date.now()
db().update(ProjectTable).set({ data }).where(eq(ProjectTable.id, projectID)).run()
const result = db()
.update(ProjectTable)
.set({ sandboxes, time_updated: Date.now() })
.where(eq(ProjectTable.id, projectID))
.returning()
.get()
if (!result) throw new Error(`Project not found: ${projectID}`)
const data = fromRow(result)
GlobalBus.emit("event", {
payload: {
type: Event.Updated.type,
@ -365,11 +406,15 @@ export namespace Project {
export async function removeSandbox(projectID: string, directory: string) {
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get()
if (!row) throw new Error(`Project not found: ${projectID}`)
const data = { ...row.data }
const sandboxes = data.sandboxes ?? []
data.sandboxes = sandboxes.filter((sandbox) => sandbox !== directory)
data.time.updated = Date.now()
db().update(ProjectTable).set({ data }).where(eq(ProjectTable.id, projectID)).run()
const sandboxes = (row.sandboxes ?? []).filter((s) => s !== directory)
const result = db()
.update(ProjectTable)
.set({ sandboxes, time_updated: Date.now() })
.where(eq(ProjectTable.id, projectID))
.returning()
.get()
if (!result) throw new Error(`Project not found: ${projectID}`)
const data = fromRow(result)
GlobalBus.emit("event", {
payload: {
type: Event.Updated.type,

View File

@ -20,32 +20,27 @@ const log = Log.create({ service: "db" })
export type DB = ReturnType<typeof drizzle>
let initialized = false
const connection = lazy(() => {
const dbPath = path.join(Global.Path.data, "opencode.db")
log.info("opening database", { path: dbPath })
const sqlite = new Database(dbPath, { create: true })
sqlite.exec("PRAGMA journal_mode = WAL")
sqlite.exec("PRAGMA synchronous = NORMAL")
sqlite.exec("PRAGMA busy_timeout = 5000")
sqlite.exec("PRAGMA cache_size = -64000")
sqlite.exec("PRAGMA foreign_keys = ON")
sqlite.run("PRAGMA journal_mode = WAL")
sqlite.run("PRAGMA synchronous = NORMAL")
sqlite.run("PRAGMA busy_timeout = 5000")
sqlite.run("PRAGMA cache_size = -64000")
sqlite.run("PRAGMA foreign_keys = ON")
runMigrations(sqlite)
migrate(sqlite)
// Run JSON migration asynchronously after schema is ready
if (!initialized) {
initialized = true
migrateFromJson(sqlite).catch((e) => log.error("json migration failed", { error: e }))
}
migrateFromJson(sqlite).catch((e) => log.error("json migration failed", { error: e }))
return drizzle(sqlite)
})
function runMigrations(sqlite: Database) {
function migrate(sqlite: Database) {
sqlite.exec(`
CREATE TABLE IF NOT EXISTS _migrations (
name TEXT PRIMARY KEY,

View File

@ -56,7 +56,21 @@ export async function migrateFromJson(sqlite: Database, customStorageDir?: strin
stats.errors.push(`project missing id: ${file}`)
continue
}
db.insert(ProjectTable).values({ id: data.id, data }).onConflictDoNothing().run()
db.insert(ProjectTable)
.values({
id: data.id,
worktree: data.worktree ?? "/",
vcs: data.vcs,
name: data.name ?? undefined,
icon_url: data.icon?.url,
icon_color: data.icon?.color,
time_created: data.time?.created ?? Date.now(),
time_updated: data.time?.updated ?? Date.now(),
time_initialized: data.time?.initialized,
sandboxes: data.sandboxes ?? [],
})
.onConflictDoNothing()
.run()
stats.projects++
} catch (e) {
stats.errors.push(`failed to migrate project ${file}: ${e}`)

View File

@ -1,6 +1,4 @@
// Auto-generated - do not edit
import m0 from "../../drizzle/0000_initial.sql" with { type: "text" }
import m0 from "../../migration/0000_easy_albert_cleary.sql" with { type: "text" }
export const migrations = [
{ name: "0000_initial", sql: m0 },
]
export const migrations = [{ name: "0000_easy_albert_cleary", sql: m0 }]

View File

@ -102,7 +102,7 @@ describe("Project.discover", () => {
await Project.discover(project)
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, project.id)).get()
const updated = row?.data
const updated = row ? Project.fromRow(row) : undefined
expect(updated?.icon).toBeDefined()
expect(updated?.icon?.url).toStartWith("data:")
expect(updated?.icon?.url).toContain("base64")
@ -118,7 +118,7 @@ describe("Project.discover", () => {
await Project.discover(project)
const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, project.id)).get()
const updated = row?.data
const updated = row ? Project.fromRow(row) : undefined
expect(updated?.icon).toBeUndefined()
})
})

View File

@ -7,6 +7,7 @@ import fs from "fs/promises"
import os from "os"
import { migrateFromJson } from "../../src/storage/json-migration"
import { ProjectTable } from "../../src/project/project.sql"
import { Project } from "../../src/project/project"
import {
SessionTable,
MessageTable,
@ -198,8 +199,9 @@ describe("JSON to SQLite migration", () => {
expect(stats?.projects).toBe(1)
const db = drizzle(sqlite)
const row = db.select().from(ProjectTable).where(eq(ProjectTable.id, project.id)).get()
expect(row?.data.id).toBe(project.id)
expect(row?.data.icon?.url).toBe(project.icon.url)
const migrated = row ? Project.fromRow(row) : undefined
expect(migrated?.id).toBe(project.id)
expect(migrated?.icon?.url).toBe(project.icon.url)
})
test("skips project with missing id field", async () => {
@ -583,10 +585,11 @@ describe("JSON to SQLite migration", () => {
const db = drizzle(sqlite)
const row = db.select().from(ProjectTable).where(eq(ProjectTable.id, fullProject.id)).get()
expect(row?.data.id).toBe(fullProject.id)
expect(row?.data.name).toBe(fullProject.name)
expect(row?.data.sandboxes).toEqual(fullProject.sandboxes)
expect(row?.data.icon?.color).toBe("#ff0000")
const data = row ? Project.fromRow(row) : undefined
expect(data?.id).toBe(fullProject.id)
expect(data?.name).toBe(fullProject.name)
expect(data?.sandboxes).toEqual(fullProject.sandboxes)
expect(data?.icon?.color).toBe("#ff0000")
})
test("handles unicode in text fields", async () => {
@ -603,8 +606,9 @@ describe("JSON to SQLite migration", () => {
const db = drizzle(sqlite)
const row = db.select().from(ProjectTable).where(eq(ProjectTable.id, unicodeProject.id)).get()
expect(row?.data.name).toBe("Проект с юникодом 🚀")
expect(row?.data.worktree).toBe("/path/测试")
const data = row ? Project.fromRow(row) : undefined
expect(data?.name).toBe("Проект с юникодом 🚀")
expect(data?.worktree).toBe("/path/测试")
})
test("migration is idempotent with onConflictDoNothing", async () => {