diff --git a/packages/opencode/.gitignore b/packages/opencode/.gitignore index 54ac0e8617..e057ca61f9 100644 --- a/packages/opencode/.gitignore +++ b/packages/opencode/.gitignore @@ -2,4 +2,3 @@ research dist gen app.log -src/storage/migrations.generated.ts diff --git a/packages/opencode/drizzle.config.ts b/packages/opencode/drizzle.config.ts index 191a853409..1c2bd77f9e 100644 --- a/packages/opencode/drizzle.config.ts +++ b/packages/opencode/drizzle.config.ts @@ -2,13 +2,6 @@ import { defineConfig } from "drizzle-kit" export default defineConfig({ dialect: "sqlite", - schema: [ - "./src/project/project.sql.ts", - "./src/session/session.sql.ts", - "./src/session/message.sql.ts", - "./src/session/part.sql.ts", - "./src/session/session-aux.sql.ts", - "./src/share/share.sql.ts", - ], + schema: "./src/**/*.sql.ts", out: "./drizzle", }) diff --git a/packages/opencode/src/cli/cmd/database.ts b/packages/opencode/src/cli/cmd/database.ts new file mode 100644 index 0000000000..15905e9161 --- /dev/null +++ b/packages/opencode/src/cli/cmd/database.ts @@ -0,0 +1,144 @@ +import type { Argv } from "yargs" +import { cmd } from "./cmd" +import { bootstrap } from "../bootstrap" +import { UI } from "../ui" +import { db } from "../../storage/db" +import { ProjectTable } from "../../project/project.sql" +import { + SessionTable, + MessageTable, + PartTable, + SessionDiffTable, + TodoTable, + PermissionTable, +} from "../../session/session.sql" +import { SessionShareTable, ShareTable } from "../../share/share.sql" +import path from "path" +import fs from "fs/promises" + +export const DatabaseCommand = cmd({ + command: "database", + describe: "database management commands", + builder: (yargs) => yargs.command(ExportCommand).demandCommand(), + async handler() {}, +}) + +const ExportCommand = cmd({ + command: "export", + describe: "export database to JSON files", + builder: (yargs: Argv) => { + return yargs.option("output", { + alias: ["o"], + describe: "output directory", + type: "string", + demandOption: true, + }) + }, + handler: async (args) => { + await bootstrap(process.cwd(), async () => { + const outDir = path.resolve(args.output) + await fs.mkdir(outDir, { recursive: true }) + + const stats = { + projects: 0, + sessions: 0, + messages: 0, + parts: 0, + diffs: 0, + todos: 0, + permissions: 0, + sessionShares: 0, + shares: 0, + } + + // Export projects + const projectDir = path.join(outDir, "project") + await fs.mkdir(projectDir, { recursive: true }) + for (const row of db().select().from(ProjectTable).all()) { + await Bun.write(path.join(projectDir, `${row.id}.json`), JSON.stringify(row.data, null, 2)) + stats.projects++ + } + + // Export sessions (organized by projectID) + const sessionDir = path.join(outDir, "session") + for (const row of db().select().from(SessionTable).all()) { + const dir = path.join(sessionDir, row.projectID) + await fs.mkdir(dir, { recursive: true }) + await Bun.write(path.join(dir, `${row.id}.json`), JSON.stringify(row.data, null, 2)) + stats.sessions++ + } + + // Export messages (organized by sessionID) + const messageDir = path.join(outDir, "message") + for (const row of db().select().from(MessageTable).all()) { + const dir = path.join(messageDir, row.sessionID) + await fs.mkdir(dir, { recursive: true }) + await Bun.write(path.join(dir, `${row.id}.json`), JSON.stringify(row.data, null, 2)) + stats.messages++ + } + + // Export parts (organized by messageID) + const partDir = path.join(outDir, "part") + for (const row of db().select().from(PartTable).all()) { + const dir = path.join(partDir, row.messageID) + await fs.mkdir(dir, { recursive: true }) + await Bun.write(path.join(dir, `${row.id}.json`), JSON.stringify(row.data, null, 2)) + stats.parts++ + } + + // Export session diffs + const diffDir = path.join(outDir, "session_diff") + await fs.mkdir(diffDir, { recursive: true }) + for (const row of db().select().from(SessionDiffTable).all()) { + await Bun.write(path.join(diffDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.diffs++ + } + + // Export todos + const todoDir = path.join(outDir, "todo") + await fs.mkdir(todoDir, { recursive: true }) + for (const row of db().select().from(TodoTable).all()) { + await Bun.write(path.join(todoDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.todos++ + } + + // Export permissions + const permDir = path.join(outDir, "permission") + await fs.mkdir(permDir, { recursive: true }) + for (const row of db().select().from(PermissionTable).all()) { + await Bun.write(path.join(permDir, `${row.projectID}.json`), JSON.stringify(row.data, null, 2)) + stats.permissions++ + } + + // Export session shares + const sessionShareDir = path.join(outDir, "session_share") + await fs.mkdir(sessionShareDir, { recursive: true }) + for (const row of db().select().from(SessionShareTable).all()) { + await Bun.write(path.join(sessionShareDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.sessionShares++ + } + + // Export shares + const shareDir = path.join(outDir, "share") + await fs.mkdir(shareDir, { recursive: true }) + for (const row of db().select().from(ShareTable).all()) { + await Bun.write(path.join(shareDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.shares++ + } + + // Create migration marker so this can be imported back + await Bun.write(path.join(outDir, "migration"), Date.now().toString()) + + UI.println(`Exported to ${outDir}:`) + UI.println(` ${stats.projects} projects`) + UI.println(` ${stats.sessions} sessions`) + UI.println(` ${stats.messages} messages`) + UI.println(` ${stats.parts} parts`) + UI.println(` ${stats.diffs} session diffs`) + UI.println(` ${stats.todos} todos`) + UI.println(` ${stats.permissions} permissions`) + UI.println(` ${stats.sessionShares} session shares`) + UI.println(` ${stats.shares} shares`) + }) + }, +}) diff --git a/packages/opencode/src/cli/cmd/import.ts b/packages/opencode/src/cli/cmd/import.ts index f612c0bea9..6980e0b954 100644 --- a/packages/opencode/src/cli/cmd/import.ts +++ b/packages/opencode/src/cli/cmd/import.ts @@ -3,9 +3,7 @@ import { Session } from "../../session" import { cmd } from "./cmd" import { bootstrap } from "../bootstrap" import { db } from "../../storage/db" -import { SessionTable } from "../../session/session.sql" -import { MessageTable } from "../../session/message.sql" -import { PartTable } from "../../session/part.sql" +import { SessionTable, MessageTable, PartTable } from "../../session/session.sql" import { Instance } from "../../project/instance" import { EOL } from "os" diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 6dc5e99e91..e73fda21b7 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -26,6 +26,7 @@ import { EOL } from "os" import { WebCommand } from "./cli/cmd/web" import { PrCommand } from "./cli/cmd/pr" import { SessionCommand } from "./cli/cmd/session" +import { DatabaseCommand } from "./cli/cmd/database" process.on("unhandledRejection", (e) => { Log.Default.error("rejection", { @@ -97,6 +98,7 @@ const cli = yargs(hideBin(process.argv)) .command(GithubCommand) .command(PrCommand) .command(SessionCommand) + .command(DatabaseCommand) .fail((msg, err) => { if ( msg?.startsWith("Unknown argument") || diff --git a/packages/opencode/src/permission/next.ts b/packages/opencode/src/permission/next.ts index 285d373e1d..2bd19b5e1d 100644 --- a/packages/opencode/src/permission/next.ts +++ b/packages/opencode/src/permission/next.ts @@ -4,7 +4,7 @@ import { Config } from "@/config/config" import { Identifier } from "@/id/id" import { Instance } from "@/project/instance" import { db } from "@/storage/db" -import { PermissionTable } from "@/session/session-aux.sql" +import { PermissionTable } from "@/session/session.sql" import { eq } from "drizzle-orm" import { fn } from "@/util/fn" import { Log } from "@/util/log" diff --git a/packages/opencode/src/session/index.ts b/packages/opencode/src/session/index.ts index 4691644a2b..57500682cd 100644 --- a/packages/opencode/src/session/index.ts +++ b/packages/opencode/src/session/index.ts @@ -11,10 +11,7 @@ import { Identifier } from "../id/id" import { Installation } from "../installation" import { db, NotFoundError } from "../storage/db" -import { SessionTable } from "./session.sql" -import { MessageTable } from "./message.sql" -import { PartTable } from "./part.sql" -import { SessionDiffTable } from "./session-aux.sql" +import { SessionTable, MessageTable, PartTable, SessionDiffTable } from "./session.sql" import { ShareTable } from "../share/share.sql" import { eq } from "drizzle-orm" import { Log } from "../util/log" diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 2c84801920..9642eb9e4d 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -7,8 +7,7 @@ import { LSP } from "../lsp" import { Snapshot } from "@/snapshot" import { fn } from "@/util/fn" import { db } from "@/storage/db" -import { MessageTable } from "./message.sql" -import { PartTable } from "./part.sql" +import { MessageTable, PartTable } from "./session.sql" import { eq, desc } from "drizzle-orm" import { ProviderTransform } from "@/provider/transform" import { STATUS_CODES } from "http" diff --git a/packages/opencode/src/session/message.sql.ts b/packages/opencode/src/session/message.sql.ts deleted file mode 100644 index c4ab82b222..0000000000 --- a/packages/opencode/src/session/message.sql.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core" -import { SessionTable } from "./session.sql" -import type { MessageV2 } from "./message-v2" - -export const MessageTable = sqliteTable( - "message", - { - id: text("id").primaryKey(), - sessionID: text("session_id") - .notNull() - .references(() => SessionTable.id, { onDelete: "cascade" }), - createdAt: integer("created_at").notNull(), - data: text("data", { mode: "json" }).notNull().$type(), - }, - (table) => [index("message_session_idx").on(table.sessionID)], -) diff --git a/packages/opencode/src/session/part.sql.ts b/packages/opencode/src/session/part.sql.ts deleted file mode 100644 index b73d874743..0000000000 --- a/packages/opencode/src/session/part.sql.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { sqliteTable, text, index } from "drizzle-orm/sqlite-core" -import { MessageTable } from "./message.sql" -import type { MessageV2 } from "./message-v2" - -export const PartTable = sqliteTable( - "part", - { - id: text("id").primaryKey(), - messageID: text("message_id") - .notNull() - .references(() => MessageTable.id, { onDelete: "cascade" }), - sessionID: text("session_id").notNull(), - data: text("data", { mode: "json" }).notNull().$type(), - }, - (table) => [index("part_message_idx").on(table.messageID), index("part_session_idx").on(table.sessionID)], -) diff --git a/packages/opencode/src/session/revert.ts b/packages/opencode/src/session/revert.ts index 5951e339a8..9543e598a1 100644 --- a/packages/opencode/src/session/revert.ts +++ b/packages/opencode/src/session/revert.ts @@ -6,8 +6,7 @@ import { Session } from "." import { Log } from "../util/log" import { splitWhen } from "remeda" import { db } from "../storage/db" -import { MessageTable } from "./message.sql" -import { PartTable } from "./part.sql" +import { MessageTable, PartTable } from "./session.sql" import { eq } from "drizzle-orm" import { Bus } from "../bus" import { SessionPrompt } from "./prompt" diff --git a/packages/opencode/src/session/session-aux.sql.ts b/packages/opencode/src/session/session-aux.sql.ts deleted file mode 100644 index e43b8f00a4..0000000000 --- a/packages/opencode/src/session/session-aux.sql.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { sqliteTable, text } from "drizzle-orm/sqlite-core" -import { SessionTable } from "./session.sql" -import { ProjectTable } from "../project/project.sql" -import type { Snapshot } from "@/snapshot" -import type { Todo } from "./todo" -import type { PermissionNext } from "@/permission/next" - -export const SessionDiffTable = sqliteTable("session_diff", { - sessionID: text("session_id") - .primaryKey() - .references(() => SessionTable.id, { onDelete: "cascade" }), - data: text("data", { mode: "json" }).notNull().$type(), -}) - -export const TodoTable = sqliteTable("todo", { - sessionID: text("session_id") - .primaryKey() - .references(() => SessionTable.id, { onDelete: "cascade" }), - data: text("data", { mode: "json" }).notNull().$type(), -}) - -export const PermissionTable = sqliteTable("permission", { - projectID: text("project_id") - .primaryKey() - .references(() => ProjectTable.id, { onDelete: "cascade" }), - data: text("data", { mode: "json" }).notNull().$type(), -}) diff --git a/packages/opencode/src/session/session.sql.ts b/packages/opencode/src/session/session.sql.ts index b0080d915f..bb3a528e02 100644 --- a/packages/opencode/src/session/session.sql.ts +++ b/packages/opencode/src/session/session.sql.ts @@ -1,6 +1,10 @@ import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core" import { ProjectTable } from "../project/project.sql" import type { Session } from "./index" +import type { MessageV2 } from "./message-v2" +import type { Snapshot } from "@/snapshot" +import type { Todo } from "./todo" +import type { PermissionNext } from "@/permission/next" export const SessionTable = sqliteTable( "session", @@ -16,3 +20,50 @@ export const SessionTable = sqliteTable( }, (table) => [index("session_project_idx").on(table.projectID), index("session_parent_idx").on(table.parentID)], ) + +export const MessageTable = sqliteTable( + "message", + { + id: text("id").primaryKey(), + sessionID: text("session_id") + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + createdAt: integer("created_at").notNull(), + data: text("data", { mode: "json" }).notNull().$type(), + }, + (table) => [index("message_session_idx").on(table.sessionID)], +) + +export const PartTable = sqliteTable( + "part", + { + id: text("id").primaryKey(), + messageID: text("message_id") + .notNull() + .references(() => MessageTable.id, { onDelete: "cascade" }), + sessionID: text("session_id").notNull(), + data: text("data", { mode: "json" }).notNull().$type(), + }, + (table) => [index("part_message_idx").on(table.messageID), index("part_session_idx").on(table.sessionID)], +) + +export const SessionDiffTable = sqliteTable("session_diff", { + sessionID: text("session_id") + .primaryKey() + .references(() => SessionTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type(), +}) + +export const TodoTable = sqliteTable("todo", { + sessionID: text("session_id") + .primaryKey() + .references(() => SessionTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type(), +}) + +export const PermissionTable = sqliteTable("permission", { + projectID: text("project_id") + .primaryKey() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type(), +}) diff --git a/packages/opencode/src/session/summary.ts b/packages/opencode/src/session/summary.ts index d3bc10dbd0..a79850046d 100644 --- a/packages/opencode/src/session/summary.ts +++ b/packages/opencode/src/session/summary.ts @@ -12,7 +12,7 @@ import { Log } from "@/util/log" import path from "path" import { Instance } from "@/project/instance" import { db } from "@/storage/db" -import { SessionDiffTable } from "./session-aux.sql" +import { SessionDiffTable } from "./session.sql" import { eq } from "drizzle-orm" import { Bus } from "@/bus" diff --git a/packages/opencode/src/session/todo.ts b/packages/opencode/src/session/todo.ts index f32c2ee71e..3280744662 100644 --- a/packages/opencode/src/session/todo.ts +++ b/packages/opencode/src/session/todo.ts @@ -2,7 +2,7 @@ import { BusEvent } from "@/bus/bus-event" import { Bus } from "@/bus" import z from "zod" import { db } from "../storage/db" -import { TodoTable } from "./session-aux.sql" +import { TodoTable } from "./session.sql" import { eq } from "drizzle-orm" export namespace Todo { diff --git a/packages/opencode/src/storage/json-migration.ts b/packages/opencode/src/storage/json-migration.ts index cc7923f691..436385ee7b 100644 --- a/packages/opencode/src/storage/json-migration.ts +++ b/packages/opencode/src/storage/json-migration.ts @@ -4,10 +4,14 @@ import { eq } from "drizzle-orm" import { Global } from "../global" import { Log } from "../util/log" import { ProjectTable } from "../project/project.sql" -import { SessionTable } from "../session/session.sql" -import { MessageTable } from "../session/message.sql" -import { PartTable } from "../session/part.sql" -import { SessionDiffTable, TodoTable, PermissionTable } from "../session/session-aux.sql" +import { + SessionTable, + MessageTable, + PartTable, + SessionDiffTable, + TodoTable, + PermissionTable, +} from "../session/session.sql" import { SessionShareTable, ShareTable } from "../share/share.sql" import path from "path" diff --git a/packages/opencode/src/storage/migrations.generated.ts b/packages/opencode/src/storage/migrations.generated.ts new file mode 100644 index 0000000000..daa960bce6 --- /dev/null +++ b/packages/opencode/src/storage/migrations.generated.ts @@ -0,0 +1,6 @@ +// Auto-generated - do not edit +import m0 from "../../drizzle/0000_initial.sql" with { type: "text" } + +export const migrations = [ + { name: "0000_initial", sql: m0 }, +] diff --git a/packages/opencode/test/storage/json-migration.test.ts b/packages/opencode/test/storage/json-migration.test.ts index 92a8a5a204..e6771d57a0 100644 --- a/packages/opencode/test/storage/json-migration.test.ts +++ b/packages/opencode/test/storage/json-migration.test.ts @@ -7,10 +7,14 @@ import fs from "fs/promises" import os from "os" import { migrateFromJson } from "../../src/storage/json-migration" import { ProjectTable } from "../../src/project/project.sql" -import { SessionTable } from "../../src/session/session.sql" -import { MessageTable } from "../../src/session/message.sql" -import { PartTable } from "../../src/session/part.sql" -import { SessionDiffTable, TodoTable, PermissionTable } from "../../src/session/session-aux.sql" +import { + SessionTable, + MessageTable, + PartTable, + SessionDiffTable, + TodoTable, + PermissionTable, +} from "../../src/session/session.sql" import { SessionShareTable, ShareTable } from "../../src/share/share.sql" import { migrations } from "../../src/storage/migrations.generated"