From 1e7b4768b13d90043fc29377fdfa573b3c0373d0 Mon Sep 17 00:00:00 2001 From: Dax Raad Date: Sat, 24 Jan 2026 11:50:25 -0500 Subject: [PATCH] sync --- bun.lock | 58 ++ packages/opencode/drizzle.config.ts | 7 + .../migration/0000_magical_strong_guy.sql | 87 +++ .../migration/meta/0000_snapshot.json | 587 ++++++++++++++++++ .../opencode/migration/meta/_journal.json | 13 + packages/opencode/package.json | 2 + packages/opencode/script/check-migrations.ts | 16 + .../opencode/script/generate-migrations.ts | 49 ++ packages/opencode/src/cli/cmd/database.ts | 147 +++++ packages/opencode/src/cli/cmd/import.ts | 27 +- packages/opencode/src/cli/cmd/stats.ts | 24 +- packages/opencode/src/index.ts | 2 + packages/opencode/src/permission/next.ts | 15 +- packages/opencode/src/project/project.sql.ts | 14 + packages/opencode/src/project/project.ts | 204 +++--- packages/opencode/src/server/error.ts | 4 +- packages/opencode/src/server/routes/pty.ts | 4 +- packages/opencode/src/server/server.ts | 4 +- packages/opencode/src/session/index.ts | 163 +++-- packages/opencode/src/session/message-v2.ts | 32 +- packages/opencode/src/session/revert.ts | 14 +- packages/opencode/src/session/session.sql.ts | 83 +++ packages/opencode/src/session/summary.ts | 13 +- packages/opencode/src/session/todo.ts | 19 +- packages/opencode/src/share/share-next.ts | 23 +- packages/opencode/src/share/share.sql.ts | 19 + packages/opencode/src/sql.d.ts | 4 + packages/opencode/src/storage/db.ts | 87 +++ .../opencode/src/storage/json-migration.ts | 300 +++++++++ .../src/storage/migrations.generated.ts | 6 + packages/opencode/src/storage/storage.ts | 227 ------- packages/opencode/src/util/lazy.ts | 11 +- packages/opencode/src/worktree/index.ts | 7 +- .../opencode/test/permission/next.test.ts | 1 - packages/opencode/test/preload.ts | 14 +- .../opencode/test/project/project.test.ts | 17 +- .../test/storage/json-migration.test.ts | 279 +++++++++ 37 files changed, 2151 insertions(+), 432 deletions(-) create mode 100644 packages/opencode/drizzle.config.ts create mode 100644 packages/opencode/migration/0000_magical_strong_guy.sql create mode 100644 packages/opencode/migration/meta/0000_snapshot.json create mode 100644 packages/opencode/migration/meta/_journal.json create mode 100644 packages/opencode/script/check-migrations.ts create mode 100644 packages/opencode/script/generate-migrations.ts create mode 100644 packages/opencode/src/cli/cmd/database.ts create mode 100644 packages/opencode/src/project/project.sql.ts create mode 100644 packages/opencode/src/session/session.sql.ts create mode 100644 packages/opencode/src/share/share.sql.ts create mode 100644 packages/opencode/src/sql.d.ts create mode 100644 packages/opencode/src/storage/db.ts create mode 100644 packages/opencode/src/storage/json-migration.ts create mode 100644 packages/opencode/src/storage/migrations.generated.ts delete mode 100644 packages/opencode/src/storage/storage.ts create mode 100644 packages/opencode/test/storage/json-migration.test.ts diff --git a/bun.lock b/bun.lock index 34a6488ba0..6211ff1482 100644 --- a/bun.lock +++ b/bun.lock @@ -311,6 +311,7 @@ "clipboardy": "4.0.0", "decimal.js": "10.5.0", "diff": "catalog:", + "drizzle-orm": "0.44.2", "fuzzysort": "3.1.0", "gray-matter": "4.0.3", "hono": "catalog:", @@ -352,6 +353,7 @@ "@types/turndown": "5.0.5", "@types/yargs": "17.0.33", "@typescript/native-preview": "catalog:", + "drizzle-kit": "0.31.0", "typescript": "catalog:", "vscode-languageserver-types": "3.17.5", "why-is-node-running": "3.2.2", @@ -4414,6 +4416,10 @@ "opencode/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@1.0.30", "", { "dependencies": { "@ai-sdk/provider": "2.0.1", "@ai-sdk/provider-utils": "3.0.20" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-thubwhRtv9uicAxSWwNpinM7hiL/0CkhL/ymPaHuKvI494J7HIzn8KQZQ2ymRz284WTIZnI7VMyyejxW4RMM6w=="], + "opencode/drizzle-kit": ["drizzle-kit@0.31.0", "", { "dependencies": { "@drizzle-team/brocli": "^0.10.2", "@esbuild-kit/esm-loader": "^2.5.5", "esbuild": "^0.25.2", "esbuild-register": "^3.5.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-pcKVT+GbfPA+bUovPIilgVOoq+onNBo/YQBG86sf3/GFHkN6lRJPm1l7dKN0IMAk57RQoIm4GUllRrasLlcaSg=="], + + "opencode/drizzle-orm": ["drizzle-orm@0.44.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-zGAqBzWWkVSFjZpwPOrmCrgO++1kZ5H/rZ4qTGeGOe18iXGVJWf3WPfHOVwFIbmi8kHjfJstC6rJomzGx8g/dQ=="], + "opencontrol/@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.6.1", "", { "dependencies": { "content-type": "^1.0.5", "cors": "^2.8.5", "eventsource": "^3.0.2", "express": "^5.0.1", "express-rate-limit": "^7.5.0", "pkce-challenge": "^4.1.0", "raw-body": "^3.0.0", "zod": "^3.23.8", "zod-to-json-schema": "^3.24.1" } }, "sha512-oxzMzYCkZHMntzuyerehK3fV6A2Kwh5BD6CGEJSVDU2QNEhfLOptf2X7esQgaHZXHZY0oHmMsOtIDLP71UJXgA=="], "opencontrol/@tsconfig/bun": ["@tsconfig/bun@1.0.7", "", {}, "sha512-udGrGJBNQdXGVulehc1aWT73wkR9wdaGBtB6yL70RJsqwW/yJhIg6ZbRlPOfIUiFNrnBuYLBi9CSmMKfDC7dvA=="], @@ -5020,6 +5026,8 @@ "lazystream/readable-stream/string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], + "opencode/drizzle-kit/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "opencontrol/@modelcontextprotocol/sdk/express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="], "opencontrol/@modelcontextprotocol/sdk/pkce-challenge": ["pkce-challenge@4.1.0", "", {}, "sha512-ZBmhE1C9LcPoH9XZSdwiPtbPHZROwAnMy+kIFQVrnMCxY4Cudlz3gBOpzilgc0jOgRaiT3sIWfpMomW2ar2orQ=="], @@ -5192,6 +5200,56 @@ "js-beautify/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "opencode/drizzle-kit/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], + + "opencode/drizzle-kit/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.12", "", { "os": "android", "cpu": "arm64" }, "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.25.12", "", { "os": "android", "cpu": "x64" }, "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA=="], + + "opencode/drizzle-kit/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.12", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.12", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.12", "", { "os": "linux", "cpu": "arm" }, "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.12", "", { "os": "linux", "cpu": "ia32" }, "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.12", "", { "os": "linux", "cpu": "ppc64" }, "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.12", "", { "os": "linux", "cpu": "none" }, "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.12", "", { "os": "linux", "cpu": "s390x" }, "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.12", "", { "os": "linux", "cpu": "x64" }, "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw=="], + + "opencode/drizzle-kit/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.12", "", { "os": "none", "cpu": "arm64" }, "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.12", "", { "os": "none", "cpu": "x64" }, "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ=="], + + "opencode/drizzle-kit/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.12", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A=="], + + "opencode/drizzle-kit/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.12", "", { "os": "openbsd", "cpu": "x64" }, "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw=="], + + "opencode/drizzle-kit/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.12", "", { "os": "sunos", "cpu": "x64" }, "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w=="], + + "opencode/drizzle-kit/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg=="], + + "opencode/drizzle-kit/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.12", "", { "os": "win32", "cpu": "ia32" }, "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ=="], + + "opencode/drizzle-kit/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "opencontrol/@modelcontextprotocol/sdk/express/accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], "opencontrol/@modelcontextprotocol/sdk/express/body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="], diff --git a/packages/opencode/drizzle.config.ts b/packages/opencode/drizzle.config.ts new file mode 100644 index 0000000000..551a2384c5 --- /dev/null +++ b/packages/opencode/drizzle.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "drizzle-kit" + +export default defineConfig({ + dialect: "sqlite", + schema: "./src/**/*.sql.ts", + out: "./migration", +}) diff --git a/packages/opencode/migration/0000_magical_strong_guy.sql b/packages/opencode/migration/0000_magical_strong_guy.sql new file mode 100644 index 0000000000..e25f0d3d56 --- /dev/null +++ b/packages/opencode/migration/0000_magical_strong_guy.sql @@ -0,0 +1,87 @@ +CREATE TABLE `project` ( + `id` text PRIMARY KEY NOT NULL, + `worktree` text NOT NULL, + `vcs` text, + `name` text, + `icon_url` text, + `icon_color` text, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + `time_initialized` integer, + `sandboxes` text NOT NULL +); +--> statement-breakpoint +CREATE TABLE `message` ( + `id` text PRIMARY KEY NOT NULL, + `session_id` text NOT NULL, + `created_at` integer NOT NULL, + `data` text NOT NULL, + FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `message_session_idx` ON `message` (`session_id`);--> statement-breakpoint +CREATE TABLE `part` ( + `id` text PRIMARY KEY NOT NULL, + `message_id` text NOT NULL, + `session_id` text NOT NULL, + `data` text NOT NULL, + FOREIGN KEY (`message_id`) REFERENCES `message`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `part_message_idx` ON `part` (`message_id`);--> statement-breakpoint +CREATE INDEX `part_session_idx` ON `part` (`session_id`);--> statement-breakpoint +CREATE TABLE `permission` ( + `project_id` text PRIMARY KEY NOT NULL, + `data` text NOT NULL, + FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `session_diff` ( + `session_id` text PRIMARY KEY NOT NULL, + `data` text NOT NULL, + FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `session` ( + `id` text PRIMARY KEY NOT NULL, + `project_id` text NOT NULL, + `parent_id` text, + `slug` text NOT NULL, + `directory` text NOT NULL, + `title` text NOT NULL, + `version` text NOT NULL, + `share_url` text, + `summary_additions` integer, + `summary_deletions` integer, + `summary_files` integer, + `summary_diffs` text, + `revert_message_id` text, + `revert_part_id` text, + `revert_snapshot` text, + `revert_diff` text, + `permission` text, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + `time_compacting` integer, + `time_archived` integer, + FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `session_project_idx` ON `session` (`project_id`);--> statement-breakpoint +CREATE INDEX `session_parent_idx` ON `session` (`parent_id`);--> statement-breakpoint +CREATE TABLE `todo` ( + `session_id` text PRIMARY KEY NOT NULL, + `data` text NOT NULL, + FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `session_share` ( + `session_id` text PRIMARY KEY NOT NULL, + `data` text NOT NULL, + FOREIGN KEY (`session_id`) REFERENCES `session`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `share` ( + `session_id` text PRIMARY KEY NOT NULL, + `data` text NOT NULL +); diff --git a/packages/opencode/migration/meta/0000_snapshot.json b/packages/opencode/migration/meta/0000_snapshot.json new file mode 100644 index 0000000000..efec141ea6 --- /dev/null +++ b/packages/opencode/migration/meta/0000_snapshot.json @@ -0,0 +1,587 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "797eb060-2c45-4abf-925d-6b8375dd8a64", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "project": { + "name": "project", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "worktree": { + "name": "worktree", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "vcs": { + "name": "vcs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_url": { + "name": "icon_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon_color": { + "name": "icon_color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "time_created": { + "name": "time_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "time_updated": { + "name": "time_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "time_initialized": { + "name": "time_initialized", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sandboxes": { + "name": "sandboxes", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "message": { + "name": "message", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "message_session_idx": { + "name": "message_session_idx", + "columns": [ + "session_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "message_session_id_session_id_fk": { + "name": "message_session_id_session_id_fk", + "tableFrom": "message", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "part": { + "name": "part", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "part_message_idx": { + "name": "part_message_idx", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "part_session_idx": { + "name": "part_session_idx", + "columns": [ + "session_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "part_message_id_message_id_fk": { + "name": "part_message_id_message_id_fk", + "tableFrom": "part", + "tableTo": "message", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "permission": { + "name": "permission", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "permission_project_id_project_id_fk": { + "name": "permission_project_id_project_id_fk", + "tableFrom": "permission", + "tableTo": "project", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session_diff": { + "name": "session_diff", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_diff_session_id_session_id_fk": { + "name": "session_diff_session_id_session_id_fk", + "tableFrom": "session_diff", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "directory": { + "name": "directory", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "share_url": { + "name": "share_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_additions": { + "name": "summary_additions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_deletions": { + "name": "summary_deletions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_files": { + "name": "summary_files", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary_diffs": { + "name": "summary_diffs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revert_message_id": { + "name": "revert_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revert_part_id": { + "name": "revert_part_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revert_snapshot": { + "name": "revert_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revert_diff": { + "name": "revert_diff", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permission": { + "name": "permission", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "time_created": { + "name": "time_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "time_updated": { + "name": "time_updated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "time_compacting": { + "name": "time_compacting", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "time_archived": { + "name": "time_archived", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_project_idx": { + "name": "session_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "session_parent_idx": { + "name": "session_parent_idx", + "columns": [ + "parent_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_project_id_project_id_fk": { + "name": "session_project_id_project_id_fk", + "tableFrom": "session", + "tableTo": "project", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "todo": { + "name": "todo", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "todo_session_id_session_id_fk": { + "name": "todo_session_id_session_id_fk", + "tableFrom": "todo", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session_share": { + "name": "session_share", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_share_session_id_session_id_fk": { + "name": "session_share_session_id_session_id_fk", + "tableFrom": "session_share", + "tableTo": "session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share": { + "name": "share", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/opencode/migration/meta/_journal.json b/packages/opencode/migration/meta/_journal.json new file mode 100644 index 0000000000..4ab81e184d --- /dev/null +++ b/packages/opencode/migration/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1769232577135, + "tag": "0000_magical_strong_guy", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 822b581dd7..6c3f3a5cca 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -25,6 +25,7 @@ }, "devDependencies": { "@babel/core": "7.28.4", + "drizzle-kit": "0.31.0", "@octokit/webhooks-types": "7.6.1", "@opencode-ai/script": "workspace:*", "@parcel/watcher-darwin-arm64": "2.5.1", @@ -96,6 +97,7 @@ "chokidar": "4.0.3", "clipboardy": "4.0.0", "decimal.js": "10.5.0", + "drizzle-orm": "0.44.2", "diff": "catalog:", "fuzzysort": "3.1.0", "gray-matter": "4.0.3", diff --git a/packages/opencode/script/check-migrations.ts b/packages/opencode/script/check-migrations.ts new file mode 100644 index 0000000000..f5eaf79323 --- /dev/null +++ b/packages/opencode/script/check-migrations.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env bun + +import { $ } from "bun" + +// drizzle-kit check compares schema to migrations, exits non-zero if drift +const result = await $`bun drizzle-kit check`.quiet().nothrow() + +if (result.exitCode !== 0) { + console.error("Schema has changes not captured in migrations!") + console.error("Run: bun drizzle-kit generate") + console.error("") + console.error(result.stderr.toString()) + process.exit(1) +} + +console.log("Migrations are up to date") diff --git a/packages/opencode/script/generate-migrations.ts b/packages/opencode/script/generate-migrations.ts new file mode 100644 index 0000000000..47c2e0c5e1 --- /dev/null +++ b/packages/opencode/script/generate-migrations.ts @@ -0,0 +1,49 @@ +#!/usr/bin/env bun + +import { Glob } from "bun" +import path from "path" +import fs from "fs" + +const migrationsDir = "./migration" +const outFile = "./src/storage/migrations.generated.ts" + +if (!fs.existsSync(migrationsDir)) { + console.log("No migrations directory found, creating empty migrations file") + await Bun.write( + outFile, + `// Auto-generated - do not edit +export const migrations: { name: string; sql: string }[] = [] +`, + ) + process.exit(0) +} + +const files = Array.from(new Glob("*.sql").scanSync({ cwd: migrationsDir })).sort() + +if (files.length === 0) { + console.log("No migrations found, creating empty migrations file") + await Bun.write( + outFile, + `// Auto-generated - do not edit +export const migrations: { name: string; sql: string }[] = [] +`, + ) + process.exit(0) +} + +const imports = files.map((f, i) => `import m${i} from "../../migration/${f}" with { type: "text" }`).join("\n") + +const entries = files.map((f, i) => ` { name: "${path.basename(f, ".sql")}", sql: m${i} },`).join("\n") + +await Bun.write( + outFile, + `// Auto-generated - do not edit +${imports} + +export const migrations = [ +${entries} +] +`, +) + +console.log(`Generated migrations file with ${files.length} migrations`) diff --git a/packages/opencode/src/cli/cmd/database.ts b/packages/opencode/src/cli/cmd/database.ts new file mode 100644 index 0000000000..5b3c1485f3 --- /dev/null +++ b/packages/opencode/src/cli/cmd/database.ts @@ -0,0 +1,147 @@ +import type { Argv } from "yargs" +import { cmd } from "./cmd" +import { bootstrap } from "../bootstrap" +import { UI } from "../ui" +import { db } from "../../storage/db" +import { ProjectTable } from "../../project/project.sql" +import { Project } from "../../project/project" +import { + SessionTable, + MessageTable, + PartTable, + SessionDiffTable, + TodoTable, + PermissionTable, +} from "../../session/session.sql" +import { Session } from "../../session" +import { SessionShareTable, ShareTable } from "../../share/share.sql" +import path from "path" +import fs from "fs/promises" + +export const DatabaseCommand = cmd({ + command: "database", + describe: "database management commands", + builder: (yargs) => yargs.command(ExportCommand).demandCommand(), + async handler() {}, +}) + +const ExportCommand = cmd({ + command: "export", + describe: "export database to JSON files", + builder: (yargs: Argv) => { + return yargs.option("output", { + alias: ["o"], + describe: "output directory", + type: "string", + demandOption: true, + }) + }, + handler: async (args) => { + await bootstrap(process.cwd(), async () => { + const outDir = path.resolve(args.output) + await fs.mkdir(outDir, { recursive: true }) + + const stats = { + projects: 0, + sessions: 0, + messages: 0, + parts: 0, + diffs: 0, + todos: 0, + permissions: 0, + sessionShares: 0, + shares: 0, + } + + // Export projects + const projectDir = path.join(outDir, "project") + await fs.mkdir(projectDir, { recursive: true }) + for (const row of db().select().from(ProjectTable).all()) { + const project = Project.fromRow(row) + await Bun.write(path.join(projectDir, `${row.id}.json`), JSON.stringify(project, null, 2)) + stats.projects++ + } + + // Export sessions (organized by projectID) + const sessionDir = path.join(outDir, "session") + for (const row of db().select().from(SessionTable).all()) { + const dir = path.join(sessionDir, row.projectID) + await fs.mkdir(dir, { recursive: true }) + await Bun.write(path.join(dir, `${row.id}.json`), JSON.stringify(Session.fromRow(row), null, 2)) + stats.sessions++ + } + + // Export messages (organized by sessionID) + const messageDir = path.join(outDir, "message") + for (const row of db().select().from(MessageTable).all()) { + const dir = path.join(messageDir, row.sessionID) + await fs.mkdir(dir, { recursive: true }) + await Bun.write(path.join(dir, `${row.id}.json`), JSON.stringify(row.data, null, 2)) + stats.messages++ + } + + // Export parts (organized by messageID) + const partDir = path.join(outDir, "part") + for (const row of db().select().from(PartTable).all()) { + const dir = path.join(partDir, row.messageID) + await fs.mkdir(dir, { recursive: true }) + await Bun.write(path.join(dir, `${row.id}.json`), JSON.stringify(row.data, null, 2)) + stats.parts++ + } + + // Export session diffs + const diffDir = path.join(outDir, "session_diff") + await fs.mkdir(diffDir, { recursive: true }) + for (const row of db().select().from(SessionDiffTable).all()) { + await Bun.write(path.join(diffDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.diffs++ + } + + // Export todos + const todoDir = path.join(outDir, "todo") + await fs.mkdir(todoDir, { recursive: true }) + for (const row of db().select().from(TodoTable).all()) { + await Bun.write(path.join(todoDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.todos++ + } + + // Export permissions + const permDir = path.join(outDir, "permission") + await fs.mkdir(permDir, { recursive: true }) + for (const row of db().select().from(PermissionTable).all()) { + await Bun.write(path.join(permDir, `${row.projectID}.json`), JSON.stringify(row.data, null, 2)) + stats.permissions++ + } + + // Export session shares + const sessionShareDir = path.join(outDir, "session_share") + await fs.mkdir(sessionShareDir, { recursive: true }) + for (const row of db().select().from(SessionShareTable).all()) { + await Bun.write(path.join(sessionShareDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.sessionShares++ + } + + // Export shares + const shareDir = path.join(outDir, "share") + await fs.mkdir(shareDir, { recursive: true }) + for (const row of db().select().from(ShareTable).all()) { + await Bun.write(path.join(shareDir, `${row.sessionID}.json`), JSON.stringify(row.data, null, 2)) + stats.shares++ + } + + // Create migration marker so this can be imported back + await Bun.write(path.join(outDir, "migration"), Date.now().toString()) + + UI.println(`Exported to ${outDir}:`) + UI.println(` ${stats.projects} projects`) + UI.println(` ${stats.sessions} sessions`) + UI.println(` ${stats.messages} messages`) + UI.println(` ${stats.parts} parts`) + UI.println(` ${stats.diffs} session diffs`) + UI.println(` ${stats.todos} todos`) + UI.println(` ${stats.permissions} permissions`) + UI.println(` ${stats.sessionShares} session shares`) + UI.println(` ${stats.shares} shares`) + }) + }, +}) diff --git a/packages/opencode/src/cli/cmd/import.ts b/packages/opencode/src/cli/cmd/import.ts index 9d7e8c5617..c78776b9d7 100644 --- a/packages/opencode/src/cli/cmd/import.ts +++ b/packages/opencode/src/cli/cmd/import.ts @@ -2,7 +2,8 @@ import type { Argv } from "yargs" import { Session } from "../../session" import { cmd } from "./cmd" import { bootstrap } from "../bootstrap" -import { Storage } from "../../storage/storage" +import { db } from "../../storage/db" +import { SessionTable, MessageTable, PartTable } from "../../session/session.sql" import { Instance } from "../../project/instance" import { EOL } from "os" @@ -81,13 +82,31 @@ export const ImportCommand = cmd({ return } - await Storage.write(["session", Instance.project.id, exportData.info.id], exportData.info) + db().insert(SessionTable).values(Session.toRow(exportData.info)).onConflictDoNothing().run() for (const msg of exportData.messages) { - await Storage.write(["message", exportData.info.id, msg.info.id], msg.info) + db() + .insert(MessageTable) + .values({ + id: msg.info.id, + sessionID: exportData.info.id, + createdAt: msg.info.time?.created ?? Date.now(), + data: msg.info, + }) + .onConflictDoNothing() + .run() for (const part of msg.parts) { - await Storage.write(["part", msg.info.id, part.id], part) + db() + .insert(PartTable) + .values({ + id: part.id, + messageID: msg.info.id, + sessionID: exportData.info.id, + data: part, + }) + .onConflictDoNothing() + .run() } } diff --git a/packages/opencode/src/cli/cmd/stats.ts b/packages/opencode/src/cli/cmd/stats.ts index d78c4f0abd..21ee97fa82 100644 --- a/packages/opencode/src/cli/cmd/stats.ts +++ b/packages/opencode/src/cli/cmd/stats.ts @@ -2,7 +2,8 @@ import type { Argv } from "yargs" import { cmd } from "./cmd" import { Session } from "../../session" import { bootstrap } from "../bootstrap" -import { Storage } from "../../storage/storage" +import { db } from "../../storage/db" +import { SessionTable } from "../../session/session.sql" import { Project } from "../../project/project" import { Instance } from "../../project/instance" @@ -83,25 +84,8 @@ async function getCurrentProject(): Promise { } async function getAllSessions(): Promise { - const sessions: Session.Info[] = [] - - const projectKeys = await Storage.list(["project"]) - const projects = await Promise.all(projectKeys.map((key) => Storage.read(key))) - - for (const project of projects) { - if (!project) continue - - const sessionKeys = await Storage.list(["session", project.id]) - const projectSessions = await Promise.all(sessionKeys.map((key) => Storage.read(key))) - - for (const session of projectSessions) { - if (session) { - sessions.push(session) - } - } - } - - return sessions + const rows = db().select().from(SessionTable).all() + return rows.map((row) => Session.fromRow(row)) } export async function aggregateSessionStats(days?: number, projectFilter?: string): Promise { diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index 6dc5e99e91..e73fda21b7 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -26,6 +26,7 @@ import { EOL } from "os" import { WebCommand } from "./cli/cmd/web" import { PrCommand } from "./cli/cmd/pr" import { SessionCommand } from "./cli/cmd/session" +import { DatabaseCommand } from "./cli/cmd/database" process.on("unhandledRejection", (e) => { Log.Default.error("rejection", { @@ -97,6 +98,7 @@ const cli = yargs(hideBin(process.argv)) .command(GithubCommand) .command(PrCommand) .command(SessionCommand) + .command(DatabaseCommand) .fail((msg, err) => { if ( msg?.startsWith("Unknown argument") || diff --git a/packages/opencode/src/permission/next.ts b/packages/opencode/src/permission/next.ts index 2481f104ed..b625bb57fc 100644 --- a/packages/opencode/src/permission/next.ts +++ b/packages/opencode/src/permission/next.ts @@ -3,7 +3,9 @@ import { BusEvent } from "@/bus/bus-event" import { Config } from "@/config/config" import { Identifier } from "@/id/id" import { Instance } from "@/project/instance" -import { Storage } from "@/storage/storage" +import { db } from "@/storage/db" +import { PermissionTable } from "@/session/session.sql" +import { eq } from "drizzle-orm" import { fn } from "@/util/fn" import { Log } from "@/util/log" import { Wildcard } from "@/util/wildcard" @@ -105,9 +107,10 @@ export namespace PermissionNext { ), } - const state = Instance.state(async () => { + const state = Instance.state(() => { const projectID = Instance.project.id - const stored = await Storage.read(["permission", projectID]).catch(() => [] as Ruleset) + const row = db().select().from(PermissionTable).where(eq(PermissionTable.projectID, projectID)).get() + const stored = row?.data ?? ([] as Ruleset) const pending: Record< string, @@ -222,7 +225,8 @@ export namespace PermissionNext { // TODO: we don't save the permission ruleset to disk yet until there's // UI to manage it - // await Storage.write(["permission", Instance.project.id], s.approved) + // db().insert(PermissionTable).values({ projectID: Instance.project.id, data: s.approved }) + // .onConflictDoUpdate({ target: PermissionTable.projectID, set: { data: s.approved } }).run() return } }, @@ -275,6 +279,7 @@ export namespace PermissionNext { } export async function list() { - return state().then((x) => Object.values(x.pending).map((x) => x.info)) + const s = await state() + return Object.values(s.pending).map((x) => x.info) } } diff --git a/packages/opencode/src/project/project.sql.ts b/packages/opencode/src/project/project.sql.ts new file mode 100644 index 0000000000..651d537cf2 --- /dev/null +++ b/packages/opencode/src/project/project.sql.ts @@ -0,0 +1,14 @@ +import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core" + +export const ProjectTable = sqliteTable("project", { + id: text("id").primaryKey(), + worktree: text("worktree").notNull(), + vcs: text("vcs"), + name: text("name"), + icon_url: text("icon_url"), + icon_color: text("icon_color"), + time_created: integer("time_created").notNull(), + time_updated: integer("time_updated").notNull(), + time_initialized: integer("time_initialized"), + sandboxes: text("sandboxes", { mode: "json" }).notNull().$type(), +}) diff --git a/packages/opencode/src/project/project.ts b/packages/opencode/src/project/project.ts index f6902de4e1..71c6a9bc7b 100644 --- a/packages/opencode/src/project/project.ts +++ b/packages/opencode/src/project/project.ts @@ -3,10 +3,12 @@ import fs from "fs/promises" import { Filesystem } from "../util/filesystem" import path from "path" import { $ } from "bun" -import { Storage } from "../storage/storage" +import { db } from "../storage/db" +import { ProjectTable } from "./project.sql" +import { SessionTable } from "../session/session.sql" +import { eq } from "drizzle-orm" import { Log } from "../util/log" import { Flag } from "@/flag/flag" -import { Session } from "../session" import { work } from "../util/queue" import { fn } from "@opencode-ai/util/fn" import { BusEvent } from "@/bus/bus-event" @@ -50,6 +52,28 @@ export namespace Project { Updated: BusEvent.define("project.updated", Info), } + type Row = typeof ProjectTable.$inferSelect + + export function fromRow(row: Row): Info { + const icon = + row.icon_url || row.icon_color + ? { url: row.icon_url ?? undefined, color: row.icon_color ?? undefined } + : undefined + return { + id: row.id, + worktree: row.worktree, + vcs: row.vcs as Info["vcs"], + name: row.name ?? undefined, + icon, + time: { + created: row.time_created, + updated: row.time_updated, + initialized: row.time_initialized ?? undefined, + }, + sandboxes: row.sandboxes, + } + } + export async function fromDirectory(directory: string) { log.info("fromDirectory", { directory }) @@ -175,9 +199,10 @@ export namespace Project { } }) - let existing = await Storage.read(["project", id]).catch(() => undefined) - if (!existing) { - existing = { + const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, id)).get() + const existing = await iife(async () => { + if (row) return fromRow(row) + const fresh: Info = { id, worktree, vcs: vcs as Info["vcs"], @@ -190,10 +215,8 @@ export namespace Project { if (id !== "global") { await migrateFromGlobal(id, worktree) } - } - - // migrate old projects before sandboxes - if (!existing.sandboxes) existing.sandboxes = [] + return fresh + }) if (Flag.OPENCODE_EXPERIMENTAL_ICON_DISCOVERY) discover(existing) @@ -208,7 +231,29 @@ export namespace Project { } if (sandbox !== result.worktree && !result.sandboxes.includes(sandbox)) result.sandboxes.push(sandbox) result.sandboxes = result.sandboxes.filter((x) => existsSync(x)) - await Storage.write(["project", id], result) + const insert = { + id: result.id, + worktree: result.worktree, + vcs: result.vcs, + name: result.name, + icon_url: result.icon?.url, + icon_color: result.icon?.color, + time_created: result.time.created, + time_updated: result.time.updated, + time_initialized: result.time.initialized, + sandboxes: result.sandboxes, + } + const updateSet = { + worktree: result.worktree, + vcs: result.vcs, + name: result.name, + icon_url: result.icon?.url, + icon_color: result.icon?.color, + time_updated: result.time.updated, + time_initialized: result.time.initialized, + sandboxes: result.sandboxes, + } + db().insert(ProjectTable).values(insert).onConflictDoUpdate({ target: ProjectTable.id, set: updateSet }).run() GlobalBus.emit("event", { payload: { type: Event.Updated.type, @@ -249,42 +294,47 @@ export namespace Project { } async function migrateFromGlobal(newProjectID: string, worktree: string) { - const globalProject = await Storage.read(["project", "global"]).catch(() => undefined) - if (!globalProject) return + const globalRow = db().select().from(ProjectTable).where(eq(ProjectTable.id, "global")).get() + if (!globalRow) return - const globalSessions = await Storage.list(["session", "global"]).catch(() => []) + const globalSessions = db().select().from(SessionTable).where(eq(SessionTable.projectID, "global")).all() if (globalSessions.length === 0) return log.info("migrating sessions from global", { newProjectID, worktree, count: globalSessions.length }) - await work(10, globalSessions, async (key) => { - const sessionID = key[key.length - 1] - const session = await Storage.read(key).catch(() => undefined) - if (!session) return - if (session.directory && session.directory !== worktree) return + await work(10, globalSessions, async (row) => { + // Skip sessions that belong to a different directory + if (row.directory && row.directory !== worktree) return - session.projectID = newProjectID - log.info("migrating session", { sessionID, from: "global", to: newProjectID }) - await Storage.write(["session", newProjectID, sessionID], session) - await Storage.remove(key) + log.info("migrating session", { sessionID: row.id, from: "global", to: newProjectID }) + db().update(SessionTable).set({ projectID: newProjectID }).where(eq(SessionTable.id, row.id)).run() }).catch((error) => { log.error("failed to migrate sessions from global to project", { error, projectId: newProjectID }) }) } - export async function setInitialized(projectID: string) { - await Storage.update(["project", projectID], (draft) => { - draft.time.initialized = Date.now() - }) + export function setInitialized(projectID: string) { + db() + .update(ProjectTable) + .set({ + time_initialized: Date.now(), + }) + .where(eq(ProjectTable.id, projectID)) + .run() } - export async function list() { - const keys = await Storage.list(["project"]) - const projects = await Promise.all(keys.map((x) => Storage.read(x))) - return projects.map((project) => ({ - ...project, - sandboxes: project.sandboxes?.filter((x) => existsSync(x)), - })) + export function list() { + return db() + .select() + .from(ProjectTable) + .all() + .map((row) => fromRow(row)) + } + + export function get(projectID: string): Info | undefined { + const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get() + if (!row) return undefined + return fromRow(row) } export const update = fn( @@ -295,43 +345,35 @@ export namespace Project { commands: Info.shape.commands.optional(), }), async (input) => { - const result = await Storage.update(["project", input.projectID], (draft) => { - if (input.name !== undefined) draft.name = input.name - if (input.icon !== undefined) { - draft.icon = { - ...draft.icon, - } - if (input.icon.url !== undefined) draft.icon.url = input.icon.url - if (input.icon.override !== undefined) draft.icon.override = input.icon.override || undefined - if (input.icon.color !== undefined) draft.icon.color = input.icon.color - } - - if (input.commands?.start !== undefined) { - const start = input.commands.start || undefined - draft.commands = { - ...(draft.commands ?? {}), - } - draft.commands.start = start - if (!draft.commands.start) draft.commands = undefined - } - - draft.time.updated = Date.now() - }) + const result = db() + .update(ProjectTable) + .set({ + name: input.name, + icon_url: input.icon?.url, + icon_color: input.icon?.color, + time_updated: Date.now(), + }) + .where(eq(ProjectTable.id, input.projectID)) + .returning() + .get() + if (!result) throw new Error(`Project not found: ${input.projectID}`) + const data = fromRow(result) GlobalBus.emit("event", { payload: { type: Event.Updated.type, - properties: result, + properties: data, }, }) - return result + return data }, ) export async function sandboxes(projectID: string) { - const project = await Storage.read(["project", projectID]).catch(() => undefined) - if (!project?.sandboxes) return [] + const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get() + if (!row) return [] + const data = fromRow(row) const valid: string[] = [] - for (const dir of project.sandboxes) { + for (const dir of data.sandboxes) { const stat = await fs.stat(dir).catch(() => undefined) if (stat?.isDirectory()) valid.push(dir) } @@ -339,33 +381,45 @@ export namespace Project { } export async function addSandbox(projectID: string, directory: string) { - const result = await Storage.update(["project", projectID], (draft) => { - const sandboxes = draft.sandboxes ?? [] - if (!sandboxes.includes(directory)) sandboxes.push(directory) - draft.sandboxes = sandboxes - draft.time.updated = Date.now() - }) + const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get() + if (!row) throw new Error(`Project not found: ${projectID}`) + const sandboxes = [...row.sandboxes] + if (!sandboxes.includes(directory)) sandboxes.push(directory) + const result = db() + .update(ProjectTable) + .set({ sandboxes, time_updated: Date.now() }) + .where(eq(ProjectTable.id, projectID)) + .returning() + .get() + if (!result) throw new Error(`Project not found: ${projectID}`) + const data = fromRow(result) GlobalBus.emit("event", { payload: { type: Event.Updated.type, - properties: result, + properties: data, }, }) - return result + return data } export async function removeSandbox(projectID: string, directory: string) { - const result = await Storage.update(["project", projectID], (draft) => { - const sandboxes = draft.sandboxes ?? [] - draft.sandboxes = sandboxes.filter((sandbox) => sandbox !== directory) - draft.time.updated = Date.now() - }) + const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get() + if (!row) throw new Error(`Project not found: ${projectID}`) + const sandboxes = row.sandboxes.filter((s: string) => s !== directory) + const result = db() + .update(ProjectTable) + .set({ sandboxes, time_updated: Date.now() }) + .where(eq(ProjectTable.id, projectID)) + .returning() + .get() + if (!result) throw new Error(`Project not found: ${projectID}`) + const data = fromRow(result) GlobalBus.emit("event", { payload: { type: Event.Updated.type, - properties: result, + properties: data, }, }) - return result + return data } } diff --git a/packages/opencode/src/server/error.ts b/packages/opencode/src/server/error.ts index 26e2dfcb12..cc5fa96187 100644 --- a/packages/opencode/src/server/error.ts +++ b/packages/opencode/src/server/error.ts @@ -1,6 +1,6 @@ import { resolver } from "hono-openapi" import z from "zod" -import { Storage } from "../storage/storage" +import { NotFoundError } from "../storage/db" export const ERRORS = { 400: { @@ -25,7 +25,7 @@ export const ERRORS = { description: "Not found", content: { "application/json": { - schema: resolver(Storage.NotFoundError.Schema), + schema: resolver(NotFoundError.Schema), }, }, }, diff --git a/packages/opencode/src/server/routes/pty.ts b/packages/opencode/src/server/routes/pty.ts index 1ac6cf7971..d1f3820cb8 100644 --- a/packages/opencode/src/server/routes/pty.ts +++ b/packages/opencode/src/server/routes/pty.ts @@ -3,7 +3,7 @@ import { describeRoute, validator, resolver } from "hono-openapi" import { upgradeWebSocket } from "hono/bun" import z from "zod" import { Pty } from "@/pty" -import { Storage } from "../../storage/storage" +import { NotFoundError } from "../../storage/db" import { errors } from "../error" import { lazy } from "../../util/lazy" @@ -76,7 +76,7 @@ export const PtyRoutes = lazy(() => async (c) => { const info = Pty.get(c.req.valid("param").ptyID) if (!info) { - throw new Storage.NotFoundError({ message: "Session not found" }) + throw new NotFoundError({ message: "Session not found" }) } return c.json(info) }, diff --git a/packages/opencode/src/server/server.ts b/packages/opencode/src/server/server.ts index fa646f21ea..0ffb10e0b6 100644 --- a/packages/opencode/src/server/server.ts +++ b/packages/opencode/src/server/server.ts @@ -31,7 +31,7 @@ import { ExperimentalRoutes } from "./routes/experimental" import { ProviderRoutes } from "./routes/provider" import { lazy } from "../util/lazy" import { InstanceBootstrap } from "../project/bootstrap" -import { Storage } from "../storage/storage" +import { NotFoundError } from "../storage/db" import type { ContentfulStatusCode } from "hono/utils/http-status" import { websocket } from "hono/bun" import { HTTPException } from "hono/http-exception" @@ -65,7 +65,7 @@ export namespace Server { }) if (err instanceof NamedError) { let status: ContentfulStatusCode - if (err instanceof Storage.NotFoundError) status = 404 + if (err instanceof NotFoundError) status = 404 else if (err instanceof Provider.ModelNotFoundError) status = 400 else if (err.name.startsWith("Worktree")) status = 400 else status = 500 diff --git a/packages/opencode/src/session/index.ts b/packages/opencode/src/session/index.ts index b81a21a57b..84b567b942 100644 --- a/packages/opencode/src/session/index.ts +++ b/packages/opencode/src/session/index.ts @@ -10,7 +10,10 @@ import { Flag } from "../flag/flag" import { Identifier } from "../id/id" import { Installation } from "../installation" -import { Storage } from "../storage/storage" +import { db, NotFoundError } from "../storage/db" +import { SessionTable, MessageTable, PartTable, SessionDiffTable } from "./session.sql" +import { ShareTable } from "../share/share.sql" +import { eq } from "drizzle-orm" import { Log } from "../util/log" import { MessageV2 } from "./message-v2" import { Instance } from "../project/instance" @@ -39,6 +42,75 @@ export namespace Session { ).test(title) } + type SessionRow = typeof SessionTable.$inferSelect + + export function fromRow(row: SessionRow): Info { + const summary = + row.summary_additions !== null || row.summary_deletions !== null || row.summary_files !== null + ? { + additions: row.summary_additions ?? 0, + deletions: row.summary_deletions ?? 0, + files: row.summary_files ?? 0, + diffs: row.summary_diffs ?? undefined, + } + : undefined + const share = row.share_url ? { url: row.share_url } : undefined + const revert = + row.revert_messageID !== null + ? { + messageID: row.revert_messageID, + partID: row.revert_partID ?? undefined, + snapshot: row.revert_snapshot ?? undefined, + diff: row.revert_diff ?? undefined, + } + : undefined + return { + id: row.id, + slug: row.slug, + projectID: row.projectID, + directory: row.directory, + parentID: row.parentID ?? undefined, + title: row.title, + version: row.version, + summary, + share, + revert, + permission: row.permission ?? undefined, + time: { + created: row.time_created, + updated: row.time_updated, + compacting: row.time_compacting ?? undefined, + archived: row.time_archived ?? undefined, + }, + } + } + + export function toRow(info: Info) { + return { + id: info.id, + projectID: info.projectID, + parentID: info.parentID, + slug: info.slug, + directory: info.directory, + title: info.title, + version: info.version, + share_url: info.share?.url, + summary_additions: info.summary?.additions, + summary_deletions: info.summary?.deletions, + summary_files: info.summary?.files, + summary_diffs: info.summary?.diffs, + revert_messageID: info.revert?.messageID ?? null, + revert_partID: info.revert?.partID ?? null, + revert_snapshot: info.revert?.snapshot ?? null, + revert_diff: info.revert?.diff ?? null, + permission: info.permission, + time_created: info.time.created, + time_updated: info.time.updated, + time_compacting: info.time.compacting, + time_archived: info.time.archived, + } + } + export const Info = z .object({ id: Identifier.schema("session"), @@ -211,7 +283,7 @@ export namespace Session { }, } log.info("created", result) - await Storage.write(["session", Instance.project.id, result.id], result) + db().insert(SessionTable).values(toRow(result)).run() Bus.publish(Event.Created, { info: result, }) @@ -240,12 +312,14 @@ export namespace Session { } export const get = fn(Identifier.schema("session"), async (id) => { - const read = await Storage.read(["session", Instance.project.id, id]) - return read as Info + const row = db().select().from(SessionTable).where(eq(SessionTable.id, id)).get() + if (!row) throw new NotFoundError({ message: `Session not found: ${id}` }) + return fromRow(row) }) export const getShare = fn(Identifier.schema("session"), async (id) => { - return Storage.read(["share", id]) + const row = db().select().from(ShareTable).where(eq(ShareTable.sessionID, id)).get() + return row?.data }) export const share = fn(Identifier.schema("session"), async (id) => { @@ -280,23 +354,24 @@ export namespace Session { ) }) - export async function update(id: string, editor: (session: Info) => void, options?: { touch?: boolean }) { - const project = Instance.project - const result = await Storage.update(["session", project.id, id], (draft) => { - editor(draft) - if (options?.touch !== false) { - draft.time.updated = Date.now() - } - }) + export function update(id: string, editor: (session: Info) => void, options?: { touch?: boolean }) { + const row = db().select().from(SessionTable).where(eq(SessionTable.id, id)).get() + if (!row) throw new Error(`Session not found: ${id}`) + const data = fromRow(row) + editor(data) + if (options?.touch !== false) { + data.time.updated = Date.now() + } + db().update(SessionTable).set(toRow(data)).where(eq(SessionTable.id, id)).run() Bus.publish(Event.Updated, { - info: result, + info: data, }) - return result + return data } export const diff = fn(Identifier.schema("session"), async (sessionID) => { - const diffs = await Storage.read(["session_diff", sessionID]) - return diffs ?? [] + const row = db().select().from(SessionDiffTable).where(eq(SessionDiffTable.sessionID, sessionID)).get() + return row?.data ?? [] }) export const messages = fn( @@ -315,22 +390,17 @@ export namespace Session { }, ) - export async function* list() { + export function* list() { const project = Instance.project - for (const item of await Storage.list(["session", project.id])) { - yield Storage.read(item) + const rows = db().select().from(SessionTable).where(eq(SessionTable.projectID, project.id)).all() + for (const row of rows) { + yield fromRow(row) } } export const children = fn(Identifier.schema("session"), async (parentID) => { - const project = Instance.project - const result = [] as Session.Info[] - for (const item of await Storage.list(["session", project.id])) { - const session = await Storage.read(item) - if (session.parentID !== parentID) continue - result.push(session) - } - return result + const rows = db().select().from(SessionTable).where(eq(SessionTable.parentID, parentID)).all() + return rows.map((row) => fromRow(row)) }) export const remove = fn(Identifier.schema("session"), async (sessionID) => { @@ -341,13 +411,8 @@ export namespace Session { await remove(child.id) } await unshare(sessionID).catch(() => {}) - for (const msg of await Storage.list(["message", sessionID])) { - for (const part of await Storage.list(["part", msg.at(-1)!])) { - await Storage.remove(part) - } - await Storage.remove(msg) - } - await Storage.remove(["session", project.id, sessionID]) + // CASCADE delete handles messages and parts automatically + db().delete(SessionTable).where(eq(SessionTable.id, sessionID)).run() Bus.publish(Event.Deleted, { info: session, }) @@ -357,7 +422,17 @@ export namespace Session { }) export const updateMessage = fn(MessageV2.Info, async (msg) => { - await Storage.write(["message", msg.sessionID, msg.id], msg) + const createdAt = msg.role === "user" ? msg.time.created : msg.time.created + db() + .insert(MessageTable) + .values({ + id: msg.id, + sessionID: msg.sessionID, + createdAt, + data: msg, + }) + .onConflictDoUpdate({ target: MessageTable.id, set: { data: msg } }) + .run() Bus.publish(MessageV2.Event.Updated, { info: msg, }) @@ -370,7 +445,8 @@ export namespace Session { messageID: Identifier.schema("message"), }), async (input) => { - await Storage.remove(["message", input.sessionID, input.messageID]) + // CASCADE delete handles parts automatically + db().delete(MessageTable).where(eq(MessageTable.id, input.messageID)).run() Bus.publish(MessageV2.Event.Removed, { sessionID: input.sessionID, messageID: input.messageID, @@ -386,7 +462,7 @@ export namespace Session { partID: Identifier.schema("part"), }), async (input) => { - await Storage.remove(["part", input.messageID, input.partID]) + db().delete(PartTable).where(eq(PartTable.id, input.partID)).run() Bus.publish(MessageV2.Event.PartRemoved, { sessionID: input.sessionID, messageID: input.messageID, @@ -411,7 +487,16 @@ export namespace Session { export const updatePart = fn(UpdatePartInput, async (input) => { const part = "delta" in input ? input.part : input const delta = "delta" in input ? input.delta : undefined - await Storage.write(["part", part.messageID, part.id], part) + db() + .insert(PartTable) + .values({ + id: part.id, + messageID: part.messageID, + sessionID: part.sessionID, + data: part, + }) + .onConflictDoUpdate({ target: PartTable.id, set: { data: part } }) + .run() Bus.publish(MessageV2.Event.PartUpdated, { part, delta, diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 83ca72addb..2dab09918a 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -6,7 +6,9 @@ import { Identifier } from "../id/id" import { LSP } from "../lsp" import { Snapshot } from "@/snapshot" import { fn } from "@/util/fn" -import { Storage } from "@/storage/storage" +import { db } from "@/storage/db" +import { MessageTable, PartTable } from "./session.sql" +import { eq, desc } from "drizzle-orm" import { ProviderTransform } from "@/provider/transform" import { STATUS_CODES } from "http" import { iife } from "@/util/iife" @@ -607,21 +609,23 @@ export namespace MessageV2 { } export const stream = fn(Identifier.schema("session"), async function* (sessionID) { - const list = await Array.fromAsync(await Storage.list(["message", sessionID])) - for (let i = list.length - 1; i >= 0; i--) { - yield await get({ - sessionID, - messageID: list[i][2], - }) + const rows = db() + .select() + .from(MessageTable) + .where(eq(MessageTable.sessionID, sessionID)) + .orderBy(desc(MessageTable.createdAt)) + .all() + for (const row of rows) { + yield { + info: row.data, + parts: await parts(row.id), + } } }) export const parts = fn(Identifier.schema("message"), async (messageID) => { - const result = [] as MessageV2.Part[] - for (const item of await Storage.list(["part", messageID])) { - const read = await Storage.read(item) - result.push(read) - } + const rows = db().select().from(PartTable).where(eq(PartTable.messageID, messageID)).all() + const result = rows.map((row) => row.data) result.sort((a, b) => (a.id > b.id ? 1 : -1)) return result }) @@ -632,8 +636,10 @@ export namespace MessageV2 { messageID: Identifier.schema("message"), }), async (input) => { + const row = db().select().from(MessageTable).where(eq(MessageTable.id, input.messageID)).get() + if (!row) throw new Error(`Message not found: ${input.messageID}`) return { - info: await Storage.read(["message", input.sessionID, input.messageID]), + info: row.data, parts: await parts(input.messageID), } }, diff --git a/packages/opencode/src/session/revert.ts b/packages/opencode/src/session/revert.ts index 7afe44e2ce..fb6e0a5ec3 100644 --- a/packages/opencode/src/session/revert.ts +++ b/packages/opencode/src/session/revert.ts @@ -5,7 +5,9 @@ import { MessageV2 } from "./message-v2" import { Session } from "." import { Log } from "../util/log" import { splitWhen } from "remeda" -import { Storage } from "../storage/storage" +import { db } from "../storage/db" +import { SessionDiffTable, MessageTable, PartTable } from "./session.sql" +import { eq } from "drizzle-orm" import { Bus } from "../bus" import { SessionPrompt } from "./prompt" import { SessionSummary } from "./summary" @@ -60,7 +62,11 @@ export namespace SessionRevert { if (revert.snapshot) revert.diff = await Snapshot.diff(revert.snapshot) const rangeMessages = all.filter((msg) => msg.info.id >= revert!.messageID) const diffs = await SessionSummary.computeDiff({ messages: rangeMessages }) - await Storage.write(["session_diff", input.sessionID], diffs) + db() + .insert(SessionDiffTable) + .values({ sessionID: input.sessionID, data: diffs }) + .onConflictDoUpdate({ target: SessionDiffTable.sessionID, set: { data: diffs } }) + .run() Bus.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: diffs, @@ -97,7 +103,7 @@ export namespace SessionRevert { const [preserve, remove] = splitWhen(msgs, (x) => x.info.id === messageID) msgs = preserve for (const msg of remove) { - await Storage.remove(["message", sessionID, msg.info.id]) + db().delete(MessageTable).where(eq(MessageTable.id, msg.info.id)).run() await Bus.publish(MessageV2.Event.Removed, { sessionID: sessionID, messageID: msg.info.id }) } const last = preserve.at(-1) @@ -106,7 +112,7 @@ export namespace SessionRevert { const [preserveParts, removeParts] = splitWhen(last.parts, (x) => x.id === partID) last.parts = preserveParts for (const part of removeParts) { - await Storage.remove(["part", last.info.id, part.id]) + db().delete(PartTable).where(eq(PartTable.id, part.id)).run() await Bus.publish(MessageV2.Event.PartRemoved, { sessionID: sessionID, messageID: last.info.id, diff --git a/packages/opencode/src/session/session.sql.ts b/packages/opencode/src/session/session.sql.ts new file mode 100644 index 0000000000..be35dd1703 --- /dev/null +++ b/packages/opencode/src/session/session.sql.ts @@ -0,0 +1,83 @@ +import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core" +import { ProjectTable } from "../project/project.sql" +import type { MessageV2 } from "./message-v2" +import type { Snapshot } from "@/snapshot" +import type { Todo } from "./todo" +import type { PermissionNext } from "@/permission/next" + +export const SessionTable = sqliteTable( + "session", + { + id: text("id").primaryKey(), + projectID: text("project_id") + .notNull() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + parentID: text("parent_id"), + slug: text("slug").notNull(), + directory: text("directory").notNull(), + title: text("title").notNull(), + version: text("version").notNull(), + share_url: text("share_url"), + summary_additions: integer("summary_additions"), + summary_deletions: integer("summary_deletions"), + summary_files: integer("summary_files"), + summary_diffs: text("summary_diffs", { mode: "json" }).$type(), + revert_messageID: text("revert_message_id"), + revert_partID: text("revert_part_id"), + revert_snapshot: text("revert_snapshot"), + revert_diff: text("revert_diff"), + permission: text("permission", { mode: "json" }).$type(), + time_created: integer("time_created").notNull(), + time_updated: integer("time_updated").notNull(), + time_compacting: integer("time_compacting"), + time_archived: integer("time_archived"), + }, + (table) => [index("session_project_idx").on(table.projectID), index("session_parent_idx").on(table.parentID)], +) + +export const MessageTable = sqliteTable( + "message", + { + id: text("id").primaryKey(), + sessionID: text("session_id") + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + createdAt: integer("created_at").notNull(), + data: text("data", { mode: "json" }).notNull().$type(), + }, + (table) => [index("message_session_idx").on(table.sessionID)], +) + +export const PartTable = sqliteTable( + "part", + { + id: text("id").primaryKey(), + messageID: text("message_id") + .notNull() + .references(() => MessageTable.id, { onDelete: "cascade" }), + sessionID: text("session_id").notNull(), + data: text("data", { mode: "json" }).notNull().$type(), + }, + (table) => [index("part_message_idx").on(table.messageID), index("part_session_idx").on(table.sessionID)], +) + +export const SessionDiffTable = sqliteTable("session_diff", { + sessionID: text("session_id") + .primaryKey() + .references(() => SessionTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type(), +}) + +export const TodoTable = sqliteTable("todo", { + sessionID: text("session_id") + .primaryKey() + .references(() => SessionTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type(), +}) + +export const PermissionTable = sqliteTable("permission", { + projectID: text("project_id") + .primaryKey() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type(), +}) diff --git a/packages/opencode/src/session/summary.ts b/packages/opencode/src/session/summary.ts index 611d5f1c62..a79850046d 100644 --- a/packages/opencode/src/session/summary.ts +++ b/packages/opencode/src/session/summary.ts @@ -11,7 +11,9 @@ import { Snapshot } from "@/snapshot" import { Log } from "@/util/log" import path from "path" import { Instance } from "@/project/instance" -import { Storage } from "@/storage/storage" +import { db } from "@/storage/db" +import { SessionDiffTable } from "./session.sql" +import { eq } from "drizzle-orm" import { Bus } from "@/bus" import { LLM } from "./llm" @@ -54,7 +56,11 @@ export namespace SessionSummary { files: diffs.length, } }) - await Storage.write(["session_diff", input.sessionID], diffs) + db() + .insert(SessionDiffTable) + .values({ sessionID: input.sessionID, data: diffs }) + .onConflictDoUpdate({ target: SessionDiffTable.sessionID, set: { data: diffs } }) + .run() Bus.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: diffs, @@ -116,7 +122,8 @@ export namespace SessionSummary { messageID: Identifier.schema("message").optional(), }), async (input) => { - return Storage.read(["session_diff", input.sessionID]).catch(() => []) + const row = db().select().from(SessionDiffTable).where(eq(SessionDiffTable.sessionID, input.sessionID)).get() + return row?.data ?? [] }, ) diff --git a/packages/opencode/src/session/todo.ts b/packages/opencode/src/session/todo.ts index aa7df7e981..3280744662 100644 --- a/packages/opencode/src/session/todo.ts +++ b/packages/opencode/src/session/todo.ts @@ -1,7 +1,9 @@ import { BusEvent } from "@/bus/bus-event" import { Bus } from "@/bus" import z from "zod" -import { Storage } from "../storage/storage" +import { db } from "../storage/db" +import { TodoTable } from "./session.sql" +import { eq } from "drizzle-orm" export namespace Todo { export const Info = z @@ -24,14 +26,17 @@ export namespace Todo { ), } - export async function update(input: { sessionID: string; todos: Info[] }) { - await Storage.write(["todo", input.sessionID], input.todos) + export function update(input: { sessionID: string; todos: Info[] }) { + db() + .insert(TodoTable) + .values({ sessionID: input.sessionID, data: input.todos }) + .onConflictDoUpdate({ target: TodoTable.sessionID, set: { data: input.todos } }) + .run() Bus.publish(Event.Updated, input) } - export async function get(sessionID: string) { - return Storage.read(["todo", sessionID]) - .then((x) => x || []) - .catch(() => []) + export function get(sessionID: string) { + const row = db().select().from(TodoTable).where(eq(TodoTable.sessionID, sessionID)).get() + return row?.data ?? [] } } diff --git a/packages/opencode/src/share/share-next.ts b/packages/opencode/src/share/share-next.ts index dddce95cb4..2d16820459 100644 --- a/packages/opencode/src/share/share-next.ts +++ b/packages/opencode/src/share/share-next.ts @@ -4,7 +4,9 @@ import { ulid } from "ulid" import { Provider } from "@/provider/provider" import { Session } from "@/session" import { MessageV2 } from "@/session/message-v2" -import { Storage } from "@/storage/storage" +import { db } from "@/storage/db" +import { SessionShareTable } from "./share.sql" +import { eq } from "drizzle-orm" import { Log } from "@/util/log" import type * as SDK from "@opencode-ai/sdk/v2" @@ -77,17 +79,18 @@ export namespace ShareNext { }) .then((x) => x.json()) .then((x) => x as { id: string; url: string; secret: string }) - await Storage.write(["session_share", sessionID], result) + db() + .insert(SessionShareTable) + .values({ sessionID, data: result }) + .onConflictDoUpdate({ target: SessionShareTable.sessionID, set: { data: result } }) + .run() fullSync(sessionID) return result } function get(sessionID: string) { - return Storage.read<{ - id: string - secret: string - url: string - }>(["session_share", sessionID]) + const row = db().select().from(SessionShareTable).where(eq(SessionShareTable.sessionID, sessionID)).get() + return row?.data } type Data = @@ -132,7 +135,7 @@ export namespace ShareNext { const queued = queue.get(sessionID) if (!queued) return queue.delete(sessionID) - const share = await get(sessionID).catch(() => undefined) + const share = get(sessionID) if (!share) return await fetch(`${await url()}/api/share/${share.id}/sync`, { @@ -152,7 +155,7 @@ export namespace ShareNext { export async function remove(sessionID: string) { if (disabled) return log.info("removing share", { sessionID }) - const share = await get(sessionID) + const share = get(sessionID) if (!share) return await fetch(`${await url()}/api/share/${share.id}`, { method: "DELETE", @@ -163,7 +166,7 @@ export namespace ShareNext { secret: share.secret, }), }) - await Storage.remove(["session_share", sessionID]) + db().delete(SessionShareTable).where(eq(SessionShareTable.sessionID, sessionID)).run() } async function fullSync(sessionID: string) { diff --git a/packages/opencode/src/share/share.sql.ts b/packages/opencode/src/share/share.sql.ts new file mode 100644 index 0000000000..7a65fd764b --- /dev/null +++ b/packages/opencode/src/share/share.sql.ts @@ -0,0 +1,19 @@ +import { sqliteTable, text } from "drizzle-orm/sqlite-core" +import { SessionTable } from "../session/session.sql" +import type { Session } from "../session" + +export const SessionShareTable = sqliteTable("session_share", { + sessionID: text("session_id") + .primaryKey() + .references(() => SessionTable.id, { onDelete: "cascade" }), + data: text("data", { mode: "json" }).notNull().$type<{ + id: string + secret: string + url: string + }>(), +}) + +export const ShareTable = sqliteTable("share", { + sessionID: text("session_id").primaryKey(), + data: text("data", { mode: "json" }).notNull().$type(), +}) diff --git a/packages/opencode/src/sql.d.ts b/packages/opencode/src/sql.d.ts new file mode 100644 index 0000000000..28cb1e2649 --- /dev/null +++ b/packages/opencode/src/sql.d.ts @@ -0,0 +1,4 @@ +declare module "*.sql" { + const content: string + export default content +} diff --git a/packages/opencode/src/storage/db.ts b/packages/opencode/src/storage/db.ts new file mode 100644 index 0000000000..3c1a159305 --- /dev/null +++ b/packages/opencode/src/storage/db.ts @@ -0,0 +1,87 @@ +import { Database } from "bun:sqlite" +import { drizzle } from "drizzle-orm/bun-sqlite" +import { lazy } from "../util/lazy" +import { Global } from "../global" +import { Log } from "../util/log" +import { migrations } from "./migrations.generated" +import { migrateFromJson } from "./json-migration" +import { NamedError } from "@opencode-ai/util/error" +import z from "zod" +import path from "path" + +export const NotFoundError = NamedError.create( + "NotFoundError", + z.object({ + message: z.string(), + }), +) + +const log = Log.create({ service: "db" }) + +export type DB = ReturnType + +const connection = lazy(() => { + const dbPath = path.join(Global.Path.data, "opencode.db") + log.info("opening database", { path: dbPath }) + + const sqlite = new Database(dbPath, { create: true }) + + sqlite.run("PRAGMA journal_mode = WAL") + sqlite.run("PRAGMA synchronous = NORMAL") + sqlite.run("PRAGMA busy_timeout = 5000") + sqlite.run("PRAGMA cache_size = -64000") + sqlite.run("PRAGMA foreign_keys = ON") + + migrate(sqlite) + + // Run JSON migration asynchronously after schema is ready + migrateFromJson(sqlite).catch((e) => log.error("json migration failed", { error: e })) + + return drizzle(sqlite) +}) + +function migrate(sqlite: Database) { + sqlite.exec(` + CREATE TABLE IF NOT EXISTS _migrations ( + name TEXT PRIMARY KEY, + applied_at INTEGER NOT NULL + ) + `) + + const applied = new Set( + sqlite + .query<{ name: string }, []>("SELECT name FROM _migrations") + .all() + .map((r) => r.name), + ) + + for (const migration of migrations) { + if (applied.has(migration.name)) continue + log.info("applying migration", { name: migration.name }) + + // Split by statement breakpoint and execute each statement + // Use IF NOT EXISTS variants to handle partial migrations + const statements = migration.sql.split("--> statement-breakpoint") + for (const stmt of statements) { + const trimmed = stmt.trim() + if (!trimmed) continue + + try { + sqlite.exec(trimmed) + } catch (e: any) { + // Ignore "already exists" errors for idempotency + if (e?.message?.includes("already exists")) { + log.info("skipping existing object", { statement: trimmed.slice(0, 50) }) + continue + } + throw e + } + } + + sqlite.run("INSERT INTO _migrations (name, applied_at) VALUES (?, ?)", [migration.name, Date.now()]) + } +} + +export function db() { + return connection() +} diff --git a/packages/opencode/src/storage/json-migration.ts b/packages/opencode/src/storage/json-migration.ts new file mode 100644 index 0000000000..96f3714c2a --- /dev/null +++ b/packages/opencode/src/storage/json-migration.ts @@ -0,0 +1,300 @@ +import { Database } from "bun:sqlite" +import { drizzle } from "drizzle-orm/bun-sqlite" +import { eq } from "drizzle-orm" +import { Global } from "../global" +import { Log } from "../util/log" +import { ProjectTable } from "../project/project.sql" +import { + SessionTable, + MessageTable, + PartTable, + SessionDiffTable, + TodoTable, + PermissionTable, +} from "../session/session.sql" +import { SessionShareTable, ShareTable } from "../share/share.sql" +import path from "path" + +const log = Log.create({ service: "json-migration" }) + +export async function migrateFromJson(sqlite: Database, customStorageDir?: string) { + const storageDir = customStorageDir ?? path.join(Global.Path.data, "storage") + const migrationMarker = path.join(storageDir, "sqlite-migrated") + + if (await Bun.file(migrationMarker).exists()) { + log.info("json migration already completed") + return + } + + if (!(await Bun.file(path.join(storageDir, "migration")).exists())) { + log.info("no json storage found, skipping migration") + await Bun.write(migrationMarker, Date.now().toString()) + return + } + + log.info("starting json to sqlite migration", { storageDir }) + + const db = drizzle(sqlite) + const stats = { + projects: 0, + sessions: 0, + messages: 0, + parts: 0, + diffs: 0, + todos: 0, + permissions: 0, + shares: 0, + errors: [] as string[], + } + + // Migrate projects first (no FK deps) + const projectGlob = new Bun.Glob("project/*.json") + for await (const file of projectGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + if (!data.id) { + stats.errors.push(`project missing id: ${file}`) + continue + } + db.insert(ProjectTable) + .values({ + id: data.id, + worktree: data.worktree ?? "/", + vcs: data.vcs, + name: data.name ?? undefined, + icon_url: data.icon?.url, + icon_color: data.icon?.color, + time_created: data.time?.created ?? Date.now(), + time_updated: data.time?.updated ?? Date.now(), + time_initialized: data.time?.initialized, + sandboxes: data.sandboxes ?? [], + }) + .onConflictDoNothing() + .run() + stats.projects++ + } catch (e) { + stats.errors.push(`failed to migrate project ${file}: ${e}`) + } + } + log.info("migrated projects", { count: stats.projects }) + + // Migrate sessions (depends on projects) + const sessionGlob = new Bun.Glob("session/*/*.json") + for await (const file of sessionGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + if (!data.id || !data.projectID) { + stats.errors.push(`session missing id or projectID: ${file}`) + continue + } + // Check if project exists (skip orphaned sessions) + const project = db.select().from(ProjectTable).where(eq(ProjectTable.id, data.projectID)).get() + if (!project) { + log.warn("skipping orphaned session", { sessionID: data.id, projectID: data.projectID }) + continue + } + db.insert(SessionTable) + .values({ + id: data.id, + projectID: data.projectID, + parentID: data.parentID ?? null, + slug: data.slug ?? "", + directory: data.directory ?? "", + title: data.title ?? "", + version: data.version ?? "", + share_url: data.share?.url ?? null, + summary_additions: data.summary?.additions ?? null, + summary_deletions: data.summary?.deletions ?? null, + summary_files: data.summary?.files ?? null, + summary_diffs: data.summary?.diffs ?? null, + revert_messageID: data.revert?.messageID ?? null, + revert_partID: data.revert?.partID ?? null, + revert_snapshot: data.revert?.snapshot ?? null, + revert_diff: data.revert?.diff ?? null, + permission: data.permission ?? null, + time_created: data.time?.created ?? Date.now(), + time_updated: data.time?.updated ?? Date.now(), + time_compacting: data.time?.compacting ?? null, + time_archived: data.time?.archived ?? null, + }) + .onConflictDoNothing() + .run() + stats.sessions++ + } catch (e) { + stats.errors.push(`failed to migrate session ${file}: ${e}`) + } + } + log.info("migrated sessions", { count: stats.sessions }) + + // Migrate messages (depends on sessions) + const messageGlob = new Bun.Glob("message/*/*.json") + for await (const file of messageGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + if (!data.id || !data.sessionID) { + stats.errors.push(`message missing id or sessionID: ${file}`) + continue + } + // Check if session exists + const session = db.select().from(SessionTable).where(eq(SessionTable.id, data.sessionID)).get() + if (!session) { + log.warn("skipping orphaned message", { messageID: data.id, sessionID: data.sessionID }) + continue + } + db.insert(MessageTable) + .values({ + id: data.id, + sessionID: data.sessionID, + createdAt: data.time?.created ?? Date.now(), + data, + }) + .onConflictDoNothing() + .run() + stats.messages++ + } catch (e) { + stats.errors.push(`failed to migrate message ${file}: ${e}`) + } + } + log.info("migrated messages", { count: stats.messages }) + + // Migrate parts (depends on messages) + const partGlob = new Bun.Glob("part/*/*.json") + for await (const file of partGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + if (!data.id || !data.messageID || !data.sessionID) { + stats.errors.push(`part missing id, messageID, or sessionID: ${file}`) + continue + } + // Check if message exists + const message = db.select().from(MessageTable).where(eq(MessageTable.id, data.messageID)).get() + if (!message) { + log.warn("skipping orphaned part", { partID: data.id, messageID: data.messageID }) + continue + } + db.insert(PartTable) + .values({ + id: data.id, + messageID: data.messageID, + sessionID: data.sessionID, + data, + }) + .onConflictDoNothing() + .run() + stats.parts++ + } catch (e) { + stats.errors.push(`failed to migrate part ${file}: ${e}`) + } + } + log.info("migrated parts", { count: stats.parts }) + + // Migrate session diffs + const diffGlob = new Bun.Glob("session_diff/*.json") + for await (const file of diffGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + const sessionID = path.basename(file, ".json") + // Check if session exists + const session = db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get() + if (!session) { + log.warn("skipping orphaned session_diff", { sessionID }) + continue + } + db.insert(SessionDiffTable).values({ sessionID, data }).onConflictDoNothing().run() + stats.diffs++ + } catch (e) { + stats.errors.push(`failed to migrate session_diff ${file}: ${e}`) + } + } + log.info("migrated session diffs", { count: stats.diffs }) + + // Migrate todos + const todoGlob = new Bun.Glob("todo/*.json") + for await (const file of todoGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + const sessionID = path.basename(file, ".json") + const session = db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get() + if (!session) { + log.warn("skipping orphaned todo", { sessionID }) + continue + } + db.insert(TodoTable).values({ sessionID, data }).onConflictDoNothing().run() + stats.todos++ + } catch (e) { + stats.errors.push(`failed to migrate todo ${file}: ${e}`) + } + } + log.info("migrated todos", { count: stats.todos }) + + // Migrate permissions + const permGlob = new Bun.Glob("permission/*.json") + for await (const file of permGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + const projectID = path.basename(file, ".json") + const project = db.select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get() + if (!project) { + log.warn("skipping orphaned permission", { projectID }) + continue + } + db.insert(PermissionTable).values({ projectID, data }).onConflictDoNothing().run() + stats.permissions++ + } catch (e) { + stats.errors.push(`failed to migrate permission ${file}: ${e}`) + } + } + log.info("migrated permissions", { count: stats.permissions }) + + // Migrate session shares + const shareGlob = new Bun.Glob("session_share/*.json") + for await (const file of shareGlob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + const sessionID = path.basename(file, ".json") + const session = db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get() + if (!session) { + log.warn("skipping orphaned session_share", { sessionID }) + continue + } + db.insert(SessionShareTable).values({ sessionID, data }).onConflictDoNothing().run() + stats.shares++ + } catch (e) { + stats.errors.push(`failed to migrate session_share ${file}: ${e}`) + } + } + log.info("migrated session shares", { count: stats.shares }) + + // Migrate shares (downloaded shared sessions, no FK) + const share2Glob = new Bun.Glob("share/*.json") + for await (const file of share2Glob.scan({ cwd: storageDir, absolute: true })) { + try { + const data = await Bun.file(file).json() + const sessionID = path.basename(file, ".json") + db.insert(ShareTable).values({ sessionID, data }).onConflictDoNothing().run() + } catch (e) { + stats.errors.push(`failed to migrate share ${file}: ${e}`) + } + } + + // Mark migration complete + await Bun.write(migrationMarker, Date.now().toString()) + + log.info("json migration complete", { + projects: stats.projects, + sessions: stats.sessions, + messages: stats.messages, + parts: stats.parts, + diffs: stats.diffs, + todos: stats.todos, + permissions: stats.permissions, + shares: stats.shares, + errorCount: stats.errors.length, + }) + + if (stats.errors.length > 0) { + log.warn("migration errors", { errors: stats.errors.slice(0, 20) }) + } + + return stats +} diff --git a/packages/opencode/src/storage/migrations.generated.ts b/packages/opencode/src/storage/migrations.generated.ts new file mode 100644 index 0000000000..a048c61efb --- /dev/null +++ b/packages/opencode/src/storage/migrations.generated.ts @@ -0,0 +1,6 @@ +// Auto-generated - do not edit +import m0 from "../../migration/0000_magical_strong_guy.sql" with { type: "text" } + +export const migrations = [ + { name: "0000_magical_strong_guy", sql: m0 }, +] diff --git a/packages/opencode/src/storage/storage.ts b/packages/opencode/src/storage/storage.ts deleted file mode 100644 index 18f2d67e7a..0000000000 --- a/packages/opencode/src/storage/storage.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { Log } from "../util/log" -import path from "path" -import fs from "fs/promises" -import { Global } from "../global" -import { Filesystem } from "../util/filesystem" -import { lazy } from "../util/lazy" -import { Lock } from "../util/lock" -import { $ } from "bun" -import { NamedError } from "@opencode-ai/util/error" -import z from "zod" - -export namespace Storage { - const log = Log.create({ service: "storage" }) - - type Migration = (dir: string) => Promise - - export const NotFoundError = NamedError.create( - "NotFoundError", - z.object({ - message: z.string(), - }), - ) - - const MIGRATIONS: Migration[] = [ - async (dir) => { - const project = path.resolve(dir, "../project") - if (!(await Filesystem.isDir(project))) return - for await (const projectDir of new Bun.Glob("*").scan({ - cwd: project, - onlyFiles: false, - })) { - log.info(`migrating project ${projectDir}`) - let projectID = projectDir - const fullProjectDir = path.join(project, projectDir) - let worktree = "/" - - if (projectID !== "global") { - for await (const msgFile of new Bun.Glob("storage/session/message/*/*.json").scan({ - cwd: path.join(project, projectDir), - absolute: true, - })) { - const json = await Bun.file(msgFile).json() - worktree = json.path?.root - if (worktree) break - } - if (!worktree) continue - if (!(await Filesystem.isDir(worktree))) continue - const [id] = await $`git rev-list --max-parents=0 --all` - .quiet() - .nothrow() - .cwd(worktree) - .text() - .then((x) => - x - .split("\n") - .filter(Boolean) - .map((x) => x.trim()) - .toSorted(), - ) - if (!id) continue - projectID = id - - await Bun.write( - path.join(dir, "project", projectID + ".json"), - JSON.stringify({ - id, - vcs: "git", - worktree, - time: { - created: Date.now(), - initialized: Date.now(), - }, - }), - ) - - log.info(`migrating sessions for project ${projectID}`) - for await (const sessionFile of new Bun.Glob("storage/session/info/*.json").scan({ - cwd: fullProjectDir, - absolute: true, - })) { - const dest = path.join(dir, "session", projectID, path.basename(sessionFile)) - log.info("copying", { - sessionFile, - dest, - }) - const session = await Bun.file(sessionFile).json() - await Bun.write(dest, JSON.stringify(session)) - log.info(`migrating messages for session ${session.id}`) - for await (const msgFile of new Bun.Glob(`storage/session/message/${session.id}/*.json`).scan({ - cwd: fullProjectDir, - absolute: true, - })) { - const dest = path.join(dir, "message", session.id, path.basename(msgFile)) - log.info("copying", { - msgFile, - dest, - }) - const message = await Bun.file(msgFile).json() - await Bun.write(dest, JSON.stringify(message)) - - log.info(`migrating parts for message ${message.id}`) - for await (const partFile of new Bun.Glob(`storage/session/part/${session.id}/${message.id}/*.json`).scan( - { - cwd: fullProjectDir, - absolute: true, - }, - )) { - const dest = path.join(dir, "part", message.id, path.basename(partFile)) - const part = await Bun.file(partFile).json() - log.info("copying", { - partFile, - dest, - }) - await Bun.write(dest, JSON.stringify(part)) - } - } - } - } - } - }, - async (dir) => { - for await (const item of new Bun.Glob("session/*/*.json").scan({ - cwd: dir, - absolute: true, - })) { - const session = await Bun.file(item).json() - if (!session.projectID) continue - if (!session.summary?.diffs) continue - const { diffs } = session.summary - await Bun.file(path.join(dir, "session_diff", session.id + ".json")).write(JSON.stringify(diffs)) - await Bun.file(path.join(dir, "session", session.projectID, session.id + ".json")).write( - JSON.stringify({ - ...session, - summary: { - additions: diffs.reduce((sum: any, x: any) => sum + x.additions, 0), - deletions: diffs.reduce((sum: any, x: any) => sum + x.deletions, 0), - }, - }), - ) - } - }, - ] - - const state = lazy(async () => { - const dir = path.join(Global.Path.data, "storage") - const migration = await Bun.file(path.join(dir, "migration")) - .json() - .then((x) => parseInt(x)) - .catch(() => 0) - for (let index = migration; index < MIGRATIONS.length; index++) { - log.info("running migration", { index }) - const migration = MIGRATIONS[index] - await migration(dir).catch(() => log.error("failed to run migration", { index })) - await Bun.write(path.join(dir, "migration"), (index + 1).toString()) - } - return { - dir, - } - }) - - export async function remove(key: string[]) { - const dir = await state().then((x) => x.dir) - const target = path.join(dir, ...key) + ".json" - return withErrorHandling(async () => { - await fs.unlink(target).catch(() => {}) - }) - } - - export async function read(key: string[]) { - const dir = await state().then((x) => x.dir) - const target = path.join(dir, ...key) + ".json" - return withErrorHandling(async () => { - using _ = await Lock.read(target) - const result = await Bun.file(target).json() - return result as T - }) - } - - export async function update(key: string[], fn: (draft: T) => void) { - const dir = await state().then((x) => x.dir) - const target = path.join(dir, ...key) + ".json" - return withErrorHandling(async () => { - using _ = await Lock.write(target) - const content = await Bun.file(target).json() - fn(content) - await Bun.write(target, JSON.stringify(content, null, 2)) - return content as T - }) - } - - export async function write(key: string[], content: T) { - const dir = await state().then((x) => x.dir) - const target = path.join(dir, ...key) + ".json" - return withErrorHandling(async () => { - using _ = await Lock.write(target) - await Bun.write(target, JSON.stringify(content, null, 2)) - }) - } - - async function withErrorHandling(body: () => Promise) { - return body().catch((e) => { - if (!(e instanceof Error)) throw e - const errnoException = e as NodeJS.ErrnoException - if (errnoException.code === "ENOENT") { - throw new NotFoundError({ message: `Resource not found: ${errnoException.path}` }) - } - throw e - }) - } - - const glob = new Bun.Glob("**/*") - export async function list(prefix: string[]) { - const dir = await state().then((x) => x.dir) - try { - const result = await Array.fromAsync( - glob.scan({ - cwd: path.join(dir, ...prefix), - onlyFiles: true, - }), - ).then((results) => results.map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)])) - result.sort() - return result - } catch { - return [] - } - } -} diff --git a/packages/opencode/src/util/lazy.ts b/packages/opencode/src/util/lazy.ts index 0cc6d8d5c4..55643dc6a7 100644 --- a/packages/opencode/src/util/lazy.ts +++ b/packages/opencode/src/util/lazy.ts @@ -4,9 +4,14 @@ export function lazy(fn: () => T) { const result = (): T => { if (loaded) return value as T - loaded = true - value = fn() - return value as T + try { + value = fn() + loaded = true + return value as T + } catch (e) { + // Don't mark as loaded if initialization failed + throw e + } } result.reset = () => { diff --git a/packages/opencode/src/worktree/index.ts b/packages/opencode/src/worktree/index.ts index 97fe2c4fc0..30443d36b1 100644 --- a/packages/opencode/src/worktree/index.ts +++ b/packages/opencode/src/worktree/index.ts @@ -7,7 +7,9 @@ import { Global } from "../global" import { Instance } from "../project/instance" import { InstanceBootstrap } from "../project/bootstrap" import { Project } from "../project/project" -import { Storage } from "../storage/storage" +import { db } from "../storage/db" +import { ProjectTable } from "../project/project.sql" +import { eq } from "drizzle-orm" import { fn } from "../util/fn" import { Log } from "../util/log" import { BusEvent } from "@/bus/bus-event" @@ -318,7 +320,8 @@ export namespace Worktree { }, }) - const project = await Storage.read(["project", projectID]).catch(() => undefined) + const row = db().select().from(ProjectTable).where(eq(ProjectTable.id, projectID)).get() + const project = row ? Project.fromRow(row) : undefined const startup = project?.commands?.start?.trim() ?? "" const run = async (cmd: string, kind: "project" | "worktree") => { diff --git a/packages/opencode/test/permission/next.test.ts b/packages/opencode/test/permission/next.test.ts index 29f1efa401..add3332048 100644 --- a/packages/opencode/test/permission/next.test.ts +++ b/packages/opencode/test/permission/next.test.ts @@ -2,7 +2,6 @@ import { test, expect } from "bun:test" import os from "os" import { PermissionNext } from "../../src/permission/next" import { Instance } from "../../src/project/instance" -import { Storage } from "../../src/storage/storage" import { tmpdir } from "../fixture/fixture" // fromConfig tests diff --git a/packages/opencode/test/preload.ts b/packages/opencode/test/preload.ts index 819166c94c..f40e4b1e9a 100644 --- a/packages/opencode/test/preload.ts +++ b/packages/opencode/test/preload.ts @@ -5,23 +5,27 @@ import path from "path" import fs from "fs/promises" import fsSync from "fs" import { afterAll } from "bun:test" -const { Global } = await import("../src/global") +// Set XDG env vars FIRST, before any src/ imports const dir = path.join(os.tmpdir(), "opencode-test-data-" + process.pid) await fs.mkdir(dir, { recursive: true }) afterAll(() => { fsSync.rmSync(dir, { recursive: true, force: true }) }) + +process.env["XDG_DATA_HOME"] = path.join(dir, "share") +process.env["XDG_CACHE_HOME"] = path.join(dir, "cache") +process.env["XDG_CONFIG_HOME"] = path.join(dir, "config") +process.env["XDG_STATE_HOME"] = path.join(dir, "state") + // Set test home directory to isolate tests from user's actual home directory // This prevents tests from picking up real user configs/skills from ~/.claude/skills const testHome = path.join(dir, "home") await fs.mkdir(testHome, { recursive: true }) process.env["OPENCODE_TEST_HOME"] = testHome -process.env["XDG_DATA_HOME"] = path.join(dir, "share") -process.env["XDG_CACHE_HOME"] = path.join(dir, "cache") -process.env["XDG_CONFIG_HOME"] = path.join(dir, "config") -process.env["XDG_STATE_HOME"] = path.join(dir, "state") +// Now safe to import Global (after XDG vars are set) +const { Global } = await import("../src/global") // Pre-fetch models.json so tests don't need the macro fallback // Also write the cache version file to prevent global/index.ts from clearing the cache diff --git a/packages/opencode/test/project/project.test.ts b/packages/opencode/test/project/project.test.ts index d44e606746..bee8b77dd1 100644 --- a/packages/opencode/test/project/project.test.ts +++ b/packages/opencode/test/project/project.test.ts @@ -1,7 +1,6 @@ import { describe, expect, test } from "bun:test" import { Project } from "../../src/project/project" import { Log } from "../../src/util/log" -import { Storage } from "../../src/storage/storage" import { $ } from "bun" import path from "path" import { tmpdir } from "../fixture/fixture" @@ -99,11 +98,12 @@ describe("Project.discover", () => { await Project.discover(project) - const updated = await Storage.read(["project", project.id]) - expect(updated.icon).toBeDefined() - expect(updated.icon?.url).toStartWith("data:") - expect(updated.icon?.url).toContain("base64") - expect(updated.icon?.color).toBeUndefined() + const updated = Project.get(project.id) + expect(updated).toBeDefined() + expect(updated!.icon).toBeDefined() + expect(updated!.icon?.url).toStartWith("data:") + expect(updated!.icon?.url).toContain("base64") + expect(updated!.icon?.color).toBeUndefined() }) test("should not discover non-image files", async () => { @@ -114,7 +114,8 @@ describe("Project.discover", () => { await Project.discover(project) - const updated = await Storage.read(["project", project.id]) - expect(updated.icon).toBeUndefined() + const updated = Project.get(project.id) + expect(updated).toBeDefined() + expect(updated!.icon).toBeUndefined() }) }) diff --git a/packages/opencode/test/storage/json-migration.test.ts b/packages/opencode/test/storage/json-migration.test.ts new file mode 100644 index 0000000000..c1038ab311 --- /dev/null +++ b/packages/opencode/test/storage/json-migration.test.ts @@ -0,0 +1,279 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test" +import { Database } from "bun:sqlite" +import { drizzle } from "drizzle-orm/bun-sqlite" +import { eq } from "drizzle-orm" +import path from "path" +import fs from "fs/promises" +import os from "os" +import { migrateFromJson } from "../../src/storage/json-migration" +import { ProjectTable } from "../../src/project/project.sql" +import { Project } from "../../src/project/project" +import { + SessionTable, + MessageTable, + PartTable, + SessionDiffTable, + TodoTable, + PermissionTable, +} from "../../src/session/session.sql" +import { SessionShareTable, ShareTable } from "../../src/share/share.sql" +import { migrations } from "../../src/storage/migrations.generated" + +// Test fixtures +const fixtures = { + project: { + id: "proj_test123abc", + name: "Test Project", + worktree: "/test/path", + vcs: "git" as const, + sandboxes: [], + }, + session: { + id: "ses_test456def", + projectID: "proj_test123abc", + slug: "test-session", + directory: "/test/path", + title: "Test Session", + version: "1.0.0", + time: { created: 1700000000000, updated: 1700000001000 }, + }, + message: { + id: "msg_test789ghi", + sessionID: "ses_test456def", + role: "user" as const, + agent: "default", + model: { providerID: "openai", modelID: "gpt-4" }, + time: { created: 1700000000000 }, + }, + part: { + id: "prt_testabc123", + messageID: "msg_test789ghi", + sessionID: "ses_test456def", + type: "text" as const, + text: "Hello, world!", + }, +} + +// Helper to create test storage directory structure +async function setupStorageDir(baseDir: string) { + const storageDir = path.join(baseDir, "storage") + await fs.mkdir(path.join(storageDir, "project"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "session", "proj_test123abc"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "message", "ses_test456def"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "part", "msg_test789ghi"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "session_diff"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "todo"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "permission"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "session_share"), { recursive: true }) + await fs.mkdir(path.join(storageDir, "share"), { recursive: true }) + // Create legacy marker to indicate JSON storage exists + await Bun.write(path.join(storageDir, "migration"), "1") + return storageDir +} + +// Helper to create in-memory test database with schema +function createTestDb() { + const sqlite = new Database(":memory:") + sqlite.exec("PRAGMA foreign_keys = ON") + + // Apply schema migrations + for (const migration of migrations) { + const statements = migration.sql.split("--> statement-breakpoint") + for (const stmt of statements) { + const trimmed = stmt.trim() + if (trimmed) sqlite.exec(trimmed) + } + } + + return sqlite +} + +describe("JSON to SQLite migration", () => { + let tmpDir: string + let storageDir: string + let sqlite: Database + + beforeEach(async () => { + tmpDir = path.join(os.tmpdir(), "opencode-migration-test-" + Math.random().toString(36).slice(2)) + await fs.mkdir(tmpDir, { recursive: true }) + storageDir = await setupStorageDir(tmpDir) + sqlite = createTestDb() + }) + + afterEach(async () => { + sqlite.close() + await fs.rm(tmpDir, { recursive: true, force: true }) + }) + + test("migrates project", async () => { + await Bun.write( + path.join(storageDir, "project", "proj_test123abc.json"), + JSON.stringify({ + id: "proj_test123abc", + worktree: "/test/path", + vcs: "git", + name: "Test Project", + time: { created: 1700000000000, updated: 1700000001000 }, + sandboxes: ["/test/sandbox"], + }), + ) + + const stats = await migrateFromJson(sqlite, storageDir) + + expect(stats?.projects).toBe(1) + + const db = drizzle(sqlite) + const projects = db.select().from(ProjectTable).all() + expect(projects.length).toBe(1) + expect(projects[0].id).toBe("proj_test123abc") + expect(projects[0].worktree).toBe("/test/path") + expect(projects[0].name).toBe("Test Project") + expect(projects[0].sandboxes).toEqual(["/test/sandbox"]) + }) + + test("migrates session with individual columns", async () => { + // First create the project + await Bun.write( + path.join(storageDir, "project", "proj_test123abc.json"), + JSON.stringify({ + id: "proj_test123abc", + worktree: "/test/path", + time: { created: Date.now(), updated: Date.now() }, + sandboxes: [], + }), + ) + + await Bun.write( + path.join(storageDir, "session", "proj_test123abc", "ses_test456def.json"), + JSON.stringify({ + id: "ses_test456def", + projectID: "proj_test123abc", + slug: "test-session", + directory: "/test/dir", + title: "Test Session Title", + version: "1.0.0", + time: { created: 1700000000000, updated: 1700000001000 }, + summary: { additions: 10, deletions: 5, files: 3 }, + share: { url: "https://example.com/share" }, + }), + ) + + await migrateFromJson(sqlite, storageDir) + + const db = drizzle(sqlite) + const sessions = db.select().from(SessionTable).all() + expect(sessions.length).toBe(1) + expect(sessions[0].id).toBe("ses_test456def") + expect(sessions[0].projectID).toBe("proj_test123abc") + expect(sessions[0].slug).toBe("test-session") + expect(sessions[0].title).toBe("Test Session Title") + expect(sessions[0].summary_additions).toBe(10) + expect(sessions[0].summary_deletions).toBe(5) + expect(sessions[0].share_url).toBe("https://example.com/share") + }) + + test("migrates messages and parts", async () => { + await Bun.write( + path.join(storageDir, "project", "proj_test123abc.json"), + JSON.stringify({ + id: "proj_test123abc", + worktree: "/", + time: { created: Date.now(), updated: Date.now() }, + sandboxes: [], + }), + ) + await Bun.write( + path.join(storageDir, "session", "proj_test123abc", "ses_test456def.json"), + JSON.stringify({ ...fixtures.session }), + ) + await Bun.write( + path.join(storageDir, "message", "ses_test456def", "msg_test789ghi.json"), + JSON.stringify({ ...fixtures.message }), + ) + await Bun.write( + path.join(storageDir, "part", "msg_test789ghi", "prt_testabc123.json"), + JSON.stringify({ ...fixtures.part }), + ) + + const stats = await migrateFromJson(sqlite, storageDir) + + expect(stats?.messages).toBe(1) + expect(stats?.parts).toBe(1) + + const db = drizzle(sqlite) + const messages = db.select().from(MessageTable).all() + expect(messages.length).toBe(1) + expect(messages[0].data.id).toBe("msg_test789ghi") + + const parts = db.select().from(PartTable).all() + expect(parts.length).toBe(1) + expect(parts[0].data.id).toBe("prt_testabc123") + }) + + test("skips orphaned sessions (no parent project)", async () => { + await Bun.write( + path.join(storageDir, "session", "proj_test123abc", "ses_orphan.json"), + JSON.stringify({ + id: "ses_orphan", + projectID: "proj_nonexistent", + slug: "orphan", + directory: "/", + title: "Orphan", + version: "1.0.0", + time: { created: Date.now(), updated: Date.now() }, + }), + ) + + const stats = await migrateFromJson(sqlite, storageDir) + + expect(stats?.sessions).toBe(0) + }) + + test("creates sqlite-migrated marker file", async () => { + await migrateFromJson(sqlite, storageDir) + + const marker = path.join(storageDir, "sqlite-migrated") + expect(await Bun.file(marker).exists()).toBe(true) + }) + + test("skips if already migrated", async () => { + await Bun.write(path.join(storageDir, "sqlite-migrated"), Date.now().toString()) + await Bun.write( + path.join(storageDir, "project", "proj_test123abc.json"), + JSON.stringify({ + id: "proj_test123abc", + worktree: "/", + time: { created: Date.now(), updated: Date.now() }, + sandboxes: [], + }), + ) + + const stats = await migrateFromJson(sqlite, storageDir) + + // Should return undefined (skipped) since already migrated + expect(stats).toBeUndefined() + }) + + test("is idempotent (running twice doesn't duplicate)", async () => { + await Bun.write( + path.join(storageDir, "project", "proj_test123abc.json"), + JSON.stringify({ + id: "proj_test123abc", + worktree: "/", + time: { created: Date.now(), updated: Date.now() }, + sandboxes: [], + }), + ) + + await migrateFromJson(sqlite, storageDir) + + // Remove marker to run again + await fs.rm(path.join(storageDir, "sqlite-migrated")) + + await migrateFromJson(sqlite, storageDir) + + const db = drizzle(sqlite) + const projects = db.select().from(ProjectTable).all() + expect(projects.length).toBe(1) // Still only 1 due to onConflictDoNothing + }) +})