diff --git a/bun.lock b/bun.lock index ee8746c42f..677be74810 100644 --- a/bun.lock +++ b/bun.lock @@ -325,6 +325,7 @@ "@aws-sdk/credential-providers": "3.993.0", "@clack/prompts": "1.0.0-alpha.1", "@effect/platform-node": "catalog:", + "@ff-labs/fff-node": "0.4.2", "@gitlab/gitlab-ai-provider": "3.6.0", "@gitlab/opencode-gitlab-auth": "1.3.3", "@hono/standard-validator": "0.1.5", @@ -345,7 +346,6 @@ "@solid-primitives/event-bus": "1.1.2", "@solid-primitives/scheduled": "1.5.2", "@standard-schema/spec": "1.0.0", - "@zip.js/zip.js": "2.7.62", "ai": "catalog:", "ai-gateway-provider": "2.3.1", "bonjour-service": "1.3.0", @@ -1098,6 +1098,24 @@ "@fastify/rate-limit": ["@fastify/rate-limit@10.3.0", "", { "dependencies": { "@lukeed/ms": "^2.0.2", "fastify-plugin": "^5.0.0", "toad-cache": "^3.7.0" } }, "sha512-eIGkG9XKQs0nyynatApA3EVrojHOuq4l6fhB4eeCk4PIOeadvOJz9/4w3vGI44Go17uaXOWEcPkaD8kuKm7g6Q=="], + "@ff-labs/fff-bin-darwin-arm64": ["@ff-labs/fff-bin-darwin-arm64@0.4.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-R9ieZvBaAmYNbdGT3gs2HUQ0Sm4I5tBrJwOepdCoeIZvJFI71hCY2DCFzeoXH2wbxMsPF70c1FSr8qERhcrbVw=="], + + "@ff-labs/fff-bin-darwin-x64": ["@ff-labs/fff-bin-darwin-x64@0.4.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-iJNgUdkS1sVMhWe6l60ZmG9BcSB87CdO65K4AuMbwHQZTHxje9Sapf+AWPGYem6H0endS7HF7ejH+yoZmCF0uw=="], + + "@ff-labs/fff-bin-linux-arm64-gnu": ["@ff-labs/fff-bin-linux-arm64-gnu@0.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-vldJC/j/Kf0LVS599CPTeIaBAd+8J6FFJ1euWn4OoSu63P3CD+9ITrmPWkIGUrt+0myOXABAx0KgLBGADtIAKg=="], + + "@ff-labs/fff-bin-linux-arm64-musl": ["@ff-labs/fff-bin-linux-arm64-musl@0.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-RgL1Oq6QMZm+M4R14SYLtiRMUObA8k+EHIftaplKpLu4Cr0q5lCclRszS0o0Le0hmFrrIvMn6pFRE7LoEzKqAQ=="], + + "@ff-labs/fff-bin-linux-x64-gnu": ["@ff-labs/fff-bin-linux-x64-gnu@0.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-ysImURWrxLT7WFTn46NrXOg4ygbuIp4NkKbWzOAzLYoMOU5JRllUxb3huw3sZNbXn+/9tpq3OE9VmWuAi0YZ/w=="], + + "@ff-labs/fff-bin-linux-x64-musl": ["@ff-labs/fff-bin-linux-x64-musl@0.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-Us4ysq/oCrcf+h5lOPzDbxFJ8WI8bSVbSVudYHYFpe54417oWtbokdbzgb5Yx7108dW7jCDtGkxq+Cnau2002A=="], + + "@ff-labs/fff-bin-win32-arm64": ["@ff-labs/fff-bin-win32-arm64@0.4.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-rkF8lNEUhaZmGESJdQGSnIReg5hMDbb7giFxTlEzgeFRkWZpjKkDajGVuJ+Rd2bI5AoxUNuTDUfvta5EkQ2S5g=="], + + "@ff-labs/fff-bin-win32-x64": ["@ff-labs/fff-bin-win32-x64@0.4.2", "", { "os": "win32", "cpu": "x64" }, "sha512-wtSZiI2/7Z61GdVlGxPtXcuQV4EyoHgVBLhJ5wXcGwEQLp/r8GUWzSpN7iDQaOKKEvHbT2XiEbcbdw+jhDR7qQ=="], + + "@ff-labs/fff-node": ["@ff-labs/fff-node@0.4.2", "", { "dependencies": { "ffi-rs": "^1.0.0" }, "optionalDependencies": { "@ff-labs/fff-bin-darwin-arm64": "0.4.2", "@ff-labs/fff-bin-darwin-x64": "0.4.2", "@ff-labs/fff-bin-linux-arm64-gnu": "0.4.2", "@ff-labs/fff-bin-linux-arm64-musl": "0.4.2", "@ff-labs/fff-bin-linux-x64-gnu": "0.4.2", "@ff-labs/fff-bin-linux-x64-musl": "0.4.2", "@ff-labs/fff-bin-win32-arm64": "0.4.2", "@ff-labs/fff-bin-win32-x64": "0.4.2" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ] }, "sha512-R1jjyvWmLC6qLOxFwdZhhA4UrOZY6r5nuqsuMpdsrDOhMMktJsbhMDZzRqXIy+GXTQBqAF1oBhW6FN6ahTCPBA=="], + "@floating-ui/core": ["@floating-ui/core@1.7.5", "", { "dependencies": { "@floating-ui/utils": "^0.2.11" } }, "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ=="], "@floating-ui/dom": ["@floating-ui/dom@1.7.6", "", { "dependencies": { "@floating-ui/core": "^1.7.5", "@floating-ui/utils": "^0.2.11" } }, "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ=="], @@ -2234,7 +2252,27 @@ "@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="], - "@zip.js/zip.js": ["@zip.js/zip.js@2.7.62", "", {}, "sha512-OaLvZ8j4gCkLn048ypkZu29KX30r8/OfFF2w4Jo5WXFr+J04J+lzJ5TKZBVgFXhlvSkqNFQdfnY1Q8TMTCyBVA=="], + "@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="], + + "@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="], + + "@yuuang/ffi-rs-darwin-x64": ["@yuuang/ffi-rs-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-sI3LpQQ34SX4nyOHc5yxA7FSqs9qPEUMqW/y/wWo9cuyPpaHMFsi/BeOVYsnC0syp3FrY7gzn6RnD6PlXCktXg=="], + + "@yuuang/ffi-rs-linux-arm-gnueabihf": ["@yuuang/ffi-rs-linux-arm-gnueabihf@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-1WkcGkJTlwh4ZA59htKI+RXhiL3oKiYwLv7PO8LUf6FuADK73s5GcXp67iakKu243uYu+qGYr4RHco4ySddYhQ=="], + + "@yuuang/ffi-rs-linux-arm64-gnu": ["@yuuang/ffi-rs-linux-arm64-gnu@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-J2PwqviycZxaEVA0Bwv38LqGDGSB9A1DPN4iYginYJZSvTvKW8kh7Tis0HbZrX1YDKnY8hi3lt0N0tCTNPDH5Q=="], + + "@yuuang/ffi-rs-linux-arm64-musl": ["@yuuang/ffi-rs-linux-arm64-musl@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Hn1W1hBPssTaqikU1Bqp1XUdDdOgbnYVIOtR++LVx66hhrtjf/xrIUQOhTm+NmOFDG16JUKXe1skfM4gpaqYwg=="], + + "@yuuang/ffi-rs-linux-x64-gnu": ["@yuuang/ffi-rs-linux-x64-gnu@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-kW6e+oCYZPvpH2ppPsffA18e1aLowtmWTRjVlyHtY04g/nQDepQvDUkkcvInh9fW5jLna7PjHvktW1tVgYIj2A=="], + + "@yuuang/ffi-rs-linux-x64-musl": ["@yuuang/ffi-rs-linux-x64-musl@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-HTwblAzruUS16nQPrez3ozvEHm1Xxh8J8w7rZYrpmAcNl1hzyOT8z/hY70M9Rt9fOqQ4Ovgor9qVy/U3ZJo0ZA=="], + + "@yuuang/ffi-rs-win32-arm64-msvc": ["@yuuang/ffi-rs-win32-arm64-msvc@1.3.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-WeZkGl2BP1U4tRhEQH+FXLQS52N8obp74smK5AAGOfzPAT1pHkq6+dVkC1QCSIt7dHJs7SPtlnQw+5DkdZYlWA=="], + + "@yuuang/ffi-rs-win32-ia32-msvc": ["@yuuang/ffi-rs-win32-ia32-msvc@1.3.1", "", { "os": "win32", "cpu": [ "x64", "ia32", ] }, "sha512-rNGgMeCH5mdeHiMiJgt7wWXovZ+FHEfXhU9p4zZBH4n8M1/QnEsRUwlapISPLpILSGpoYS6iBuq9/fUlZY8Mhg=="], + + "@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="], "abbrev": ["abbrev@2.0.0", "", {}, "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ=="], @@ -2934,6 +2972,8 @@ "fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="], + "ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="], + "file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="], "filelist": ["filelist@1.0.6", "", { "dependencies": { "minimatch": "^5.0.1" } }, "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA=="], diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 3e73a7021d..0f3e3946b3 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -90,6 +90,7 @@ "@aws-sdk/credential-providers": "3.993.0", "@clack/prompts": "1.0.0-alpha.1", "@effect/platform-node": "catalog:", + "@ff-labs/fff-node": "0.4.2", "@gitlab/gitlab-ai-provider": "3.6.0", "@gitlab/opencode-gitlab-auth": "1.3.3", "@hono/standard-validator": "0.1.5", @@ -110,7 +111,6 @@ "@solid-primitives/event-bus": "1.1.2", "@solid-primitives/scheduled": "1.5.2", "@standard-schema/spec": "1.0.0", - "@zip.js/zip.js": "2.7.62", "ai": "catalog:", "ai-gateway-provider": "2.3.1", "bonjour-service": "1.3.0", diff --git a/packages/opencode/src/cli/cmd/debug/file.ts b/packages/opencode/src/cli/cmd/debug/file.ts index 6faaf399ae..70e2e75761 100644 --- a/packages/opencode/src/cli/cmd/debug/file.ts +++ b/packages/opencode/src/cli/cmd/debug/file.ts @@ -2,7 +2,7 @@ import { EOL } from "os" import { File } from "../../../file" import { bootstrap } from "../../bootstrap" import { cmd } from "../cmd" -import { Ripgrep } from "@/file/ripgrep" +import { Fff } from "@/file/fff" const FileSearchCommand = cmd({ command: "search ", @@ -77,7 +77,7 @@ const FileTreeCommand = cmd({ default: process.cwd(), }), async handler(args) { - const files = await Ripgrep.tree({ cwd: args.dir, limit: 200 }) + const files = await Fff.tree({ cwd: args.dir, limit: 200 }) console.log(JSON.stringify(files, null, 2)) }, }) diff --git a/packages/opencode/src/cli/cmd/debug/index.ts b/packages/opencode/src/cli/cmd/debug/index.ts index 8da6ff5593..65172dc604 100644 --- a/packages/opencode/src/cli/cmd/debug/index.ts +++ b/packages/opencode/src/cli/cmd/debug/index.ts @@ -4,7 +4,7 @@ import { cmd } from "../cmd" import { ConfigCommand } from "./config" import { FileCommand } from "./file" import { LSPCommand } from "./lsp" -import { RipgrepCommand } from "./ripgrep" +import { SearchCommand } from "./search" import { ScrapCommand } from "./scrap" import { SkillCommand } from "./skill" import { SnapshotCommand } from "./snapshot" @@ -17,7 +17,7 @@ export const DebugCommand = cmd({ yargs .command(ConfigCommand) .command(LSPCommand) - .command(RipgrepCommand) + .command(SearchCommand) .command(FileCommand) .command(ScrapCommand) .command(SkillCommand) diff --git a/packages/opencode/src/cli/cmd/debug/ripgrep.ts b/packages/opencode/src/cli/cmd/debug/search.ts similarity index 61% rename from packages/opencode/src/cli/cmd/debug/ripgrep.ts rename to packages/opencode/src/cli/cmd/debug/search.ts index a4cebc5b8f..7f804ebbf5 100644 --- a/packages/opencode/src/cli/cmd/debug/ripgrep.ts +++ b/packages/opencode/src/cli/cmd/debug/search.ts @@ -1,33 +1,34 @@ import { EOL } from "os" -import { Ripgrep } from "../../../file/ripgrep" +import { Fff } from "../../../file/fff" import { Instance } from "../../../project/instance" import { bootstrap } from "../../bootstrap" import { cmd } from "../cmd" +import { Glob } from "@/util/glob" -export const RipgrepCommand = cmd({ - command: "rg", - describe: "ripgrep debugging utilities", - builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(SearchCommand).demandCommand(), +export const SearchCommand = cmd({ + command: "search", + describe: "fff search debugging utilities", + builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(ContentCommand).demandCommand(), async handler() {}, }) const TreeCommand = cmd({ command: "tree", - describe: "show file tree using ripgrep", + describe: "show file tree using fff", builder: (yargs) => yargs.option("limit", { type: "number", }), async handler(args) { await bootstrap(process.cwd(), async () => { - process.stdout.write((await Ripgrep.tree({ cwd: Instance.directory, limit: args.limit })) + EOL) + process.stdout.write((await Fff.tree({ cwd: Instance.directory, limit: args.limit })) + EOL) }) }, }) const FilesCommand = cmd({ command: "files", - describe: "list files using ripgrep", + describe: "list files using fff", builder: (yargs) => yargs .option("query", { @@ -44,22 +45,24 @@ const FilesCommand = cmd({ }), async handler(args) { await bootstrap(process.cwd(), async () => { - const files: string[] = [] - for await (const file of Ripgrep.files({ + const limit = args.limit ?? 100 + const files = (await Glob.scan("**/*", { cwd: Instance.directory, - glob: args.glob ? [args.glob] : undefined, - })) { - files.push(file) - if (args.limit && files.length >= args.limit) break - } + include: "file", + dot: true, + })) + .map((x) => x.replaceAll("\\", "/")) + .filter((x) => Fff.allowed({ rel: x, hidden: true, glob: args.glob ? [args.glob] : undefined })) + .filter((x) => !args.query || x.includes(args.query)) + .slice(0, limit) process.stdout.write(files.join(EOL) + EOL) }) }, }) -const SearchCommand = cmd({ - command: "search ", - describe: "search file contents using ripgrep", +const ContentCommand = cmd({ + command: "content ", + describe: "search file contents using fff", builder: (yargs) => yargs .positional("pattern", { @@ -76,12 +79,12 @@ const SearchCommand = cmd({ description: "Limit number of results", }), async handler(args) { - const results = await Ripgrep.search({ + const rows = await Fff.search({ cwd: process.cwd(), pattern: args.pattern, glob: args.glob as string[] | undefined, limit: args.limit, }) - process.stdout.write(JSON.stringify(results, null, 2) + EOL) + process.stdout.write(JSON.stringify(rows, null, 2) + EOL) }, }) diff --git a/packages/opencode/src/file/fff.ts b/packages/opencode/src/file/fff.ts new file mode 100644 index 0000000000..8996223d14 --- /dev/null +++ b/packages/opencode/src/file/fff.ts @@ -0,0 +1,273 @@ +import fs from "fs/promises" +import path from "path" +import { + FileFinder, + type FileItem, + type GrepCursor, + type GrepMatch, + type GrepMode, + type SearchResult, +} from "@ff-labs/fff-node" +import z from "zod" +import { Global } from "../global" +import { Instance } from "../project/instance" +import { Filesystem } from "../util/filesystem" +import { Glob } from "../util/glob" +import { Log } from "../util/log" + +export namespace Fff { + const log = Log.create({ service: "file.fff" }) + + export const Match = z.object({ + path: z.object({ + text: z.string(), + }), + lines: z.object({ + text: z.string(), + }), + line_number: z.number(), + absolute_offset: z.number(), + submatches: z.array( + z.object({ + match: z.object({ + text: z.string(), + }), + start: z.number(), + end: z.number(), + }), + ), + }) + + const state = Instance.state( + async () => ({ + map: new Map(), + pending: new Map>(), + }), + async (state) => { + for (const pick of state.map.values()) pick.destroy() + }, + ) + + const root = path.join(Global.Path.cache, "fff") + + function key(dir: string) { + return Buffer.from(dir).toString("base64url") + } + + async function db(dir: string) { + await fs.mkdir(root, { recursive: true }) + const id = key(dir) + return { + frecency: path.join(root, `${id}.frecency.mdb`), + history: path.join(root, `${id}.history.mdb`), + } + } + + function refresh(pick: FileFinder) { + const git = pick.refreshGitStatus() + if (!git.ok) { + log.warn("git refresh failed", { error: git.error }) + return + } + } + + export async function picker(cwd: string) { + const dir = Filesystem.resolve(cwd) + const memo = await state() + const cached = memo.map.get(dir) + if (cached) return cached + + const wait = memo.pending.get(dir) + if (wait) return wait + + const next = (async () => { + const files = await db(dir) + const made = FileFinder.create({ + basePath: dir, + frecencyDbPath: files.frecency, + historyDbPath: files.history, + aiMode: true, + }) + if (!made.ok) throw new Error(made.error) + + const pick = made.value + const done = await pick.waitForScan(5000) + if (!done.ok) { + pick.destroy() + throw new Error(done.error) + } + + memo.map.set(dir, pick) + refresh(pick) + return pick + })() + + memo.pending.set(dir, next) + try { + return await next + } finally { + if (memo.pending.get(dir) === next) memo.pending.delete(dir) + } + } + + export async function files(input: { cwd: string; query: string; page?: number; size?: number; current?: string }) { + const pick = await picker(input.cwd) + const out = pick.fileSearch(input.query, { + pageIndex: input.page ?? 0, + pageSize: input.size ?? 100, + currentFile: input.current, + }) + if (!out.ok) throw new Error(out.error) + return out.value + } + + export async function grep(input: { + cwd: string + query: string + mode?: GrepMode + max?: number + before?: number + after?: number + budget?: number + cursor?: GrepCursor | null + }) { + const pick = await picker(input.cwd) + const out = pick.grep(input.query, { + mode: input.mode, + maxMatchesPerFile: input.max, + beforeContext: input.before, + afterContext: input.after, + timeBudgetMs: input.budget, + cursor: input.cursor, + }) + if (!out.ok) throw new Error(out.error) + return out.value + } + + function norm(text: string) { + return text.replaceAll("\\", "/") + } + + function hidden(rel: string) { + return norm(rel) + .split("/") + .some((part) => part.startsWith(".")) + } + + function accept(rel: string, file: string, glob?: string[], show?: boolean) { + if (show === false && hidden(rel)) return false + if (!glob?.length) return true + const allow = glob.filter((x) => !x.startsWith("!")) + const deny = glob.filter((x) => x.startsWith("!")).map((x) => x.slice(1)) + if (allow.length > 0 && !allow.some((x) => Glob.match(x, rel) || Glob.match(x, file))) return false + if (deny.some((x) => Glob.match(x, rel) || Glob.match(x, file))) return false + return true + } + + export function allowed(input: { rel: string; file?: string; glob?: string[]; hidden?: boolean }) { + return accept(input.rel, input.file ?? input.rel.split("/").at(-1) ?? input.rel, input.glob, input.hidden !== false) + } + + export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) { + input.signal?.throwIfAborted() + const files = (await Glob.scan("**/*", { + cwd: input.cwd, + include: "file", + dot: true, + })) + .map((row) => norm(row)) + .filter((row) => allowed({ rel: row, hidden: true })) + .toSorted((a, b) => a.localeCompare(b)) + input.signal?.throwIfAborted() + interface Node { + name: string + children: Map + } + + function dir(node: Node, name: string) { + const old = node.children.get(name) + if (old) return old + const next = { name, children: new Map() } + node.children.set(name, next) + return next + } + + const root = { name: "", children: new Map() } + for (const file of files) { + if (file.includes(".opencode")) continue + const parts = file.split("/") + if (parts.length < 2) continue + let node = root + for (const part of parts.slice(0, -1)) { + node = dir(node, part) + } + } + + function count(node: Node): number { + return Array.from(node.children.values()).reduce((sum, child) => sum + 1 + count(child), 0) + } + + const total = count(root) + const limit = input.limit ?? total + const lines: string[] = [] + const queue = Array.from(root.children.values()) + .toSorted((a, b) => a.name.localeCompare(b.name)) + .map((node) => ({ node, path: node.name })) + + let used = 0 + for (let i = 0; i < queue.length && used < limit; i++) { + input.signal?.throwIfAborted() + const row = queue[i] + lines.push(row.path) + used++ + queue.push( + ...Array.from(row.node.children.values()) + .toSorted((a, b) => a.name.localeCompare(b.name)) + .map((node) => ({ node, path: `${row.path}/${node.name}` })), + ) + } + if (total > used) lines.push(`[${total - used} truncated]`) + input.signal?.throwIfAborted() + return lines.join("\n") + } + + export async function search(input: { + cwd: string + pattern: string + glob?: string[] + limit?: number + follow?: boolean + }) { + const out = await grep({ + cwd: input.cwd, + query: input.pattern, + mode: "regex", + max: input.limit, + }) + const rows = out.items + .filter((row) => accept(norm(row.relativePath), row.fileName, input.glob, true)) + .slice(0, input.limit) + .map((row) => ({ + path: { text: row.relativePath }, + lines: { text: row.lineContent }, + line_number: row.lineNumber, + absolute_offset: row.byteOffset, + submatches: row.matchRanges + .map(([start, end]) => { + const text = row.lineContent.slice(start, end) + if (!text) return undefined + return { + match: { text }, + start, + end, + } + }) + .filter((row) => row !== undefined), + })) + return Match.array().parse(rows) + } + + export type Search = SearchResult + export type File = FileItem + export type Hit = GrepMatch +} diff --git a/packages/opencode/src/file/index.ts b/packages/opencode/src/file/index.ts index 6e9b917271..26b5fefce1 100644 --- a/packages/opencode/src/file/index.ts +++ b/packages/opencode/src/file/index.ts @@ -12,9 +12,9 @@ import z from "zod" import { Global } from "../global" import { Instance } from "../project/instance" import { Filesystem } from "../util/filesystem" +import { Glob } from "../util/glob" import { Log } from "../util/log" import { Protected } from "./protected" -import { Ripgrep } from "./ripgrep" export namespace File { export const Info = z @@ -401,7 +401,11 @@ export namespace File { next.dirs = Array.from(dirs).toSorted() } else { const seen = new Set() - for await (const file of Ripgrep.files({ cwd: instance.directory })) { + for (const file of (await Glob.scan("**/*", { + cwd: instance.directory, + include: "file", + dot: true, + })).toSorted((a, b) => a.localeCompare(b))) { next.files.push(file) let current = file while (true) { diff --git a/packages/opencode/src/file/ripgrep.ts b/packages/opencode/src/file/ripgrep.ts deleted file mode 100644 index 601c82e94f..0000000000 --- a/packages/opencode/src/file/ripgrep.ts +++ /dev/null @@ -1,376 +0,0 @@ -// Ripgrep utility functions -import path from "path" -import { Global } from "../global" -import fs from "fs/promises" -import z from "zod" -import { NamedError } from "@opencode-ai/util/error" -import { lazy } from "../util/lazy" - -import { Filesystem } from "../util/filesystem" -import { Process } from "../util/process" -import { which } from "../util/which" -import { text } from "node:stream/consumers" - -import { ZipReader, BlobReader, BlobWriter } from "@zip.js/zip.js" -import { Log } from "@/util/log" - -export namespace Ripgrep { - const log = Log.create({ service: "ripgrep" }) - const Stats = z.object({ - elapsed: z.object({ - secs: z.number(), - nanos: z.number(), - human: z.string(), - }), - searches: z.number(), - searches_with_match: z.number(), - bytes_searched: z.number(), - bytes_printed: z.number(), - matched_lines: z.number(), - matches: z.number(), - }) - - const Begin = z.object({ - type: z.literal("begin"), - data: z.object({ - path: z.object({ - text: z.string(), - }), - }), - }) - - export const Match = z.object({ - type: z.literal("match"), - data: z.object({ - path: z.object({ - text: z.string(), - }), - lines: z.object({ - text: z.string(), - }), - line_number: z.number(), - absolute_offset: z.number(), - submatches: z.array( - z.object({ - match: z.object({ - text: z.string(), - }), - start: z.number(), - end: z.number(), - }), - ), - }), - }) - - const End = z.object({ - type: z.literal("end"), - data: z.object({ - path: z.object({ - text: z.string(), - }), - binary_offset: z.number().nullable(), - stats: Stats, - }), - }) - - const Summary = z.object({ - type: z.literal("summary"), - data: z.object({ - elapsed_total: z.object({ - human: z.string(), - nanos: z.number(), - secs: z.number(), - }), - stats: Stats, - }), - }) - - const Result = z.union([Begin, Match, End, Summary]) - - export type Result = z.infer - export type Match = z.infer - export type Begin = z.infer - export type End = z.infer - export type Summary = z.infer - const PLATFORM = { - "arm64-darwin": { platform: "aarch64-apple-darwin", extension: "tar.gz" }, - "arm64-linux": { - platform: "aarch64-unknown-linux-gnu", - extension: "tar.gz", - }, - "x64-darwin": { platform: "x86_64-apple-darwin", extension: "tar.gz" }, - "x64-linux": { platform: "x86_64-unknown-linux-musl", extension: "tar.gz" }, - "arm64-win32": { platform: "aarch64-pc-windows-msvc", extension: "zip" }, - "x64-win32": { platform: "x86_64-pc-windows-msvc", extension: "zip" }, - } as const - - export const ExtractionFailedError = NamedError.create( - "RipgrepExtractionFailedError", - z.object({ - filepath: z.string(), - stderr: z.string(), - }), - ) - - export const UnsupportedPlatformError = NamedError.create( - "RipgrepUnsupportedPlatformError", - z.object({ - platform: z.string(), - }), - ) - - export const DownloadFailedError = NamedError.create( - "RipgrepDownloadFailedError", - z.object({ - url: z.string(), - status: z.number(), - }), - ) - - const state = lazy(async () => { - const system = which("rg") - if (system) { - const stat = await fs.stat(system).catch(() => undefined) - if (stat?.isFile()) return { filepath: system } - log.warn("bun.which returned invalid rg path", { filepath: system }) - } - const filepath = path.join(Global.Path.bin, "rg" + (process.platform === "win32" ? ".exe" : "")) - - if (!(await Filesystem.exists(filepath))) { - const platformKey = `${process.arch}-${process.platform}` as keyof typeof PLATFORM - const config = PLATFORM[platformKey] - if (!config) throw new UnsupportedPlatformError({ platform: platformKey }) - - const version = "14.1.1" - const filename = `ripgrep-${version}-${config.platform}.${config.extension}` - const url = `https://github.com/BurntSushi/ripgrep/releases/download/${version}/${filename}` - - const response = await fetch(url) - if (!response.ok) throw new DownloadFailedError({ url, status: response.status }) - - const arrayBuffer = await response.arrayBuffer() - const archivePath = path.join(Global.Path.bin, filename) - await Filesystem.write(archivePath, Buffer.from(arrayBuffer)) - if (config.extension === "tar.gz") { - const args = ["tar", "-xzf", archivePath, "--strip-components=1"] - - if (platformKey.endsWith("-darwin")) args.push("--include=*/rg") - if (platformKey.endsWith("-linux")) args.push("--wildcards", "*/rg") - - const proc = Process.spawn(args, { - cwd: Global.Path.bin, - stderr: "pipe", - stdout: "pipe", - }) - const exit = await proc.exited - if (exit !== 0) { - const stderr = proc.stderr ? await text(proc.stderr) : "" - throw new ExtractionFailedError({ - filepath, - stderr, - }) - } - } - if (config.extension === "zip") { - const zipFileReader = new ZipReader(new BlobReader(new Blob([arrayBuffer]))) - const entries = await zipFileReader.getEntries() - let rgEntry: any - for (const entry of entries) { - if (entry.filename.endsWith("rg.exe")) { - rgEntry = entry - break - } - } - - if (!rgEntry) { - throw new ExtractionFailedError({ - filepath: archivePath, - stderr: "rg.exe not found in zip archive", - }) - } - - const rgBlob = await rgEntry.getData(new BlobWriter()) - if (!rgBlob) { - throw new ExtractionFailedError({ - filepath: archivePath, - stderr: "Failed to extract rg.exe from zip archive", - }) - } - await Filesystem.write(filepath, Buffer.from(await rgBlob.arrayBuffer())) - await zipFileReader.close() - } - await fs.unlink(archivePath) - if (!platformKey.endsWith("-win32")) await fs.chmod(filepath, 0o755) - } - - return { - filepath, - } - }) - - export async function filepath() { - const { filepath } = await state() - return filepath - } - - export async function* files(input: { - cwd: string - glob?: string[] - hidden?: boolean - follow?: boolean - maxDepth?: number - signal?: AbortSignal - }) { - input.signal?.throwIfAborted() - - const args = [await filepath(), "--files", "--glob=!.git/*"] - if (input.follow) args.push("--follow") - if (input.hidden !== false) args.push("--hidden") - if (input.maxDepth !== undefined) args.push(`--max-depth=${input.maxDepth}`) - if (input.glob) { - for (const g of input.glob) { - args.push(`--glob=${g}`) - } - } - - // Guard against invalid cwd to provide a consistent ENOENT error. - if (!(await fs.stat(input.cwd).catch(() => undefined))?.isDirectory()) { - throw Object.assign(new Error(`No such file or directory: '${input.cwd}'`), { - code: "ENOENT", - errno: -2, - path: input.cwd, - }) - } - - const proc = Process.spawn(args, { - cwd: input.cwd, - stdout: "pipe", - stderr: "ignore", - abort: input.signal, - }) - - if (!proc.stdout) { - throw new Error("Process output not available") - } - - let buffer = "" - const stream = proc.stdout as AsyncIterable - for await (const chunk of stream) { - input.signal?.throwIfAborted() - - buffer += typeof chunk === "string" ? chunk : chunk.toString() - // Handle both Unix (\n) and Windows (\r\n) line endings - const lines = buffer.split(/\r?\n/) - buffer = lines.pop() || "" - - for (const line of lines) { - if (line) yield line - } - } - - if (buffer) yield buffer - await proc.exited - - input.signal?.throwIfAborted() - } - - export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) { - log.info("tree", input) - const files = await Array.fromAsync(Ripgrep.files({ cwd: input.cwd, signal: input.signal })) - interface Node { - name: string - children: Map - } - - function dir(node: Node, name: string) { - const existing = node.children.get(name) - if (existing) return existing - const next = { name, children: new Map() } - node.children.set(name, next) - return next - } - - const root: Node = { name: "", children: new Map() } - for (const file of files) { - if (file.includes(".opencode")) continue - const parts = file.split(path.sep) - if (parts.length < 2) continue - let node = root - for (const part of parts.slice(0, -1)) { - node = dir(node, part) - } - } - - function count(node: Node): number { - let total = 0 - for (const child of node.children.values()) { - total += 1 + count(child) - } - return total - } - - const total = count(root) - const limit = input.limit ?? total - const lines: string[] = [] - const queue: { node: Node; path: string }[] = [] - for (const child of Array.from(root.children.values()).sort((a, b) => a.name.localeCompare(b.name))) { - queue.push({ node: child, path: child.name }) - } - - let used = 0 - for (let i = 0; i < queue.length && used < limit; i++) { - const { node, path } = queue[i] - lines.push(path) - used++ - for (const child of Array.from(node.children.values()).sort((a, b) => a.name.localeCompare(b.name))) { - queue.push({ node: child, path: `${path}/${child.name}` }) - } - } - - if (total > used) lines.push(`[${total - used} truncated]`) - - return lines.join("\n") - } - - export async function search(input: { - cwd: string - pattern: string - glob?: string[] - limit?: number - follow?: boolean - }) { - const args = [`${await filepath()}`, "--json", "--hidden", "--glob=!.git/*"] - if (input.follow) args.push("--follow") - - if (input.glob) { - for (const g of input.glob) { - args.push(`--glob=${g}`) - } - } - - if (input.limit) { - args.push(`--max-count=${input.limit}`) - } - - args.push("--") - args.push(input.pattern) - - const result = await Process.text(args, { - cwd: input.cwd, - nothrow: true, - }) - if (result.code !== 0) { - return [] - } - - // Handle both Unix (\n) and Windows (\r\n) line endings - const lines = result.text.trim().split(/\r?\n/).filter(Boolean) - // Parse JSON lines from ripgrep output - - return lines - .map((line) => JSON.parse(line)) - .map((parsed) => Result.parse(parsed)) - .filter((r) => r.type === "match") - .map((r) => r.data) - } -} diff --git a/packages/opencode/src/server/routes/file.ts b/packages/opencode/src/server/routes/file.ts index 60789ef4b7..1061d2d12e 100644 --- a/packages/opencode/src/server/routes/file.ts +++ b/packages/opencode/src/server/routes/file.ts @@ -2,7 +2,7 @@ import { Hono } from "hono" import { describeRoute, validator, resolver } from "hono-openapi" import z from "zod" import { File } from "../../file" -import { Ripgrep } from "../../file/ripgrep" +import { Fff } from "../../file/fff" import { LSP } from "../../lsp" import { Instance } from "../../project/instance" import { lazy } from "../../util/lazy" @@ -13,14 +13,14 @@ export const FileRoutes = lazy(() => "/find", describeRoute({ summary: "Find text", - description: "Search for text patterns across files in the project using ripgrep.", + description: "Search for text patterns across files in the project.", operationId: "find.text", responses: { 200: { description: "Matches", content: { "application/json": { - schema: resolver(Ripgrep.Match.shape.data.array()), + schema: resolver(Fff.Match.array()), }, }, }, @@ -34,7 +34,7 @@ export const FileRoutes = lazy(() => ), async (c) => { const pattern = c.req.valid("query").pattern - const result = await Ripgrep.search({ + const result = await Fff.search({ cwd: Instance.directory, pattern, limit: 10, diff --git a/packages/opencode/test/file/ripgrep.test.ts b/packages/opencode/test/file/ripgrep.test.ts deleted file mode 100644 index 5eb56e53de..0000000000 --- a/packages/opencode/test/file/ripgrep.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, test } from "bun:test" -import fs from "fs/promises" -import path from "path" -import { tmpdir } from "../fixture/fixture" -import { Ripgrep } from "../../src/file/ripgrep" - -describe("file.ripgrep", () => { - test("defaults to include hidden", async () => { - await using tmp = await tmpdir({ - init: async (dir) => { - await Bun.write(path.join(dir, "visible.txt"), "hello") - await fs.mkdir(path.join(dir, ".opencode"), { recursive: true }) - await Bun.write(path.join(dir, ".opencode", "thing.json"), "{}") - }, - }) - - const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path })) - const hasVisible = files.includes("visible.txt") - const hasHidden = files.includes(path.join(".opencode", "thing.json")) - expect(hasVisible).toBe(true) - expect(hasHidden).toBe(true) - }) - - test("hidden false excludes hidden", async () => { - await using tmp = await tmpdir({ - init: async (dir) => { - await Bun.write(path.join(dir, "visible.txt"), "hello") - await fs.mkdir(path.join(dir, ".opencode"), { recursive: true }) - await Bun.write(path.join(dir, ".opencode", "thing.json"), "{}") - }, - }) - - const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path, hidden: false })) - const hasVisible = files.includes("visible.txt") - const hasHidden = files.includes(path.join(".opencode", "thing.json")) - expect(hasVisible).toBe(true) - expect(hasHidden).toBe(false) - }) - - test("search returns empty when nothing matches", async () => { - await using tmp = await tmpdir({ - init: async (dir) => { - await Bun.write(path.join(dir, "match.ts"), "const value = 'other'\n") - }, - }) - - const hits = await Ripgrep.search({ - cwd: tmp.path, - pattern: "needle", - }) - - expect(hits).toEqual([]) - }) -})