feat(file): add fff-backed search

pull/18419/head
Shoubhit Dash 2026-03-20 22:25:05 +05:30
parent 0bbf26a1ce
commit b5a7ad7085
10 changed files with 353 additions and 463 deletions

View File

@ -325,6 +325,7 @@
"@aws-sdk/credential-providers": "3.993.0",
"@clack/prompts": "1.0.0-alpha.1",
"@effect/platform-node": "catalog:",
"@ff-labs/fff-node": "0.4.2",
"@gitlab/gitlab-ai-provider": "3.6.0",
"@gitlab/opencode-gitlab-auth": "1.3.3",
"@hono/standard-validator": "0.1.5",
@ -345,7 +346,6 @@
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@standard-schema/spec": "1.0.0",
"@zip.js/zip.js": "2.7.62",
"ai": "catalog:",
"ai-gateway-provider": "2.3.1",
"bonjour-service": "1.3.0",
@ -1098,6 +1098,24 @@
"@fastify/rate-limit": ["@fastify/rate-limit@10.3.0", "", { "dependencies": { "@lukeed/ms": "^2.0.2", "fastify-plugin": "^5.0.0", "toad-cache": "^3.7.0" } }, "sha512-eIGkG9XKQs0nyynatApA3EVrojHOuq4l6fhB4eeCk4PIOeadvOJz9/4w3vGI44Go17uaXOWEcPkaD8kuKm7g6Q=="],
"@ff-labs/fff-bin-darwin-arm64": ["@ff-labs/fff-bin-darwin-arm64@0.4.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-R9ieZvBaAmYNbdGT3gs2HUQ0Sm4I5tBrJwOepdCoeIZvJFI71hCY2DCFzeoXH2wbxMsPF70c1FSr8qERhcrbVw=="],
"@ff-labs/fff-bin-darwin-x64": ["@ff-labs/fff-bin-darwin-x64@0.4.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-iJNgUdkS1sVMhWe6l60ZmG9BcSB87CdO65K4AuMbwHQZTHxje9Sapf+AWPGYem6H0endS7HF7ejH+yoZmCF0uw=="],
"@ff-labs/fff-bin-linux-arm64-gnu": ["@ff-labs/fff-bin-linux-arm64-gnu@0.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-vldJC/j/Kf0LVS599CPTeIaBAd+8J6FFJ1euWn4OoSu63P3CD+9ITrmPWkIGUrt+0myOXABAx0KgLBGADtIAKg=="],
"@ff-labs/fff-bin-linux-arm64-musl": ["@ff-labs/fff-bin-linux-arm64-musl@0.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-RgL1Oq6QMZm+M4R14SYLtiRMUObA8k+EHIftaplKpLu4Cr0q5lCclRszS0o0Le0hmFrrIvMn6pFRE7LoEzKqAQ=="],
"@ff-labs/fff-bin-linux-x64-gnu": ["@ff-labs/fff-bin-linux-x64-gnu@0.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-ysImURWrxLT7WFTn46NrXOg4ygbuIp4NkKbWzOAzLYoMOU5JRllUxb3huw3sZNbXn+/9tpq3OE9VmWuAi0YZ/w=="],
"@ff-labs/fff-bin-linux-x64-musl": ["@ff-labs/fff-bin-linux-x64-musl@0.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-Us4ysq/oCrcf+h5lOPzDbxFJ8WI8bSVbSVudYHYFpe54417oWtbokdbzgb5Yx7108dW7jCDtGkxq+Cnau2002A=="],
"@ff-labs/fff-bin-win32-arm64": ["@ff-labs/fff-bin-win32-arm64@0.4.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-rkF8lNEUhaZmGESJdQGSnIReg5hMDbb7giFxTlEzgeFRkWZpjKkDajGVuJ+Rd2bI5AoxUNuTDUfvta5EkQ2S5g=="],
"@ff-labs/fff-bin-win32-x64": ["@ff-labs/fff-bin-win32-x64@0.4.2", "", { "os": "win32", "cpu": "x64" }, "sha512-wtSZiI2/7Z61GdVlGxPtXcuQV4EyoHgVBLhJ5wXcGwEQLp/r8GUWzSpN7iDQaOKKEvHbT2XiEbcbdw+jhDR7qQ=="],
"@ff-labs/fff-node": ["@ff-labs/fff-node@0.4.2", "", { "dependencies": { "ffi-rs": "^1.0.0" }, "optionalDependencies": { "@ff-labs/fff-bin-darwin-arm64": "0.4.2", "@ff-labs/fff-bin-darwin-x64": "0.4.2", "@ff-labs/fff-bin-linux-arm64-gnu": "0.4.2", "@ff-labs/fff-bin-linux-arm64-musl": "0.4.2", "@ff-labs/fff-bin-linux-x64-gnu": "0.4.2", "@ff-labs/fff-bin-linux-x64-musl": "0.4.2", "@ff-labs/fff-bin-win32-arm64": "0.4.2", "@ff-labs/fff-bin-win32-x64": "0.4.2" }, "os": [ "linux", "win32", "darwin", ], "cpu": [ "x64", "arm64", ] }, "sha512-R1jjyvWmLC6qLOxFwdZhhA4UrOZY6r5nuqsuMpdsrDOhMMktJsbhMDZzRqXIy+GXTQBqAF1oBhW6FN6ahTCPBA=="],
"@floating-ui/core": ["@floating-ui/core@1.7.5", "", { "dependencies": { "@floating-ui/utils": "^0.2.11" } }, "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ=="],
"@floating-ui/dom": ["@floating-ui/dom@1.7.6", "", { "dependencies": { "@floating-ui/core": "^1.7.5", "@floating-ui/utils": "^0.2.11" } }, "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ=="],
@ -2234,7 +2252,27 @@
"@xmldom/xmldom": ["@xmldom/xmldom@0.8.11", "", {}, "sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw=="],
"@zip.js/zip.js": ["@zip.js/zip.js@2.7.62", "", {}, "sha512-OaLvZ8j4gCkLn048ypkZu29KX30r8/OfFF2w4Jo5WXFr+J04J+lzJ5TKZBVgFXhlvSkqNFQdfnY1Q8TMTCyBVA=="],
"@yuuang/ffi-rs-android-arm64": ["@yuuang/ffi-rs-android-arm64@1.3.1", "", { "os": "android", "cpu": "arm64" }, "sha512-V4nmlXdOYZEa7GOxSExVG95SLp8FE0iTq2yKeN54UlfNMr3Sik+1Ff57LcCv7qYcn4TBqnBAt5rT3FAM6T6caQ=="],
"@yuuang/ffi-rs-darwin-arm64": ["@yuuang/ffi-rs-darwin-arm64@1.3.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-YlnTMIyzfW3mAULC5ZA774nzQfFlYXM0rrfq/8ZzWt+IMbYk55a++jrI+6JeKV+1EqlDS3TFBEFtjdBNG94KzQ=="],
"@yuuang/ffi-rs-darwin-x64": ["@yuuang/ffi-rs-darwin-x64@1.3.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-sI3LpQQ34SX4nyOHc5yxA7FSqs9qPEUMqW/y/wWo9cuyPpaHMFsi/BeOVYsnC0syp3FrY7gzn6RnD6PlXCktXg=="],
"@yuuang/ffi-rs-linux-arm-gnueabihf": ["@yuuang/ffi-rs-linux-arm-gnueabihf@1.3.1", "", { "os": "linux", "cpu": "arm" }, "sha512-1WkcGkJTlwh4ZA59htKI+RXhiL3oKiYwLv7PO8LUf6FuADK73s5GcXp67iakKu243uYu+qGYr4RHco4ySddYhQ=="],
"@yuuang/ffi-rs-linux-arm64-gnu": ["@yuuang/ffi-rs-linux-arm64-gnu@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-J2PwqviycZxaEVA0Bwv38LqGDGSB9A1DPN4iYginYJZSvTvKW8kh7Tis0HbZrX1YDKnY8hi3lt0N0tCTNPDH5Q=="],
"@yuuang/ffi-rs-linux-arm64-musl": ["@yuuang/ffi-rs-linux-arm64-musl@1.3.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-Hn1W1hBPssTaqikU1Bqp1XUdDdOgbnYVIOtR++LVx66hhrtjf/xrIUQOhTm+NmOFDG16JUKXe1skfM4gpaqYwg=="],
"@yuuang/ffi-rs-linux-x64-gnu": ["@yuuang/ffi-rs-linux-x64-gnu@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-kW6e+oCYZPvpH2ppPsffA18e1aLowtmWTRjVlyHtY04g/nQDepQvDUkkcvInh9fW5jLna7PjHvktW1tVgYIj2A=="],
"@yuuang/ffi-rs-linux-x64-musl": ["@yuuang/ffi-rs-linux-x64-musl@1.3.1", "", { "os": "linux", "cpu": "x64" }, "sha512-HTwblAzruUS16nQPrez3ozvEHm1Xxh8J8w7rZYrpmAcNl1hzyOT8z/hY70M9Rt9fOqQ4Ovgor9qVy/U3ZJo0ZA=="],
"@yuuang/ffi-rs-win32-arm64-msvc": ["@yuuang/ffi-rs-win32-arm64-msvc@1.3.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-WeZkGl2BP1U4tRhEQH+FXLQS52N8obp74smK5AAGOfzPAT1pHkq6+dVkC1QCSIt7dHJs7SPtlnQw+5DkdZYlWA=="],
"@yuuang/ffi-rs-win32-ia32-msvc": ["@yuuang/ffi-rs-win32-ia32-msvc@1.3.1", "", { "os": "win32", "cpu": [ "x64", "ia32", ] }, "sha512-rNGgMeCH5mdeHiMiJgt7wWXovZ+FHEfXhU9p4zZBH4n8M1/QnEsRUwlapISPLpILSGpoYS6iBuq9/fUlZY8Mhg=="],
"@yuuang/ffi-rs-win32-x64-msvc": ["@yuuang/ffi-rs-win32-x64-msvc@1.3.1", "", { "os": "win32", "cpu": "x64" }, "sha512-dr2LcLD2CXo2a7BktlOpV68QhayqiI112KxIJC9tBgQO/Dkdg4CPsdqmvzzLhFo64iC5RLl2BT7M5lJImrfUWw=="],
"abbrev": ["abbrev@2.0.0", "", {}, "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ=="],
@ -2934,6 +2972,8 @@
"fetch-blob": ["fetch-blob@3.2.0", "", { "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" } }, "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ=="],
"ffi-rs": ["ffi-rs@1.3.1", "", { "optionalDependencies": { "@yuuang/ffi-rs-android-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-arm64": "1.3.1", "@yuuang/ffi-rs-darwin-x64": "1.3.1", "@yuuang/ffi-rs-linux-arm-gnueabihf": "1.3.1", "@yuuang/ffi-rs-linux-arm64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-arm64-musl": "1.3.1", "@yuuang/ffi-rs-linux-x64-gnu": "1.3.1", "@yuuang/ffi-rs-linux-x64-musl": "1.3.1", "@yuuang/ffi-rs-win32-arm64-msvc": "1.3.1", "@yuuang/ffi-rs-win32-ia32-msvc": "1.3.1", "@yuuang/ffi-rs-win32-x64-msvc": "1.3.1" } }, "sha512-ZyNXL9fnclnZV+waQmWB9JrfbIEyxQa1OWtMrHOrAgcC04PgP5hBMG5TdhVN8N4uT/eul8zCFMVnJUukAFFlXA=="],
"file-type": ["file-type@16.5.4", "", { "dependencies": { "readable-web-to-node-stream": "^3.0.0", "strtok3": "^6.2.4", "token-types": "^4.1.1" } }, "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw=="],
"filelist": ["filelist@1.0.6", "", { "dependencies": { "minimatch": "^5.0.1" } }, "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA=="],

View File

@ -90,6 +90,7 @@
"@aws-sdk/credential-providers": "3.993.0",
"@clack/prompts": "1.0.0-alpha.1",
"@effect/platform-node": "catalog:",
"@ff-labs/fff-node": "0.4.2",
"@gitlab/gitlab-ai-provider": "3.6.0",
"@gitlab/opencode-gitlab-auth": "1.3.3",
"@hono/standard-validator": "0.1.5",
@ -110,7 +111,6 @@
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@standard-schema/spec": "1.0.0",
"@zip.js/zip.js": "2.7.62",
"ai": "catalog:",
"ai-gateway-provider": "2.3.1",
"bonjour-service": "1.3.0",

View File

@ -2,7 +2,7 @@ import { EOL } from "os"
import { File } from "../../../file"
import { bootstrap } from "../../bootstrap"
import { cmd } from "../cmd"
import { Ripgrep } from "@/file/ripgrep"
import { Fff } from "@/file/fff"
const FileSearchCommand = cmd({
command: "search <query>",
@ -77,7 +77,7 @@ const FileTreeCommand = cmd({
default: process.cwd(),
}),
async handler(args) {
const files = await Ripgrep.tree({ cwd: args.dir, limit: 200 })
const files = await Fff.tree({ cwd: args.dir, limit: 200 })
console.log(JSON.stringify(files, null, 2))
},
})

View File

@ -4,7 +4,7 @@ import { cmd } from "../cmd"
import { ConfigCommand } from "./config"
import { FileCommand } from "./file"
import { LSPCommand } from "./lsp"
import { RipgrepCommand } from "./ripgrep"
import { SearchCommand } from "./search"
import { ScrapCommand } from "./scrap"
import { SkillCommand } from "./skill"
import { SnapshotCommand } from "./snapshot"
@ -17,7 +17,7 @@ export const DebugCommand = cmd({
yargs
.command(ConfigCommand)
.command(LSPCommand)
.command(RipgrepCommand)
.command(SearchCommand)
.command(FileCommand)
.command(ScrapCommand)
.command(SkillCommand)

View File

@ -1,33 +1,34 @@
import { EOL } from "os"
import { Ripgrep } from "../../../file/ripgrep"
import { Fff } from "../../../file/fff"
import { Instance } from "../../../project/instance"
import { bootstrap } from "../../bootstrap"
import { cmd } from "../cmd"
import { Glob } from "@/util/glob"
export const RipgrepCommand = cmd({
command: "rg",
describe: "ripgrep debugging utilities",
builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(SearchCommand).demandCommand(),
export const SearchCommand = cmd({
command: "search",
describe: "fff search debugging utilities",
builder: (yargs) => yargs.command(TreeCommand).command(FilesCommand).command(ContentCommand).demandCommand(),
async handler() {},
})
const TreeCommand = cmd({
command: "tree",
describe: "show file tree using ripgrep",
describe: "show file tree using fff",
builder: (yargs) =>
yargs.option("limit", {
type: "number",
}),
async handler(args) {
await bootstrap(process.cwd(), async () => {
process.stdout.write((await Ripgrep.tree({ cwd: Instance.directory, limit: args.limit })) + EOL)
process.stdout.write((await Fff.tree({ cwd: Instance.directory, limit: args.limit })) + EOL)
})
},
})
const FilesCommand = cmd({
command: "files",
describe: "list files using ripgrep",
describe: "list files using fff",
builder: (yargs) =>
yargs
.option("query", {
@ -44,22 +45,24 @@ const FilesCommand = cmd({
}),
async handler(args) {
await bootstrap(process.cwd(), async () => {
const files: string[] = []
for await (const file of Ripgrep.files({
const limit = args.limit ?? 100
const files = (await Glob.scan("**/*", {
cwd: Instance.directory,
glob: args.glob ? [args.glob] : undefined,
})) {
files.push(file)
if (args.limit && files.length >= args.limit) break
}
include: "file",
dot: true,
}))
.map((x) => x.replaceAll("\\", "/"))
.filter((x) => Fff.allowed({ rel: x, hidden: true, glob: args.glob ? [args.glob] : undefined }))
.filter((x) => !args.query || x.includes(args.query))
.slice(0, limit)
process.stdout.write(files.join(EOL) + EOL)
})
},
})
const SearchCommand = cmd({
command: "search <pattern>",
describe: "search file contents using ripgrep",
const ContentCommand = cmd({
command: "content <pattern>",
describe: "search file contents using fff",
builder: (yargs) =>
yargs
.positional("pattern", {
@ -76,12 +79,12 @@ const SearchCommand = cmd({
description: "Limit number of results",
}),
async handler(args) {
const results = await Ripgrep.search({
const rows = await Fff.search({
cwd: process.cwd(),
pattern: args.pattern,
glob: args.glob as string[] | undefined,
limit: args.limit,
})
process.stdout.write(JSON.stringify(results, null, 2) + EOL)
process.stdout.write(JSON.stringify(rows, null, 2) + EOL)
},
})

View File

@ -0,0 +1,273 @@
import fs from "fs/promises"
import path from "path"
import {
FileFinder,
type FileItem,
type GrepCursor,
type GrepMatch,
type GrepMode,
type SearchResult,
} from "@ff-labs/fff-node"
import z from "zod"
import { Global } from "../global"
import { Instance } from "../project/instance"
import { Filesystem } from "../util/filesystem"
import { Glob } from "../util/glob"
import { Log } from "../util/log"
export namespace Fff {
const log = Log.create({ service: "file.fff" })
export const Match = z.object({
path: z.object({
text: z.string(),
}),
lines: z.object({
text: z.string(),
}),
line_number: z.number(),
absolute_offset: z.number(),
submatches: z.array(
z.object({
match: z.object({
text: z.string(),
}),
start: z.number(),
end: z.number(),
}),
),
})
const state = Instance.state(
async () => ({
map: new Map<string, FileFinder>(),
pending: new Map<string, Promise<FileFinder>>(),
}),
async (state) => {
for (const pick of state.map.values()) pick.destroy()
},
)
const root = path.join(Global.Path.cache, "fff")
function key(dir: string) {
return Buffer.from(dir).toString("base64url")
}
async function db(dir: string) {
await fs.mkdir(root, { recursive: true })
const id = key(dir)
return {
frecency: path.join(root, `${id}.frecency.mdb`),
history: path.join(root, `${id}.history.mdb`),
}
}
function refresh(pick: FileFinder) {
const git = pick.refreshGitStatus()
if (!git.ok) {
log.warn("git refresh failed", { error: git.error })
return
}
}
export async function picker(cwd: string) {
const dir = Filesystem.resolve(cwd)
const memo = await state()
const cached = memo.map.get(dir)
if (cached) return cached
const wait = memo.pending.get(dir)
if (wait) return wait
const next = (async () => {
const files = await db(dir)
const made = FileFinder.create({
basePath: dir,
frecencyDbPath: files.frecency,
historyDbPath: files.history,
aiMode: true,
})
if (!made.ok) throw new Error(made.error)
const pick = made.value
const done = await pick.waitForScan(5000)
if (!done.ok) {
pick.destroy()
throw new Error(done.error)
}
memo.map.set(dir, pick)
refresh(pick)
return pick
})()
memo.pending.set(dir, next)
try {
return await next
} finally {
if (memo.pending.get(dir) === next) memo.pending.delete(dir)
}
}
export async function files(input: { cwd: string; query: string; page?: number; size?: number; current?: string }) {
const pick = await picker(input.cwd)
const out = pick.fileSearch(input.query, {
pageIndex: input.page ?? 0,
pageSize: input.size ?? 100,
currentFile: input.current,
})
if (!out.ok) throw new Error(out.error)
return out.value
}
export async function grep(input: {
cwd: string
query: string
mode?: GrepMode
max?: number
before?: number
after?: number
budget?: number
cursor?: GrepCursor | null
}) {
const pick = await picker(input.cwd)
const out = pick.grep(input.query, {
mode: input.mode,
maxMatchesPerFile: input.max,
beforeContext: input.before,
afterContext: input.after,
timeBudgetMs: input.budget,
cursor: input.cursor,
})
if (!out.ok) throw new Error(out.error)
return out.value
}
function norm(text: string) {
return text.replaceAll("\\", "/")
}
function hidden(rel: string) {
return norm(rel)
.split("/")
.some((part) => part.startsWith("."))
}
function accept(rel: string, file: string, glob?: string[], show?: boolean) {
if (show === false && hidden(rel)) return false
if (!glob?.length) return true
const allow = glob.filter((x) => !x.startsWith("!"))
const deny = glob.filter((x) => x.startsWith("!")).map((x) => x.slice(1))
if (allow.length > 0 && !allow.some((x) => Glob.match(x, rel) || Glob.match(x, file))) return false
if (deny.some((x) => Glob.match(x, rel) || Glob.match(x, file))) return false
return true
}
export function allowed(input: { rel: string; file?: string; glob?: string[]; hidden?: boolean }) {
return accept(input.rel, input.file ?? input.rel.split("/").at(-1) ?? input.rel, input.glob, input.hidden !== false)
}
export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) {
input.signal?.throwIfAborted()
const files = (await Glob.scan("**/*", {
cwd: input.cwd,
include: "file",
dot: true,
}))
.map((row) => norm(row))
.filter((row) => allowed({ rel: row, hidden: true }))
.toSorted((a, b) => a.localeCompare(b))
input.signal?.throwIfAborted()
interface Node {
name: string
children: Map<string, Node>
}
function dir(node: Node, name: string) {
const old = node.children.get(name)
if (old) return old
const next = { name, children: new Map<string, Node>() }
node.children.set(name, next)
return next
}
const root = { name: "", children: new Map<string, Node>() }
for (const file of files) {
if (file.includes(".opencode")) continue
const parts = file.split("/")
if (parts.length < 2) continue
let node = root
for (const part of parts.slice(0, -1)) {
node = dir(node, part)
}
}
function count(node: Node): number {
return Array.from(node.children.values()).reduce((sum, child) => sum + 1 + count(child), 0)
}
const total = count(root)
const limit = input.limit ?? total
const lines: string[] = []
const queue = Array.from(root.children.values())
.toSorted((a, b) => a.name.localeCompare(b.name))
.map((node) => ({ node, path: node.name }))
let used = 0
for (let i = 0; i < queue.length && used < limit; i++) {
input.signal?.throwIfAborted()
const row = queue[i]
lines.push(row.path)
used++
queue.push(
...Array.from(row.node.children.values())
.toSorted((a, b) => a.name.localeCompare(b.name))
.map((node) => ({ node, path: `${row.path}/${node.name}` })),
)
}
if (total > used) lines.push(`[${total - used} truncated]`)
input.signal?.throwIfAborted()
return lines.join("\n")
}
export async function search(input: {
cwd: string
pattern: string
glob?: string[]
limit?: number
follow?: boolean
}) {
const out = await grep({
cwd: input.cwd,
query: input.pattern,
mode: "regex",
max: input.limit,
})
const rows = out.items
.filter((row) => accept(norm(row.relativePath), row.fileName, input.glob, true))
.slice(0, input.limit)
.map((row) => ({
path: { text: row.relativePath },
lines: { text: row.lineContent },
line_number: row.lineNumber,
absolute_offset: row.byteOffset,
submatches: row.matchRanges
.map(([start, end]) => {
const text = row.lineContent.slice(start, end)
if (!text) return undefined
return {
match: { text },
start,
end,
}
})
.filter((row) => row !== undefined),
}))
return Match.array().parse(rows)
}
export type Search = SearchResult
export type File = FileItem
export type Hit = GrepMatch
}

View File

@ -12,9 +12,9 @@ import z from "zod"
import { Global } from "../global"
import { Instance } from "../project/instance"
import { Filesystem } from "../util/filesystem"
import { Glob } from "../util/glob"
import { Log } from "../util/log"
import { Protected } from "./protected"
import { Ripgrep } from "./ripgrep"
export namespace File {
export const Info = z
@ -401,7 +401,11 @@ export namespace File {
next.dirs = Array.from(dirs).toSorted()
} else {
const seen = new Set<string>()
for await (const file of Ripgrep.files({ cwd: instance.directory })) {
for (const file of (await Glob.scan("**/*", {
cwd: instance.directory,
include: "file",
dot: true,
})).toSorted((a, b) => a.localeCompare(b))) {
next.files.push(file)
let current = file
while (true) {

View File

@ -1,376 +0,0 @@
// Ripgrep utility functions
import path from "path"
import { Global } from "../global"
import fs from "fs/promises"
import z from "zod"
import { NamedError } from "@opencode-ai/util/error"
import { lazy } from "../util/lazy"
import { Filesystem } from "../util/filesystem"
import { Process } from "../util/process"
import { which } from "../util/which"
import { text } from "node:stream/consumers"
import { ZipReader, BlobReader, BlobWriter } from "@zip.js/zip.js"
import { Log } from "@/util/log"
export namespace Ripgrep {
const log = Log.create({ service: "ripgrep" })
const Stats = z.object({
elapsed: z.object({
secs: z.number(),
nanos: z.number(),
human: z.string(),
}),
searches: z.number(),
searches_with_match: z.number(),
bytes_searched: z.number(),
bytes_printed: z.number(),
matched_lines: z.number(),
matches: z.number(),
})
const Begin = z.object({
type: z.literal("begin"),
data: z.object({
path: z.object({
text: z.string(),
}),
}),
})
export const Match = z.object({
type: z.literal("match"),
data: z.object({
path: z.object({
text: z.string(),
}),
lines: z.object({
text: z.string(),
}),
line_number: z.number(),
absolute_offset: z.number(),
submatches: z.array(
z.object({
match: z.object({
text: z.string(),
}),
start: z.number(),
end: z.number(),
}),
),
}),
})
const End = z.object({
type: z.literal("end"),
data: z.object({
path: z.object({
text: z.string(),
}),
binary_offset: z.number().nullable(),
stats: Stats,
}),
})
const Summary = z.object({
type: z.literal("summary"),
data: z.object({
elapsed_total: z.object({
human: z.string(),
nanos: z.number(),
secs: z.number(),
}),
stats: Stats,
}),
})
const Result = z.union([Begin, Match, End, Summary])
export type Result = z.infer<typeof Result>
export type Match = z.infer<typeof Match>
export type Begin = z.infer<typeof Begin>
export type End = z.infer<typeof End>
export type Summary = z.infer<typeof Summary>
const PLATFORM = {
"arm64-darwin": { platform: "aarch64-apple-darwin", extension: "tar.gz" },
"arm64-linux": {
platform: "aarch64-unknown-linux-gnu",
extension: "tar.gz",
},
"x64-darwin": { platform: "x86_64-apple-darwin", extension: "tar.gz" },
"x64-linux": { platform: "x86_64-unknown-linux-musl", extension: "tar.gz" },
"arm64-win32": { platform: "aarch64-pc-windows-msvc", extension: "zip" },
"x64-win32": { platform: "x86_64-pc-windows-msvc", extension: "zip" },
} as const
export const ExtractionFailedError = NamedError.create(
"RipgrepExtractionFailedError",
z.object({
filepath: z.string(),
stderr: z.string(),
}),
)
export const UnsupportedPlatformError = NamedError.create(
"RipgrepUnsupportedPlatformError",
z.object({
platform: z.string(),
}),
)
export const DownloadFailedError = NamedError.create(
"RipgrepDownloadFailedError",
z.object({
url: z.string(),
status: z.number(),
}),
)
const state = lazy(async () => {
const system = which("rg")
if (system) {
const stat = await fs.stat(system).catch(() => undefined)
if (stat?.isFile()) return { filepath: system }
log.warn("bun.which returned invalid rg path", { filepath: system })
}
const filepath = path.join(Global.Path.bin, "rg" + (process.platform === "win32" ? ".exe" : ""))
if (!(await Filesystem.exists(filepath))) {
const platformKey = `${process.arch}-${process.platform}` as keyof typeof PLATFORM
const config = PLATFORM[platformKey]
if (!config) throw new UnsupportedPlatformError({ platform: platformKey })
const version = "14.1.1"
const filename = `ripgrep-${version}-${config.platform}.${config.extension}`
const url = `https://github.com/BurntSushi/ripgrep/releases/download/${version}/${filename}`
const response = await fetch(url)
if (!response.ok) throw new DownloadFailedError({ url, status: response.status })
const arrayBuffer = await response.arrayBuffer()
const archivePath = path.join(Global.Path.bin, filename)
await Filesystem.write(archivePath, Buffer.from(arrayBuffer))
if (config.extension === "tar.gz") {
const args = ["tar", "-xzf", archivePath, "--strip-components=1"]
if (platformKey.endsWith("-darwin")) args.push("--include=*/rg")
if (platformKey.endsWith("-linux")) args.push("--wildcards", "*/rg")
const proc = Process.spawn(args, {
cwd: Global.Path.bin,
stderr: "pipe",
stdout: "pipe",
})
const exit = await proc.exited
if (exit !== 0) {
const stderr = proc.stderr ? await text(proc.stderr) : ""
throw new ExtractionFailedError({
filepath,
stderr,
})
}
}
if (config.extension === "zip") {
const zipFileReader = new ZipReader(new BlobReader(new Blob([arrayBuffer])))
const entries = await zipFileReader.getEntries()
let rgEntry: any
for (const entry of entries) {
if (entry.filename.endsWith("rg.exe")) {
rgEntry = entry
break
}
}
if (!rgEntry) {
throw new ExtractionFailedError({
filepath: archivePath,
stderr: "rg.exe not found in zip archive",
})
}
const rgBlob = await rgEntry.getData(new BlobWriter())
if (!rgBlob) {
throw new ExtractionFailedError({
filepath: archivePath,
stderr: "Failed to extract rg.exe from zip archive",
})
}
await Filesystem.write(filepath, Buffer.from(await rgBlob.arrayBuffer()))
await zipFileReader.close()
}
await fs.unlink(archivePath)
if (!platformKey.endsWith("-win32")) await fs.chmod(filepath, 0o755)
}
return {
filepath,
}
})
export async function filepath() {
const { filepath } = await state()
return filepath
}
export async function* files(input: {
cwd: string
glob?: string[]
hidden?: boolean
follow?: boolean
maxDepth?: number
signal?: AbortSignal
}) {
input.signal?.throwIfAborted()
const args = [await filepath(), "--files", "--glob=!.git/*"]
if (input.follow) args.push("--follow")
if (input.hidden !== false) args.push("--hidden")
if (input.maxDepth !== undefined) args.push(`--max-depth=${input.maxDepth}`)
if (input.glob) {
for (const g of input.glob) {
args.push(`--glob=${g}`)
}
}
// Guard against invalid cwd to provide a consistent ENOENT error.
if (!(await fs.stat(input.cwd).catch(() => undefined))?.isDirectory()) {
throw Object.assign(new Error(`No such file or directory: '${input.cwd}'`), {
code: "ENOENT",
errno: -2,
path: input.cwd,
})
}
const proc = Process.spawn(args, {
cwd: input.cwd,
stdout: "pipe",
stderr: "ignore",
abort: input.signal,
})
if (!proc.stdout) {
throw new Error("Process output not available")
}
let buffer = ""
const stream = proc.stdout as AsyncIterable<Buffer | string>
for await (const chunk of stream) {
input.signal?.throwIfAborted()
buffer += typeof chunk === "string" ? chunk : chunk.toString()
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = buffer.split(/\r?\n/)
buffer = lines.pop() || ""
for (const line of lines) {
if (line) yield line
}
}
if (buffer) yield buffer
await proc.exited
input.signal?.throwIfAborted()
}
export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) {
log.info("tree", input)
const files = await Array.fromAsync(Ripgrep.files({ cwd: input.cwd, signal: input.signal }))
interface Node {
name: string
children: Map<string, Node>
}
function dir(node: Node, name: string) {
const existing = node.children.get(name)
if (existing) return existing
const next = { name, children: new Map() }
node.children.set(name, next)
return next
}
const root: Node = { name: "", children: new Map() }
for (const file of files) {
if (file.includes(".opencode")) continue
const parts = file.split(path.sep)
if (parts.length < 2) continue
let node = root
for (const part of parts.slice(0, -1)) {
node = dir(node, part)
}
}
function count(node: Node): number {
let total = 0
for (const child of node.children.values()) {
total += 1 + count(child)
}
return total
}
const total = count(root)
const limit = input.limit ?? total
const lines: string[] = []
const queue: { node: Node; path: string }[] = []
for (const child of Array.from(root.children.values()).sort((a, b) => a.name.localeCompare(b.name))) {
queue.push({ node: child, path: child.name })
}
let used = 0
for (let i = 0; i < queue.length && used < limit; i++) {
const { node, path } = queue[i]
lines.push(path)
used++
for (const child of Array.from(node.children.values()).sort((a, b) => a.name.localeCompare(b.name))) {
queue.push({ node: child, path: `${path}/${child.name}` })
}
}
if (total > used) lines.push(`[${total - used} truncated]`)
return lines.join("\n")
}
export async function search(input: {
cwd: string
pattern: string
glob?: string[]
limit?: number
follow?: boolean
}) {
const args = [`${await filepath()}`, "--json", "--hidden", "--glob=!.git/*"]
if (input.follow) args.push("--follow")
if (input.glob) {
for (const g of input.glob) {
args.push(`--glob=${g}`)
}
}
if (input.limit) {
args.push(`--max-count=${input.limit}`)
}
args.push("--")
args.push(input.pattern)
const result = await Process.text(args, {
cwd: input.cwd,
nothrow: true,
})
if (result.code !== 0) {
return []
}
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = result.text.trim().split(/\r?\n/).filter(Boolean)
// Parse JSON lines from ripgrep output
return lines
.map((line) => JSON.parse(line))
.map((parsed) => Result.parse(parsed))
.filter((r) => r.type === "match")
.map((r) => r.data)
}
}

View File

@ -2,7 +2,7 @@ import { Hono } from "hono"
import { describeRoute, validator, resolver } from "hono-openapi"
import z from "zod"
import { File } from "../../file"
import { Ripgrep } from "../../file/ripgrep"
import { Fff } from "../../file/fff"
import { LSP } from "../../lsp"
import { Instance } from "../../project/instance"
import { lazy } from "../../util/lazy"
@ -13,14 +13,14 @@ export const FileRoutes = lazy(() =>
"/find",
describeRoute({
summary: "Find text",
description: "Search for text patterns across files in the project using ripgrep.",
description: "Search for text patterns across files in the project.",
operationId: "find.text",
responses: {
200: {
description: "Matches",
content: {
"application/json": {
schema: resolver(Ripgrep.Match.shape.data.array()),
schema: resolver(Fff.Match.array()),
},
},
},
@ -34,7 +34,7 @@ export const FileRoutes = lazy(() =>
),
async (c) => {
const pattern = c.req.valid("query").pattern
const result = await Ripgrep.search({
const result = await Fff.search({
cwd: Instance.directory,
pattern,
limit: 10,

View File

@ -1,54 +0,0 @@
import { describe, expect, test } from "bun:test"
import fs from "fs/promises"
import path from "path"
import { tmpdir } from "../fixture/fixture"
import { Ripgrep } from "../../src/file/ripgrep"
describe("file.ripgrep", () => {
test("defaults to include hidden", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "visible.txt"), "hello")
await fs.mkdir(path.join(dir, ".opencode"), { recursive: true })
await Bun.write(path.join(dir, ".opencode", "thing.json"), "{}")
},
})
const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path }))
const hasVisible = files.includes("visible.txt")
const hasHidden = files.includes(path.join(".opencode", "thing.json"))
expect(hasVisible).toBe(true)
expect(hasHidden).toBe(true)
})
test("hidden false excludes hidden", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "visible.txt"), "hello")
await fs.mkdir(path.join(dir, ".opencode"), { recursive: true })
await Bun.write(path.join(dir, ".opencode", "thing.json"), "{}")
},
})
const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path, hidden: false }))
const hasVisible = files.includes("visible.txt")
const hasHidden = files.includes(path.join(".opencode", "thing.json"))
expect(hasVisible).toBe(true)
expect(hasHidden).toBe(false)
})
test("search returns empty when nothing matches", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "match.ts"), "const value = 'other'\n")
},
})
const hits = await Ripgrep.search({
cwd: tmp.path,
pattern: "needle",
})
expect(hits).toEqual([])
})
})