feat(tools): switch search tools to fff
Shoubhit Dash
parent
b5a7ad7085
commit
2948b2fb73
|
|
@ -1,4 +1,4 @@
|
|||
import { Ripgrep } from "../file/ripgrep"
|
||||
import { Fff } from "../file/fff"
|
||||
|
||||
import { Instance } from "../project/instance"
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ export namespace SystemPrompt {
|
|||
`<directories>`,
|
||||
` ${
|
||||
project.vcs === "git" && false
|
||||
? await Ripgrep.tree({
|
||||
? await Fff.tree({
|
||||
cwd: Instance.directory,
|
||||
limit: 50,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,11 +1,95 @@
|
|||
import z from "zod"
|
||||
import path from "path"
|
||||
import { Tool } from "./tool"
|
||||
import { Filesystem } from "../util/filesystem"
|
||||
import DESCRIPTION from "./glob.txt"
|
||||
import { Ripgrep } from "../file/ripgrep"
|
||||
import { Fff } from "../file/fff"
|
||||
import { Instance } from "../project/instance"
|
||||
import { assertExternalDirectory } from "./external-directory"
|
||||
import { Glob } from "../util/glob"
|
||||
|
||||
type Row = {
|
||||
path: string
|
||||
rel: string
|
||||
}
|
||||
|
||||
function include(pattern: string) {
|
||||
const val = pattern.trim().replaceAll("\\", "/")
|
||||
if (!val) return "*"
|
||||
const flat = val.replaceAll("**/", "").replaceAll("/**", "/")
|
||||
const idx = flat.lastIndexOf("/")
|
||||
if (idx < 0) return flat
|
||||
const dir = flat.slice(0, idx + 1)
|
||||
const glob = flat.slice(idx + 1)
|
||||
if (!glob) return dir
|
||||
return `${dir} ${glob}`
|
||||
}
|
||||
|
||||
function words(text: string) {
|
||||
return text.trim().split(/\s+/).filter(Boolean)
|
||||
}
|
||||
|
||||
function norm(text: string) {
|
||||
return text.replaceAll("\\", "/")
|
||||
}
|
||||
|
||||
function hidden(rel: string) {
|
||||
return norm(rel).split("/").includes(".git")
|
||||
}
|
||||
|
||||
function broad(pattern: string) {
|
||||
const val = norm(pattern.trim())
|
||||
if (!val) return true
|
||||
if (["*", "**", "**/*", "./**", "./**/*"].includes(val)) return true
|
||||
return /^(\*\*\/)?\*$/.test(val)
|
||||
}
|
||||
|
||||
function allowed(pattern: string, rel: string) {
|
||||
if (Glob.match(pattern, rel)) return true
|
||||
const file = rel.split("/").at(-1) ?? rel
|
||||
return Glob.match(pattern, file)
|
||||
}
|
||||
|
||||
function pick(items: { path: string; relativePath: string }[]) {
|
||||
return items
|
||||
.map((item) => ({
|
||||
path: item.path,
|
||||
rel: norm(item.relativePath),
|
||||
}))
|
||||
.filter((item) => !hidden(item.rel))
|
||||
}
|
||||
|
||||
function top(rows: Row[]) {
|
||||
const out = new Map<string, number>()
|
||||
for (const row of rows) {
|
||||
const parts = row.rel.split("/")
|
||||
const key = parts.length < 2 ? "." : parts.slice(0, Math.min(2, parts.length - 1)).join("/") + "/"
|
||||
out.set(key, (out.get(key) ?? 0) + 1)
|
||||
}
|
||||
return Array.from(out.entries())
|
||||
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
||||
.slice(0, 12)
|
||||
}
|
||||
|
||||
async function scan(pattern: string, dir: string) {
|
||||
const direct = await Glob.scan(pattern, {
|
||||
cwd: dir,
|
||||
absolute: true,
|
||||
include: "file",
|
||||
dot: true,
|
||||
})
|
||||
const out = direct.length > 0 ? direct : await Glob.scan(`**/${pattern}`, {
|
||||
cwd: dir,
|
||||
absolute: true,
|
||||
include: "file",
|
||||
dot: true,
|
||||
})
|
||||
return out
|
||||
.map((file) => ({
|
||||
path: file,
|
||||
rel: norm(path.relative(dir, file)),
|
||||
}))
|
||||
.filter((item) => !hidden(item.rel))
|
||||
}
|
||||
|
||||
export const GlobTool = Tool.define("glob", {
|
||||
description: DESCRIPTION,
|
||||
|
|
@ -29,35 +113,60 @@ export const GlobTool = Tool.define("glob", {
|
|||
},
|
||||
})
|
||||
|
||||
let search = params.path ?? Instance.directory
|
||||
search = path.isAbsolute(search) ? search : path.resolve(Instance.directory, search)
|
||||
await assertExternalDirectory(ctx, search, { kind: "directory" })
|
||||
let dir = params.path ?? Instance.directory
|
||||
dir = path.isAbsolute(dir) ? dir : path.resolve(Instance.directory, dir)
|
||||
await assertExternalDirectory(ctx, dir, { kind: "directory" })
|
||||
|
||||
const limit = 100
|
||||
const files = []
|
||||
let truncated = false
|
||||
for await (const file of Ripgrep.files({
|
||||
cwd: search,
|
||||
glob: [params.pattern],
|
||||
signal: ctx.abort,
|
||||
})) {
|
||||
if (files.length >= limit) {
|
||||
truncated = true
|
||||
break
|
||||
const wide = broad(params.pattern)
|
||||
const size = wide ? 400 : limit + 1
|
||||
|
||||
const first = await Fff.files({
|
||||
cwd: dir,
|
||||
query: include(params.pattern),
|
||||
size,
|
||||
current: path.join(dir, ".opencode"),
|
||||
})
|
||||
|
||||
let fallback = false
|
||||
let rows = pick(first.items).filter((row) => allowed(params.pattern, row.rel))
|
||||
if (!rows.length) {
|
||||
const list = words(params.pattern)
|
||||
if (list.length >= 3) {
|
||||
const short = list.slice(0, 2).join(" ")
|
||||
const next = await Fff.files({
|
||||
cwd: dir,
|
||||
query: include(short),
|
||||
size,
|
||||
current: path.join(dir, ".opencode"),
|
||||
})
|
||||
rows = pick(next.items).filter((row) => allowed(params.pattern, row.rel))
|
||||
}
|
||||
const full = path.resolve(search, file)
|
||||
const stats = Filesystem.stat(full)?.mtime.getTime() ?? 0
|
||||
files.push({
|
||||
path: full,
|
||||
mtime: stats,
|
||||
})
|
||||
}
|
||||
files.sort((a, b) => b.mtime - a.mtime)
|
||||
if (!rows.length) {
|
||||
fallback = true
|
||||
rows = (await scan(params.pattern, dir)).filter((row) => allowed(params.pattern, row.rel))
|
||||
}
|
||||
|
||||
const truncated = rows.length > limit
|
||||
const files = rows.slice(0, limit).map((row) => row.path)
|
||||
|
||||
const output = []
|
||||
if (files.length === 0) output.push("No files found")
|
||||
if (files.length > 0) {
|
||||
output.push(...files.map((f) => f.path))
|
||||
output.push(...files)
|
||||
if (wide && truncated) {
|
||||
const dirs = top(rows)
|
||||
if (dirs.length > 0) {
|
||||
output.push("")
|
||||
output.push("Top directories in this result set:")
|
||||
output.push(...dirs.map(([dir, count]) => `${dir} (${count})`))
|
||||
}
|
||||
}
|
||||
if (fallback) {
|
||||
output.push("")
|
||||
output.push("(Used filesystem glob fallback for this pattern.)")
|
||||
}
|
||||
if (truncated) {
|
||||
output.push("")
|
||||
output.push(
|
||||
|
|
@ -67,7 +176,7 @@ export const GlobTool = Tool.define("glob", {
|
|||
}
|
||||
|
||||
return {
|
||||
title: path.relative(Instance.worktree, search),
|
||||
title: path.relative(Instance.worktree, dir),
|
||||
metadata: {
|
||||
count: files.length,
|
||||
truncated,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
- Fast file pattern matching tool that works with any codebase size
|
||||
- Fast file pattern matching tool that uses fuzzy-first indexing and frecency ranking
|
||||
- Supports glob patterns like "**/*.js" or "src/**/*.ts"
|
||||
- Returns matching file paths sorted by modification time
|
||||
- Returns matching file paths prioritized by recent and relevant files
|
||||
- Use this tool when you need to find files by name patterns
|
||||
- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead
|
||||
- You have the capability to call multiple tools in a single response. It is always better to speculatively perform multiple searches as a batch that are potentially useful.
|
||||
|
|
|
|||
|
|
@ -1,16 +1,136 @@
|
|||
import z from "zod"
|
||||
import { text } from "node:stream/consumers"
|
||||
import { Tool } from "./tool"
|
||||
import { Filesystem } from "../util/filesystem"
|
||||
import { Ripgrep } from "../file/ripgrep"
|
||||
import { Process } from "../util/process"
|
||||
import { Fff } from "../file/fff"
|
||||
import type { GrepMode } from "@ff-labs/fff-node"
|
||||
|
||||
import DESCRIPTION from "./grep.txt"
|
||||
import { Instance } from "../project/instance"
|
||||
import path from "path"
|
||||
import { assertExternalDirectory } from "./external-directory"
|
||||
import { Glob } from "../util/glob"
|
||||
|
||||
const MAX_LINE_LENGTH = 2000
|
||||
const MAX_LINE = 180
|
||||
const MAX_MATCH = 100
|
||||
const MAX_DEF_FIRST = 8
|
||||
const MAX_DEF_NEXT = 5
|
||||
|
||||
function isRegex(pattern: string) {
|
||||
return /[.*+?^${}()|[\]\\]/.test(pattern)
|
||||
}
|
||||
|
||||
function isConstraint(text: string) {
|
||||
return text.startsWith("!") || text.startsWith("*") || text.endsWith("/")
|
||||
}
|
||||
|
||||
function clean(text: string) {
|
||||
return text.replaceAll(":", "").replaceAll("-", "").replaceAll("_", "").toLowerCase().trim()
|
||||
}
|
||||
|
||||
function include(text?: string) {
|
||||
if (!text) return undefined
|
||||
const val = text.trim().replaceAll("\\", "/")
|
||||
if (!val) return undefined
|
||||
const flat = val.replaceAll("**/", "").replaceAll("/**", "/")
|
||||
const idx = flat.lastIndexOf("/")
|
||||
if (idx < 0) return flat
|
||||
const dir = flat.slice(0, idx + 1)
|
||||
const glob = flat.slice(idx + 1)
|
||||
if (!glob) return dir
|
||||
return `${dir} ${glob}`
|
||||
}
|
||||
|
||||
function query(pattern: string, inc?: string) {
|
||||
if (!inc) return pattern
|
||||
return `${inc} ${pattern}`.trim()
|
||||
}
|
||||
|
||||
function norm(text: string) {
|
||||
return text.replaceAll("\\", "/")
|
||||
}
|
||||
|
||||
function allowed(hit: Fff.Hit, inc?: string) {
|
||||
if (!inc) return true
|
||||
const rel = norm(hit.relativePath)
|
||||
if (Glob.match(inc, rel)) return true
|
||||
return Glob.match(inc, norm(hit.fileName))
|
||||
}
|
||||
|
||||
function def(line: string) {
|
||||
const text = line.trim()
|
||||
if (!text) return false
|
||||
return /^(export\s+)?(default\s+)?(async\s+)?(function|class|interface|type|enum|const|let|var)\b/.test(text)
|
||||
}
|
||||
|
||||
function imp(line: string) {
|
||||
return /^(import\b|export\s+\{.*\}\s+from\b|use\b|#include\b|require\()/.test(line.trim())
|
||||
}
|
||||
|
||||
function line(text: string, ranges: [number, number][]) {
|
||||
const trim = text.trim()
|
||||
if (trim.length <= MAX_LINE) return trim
|
||||
const first = ranges[0]
|
||||
if (!first) return trim.slice(0, MAX_LINE - 3) + "..."
|
||||
const start = Math.max(0, first[0] - Math.floor(MAX_LINE / 3))
|
||||
const end = Math.min(trim.length, start + MAX_LINE)
|
||||
const body = trim.slice(start, end)
|
||||
const pre = start > 0 ? "..." : ""
|
||||
const post = end < trim.length ? "..." : ""
|
||||
return pre + body + post
|
||||
}
|
||||
|
||||
function group(rows: Item[]) {
|
||||
const out = new Map<string, Item[]>()
|
||||
for (const row of rows) {
|
||||
const list = out.get(row.hit.path)
|
||||
if (list) {
|
||||
list.push(row)
|
||||
continue
|
||||
}
|
||||
out.set(row.hit.path, [row])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type Item = {
|
||||
hit: Fff.Hit
|
||||
def: boolean
|
||||
imp: boolean
|
||||
idx: number
|
||||
}
|
||||
|
||||
async function run(input: {
|
||||
cwd: string
|
||||
pattern: string
|
||||
inc?: string
|
||||
mode: GrepMode
|
||||
max: number
|
||||
before: number
|
||||
after: number
|
||||
}) {
|
||||
const first = await Fff.grep({
|
||||
cwd: input.cwd,
|
||||
query: query(input.pattern, include(input.inc)),
|
||||
mode: input.mode,
|
||||
max: input.max,
|
||||
before: input.before,
|
||||
after: input.after,
|
||||
})
|
||||
const head = first.items.filter((hit) => allowed(hit, input.inc))
|
||||
if (head.length) return { out: first, hits: head }
|
||||
if (!input.inc) return { out: first, hits: head }
|
||||
const raw = await Fff.grep({
|
||||
cwd: input.cwd,
|
||||
query: input.pattern,
|
||||
mode: input.mode,
|
||||
max: input.max,
|
||||
before: input.before,
|
||||
after: input.after,
|
||||
})
|
||||
return {
|
||||
out: raw,
|
||||
hits: raw.items.filter((hit) => allowed(hit, input.inc)),
|
||||
}
|
||||
}
|
||||
|
||||
export const GrepTool = Tool.define("grep", {
|
||||
description: DESCRIPTION,
|
||||
|
|
@ -35,122 +155,161 @@ export const GrepTool = Tool.define("grep", {
|
|||
},
|
||||
})
|
||||
|
||||
let searchPath = params.path ?? Instance.directory
|
||||
searchPath = path.isAbsolute(searchPath) ? searchPath : path.resolve(Instance.directory, searchPath)
|
||||
await assertExternalDirectory(ctx, searchPath, { kind: "directory" })
|
||||
let dir = params.path ?? Instance.directory
|
||||
dir = path.isAbsolute(dir) ? dir : path.resolve(Instance.directory, dir)
|
||||
await assertExternalDirectory(ctx, dir, { kind: "directory" })
|
||||
|
||||
const rgPath = await Ripgrep.filepath()
|
||||
const args = ["-nH", "--hidden", "--no-messages", "--field-match-separator=|", "--regexp", params.pattern]
|
||||
if (params.include) {
|
||||
args.push("--glob", params.include)
|
||||
}
|
||||
args.push(searchPath)
|
||||
|
||||
const proc = Process.spawn([rgPath, ...args], {
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
abort: ctx.abort,
|
||||
const mode = isRegex(params.pattern) ? "regex" : "plain"
|
||||
const exact = await run({
|
||||
cwd: dir,
|
||||
pattern: params.pattern,
|
||||
inc: params.include,
|
||||
mode,
|
||||
max: 10,
|
||||
before: 0,
|
||||
after: 4,
|
||||
})
|
||||
|
||||
if (!proc.stdout || !proc.stderr) {
|
||||
throw new Error("Process output not available")
|
||||
}
|
||||
let phase = "exact"
|
||||
let note = ""
|
||||
let warn = exact.out.regexFallbackError
|
||||
let hits = exact.hits
|
||||
|
||||
const output = await text(proc.stdout)
|
||||
const errorOutput = await text(proc.stderr)
|
||||
const exitCode = await proc.exited
|
||||
|
||||
// Exit codes: 0 = matches found, 1 = no matches, 2 = errors (but may still have matches)
|
||||
// With --no-messages, we suppress error output but still get exit code 2 for broken symlinks etc.
|
||||
// Only fail if exit code is 2 AND no output was produced
|
||||
if (exitCode === 1 || (exitCode === 2 && !output.trim())) {
|
||||
return {
|
||||
title: params.pattern,
|
||||
metadata: { matches: 0, truncated: false },
|
||||
output: "No files found",
|
||||
}
|
||||
}
|
||||
|
||||
if (exitCode !== 0 && exitCode !== 2) {
|
||||
throw new Error(`ripgrep failed: ${errorOutput}`)
|
||||
}
|
||||
|
||||
const hasErrors = exitCode === 2
|
||||
|
||||
// Handle both Unix (\n) and Windows (\r\n) line endings
|
||||
const lines = output.trim().split(/\r?\n/)
|
||||
const matches = []
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line) continue
|
||||
|
||||
const [filePath, lineNumStr, ...lineTextParts] = line.split("|")
|
||||
if (!filePath || !lineNumStr || lineTextParts.length === 0) continue
|
||||
|
||||
const lineNum = parseInt(lineNumStr, 10)
|
||||
const lineText = lineTextParts.join("|")
|
||||
|
||||
const stats = Filesystem.stat(filePath)
|
||||
if (!stats) continue
|
||||
|
||||
matches.push({
|
||||
path: filePath,
|
||||
modTime: stats.mtime.getTime(),
|
||||
lineNum,
|
||||
lineText,
|
||||
})
|
||||
}
|
||||
|
||||
matches.sort((a, b) => b.modTime - a.modTime)
|
||||
|
||||
const limit = 100
|
||||
const truncated = matches.length > limit
|
||||
const finalMatches = truncated ? matches.slice(0, limit) : matches
|
||||
|
||||
if (finalMatches.length === 0) {
|
||||
return {
|
||||
title: params.pattern,
|
||||
metadata: { matches: 0, truncated: false },
|
||||
output: "No files found",
|
||||
}
|
||||
}
|
||||
|
||||
const totalMatches = matches.length
|
||||
const outputLines = [`Found ${totalMatches} matches${truncated ? ` (showing first ${limit})` : ""}`]
|
||||
|
||||
let currentFile = ""
|
||||
for (const match of finalMatches) {
|
||||
if (currentFile !== match.path) {
|
||||
if (currentFile !== "") {
|
||||
outputLines.push("")
|
||||
if (!hits.length) {
|
||||
const words = params.pattern.trim().split(/\s+/).filter(Boolean)
|
||||
if (words.length >= 2 && !isConstraint(words[0])) {
|
||||
const next = words.slice(1).join(" ")
|
||||
const step = await run({
|
||||
cwd: dir,
|
||||
pattern: next,
|
||||
inc: params.include,
|
||||
mode: isRegex(next) ? "regex" : "plain",
|
||||
max: 10,
|
||||
before: 0,
|
||||
after: 4,
|
||||
})
|
||||
warn = warn ?? step.out.regexFallbackError
|
||||
if (step.hits.length > 0 && step.hits.length <= 10) {
|
||||
phase = "broad"
|
||||
note = `0 exact matches. Broadened query \`${next}\`:`
|
||||
hits = step.hits
|
||||
}
|
||||
currentFile = match.path
|
||||
outputLines.push(`${match.path}:`)
|
||||
}
|
||||
const truncatedLineText =
|
||||
match.lineText.length > MAX_LINE_LENGTH ? match.lineText.substring(0, MAX_LINE_LENGTH) + "..." : match.lineText
|
||||
outputLines.push(` Line ${match.lineNum}: ${truncatedLineText}`)
|
||||
}
|
||||
|
||||
if (truncated) {
|
||||
outputLines.push("")
|
||||
outputLines.push(
|
||||
`(Results truncated: showing ${limit} of ${totalMatches} matches (${totalMatches - limit} hidden). Consider using a more specific path or pattern.)`,
|
||||
)
|
||||
if (!hits.length) {
|
||||
const fuzzy = clean(params.pattern)
|
||||
if (fuzzy) {
|
||||
const step = await run({
|
||||
cwd: dir,
|
||||
pattern: fuzzy,
|
||||
inc: params.include,
|
||||
mode: "fuzzy",
|
||||
max: 3,
|
||||
before: 0,
|
||||
after: 2,
|
||||
})
|
||||
if (step.hits.length) {
|
||||
phase = "fuzzy"
|
||||
note = `0 exact matches. ${step.hits.length} approximate:`
|
||||
hits = step.hits
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hasErrors) {
|
||||
outputLines.push("")
|
||||
outputLines.push("(Some paths were inaccessible and skipped)")
|
||||
if (!hits.length && params.pattern.includes("/")) {
|
||||
const files = await Fff.files({
|
||||
cwd: dir,
|
||||
query: params.pattern,
|
||||
size: 1,
|
||||
})
|
||||
const row = files.items[0]
|
||||
const score = files.scores[0]
|
||||
if (row && score && score.baseScore > params.pattern.length * 10) {
|
||||
return {
|
||||
title: params.pattern,
|
||||
metadata: { matches: 0, truncated: false },
|
||||
output: `0 content matches. But there is a relevant file path:\n${row.path}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!hits.length) {
|
||||
return {
|
||||
title: params.pattern,
|
||||
metadata: { matches: 0, truncated: false },
|
||||
output: "No files found",
|
||||
}
|
||||
}
|
||||
|
||||
const rows = hits.map((hit, idx) => ({
|
||||
hit,
|
||||
idx,
|
||||
def: def(hit.lineContent),
|
||||
imp: imp(hit.lineContent),
|
||||
}))
|
||||
const hasDef = rows.some((row) => row.def)
|
||||
const show = hasDef ? rows.filter((row) => !row.imp || row.def) : rows
|
||||
show.sort((a, b) => {
|
||||
const ak = a.def ? 0 : a.imp ? 2 : 1
|
||||
const bk = b.def ? 0 : b.imp ? 2 : 1
|
||||
if (ak !== bk) return ak - bk
|
||||
return a.idx - b.idx
|
||||
})
|
||||
|
||||
const total = show.length
|
||||
const trim = show.slice(0, MAX_MATCH)
|
||||
const over = total > MAX_MATCH
|
||||
const files = new Set(trim.map((row) => row.hit.path)).size
|
||||
const budget = files <= 3 ? 5000 : files <= 8 ? 3500 : 2500
|
||||
const read = (trim.find((row) => row.def) ?? trim[0]).hit.path
|
||||
|
||||
const out: string[] = []
|
||||
if (phase === "exact") out.push(`Found ${total} matches${over ? ` (showing first ${MAX_MATCH})` : ""}`)
|
||||
if (phase !== "exact") out.push(note)
|
||||
out.push(`Read ${read}`)
|
||||
if (warn) out.push(`! regex failed: ${warn}`)
|
||||
|
||||
const by = group(trim)
|
||||
let used = out.join("\n").length
|
||||
let cut = false
|
||||
let firstDef = true
|
||||
let shown = 0
|
||||
for (const [file, list] of by.entries()) {
|
||||
const chunk = ["", `${file}:`]
|
||||
let add = 0
|
||||
for (const row of list) {
|
||||
add++
|
||||
chunk.push(` Line ${row.hit.lineNumber}: ${line(row.hit.lineContent, row.hit.matchRanges)}`)
|
||||
if (!row.def) continue
|
||||
const max = firstDef ? MAX_DEF_FIRST : MAX_DEF_NEXT
|
||||
firstDef = false
|
||||
for (const extra of (row.hit.contextAfter ?? []).slice(0, max)) {
|
||||
chunk.push(` ${line(extra, [])}`)
|
||||
}
|
||||
}
|
||||
const text = chunk.join("\n")
|
||||
if (used + text.length > budget && shown > 0) {
|
||||
cut = true
|
||||
break
|
||||
}
|
||||
out.push(...chunk)
|
||||
used += text.length
|
||||
shown += add
|
||||
}
|
||||
|
||||
if (over || cut) {
|
||||
out.push("")
|
||||
out.push(`(Results truncated: showing first ${shown} results. Consider using a more specific path or pattern.)`)
|
||||
}
|
||||
|
||||
return {
|
||||
title: params.pattern,
|
||||
metadata: {
|
||||
matches: totalMatches,
|
||||
truncated,
|
||||
matches: total,
|
||||
truncated: over || cut,
|
||||
},
|
||||
output: outputLines.join("\n"),
|
||||
output: out.join("\n"),
|
||||
}
|
||||
},
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
- Fast content search tool that works with any codebase size
|
||||
- Searches file contents using regular expressions
|
||||
- Supports full regex syntax (eg. "log.*Error", "function\s+\w+", etc.)
|
||||
- Fast content search tool that uses fuzzy-first indexing and frecency ranking
|
||||
- Searches file contents with plain text, regex, and typo-tolerant fuzzy fallback
|
||||
- Supports regex syntax (eg. "log.*Error", "function\s+\w+", etc.)
|
||||
- Filter files by pattern with the include parameter (eg. "*.js", "*.{ts,tsx}")
|
||||
- Returns file paths and line numbers with at least one match sorted by modification time
|
||||
- Returns file paths and line numbers, prioritizing likely definitions and high-signal results
|
||||
- Includes smart retries (query broadening and path suggestions) when exact matches fail
|
||||
- Use this tool when you need to find files containing specific patterns
|
||||
- If you need to identify/count the number of matches within files, use the Bash tool with `rg` (ripgrep) directly. Do NOT use `grep`.
|
||||
- When you are doing an open-ended search that may require multiple rounds of globbing and grepping, use the Task tool instead
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ import { Tool } from "./tool"
|
|||
import * as path from "path"
|
||||
import DESCRIPTION from "./ls.txt"
|
||||
import { Instance } from "../project/instance"
|
||||
import { Ripgrep } from "../file/ripgrep"
|
||||
import { Fff } from "../file/fff"
|
||||
import { assertExternalDirectory } from "./external-directory"
|
||||
import { Glob } from "../util/glob"
|
||||
|
||||
export const IGNORE_PATTERNS = [
|
||||
"node_modules/",
|
||||
|
|
@ -55,11 +56,18 @@ export const ListTool = Tool.define("list", {
|
|||
})
|
||||
|
||||
const ignoreGlobs = IGNORE_PATTERNS.map((p) => `!${p}*`).concat(params.ignore?.map((p) => `!${p}`) || [])
|
||||
const files = []
|
||||
for await (const file of Ripgrep.files({ cwd: searchPath, glob: ignoreGlobs, signal: ctx.abort })) {
|
||||
files.push(file)
|
||||
if (files.length >= LIMIT) break
|
||||
}
|
||||
const rows = (await Glob.scan("**/*", {
|
||||
cwd: searchPath,
|
||||
include: "file",
|
||||
dot: true,
|
||||
}))
|
||||
.map((row) => row.replaceAll("\\", "/"))
|
||||
.filter((row) => {
|
||||
ctx.abort.throwIfAborted()
|
||||
return Fff.allowed({ rel: row, glob: ignoreGlobs, hidden: true })
|
||||
})
|
||||
.toSorted((a, b) => a.localeCompare(b))
|
||||
const files = rows.slice(0, LIMIT)
|
||||
|
||||
// Build directory structure
|
||||
const dirs = new Set<string>()
|
||||
|
|
@ -113,7 +121,7 @@ export const ListTool = Tool.define("list", {
|
|||
title: path.relative(Instance.worktree, searchPath),
|
||||
metadata: {
|
||||
count: files.length,
|
||||
truncated: files.length >= LIMIT,
|
||||
truncated: rows.length > LIMIT,
|
||||
},
|
||||
output,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ import { pathToFileURL } from "url"
|
|||
import z from "zod"
|
||||
import { Tool } from "./tool"
|
||||
import { Skill } from "../skill"
|
||||
import { Ripgrep } from "../file/ripgrep"
|
||||
import { Fff } from "../file/fff"
|
||||
import { iife } from "@/util/iife"
|
||||
import { Glob } from "../util/glob"
|
||||
|
||||
export const SkillTool = Tool.define("skill", async (ctx) => {
|
||||
const list = await Skill.available(ctx?.agent)
|
||||
|
|
@ -60,22 +61,17 @@ export const SkillTool = Tool.define("skill", async (ctx) => {
|
|||
|
||||
const limit = 10
|
||||
const files = await iife(async () => {
|
||||
const arr = []
|
||||
for await (const file of Ripgrep.files({
|
||||
ctx.abort.throwIfAborted()
|
||||
return (await Glob.scan("**/*", {
|
||||
cwd: dir,
|
||||
follow: false,
|
||||
hidden: true,
|
||||
signal: ctx.abort,
|
||||
})) {
|
||||
if (file.includes("SKILL.md")) {
|
||||
continue
|
||||
}
|
||||
arr.push(path.resolve(dir, file))
|
||||
if (arr.length >= limit) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return arr
|
||||
include: "file",
|
||||
dot: true,
|
||||
}))
|
||||
.map((file) => file.replaceAll("\\", "/"))
|
||||
.filter((file) => Fff.allowed({ rel: file, hidden: true, glob: ["!node_modules/*", "!.git/*"] }))
|
||||
.filter((file) => !file.includes("SKILL.md"))
|
||||
.slice(0, limit)
|
||||
.map((file) => path.resolve(dir, file))
|
||||
}).then((f) => f.map((file) => `<file>${file}</file>`).join("\n"))
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import { describe, expect, test } from "bun:test"
|
||||
import fs from "fs/promises"
|
||||
import path from "path"
|
||||
import { tmpdir } from "../fixture/fixture"
|
||||
import { Instance } from "../../src/project/instance"
|
||||
import { Fff } from "../../src/file/fff"
|
||||
|
||||
describe("file.fff", () => {
|
||||
test("allowed respects hidden filter", async () => {
|
||||
expect(Fff.allowed({ rel: "visible.txt", hidden: true })).toBe(true)
|
||||
expect(Fff.allowed({ rel: ".opencode/thing.json", hidden: true })).toBe(true)
|
||||
expect(Fff.allowed({ rel: ".opencode/thing.json", hidden: false })).toBe(false)
|
||||
})
|
||||
|
||||
test("search returns empty when nothing matches", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(path.join(dir, "match.ts"), "const value = 'other'\n")
|
||||
},
|
||||
})
|
||||
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const hits = await Fff.search({
|
||||
cwd: tmp.path,
|
||||
pattern: "needle",
|
||||
})
|
||||
expect(hits).toEqual([])
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("tree builds and truncates", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await fs.mkdir(path.join(dir, "a", "b"), { recursive: true })
|
||||
await Bun.write(path.join(dir, "a", "b", "c.ts"), "export const x = 1\n")
|
||||
await Bun.write(path.join(dir, "a", "d.ts"), "export const y = 1\n")
|
||||
},
|
||||
})
|
||||
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const tree = await Fff.tree({ cwd: tmp.path, limit: 1 })
|
||||
expect(tree).toContain("a")
|
||||
expect(tree).toContain("truncated")
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
import { describe, expect, test } from "bun:test"
|
||||
import path from "path"
|
||||
import { GlobTool } from "../../src/tool/glob"
|
||||
import { Instance } from "../../src/project/instance"
|
||||
import { tmpdir } from "../fixture/fixture"
|
||||
import { SessionID, MessageID } from "../../src/session/schema"
|
||||
|
||||
const ctx = {
|
||||
sessionID: SessionID.make("ses_test"),
|
||||
messageID: MessageID.make(""),
|
||||
callID: "",
|
||||
agent: "build",
|
||||
abort: AbortSignal.any([]),
|
||||
messages: [],
|
||||
metadata: () => {},
|
||||
ask: async () => {},
|
||||
}
|
||||
|
||||
describe("tool.glob", () => {
|
||||
test("finds files by glob pattern", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(path.join(dir, "src", "foo.ts"), "export const foo = 1\n")
|
||||
await Bun.write(path.join(dir, "src", "bar.ts"), "export const bar = 1\n")
|
||||
await Bun.write(path.join(dir, "src", "baz.js"), "export const baz = 1\n")
|
||||
},
|
||||
})
|
||||
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const glob = await GlobTool.init()
|
||||
const result = await glob.execute(
|
||||
{
|
||||
pattern: "*.ts",
|
||||
path: tmp.path,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
|
||||
expect(result.metadata.count).toBe(2)
|
||||
expect(result.output).toContain(path.join(tmp.path, "src", "foo.ts"))
|
||||
expect(result.output).toContain(path.join(tmp.path, "src", "bar.ts"))
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("returns no files found for unmatched patterns", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(path.join(dir, "src", "foo.ts"), "export const foo = 1\n")
|
||||
},
|
||||
})
|
||||
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const glob = await GlobTool.init()
|
||||
const result = await glob.execute(
|
||||
{
|
||||
pattern: "*.py",
|
||||
path: tmp.path,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
|
||||
expect(result.metadata.count).toBe(0)
|
||||
expect(result.output).toBe("No files found")
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("falls back for brace glob patterns", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(path.join(dir, "src", "foo.ts"), "export const foo = 1\n")
|
||||
await Bun.write(path.join(dir, "src", "bar.js"), "export const bar = 1\n")
|
||||
await Bun.write(path.join(dir, "src", "baz.py"), "print('baz')\n")
|
||||
},
|
||||
})
|
||||
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const glob = await GlobTool.init()
|
||||
const result = await glob.execute(
|
||||
{
|
||||
pattern: "*.{ts,js}",
|
||||
path: tmp.path,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
|
||||
expect(result.metadata.count).toBe(2)
|
||||
expect(result.output).toContain(path.join(tmp.path, "src", "foo.ts"))
|
||||
expect(result.output).toContain(path.join(tmp.path, "src", "bar.js"))
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -84,28 +84,50 @@ describe("tool.grep", () => {
|
|||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("CRLF regex handling", () => {
|
||||
test("regex correctly splits Unix line endings", () => {
|
||||
const unixOutput = "file1.txt|1|content1\nfile2.txt|2|content2\nfile3.txt|3|content3"
|
||||
const lines = unixOutput.trim().split(/\r?\n/)
|
||||
expect(lines.length).toBe(3)
|
||||
expect(lines[0]).toBe("file1.txt|1|content1")
|
||||
expect(lines[2]).toBe("file3.txt|3|content3")
|
||||
test("broadens multi-word query when exact has no match", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(path.join(dir, "test.txt"), "upload completed\n")
|
||||
},
|
||||
})
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const grep = await GrepTool.init()
|
||||
const result = await grep.execute(
|
||||
{
|
||||
pattern: "prepare upload",
|
||||
path: tmp.path,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
expect(result.metadata.matches).toBeGreaterThan(0)
|
||||
expect(result.output).toContain("Broadened query")
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("regex correctly splits Windows CRLF line endings", () => {
|
||||
const windowsOutput = "file1.txt|1|content1\r\nfile2.txt|2|content2\r\nfile3.txt|3|content3"
|
||||
const lines = windowsOutput.trim().split(/\r?\n/)
|
||||
expect(lines.length).toBe(3)
|
||||
expect(lines[0]).toBe("file1.txt|1|content1")
|
||||
expect(lines[2]).toBe("file3.txt|3|content3")
|
||||
})
|
||||
|
||||
test("regex handles mixed line endings", () => {
|
||||
const mixedOutput = "file1.txt|1|content1\nfile2.txt|2|content2\r\nfile3.txt|3|content3"
|
||||
const lines = mixedOutput.trim().split(/\r?\n/)
|
||||
expect(lines.length).toBe(3)
|
||||
test("suggests path when content has no match", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
init: async (dir) => {
|
||||
await Bun.write(path.join(dir, "src", "server", "auth.ts"), "export const token = 1\n")
|
||||
},
|
||||
})
|
||||
await Instance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const grep = await GrepTool.init()
|
||||
const result = await grep.execute(
|
||||
{
|
||||
pattern: "src/server/auth.ts",
|
||||
path: tmp.path,
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
expect(result.metadata.matches).toBe(0)
|
||||
expect(result.output).toContain("relevant file path")
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue