tui: parallelize skill downloads for faster loading
Refactored skill discovery to download skills in parallel instead of sequentially, reducing load times when multiple skills need to be fetched from remote URLs. Also updated AGENTS.md with guidance on using dev branch for diffs.pull/12307/head^2
parent
266de27a0b
commit
c35bd39829
|
|
@ -1,6 +1,7 @@
|
||||||
- To regenerate the JavaScript SDK, run `./packages/sdk/js/script/build.ts`.
|
- To regenerate the JavaScript SDK, run `./packages/sdk/js/script/build.ts`.
|
||||||
- ALWAYS USE PARALLEL TOOLS WHEN APPLICABLE.
|
- ALWAYS USE PARALLEL TOOLS WHEN APPLICABLE.
|
||||||
- The default branch in this repo is `dev`.
|
- The default branch in this repo is `dev`.
|
||||||
|
- Local `main` ref may not exist; use `dev` or `origin/dev` for diffs.
|
||||||
- Prefer automation: execute requested actions without confirmation unless blocked by missing info or safety/irreversibility.
|
- Prefer automation: execute requested actions without confirmation unless blocked by missing info or safety/irreversibility.
|
||||||
|
|
||||||
## Style Guide
|
## Style Guide
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import path from "path"
|
import path from "path"
|
||||||
import { mkdir } from "fs/promises"
|
import { mkdir } from "fs/promises"
|
||||||
import { Log } from "../util/log"
|
import { Log } from "../util/log"
|
||||||
import { Global } from "@/global"
|
import { Global } from "../global"
|
||||||
|
|
||||||
export namespace Discovery {
|
export namespace Discovery {
|
||||||
const log = Log.create({ service: "skill-discovery" })
|
const log = Log.create({ service: "skill-discovery" })
|
||||||
|
|
@ -20,61 +20,78 @@ export namespace Discovery {
|
||||||
|
|
||||||
async function get(url: string, dest: string): Promise<boolean> {
|
async function get(url: string, dest: string): Promise<boolean> {
|
||||||
if (await Bun.file(dest).exists()) return true
|
if (await Bun.file(dest).exists()) return true
|
||||||
try {
|
return fetch(url)
|
||||||
const response = await fetch(url)
|
.then(async (response) => {
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
log.error("failed to download", { url, status: response.status })
|
log.error("failed to download", { url, status: response.status })
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
await Bun.write(dest, await response.text())
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
log.error("failed to download", { url, err })
|
||||||
return false
|
return false
|
||||||
}
|
})
|
||||||
const content = await response.text()
|
|
||||||
await Bun.write(dest, content)
|
|
||||||
return true
|
|
||||||
} catch (err) {
|
|
||||||
log.error("failed to download", { url, err })
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function pull(url: string): Promise<string[]> {
|
export async function pull(url: string): Promise<string[]> {
|
||||||
const result: string[] = []
|
const result: string[] = []
|
||||||
const indexUrl = new URL("index.json", url.endsWith("/") ? url : `${url}/`).href
|
const base = url.endsWith("/") ? url : `${url}/`
|
||||||
const cacheDir = dir()
|
const index = new URL("index.json", base).href
|
||||||
|
const cache = dir()
|
||||||
|
const host = base.slice(0, -1)
|
||||||
|
|
||||||
try {
|
log.info("fetching index", { url: index })
|
||||||
log.info("fetching index", { url: indexUrl })
|
const data = await fetch(index)
|
||||||
const response = await fetch(indexUrl)
|
.then(async (response) => {
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
log.error("failed to fetch index", { url: indexUrl, status: response.status })
|
log.error("failed to fetch index", { url: index, status: response.status })
|
||||||
return result
|
return undefined
|
||||||
}
|
|
||||||
|
|
||||||
const index = (await response.json()) as Index
|
|
||||||
if (!index.skills || !Array.isArray(index.skills)) {
|
|
||||||
log.warn("invalid index format", { url: indexUrl })
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const skill of index.skills) {
|
|
||||||
if (!skill.name || !skill.files || !Array.isArray(skill.files)) {
|
|
||||||
log.warn("invalid skill entry", { url: indexUrl, skill })
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
return response
|
||||||
|
.json()
|
||||||
|
.then((json) => json as Index)
|
||||||
|
.catch((err) => {
|
||||||
|
log.error("failed to parse index", { url: index, err })
|
||||||
|
return undefined
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
log.error("failed to fetch index", { url: index, err })
|
||||||
|
return undefined
|
||||||
|
})
|
||||||
|
|
||||||
const skillDir = path.join(cacheDir, skill.name)
|
if (!data?.skills || !Array.isArray(data.skills)) {
|
||||||
for (const file of skill.files) {
|
log.warn("invalid index format", { url: index })
|
||||||
const fileUrl = new URL(file, `${url.replace(/\/$/, "")}/${skill.name}/`).href
|
return result
|
||||||
const localPath = path.join(skillDir, file)
|
|
||||||
await mkdir(path.dirname(localPath), { recursive: true })
|
|
||||||
await get(fileUrl, localPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
const skillMd = path.join(skillDir, "SKILL.md")
|
|
||||||
if (await Bun.file(skillMd).exists()) result.push(skillDir)
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
log.error("failed to fetch from URL", { url, err })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const list = data.skills.filter((skill) => {
|
||||||
|
if (!skill?.name || !Array.isArray(skill.files)) {
|
||||||
|
log.warn("invalid skill entry", { url: index, skill })
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
|
||||||
|
await Promise.all(
|
||||||
|
list.map(async (skill) => {
|
||||||
|
const root = path.join(cache, skill.name)
|
||||||
|
await Promise.all(
|
||||||
|
skill.files.map(async (file) => {
|
||||||
|
const link = new URL(file, `${host}/${skill.name}/`).href
|
||||||
|
const dest = path.join(root, file)
|
||||||
|
await mkdir(path.dirname(dest), { recursive: true })
|
||||||
|
await get(link, dest)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
const md = path.join(root, "SKILL.md")
|
||||||
|
if (await Bun.file(md).exists()) result.push(root)
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -153,9 +153,9 @@ export namespace Skill {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download and load skills from URLs
|
// Download and load skills from URLs
|
||||||
for (const skillUrl of config.skills?.urls ?? []) {
|
for (const url of config.skills?.urls ?? []) {
|
||||||
const downloadedDirs = await Discovery.pull(skillUrl)
|
const list = await Discovery.pull(url)
|
||||||
for (const dir of downloadedDirs) {
|
for (const dir of list) {
|
||||||
dirs.add(dir)
|
dirs.add(dir)
|
||||||
for await (const match of SKILL_GLOB.scan({
|
for await (const match of SKILL_GLOB.scan({
|
||||||
cwd: dir,
|
cwd: dir,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue