sandvich
/
opencode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update for whisper

pull/19545/head
Ryan Vogel 2026-03-28 21:12:24 -04:00
parent bd2e34f3bd
commit 2abf1100ee
7 changed files with 1365 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
bun.lock
View File

@ -317,6 +317,7 @@
"name": "mobile-voice",
"version": "1.0.0",
"dependencies": {
"@fugood/react-native-audio-pcm-stream": "1.1.4",
"@react-navigation/bottom-tabs": "^7.15.5",
"@react-navigation/elements": "^2.9.10",
"@react-navigation/native": "^7.1.33",
@ -345,14 +346,13 @@
"react-dom": "19.2.0",
"react-native": "0.83.4",
"react-native-audio-api": "^0.11.7",
"react-native-executorch": "^0.8.0",
"react-native-executorch-expo-resource-fetcher": "^0.8.0",
"react-native-gesture-handler": "~2.30.0",
"react-native-reanimated": "4.2.1",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0",
"react-native-web": "~0.21.0",
"react-native-worklets": "0.7.2",
"whisper.rn": "0.5.5",
},
"devDependencies": {
"@types/react": "~19.2.2",
@ -1378,6 +1378,8 @@
"@fontsource/inter": ["@fontsource/inter@5.2.8", "", {}, "sha512-P6r5WnJoKiNVV+zvW2xM13gNdFhAEpQ9dQJHt3naLvfg+LkF2ldgSLiF4T41lf1SQCM9QmkqPTn4TH568IRagg=="],
"@fugood/react-native-audio-pcm-stream": ["@fugood/react-native-audio-pcm-stream@1.1.4", "", {}, "sha512-M6H6ay4ea0vpioII9T/C9qXFPeGpxGN24nl0REP2/wtsorZXg3zzHjZbf3UUUwjf6lEEHMlGCJfXUsxwC/vV8w=="],
"@graphql-typed-document-node/core": ["@graphql-typed-document-node/core@3.2.0", "", { "peerDependencies": { "graphql": "^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0" } }, "sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ=="],
"@happy-dom/global-registrator": ["@happy-dom/global-registrator@20.0.11", "", { "dependencies": { "@types/node": "^20.0.0", "happy-dom": "^20.0.11" } }, "sha512-GqNqiShBT/lzkHTMC/slKBrvN0DsD4Di8ssBk4aDaVgEn+2WMzE6DXxq701ndSXj7/0cJ8mNT71pM7Bnrr6JRw=="],
@ -1396,8 +1398,6 @@
"@hono/zod-validator": ["@hono/zod-validator@0.4.2", "", { "peerDependencies": { "hono": ">=3.9.0", "zod": "^3.19.1" } }, "sha512-1rrlBg+EpDPhzOV4hT9pxr5+xDVmKuz6YJl+la7VCwK6ass5ldyKm5fD+umJdV2zhHD6jROoCCv8NbTwyfhT0g=="],
"@huggingface/jinja": ["@huggingface/jinja@0.5.6", "", {}, "sha512-MyMWyLnjqo+KRJYSH7oWNbsOn5onuIvfXYPcc0WOGxU0eHUV7oAYUoQTl2BMdu7ml+ea/bu11UM+EshbeHwtIA=="],
"@ibm/plex": ["@ibm/plex@6.4.1", "", { "dependencies": { "@ibm/telemetry-js": "^1.5.1" } }, "sha512-fnsipQywHt3zWvsnlyYKMikcVI7E2fEwpiPnIHFqlbByXVfQfANAAeJk1IV4mNnxhppUIDlhU0TzwYwL++Rn2g=="],
"@ibm/telemetry-js": ["@ibm/telemetry-js@1.11.0", "", { "bin": { "ibmtelemetry": "dist/collect.js" } }, "sha512-RO/9j+URJnSfseWg9ZkEX9p+a3Ousd33DBU7rOafoZB08RqdzxFVYJ2/iM50dkBuD0o7WX7GYt1sLbNgCoE+pA=="],
@ -3918,10 +3918,6 @@
"jsonfile": ["jsonfile@4.0.0", "", { "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg=="],
"jsonrepair": ["jsonrepair@3.13.3", "", { "bin": { "jsonrepair": "bin/cli.js" } }, "sha512-BTznj0owIt2CBAH/LTo7+1I5pMvl1e1033LRl/HUowlZmJOIhzC0zbX5bxMngLkfT4WnzPP26QnW5wMr2g9tsQ=="],
"jsonschema": ["jsonschema@1.5.0", "", {}, "sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw=="],
"jsonwebtoken": ["jsonwebtoken@9.0.3", "", { "dependencies": { "jws": "^4.0.1", "lodash.includes": "^4.3.0", "lodash.isboolean": "^3.0.3", "lodash.isinteger": "^4.0.4", "lodash.isnumber": "^3.0.3", "lodash.isplainobject": "^4.0.6", "lodash.isstring": "^4.0.1", "lodash.once": "^4.0.0", "ms": "^2.1.1", "semver": "^7.5.4" } }, "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g=="],
"jwa": ["jwa@2.0.1", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg=="],
@ -4536,7 +4532,7 @@
"plist": ["plist@3.1.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.8", "base64-js": "^1.5.1", "xmlbuilder": "^15.1.1" } }, "sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ=="],
"pngjs": ["pngjs@7.0.0", "", {}, "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow=="],
"pngjs": ["pngjs@5.0.0", "", {}, "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw=="],
"poe-oauth": ["poe-oauth@0.0.3", "", {}, "sha512-KgxDylcuq/mov8URSplrBGjrIjkQwjN/Ml8BhqaGsAvHzYN3yhuROdv1sDRfwqncg7TT8XzJvMeJAWmv/4NDLw=="],
@ -4650,10 +4646,6 @@
"react-native-audio-api": ["react-native-audio-api@0.11.7", "", { "dependencies": { "semver": "^7.7.3" }, "peerDependencies": { "react": "*", "react-native": "*" }, "bin": { "setup-rn-audio-api-web": "scripts/setup-rn-audio-api-web.js" } }, "sha512-2oIoP77Tn2nlouRVfEC3bAsuSyKU6xhGNkSnVXTLLQQZslEDoYX2cN9pVRZoWOqhFrLT8q4IZI9HaFgYL13L1A=="],
"react-native-executorch": ["react-native-executorch@0.8.0", "", { "dependencies": { "@huggingface/jinja": "^0.5.0", "jsonrepair": "^3.12.0", "jsonschema": "^1.5.0", "pngjs": "^7.0.0", "zod": "^4.3.6" }, "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-9zRiJiCSTOYbES4htuk+yqkhgec/i4L1E63ZYgJ1AHkDbvHyoYLH3KkKjjzxDw7NYJCCOx+6vj9l9JrodoCbzg=="],
"react-native-executorch-expo-resource-fetcher": ["react-native-executorch-expo-resource-fetcher@0.8.0", "", { "peerDependencies": { "expo": ">=54.0.0", "expo-asset": ">=12.0.0", "expo-file-system": ">=19.0.0", "react-native": "*", "react-native-executorch": "*" } }, "sha512-vdAne2FBL0nCQ2c2yHTSt8Uttm0Klmo/K7tirSVlKxgVtli4cmsfl+UpR5giaNtlRZ3ImMAMXNW34j0fItmRfQ=="],
"react-native-gesture-handler": ["react-native-gesture-handler@2.30.1", "", { "dependencies": { "@egjs/hammerjs": "^2.0.17", "hoist-non-react-statics": "^3.3.0", "invariant": "^2.2.4" }, "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-xIUBDo5ktmJs++0fZlavQNvDEE4PsihWhSeJsJtoz4Q6p0MiTM9TgrTgfEgzRR36qGPytFoeq+ShLrVwGdpUdA=="],
"react-native-is-edge-to-edge": ["react-native-is-edge-to-edge@1.3.1", "", { "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-NIXU/iT5+ORyCc7p0z2nnlkouYKX425vuU1OEm6bMMtWWR9yvb+Xg5AZmImTKoF9abxCPqrKC3rOZsKzUYgYZA=="],
@ -5446,6 +5438,8 @@
"which-typed-array": ["which-typed-array@1.1.20", "", { "dependencies": { "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.8", "call-bound": "^1.0.4", "for-each": "^0.3.5", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-tostringtag": "^1.0.2" } }, "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg=="],
"whisper.rn": ["whisper.rn@0.5.5", "", { "dependencies": { "safe-buffer": "^5.2.1" }, "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-awFE+ImMtRdGhA+hjm3GEwnSvyEVP1sdhMb+MyCa5bVdoOCpaxrwVwXDo9U46Qwkhwml3PCFaauTsGmRkTyhdw=="],
"why-is-node-running": ["why-is-node-running@3.2.2", "", { "bin": { "why-is-node-running": "cli.js" } }, "sha512-NKUzAelcoCXhXL4dJzKIwXeR8iEVqsA0Lq6Vnd0UXvgaKbzVo4ZTHROF2Jidrv+SgxOQ03fMinnNhzZATxOD3A=="],
"widest-line": ["widest-line@5.0.0", "", { "dependencies": { "string-width": "^7.0.0" } }, "sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA=="],
@ -5846,6 +5840,8 @@
"@jimp/core/mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="],
"@jimp/js-png/pngjs": ["pngjs@7.0.0", "", {}, "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow=="],
"@jimp/plugin-blit/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"@jimp/plugin-circle/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
@ -6556,8 +6552,6 @@
"proper-lockfile/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="],
"qrcode/pngjs": ["pngjs@5.0.0", "", {}, "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw=="],
"qrcode/yargs": ["yargs@15.4.1", "", { "dependencies": { "cliui": "^6.0.0", "decamelize": "^1.2.0", "find-up": "^4.1.0", "get-caller-file": "^2.0.1", "require-directory": "^2.1.1", "require-main-filename": "^2.0.0", "set-blocking": "^2.0.0", "string-width": "^4.2.0", "which-module": "^2.0.0", "y18n": "^4.0.0", "yargs-parser": "^18.1.2" } }, "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A=="],
"raw-body/iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="],
@ -6582,8 +6576,6 @@
"react-native/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="],
"react-native-executorch/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
"react-native-reanimated/react-native-is-edge-to-edge": ["react-native-is-edge-to-edge@1.2.1", "", { "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-FLbPWl/MyYQWz+KwqOZsSyj2JmLKglHatd3xLZWskXOpRaio4LfEDEz8E/A6uD8QoTHW6Aobw1jbEwK7KMgR7Q=="],
"react-native-reanimated/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="],

5
packages/mobile-voice/app.json
View File

@ -1,6 +1,6 @@
{
"expo": {
"name": "mobile-voice",
"name": "Control",
"slug": "mobile-voice",
"version": "1.0.0",
"orientation": "portrait",
@ -10,6 +10,9 @@
"ios": {
"icon": "./assets/images/icon.png",
"bundleIdentifier": "com.anomalyco.mobilevoice",
"entitlements": {
"com.apple.developer.kernel.extended-virtual-addressing": true
},
"infoPlist": {
"NSMicrophoneUsageDescription": "This app needs microphone access for live speech-to-text dictation.",
"NSAppTransportSecurity": {

6
packages/mobile-voice/package.json
View File

@ -13,6 +13,7 @@
"lint": "expo lint"
},
"dependencies": {
"@fugood/react-native-audio-pcm-stream": "1.1.4",
"@react-navigation/bottom-tabs": "^7.15.5",
"@react-navigation/elements": "^2.9.10",
"@react-navigation/native": "^7.1.33",
@ -41,14 +42,13 @@
"react-dom": "19.2.0",
"react-native": "0.83.4",
"react-native-audio-api": "^0.11.7",
"react-native-executorch": "^0.8.0",
"react-native-executorch-expo-resource-fetcher": "^0.8.0",
"react-native-gesture-handler": "~2.30.0",
"react-native-reanimated": "4.2.1",
"react-native-safe-area-context": "~5.6.2",
"react-native-screens": "~4.23.0",
"react-native-web": "~0.21.0",
"react-native-worklets": "0.7.2"
"react-native-worklets": "0.7.2",
"whisper.rn": "0.5.5"
},
"devDependencies": {
"@types/react": "~19.2.2",

25
packages/mobile-voice/src/app/_layout.tsx
View File

@ -1,25 +1,20 @@
import React from 'react';
import { Slot } from 'expo-router';
import { LogBox } from 'react-native';
import { initExecutorch } from 'react-native-executorch';
import { ExpoResourceFetcher } from 'react-native-executorch-expo-resource-fetcher';
import React from "react"
import { Slot } from "expo-router"
import { LogBox } from "react-native"
import {
configureNotificationBehavior,
registerBackgroundNotificationTask,
} from '@/notifications/monitoring-notifications';
// Initialize the ExecuTorch resource fetcher before any model hooks run
initExecutorch({ resourceFetcher: ExpoResourceFetcher });
} from "@/notifications/monitoring-notifications"
// Suppress known non-actionable warnings from third-party libs.
LogBox.ignoreLogs([
'RecordingNotificationManager is not implemented on iOS',
'[React Native ExecuTorch] No content-length header',
]);
"RecordingNotificationManager is not implemented on iOS",
"`transcribeRealtime` is deprecated, use `RealtimeTranscriber` instead",
])
configureNotificationBehavior();
registerBackgroundNotificationTask().catch(() => {});
configureNotificationBehavior()
registerBackgroundNotificationTask().catch(() => {})
export default function RootLayout() {
return <Slot />;
return <Slot />
}

1460
packages/mobile-voice/src/app/index.tsx
View File

File diff suppressed because it is too large Load Diff

138
packages/mobile-voice/src/types/whisper-rn.d.ts vendored 100644
View File

@ -0,0 +1,138 @@
declare module "whisper.rn" {
export type TranscribeOptions = {
language?: string
translate?: boolean
maxLen?: number
prompt?: string
[key: string]: unknown
}
export type TranscribeResult = {
result: string
language: string
segments: {
text: string
t0: number
t1: number
}[]
isAborted?: boolean
}
export type TranscribeRealtimeEvent = {
contextId: number
jobId: number
isCapturing: boolean
isStoppedByAction?: boolean
code: number
data?: TranscribeResult
error?: string
processTime: number
recordingTime: number
}
export type TranscribeRealtimeOptions = TranscribeOptions & {
realtimeAudioSec?: number
realtimeAudioSliceSec?: number
realtimeAudioMinSec?: number
[key: string]: unknown
}
export type WhisperContext = {
id: number
gpu: boolean
reasonNoGPU: string
transcribeRealtime(options?: TranscribeRealtimeOptions): Promise<{
stop: () => Promise<void>
subscribe: (callback: (event: TranscribeRealtimeEvent) => void) => void
}>
transcribeData(
data: ArrayBuffer,
options?: TranscribeOptions,
): {
stop: () => Promise<void>
promise: Promise<TranscribeResult>
}
release(): Promise<void>
}
export type ContextOptions = {
filePath: string | number
useGpu?: boolean
useCoreMLIos?: boolean
useFlashAttn?: boolean
}
export function initWhisper(options: ContextOptions): Promise<WhisperContext>
export function releaseAllWhisper(): Promise<void>
}
declare module "whisper.rn/realtime-transcription/index" {
import type { TranscribeOptions, TranscribeResult, WhisperContext } from "whisper.rn"
export type RealtimeTranscribeEvent = {
type: "start" | "transcribe" | "end" | "error"
sliceIndex: number
data?: TranscribeResult
isCapturing: boolean
processTime: number
recordingTime: number
}
export type RealtimeOptions = {
audioSliceSec?: number
audioMinSec?: number
maxSlicesInMemory?: number
transcribeOptions?: TranscribeOptions
logger?: (message: string) => void
}
export type RealtimeTranscriberCallbacks = {
onTranscribe?: (event: RealtimeTranscribeEvent) => void
onError?: (error: string) => void
onStatusChange?: (isActive: boolean) => void
}
export type RealtimeTranscriberDependencies = {
whisperContext: WhisperContext
audioStream: unknown
vadContext?: unknown
fs?: unknown
}
export class RealtimeTranscriber {
constructor(
dependencies: RealtimeTranscriberDependencies,
options?: RealtimeOptions,
callbacks?: RealtimeTranscriberCallbacks,
)
start(): Promise<void>
stop(): Promise<void>
release(): Promise<void>
updateCallbacks(callbacks: Partial<RealtimeTranscriberCallbacks>): void
}
}
declare module "whisper.rn/realtime-transcription" {
export * from "whisper.rn/realtime-transcription/index"
}
declare module "whisper.rn/src/realtime-transcription" {
export * from "whisper.rn/realtime-transcription/index"
}
declare module "whisper.rn/realtime-transcription/adapters/AudioPcmStreamAdapter" {
export class AudioPcmStreamAdapter {
initialize(config: Record<string, unknown>): Promise<void>
start(): Promise<void>
stop(): Promise<void>
isRecording(): boolean
onData(callback: (data: unknown) => void): void
onError(callback: (error: string) => void): void
onStatusChange(callback: (isRecording: boolean) => void): void
release(): Promise<void>
}
}
declare module "whisper.rn/src/realtime-transcription/adapters/AudioPcmStreamAdapter" {
export * from "whisper.rn/realtime-transcription/adapters/AudioPcmStreamAdapter"
}

17
packages/opencode/src/server/push-relay.ts
View File

@ -139,8 +139,8 @@ async function notify(input: { type: Type; sessionID: string }): Promise<Notify>
const session = await Session.get(sessionID)
out.title = session.title
let latestUser: string | undefined
for await (const msg of MessageV2.stream(sessionID)) {
if (msg.info.role !== "user") continue
const body = msg.parts
.map((part) => {
if (part.type !== "text") return ""
@ -151,8 +151,19 @@ async function notify(input: { type: Type; sessionID: string }): Promise<Notify>
.join(" ")
const next = words(body)
if (!next) continue
out.body = next
break
if (msg.info.role === "assistant") {
out.body = next
break
}
if (!latestUser && msg.info.role === "user") {
latestUser = next
}
}
if (!out.body) {
out.body = latestUser
}
} catch (error) {
log.info("notification metadata unavailable", {