From a45c3a0049450b09b26a9c4fce6a47f7101cb34c Mon Sep 17 00:00:00 2001 From: Ryan Vogel Date: Sat, 28 Mar 2026 18:10:35 -0400 Subject: [PATCH] feat: harden mobile server flow and enrich push alerts Persist scanned servers across reloads, smooth server/session UI states, and make recording feel immediate. Add session-aware push notification title/body metadata from the OpenCode server. --- packages/mobile-voice/app.json | 8 + packages/mobile-voice/src/app/index.tsx | 345 +++++++++++++++++---- packages/opencode/src/server/push-relay.ts | 81 ++++- 3 files changed, 371 insertions(+), 63 deletions(-) diff --git a/packages/mobile-voice/app.json b/packages/mobile-voice/app.json index 1cbc511af1..43a9be652a 100644 --- a/packages/mobile-voice/app.json +++ b/packages/mobile-voice/app.json @@ -12,6 +12,14 @@ "bundleIdentifier": "com.anomalyco.mobilevoice", "infoPlist": { "NSMicrophoneUsageDescription": "This app needs microphone access for live speech-to-text dictation.", + "NSAppTransportSecurity": { + "NSExceptionDomains": { + "ts.net": { + "NSIncludesSubdomains": true, + "NSExceptionAllowsInsecureHTTPLoads": true + } + } + }, "ITSAppUsesNonExemptEncryption": false } }, diff --git a/packages/mobile-voice/src/app/index.tsx b/packages/mobile-voice/src/app/index.tsx index cac8f8db46..3b2dd415f9 100644 --- a/packages/mobile-voice/src/app/index.tsx +++ b/packages/mobile-voice/src/app/index.tsx @@ -31,6 +31,7 @@ import { useSpeechToText, WHISPER_BASE_EN } from "react-native-executorch" import { ExpoResourceFetcher } from "react-native-executorch-expo-resource-fetcher" import { AudioManager, AudioRecorder } from "react-native-audio-api" import * as Notifications from "expo-notifications" +import * as FileSystem from "expo-file-system/legacy" import Constants from "expo-constants" import { fetch as expoFetch } from "expo/fetch" import { @@ -59,6 +60,7 @@ const DROPDOWN_VISIBLE_ROWS = 6 // If the press duration is shorter than this, treat it as a tap (toggle) const TAP_THRESHOLD_MS = 300 const DEFAULT_RELAY_URL = "https://apn.dev.opencode.ai" +const SERVER_STATE_FILE = `${FileSystem.documentDirectory}mobile-voice-servers.json` type ServerItem = { id: string @@ -119,6 +121,20 @@ type Scan = { data: string } +type SavedServer = { + id: string + name: string + url: string + relayURL: string + relaySecret: string +} + +type SavedState = { + servers: SavedServer[] + activeServerId: string | null + activeSessionId: string | null +} + type Cam = { CameraView: (typeof import("expo-camera"))["CameraView"] requestCameraPermissionsAsync: (typeof import("expo-camera"))["Camera"]["requestCameraPermissionsAsync"] @@ -161,6 +177,72 @@ function pickHost(list: string[]): string | undefined { return next ?? list[0] } +function serverBases(input: string) { + const base = input.replace(/\/+$/, "") + const list = [base] + try { + const url = new URL(base) + const local = + url.hostname === "127.0.0.1" || + url.hostname === "localhost" || + url.hostname === "::1" || + url.hostname.startsWith("10.") + const tailnet = url.hostname.endsWith(".ts.net") + const secure = `https://${url.host}` + const insecure = `http://${url.host}` + if (url.protocol === "http:" && !local) { + if (tailnet) { + list.unshift(secure) + } else { + list.push(secure) + } + } else if (url.protocol === "https:" && tailnet) { + list.push(insecure) + } + } catch { + // Keep original base only. + } + return [...new Set(list)] +} + +function toSaved(servers: ServerItem[], activeServerId: string | null, activeSessionId: string | null): SavedState { + return { + servers: servers.map((item) => ({ + id: item.id, + name: item.name, + url: item.url, + relayURL: item.relayURL, + relaySecret: item.relaySecret, + })), + activeServerId, + activeSessionId, + } +} + +function fromSaved(input: SavedState): { + servers: ServerItem[] + activeServerId: string | null + activeSessionId: string | null +} { + const servers = input.servers.map((item) => ({ + id: item.id, + name: item.name, + url: item.url, + relayURL: item.relayURL, + relaySecret: item.relaySecret, + status: "checking" as const, + sessions: [] as SessionItem[], + sessionsLoading: false, + })) + const hasActive = input.activeServerId && servers.some((item) => item.id === input.activeServerId) + const activeServerId = hasActive ? input.activeServerId : (servers[0]?.id ?? null) + return { + servers, + activeServerId, + activeSessionId: hasActive ? input.activeSessionId : null, + } +} + export default function DictationScreen() { const [camera, setCamera] = useState(null) const [modelReset, setModelReset] = useState(false) @@ -183,28 +265,7 @@ export default function DictationScreen() { const [dropdownRenderMode, setDropdownRenderMode] = useState>("server") const [scanOpen, setScanOpen] = useState(false) const [camGranted, setCamGranted] = useState(false) - const [servers, setServers] = useState([ - { - id: "srv-1", - name: "Local OpenCode", - url: "http://127.0.0.1:4096", - relayURL: DEFAULT_RELAY_URL, - relaySecret: "", - status: "checking", - sessions: [], - sessionsLoading: false, - }, - { - id: "srv-2", - name: "Staging OpenCode", - url: "http://127.0.0.1:4097", - relayURL: "http://127.0.0.1:8788", - relaySecret: "", - status: "offline", - sessions: [], - sessionsLoading: false, - }, - ]) + const [servers, setServers] = useState([]) const [activeServerId, setActiveServerId] = useState(null) const [activeSessionId, setActiveSessionId] = useState(null) const [waveformLevels, setWaveformLevels] = useState(Array.from({ length: 24 }, () => 0)) @@ -232,6 +293,8 @@ export default function DictationScreen() { const monitorJobRef = useRef(null) const previousPushTokenRef = useRef(null) const scanLockRef = useRef(false) + const restoredRef = useRef(false) + const refreshSeqRef = useRef>({}) const [recorder] = useState(() => new AudioRecorder()) @@ -239,6 +302,38 @@ export default function DictationScreen() { serversRef.current = servers }, [servers]) + useEffect(() => { + let mounted = true + ;(async () => { + try { + const data = await FileSystem.readAsStringAsync(SERVER_STATE_FILE) + if (!mounted || !data) return + const parsed = JSON.parse(data) as SavedState + const next = fromSaved(parsed) + setServers(next.servers) + setActiveServerId(next.activeServerId) + setActiveSessionId(next.activeSessionId) + console.log("[Server] restore", { + count: next.servers.length, + activeServerId: next.activeServerId, + }) + } catch { + // No saved servers yet. + } finally { + restoredRef.current = true + } + })() + return () => { + mounted = false + } + }, []) + + useEffect(() => { + if (!restoredRef.current) return + const payload = toSaved(servers, activeServerId, activeSessionId) + FileSystem.writeAsStringAsync(SERVER_STATE_FILE, JSON.stringify(payload)).catch(() => {}) + }, [activeServerId, activeSessionId, servers]) + useEffect(() => { monitorJobRef.current = monitorJob }, [monitorJob]) @@ -373,30 +468,39 @@ export default function DictationScreen() { if (!m.isReady || isRecordingRef.current || isStartingRef.current) return isStartingRef.current = true + const sessionId = Date.now() + activeSessionRef.current = sessionId + accumulatedRef.current = "" + baseTextRef.current = transcribedText + isRecordingRef.current = true + setIsRecording(true) + const cancelled = () => !isRecordingRef.current || activeSessionRef.current !== sessionId // If prewarm is still running, wait once here to avoid ModelGenerating race. if (prewarmPromiseRef.current) { await prewarmPromiseRef.current prewarmPromiseRef.current = null } + if (cancelled()) { + isStartingRef.current = false + return + } try { await ensureAudioRoute() } catch (e) { console.warn("[Dictation] Failed to ensure audio route:", e) } - - isRecordingRef.current = true - setIsRecording(true) - const sessionId = Date.now() - activeSessionRef.current = sessionId - accumulatedRef.current = "" - baseTextRef.current = transcribedText + if (cancelled()) { + isStartingRef.current = false + return + } recorder.onError((err) => { console.error("[Dictation] Recorder error:", err.message) if (activeSessionRef.current !== sessionId) return isRecordingRef.current = false + activeSessionRef.current = 0 setIsRecording(false) recorder.clearOnAudioReady() recorder.clearOnError() @@ -461,7 +565,17 @@ export default function DictationScreen() { if (readyResult.status === "error") { console.error("[Dictation] onAudioReady failed:", readyResult.message) isRecordingRef.current = false + activeSessionRef.current = 0 setIsRecording(false) + recorder.clearOnAudioReady() + recorder.clearOnError() + isStartingRef.current = false + return + } + if (cancelled()) { + recorder.clearOnAudioReady() + recorder.clearOnError() + modelRef.current.streamStop() isStartingRef.current = false return } @@ -494,10 +608,14 @@ export default function DictationScreen() { console.error("[Dictation] Recorder start failed:", startResult.message) modelRef.current.streamStop() isRecordingRef.current = false + activeSessionRef.current = 0 setIsRecording(false) + recorder.clearOnAudioReady() + recorder.clearOnError() isStartingRef.current = false return } + isStartingRef.current = false try { await streamTask @@ -506,8 +624,6 @@ export default function DictationScreen() { } } catch (error) { console.error("[Dictation] Streaming error:", error) - } finally { - isStartingRef.current = false } }, [ensureAudioRoute, recorder, transcribedText]) @@ -1048,30 +1164,71 @@ export default function DictationScreen() { const refreshServerStatusAndSessions = useCallback(async (serverID: string, includeSessions = true) => { const server = serversRef.current.find((s) => s.id === serverID) if (!server) return + const req = (refreshSeqRef.current[serverID] ?? 0) + 1 + refreshSeqRef.current[serverID] = req + const current = () => refreshSeqRef.current[serverID] === req - const base = server.url.replace(/\/+$/, "") + const candidates = serverBases(server.url) + const base = candidates[0] ?? server.url.replace(/\/+$/, "") + const healthURL = `${base}/health` + const sessionsURL = `${base}/experimental/session?limit=100` + const insecureRemote = + base.startsWith("http://") && !base.includes("127.0.0.1") && !base.includes("localhost") && !base.includes("10.") console.log("[Server] refresh:start", { id: server.id, name: server.name, base, + healthURL, + sessionsURL, includeSessions, }) - setServers((prev) => - prev.map((s) => { - if (s.id !== serverID) return s - if (s.status === "checking" && s.sessionsLoading === includeSessions) return s - return { ...s, status: "checking", sessionsLoading: includeSessions ? true : s.sessionsLoading } - }), - ) + setServers((prev) => prev.map((s) => (s.id === serverID && includeSessions ? { ...s, sessionsLoading: true } : s))) + let activeBase = base try { - const healthRes = await fetch(`${base}/health`) - const online = healthRes.ok + let healthRes: Response | null = null + let healthErr: unknown + + for (const item of candidates) { + const url = `${item}/health` + try { + const next = await fetch(url) + if (next.ok) { + healthRes = next + activeBase = item + if (item !== server.url.replace(/\/+$/, "") && current()) { + setServers((prev) => prev.map((s) => (s.id === serverID ? { ...s, url: item } : s))) + console.log("[Server] refresh:scheme-upgrade", { + id: server.id, + from: server.url, + to: item, + }) + } + break + } + healthRes = next + activeBase = item + } catch (err) { + healthErr = err + console.log("[Server] health:attempt-error", { + id: server.id, + url, + error: err instanceof Error ? `${err.name}: ${err.message}` : String(err), + }) + } + } + + const online = !!healthRes?.ok + if (!current()) { + console.log("[Server] refresh:stale-skip", { id: server.id, req }) + return + } console.log("[Server] health", { id: server.id, - base, - status: healthRes.status, + base: activeBase, + url: `${activeBase}/health`, + status: healthRes?.status ?? "fetch_error", online, }) @@ -1079,7 +1236,12 @@ export default function DictationScreen() { setServers((prev) => prev.map((s) => (s.id === serverID ? { ...s, status: "offline", sessionsLoading: false, sessions: [] } : s)), ) - console.log("[Server] refresh:offline", { id: server.id, base }) + console.log("[Server] refresh:offline", { + id: server.id, + base, + candidates, + error: healthErr instanceof Error ? `${healthErr.name}: ${healthErr.message}` : String(healthErr), + }) return } @@ -1091,7 +1253,20 @@ export default function DictationScreen() { return } - const sessionsRes = await fetch(`${base}/experimental/session?limit=100`) + const resolvedSessionsURL = `${activeBase}/experimental/session?limit=100` + const sessionsRes = await fetch(resolvedSessionsURL) + if (!current()) { + console.log("[Server] refresh:stale-skip", { id: server.id, req }) + return + } + if (!sessionsRes.ok) { + console.log("[Server] sessions:http-error", { + id: server.id, + url: resolvedSessionsURL, + status: sessionsRes.status, + }) + } + const json = sessionsRes.ok ? await sessionsRes.json() : [] const sessions: SessionItem[] = Array.isArray(json) ? json @@ -1108,14 +1283,29 @@ export default function DictationScreen() { prev.map((s) => (s.id === serverID ? { ...s, status: "online", sessionsLoading: false, sessions } : s)), ) console.log("[Server] sessions", { id: server.id, count: sessions.length }) - } catch { + } catch (err) { + if (!current()) { + console.log("[Server] refresh:stale-skip", { id: server.id, req }) + return + } setServers((prev) => prev.map((s) => (s.id === serverID ? { ...s, status: "offline", sessionsLoading: false, sessions: [] } : s)), ) console.log("[Server] refresh:error", { id: server.id, base, + healthURL, + sessionsURL, + candidates, + insecureRemote, + error: err instanceof Error ? `${err.name}: ${err.message}` : String(err), }) + if (insecureRemote) { + console.log("[Server] refresh:hint", { + id: server.id, + message: "Remote http:// host may be blocked by iOS ATS; prefer https:// for non-local hosts.", + }) + } } }, []) @@ -1212,9 +1402,9 @@ export default function DictationScreen() { const id = `srv-${Date.now()}` const relaySecret = relaySecretRaw.trim() + const url = `${parsed.protocol}//${parsed.host}` const inferredName = parsed.hostname === "127.0.0.1" || parsed.hostname === "localhost" ? "Local OpenCode" : parsed.hostname - const url = `${parsed.protocol}//${parsed.host}` const relay = `${relayParsed.protocol}//${relayParsed.host}` const existing = serversRef.current.find( (item) => item.url === url && item.relayURL === relay && item.relaySecret.trim() === relaySecret, @@ -1434,7 +1624,11 @@ export default function DictationScreen() { > - + {activeServer.name} @@ -1446,7 +1640,11 @@ export default function DictationScreen() { onPress={toggleSessionMenu} style={({ pressed }) => [styles.headerSplitRight, pressed && styles.clearButtonPressed]} > - + {activeSession?.title ?? "Select session"} @@ -1497,16 +1695,20 @@ export default function DictationScreen() { )) ) ) : activeServer ? ( - activeServer.sessionsLoading ? ( - Loading sessions… - ) : activeServer.sessions.length === 0 ? ( - No sessions available + activeServer.sessions.length === 0 ? ( + activeServer.sessionsLoading ? null : ( + No sessions available + ) ) : ( - activeServer.sessions.map((session) => ( + activeServer.sessions.map((session, index) => ( handleSelectSession(session.id)} - style={({ pressed }) => [styles.serverRow, pressed && styles.serverRowPressed]} + style={({ pressed }) => [ + styles.serverRow, + index === activeServer.sessions.length - 1 && styles.serverRowLast, + pressed && styles.serverRowPressed, + ]} > @@ -1705,6 +1907,7 @@ const styles = StyleSheet.create({ flexDirection: "row", alignItems: "center", gap: 8, + width: "100%", }, headerSplitRow: { height: 45, @@ -1712,27 +1915,46 @@ const styles = StyleSheet.create({ alignItems: "center", }, headerSplitLeft: { - maxWidth: "38%", + flex: 1, + flexBasis: 0, + minWidth: 0, height: "100%", justifyContent: "center", + alignItems: "flex-start", paddingRight: 8, }, headerSplitDivider: { - width: 1, - height: 20, - backgroundColor: "#2B3140", - marginRight: 10, + width: 4, + height: 4, + borderRadius: 2, + backgroundColor: "#3F4556", + marginHorizontal: 6, }, headerSplitRight: { flex: 1, + flexBasis: 0, + minWidth: 0, height: "100%", justifyContent: "center", + alignItems: "flex-start", + paddingLeft: 8, }, workspaceHeaderText: { color: "#8F8F8F", fontSize: 14, fontWeight: "600", }, + headerServerText: { + flex: 1, + minWidth: 0, + width: "100%", + }, + headerSessionText: { + flexShrink: 1, + minWidth: 0, + width: "100%", + textAlign: "left", + }, serverMenuInline: { marginTop: 8, paddingBottom: 8, @@ -1759,6 +1981,9 @@ const styles = StyleSheet.create({ borderBottomWidth: 1, borderBottomColor: "#222733", }, + serverRowLast: { + borderBottomWidth: 0, + }, serverRowPressed: { opacity: 0.85, }, diff --git a/packages/opencode/src/server/push-relay.ts b/packages/opencode/src/server/push-relay.ts index 7066c85ebf..d0b7698129 100644 --- a/packages/opencode/src/server/push-relay.ts +++ b/packages/opencode/src/server/push-relay.ts @@ -1,4 +1,5 @@ import os from "node:os" +import { SessionID } from "@/session/schema" import { GlobalBus } from "@/bus/global" import { Log } from "@/util/log" @@ -32,6 +33,13 @@ type Event = { properties: unknown } +type Notify = { + type: Type + sessionID: string + title?: string + body?: string +} + const log = Log.create({ service: "push-relay" }) let state: State | undefined @@ -99,6 +107,66 @@ function map(event: Event): { type: Type; sessionID: string } | undefined { return { type: "complete", sessionID } } +function text(input: string) { + return input.replace(/\s+/g, " ").trim() +} + +function words(input: string, max = 18, chars = 140) { + const clean = text(input) + if (!clean) return "" + const split = clean.split(" ") + const cut = split.slice(0, max).join(" ") + if (cut.length <= chars && split.length <= max) return cut + const short = cut.slice(0, chars).trim() + return short.endsWith("…") ? short : `${short}…` +} + +function fallback(input: Type) { + if (input === "complete") return "Session complete." + if (input === "permission") return "OpenCode needs your permission decision." + return "OpenCode reported an error for your session." +} + +async function notify(input: { type: Type; sessionID: string }): Promise { + const out: Notify = { + type: input.type, + sessionID: input.sessionID, + } + + try { + const [{ Session }, { MessageV2 }] = await Promise.all([import("@/session"), import("@/session/message-v2")]) + const sessionID = SessionID.make(input.sessionID) + const session = await Session.get(sessionID) + out.title = session.title + + for await (const msg of MessageV2.stream(sessionID)) { + if (msg.info.role !== "user") continue + const body = msg.parts + .map((part) => { + if (part.type !== "text") return "" + if (part.ignored) return "" + return part.text + }) + .filter(Boolean) + .join(" ") + const next = words(body) + if (!next) continue + out.body = next + break + } + } catch (error) { + log.info("notification metadata unavailable", { + type: input.type, + sessionID: input.sessionID, + error: String(error), + }) + } + + if (!out.title) out.title = `Session ${input.type}` + if (!out.body) out.body = fallback(input.type) + return out +} + function dedupe(input: { type: Type; sessionID: string }) { if (input.type !== "complete") return false const next = state @@ -130,11 +198,13 @@ function dedupe(input: { type: Type; sessionID: string }) { return now - prev < 5_000 } -function post(input: { type: Type; sessionID: string }) { +async function post(input: { type: Type; sessionID: string }) { const next = state if (!next) return false if (dedupe(input)) return true + const content = await notify(input) + void fetch(`${next.relayURL}/v1/event`, { method: "POST", headers: { @@ -144,6 +214,8 @@ function post(input: { type: Type; sessionID: string }) { secret: next.relaySecret, eventType: input.type, sessionID: input.sessionID, + title: content.title, + body: content.body, }), }) .then(async (res) => { @@ -153,6 +225,7 @@ function post(input: { type: Type; sessionID: string }) { status: res.status, type: input.type, sessionID: input.sessionID, + title: content.title, error, }) }) @@ -160,6 +233,7 @@ function post(input: { type: Type; sessionID: string }) { log.warn("relay post failed", { type: input.type, sessionID: input.sessionID, + title: content.title, error: String(error), }) }) @@ -186,7 +260,7 @@ export namespace PushRelay { const callback = (event: { payload: Event }) => { const next = map(event.payload) if (!next) return - post(next) + void post(next) } GlobalBus.on("event", callback) const unsub = () => { @@ -236,7 +310,8 @@ export namespace PushRelay { } export function test(input: { type: Type; sessionID: string }) { - return post(input) + void post(input) + return true } export function auth(input: string) {