Load results a few schedules at a time

The dashboard polls results every 10s, and each poll fetched every
schedule's drafts in parallel. With many video schedules that burst
overloaded the engine, which answered some with 503 and dropped those
schedules' results until the next poll. Run at most 4 at once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Iqbal
2026-10-29 17:01:21 +07:00
parent c72807dc26
commit 4658d66503
2 changed files with 40 additions and 5 deletions

View File

@@ -182,12 +182,32 @@ export function keepSettled<T>(results: PromiseSettledResult<T[]>[]): T[] {
return kept
}
// Promise.allSettled over a function, running at most `limit` at a time. Results are loaded one
// request per schedule and polled; firing them all at once floods the engine, which then answers
// some with 503 and drops those schedules' results until the next poll.
export async function settleLimited<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<PromiseSettledResult<R>[]> {
const out: PromiseSettledResult<R>[] = new Array(items.length)
let next = 0
const worker = async () => {
while (next < items.length) {
const i = next++
try { out[i] = { status: 'fulfilled', value: await fn(items[i]) } }
catch (reason) { out[i] = { status: 'rejected', reason } }
}
}
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
return out
}
const PER_SCHEDULE_LIMIT = 4
export async function listResults(): Promise<{ materials: Material[]; sources: FoundSource[] }> {
const schedules = await expressAgents({ archived: true })
const [materials, sources] = await Promise.all([
Promise.allSettled(schedules.filter(s => s.rule.type === 'content').map(materialsOf)),
Promise.allSettled(schedules.filter(s => s.rule.type === 'research').map(sourcesOf)),
])
const content = schedules.filter(s => s.rule.type === 'content')
const research = schedules.filter(s => s.rule.type === 'research')
// One shared budget: research runs after content rather than alongside it.
const materials = await settleLimited(content, PER_SCHEDULE_LIMIT, materialsOf)
const sources = await settleLimited(research, PER_SCHEDULE_LIMIT, sourcesOf)
const newest = <T extends { createdAt: string }>(a: T, b: T) => b.createdAt.localeCompare(a.createdAt)
return { materials: keepSettled(materials).sort(newest), sources: keepSettled(sources).sort(newest) }
}