Attach Brain material without blocking a schedule's status

Saving a topic or resuming sent one PUT per ready document (about 150),
and a topic change left the schedule in draft (shown as paused) until
all of them were done. Creating a schedule waited on the same loop with
no sign of progress.

A topic change now re-activates (or keeps paused) right after the PATCH,
and resuming is one activate; both then attach only the documents the
schedule does not have yet. Creating a schedule still attaches every
ready document, three at a time and paced under the proxy's rate limit,
retrying a busy answer instead of skipping it, and the Instruction chat
shows how many are done.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/express/schedules.ts

@@ -1,7 +1,7 @@
 // Each Express schedule is one engine agent. Content rules use the engine's recurring production
 // (content_goal: weekly plans with novelty checks, started automatically); research and video rules
 // use run_schedule (one run per day or week; a video run renders one video).
-import { opKey, request } from '../api/client'
+import { EngineError, limiter, opKey, request } from '../api/client'
 import { listDocuments, type Agent, type RunSchedule } from '../api/engine'
 import { getLang, t } from '../i18n'
 import {
@@ -265,41 +265,84 @@ export function canSchedule(rule: Rule): rule is ResearchRule | ContentRule {
 }
 
 // The production corpus is the agent's own sources: the chosen documents, or every ready Brain document.
-async function attachBrain(agentId: number, chosen?: number[]) {
-  const ids = chosen ?? (await listDocuments()).filter(d => d.knowledge === 'ready').map(d => d.docId)
-  for (const id of ids) {
-    await request(`${AGENTS}/${agentId}/knowledge-sources`, { method: 'PUT', json: { kind: 'document', source_id: id } })
-      .catch(() => undefined) // a document someone else cannot share is skipped, not fatal
+// The engine adds them one request at a time and checks the whole workspace on each, so a Brain of
+// 150 documents is 150 slow requests: they go a few at a time, paced under the proxy's rate limit.
+const ATTACH_AT_ONCE = 3
+const ATTACH_TRIES = 3
+
+export type AttachProgress = (done: number, total: number) => void
+
+async function attachOne(agentId: number, id: number, tries = ATTACH_TRIES): Promise<void> {
+  try {
+    await request(`${AGENTS}/${agentId}/knowledge-sources`, { method: 'PUT', paced: true, json: { kind: 'document', source_id: id } })
+  } catch (e) {
+    // A busy proxy is asked again once the pace has waited for it; a document someone else cannot
+    // share (404) is skipped, not fatal.
+    const busy = e instanceof EngineError && (e.status === 503 || e.status === 429)
+    if (busy && tries > 1) return attachOne(agentId, id, tries - 1)
+  }
+}
+
+async function attachDocuments(agentId: number, ids: number[], onProgress?: AttachProgress) {
+  const one = limiter(ATTACH_AT_ONCE)
+  let done = 0
+  onProgress?.(0, ids.length)
+  await Promise.all(ids.map(id => one(async () => {
+    await attachOne(agentId, id)
+    onProgress?.(++done, ids.length)
+  })))
+}
+
+/** The ready documents a whole-Brain schedule does not have yet. */
+export const missingDocuments = (ready: number[], attached: { kind: string; id: number }[]) => {
+  const have = new Set(attached.filter(s => s.kind === 'document').map(s => s.id))
+  return ready.filter(id => !have.has(id))
+}
+
+const readyDocuments = async () => (await listDocuments()).filter(d => d.knowledge === 'ready').map(d => d.docId)
+
+// A whole-Brain schedule picks up material added since it was made; a schedule of chosen material keeps to it.
+// The schedule's own change has gone through by then, so a failure here does not fail it: the next
+// edit or resume attaches what is still missing.
+async function attachNewMaterial(schedule: Schedule) {
+  if (usesChosen(schedule.agent)) return
+  const id = schedule.agent.id
+  try {
+    const [ready, attached] = await Promise.all([readyDocuments(), request<{ kind: string; id: number }[]>(`${AGENTS}/${id}/knowledge-sources`)])
+    await attachDocuments(id, missingDocuments(ready, attached))
+  } catch (e) {
+    console.warn(`New Brain material was not attached to schedule ${id}`, e)
   }
-  return ids.length
 }
 
 /** With `chosen` document ids the schedule uses only those; without, the whole Brain. */
-export async function createSchedule(rule: Rule, chosen?: number[]): Promise<ExpressAgent> {
+export async function createSchedule(rule: Rule, chosen?: number[], onProgress?: AttachProgress): Promise<ExpressAgent> {
   const only = chosen?.length ? chosen : undefined
   // A video schedule's run budget is today's price of one video, so it is fetched first.
   const credits = rule.type === 'content' && rule.kind === 'video' ? (await videoQuote()).max_credits : 0
   const agent = await request<ExpressAgent>(AGENTS, { method: 'POST', json: configFor(rule, credits, !!only) })
-  await attachBrain(agent.id, only)
+  await attachDocuments(agent.id, only ?? (await readyDocuments()), onProgress)
   // Activating approves the production rules as they stand; until then nothing runs.
   return request<ExpressAgent>(`${AGENTS}/${agent.id}/activate`, { method: 'POST' })
 }
 
-/** Changing a rule changes what was approved, so the agent is re-activated right after. */
+/** Changing the brief changes what was approved, so the engine sets the agent back to draft; it is
+ * re-activated (or kept paused) at once. Sources are not part of the approval, so new material
+ * is attached after that, without leaving the schedule paused meanwhile. */
 export async function updateTopic(schedule: Schedule, topic: string): Promise<ExpressAgent> {
   const rule = { ...schedule.rule, topic: topic.trim() }
-  const chosen = usesChosen(schedule.agent)
-  const { name, goal, instructions } = configFor(rule, 0, chosen)
+  const { name, goal, instructions } = configFor(rule, 0, usesChosen(schedule.agent))
   await request(`${AGENTS}/${schedule.agent.id}`, { method: 'PATCH', json: { name, goal, instructions } })
-  if (!chosen) await attachBrain(schedule.agent.id)
-  return request<ExpressAgent>(`${AGENTS}/${schedule.agent.id}/${schedule.active ? 'activate' : 'pause'}`, { method: 'POST' })
+  const agent = await request<ExpressAgent>(`${AGENTS}/${schedule.agent.id}/${schedule.active ? 'activate' : 'pause'}`, { method: 'POST' })
+  await attachNewMaterial(schedule)
+  return agent
 }
 
 export const pauseSchedule = (id: number) => request<ExpressAgent>(`${AGENTS}/${id}/pause`, { method: 'POST' })
 export async function resumeSchedule(schedule: Schedule) {
-  const id = schedule.agent.id
-  if (!usesChosen(schedule.agent)) await attachBrain(id)
-  return request<ExpressAgent>(`${AGENTS}/${id}/activate`, { method: 'POST' })
+  const agent = await request<ExpressAgent>(`${AGENTS}/${schedule.agent.id}/activate`, { method: 'POST' })
+  await attachNewMaterial(schedule)
+  return agent
 }
 export const archiveSchedule = (id: number) => request<ExpressAgent>(`${AGENTS}/${id}/archive`, { method: 'POST' })