Next.js chat with Live View

1
import { anthropic } from "@ai-sdk/anthropic";
2
import {
3
  convertToModelMessages,
4
  stepCountIs,
5
  streamText,
6
  tool,
7
  type UIMessage,
8
} from "ai";
9
import { chromium, type Browser, type Page } from "playwright";
10
import Steel from "steel-sdk";
11
import { z } from "zod";
12

13
// Playwright needs the Node.js runtime (not Edge).
14
export const runtime = "nodejs";
15
export const maxDuration = 120;
16

17
const STEEL_API_KEY = process.env.STEEL_API_KEY!;
18

19
export async function POST(req: Request) {
20
  const { messages } = (await req.json()) as { messages: UIMessage[] };
21
  const steel = new Steel({ steelAPIKey: STEEL_API_KEY });
22

23
  let session: Awaited<ReturnType<typeof steel.sessions.create>> | null = null;
24
  let browser: Browser | null = null;
25
  let page: Page | null = null;
26

27
  const cleanup = async () => {
28
    if (browser) await browser.close().catch(() => {});
29
    if (session) await steel.sessions.release(session.id).catch(() => {});
30
  };
31

32
  const result = streamText({
33
    model: anthropic("claude-haiku-4-5"),
34
    system: [
35
      "You operate a Steel cloud browser via tools.",
36
      "Workflow: (1) call openSession, (2) navigate to the target URL,",
37
      "(3) call snapshot to see the page's text and links,",
38
      "(4) only call extract when you need structured rows beyond what snapshot gives,",
39
      "(5) reply to the user in plain English.",
40
      "Prefer snapshot's links list over guessing selectors. Do not invent data.",
41
    ].join(" "),
42
    messages: await convertToModelMessages(messages),
43
    stopWhen: stepCountIs(15),
44
    tools: {
45
      openSession: tool({
46
        description:
47
          "Open a Steel cloud browser session. Call this exactly once, before anything else.",
48
        inputSchema: z.object({}),
49
        execute: async () => {
50
          session = await steel.sessions.create({});
51
          browser = await chromium.connectOverCDP(
52
            `${session.websocketUrl}&apiKey=${STEEL_API_KEY}`
53
          );
54
          const ctx = browser.contexts()[0];
55
          page = ctx.pages()[0] ?? (await ctx.newPage());
56
          return {
57
            sessionId: session.id,
58
            liveViewUrl: session.sessionViewerUrl,
59
            debugUrl: session.debugUrl,
60
          };
61
        },
62
      }),
63
      navigate: tool({
64
        description: "Navigate the open session to a URL.",
65
        inputSchema: z.object({ url: z.string().url() }),
66
        execute: async ({ url }) => {
67
          if (!page) throw new Error("openSession must be called first.");
68
          await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45_000 });
69
          return { url: page.url(), title: await page.title() };
70
        },
71
      }),
72
      snapshot: tool({
73
        description:
74
          "Return a readable snapshot of the current page: title, URL, visible text (capped), and a list of links with their text and href. Call this BEFORE extract so you never have to guess CSS selectors.",
75
        inputSchema: z.object({
76
          maxChars: z.number().int().positive().max(10_000).default(4_000),
77
          maxLinks: z.number().int().positive().max(200).default(50),
78
        }),
79
        execute: async ({ maxChars, maxLinks }) => {
80
          if (!page) throw new Error("openSession must be called first.");
81
          return (await page.evaluate(
82
            ({ maxChars, maxLinks }: { maxChars: number; maxLinks: number }) => {
83
              const text = (document.body.innerText || "").slice(0, maxChars);
84
              const links = Array.from(document.querySelectorAll("a[href]"))
85
                .slice(0, maxLinks)
86
                .map((a) => {
87
                  const anchor = a as HTMLAnchorElement;
88
                  const t = (anchor.innerText || anchor.textContent || "").trim().slice(0, 120);
89
                  return { text: t, href: anchor.href };
90
                })
91
                .filter((l) => l.text && l.href);
92
              return { url: location.href, title: document.title, text, links };
93
            },
94
            { maxChars, maxLinks }
95
          )) as { url: string; title: string; text: string; links: { text: string; href: string }[] };
96
        },
97
      }),
98
      extract: tool({
99
        description:
100
          "Extract structured data from the current page using CSS selectors.",
101
        inputSchema: z.object({
102
          rowSelector: z.string(),
103
          fields: z.array(z.object({
104
            name: z.string(),
105
            selector: z.string(),
106
            attr: z.string().optional(),
107
          })).min(1).max(10),
108
          limit: z.number().int().positive().max(20).default(10),
109
        }),
110
        execute: async ({ rowSelector, fields, limit }) => {
111
          if (!page) throw new Error("openSession must be called first.");
112
          // Batch the extraction inside one page.evaluate — N*M serial
113
          // CDP calls would cost seconds on a cloud browser.
114
          const items = (await page.evaluate(
115
            ({ rowSelector, fields, limit }: {
116
              rowSelector: string;
117
              fields: { name: string; selector: string; attr?: string }[];
118
              limit: number;
119
            }) => {
120
              const rows = Array.from(
121
                document.querySelectorAll(rowSelector)
122
              ).slice(0, limit);
123
              return rows.map((row) => {
124
                const item: Record<string, string> = {};
125
                for (const f of fields) {
126
                  const el = f.selector
127
                    ? (row.querySelector(f.selector) as Element | null)
128
                    : row;
129
                  if (!el) { item[f.name] = ""; continue; }
130
                  if (f.attr) {
131
                    item[f.name] = (el.getAttribute(f.attr) ?? "").trim();
132
                  } else {
133
                    const text = (el as HTMLElement).innerText ?? el.textContent ?? "";
134
                    item[f.name] = text.trim();
135
                  }
136
                }
137
                return item;
138
              });
139
            },
140
            { rowSelector, fields, limit }
141
          )) as Record<string, string>[];
142
          return { count: items.length, items };
143
        },
144
      }),
145
      // v6's needsApproval: destructive tools stream to the UI without executing,
146
      // wait for user confirmation, then resume. Wire up approval UI on the client
147
      // for this to run end-to-end.
148
      submitForm: tool({
149
        description: "Submit a form on the current page. Requires user approval.",
150
        inputSchema: z.object({
151
          reason: z.string().describe("Why this submission is safe."),
152
        }),
153
        needsApproval: true,
154
        execute: async ({ reason }) => {
155
          return { submitted: false, note: `Demo only. Reason: ${reason}` };
156
        },
157
      }),
158
    },
159
    // Phase-gate: no one can navigate before the session is open, and the
160
    // agent can't open a second session.
161
    prepareStep: async ({ stepNumber, steps }) => {
162
      const opened = steps.some((s: any) =>
163
        s.toolCalls?.some((tc: any) => tc.toolName === "openSession")
164
      );
165
      if (stepNumber === 0 || !opened) return { activeTools: ["openSession"] };
166
      return { activeTools: ["navigate", "snapshot", "extract", "submitForm"] };
167
    },
168
    onStepFinish: async ({ toolCalls, usage }) => {
169
      const names = toolCalls?.map((t: any) => t.toolName).join(", ") || "";
170
      console.log(`  step: ${names || "(text)"} | ${usage?.totalTokens ?? 0} tokens`);
171
    },
172
    onFinish: cleanup,
173
    onAbort: cleanup,
174
  });
175

176
  return result.toUIMessageStreamResponse();
177
}

1
"use client";
2
import { useChat } from "@ai-sdk/react";
3
import { useMemo, useState } from "react";
4

5
export default function Page() {
6
  const { messages, sendMessage, status } = useChat();
7
  const [input, setInput] = useState("");
8

9
  const { debugUrl, liveViewUrl } = useMemo(() => {
10
    for (const m of messages) {
11
      for (const part of (m.parts ?? []) as any[]) {
12
        if (part?.type === "tool-openSession" && part?.output) {
13
          return {
14
            debugUrl: (part.output.debugUrl ?? null) as string | null,
15
            liveViewUrl: (part.output.liveViewUrl ?? null) as string | null,
16
          };
17
        }
18
      }
19
    }
20
    return { debugUrl: null, liveViewUrl: null };
21
  }, [messages]);
22

23
  return (
24
    <main style={{ display: "grid", gridTemplateColumns: "1fr 1.3fr", height: "100vh" }}>
25
      <section>
26
        {messages.map((m) => (
27
          <div key={m.id}>
28
            {(m.parts ?? []).map((part: any, i: number) => {
29
              if (part.type === "text") return <span key={i}>{part.text}</span>;
30
              if (String(part.type).startsWith("tool-")) {
31
                return (
32
                  <pre key={i}>
33
                    {String(part.type)} {part.state} {JSON.stringify(part.input)}
34
                  </pre>
35
                );
36
              }
37
              return null;
38
            })}
39
          </div>
40
        ))}
41
        <form
42
          onSubmit={(e) => {
43
            e.preventDefault();
44
            if (!input.trim()) return;
45
            sendMessage({ text: input });
46
            setInput("");
47
          }}
48
        >
49
          <input value={input} onChange={(e) => setInput(e.target.value)} />
50
          <button disabled={status !== "ready"}>Send</button>
51
        </form>
52
      </section>
53
      <aside>
54
        {liveViewUrl && (
55
          <a href={liveViewUrl} target="_blank" rel="noreferrer">open in new tab ↗</a>
56
        )}
57
        {debugUrl ? (
58
          <iframe
59
            src={debugUrl}
60
            sandbox="allow-same-origin allow-scripts"
61
            style={{ width: "100%", height: "100%", border: 0 }}
62
          />
63
        ) : (
64
          <div>Live View appears once the agent opens a session.</div>
65
        )}
66
      </aside>
67
    </main>
68
  );
69
}