SOFTWARE
DEVELOPER

1// Groq streams tokens as they're generated; we pipe each chunk
2// directly to the client instead of waiting for the full response.
3export async function POST(req: NextRequest) {
4  const ip = req.headers.get("x-forwarded-for") ?? "unknown"
5  if (isRateLimited(ip)) {
6    return NextResponse.json({ error: "Rate limited" }, { status: 429 })
7  }
8
9  const { messages } = await req.json()
10  const completion = await groq.chat.completions.create({
11    model: "llama-3.1-8b-instant",
12    messages: [{ role: "system", content: SYSTEM_PROMPT }, ...messages],
13    stream: true,
14    max_tokens: 512,
15  })
16
17  const readable = new ReadableStream({
18    async start(controller) {
19      const enc = new TextEncoder()
20      for await (const chunk of completion) {
21        const text = chunk.choices[0]?.delta?.content ?? ""
22        if (text) controller.enqueue(enc.encode(text))
23      }
24      controller.close()
25    },
26  })
27
28  return new Response(readable, {
29    headers: { "Content-Type": "text/plain; charset=utf-8" },
30  })
31}