Cooking partner

You are a friendly hands-free cooking assistant. The user is actively cooking.

You have been given the recipe the user selected. Guide them step by step on request.
When they say "next step", move to the next instruction. Read measurements clearly and slowly.
When a step needs timing, call set_timer. Warn about food safety issues.
Keep answers short — the user's hands are busy.

const instructions = `${COOKING_INSTRUCTIONS}\n\nTonight's recipe:\n${recipeText}`;

import { parseSkillMd } from 'cosmo-ai';

const agent = client.agent({
  instructions: COOKING_INSTRUCTIONS,
  skills: recipes.map((md, i) => parseSkillMd(md, { defaultName: `recipe-${i}` })),
});

'use client';

import { useCallback, useEffect, useRef, useState } from 'react';
import {
  CosmoRealtimeProvider,
  RealtimeAudio,
  MicToggle,
  RealtimeClient,
  useTranscript,
  useTransportState,
  type RealtimeSession,
} from 'cosmo-ai';
import { tool } from 'cosmo-ai/tool';
import { zodInput } from 'cosmo-ai/tool/zod';
import { z } from 'zod/v4';

const COOKING_INSTRUCTIONS = `You are a friendly hands-free cooking assistant. …`;

type CookingSessionProps = {
  token: string; // short-lived end-user JWT minted on your backend
  recipeText: string;
  recipeName: string;
};

export function CookingPartner({ token, recipeText, recipeName }: CookingSessionProps) {
  const clientRef = useRef<RealtimeClient | null>(null);
  if (clientRef.current === null) {
    clientRef.current = new RealtimeClient({
      baseUrl: 'https://app.askcosmo.ai',
      token,
    });
  }
  const client = clientRef.current;
  const sessionRef = useRef<RealtimeSession | null>(null);

  useEffect(() => {
    const setTimer = tool({
      name: 'set_timer',
      description: 'Start a countdown timer for a cooking step',
      input: zodInput(
        z.object({
          label: z.string().describe('What the timer is for'),
          seconds: z.number().describe('Duration in seconds'),
        }),
      ),
      handler: async ({ label, seconds }) => {
        startKitchenTimer(label, seconds); // your app's timer UI
        return { started: true };
      },
    });

    const agent = client.agent({
      instructions: `${COOKING_INSTRUCTIONS}\n\nTonight's recipe:\n${recipeText}`,
      greeting: `Ready when you are — we're making ${recipeName}.`,
      tools: [setTimer, { kind: 'web_search' }],
    });

    let cancelled = false;
    void agent.start().then((session) => {
      if (cancelled) {
        void session.end();
        return;
      }
      sessionRef.current = session;
    });
    return () => {
      cancelled = true;
      void sessionRef.current?.end();
      sessionRef.current = null;
    };
  }, [client, recipeText, recipeName]);

  return (
    <CosmoRealtimeProvider client={client}>
      <KitchenView recipeName={recipeName} sessionRef={sessionRef} />
    </CosmoRealtimeProvider>
  );
}

function KitchenView({
  recipeName,
  sessionRef,
}: {
  recipeName: string;
  sessionRef: React.RefObject<RealtimeSession | null>;
}) {
  const transport = useTransportState();
  const transcript = useTranscript({ limit: 10 });

  const cameraHandleRef = useRef<string | null>(null);
  const videoRef = useRef<HTMLVideoElement | null>(null);
  const [cameraActive, setCameraActive] = useState(false);

  const toggleCamera = useCallback(async () => {
    const session = sessionRef.current;
    if (session === null || transport !== 'ready') return;

    if (cameraActive) {
      const handle = cameraHandleRef.current;
      if (handle !== null) {
        await session.removeVideoStream(handle);
        cameraHandleRef.current = null;
      }
      if (videoRef.current?.srcObject) {
        const s = videoRef.current.srcObject as MediaStream;
        for (const t of s.getTracks()) t.stop();
        videoRef.current.srcObject = null;
      }
      setCameraActive(false);
    } else {
      const stream = await navigator.mediaDevices.getUserMedia({
        video: { facingMode: 'environment' },
      });
      if (videoRef.current) videoRef.current.srcObject = stream;
      cameraHandleRef.current = await session.addVideoStream(stream, {
        id: 'stovetop',
        fps: 1,
      });
      setCameraActive(true);
    }
  }, [sessionRef, transport, cameraActive]);

  return (
    <div style={{ maxWidth: 480, margin: '0 auto', padding: 20, fontFamily: 'system-ui' }}>
      <h1 style={{ fontSize: 20 }}>{recipeName}</h1>
      <p style={{ color: '#555', fontSize: 13 }}>
        Status: {transport}
      </p>

      {/* Audio */}
      <RealtimeAudio />

      {/* Camera preview (optional) */}
      <video
        ref={videoRef}
        autoPlay
        muted
        playsInline
        style={{
          width: '100%',
          borderRadius: 8,
          background: '#000',
          display: cameraActive ? 'block' : 'none',
        }}
      />

      {/* Controls */}
      <div style={{ display: 'flex', gap: 12, marginTop: 12 }}>
        <MicToggle label={{ muted: 'Unmute', unmuted: 'Mute' }} />
        <button onClick={toggleCamera} disabled={transport !== 'ready'}>
          {cameraActive ? 'Stop camera' : 'Show stovetop'}
        </button>
        <button onClick={() => sessionRef.current?.end()}>End session</button>
      </div>

      {/* Transcript */}
      <div style={{ marginTop: 20, display: 'flex', flexDirection: 'column', gap: 6 }}>
        {transcript.map((item) => (
          <div
            key={item.id}
            style={{
              padding: '8px 12px',
              borderRadius: 10,
              background: item.role === 'user' ? '#dcfce7' : '#f1f5f9',
              alignSelf: item.role === 'user' ? 'flex-end' : 'flex-start',
              maxWidth: '85%',
              fontSize: 15,
              lineHeight: 1.4,
              opacity: item.isFinal ? 1 : 0.6,
            }}
          >
            {item.text}
          </div>
        ))}
      </div>
    </div>
  );
}

import asyncio
import sys
from typing import Any

from pydantic import BaseModel, Field

from cosmo_ai import (
    CosmoRealtime,
    RealtimeReady,
    RealtimeTranscriptDelta,
    WebSearchTool,
    tool,
)

COOKING_INSTRUCTIONS = "You are a friendly hands-free cooking assistant. …"


class TimerInput(BaseModel):
    label: str = Field(description="What the timer is for")
    seconds: int = Field(description="Duration in seconds")


@tool
async def set_timer(input: TimerInput) -> dict[str, Any]:
    """Start a countdown timer for a cooking step."""
    schedule_kitchen_timer(input.label, input.seconds)  # your app's timer
    return {"started": True}


async def run(api_key: str, recipe_text: str) -> None:
    async with CosmoRealtime(api_key=api_key) as client:
        agent = client.agent(
            instructions=f"{COOKING_INSTRUCTIONS}\n\nTonight's recipe:\n{recipe_text}",
            tools=[set_timer, WebSearchTool()],
        )
        async with agent.start() as session:

            async def print_events() -> None:
                async for event in session:
                    if isinstance(event, RealtimeReady):
                        print(f"Session ready: {event.session_id}")
                        print("Ask me about the recipe. Press Enter to stop.")
                    elif isinstance(event, RealtimeTranscriptDelta) and event.is_final:
                        print(f"[{event.role.value}] {event.text}")

            printer = asyncio.create_task(print_events())

            await session.set_microphone_enabled(True)
            await session.set_speaker_enabled(True)

            loop = asyncio.get_event_loop()
            try:
                await loop.run_in_executor(None, sys.stdin.readline)
            except (KeyboardInterrupt, EOFError):
                pass

            await session.end()
            await printer