Compare commits
6 Commits
fix/text-c
...
feat/add-t
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
85ab8b4b99 | ||
|
|
efb98f612b | ||
|
|
befb6228f5 | ||
|
|
2454a4729d | ||
|
|
935b10279b | ||
|
|
052c2b3ad1 |
10
README.md
10
README.md
@@ -15,7 +15,7 @@ Feel free to play around with the existing code and please leave any feedback fo
|
||||
|
||||
3. Run `npm install` (assuming you have npm installed. If not, please follow these instructions: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm/)
|
||||
|
||||
4. Enter your HeyGen Enterprise API Token or Trial Token in the `.env` file. Replace `PLACEHOLDER-API-KEY` with your API key. This will allow the Client app to generate secure Access Tokens with which to create interactive sessions.
|
||||
4. Enter your HeyGen Enterprise API Token or Trial Token in the `.env` file. Replace `HEYGEN_API_KEY` with your API key. This will allow the Client app to generate secure Access Tokens with which to create interactive sessions.
|
||||
|
||||
You can retrieve either the API Key or Trial Token by logging in to HeyGen and navigating to this page in your settings: [https://app.heygen.com/settings?nav=API]. NOTE: use the trial token if you don't have an enterprise API token yet.
|
||||
|
||||
@@ -67,14 +67,6 @@ In order to use a private Avatar created under your own account in Interactive A
|
||||
|
||||
Please note that Photo Avatars are not compatible with Interactive Avatar and cannot be used.
|
||||
|
||||
### Which voices can I use with my Interactive Avatar?
|
||||
|
||||
Most of HeyGen's AI Voices can be used with the Interactive Avatar API. To find the Voice IDs that you can use, please use the List Voices v2 endpoint from HeyGen: [https://docs.heygen.com/reference/list-voices-v2]
|
||||
|
||||
Please note that for voices that support Emotions, such as Christine and Tarquin, you need to pass in the Emotion string in the Voice Setting parameter: [https://docs.heygen.com/reference/new-session-copy#voicesetting]
|
||||
|
||||
You can also set the speed at which the Interactive Avatar speaks by passing in a Rate in the Voice Setting.
|
||||
|
||||
### Where can I read more about enterprise-level usage of the Interactive Avatar API?
|
||||
|
||||
Please read our Interactive Avatar 101 article for more information on pricing and how to increase your concurrent session limit: https://help.heygen.com/en/articles/9182113-interactive-avatar-101-your-ultimate-guide
|
||||
|
||||
@@ -13,7 +13,7 @@ export async function POST() {
|
||||
headers: {
|
||||
"x-api-key": HEYGEN_API_KEY,
|
||||
},
|
||||
}
|
||||
},
|
||||
);
|
||||
const data = await res.json();
|
||||
|
||||
|
||||
@@ -20,116 +20,3 @@ export const AVATARS = [
|
||||
name: "Joshua Heygen CEO",
|
||||
},
|
||||
];
|
||||
|
||||
export const VOICES = [
|
||||
{
|
||||
voice_id: "077ab11b14f04ce0b49b5f6e5cc20979",
|
||||
language: "English",
|
||||
gender: "Male",
|
||||
name: "Paul - Natural",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/k6dKrFe85PisZ3FMLeppUM.mp3",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "131a436c47064f708210df6628ef8f32",
|
||||
language: "English",
|
||||
gender: "Female",
|
||||
name: "Amber - Friendly",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/5HHGT48B6g6aSg2buYcBvw.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "0ebe70d83b2349529e56492c002c9572",
|
||||
language: "English",
|
||||
gender: "Male",
|
||||
name: "Antoni - Friendly",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/TwupgZ2az5RiTnmAifPmmS.mp3",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "1bd001e7e50f421d891986aad5158bc8",
|
||||
language: "English",
|
||||
gender: "Female",
|
||||
name: "Sara - Cheerful",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/func8CFnfVLKF2VzGDCDCR.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "001cc6d54eae4ca2b5fb16ca8e8eb9bb",
|
||||
language: "Spanish",
|
||||
gender: "Male",
|
||||
name: "Elias - Natural",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/JmCb3rgMZnCjCAA9aacnGj.wav",
|
||||
support_pause: false,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "00988b7d451d0722635ff7b2b9540a7b",
|
||||
language: "Portuguese",
|
||||
gender: "Female",
|
||||
name: "Brenda - Professional",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/fec6396adb73461c9997b2c0d7759b7b.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "00c8fd447ad7480ab1785825978a2215",
|
||||
language: "Chinese",
|
||||
gender: "Female",
|
||||
name: "Xiaoxuan - Serious",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/909633f8d34e408a9aaa4e1b60586865.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "00ed77fac8b84ffcb2ab52739b9dccd3",
|
||||
language: "Latvian",
|
||||
gender: "Male",
|
||||
name: "Nils - Affinity",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/KwTwAz3R4aBFN69fEYQFdX.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "02bec3b4cb514722a84e4e18d596fddf",
|
||||
language: "Arabic",
|
||||
gender: "Female",
|
||||
name: "Fatima - Professional",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/930a245487fe42158c810ac76b8ddbab.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "04e95f5bcb8b4620a2c4ef45b8a4481a",
|
||||
language: "Ukrainian",
|
||||
gender: "Female",
|
||||
name: "Polina - Professional",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/ntekV94yFpvv4RgBVPqW7c.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
{
|
||||
voice_id: "071d6bea6a7f455b82b6364dab9104a2",
|
||||
language: "German",
|
||||
gender: "Male",
|
||||
name: "Jan - Natural",
|
||||
preview_audio:
|
||||
"https://static.heygen.ai/voice_preview/fa3728bed81a4d11b8ccef10506af5f4.wav",
|
||||
support_pause: true,
|
||||
emotion_support: false,
|
||||
},
|
||||
];
|
||||
|
||||
23
app/page.tsx
23
app/page.tsx
@@ -1,34 +1,13 @@
|
||||
"use client";
|
||||
|
||||
import InteractiveAvatar from "@/components/InteractiveAvatar";
|
||||
import InteractiveAvatarCode from "@/components/InteractiveAvatarCode";
|
||||
import { Tab, Tabs } from "@nextui-org/react";
|
||||
|
||||
export default function App() {
|
||||
const tabs = [
|
||||
{
|
||||
id: "demo",
|
||||
label: "Demo",
|
||||
content: <InteractiveAvatar />,
|
||||
},
|
||||
{
|
||||
id: "code",
|
||||
label: "Code",
|
||||
content: <InteractiveAvatarCode />,
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="w-screen h-screen flex flex-col">
|
||||
<div className="w-[900px] flex flex-col items-start justify-start gap-5 mx-auto pt-4 pb-20">
|
||||
<div className="w-full">
|
||||
<Tabs items={tabs}>
|
||||
{(items) => (
|
||||
<Tab key={items.id} title={items.label}>
|
||||
{items.content}
|
||||
</Tab>
|
||||
)}
|
||||
</Tabs>
|
||||
<InteractiveAvatar />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { AVATARS, VOICES } from "@/app/lib/constants";
|
||||
import {
|
||||
Configuration,
|
||||
NewSessionData,
|
||||
StreamingAvatarApi,
|
||||
import type { StartAvatarResponse } from "@heygen/streaming-avatar";
|
||||
|
||||
import StreamingAvatar, {
|
||||
AvatarQuality,
|
||||
StreamingEvents, TaskType,
|
||||
} from "@heygen/streaming-avatar";
|
||||
import {
|
||||
Button,
|
||||
@@ -14,63 +14,30 @@ import {
|
||||
Select,
|
||||
SelectItem,
|
||||
Spinner,
|
||||
Tooltip,
|
||||
Chip,
|
||||
Tabs,
|
||||
Tab,
|
||||
} from "@nextui-org/react";
|
||||
import { Microphone, MicrophoneStage } from "@phosphor-icons/react";
|
||||
import { useChat } from "ai/react";
|
||||
import clsx from "clsx";
|
||||
import OpenAI from "openai";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useMemoizedFn, usePrevious } from "ahooks";
|
||||
|
||||
import InteractiveAvatarTextInput from "./InteractiveAvatarTextInput";
|
||||
|
||||
const openai = new OpenAI({
|
||||
apiKey: process.env.NEXT_PUBLIC_OPENAI_API_KEY,
|
||||
dangerouslyAllowBrowser: true,
|
||||
});
|
||||
import { AVATARS } from "@/app/lib/constants";
|
||||
|
||||
export default function InteractiveAvatar() {
|
||||
const [isLoadingSession, setIsLoadingSession] = useState(false);
|
||||
const [isLoadingRepeat, setIsLoadingRepeat] = useState(false);
|
||||
const [isLoadingChat, setIsLoadingChat] = useState(false);
|
||||
const [stream, setStream] = useState<MediaStream>();
|
||||
const [debug, setDebug] = useState<string>();
|
||||
const [knowledgeId, setKnowledgeId] = useState<string>("");
|
||||
const [avatarId, setAvatarId] = useState<string>("");
|
||||
const [voiceId, setVoiceId] = useState<string>("");
|
||||
const [data, setData] = useState<NewSessionData>();
|
||||
const [data, setData] = useState<StartAvatarResponse>();
|
||||
const [text, setText] = useState<string>("");
|
||||
const [initialized, setInitialized] = useState(false); // Track initialization
|
||||
const [recording, setRecording] = useState(false); // Track recording state
|
||||
const mediaStream = useRef<HTMLVideoElement>(null);
|
||||
const avatar = useRef<StreamingAvatarApi | null>(null);
|
||||
const mediaRecorder = useRef<MediaRecorder | null>(null);
|
||||
const audioChunks = useRef<Blob[]>([]);
|
||||
const { input, setInput, handleSubmit } = useChat({
|
||||
onFinish: async (message) => {
|
||||
console.log("ChatGPT Response:", message);
|
||||
|
||||
if (!initialized || !avatar.current) {
|
||||
setDebug("Avatar API not initialized");
|
||||
return;
|
||||
}
|
||||
|
||||
//send the ChatGPT response to the Interactive Avatar
|
||||
await avatar.current
|
||||
.speak({
|
||||
taskRequest: { text: message.content, sessionId: data?.sessionId },
|
||||
})
|
||||
.catch((e) => {
|
||||
setDebug(e.message);
|
||||
});
|
||||
setIsLoadingChat(false);
|
||||
},
|
||||
initialMessages: [
|
||||
{
|
||||
id: "1",
|
||||
role: "system",
|
||||
content: "You are a helpful assistant.",
|
||||
},
|
||||
],
|
||||
});
|
||||
const avatar = useRef<StreamingAvatar | null>(null);
|
||||
const [chatMode, setChatMode] = useState("text_mode");
|
||||
const [isUserTalking, setIsUserTalking] = useState(false);
|
||||
|
||||
async function fetchAccessToken() {
|
||||
try {
|
||||
@@ -78,114 +45,115 @@ export default function InteractiveAvatar() {
|
||||
method: "POST",
|
||||
});
|
||||
const token = await response.text();
|
||||
|
||||
console.log("Access Token:", token); // Log the token to verify
|
||||
|
||||
return token;
|
||||
} catch (error) {
|
||||
console.error("Error fetching access token:", error);
|
||||
return "";
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
async function startSession() {
|
||||
setIsLoadingSession(true);
|
||||
await updateToken();
|
||||
if (!avatar.current) {
|
||||
setDebug("Avatar API is not initialized");
|
||||
return;
|
||||
}
|
||||
const newToken = await fetchAccessToken();
|
||||
|
||||
avatar.current = new StreamingAvatar({
|
||||
token: newToken,
|
||||
});
|
||||
avatar.current.on(StreamingEvents.AVATAR_START_TALKING, (e) => {
|
||||
console.log("Avatar started talking", e);
|
||||
});
|
||||
avatar.current.on(StreamingEvents.AVATAR_STOP_TALKING, (e) => {
|
||||
console.log("Avatar stopped talking", e);
|
||||
});
|
||||
avatar.current.on(StreamingEvents.STREAM_DISCONNECTED, () => {
|
||||
console.log("Stream disconnected");
|
||||
endSession();
|
||||
});
|
||||
avatar.current?.on(StreamingEvents.STREAM_READY, (event) => {
|
||||
console.log(">>>>> Stream ready:", event.detail);
|
||||
setStream(event.detail);
|
||||
});
|
||||
avatar.current?.on(StreamingEvents.USER_START, (event) => {
|
||||
console.log(">>>>> User started talking:", event);
|
||||
setIsUserTalking(true);
|
||||
});
|
||||
avatar.current?.on(StreamingEvents.USER_STOP, (event) => {
|
||||
console.log(">>>>> User stopped talking:", event);
|
||||
setIsUserTalking(false);
|
||||
});
|
||||
try {
|
||||
const res = await avatar.current.createStartAvatar(
|
||||
{
|
||||
newSessionRequest: {
|
||||
quality: "low",
|
||||
avatarName: avatarId,
|
||||
voice: { voiceId: voiceId },
|
||||
},
|
||||
},
|
||||
setDebug
|
||||
);
|
||||
const res = await avatar.current.createStartAvatar({
|
||||
quality: AvatarQuality.Low,
|
||||
avatarName: avatarId,
|
||||
knowledgeId: knowledgeId,
|
||||
});
|
||||
|
||||
setData(res);
|
||||
setStream(avatar.current.mediaStream);
|
||||
// default to voice mode
|
||||
await avatar.current?.startVoiceChat();
|
||||
setChatMode("voice_mode");
|
||||
} catch (error) {
|
||||
console.error("Error starting avatar session:", error);
|
||||
setDebug(
|
||||
`There was an error starting the session. ${voiceId ? "This custom voice ID may not be supported." : ""}`
|
||||
);
|
||||
} finally {
|
||||
setIsLoadingSession(false);
|
||||
}
|
||||
setIsLoadingSession(false);
|
||||
}
|
||||
|
||||
async function updateToken() {
|
||||
const newToken = await fetchAccessToken();
|
||||
console.log("Updating Access Token:", newToken); // Log token for debugging
|
||||
avatar.current = new StreamingAvatarApi(
|
||||
new Configuration({ accessToken: newToken })
|
||||
);
|
||||
|
||||
const startTalkCallback = (e: any) => {
|
||||
console.log("Avatar started talking", e);
|
||||
};
|
||||
|
||||
const stopTalkCallback = (e: any) => {
|
||||
console.log("Avatar stopped talking", e);
|
||||
};
|
||||
|
||||
console.log("Adding event handlers:", avatar.current);
|
||||
avatar.current.addEventHandler("avatar_start_talking", startTalkCallback);
|
||||
avatar.current.addEventHandler("avatar_stop_talking", stopTalkCallback);
|
||||
|
||||
setInitialized(true);
|
||||
}
|
||||
|
||||
async function handleInterrupt() {
|
||||
if (!initialized || !avatar.current) {
|
||||
async function handleSpeak() {
|
||||
setIsLoadingRepeat(true);
|
||||
if (!avatar.current) {
|
||||
setDebug("Avatar API not initialized");
|
||||
|
||||
return;
|
||||
}
|
||||
// speak({ text: text, task_type: TaskType.REPEAT })
|
||||
await avatar.current.speak({ text: text }).catch((e) => {
|
||||
setDebug(e.message);
|
||||
});
|
||||
setIsLoadingRepeat(false);
|
||||
}
|
||||
async function handleInterrupt() {
|
||||
if (!avatar.current) {
|
||||
setDebug("Avatar API not initialized");
|
||||
|
||||
return;
|
||||
}
|
||||
await avatar.current
|
||||
.interrupt({ interruptRequest: { sessionId: data?.sessionId } })
|
||||
.interrupt()
|
||||
.catch((e) => {
|
||||
setDebug(e.message);
|
||||
});
|
||||
}
|
||||
|
||||
async function endSession() {
|
||||
if (!initialized || !avatar.current) {
|
||||
setDebug("Avatar API not initialized");
|
||||
return;
|
||||
}
|
||||
await avatar.current.stopAvatar(
|
||||
{ stopSessionRequest: { sessionId: data?.sessionId } },
|
||||
setDebug
|
||||
);
|
||||
await avatar.current?.stopAvatar();
|
||||
setStream(undefined);
|
||||
}
|
||||
|
||||
async function handleSpeak() {
|
||||
setIsLoadingRepeat(true);
|
||||
if (!initialized || !avatar.current) {
|
||||
setDebug("Avatar API not initialized");
|
||||
const handleChangeChatMode = useMemoizedFn(async (v) => {
|
||||
if (v === chatMode) {
|
||||
return;
|
||||
}
|
||||
await avatar.current
|
||||
.speak({ taskRequest: { text: text, sessionId: data?.sessionId } })
|
||||
.catch((e) => {
|
||||
setDebug(e.message);
|
||||
});
|
||||
setIsLoadingRepeat(false);
|
||||
}
|
||||
if (v === "text_mode") {
|
||||
avatar.current?.closeVoiceChat();
|
||||
} else {
|
||||
await avatar.current?.startVoiceChat();
|
||||
}
|
||||
setChatMode(v);
|
||||
});
|
||||
|
||||
const previousText = usePrevious(text);
|
||||
useEffect(() => {
|
||||
if (!previousText && text) {
|
||||
avatar.current?.startListening();
|
||||
} else if (previousText && !text) {
|
||||
avatar?.current?.stopListening();
|
||||
}
|
||||
}, [text, previousText]);
|
||||
|
||||
useEffect(() => {
|
||||
async function init() {
|
||||
const newToken = await fetchAccessToken();
|
||||
console.log("Initializing with Access Token:", newToken); // Log token for debugging
|
||||
avatar.current = new StreamingAvatarApi(
|
||||
new Configuration({ accessToken: newToken, jitterBuffer: 200 })
|
||||
);
|
||||
setInitialized(true); // Set initialized to true
|
||||
}
|
||||
init();
|
||||
|
||||
return () => {
|
||||
endSession();
|
||||
};
|
||||
@@ -201,54 +169,6 @@ export default function InteractiveAvatar() {
|
||||
}
|
||||
}, [mediaStream, stream]);
|
||||
|
||||
function startRecording() {
|
||||
navigator.mediaDevices
|
||||
.getUserMedia({ audio: true })
|
||||
.then((stream) => {
|
||||
mediaRecorder.current = new MediaRecorder(stream);
|
||||
mediaRecorder.current.ondataavailable = (event) => {
|
||||
audioChunks.current.push(event.data);
|
||||
};
|
||||
mediaRecorder.current.onstop = () => {
|
||||
const audioBlob = new Blob(audioChunks.current, {
|
||||
type: "audio/wav",
|
||||
});
|
||||
audioChunks.current = [];
|
||||
transcribeAudio(audioBlob);
|
||||
};
|
||||
mediaRecorder.current.start();
|
||||
setRecording(true);
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("Error accessing microphone:", error);
|
||||
});
|
||||
}
|
||||
|
||||
function stopRecording() {
|
||||
if (mediaRecorder.current) {
|
||||
mediaRecorder.current.stop();
|
||||
setRecording(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function transcribeAudio(audioBlob: Blob) {
|
||||
try {
|
||||
// Convert Blob to File
|
||||
const audioFile = new File([audioBlob], "recording.wav", {
|
||||
type: "audio/wav",
|
||||
});
|
||||
const response = await openai.audio.transcriptions.create({
|
||||
model: "whisper-1",
|
||||
file: audioFile,
|
||||
});
|
||||
const transcription = response.text;
|
||||
console.log("Transcription: ", transcription);
|
||||
setInput(transcription);
|
||||
} catch (error) {
|
||||
console.error("Error transcribing audio:", error);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="w-full flex flex-col gap-4">
|
||||
<Card>
|
||||
@@ -269,18 +189,18 @@ export default function InteractiveAvatar() {
|
||||
</video>
|
||||
<div className="flex flex-col gap-2 absolute bottom-3 right-3">
|
||||
<Button
|
||||
size="md"
|
||||
onClick={handleInterrupt}
|
||||
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 text-white rounded-lg"
|
||||
size="md"
|
||||
variant="shadow"
|
||||
onClick={handleInterrupt}
|
||||
>
|
||||
Interrupt task
|
||||
</Button>
|
||||
<Button
|
||||
size="md"
|
||||
onClick={endSession}
|
||||
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 text-white rounded-lg"
|
||||
size="md"
|
||||
variant="shadow"
|
||||
onClick={endSession}
|
||||
>
|
||||
End session
|
||||
</Button>
|
||||
@@ -289,13 +209,21 @@ export default function InteractiveAvatar() {
|
||||
) : !isLoadingSession ? (
|
||||
<div className="h-full justify-center items-center flex flex-col gap-8 w-[500px] self-center">
|
||||
<div className="flex flex-col gap-2 w-full">
|
||||
<p className="text-sm font-medium leading-none">
|
||||
Custom Knowledge ID (optional)
|
||||
</p>
|
||||
<Input
|
||||
placeholder="Enter a custom knowledge ID"
|
||||
value={knowledgeId}
|
||||
onChange={(e) => setKnowledgeId(e.target.value)}
|
||||
/>
|
||||
<p className="text-sm font-medium leading-none">
|
||||
Custom Avatar ID (optional)
|
||||
</p>
|
||||
<Input
|
||||
placeholder="Enter a custom avatar ID"
|
||||
value={avatarId}
|
||||
onChange={(e) => setAvatarId(e.target.value)}
|
||||
placeholder="Enter a custom avatar ID"
|
||||
/>
|
||||
<Select
|
||||
placeholder="Or select one from these example avatars"
|
||||
@@ -314,97 +242,58 @@ export default function InteractiveAvatar() {
|
||||
))}
|
||||
</Select>
|
||||
</div>
|
||||
<div className="flex flex-col gap-2 w-full">
|
||||
<p className="text-sm font-medium leading-none">
|
||||
Custom Voice ID (optional)
|
||||
</p>
|
||||
<Input
|
||||
value={voiceId}
|
||||
onChange={(e) => setVoiceId(e.target.value)}
|
||||
placeholder="Enter a custom voice ID"
|
||||
/>
|
||||
<Select
|
||||
placeholder="Or select one from these example voices"
|
||||
size="md"
|
||||
onChange={(e) => {
|
||||
setVoiceId(e.target.value);
|
||||
}}
|
||||
>
|
||||
{VOICES.map((voice) => (
|
||||
<SelectItem key={voice.voice_id} textValue={voice.voice_id}>
|
||||
{voice.name} | {voice.language} | {voice.gender}
|
||||
</SelectItem>
|
||||
))}
|
||||
</Select>
|
||||
</div>
|
||||
<Button
|
||||
size="md"
|
||||
onClick={startSession}
|
||||
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 w-full text-white"
|
||||
size="md"
|
||||
variant="shadow"
|
||||
onClick={startSession}
|
||||
>
|
||||
Start session
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<Spinner size="lg" color="default" />
|
||||
<Spinner color="default" size="lg" />
|
||||
)}
|
||||
</CardBody>
|
||||
<Divider />
|
||||
<CardFooter className="flex flex-col gap-3">
|
||||
<InteractiveAvatarTextInput
|
||||
label="Repeat"
|
||||
placeholder="Type something for the avatar to repeat"
|
||||
input={text}
|
||||
onSubmit={handleSpeak}
|
||||
setInput={setText}
|
||||
disabled={!stream}
|
||||
loading={isLoadingRepeat}
|
||||
/>
|
||||
<InteractiveAvatarTextInput
|
||||
label="Chat"
|
||||
placeholder="Chat with the avatar (uses ChatGPT)"
|
||||
input={input}
|
||||
onSubmit={() => {
|
||||
setIsLoadingChat(true);
|
||||
if (!input) {
|
||||
setDebug("Please enter text to send to ChatGPT");
|
||||
return;
|
||||
}
|
||||
handleSubmit();
|
||||
<CardFooter className="flex flex-col gap-3 relative">
|
||||
<Tabs
|
||||
aria-label="Options"
|
||||
selectedKey={chatMode}
|
||||
onSelectionChange={(v) => {
|
||||
handleChangeChatMode(v);
|
||||
}}
|
||||
setInput={setInput}
|
||||
loading={isLoadingChat}
|
||||
endContent={
|
||||
<Tooltip
|
||||
content={!recording ? "Start recording" : "Stop recording"}
|
||||
>
|
||||
<Tab key="text_mode" title="Text mode" />
|
||||
<Tab key="voice_mode" title="Voice mode" />
|
||||
</Tabs>
|
||||
{chatMode === "text_mode" ? (
|
||||
<div className="w-full flex relative">
|
||||
<InteractiveAvatarTextInput
|
||||
disabled={!stream}
|
||||
input={text}
|
||||
label="Chat"
|
||||
loading={isLoadingRepeat}
|
||||
placeholder="Type something for the avatar to respond"
|
||||
setInput={setText}
|
||||
onSubmit={handleSpeak}
|
||||
/>
|
||||
{text && (
|
||||
<Chip className="absolute right-16 top-3">Listening</Chip>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="w-full text-center">
|
||||
<Button
|
||||
isDisabled={!isUserTalking}
|
||||
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 text-white"
|
||||
size="md"
|
||||
variant="shadow"
|
||||
>
|
||||
<Button
|
||||
onClick={!recording ? startRecording : stopRecording}
|
||||
isDisabled={!stream}
|
||||
isIconOnly
|
||||
className={clsx(
|
||||
"mr-4 text-white",
|
||||
!recording
|
||||
? "bg-gradient-to-tr from-indigo-500 to-indigo-300"
|
||||
: ""
|
||||
)}
|
||||
size="sm"
|
||||
variant="shadow"
|
||||
>
|
||||
{!recording ? (
|
||||
<Microphone size={20} />
|
||||
) : (
|
||||
<>
|
||||
<div className="absolute h-full w-full bg-gradient-to-tr from-indigo-500 to-indigo-300 animate-pulse -z-10"></div>
|
||||
<MicrophoneStage size={20} />
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</Tooltip>
|
||||
}
|
||||
disabled={!stream}
|
||||
/>
|
||||
{isUserTalking ? "Listening" : "Voice chat"}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</CardFooter>
|
||||
</Card>
|
||||
<p className="font-mono text-right">
|
||||
|
||||
@@ -3,15 +3,16 @@
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev --turbo",
|
||||
"dev": "node_modules/next/dist/bin/next dev",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "eslint . --ext .ts,.tsx -c .eslintrc.json --fix"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^0.0.34",
|
||||
"@heygen/streaming-avatar": "^1.0.11",
|
||||
"@heygen/streaming-avatar": "^2.0.0-beta.1",
|
||||
"@nextui-org/button": "2.0.34",
|
||||
"@nextui-org/chip": "^2.0.32",
|
||||
"@nextui-org/code": "2.0.29",
|
||||
"@nextui-org/input": "2.2.2",
|
||||
"@nextui-org/kbd": "2.0.30",
|
||||
@@ -28,6 +29,7 @@
|
||||
"@react-aria/visually-hidden": "3.8.12",
|
||||
"@uiw/codemirror-extensions-langs": "^4.22.1",
|
||||
"@uiw/react-codemirror": "^4.22.1",
|
||||
"ahooks": "^3.8.1",
|
||||
"ai": "^3.2.15",
|
||||
"clsx": "2.1.1",
|
||||
"framer-motion": "~11.1.1",
|
||||
|
||||
Reference in New Issue
Block a user