Compare commits

...

17 Commits

Author SHA1 Message Date
2d72e83c85 Update .env 2025-05-07 15:43:57 +00:00
morozovHeyGen
a5c5a7afde Merge pull request #62 from HeyGen-Official/update-demo-and-sdk-version
Update demo
2025-05-06 21:02:44 +03:00
Vlad Morozov
66990fcff5 fix styles 2025-04-22 21:28:41 +03:00
Vlad Morozov
e8688bac2b update deps 2025-04-22 21:08:28 +03:00
Vlad Morozov
a3cdf46210 update demo 2025-04-22 19:38:59 +03:00
eddy-heygen
119cab37ea Merge pull request #60 from HeyGen-Official/update-avatar-sdk
update streaming avatar sdk to 2.0.10, optional api url env
2025-03-17 14:52:51 -07:00
Eddy Kim
8a25c1d520 update streaming avatar sdk to 2.0.10, add optional api base url environment variable for testing 2025-03-17 14:34:07 -07:00
Joby
e0150f55b3 feat: change demo avatars (#56) 2025-02-24 19:19:15 -08:00
Joby
e259bdfe23 feat: update 2.0.9 (#51) 2025-01-17 17:57:29 -08:00
Joby
d012ef3e3c fix: remove unused package (#49) 2024-12-23 11:12:14 -08:00
Joby
e2763a2cb4 feat: add package lock (#46) 2024-12-19 11:42:12 -08:00
eddy-heygen
ba8fbf7be4 Merge pull request #43 from HeyGen-Official/deprecate-openai-example
remove deprecated openai example
2024-12-03 14:39:57 -08:00
Eddy Kim
bcaea9916d remove deprecated openai example 2024-12-03 14:30:05 -08:00
Joby
0fa4f0385e fix: incorrect home page url in header banner (#40) 2024-11-19 10:23:59 -08:00
Joby
6fca8b4d42 feat: upgrade the sdk to v2.0.8 (#39) 2024-11-17 22:38:11 -08:00
Joby
431281d47c feat: task mode (#30) 2024-10-22 16:58:38 -07:00
Joby
274a307e83 chore: update sdk version (#26) 2024-09-30 18:24:59 -07:00
41 changed files with 5928 additions and 875 deletions

5
.env
View File

@@ -1,3 +1,2 @@
HEYGEN_API_KEY=your Heygen API key
OPENAI_API_KEY=your OpenAI API key
NEXT_PUBLIC_OPENAI_API_KEY=your OpenAI API key
HEYGEN_API_KEY="Nzc0ODg1OTQ5ODU1NDRhNDg5OWVjMzc3MGIxNDVhNzItMTc0MzYxNDU5NA=="
NEXT_PUBLIC_BASE_API_URL=https://api.heygen.com

View File

@@ -1,20 +0,0 @@
.now/*
*.css
.changeset
dist
esm/*
public/*
tests/*
scripts/*
*.config.js
.DS_Store
node_modules
coverage
.next
build
!.commitlintrc.cjs
!.lintstagedrc.cjs
!jest.config.js
!plopfile.js
!react-shim.js
!tsup.config.ts

View File

@@ -1,92 +0,0 @@
{
"$schema": "https://json.schemastore.org/eslintrc.json",
"env": {
"browser": false,
"es2021": true,
"node": true
},
"extends": [
"plugin:react/recommended",
"plugin:prettier/recommended",
"plugin:react-hooks/recommended",
"plugin:jsx-a11y/recommended"
],
"plugins": ["react", "unused-imports", "import", "@typescript-eslint", "jsx-a11y", "prettier"],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaFeatures": {
"jsx": true
},
"ecmaVersion": 12,
"sourceType": "module"
},
"settings": {
"react": {
"version": "detect"
}
},
"rules": {
"no-console": "warn",
"react/prop-types": "off",
"react/jsx-uses-react": "off",
"react/react-in-jsx-scope": "off",
"react-hooks/exhaustive-deps": "off",
"jsx-a11y/click-events-have-key-events": "warn",
"jsx-a11y/interactive-supports-focus": "warn",
"prettier/prettier": "warn",
"no-unused-vars": "off",
"unused-imports/no-unused-vars": "off",
"unused-imports/no-unused-imports": "warn",
"@typescript-eslint/no-unused-vars": [
"warn",
{
"args": "after-used",
"ignoreRestSiblings": false,
"argsIgnorePattern": "^_.*?$"
}
],
"import/order": [
"warn",
{
"groups": [
"type",
"builtin",
"object",
"external",
"internal",
"parent",
"sibling",
"index"
],
"pathGroups": [
{
"pattern": "~/**",
"group": "external",
"position": "after"
}
],
"newlines-between": "always"
}
],
"react/self-closing-comp": "warn",
"react/jsx-sort-props": [
"warn",
{
"callbacksLast": true,
"shorthandFirst": true,
"noSortAlphabetically": false,
"reservedFirst": true
}
],
"padding-line-between-statements": [
"warn",
{"blankLine": "always", "prev": "*", "next": "return"},
{"blankLine": "always", "prev": ["const", "let", "var"], "next": "*"},
{
"blankLine": "any",
"prev": ["const", "let", "var"],
"next": ["const", "let", "var"]
}
]
}
}

2
.npmrc
View File

@@ -1 +1 @@
package-lock=false

View File

@@ -41,24 +41,6 @@ After you see Monica appear on the screen, you can enter text into the input lab
If you want to see a different Avatar or try a different voice, you can close the session and enter the IDs and then 'start' the session again. Please see below for information on where to retrieve different Avatar and voice IDs that you can use.
### Connecting to OpenAI
A common use case for a Interactive Avatar is to use it as the 'face' of an LLM that users can interact with. In this demo we have included functionality to showcase this by both accepting user input via voice (using OpenAI's Whisper library) and also sending that input to an OpenAI LLM model (using their Chat Completions endpoint).
Both of these features of this demo require an OpenAI API Key. If you do not have a paid OpenAI account, you can learn more on their website: [https://openai.com/index/openai-api/]
Without an OpenAI API Key, this functionality will not work, and the Interactive Avatar will only be able to repeat text input that you provide, and not demonstrate being the 'face' of an LLM. Regardless, this demo is meant to demonstrate what kinds of apps and experiences you can build with our Interactive Avatar SDK, so you can code your own connection to a different LLM if you so choose.
To add your Open AI API Key, fill copy it to the `OPENAI_API_KEY` and `NEXT_PUBLIC_OPENAI_API_KEY` variables in the `.env` file.
### How does the integration with OpenAI / ChatGPT work?
In this demo, we are calling the Chat Completions API from OpenAI in order to come up with some response to user input. You can see the relevant code in components/InteractiveAvatar.tsx.
In the initialMessages parameter, you can replace the content of the 'system' message with whatever 'knowledge base' or context that you would like the GPT-4o model to reply to the user's input with.
You can explore this API and the different parameters and models available here: [https://platform.openai.com/docs/guides/text-generation/chat-completions-api]
### Which Avatars can I use with this project?
By default, there are several Public Avatars that can be used in Interactive Avatar. (AKA Interactive Avatars.) You can find the Avatar IDs for these Public Avatars by navigating to [app.heygen.com/interactive-avatar](https://app.heygen.com/interactive-avatar) and clicking 'Select Avatar' and copying the avatar id.

View File

@@ -1,16 +0,0 @@
import { openai } from "@ai-sdk/openai";
import { streamText } from "ai";
// Allow streaming responses up to 30 seconds
export const maxDuration = 30;
export async function POST(req: Request) {
const { messages } = await req.json();
const result = await streamText({
model: openai("gpt-4-turbo"),
messages,
});
return result.toAIStreamResponse();
}

View File

@@ -5,16 +5,17 @@ export async function POST() {
if (!HEYGEN_API_KEY) {
throw new Error("API key is missing from .env");
}
const baseApiUrl = process.env.NEXT_PUBLIC_BASE_API_URL;
const res = await fetch(
"https://api.heygen.com/v1/streaming.create_token",
{
method: "POST",
headers: {
"x-api-key": HEYGEN_API_KEY,
},
const res = await fetch(`${baseApiUrl}/v1/streaming.create_token`, {
method: "POST",
headers: {
"x-api-key": HEYGEN_API_KEY,
},
);
});
console.log("Response:", res);
const data = await res.json();
return new Response(data.data.token, {

View File

@@ -11,7 +11,7 @@ export default function Error({
}) {
useEffect(() => {
// Log the error to an error reporting service
/* eslint-disable no-console */
console.error(error);
}, [error]);

View File

@@ -1,10 +1,7 @@
import "@/styles/globals.css";
import clsx from "clsx";
import { Metadata, Viewport } from "next";
import { Providers } from "./providers";
import { Metadata } from "next";
import { Fira_Code as FontMono, Inter as FontSans } from "next/font/google";
import NavBar from "@/components/NavBar";
const fontSans = FontSans({
@@ -27,13 +24,6 @@ export const metadata: Metadata = {
},
};
export const viewport: Viewport = {
themeColor: [
{ media: "(prefers-color-scheme: light)", color: "white" },
{ media: "(prefers-color-scheme: dark)", color: "black" },
],
};
export default function RootLayout({
children,
}: {
@@ -42,17 +32,15 @@ export default function RootLayout({
return (
<html
suppressHydrationWarning
lang="en"
className={`${fontSans.variable} ${fontMono.variable} font-sans`}
lang="en"
>
<head />
<body className={clsx("min-h-screen bg-background antialiased")}>
<Providers themeProps={{ attribute: "class", defaultTheme: "dark" }}>
<main className="relative flex flex-col h-screen w-screen">
<NavBar />
{children}
</main>
</Providers>
<body className="min-h-screen bg-black text-white">
<main className="relative flex flex-col gap-6 h-screen w-screen">
<NavBar />
{children}
</main>
</body>
</html>
);

View File

@@ -1,53 +1,53 @@
export const AVATARS = [
{
avatar_id: "Eric_public_pro2_20230608",
name: "Edward in Blue Shirt",
avatar_id: "Ann_Therapist_public",
name: "Ann Therapist",
},
{
avatar_id: "Tyler-incasualsuit-20220721",
name: "Tyler in Casual Suit",
avatar_id: "Shawn_Therapist_public",
name: "Shawn Therapist",
},
{
avatar_id: "Anna_public_3_20240108",
name: "Anna in Brown T-shirt",
avatar_id: "Bryan_FitnessCoach_public",
name: "Bryan Fitness Coach",
},
{
avatar_id: "Susan_public_2_20240328",
name: "Susan in Black Shirt",
avatar_id: "Dexter_Doctor_Standing2_public",
name: "Dexter Doctor Standing",
},
{
avatar_id: "josh_lite3_20230714",
name: "Joshua Heygen CEO",
avatar_id: "Elenora_IT_Sitting_public",
name: "Elenora Tech Expert",
},
];
export const STT_LANGUAGE_LIST = [
{ label: 'Bulgarian', value: 'bg', key: 'bg' },
{ label: 'Chinese', value: 'zh', key: 'zh' },
{ label: 'Czech', value: 'cs', key: 'cs' },
{ label: 'Danish', value: 'da', key: 'da' },
{ label: 'Dutch', value: 'nl', key: 'nl' },
{ label: 'English', value: 'en', key: 'en' },
{ label: 'Finnish', value: 'fi', key: 'fi' },
{ label: 'French', value: 'fr', key: 'fr' },
{ label: 'German', value: 'de', key: 'de' },
{ label: 'Greek', value: 'el', key: 'el' },
{ label: 'Hindi', value: 'hi', key: 'hi' },
{ label: 'Hungarian', value: 'hu', key: 'hu' },
{ label: 'Indonesian', value: 'id', key: 'id' },
{ label: 'Italian', value: 'it', key: 'it' },
{ label: 'Japanese', value: 'ja', key: 'ja' },
{ label: 'Korean', value: 'ko', key: 'ko' },
{ label: 'Malay', value: 'ms', key: 'ms' },
{ label: 'Norwegian', value: 'no', key: 'no' },
{ label: 'Polish', value: 'pl', key: 'pl' },
{ label: 'Portuguese', value: 'pt', key: 'pt' },
{ label: 'Romanian', value: 'ro', key: 'ro' },
{ label: 'Russian', value: 'ru', key: 'ru' },
{ label: 'Slovak', value: 'sk', key: 'sk' },
{ label: 'Spanish', value: 'es', key: 'es' },
{ label: 'Swedish', value: 'sv', key: 'sv' },
{ label: 'Turkish', value: 'tr', key: 'tr' },
{ label: 'Ukrainian', value: 'uk', key: 'uk' },
{ label: 'Vietnamese', value: 'vi', key: 'vi' },
{ label: "Bulgarian", value: "bg", key: "bg" },
{ label: "Chinese", value: "zh", key: "zh" },
{ label: "Czech", value: "cs", key: "cs" },
{ label: "Danish", value: "da", key: "da" },
{ label: "Dutch", value: "nl", key: "nl" },
{ label: "English", value: "en", key: "en" },
{ label: "Finnish", value: "fi", key: "fi" },
{ label: "French", value: "fr", key: "fr" },
{ label: "German", value: "de", key: "de" },
{ label: "Greek", value: "el", key: "el" },
{ label: "Hindi", value: "hi", key: "hi" },
{ label: "Hungarian", value: "hu", key: "hu" },
{ label: "Indonesian", value: "id", key: "id" },
{ label: "Italian", value: "it", key: "it" },
{ label: "Japanese", value: "ja", key: "ja" },
{ label: "Korean", value: "ko", key: "ko" },
{ label: "Malay", value: "ms", key: "ms" },
{ label: "Norwegian", value: "no", key: "no" },
{ label: "Polish", value: "pl", key: "pl" },
{ label: "Portuguese", value: "pt", key: "pt" },
{ label: "Romanian", value: "ro", key: "ro" },
{ label: "Russian", value: "ru", key: "ru" },
{ label: "Slovak", value: "sk", key: "sk" },
{ label: "Spanish", value: "es", key: "es" },
{ label: "Swedish", value: "sv", key: "sv" },
{ label: "Turkish", value: "tr", key: "tr" },
{ label: "Ukrainian", value: "uk", key: "uk" },
{ label: "Vietnamese", value: "vi", key: "vi" },
];

View File

@@ -2,7 +2,6 @@
import InteractiveAvatar from "@/components/InteractiveAvatar";
export default function App() {
return (
<div className="w-screen h-screen flex flex-col">
<div className="w-[900px] flex flex-col items-start justify-start gap-5 mx-auto pt-4 pb-20">

View File

@@ -1,22 +0,0 @@
"use client";
import * as React from "react";
import { NextUIProvider } from "@nextui-org/system";
import { useRouter } from "next/navigation";
import { ThemeProvider as NextThemesProvider } from "next-themes";
import { ThemeProviderProps } from "next-themes/dist/types";
export interface ProvidersProps {
children: React.ReactNode;
themeProps?: ThemeProviderProps;
}
export function Providers({ children, themeProps }: ProvidersProps) {
const router = useRouter();
return (
<NextUIProvider navigate={router.push}>
<NextThemesProvider {...themeProps}>{children}</NextThemesProvider>
</NextUIProvider>
);
}

View File

@@ -0,0 +1,14 @@
interface FieldProps {
label: string;
children: React.ReactNode;
tooltip?: string;
}
export const Field = (props: FieldProps) => {
return (
<div className="flex flex-col gap-1">
<label className="text-zinc-400 text-sm">{props.label}</label>
{props.children}
</div>
);
};

View File

@@ -0,0 +1,192 @@
import React, { useMemo, useState } from "react";
import {
AvatarQuality,
ElevenLabsModel,
STTProvider,
VoiceEmotion,
StartAvatarRequest,
VoiceChatTransport,
} from "@heygen/streaming-avatar";
import { Input } from "../Input";
import { Select } from "../Select";
import { Field } from "./Field";
import { AVATARS, STT_LANGUAGE_LIST } from "@/app/lib/constants";
interface AvatarConfigProps {
onConfigChange: (config: StartAvatarRequest) => void;
config: StartAvatarRequest;
}
export const AvatarConfig: React.FC<AvatarConfigProps> = ({
onConfigChange,
config,
}) => {
const onChange = <T extends keyof StartAvatarRequest>(
key: T,
value: StartAvatarRequest[T],
) => {
onConfigChange({ ...config, [key]: value });
};
const [showMore, setShowMore] = useState<boolean>(false);
const selectedAvatar = useMemo(() => {
const avatar = AVATARS.find(
(avatar) => avatar.avatar_id === config.avatarName,
);
if (!avatar) {
return {
isCustom: true,
name: "Custom Avatar ID",
avatarId: null,
};
} else {
return {
isCustom: false,
name: avatar.name,
avatarId: avatar.avatar_id,
};
}
}, [config.avatarName]);
return (
<div className="relative flex flex-col gap-4 w-[550px] py-8 max-h-full overflow-y-auto px-4">
<Field label="Custom Knowledge Base ID">
<Input
placeholder="Enter custom knowledge base ID"
value={config.knowledgeId}
onChange={(value) => onChange("knowledgeId", value)}
/>
</Field>
<Field label="Avatar ID">
<Select
isSelected={(option) =>
typeof option === "string"
? !!selectedAvatar?.isCustom
: option.avatar_id === selectedAvatar?.avatarId
}
options={[...AVATARS, "CUSTOM"]}
placeholder="Select Avatar"
renderOption={(option) => {
return typeof option === "string"
? "Custom Avatar ID"
: option.name;
}}
value={
selectedAvatar?.isCustom ? "Custom Avatar ID" : selectedAvatar?.name
}
onSelect={(option) => {
if (typeof option === "string") {
onChange("avatarName", "");
} else {
onChange("avatarName", option.avatar_id);
}
}}
/>
</Field>
{selectedAvatar?.isCustom && (
<Field label="Custom Avatar ID">
<Input
placeholder="Enter custom avatar ID"
value={config.avatarName}
onChange={(value) => onChange("avatarName", value)}
/>
</Field>
)}
<Field label="Language">
<Select
isSelected={(option) => option.value === config.language}
options={STT_LANGUAGE_LIST}
renderOption={(option) => option.label}
value={
STT_LANGUAGE_LIST.find((option) => option.value === config.language)
?.label
}
onSelect={(option) => onChange("language", option.value)}
/>
</Field>
<Field label="Avatar Quality">
<Select
isSelected={(option) => option === config.quality}
options={Object.values(AvatarQuality)}
renderOption={(option) => option}
value={config.quality}
onSelect={(option) => onChange("quality", option)}
/>
</Field>
<Field label="Voice Chat Transport">
<Select
isSelected={(option) => option === config.voiceChatTransport}
options={Object.values(VoiceChatTransport)}
renderOption={(option) => option}
value={config.voiceChatTransport}
onSelect={(option) => onChange("voiceChatTransport", option)}
/>
</Field>
{showMore && (
<>
<h1 className="text-zinc-100 w-full text-center mt-5">
Voice Settings
</h1>
<Field label="Custom Voice ID">
<Input
placeholder="Enter custom voice ID"
value={config.voice?.voiceId}
onChange={(value) =>
onChange("voice", { ...config.voice, voiceId: value })
}
/>
</Field>
<Field label="Emotion">
<Select
isSelected={(option) => option === config.voice?.emotion}
options={Object.values(VoiceEmotion)}
renderOption={(option) => option}
value={config.voice?.emotion}
onSelect={(option) =>
onChange("voice", { ...config.voice, emotion: option })
}
/>
</Field>
<Field label="ElevenLabs Model">
<Select
isSelected={(option) => option === config.voice?.model}
options={Object.values(ElevenLabsModel)}
renderOption={(option) => option}
value={config.voice?.model}
onSelect={(option) =>
onChange("voice", { ...config.voice, model: option })
}
/>
</Field>
<h1 className="text-zinc-100 w-full text-center mt-5">
STT Settings
</h1>
<Field label="Provider">
<Select
isSelected={(option) => option === config.sttSettings?.provider}
options={Object.values(STTProvider)}
renderOption={(option) => option}
value={config.sttSettings?.provider}
onSelect={(option) =>
onChange("sttSettings", {
...config.sttSettings,
provider: option,
})
}
/>
</Field>
</>
)}
<button
className="text-zinc-400 text-sm cursor-pointer w-full text-center bg-transparent"
onClick={() => setShowMore(!showMore)}
>
{showMore ? "Show less" : "Show more..."}
</button>
</div>
);
};

View File

@@ -0,0 +1,41 @@
import React from "react";
import { useVoiceChat } from "../logic/useVoiceChat";
import { Button } from "../Button";
import { LoadingIcon, MicIcon, MicOffIcon } from "../Icons";
import { useConversationState } from "../logic/useConversationState";
export const AudioInput: React.FC = () => {
const { muteInputAudio, unmuteInputAudio, isMuted, isVoiceChatLoading } =
useVoiceChat();
const { isUserTalking } = useConversationState();
const handleMuteClick = () => {
if (isMuted) {
unmuteInputAudio();
} else {
muteInputAudio();
}
};
return (
<div>
<Button
className={`!p-2 relative`}
disabled={isVoiceChatLoading}
onClick={handleMuteClick}
>
<div
className={`absolute left-0 top-0 rounded-lg border-2 border-[#7559FF] w-full h-full ${isUserTalking ? "animate-ping" : ""}`}
/>
{isVoiceChatLoading ? (
<LoadingIcon className="animate-spin" size={20} />
) : isMuted ? (
<MicOffIcon size={20} />
) : (
<MicIcon size={20} />
)}
</Button>
</div>
);
};

View File

@@ -0,0 +1,60 @@
import { ToggleGroup, ToggleGroupItem } from "@radix-ui/react-toggle-group";
import React from "react";
import { useVoiceChat } from "../logic/useVoiceChat";
import { Button } from "../Button";
import { useInterrupt } from "../logic/useInterrupt";
import { AudioInput } from "./AudioInput";
import { TextInput } from "./TextInput";
export const AvatarControls: React.FC = () => {
const {
isVoiceChatLoading,
isVoiceChatActive,
startVoiceChat,
stopVoiceChat,
} = useVoiceChat();
const { interrupt } = useInterrupt();
return (
<div className="flex flex-col gap-3 relative w-full items-center">
<ToggleGroup
className={`bg-zinc-700 rounded-lg p-1 ${isVoiceChatLoading ? "opacity-50" : ""}`}
disabled={isVoiceChatLoading}
type="single"
value={isVoiceChatActive || isVoiceChatLoading ? "voice" : "text"}
onValueChange={(value) => {
if (value === "voice" && !isVoiceChatActive && !isVoiceChatLoading) {
startVoiceChat();
} else if (
value === "text" &&
isVoiceChatActive &&
!isVoiceChatLoading
) {
stopVoiceChat();
}
}}
>
<ToggleGroupItem
className="data-[state=on]:bg-zinc-800 rounded-lg p-2 text-sm w-[90px] text-center"
value="voice"
>
Voice Chat
</ToggleGroupItem>
<ToggleGroupItem
className="data-[state=on]:bg-zinc-800 rounded-lg p-2 text-sm w-[90px] text-center"
value="text"
>
Text Chat
</ToggleGroupItem>
</ToggleGroup>
{isVoiceChatActive || isVoiceChatLoading ? <AudioInput /> : <TextInput />}
<div className="absolute top-[-70px] right-3">
<Button className="!bg-zinc-700 !text-white" onClick={interrupt}>
Interrupt
</Button>
</div>
</div>
);
};

View File

@@ -0,0 +1,51 @@
import React, { forwardRef } from "react";
import { ConnectionQuality } from "@heygen/streaming-avatar";
import { useConnectionQuality } from "../logic/useConnectionQuality";
import { useStreamingAvatarSession } from "../logic/useStreamingAvatarSession";
import { StreamingAvatarSessionState } from "../logic";
import { CloseIcon } from "../Icons";
import { Button } from "../Button";
export const AvatarVideo = forwardRef<HTMLVideoElement>(({}, ref) => {
const { sessionState, stopAvatar } = useStreamingAvatarSession();
const { connectionQuality } = useConnectionQuality();
const isLoaded = sessionState === StreamingAvatarSessionState.CONNECTED;
return (
<>
{connectionQuality !== ConnectionQuality.UNKNOWN && (
<div className="absolute top-3 left-3 bg-black text-white rounded-lg px-3 py-2">
Connection Quality: {connectionQuality}
</div>
)}
{isLoaded && (
<Button
className="absolute top-3 right-3 !p-2 bg-zinc-700 bg-opacity-50 z-10"
onClick={stopAvatar}
>
<CloseIcon />
</Button>
)}
<video
ref={ref}
autoPlay
playsInline
style={{
width: "100%",
height: "100%",
objectFit: "contain",
}}
>
<track kind="captions" />
</video>
{!isLoaded && (
<div className="w-full h-full flex items-center justify-center absolute top-0 left-0">
Loading...
</div>
)}
</>
);
});
AvatarVideo.displayName = "AvatarVideo";

View File

@@ -0,0 +1,39 @@
import React, { useEffect, useRef } from "react";
import { useMessageHistory, MessageSender } from "../logic";
export const MessageHistory: React.FC = () => {
const { messages } = useMessageHistory();
const containerRef = useRef<HTMLDivElement>(null);
useEffect(() => {
const container = containerRef.current;
if (!container || messages.length === 0) return;
container.scrollTop = container.scrollHeight;
}, [messages]);
return (
<div
ref={containerRef}
className="w-[600px] overflow-y-auto flex flex-col gap-2 px-2 py-2 text-white self-center max-h-[150px]"
>
{messages.map((message) => (
<div
key={message.id}
className={`flex flex-col gap-1 max-w-[350px] ${
message.sender === MessageSender.CLIENT
? "self-end items-end"
: "self-start items-start"
}`}
>
<p className="text-xs text-zinc-400">
{message.sender === MessageSender.AVATAR ? "Avatar" : "You"}
</p>
<p className="text-sm">{message.content}</p>
</div>
))}
</div>
);
};

View File

@@ -0,0 +1,93 @@
import { TaskType, TaskMode } from "@heygen/streaming-avatar";
import React, { useCallback, useEffect, useState } from "react";
import { usePrevious } from "ahooks";
import { Select } from "../Select";
import { Button } from "../Button";
import { SendIcon } from "../Icons";
import { useTextChat } from "../logic/useTextChat";
import { Input } from "../Input";
import { useConversationState } from "../logic/useConversationState";
export const TextInput: React.FC = () => {
const { sendMessage, sendMessageSync, repeatMessage, repeatMessageSync } =
useTextChat();
const { startListening, stopListening } = useConversationState();
const [taskType, setTaskType] = useState<TaskType>(TaskType.TALK);
const [taskMode, setTaskMode] = useState<TaskMode>(TaskMode.ASYNC);
const [message, setMessage] = useState("");
const handleSend = useCallback(() => {
if (message.trim() === "") {
return;
}
if (taskType === TaskType.TALK) {
taskMode === TaskMode.SYNC
? sendMessageSync(message)
: sendMessage(message);
} else {
taskMode === TaskMode.SYNC
? repeatMessageSync(message)
: repeatMessage(message);
}
setMessage("");
}, [
taskType,
taskMode,
message,
sendMessage,
sendMessageSync,
repeatMessage,
repeatMessageSync,
]);
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (event.key === "Enter") {
handleSend();
}
};
window.addEventListener("keydown", handleKeyDown);
return () => window.removeEventListener("keydown", handleKeyDown);
}, [handleSend]);
const previousText = usePrevious(message);
useEffect(() => {
if (!previousText && message) {
startListening();
} else if (previousText && !message) {
stopListening();
}
}, [message, previousText, startListening, stopListening]);
return (
<div className="flex flex-row gap-2 items-end w-full">
<Select
isSelected={(option) => option === taskType}
options={Object.values(TaskType)}
renderOption={(option) => option.toUpperCase()}
value={taskType.toUpperCase()}
onSelect={setTaskType}
/>
<Select
isSelected={(option) => option === taskMode}
options={Object.values(TaskMode)}
renderOption={(option) => option.toUpperCase()}
value={taskMode.toUpperCase()}
onSelect={setTaskMode}
/>
<Input
className="min-w-[500px]"
placeholder={`Type something for the avatar to ${taskType === TaskType.REPEAT ? "repeat" : "respond"}...`}
value={message}
onChange={setMessage}
/>
<Button className="!p-2" onClick={handleSend}>
<SendIcon size={20} />
</Button>
</div>
);
};

15
components/Button.tsx Normal file
View File

@@ -0,0 +1,15 @@
import React from "react";
export const Button: React.FC<
React.ButtonHTMLAttributes<HTMLButtonElement>
> = ({ children, className, onClick, ...props }) => {
return (
<button
className={`bg-[#7559FF] text-white text-sm px-6 py-2 rounded-lg disabled:opacity-50 h-fit ${className}`}
onClick={props.disabled ? undefined : onClick}
{...props}
>
{children}
</button>
);
};

View File

@@ -1,5 +1,5 @@
export function HeyGenLogo() {
return <img src="/heygen-logo.png" className="h-8" alt="HeyGen Logo" />;
return <img alt="HeyGen Logo" className="h-8" src="/heygen-logo.png" />;
}
type IconSvgProps = {
@@ -32,7 +32,7 @@ export function GithubIcon({
);
}
export function MoonFilledIcon({
export function ChevronDownIcon({
size = 24,
width,
height,
@@ -40,23 +40,109 @@ export function MoonFilledIcon({
}: IconSvgProps) {
return (
<svg
aria-hidden="true"
focusable="false"
fill="none"
height={size || height}
role="presentation"
viewBox="0 0 24 24"
viewBox="0 0 20 20"
width={size || width}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<path
d="M21.53 15.93c-.16-.27-.61-.69-1.73-.49a8.46 8.46 0 01-1.88.13 8.409 8.409 0 01-5.91-2.82 8.068 8.068 0 01-1.44-8.66c.44-1.01.13-1.54-.09-1.76s-.77-.55-1.83-.11a10.318 10.318 0 00-6.32 10.21 10.475 10.475 0 007.04 8.99 10 10 0 002.89.55c.16.01.32.02.48.02a10.5 10.5 0 008.47-4.27c.67-.93.49-1.519.32-1.79z"
d="M4.88231 7.6185C4.7173 7.78351 4.6348 7.86601 4.60389 7.96115C4.5767 8.04483 4.5767 8.13498 4.60389 8.21866C4.6348 8.3138 4.7173 8.3963 4.88231 8.56131L8.82165 12.5007C9.47253 13.1515 10.5278 13.1515 11.1787 12.5007L15.118 8.56131C15.283 8.39631 15.3655 8.3138 15.3964 8.21866C15.4236 8.13498 15.4236 8.04484 15.3964 7.96115C15.3655 7.86601 15.283 7.78351 15.118 7.6185L14.8823 7.3828C14.7173 7.21779 14.6348 7.13529 14.5397 7.10438C14.456 7.07719 14.3658 7.07719 14.2822 7.10438C14.187 7.13529 14.1045 7.21779 13.9395 7.3828L10.4716 10.8507C10.3066 11.0157 10.2241 11.0982 10.1289 11.1292C10.0452 11.1563 9.95509 11.1563 9.87141 11.1292C9.77627 11.0982 9.69377 11.0157 9.52876 10.8507L6.06082 7.3828C5.89582 7.21779 5.81331 7.13529 5.71818 7.10438C5.63449 7.07719 5.54435 7.07719 5.46066 7.10438C5.36552 7.13529 5.28302 7.21779 5.11801 7.3828L4.88231 7.6185Z"
fill="currentColor"
/>
</svg>
);
}
export function SunFilledIcon({
export function SendIcon({ size = 24, width, height, ...props }: IconSvgProps) {
return (
<svg
fill="none"
height={size || height}
viewBox="0 0 16 16"
width={size || width}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<path
clipRule="evenodd"
d="M14.686 7.41437C14.8667 7.78396 14.8667 8.21629 14.686 8.58588C14.5413 8.8817 14.2792 9.05684 14.08 9.17191C13.8742 9.29079 13.6015 9.41707 13.2919 9.56042L3.52078 14.0855C3.29008 14.1924 3.07741 14.2909 2.89693 14.3553C2.70994 14.422 2.46552 14.4879 2.19444 14.442C1.8383 14.3817 1.52185 14.1796 1.3175 13.8817C1.16195 13.655 1.11903 13.4055 1.10097 13.2078C1.08355 13.017 1.08357 12.7826 1.08359 12.5284L1.08359 10.1207C1.08359 10.1021 1.08351 10.0829 1.08343 10.0633C1.08255 9.85606 1.08146 9.59598 1.17301 9.35874C1.252 9.15409 1.38025 8.97208 1.54641 8.82886C1.73903 8.66284 1.98433 8.57639 2.17979 8.5075C2.19829 8.50098 2.21635 8.49461 2.23387 8.48835L3.3612 8.08569L2.23387 7.68302C2.21635 7.67676 2.19829 7.67039 2.17979 7.66387C1.98433 7.59498 1.73903 7.50853 1.54641 7.34251C1.38025 7.19929 1.252 7.01728 1.17301 6.81263C1.08146 6.57539 1.08255 6.3153 1.08343 6.10806C1.08351 6.08844 1.08359 6.0693 1.08359 6.05069L1.08359 3.47182C1.08357 3.21759 1.08355 2.98324 1.10097 2.79242C1.11903 2.59472 1.16195 2.34523 1.3175 2.11853C1.52185 1.82069 1.8383 1.61851 2.19444 1.55824C2.46552 1.51236 2.70994 1.57825 2.89693 1.64495C3.07741 1.70933 3.29007 1.80784 3.52076 1.9147L13.2919 6.43983C13.6015 6.58318 13.8742 6.70946 14.08 6.82834C14.2792 6.94341 14.5413 7.11855 14.686 7.41437ZM13.413 7.98287C13.266 7.89792 13.0493 7.79688 12.7045 7.63716L2.98502 3.13597C2.7214 3.01388 2.56493 2.94215 2.44896 2.90078C2.44246 2.89846 2.43638 2.89635 2.4307 2.89443C2.43005 2.90039 2.42941 2.9068 2.42878 2.91367C2.41759 3.03629 2.41693 3.20842 2.41693 3.49893L2.41693 6.05069C2.41693 6.19492 2.41728 6.27013 2.42098 6.32446C2.4211 6.32621 2.42121 6.32787 2.42133 6.32946C2.42279 6.3301 2.42431 6.33077 2.42591 6.33147C2.47584 6.35323 2.54655 6.37886 2.68238 6.42738L5.56736 7.45787C5.83268 7.55263 6.00978 7.80395 6.00978 8.08569C6.00978 8.36742 5.83268 8.61874 5.56736 8.7135L2.68238 9.74399C2.54655 9.79251 2.47584 9.81814 2.42591 9.8399C2.42431 9.8406 2.42279 9.84127 2.42133 9.84191C2.42121 9.8435 2.4211 9.84516 2.42098 9.84691C2.41728 9.90124 2.41693 9.97645 2.41693 10.1207L2.41693 12.5013C2.41693 12.7918 2.41759 12.964 2.42878 13.0866C2.42941 13.0935 2.43005 13.0999 2.4307 13.1058C2.43638 13.1039 2.44246 13.1018 2.44896 13.0995C2.56493 13.0581 2.7214 12.9864 2.98502 12.8643L12.7045 8.36309C13.0493 8.20337 13.266 8.10233 13.413 8.01737C13.4236 8.01125 13.4333 8.0055 13.4422 8.00012C13.4333 7.99474 13.4236 7.98899 13.413 7.98287Z"
fill="currentColor"
fillRule="evenodd"
/>
</svg>
);
}
export function MicIcon({ size = 24, width, height, ...props }: IconSvgProps) {
return (
<svg
fill="none"
height={size || height}
viewBox="0 0 20 20"
width={size || width}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<g clipPath="url(#clip0_9098_19437)">
<g filter="url(#filter0_d_9098_19437)">
<path
clipRule="evenodd"
d="M5.83341 5.00065C5.83341 2.69946 7.6989 0.833984 10.0001 0.833984C12.3013 0.833984 14.1667 2.69946 14.1667 5.00065L14.1667 8.33398C14.1667 10.6352 12.3013 12.5007 10.0001 12.5007C7.6989 12.5007 5.83341 10.6352 5.83341 8.33398L5.83341 5.00065ZM12.5001 5.00065L12.5001 8.33398C12.5001 9.7147 11.3808 10.834 10.0001 10.834C8.61937 10.834 7.50008 9.7147 7.50008 8.33398V5.00065C7.50008 3.61994 8.61937 2.50065 10.0001 2.50065C11.3808 2.50065 12.5001 3.61994 12.5001 5.00065Z"
fill="#232833"
fillRule="evenodd"
/>
<path
d="M5.66675 17.5007H9.16675V15.3688C5.64744 14.9564 2.91675 11.9641 2.91675 8.33398V8.16732C2.91675 7.93396 2.91675 7.81729 2.96216 7.72816C3.00211 7.64975 3.06585 7.58601 3.14425 7.54607C3.23338 7.50065 3.35006 7.50065 3.58341 7.50065H3.91675C4.1501 7.50065 4.26678 7.50065 4.35591 7.54607C4.43431 7.58601 4.49805 7.64975 4.538 7.72816C4.58341 7.81729 4.58341 7.93396 4.58341 8.16732V8.33398C4.58341 11.3255 7.00854 13.7507 10.0001 13.7507C12.9916 13.7507 15.4167 11.3255 15.4167 8.33398V8.16732C15.4167 7.93396 15.4167 7.81729 15.4622 7.72816C15.5021 7.64975 15.5659 7.58601 15.6443 7.54607C15.7334 7.50065 15.8501 7.50065 16.0834 7.50065H16.4167C16.6501 7.50065 16.7668 7.50065 16.8559 7.54607C16.9343 7.58601 16.9981 7.64975 17.038 7.72816C17.0834 7.81729 17.0834 7.93396 17.0834 8.16732V8.33398C17.0834 11.9641 14.3527 14.9564 10.8334 15.3688V17.5007L14.3334 17.5007C14.5668 17.5007 14.6834 17.5007 14.7726 17.5461C14.851 17.586 14.9147 17.6498 14.9547 17.7282C15.0001 17.8173 15.0001 17.934 15.0001 18.1673V18.5007C15.0001 18.734 15.0001 18.8507 14.9547 18.9398C14.9147 19.0182 14.851 19.082 14.7726 19.1219C14.6834 19.1673 14.5668 19.1673 14.3334 19.1673L5.66675 19.1673C5.43339 19.1673 5.31672 19.1673 5.22759 19.1219C5.14918 19.082 5.08544 19.0182 5.0455 18.9398C5.00008 18.8507 5.00008 18.734 5.00008 18.5007V18.1673C5.00008 17.934 5.00008 17.8173 5.0455 17.7282C5.08544 17.6498 5.14918 17.586 5.22759 17.5461C5.31672 17.5007 5.43339 17.5007 5.66675 17.5007Z"
fill="#232833"
/>
</g>
</g>
<defs>
<filter
colorInterpolationFilters="sRGB"
filterUnits="userSpaceOnUse"
height="22.334"
id="filter0_d_9098_19437"
width="18.1667"
x="0.916748"
y="-0.166016"
>
<feFlood floodOpacity="0" result="BackgroundImageFix" />
<feColorMatrix
in="SourceAlpha"
result="hardAlpha"
type="matrix"
values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0"
/>
<feOffset dy="1" />
<feGaussianBlur stdDeviation="1" />
<feColorMatrix
type="matrix"
values="0 0 0 0 0.0627451 0 0 0 0 0.0941176 0 0 0 0 0.156863 0 0 0 0.05 0"
/>
<feBlend
in2="BackgroundImageFix"
mode="normal"
result="effect1_dropShadow_9098_19437"
/>
<feBlend
in="SourceGraphic"
in2="effect1_dropShadow_9098_19437"
mode="normal"
result="shape"
/>
</filter>
<clipPath id="clip0_9098_19437">
<rect fill="white" height="20" width="20" />
</clipPath>
</defs>
</svg>
);
}
export function MicOffIcon({
size = 24,
width,
height,
@@ -64,18 +150,84 @@ export function SunFilledIcon({
}: IconSvgProps) {
return (
<svg
aria-hidden="true"
focusable="false"
fill="none"
height={size || height}
role="presentation"
viewBox="0 0 24 24"
viewBox="0 0 48 48"
width={size || width}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<g fill="currentColor">
<path d="M19 12a7 7 0 11-7-7 7 7 0 017 7z" />
<path d="M12 22.96a.969.969 0 01-1-.96v-.08a1 1 0 012 0 1.038 1.038 0 01-1 1.04zm7.14-2.82a1.024 1.024 0 01-.71-.29l-.13-.13a1 1 0 011.41-1.41l.13.13a1 1 0 010 1.41.984.984 0 01-.7.29zm-14.28 0a1.024 1.024 0 01-.71-.29 1 1 0 010-1.41l.13-.13a1 1 0 011.41 1.41l-.13.13a1 1 0 01-.7.29zM22 13h-.08a1 1 0 010-2 1.038 1.038 0 011.04 1 .969.969 0 01-.96 1zM2.08 13H2a1 1 0 010-2 1.038 1.038 0 011.04 1 .969.969 0 01-.96 1zm16.93-7.01a1.024 1.024 0 01-.71-.29 1 1 0 010-1.41l.13-.13a1 1 0 011.41 1.41l-.13.13a.984.984 0 01-.7.29zm-14.02 0a1.024 1.024 0 01-.71-.29l-.13-.14a1 1 0 011.41-1.41l.13.13a1 1 0 010 1.41.97.97 0 01-.7.3zM12 3.04a.969.969 0 01-1-.96V2a1 1 0 012 0 1.038 1.038 0 01-1 1.04z" />
</g>
<path
d="M24 2c2.75 0 5.24 1.11 7.047 2.905l-2.864 2.793A6 6 0 0 0 18 12v5.633l-3.898 3.803A10.09 10.09 0 0 1 14 20v-8c0-5.523 4.477-10 10-10Z"
data-follow-fill="#232833"
fill="#232833"
/>
<path
clipRule="evenodd"
d="m18.151 28.112-2.172 2.12A12.945 12.945 0 0 0 24 33c7.18 0 13-5.82 13-13v-1a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1v1c0 8.712-6.554 15.894-15 16.884V42h9a1 1 0 0 1 1 1v2a1 1 0 0 1-1 1H13a1 1 0 0 1-1-1v-2a1 1 0 0 1 1-1h9v-5.116a16.925 16.925 0 0 1-8.904-3.84l-6.185 6.033a1 1 0 0 1-1.414-.017l-1.292-1.324a1 1 0 0 1 .018-1.414l33.642-32.82a1 1 0 0 1 1.415.017l1.291 1.324a1 1 0 0 1-.017 1.414L34 12.651V20c0 5.523-4.477 10-10 10-2.184 0-4.204-.7-5.849-1.888ZM30 16.552l-8.912 8.695A6 6 0 0 0 30 20v-3.447Z"
data-follow-fill="#232833"
fill="#232833"
fillRule="evenodd"
/>
<path
d="M11 20c0 1.353.207 2.658.59 3.885l-3.119 3.043A16.94 16.94 0 0 1 7 20v-1a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1v1Z"
data-follow-fill="#232833"
fill="#232833"
/>
</svg>
);
}
export function LoadingIcon({
size = 24,
width,
height,
className,
...props
}: IconSvgProps) {
return (
<svg
className={`animate-spin ${className}`}
fill="none"
height={size || height}
viewBox="0 0 1024 1024"
width={size || width}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<path
d="M512 170.667c-188.523 0-341.333 152.81-341.333 341.333S323.477 853.333 512 853.333a339.84 339.84 0 0 0 208.704-71.232c22.4-17.322 33.621-26.005 39.403-28.842 4.202-2.07 5.781-2.944 7.466-3.499 1.664-.576 3.435-.853 8.043-1.813 6.315-1.28 22.997-1.28 56.384-1.28 13.333 0 20.01 0 22.933 2.325a10.347 10.347 0 0 1 4.011 8.256c.043 3.733-3.627 8.384-10.923 17.707C769.941 874.624 648.448 938.667 512 938.667 276.352 938.667 85.333 747.648 85.333 512S276.352 85.333 512 85.333c136.448 0 257.92 64.043 336.021 163.712 7.318 9.323 10.966 13.974 10.923 17.707a10.347 10.347 0 0 1-4.01 8.256c-2.923 2.325-9.6 2.325-22.934 2.325-33.387 0-50.07 0-56.384-1.28-4.608-.938-6.379-1.237-8.043-1.813-1.685-.555-3.264-1.43-7.466-3.499-5.782-2.837-16.982-11.52-39.403-28.842A339.84 339.84 0 0 0 512 170.667z"
data-follow-fill="#2c2c2c"
fill="currentColor"
/>
<path
d="M832 682.667a170.667 170.667 0 1 0 0-341.334 170.667 170.667 0 0 0 0 341.334z"
data-follow-fill="#2c2c2c"
fill="currentColor"
/>
</svg>
);
}
export function CloseIcon({
size = 24,
width,
height,
...props
}: IconSvgProps) {
return (
<svg
fill="none"
height={size || height}
viewBox="0 0 20 20"
width={size || width}
xmlns="http://www.w3.org/2000/svg"
{...props}
>
<path
d="M15.7135 3.43055C15.8086 3.46146 15.8911 3.54396 16.0562 3.70897L16.2919 3.94467C16.4569 4.10968 16.5394 4.19218 16.5703 4.28732C16.5975 4.37101 16.5975 4.46115 16.5703 4.54484C16.5394 4.63997 16.4569 4.72248 16.2919 4.88748L11.1785 10.0008L16.2918 15.1141C16.4568 15.2791 16.5393 15.3616 16.5702 15.4568C16.5974 15.5404 16.5974 15.6306 16.5702 15.7143C16.5393 15.8094 16.4568 15.8919 16.2918 16.0569L16.0561 16.2926C15.8911 16.4576 15.8086 16.5401 15.7135 16.571C15.6298 16.5982 15.5396 16.5982 15.4559 16.571C15.3608 16.5401 15.2783 16.4576 15.1133 16.2926L10 11.1793L4.8868 16.2925C4.7218 16.4575 4.63929 16.5401 4.54415 16.571C4.46047 16.5982 4.37032 16.5982 4.28664 16.571C4.1915 16.5401 4.109 16.4575 3.94399 16.2925L3.70829 16.0568C3.54328 15.8918 3.46078 15.8093 3.42987 15.7142C3.40267 15.6305 3.40267 15.5404 3.42987 15.4567C3.46078 15.3615 3.54328 15.279 3.70829 15.114L8.8215 10.0008L3.70824 4.88756C3.54323 4.72255 3.46073 4.64005 3.42982 4.54491C3.40263 4.46123 3.40263 4.37108 3.42982 4.2874C3.46073 4.19226 3.54323 4.10976 3.70824 3.94475L3.94394 3.70905C4.10895 3.54404 4.19145 3.46154 4.28659 3.43062C4.37027 3.40343 4.46042 3.40343 4.5441 3.43062C4.63924 3.46154 4.72174 3.54404 4.88675 3.70905L10 8.82231L15.1133 3.70897C15.2784 3.54396 15.3609 3.46146 15.456 3.43055C15.5397 3.40336 15.6298 3.40336 15.7135 3.43055Z"
fill="currentColor"
/>
</svg>
);
}

20
components/Input.tsx Normal file
View File

@@ -0,0 +1,20 @@
import React from "react";
interface InputProps {
value: string | undefined | null;
onChange: (value: string) => void;
placeholder?: string;
className?: string;
}
export const Input = (props: InputProps) => {
return (
<input
className={`w-full text-white text-sm bg-zinc-700 py-2 px-6 rounded-lg outline-none ${props.className}`}
placeholder={props.placeholder}
type="text"
value={props.value || ""}
onChange={(e) => props.onChange(e.target.value)}
/>
);
};

View File

@@ -1,45 +1,52 @@
import type { StartAvatarResponse } from "@heygen/streaming-avatar";
import StreamingAvatar, {
AvatarQuality,
StreamingEvents, TaskType, VoiceEmotion,
} from "@heygen/streaming-avatar";
import {
Button,
Card,
CardBody,
CardFooter,
Divider,
Input,
Select,
SelectItem,
Spinner,
Chip,
Tabs,
Tab,
} from "@nextui-org/react";
AvatarQuality,
StreamingEvents,
VoiceChatTransport,
VoiceEmotion,
StartAvatarRequest,
STTProvider,
ElevenLabsModel,
} from "@heygen/streaming-avatar";
import { useEffect, useRef, useState } from "react";
import { useMemoizedFn, usePrevious } from "ahooks";
import { useMemoizedFn, useUnmount } from "ahooks";
import InteractiveAvatarTextInput from "./InteractiveAvatarTextInput";
import { Button } from "./Button";
import { AvatarConfig } from "./AvatarConfig";
import { AvatarVideo } from "./AvatarSession/AvatarVideo";
import { useStreamingAvatarSession } from "./logic/useStreamingAvatarSession";
import { AvatarControls } from "./AvatarSession/AvatarControls";
import { useVoiceChat } from "./logic/useVoiceChat";
import { StreamingAvatarProvider, StreamingAvatarSessionState } from "./logic";
import { LoadingIcon } from "./Icons";
import { MessageHistory } from "./AvatarSession/MessageHistory";
import {AVATARS, STT_LANGUAGE_LIST} from "@/app/lib/constants";
import { AVATARS } from "@/app/lib/constants";
export default function InteractiveAvatar() {
const [isLoadingSession, setIsLoadingSession] = useState(false);
const [isLoadingRepeat, setIsLoadingRepeat] = useState(false);
const [stream, setStream] = useState<MediaStream>();
const [debug, setDebug] = useState<string>();
const [knowledgeId, setKnowledgeId] = useState<string>("");
const [avatarId, setAvatarId] = useState<string>("");
const [language, setLanguage] = useState<string>('en');
const DEFAULT_CONFIG: StartAvatarRequest = {
quality: AvatarQuality.Low,
avatarName: AVATARS[0].avatar_id,
knowledgeId: undefined,
voice: {
rate: 1.5,
emotion: VoiceEmotion.EXCITED,
model: ElevenLabsModel.eleven_flash_v2_5,
},
language: "en",
disableIdleTimeout: true,
voiceChatTransport: VoiceChatTransport.LIVEKIT,
sttSettings: {
provider: STTProvider.DEEPGRAM,
},
};
function InteractiveAvatar() {
const { initAvatar, startAvatar, stopAvatar, sessionState, stream } =
useStreamingAvatarSession();
const { startVoiceChat } = useVoiceChat();
const [config, setConfig] = useState<StartAvatarRequest>(DEFAULT_CONFIG);
const [data, setData] = useState<StartAvatarResponse>();
const [text, setText] = useState<string>("");
const mediaStream = useRef<HTMLVideoElement>(null);
const avatar = useRef<StreamingAvatar | null>(null);
const [chatMode, setChatMode] = useState("text_mode");
const [isUserTalking, setIsUserTalking] = useState(false);
async function fetchAccessToken() {
try {
@@ -53,276 +60,107 @@ export default function InteractiveAvatar() {
return token;
} catch (error) {
console.error("Error fetching access token:", error);
throw error;
}
return "";
}
async function startSession() {
setIsLoadingSession(true);
const newToken = await fetchAccessToken();
avatar.current = new StreamingAvatar({
token: newToken,
});
avatar.current.on(StreamingEvents.AVATAR_START_TALKING, (e) => {
console.log("Avatar started talking", e);
});
avatar.current.on(StreamingEvents.AVATAR_STOP_TALKING, (e) => {
console.log("Avatar stopped talking", e);
});
avatar.current.on(StreamingEvents.STREAM_DISCONNECTED, () => {
console.log("Stream disconnected");
endSession();
});
avatar.current?.on(StreamingEvents.STREAM_READY, (event) => {
console.log(">>>>> Stream ready:", event.detail);
setStream(event.detail);
});
avatar.current?.on(StreamingEvents.USER_START, (event) => {
console.log(">>>>> User started talking:", event);
setIsUserTalking(true);
});
avatar.current?.on(StreamingEvents.USER_STOP, (event) => {
console.log(">>>>> User stopped talking:", event);
setIsUserTalking(false);
});
const startSessionV2 = useMemoizedFn(async (isVoiceChat: boolean) => {
try {
const res = await avatar.current.createStartAvatar({
quality: AvatarQuality.Low,
avatarName: avatarId,
knowledgeId: knowledgeId, // Or use a custom `knowledgeBase`.
voice: {
rate: 1.5, // 0.5 ~ 1.5
emotion: VoiceEmotion.EXCITED,
},
language: language,
const newToken = await fetchAccessToken();
const avatar = initAvatar(newToken);
avatar.on(StreamingEvents.AVATAR_START_TALKING, (e) => {
console.log("Avatar started talking", e);
});
avatar.on(StreamingEvents.AVATAR_STOP_TALKING, (e) => {
console.log("Avatar stopped talking", e);
});
avatar.on(StreamingEvents.STREAM_DISCONNECTED, () => {
console.log("Stream disconnected");
});
avatar.on(StreamingEvents.STREAM_READY, (event) => {
console.log(">>>>> Stream ready:", event.detail);
});
avatar.on(StreamingEvents.USER_START, (event) => {
console.log(">>>>> User started talking:", event);
});
avatar.on(StreamingEvents.USER_STOP, (event) => {
console.log(">>>>> User stopped talking:", event);
});
avatar.on(StreamingEvents.USER_END_MESSAGE, (event) => {
console.log(">>>>> User end message:", event);
});
avatar.on(StreamingEvents.USER_TALKING_MESSAGE, (event) => {
console.log(">>>>> User talking message:", event);
});
avatar.on(StreamingEvents.AVATAR_TALKING_MESSAGE, (event) => {
console.log(">>>>> Avatar talking message:", event);
});
avatar.on(StreamingEvents.AVATAR_END_MESSAGE, (event) => {
console.log(">>>>> Avatar end message:", event);
});
setData(res);
// default to voice mode
await avatar.current?.startVoiceChat();
setChatMode("voice_mode");
await startAvatar(config);
if (isVoiceChat) {
await startVoiceChat();
}
} catch (error) {
console.error("Error starting avatar session:", error);
} finally {
setIsLoadingSession(false);
}
}
async function handleSpeak() {
setIsLoadingRepeat(true);
if (!avatar.current) {
setDebug("Avatar API not initialized");
return;
}
// speak({ text: text, task_type: TaskType.REPEAT })
await avatar.current.speak({ text: text }).catch((e) => {
setDebug(e.message);
});
setIsLoadingRepeat(false);
}
async function handleInterrupt() {
if (!avatar.current) {
setDebug("Avatar API not initialized");
return;
}
await avatar.current
.interrupt()
.catch((e) => {
setDebug(e.message);
});
}
async function endSession() {
await avatar.current?.stopAvatar();
setStream(undefined);
}
const handleChangeChatMode = useMemoizedFn(async (v) => {
if (v === chatMode) {
return;
}
if (v === "text_mode") {
avatar.current?.closeVoiceChat();
} else {
await avatar.current?.startVoiceChat();
}
setChatMode(v);
});
const previousText = usePrevious(text);
useEffect(() => {
if (!previousText && text) {
avatar.current?.startListening();
} else if (previousText && !text) {
avatar?.current?.stopListening();
}
}, [text, previousText]);
useEffect(() => {
return () => {
endSession();
};
}, []);
useUnmount(() => {
stopAvatar();
});
useEffect(() => {
if (stream && mediaStream.current) {
mediaStream.current.srcObject = stream;
mediaStream.current.onloadedmetadata = () => {
mediaStream.current!.play();
setDebug("Playing");
};
}
}, [mediaStream, stream]);
return (
<div className="w-full flex flex-col gap-4">
<Card>
<CardBody className="h-[500px] flex flex-col justify-center items-center">
{stream ? (
<div className="h-[500px] w-[900px] justify-center items-center flex rounded-lg overflow-hidden">
<video
ref={mediaStream}
autoPlay
playsInline
style={{
width: "100%",
height: "100%",
objectFit: "contain",
}}
>
<track kind="captions" />
</video>
<div className="flex flex-col gap-2 absolute bottom-3 right-3">
<Button
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 text-white rounded-lg"
size="md"
variant="shadow"
onClick={handleInterrupt}
>
Interrupt task
</Button>
<Button
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 text-white rounded-lg"
size="md"
variant="shadow"
onClick={endSession}
>
End session
</Button>
</div>
</div>
) : !isLoadingSession ? (
<div className="h-full justify-center items-center flex flex-col gap-8 w-[500px] self-center">
<div className="flex flex-col gap-2 w-full">
<p className="text-sm font-medium leading-none">
Custom Knowledge ID (optional)
</p>
<Input
placeholder="Enter a custom knowledge ID"
value={knowledgeId}
onChange={(e) => setKnowledgeId(e.target.value)}
/>
<p className="text-sm font-medium leading-none">
Custom Avatar ID (optional)
</p>
<Input
placeholder="Enter a custom avatar ID"
value={avatarId}
onChange={(e) => setAvatarId(e.target.value)}
/>
<Select
placeholder="Or select one from these example avatars"
size="md"
onChange={(e) => {
setAvatarId(e.target.value);
}}
>
{AVATARS.map((avatar) => (
<SelectItem
key={avatar.avatar_id}
textValue={avatar.avatar_id}
>
{avatar.name}
</SelectItem>
))}
</Select>
<Select
label="Select language"
placeholder="Select language"
className="max-w-xs"
selectedKeys={[language]}
onChange={(e) => {
setLanguage(e.target.value);
}}
>
{STT_LANGUAGE_LIST.map((lang) => (
<SelectItem key={lang.key}>
{lang.label}
</SelectItem>
))}
</Select>
</div>
<Button
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 w-full text-white"
size="md"
variant="shadow"
onClick={startSession}
>
Start session
<div className="flex flex-col rounded-xl bg-zinc-900 overflow-hidden">
<div className="relative w-full aspect-video overflow-hidden flex flex-col items-center justify-center">
{sessionState !== StreamingAvatarSessionState.INACTIVE ? (
<AvatarVideo ref={mediaStream} />
) : (
<AvatarConfig config={config} onConfigChange={setConfig} />
)}
</div>
<div className="flex flex-col gap-3 items-center justify-center p-4 border-t border-zinc-700 w-full">
{sessionState === StreamingAvatarSessionState.CONNECTED ? (
<AvatarControls />
) : sessionState === StreamingAvatarSessionState.INACTIVE ? (
<div className="flex flex-row gap-4">
<Button onClick={() => startSessionV2(true)}>
Start Voice Chat
</Button>
<Button onClick={() => startSessionV2(false)}>
Start Text Chat
</Button>
</div>
) : (
<Spinner color="default" size="lg" />
<LoadingIcon />
)}
</CardBody>
<Divider />
<CardFooter className="flex flex-col gap-3 relative">
<Tabs
aria-label="Options"
selectedKey={chatMode}
onSelectionChange={(v) => {
handleChangeChatMode(v);
}}
>
<Tab key="text_mode" title="Text mode" />
<Tab key="voice_mode" title="Voice mode" />
</Tabs>
{chatMode === "text_mode" ? (
<div className="w-full flex relative">
<InteractiveAvatarTextInput
disabled={!stream}
input={text}
label="Chat"
loading={isLoadingRepeat}
placeholder="Type something for the avatar to respond"
setInput={setText}
onSubmit={handleSpeak}
/>
{text && (
<Chip className="absolute right-16 top-3">Listening</Chip>
)}
</div>
) : (
<div className="w-full text-center">
<Button
isDisabled={!isUserTalking}
className="bg-gradient-to-tr from-indigo-500 to-indigo-300 text-white"
size="md"
variant="shadow"
>
{isUserTalking ? "Listening" : "Voice chat"}
</Button>
</div>
)}
</CardFooter>
</Card>
<p className="font-mono text-right">
<span className="font-bold">Console:</span>
<br />
{debug}
</p>
</div>
</div>
{sessionState === StreamingAvatarSessionState.CONNECTED && (
<MessageHistory />
)}
</div>
);
}
export default function InteractiveAvatarWrapper() {
return (
<StreamingAvatarProvider basePath={process.env.NEXT_PUBLIC_BASE_API_URL}>
<InteractiveAvatar />
</StreamingAvatarProvider>
);
}

View File

@@ -1,99 +0,0 @@
import { Card, CardBody } from "@nextui-org/react";
import { langs } from "@uiw/codemirror-extensions-langs";
import ReactCodeMirror from "@uiw/react-codemirror";
export default function InteractiveAvatarCode() {
return (
<div className="w-full flex flex-col gap-2">
<p>This SDK supports the following behavior:</p>
<ul>
<li>
<div className="flex flex-row gap-2">
<p className="text-indigo-400 font-semibold">Start:</p> Start the
Interactive Avatar session
</div>
</li>
<li>
<div className="flex flex-row gap-2">
<p className="text-indigo-400 font-semibold">Close:</p> Close the
Interactive Avatar session
</div>
</li>
<li>
<div className="flex flex-row gap-2">
<p className="text-indigo-400 font-semibold">Speak:</p> Repeat the
input
</div>
</li>
</ul>
<Card>
<CardBody>
<ReactCodeMirror
editable={false}
extensions={[langs.typescript()]}
height="700px"
theme="dark"
value={TEXT}
/>
</CardBody>
</Card>
</div>
);
}
const TEXT = `
export default function App() {
// Media stream used by the video player to display the avatar
const [stream, setStream] = useState<MediaStream> ();
const mediaStream = useRef<HTMLVideoElement>(null);
// Instantiate the Interactive Avatar api using your access token
const avatar = useRef(new StreamingAvatarApi(
new Configuration({accessToken: '<REPLACE_WITH_ACCESS_TOKEN>'})
));
// State holding Interactive Avatar session data
const [sessionData, setSessionData] = useState<NewSessionData>();
// Function to start the Interactive Avatar session
async function start(){
const res = await avatar.current.createStartAvatar(
{ newSessionRequest:
// Define the session variables during creation
{ quality: "medium", // low, medium, high
avatarName: <REPLACE_WITH_AVATAR_ID>,
voice:{voiceId: <REPLACE_WITH_VOICE_ID>}
}
});
setSessionData(res);
}
// Function to stop the Interactive Avatar session
async function stop(){
await avatar.current.stopAvatar({stopSessionRequest: {sessionId: sessionData?.sessionId}});
}
// Function which passes in text to the avatar to repeat
async function handleSpeak(){
await avatar.current.speak({taskRequest: {text: <TEXT_TO_SAY>, sessionId: sessionData?.sessionId}}).catch((e) => {
});
}
useEffect(()=>{
// Handles the display of the Interactive Avatar
if(stream && mediaStream.current){
mediaStream.current.srcObject = stream;
mediaStream.current.onloadedmetadata = () => {
mediaStream.current!.play();
}
}
}, [mediaStream, stream])
return (
<div className="w-full">
<video playsInline autoPlay width={500} ref={mediaStream}/>
</div>
)
}`;

View File

@@ -1,77 +0,0 @@
import { Input, Spinner, Tooltip } from "@nextui-org/react";
import { Airplane, ArrowRight, PaperPlaneRight } from "@phosphor-icons/react";
import clsx from "clsx";
interface StreamingAvatarTextInputProps {
label: string;
placeholder: string;
input: string;
onSubmit: () => void;
setInput: (value: string) => void;
endContent?: React.ReactNode;
disabled?: boolean;
loading?: boolean;
}
export default function InteractiveAvatarTextInput({
label,
placeholder,
input,
onSubmit,
setInput,
endContent,
disabled = false,
loading = false,
}: StreamingAvatarTextInputProps) {
function handleSubmit() {
if (input.trim() === "") {
return;
}
onSubmit();
setInput("");
}
return (
<Input
endContent={
<div className="flex flex-row items-center h-full">
{endContent}
<Tooltip content="Send message">
{loading ? (
<Spinner
className="text-indigo-300 hover:text-indigo-200"
size="sm"
color="default"
/>
) : (
<button
type="submit"
className="focus:outline-none"
onClick={handleSubmit}
>
<PaperPlaneRight
className={clsx(
"text-indigo-300 hover:text-indigo-200",
disabled && "opacity-50"
)}
size={24}
/>
</button>
)}
</Tooltip>
</div>
}
label={label}
placeholder={placeholder}
size="sm"
value={input}
onKeyDown={(e) => {
if (e.key === "Enter") {
handleSubmit();
}
}}
onValueChange={setInput}
isDisabled={disabled}
/>
);
}

View File

@@ -1,70 +1,59 @@
"use client";
import {
Link,
Navbar,
NavbarBrand,
NavbarContent,
NavbarItem,
} from "@nextui-org/react";
import Link from "next/link";
import { GithubIcon, HeyGenLogo } from "./Icons";
import { ThemeSwitch } from "./ThemeSwitch";
export default function NavBar() {
return (
<Navbar className="w-full">
<NavbarBrand>
<Link isExternal aria-label="HeyGen" href="https://app.heygen.com/">
<HeyGenLogo />
</Link>
<div className="bg-gradient-to-br from-sky-300 to-indigo-500 bg-clip-text ml-4">
<p className="text-xl font-semibold text-transparent">
HeyGen Interactive Avatar SDK NextJS Demo
</p>
<>
<div className="flex flex-row justify-between items-center w-[1000px] m-auto p-6">
<div className="flex flex-row items-center gap-4">
<Link href="https://app.heygen.com/" target="_blank">
<HeyGenLogo />
</Link>
<div className="bg-gradient-to-br from-sky-300 to-indigo-500 bg-clip-text">
<p className="text-xl font-semibold text-transparent">
HeyGen Interactive Avatar SDK NextJS Demo
</p>
</div>
</div>
</NavbarBrand>
<NavbarContent justify="center">
<NavbarItem className="flex flex-row items-center gap-4">
<div className="flex flex-row items-center gap-6">
<Link
isExternal
color="foreground"
href="https://app.heygen.com/interactive-avatar"
href="https://labs.heygen.com/interactive-avatar"
target="_blank"
>
Avatars
</Link>
<Link
isExternal
color="foreground"
href="https://docs.heygen.com/reference/list-voices-v2"
target="_blank"
>
Voices
</Link>
<Link
isExternal
color="foreground"
href="https://docs.heygen.com/reference/new-session-copy"
target="_blank"
>
API Docs
</Link>
<Link
isExternal
color="foreground"
href="https://help.heygen.com/en/articles/9182113-interactive-avatar-101-your-ultimate-guide"
target="_blank"
>
Guide
</Link>
<Link
isExternal
aria-label="Github"
href="https://github.com/HeyGen-Official/StreamingAvatarSDK"
className="flex flex-row justify-center gap-1 text-foreground"
href="https://github.com/HeyGen-Official/StreamingAvatarSDK"
target="_blank"
>
<GithubIcon className="text-default-500" />
SDK
</Link>
<ThemeSwitch />
</NavbarItem>
</NavbarContent>
</Navbar>
</div>
</div>
</>
);
}

62
components/Select.tsx Normal file
View File

@@ -0,0 +1,62 @@
import * as SelectPrimitive from "@radix-ui/react-select";
import { useState } from "react";
import { ChevronDownIcon } from "./Icons";
interface SelectProps<T> {
options: T[];
renderOption: (option: T) => React.ReactNode;
onSelect: (option: T) => void;
isSelected: (option: T) => boolean;
value: string | null | undefined;
placeholder?: string;
disabled?: boolean;
}
export function Select<T>(props: SelectProps<T>) {
const [isOpen, setIsOpen] = useState(false);
return (
<SelectPrimitive.Root
disabled={props.disabled}
open={isOpen}
onOpenChange={setIsOpen}
>
<SelectPrimitive.Trigger className="w-full text-white text-sm bg-zinc-700 py-2 px-6 rounded-lg cursor-pointer flex items-center justify-between h-fit disabled:opacity-50 min-h-[36px]">
<div className={`${props.value ? "text-white" : "text-zinc-400"}`}>
{props.value ? props.value : props.placeholder}
</div>
<ChevronDownIcon className="w-4 h-4" />
</SelectPrimitive.Trigger>
<SelectPrimitive.Portal>
<SelectPrimitive.Content
className="z-50 w-[var(--radix-select-trigger-width)] max-h-[300px] overflow-y-auto"
position="popper"
sideOffset={5}
>
<SelectPrimitive.Viewport className="rounded-lg border border-zinc-600 bg-zinc-700 shadow-lg py-1">
{props.options.map((option) => {
const isSelected = props.isSelected(option);
return (
<div
key={props.renderOption(option)?.toString()}
className={`py-2 px-4 cursor-pointer hover:bg-zinc-600 outline-none text-sm ${
isSelected ? "text-white bg-zinc-500" : "text-zinc-400"
}`}
onClick={() => {
props.onSelect(option);
setIsOpen(false);
}}
>
{props.renderOption(option)}
</div>
);
})}
</SelectPrimitive.Viewport>
</SelectPrimitive.Content>
</SelectPrimitive.Portal>
</SelectPrimitive.Root>
);
}

View File

@@ -1,80 +0,0 @@
"use client";
import { FC } from "react";
import { VisuallyHidden } from "@react-aria/visually-hidden";
import { SwitchProps, useSwitch } from "@nextui-org/switch";
import { useTheme } from "next-themes";
import { useIsSSR } from "@react-aria/ssr";
import clsx from "clsx";
import { MoonFilledIcon, SunFilledIcon } from "./Icons";
export interface ThemeSwitchProps {
className?: string;
classNames?: SwitchProps["classNames"];
}
export const ThemeSwitch: FC<ThemeSwitchProps> = ({
className,
classNames,
}) => {
const { theme, setTheme } = useTheme();
const isSSR = useIsSSR();
const onChange = () => {
theme === "light" ? setTheme("dark") : setTheme("light");
};
const {
Component,
slots,
isSelected,
getBaseProps,
getInputProps,
getWrapperProps,
} = useSwitch({
isSelected: theme === "light" || isSSR,
"aria-label": `Switch to ${theme === "light" || isSSR ? "dark" : "light"} mode`,
onChange,
});
return (
<Component
{...getBaseProps({
className: clsx(
"px-px transition-opacity hover:opacity-80 cursor-pointer",
className,
classNames?.base
),
})}
>
<VisuallyHidden>
<input {...getInputProps()} />
</VisuallyHidden>
<div
{...getWrapperProps()}
className={slots.wrapper({
class: clsx(
[
"w-auto h-auto",
"bg-transparent",
"rounded-lg",
"flex items-center justify-center",
"group-data-[selected=true]:bg-transparent",
"!text-default-500",
"pt-px",
"px-0",
"mx-0",
],
classNames?.wrapper
),
})}
>
{!isSelected || isSSR ? (
<SunFilledIcon size={24} />
) : (
<MoonFilledIcon size={24} />
)}
</div>
</Component>
);
};

View File

@@ -0,0 +1,257 @@
import StreamingAvatar, {
ConnectionQuality,
StreamingTalkingMessageEvent,
UserTalkingMessageEvent,
} from "@heygen/streaming-avatar";
import React, { useRef, useState } from "react";
export enum StreamingAvatarSessionState {
INACTIVE = "inactive",
CONNECTING = "connecting",
CONNECTED = "connected",
}
export enum MessageSender {
CLIENT = "CLIENT",
AVATAR = "AVATAR",
}
export interface Message {
id: string;
sender: MessageSender;
content: string;
}
type StreamingAvatarContextProps = {
avatarRef: React.MutableRefObject<StreamingAvatar | null>;
basePath?: string;
isMuted: boolean;
setIsMuted: (isMuted: boolean) => void;
isVoiceChatLoading: boolean;
setIsVoiceChatLoading: (isVoiceChatLoading: boolean) => void;
isVoiceChatActive: boolean;
setIsVoiceChatActive: (isVoiceChatActive: boolean) => void;
sessionState: StreamingAvatarSessionState;
setSessionState: (sessionState: StreamingAvatarSessionState) => void;
stream: MediaStream | null;
setStream: (stream: MediaStream | null) => void;
messages: Message[];
clearMessages: () => void;
handleUserTalkingMessage: ({
detail,
}: {
detail: UserTalkingMessageEvent;
}) => void;
handleStreamingTalkingMessage: ({
detail,
}: {
detail: StreamingTalkingMessageEvent;
}) => void;
handleEndMessage: () => void;
isListening: boolean;
setIsListening: (isListening: boolean) => void;
isUserTalking: boolean;
setIsUserTalking: (isUserTalking: boolean) => void;
isAvatarTalking: boolean;
setIsAvatarTalking: (isAvatarTalking: boolean) => void;
connectionQuality: ConnectionQuality;
setConnectionQuality: (connectionQuality: ConnectionQuality) => void;
};
const StreamingAvatarContext = React.createContext<StreamingAvatarContextProps>(
{
avatarRef: { current: null },
isMuted: true,
setIsMuted: () => {},
isVoiceChatLoading: false,
setIsVoiceChatLoading: () => {},
sessionState: StreamingAvatarSessionState.INACTIVE,
setSessionState: () => {},
isVoiceChatActive: false,
setIsVoiceChatActive: () => {},
stream: null,
setStream: () => {},
messages: [],
clearMessages: () => {},
handleUserTalkingMessage: () => {},
handleStreamingTalkingMessage: () => {},
handleEndMessage: () => {},
isListening: false,
setIsListening: () => {},
isUserTalking: false,
setIsUserTalking: () => {},
isAvatarTalking: false,
setIsAvatarTalking: () => {},
connectionQuality: ConnectionQuality.UNKNOWN,
setConnectionQuality: () => {},
},
);
const useStreamingAvatarSessionState = () => {
const [sessionState, setSessionState] = useState(
StreamingAvatarSessionState.INACTIVE,
);
const [stream, setStream] = useState<MediaStream | null>(null);
return {
sessionState,
setSessionState,
stream,
setStream,
};
};
const useStreamingAvatarVoiceChatState = () => {
const [isMuted, setIsMuted] = useState(true);
const [isVoiceChatLoading, setIsVoiceChatLoading] = useState(false);
const [isVoiceChatActive, setIsVoiceChatActive] = useState(false);
return {
isMuted,
setIsMuted,
isVoiceChatLoading,
setIsVoiceChatLoading,
isVoiceChatActive,
setIsVoiceChatActive,
};
};
const useStreamingAvatarMessageState = () => {
const [messages, setMessages] = useState<Message[]>([]);
const currentSenderRef = useRef<MessageSender | null>(null);
const handleUserTalkingMessage = ({
detail,
}: {
detail: UserTalkingMessageEvent;
}) => {
if (currentSenderRef.current === MessageSender.CLIENT) {
setMessages((prev) => [
...prev.slice(0, -1),
{
...prev[prev.length - 1],
content: [prev[prev.length - 1].content, detail.message].join(""),
},
]);
} else {
currentSenderRef.current = MessageSender.CLIENT;
setMessages((prev) => [
...prev,
{
id: Date.now().toString(),
sender: MessageSender.CLIENT,
content: detail.message,
},
]);
}
};
const handleStreamingTalkingMessage = ({
detail,
}: {
detail: StreamingTalkingMessageEvent;
}) => {
if (currentSenderRef.current === MessageSender.AVATAR) {
setMessages((prev) => [
...prev.slice(0, -1),
{
...prev[prev.length - 1],
content: [prev[prev.length - 1].content, detail.message].join(""),
},
]);
} else {
currentSenderRef.current = MessageSender.AVATAR;
setMessages((prev) => [
...prev,
{
id: Date.now().toString(),
sender: MessageSender.AVATAR,
content: detail.message,
},
]);
}
};
const handleEndMessage = () => {
currentSenderRef.current = null;
};
return {
messages,
clearMessages: () => {
setMessages([]);
currentSenderRef.current = null;
},
handleUserTalkingMessage,
handleStreamingTalkingMessage,
handleEndMessage,
};
};
const useStreamingAvatarListeningState = () => {
const [isListening, setIsListening] = useState(false);
return { isListening, setIsListening };
};
const useStreamingAvatarTalkingState = () => {
const [isUserTalking, setIsUserTalking] = useState(false);
const [isAvatarTalking, setIsAvatarTalking] = useState(false);
return {
isUserTalking,
setIsUserTalking,
isAvatarTalking,
setIsAvatarTalking,
};
};
const useStreamingAvatarConnectionQualityState = () => {
const [connectionQuality, setConnectionQuality] = useState(
ConnectionQuality.UNKNOWN,
);
return { connectionQuality, setConnectionQuality };
};
export const StreamingAvatarProvider = ({
children,
basePath,
}: {
children: React.ReactNode;
basePath?: string;
}) => {
const avatarRef = React.useRef<StreamingAvatar>(null);
const voiceChatState = useStreamingAvatarVoiceChatState();
const sessionState = useStreamingAvatarSessionState();
const messageState = useStreamingAvatarMessageState();
const listeningState = useStreamingAvatarListeningState();
const talkingState = useStreamingAvatarTalkingState();
const connectionQualityState = useStreamingAvatarConnectionQualityState();
return (
<StreamingAvatarContext.Provider
value={{
avatarRef,
basePath,
...voiceChatState,
...sessionState,
...messageState,
...listeningState,
...talkingState,
...connectionQualityState,
}}
>
{children}
</StreamingAvatarContext.Provider>
);
};
export const useStreamingAvatarContext = () => {
return React.useContext(StreamingAvatarContext);
};

10
components/logic/index.ts Normal file
View File

@@ -0,0 +1,10 @@
export { useStreamingAvatarSession } from "./useStreamingAvatarSession";
export { useVoiceChat } from "./useVoiceChat";
export { useConnectionQuality } from "./useConnectionQuality";
export { useMessageHistory } from "./useMessageHistory";
export { useInterrupt } from "./useInterrupt";
export {
StreamingAvatarSessionState,
StreamingAvatarProvider,
MessageSender,
} from "./context";

View File

@@ -0,0 +1,9 @@
import { useStreamingAvatarContext } from "./context";
export const useConnectionQuality = () => {
const { connectionQuality } = useStreamingAvatarContext();
return {
connectionQuality,
};
};

View File

@@ -0,0 +1,26 @@
import { useCallback } from "react";
import { useStreamingAvatarContext } from "./context";
export const useConversationState = () => {
const { avatarRef, isAvatarTalking, isUserTalking, isListening } =
useStreamingAvatarContext();
const startListening = useCallback(() => {
if (!avatarRef.current) return;
avatarRef.current.startListening();
}, [avatarRef]);
const stopListening = useCallback(() => {
if (!avatarRef.current) return;
avatarRef.current.stopListening();
}, [avatarRef]);
return {
isAvatarListening: isListening,
startListening,
stopListening,
isUserTalking,
isAvatarTalking,
};
};

View File

@@ -0,0 +1,14 @@
import { useCallback } from "react";
import { useStreamingAvatarContext } from "./context";
export const useInterrupt = () => {
const { avatarRef } = useStreamingAvatarContext();
const interrupt = useCallback(() => {
if (!avatarRef.current) return;
avatarRef.current.interrupt();
}, [avatarRef]);
return { interrupt };
};

View File

@@ -0,0 +1,7 @@
import { useStreamingAvatarContext } from "./context";
export const useMessageHistory = () => {
const { messages } = useStreamingAvatarContext();
return { messages };
};

View File

@@ -0,0 +1,158 @@
import StreamingAvatar, {
ConnectionQuality,
StartAvatarRequest,
StreamingEvents,
} from "@heygen/streaming-avatar";
import { useCallback } from "react";
import {
StreamingAvatarSessionState,
useStreamingAvatarContext,
} from "./context";
import { useVoiceChat } from "./useVoiceChat";
import { useMessageHistory } from "./useMessageHistory";
export const useStreamingAvatarSession = () => {
const {
avatarRef,
basePath,
sessionState,
setSessionState,
stream,
setStream,
setIsListening,
setIsUserTalking,
setIsAvatarTalking,
setConnectionQuality,
handleUserTalkingMessage,
handleStreamingTalkingMessage,
handleEndMessage,
clearMessages,
} = useStreamingAvatarContext();
const { stopVoiceChat } = useVoiceChat();
useMessageHistory();
const init = useCallback(
(token: string) => {
avatarRef.current = new StreamingAvatar({
token,
basePath: basePath,
});
return avatarRef.current;
},
[basePath, avatarRef],
);
const handleStream = useCallback(
({ detail }: { detail: MediaStream }) => {
setStream(detail);
setSessionState(StreamingAvatarSessionState.CONNECTED);
},
[setSessionState, setStream],
);
const stop = useCallback(async () => {
avatarRef.current?.off(StreamingEvents.STREAM_READY, handleStream);
avatarRef.current?.off(StreamingEvents.STREAM_DISCONNECTED, stop);
clearMessages();
stopVoiceChat();
setIsListening(false);
setIsUserTalking(false);
setIsAvatarTalking(false);
setStream(null);
await avatarRef.current?.stopAvatar();
setSessionState(StreamingAvatarSessionState.INACTIVE);
}, [
handleStream,
setSessionState,
setStream,
avatarRef,
setIsListening,
stopVoiceChat,
clearMessages,
setIsUserTalking,
setIsAvatarTalking,
]);
const start = useCallback(
async (config: StartAvatarRequest, token?: string) => {
if (sessionState !== StreamingAvatarSessionState.INACTIVE) {
throw new Error("There is already an active session");
}
if (!avatarRef.current) {
if (!token) {
throw new Error("Token is required");
}
init(token);
}
if (!avatarRef.current) {
throw new Error("Avatar is not initialized");
}
setSessionState(StreamingAvatarSessionState.CONNECTING);
avatarRef.current.on(StreamingEvents.STREAM_READY, handleStream);
avatarRef.current.on(StreamingEvents.STREAM_DISCONNECTED, stop);
avatarRef.current.on(
StreamingEvents.CONNECTION_QUALITY_CHANGED,
({ detail }: { detail: ConnectionQuality }) =>
setConnectionQuality(detail),
);
avatarRef.current.on(StreamingEvents.USER_START, () => {
setIsUserTalking(true);
});
avatarRef.current.on(StreamingEvents.USER_STOP, () => {
setIsUserTalking(false);
});
avatarRef.current.on(StreamingEvents.AVATAR_START_TALKING, () => {
setIsAvatarTalking(true);
});
avatarRef.current.on(StreamingEvents.AVATAR_STOP_TALKING, () => {
setIsAvatarTalking(false);
});
avatarRef.current.on(
StreamingEvents.USER_TALKING_MESSAGE,
handleUserTalkingMessage,
);
avatarRef.current.on(
StreamingEvents.AVATAR_TALKING_MESSAGE,
handleStreamingTalkingMessage,
);
avatarRef.current.on(StreamingEvents.USER_END_MESSAGE, handleEndMessage);
avatarRef.current.on(
StreamingEvents.AVATAR_END_MESSAGE,
handleEndMessage,
);
await avatarRef.current.createStartAvatar(config);
return avatarRef.current;
},
[
init,
handleStream,
stop,
setSessionState,
avatarRef,
sessionState,
setConnectionQuality,
setIsUserTalking,
handleUserTalkingMessage,
handleStreamingTalkingMessage,
handleEndMessage,
setIsAvatarTalking,
],
);
return {
avatarRef,
sessionState,
stream,
initAvatar: init,
startAvatar: start,
stopAvatar: stop,
};
};

View File

@@ -0,0 +1,66 @@
import { TaskMode, TaskType } from "@heygen/streaming-avatar";
import { useCallback } from "react";
import { useStreamingAvatarContext } from "./context";
export const useTextChat = () => {
const { avatarRef } = useStreamingAvatarContext();
const sendMessage = useCallback(
(message: string) => {
if (!avatarRef.current) return;
avatarRef.current.speak({
text: message,
taskType: TaskType.TALK,
taskMode: TaskMode.ASYNC,
});
},
[avatarRef],
);
const sendMessageSync = useCallback(
async (message: string) => {
if (!avatarRef.current) return;
return await avatarRef.current?.speak({
text: message,
taskType: TaskType.TALK,
taskMode: TaskMode.SYNC,
});
},
[avatarRef],
);
const repeatMessage = useCallback(
(message: string) => {
if (!avatarRef.current) return;
return avatarRef.current?.speak({
text: message,
taskType: TaskType.REPEAT,
taskMode: TaskMode.ASYNC,
});
},
[avatarRef],
);
const repeatMessageSync = useCallback(
async (message: string) => {
if (!avatarRef.current) return;
return await avatarRef.current?.speak({
text: message,
taskType: TaskType.REPEAT,
taskMode: TaskMode.SYNC,
});
},
[avatarRef],
);
return {
sendMessage,
sendMessageSync,
repeatMessage,
repeatMessageSync,
};
};

View File

@@ -0,0 +1,58 @@
import { useCallback } from "react";
import { useStreamingAvatarContext } from "./context";
export const useVoiceChat = () => {
const {
avatarRef,
isMuted,
setIsMuted,
isVoiceChatActive,
setIsVoiceChatActive,
isVoiceChatLoading,
setIsVoiceChatLoading,
} = useStreamingAvatarContext();
const startVoiceChat = useCallback(
async (isInputAudioMuted?: boolean) => {
if (!avatarRef.current) return;
setIsVoiceChatLoading(true);
await avatarRef.current?.startVoiceChat({
isInputAudioMuted,
});
setIsVoiceChatLoading(false);
setIsVoiceChatActive(true);
setIsMuted(!!isInputAudioMuted);
},
[avatarRef, setIsMuted, setIsVoiceChatActive, setIsVoiceChatLoading],
);
const stopVoiceChat = useCallback(() => {
if (!avatarRef.current) return;
avatarRef.current?.closeVoiceChat();
setIsVoiceChatActive(false);
setIsMuted(true);
}, [avatarRef, setIsMuted, setIsVoiceChatActive]);
const muteInputAudio = useCallback(() => {
if (!avatarRef.current) return;
avatarRef.current?.muteInputAudio();
setIsMuted(true);
}, [avatarRef, setIsMuted]);
const unmuteInputAudio = useCallback(() => {
if (!avatarRef.current) return;
avatarRef.current?.unmuteInputAudio();
setIsMuted(false);
}, [avatarRef, setIsMuted]);
return {
startVoiceChat,
stopVoiceChat,
muteInputAudio,
unmuteInputAudio,
isMuted,
isVoiceChatActive,
isVoiceChatLoading,
};
};

112
eslint.config.mjs Normal file
View File

@@ -0,0 +1,112 @@
import tseslint from '@typescript-eslint/eslint-plugin';
import tsparser from '@typescript-eslint/parser';
import reactPlugin from 'eslint-plugin-react';
import reactHooksPlugin from 'eslint-plugin-react-hooks';
import unusedImports from 'eslint-plugin-unused-imports';
import prettierPlugin from 'eslint-plugin-prettier';
import importPlugin from 'eslint-plugin-import';
import nextPlugin from '@next/eslint-plugin-next';
export default [
{
ignores: ['.next', 'node_modules']
},
{
files: ['**/*.{js,jsx,ts,tsx}'],
ignores: [
'build/**/*',
'public/**/*',
'dist/**/*',
'coverage/**/*',
'*.config.js',
'*.config.ts',
'postcss.config.js',
'tailwind.config.js',
],
languageOptions: {
parser: tsparser,
parserOptions: {
ecmaVersion: 'latest',
sourceType: 'module',
ecmaFeatures: {
jsx: true,
},
},
},
plugins: {
'@typescript-eslint': tseslint,
'react': reactPlugin,
'react-hooks': reactHooksPlugin,
'unused-imports': unusedImports,
'prettier': prettierPlugin,
'import': importPlugin,
'next': nextPlugin,
},
settings: {
react: {
version: 'detect',
},
},
rules: {
'react/prop-types': 'off',
'react/jsx-uses-react': 'off',
'react/react-in-jsx-scope': 'off',
'react/self-closing-comp': 'warn',
'react/jsx-sort-props': [
'warn',
{
callbacksLast: true,
shorthandFirst: true,
noSortAlphabetically: false,
reservedFirst: true,
},
],
'react-hooks/exhaustive-deps': 'warn',
'react-hooks/rules-of-hooks': 'error',
'no-unused-vars': 'off',
'@typescript-eslint/no-unused-vars': [
'warn',
{
args: 'after-used',
ignoreRestSiblings: false,
argsIgnorePattern: '^_.*?$',
},
],
'unused-imports/no-unused-imports': 'warn',
'import/order': [
'warn',
{
groups: [
'type',
'builtin',
'object',
'external',
'internal',
'parent',
'sibling',
'index',
],
pathGroups: [
{
pattern: '~/**',
group: 'external',
position: 'after',
},
],
'newlines-between': 'always',
},
],
'prettier/prettier': 'warn',
'padding-line-between-statements': [
'warn',
{ blankLine: 'always', prev: '*', next: 'return' },
{ blankLine: 'always', prev: ['const', 'let', 'var'], next: '*' },
{
blankLine: 'any',
prev: ['const', 'let', 'var'],
next: ['const', 'let', 'var'],
},
],
},
},
];

View File

@@ -6,61 +6,37 @@
"dev": "node_modules/next/dist/bin/next dev",
"build": "next build",
"start": "next start",
"lint": "eslint . --ext .ts,.tsx -c .eslintrc.json --fix"
"lint": "eslint . --ext .ts,.tsx,.js,.jsx"
},
"dependencies": {
"@ai-sdk/openai": "^0.0.34",
"@heygen/streaming-avatar": "^2.0.4",
"@nextui-org/button": "2.0.34",
"@nextui-org/chip": "^2.0.32",
"@nextui-org/code": "2.0.29",
"@nextui-org/input": "2.2.2",
"@nextui-org/kbd": "2.0.30",
"@nextui-org/link": "2.0.32",
"@nextui-org/listbox": "2.1.21",
"@nextui-org/navbar": "2.0.33",
"@nextui-org/react": "^2.4.2",
"@nextui-org/snippet": "2.0.38",
"@nextui-org/switch": "2.0.31",
"@nextui-org/system": "2.2.1",
"@nextui-org/theme": "2.2.5",
"@phosphor-icons/react": "^2.1.5",
"@react-aria/ssr": "3.9.4",
"@react-aria/visually-hidden": "3.8.12",
"@uiw/codemirror-extensions-langs": "^4.22.1",
"@uiw/react-codemirror": "^4.22.1",
"ahooks": "^3.8.1",
"ai": "^3.2.15",
"clsx": "2.1.1",
"framer-motion": "~11.1.1",
"intl-messageformat": "^10.5.0",
"next": "14.2.4",
"next-themes": "^0.2.1",
"@heygen/streaming-avatar": "^2.0.13",
"@radix-ui/react-select": "^2.1.7",
"@radix-ui/react-switch": "^1.1.4",
"@radix-ui/react-toggle-group": "^1.1.3",
"ahooks": "^3.8.4",
"next": "^15.3.0",
"openai": "^4.52.1",
"react": "18.3.1",
"react-dom": "18.3.1",
"zod": "^3.23.8"
"react": "^19.1.0",
"react-dom": "^19.1.0"
},
"devDependencies": {
"@types/node": "20.5.7",
"@types/react": "18.3.3",
"@types/react-dom": "18.3.0",
"@typescript-eslint/eslint-plugin": "7.2.0",
"@typescript-eslint/parser": "7.2.0",
"@types/react": "^19.0.1",
"@types/react-dom": "^19.1.2",
"@typescript-eslint/eslint-plugin": "^8.31.0",
"@typescript-eslint/parser": "^8.31.0",
"@next/eslint-plugin-next": "^15.3.1",
"autoprefixer": "10.4.19",
"eslint": "^8.57.0",
"eslint-config-next": "14.2.1",
"eslint-config-prettier": "^8.2.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jsx-a11y": "^6.4.1",
"eslint": "^9.25.1",
"eslint-config-prettier": "^10.1.2",
"eslint-plugin-import": "^2.31.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.23.2",
"eslint-plugin-react-hooks": "^4.6.0",
"eslint-plugin-unused-imports": "^3.2.0",
"eslint-plugin-prettier": "^5.2.6",
"eslint-plugin-react": "^7.37.5",
"eslint-plugin-react-hooks": "^5.2.0",
"eslint-plugin-unused-imports": "^4.1.4",
"postcss": "8.4.38",
"tailwind-variants": "0.1.20",
"tailwindcss": "3.4.3",
"tailwindcss": "^3.4.17",
"typescript": "5.0.4"
}
}

4235
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,11 +1,8 @@
import {nextui} from '@nextui-org/theme'
/** @type {import('tailwindcss').Config} */
module.exports = {
content: [
'./components/**/*.{js,ts,jsx,tsx,mdx}',
'./app/**/*.{js,ts,jsx,tsx,mdx}',
'./node_modules/@nextui-org/theme/dist/**/*.{js,ts,jsx,tsx}'
],
theme: {
extend: {
@@ -16,5 +13,4 @@ module.exports = {
},
},
darkMode: "class",
plugins: [nextui()],
}