feat: multimodal support (image) (#27793)

Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

web/app/components/header/account-setting/model-provider-page/model-name/index.tsx

@@ -17,6 +17,7 @@ type ModelNameProps = PropsWithChildren<{
   showMode?: boolean
   modeClassName?: string
   showFeatures?: boolean
+  showFeaturesLabel?: boolean
   featuresClassName?: string
   showContextSize?: boolean
 }>
@@ -28,6 +29,7 @@ const ModelName: FC<ModelNameProps> = ({
   showMode,
   modeClassName,
   showFeatures,
+  showFeaturesLabel,
   featuresClassName,
   showContextSize,
   children,
@@ -59,15 +61,6 @@ const ModelName: FC<ModelNameProps> = ({
             </ModelBadge>
           )
         }
-        {
-          showFeatures && modelItem.features?.map(feature => (
-            <FeatureIcon
-              key={feature}
-              feature={feature}
-              className={featuresClassName}
-            />
-          ))
-        }
         {
           showContextSize && modelItem.model_properties.context_size && (
             <ModelBadge>
@@ -75,6 +68,16 @@ const ModelName: FC<ModelNameProps> = ({
             </ModelBadge>
           )
         }
+        {
+          showFeatures && modelItem.features?.map(feature => (
+            <FeatureIcon
+              key={feature}
+              feature={feature}
+              className={featuresClassName}
+              showFeaturesLabel={showFeaturesLabel}
+            />
+          ))
+        }
       </div>
       {children}
     </div>

web/app/components/header/account-setting/model-provider-page/model-selector/feature-icon.tsx

@@ -5,24 +5,24 @@ import {
   ModelFeatureEnum,
   ModelFeatureTextEnum,
 } from '../declarations'
-import {
-  AudioSupportIcon,
-  DocumentSupportIcon,
-  // MagicBox,
-  MagicEyes,
-  // MagicWand,
-  // Robot,
-  VideoSupportIcon,
-} from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
 import Tooltip from '@/app/components/base/tooltip'
+import {
+  RiFileTextLine,
+  RiFilmAiLine,
+  RiImageCircleAiLine,
+  RiVoiceAiFill,
+} from '@remixicon/react'
+import cn from '@/utils/classnames'
 
 type FeatureIconProps = {
   feature: ModelFeatureEnum
   className?: string
+  showFeaturesLabel?: boolean
 }
 const FeatureIcon: FC<FeatureIconProps> = ({
   className,
   feature,
+  showFeaturesLabel,
 }) => {
   const { t } = useTranslation()
 
@@ -63,13 +63,29 @@ const FeatureIcon: FC<FeatureIconProps> = ({
   // }
 
   if (feature === ModelFeatureEnum.vision) {
+    if (showFeaturesLabel) {
+      return (
+        <ModelBadge
+          className={cn('gap-x-0.5', className)}
+        >
+          <RiImageCircleAiLine className='size-3' />
+          <span>{ModelFeatureTextEnum.vision}</span>
+        </ModelBadge>
+      )
+    }
+
     return (
       <Tooltip
         popupContent={t('common.modelProvider.featureSupported', { feature: ModelFeatureTextEnum.vision })}
       >
         <div className='inline-block cursor-help'>
-          <ModelBadge className={`w-[18px] justify-center !px-0 text-text-tertiary ${className}`}>
-            <MagicEyes className='h-3 w-3' />
+          <ModelBadge
+            className={cn(
+              'w-[18px] justify-center !px-0',
+              className,
+            )}
+          >
+            <RiImageCircleAiLine className='size-3' />
           </ModelBadge>
         </div>
       </Tooltip>
@@ -77,13 +93,29 @@ const FeatureIcon: FC<FeatureIconProps> = ({
   }
 
   if (feature === ModelFeatureEnum.document) {
+    if (showFeaturesLabel) {
+      return (
+        <ModelBadge
+          className={cn('gap-x-0.5', className)}
+        >
+          <RiFileTextLine className='size-3' />
+          <span>{ModelFeatureTextEnum.document}</span>
+        </ModelBadge>
+      )
+    }
+
     return (
       <Tooltip
         popupContent={t('common.modelProvider.featureSupported', { feature: ModelFeatureTextEnum.document })}
       >
         <div className='inline-block cursor-help'>
-          <ModelBadge className={`w-[18px] justify-center !px-0 text-text-tertiary ${className}`}>
-            <DocumentSupportIcon className='h-3 w-3' />
+          <ModelBadge
+            className={cn(
+              'w-[18px] justify-center !px-0',
+              className,
+            )}
+          >
+            <RiFileTextLine className='size-3' />
           </ModelBadge>
         </div>
       </Tooltip>
@@ -91,13 +123,29 @@ const FeatureIcon: FC<FeatureIconProps> = ({
   }
 
   if (feature === ModelFeatureEnum.audio) {
+    if (showFeaturesLabel) {
+      return (
+        <ModelBadge
+          className={cn('gap-x-0.5', className)}
+        >
+          <RiVoiceAiFill className='size-3' />
+          <span>{ModelFeatureTextEnum.audio}</span>
+        </ModelBadge>
+      )
+    }
+
     return (
       <Tooltip
         popupContent={t('common.modelProvider.featureSupported', { feature: ModelFeatureTextEnum.audio })}
       >
         <div className='inline-block cursor-help'>
-          <ModelBadge className={`w-[18px] justify-center !px-0 text-text-tertiary ${className}`}>
-            <AudioSupportIcon className='h-3 w-3' />
+          <ModelBadge
+            className={cn(
+              'w-[18px] justify-center !px-0',
+              className,
+            )}
+          >
+            <RiVoiceAiFill className='size-3' />
           </ModelBadge>
         </div>
       </Tooltip>
@@ -105,13 +153,29 @@ const FeatureIcon: FC<FeatureIconProps> = ({
   }
 
   if (feature === ModelFeatureEnum.video) {
+    if (showFeaturesLabel) {
+      return (
+        <ModelBadge
+          className={cn('gap-x-0.5', className)}
+        >
+          <RiFilmAiLine className='size-3' />
+          <span>{ModelFeatureTextEnum.video}</span>
+        </ModelBadge>
+      )
+    }
+
     return (
       <Tooltip
         popupContent={t('common.modelProvider.featureSupported', { feature: ModelFeatureTextEnum.video })}
       >
         <div className='inline-block cursor-help'>
-          <ModelBadge className={`w-[18px] justify-center !px-0 text-text-tertiary ${className}`}>
-            <VideoSupportIcon className='h-3 w-3' />
+          <ModelBadge
+            className={cn(
+              'w-[18px] justify-center !px-0',
+              className,
+            )}
+          >
+            <RiFilmAiLine className='size-3' />
           </ModelBadge>
         </div>
       </Tooltip>

web/app/components/header/account-setting/model-provider-page/model-selector/popup-item.tsx

@@ -1,11 +1,6 @@
 import type { FC } from 'react'
 import { useTranslation } from 'react-i18next'
-import {
-  RiFileTextLine,
-  RiFilmAiLine,
-  RiImageCircleAiLine,
-  RiVoiceAiFill,
-} from '@remixicon/react'
+
 import type {
   DefaultModel,
   Model,
@@ -13,7 +8,6 @@ import type {
 } from '../declarations'
 import {
   ModelFeatureEnum,
-  ModelFeatureTextEnum,
   ModelTypeEnum,
 } from '../declarations'
 import {
@@ -37,6 +31,7 @@ import { useModalContext } from '@/context/modal-context'
 import { useProviderContext } from '@/context/provider-context'
 import Tooltip from '@/app/components/base/tooltip'
 import cn from '@/utils/classnames'
+import FeatureIcon from './feature-icon'
 
 type PopupItemProps = {
   defaultModel?: DefaultModel
@@ -119,37 +114,23 @@ const PopupItem: FC<PopupItemProps> = ({
                     </ModelBadge>
                   )}
                 </div>
-                {modelItem.model_type === ModelTypeEnum.textGeneration && modelItem.features?.some(feature => [ModelFeatureEnum.vision, ModelFeatureEnum.audio, ModelFeatureEnum.video, ModelFeatureEnum.document].includes(feature)) && (
-                  <div className='pt-2'>
-                    <div className='system-2xs-medium-uppercase mb-1 text-text-tertiary'>{t('common.model.capabilities')}</div>
-                    <div className='flex flex-wrap gap-1'>
-                      {modelItem.features?.includes(ModelFeatureEnum.vision) && (
-                        <ModelBadge>
-                          <RiImageCircleAiLine className='mr-0.5 h-3.5 w-3.5' />
-                          <span>{ModelFeatureTextEnum.vision}</span>
-                        </ModelBadge>
-                      )}
-                      {modelItem.features?.includes(ModelFeatureEnum.audio) && (
-                        <ModelBadge>
-                          <RiVoiceAiFill className='mr-0.5 h-3.5 w-3.5' />
-                          <span>{ModelFeatureTextEnum.audio}</span>
-                        </ModelBadge>
-                      )}
-                      {modelItem.features?.includes(ModelFeatureEnum.video) && (
-                        <ModelBadge>
-                          <RiFilmAiLine className='mr-0.5 h-3.5 w-3.5' />
-                          <span>{ModelFeatureTextEnum.video}</span>
-                        </ModelBadge>
-                      )}
-                      {modelItem.features?.includes(ModelFeatureEnum.document) && (
-                        <ModelBadge>
-                          <RiFileTextLine className='mr-0.5 h-3.5 w-3.5' />
-                          <span>{ModelFeatureTextEnum.document}</span>
-                        </ModelBadge>
-                      )}
+                {[ModelTypeEnum.textGeneration, ModelTypeEnum.textEmbedding, ModelTypeEnum.rerank].includes(modelItem.model_type as ModelTypeEnum)
+                  && modelItem.features?.some(feature => [ModelFeatureEnum.vision, ModelFeatureEnum.audio, ModelFeatureEnum.video, ModelFeatureEnum.document].includes(feature))
+                  && (
+                    <div className='pt-2'>
+                      <div className='system-2xs-medium-uppercase mb-1 text-text-tertiary'>{t('common.model.capabilities')}</div>
+                      <div className='flex flex-wrap gap-1'>
+                        {modelItem.features?.map(feature => (
+                          <FeatureIcon
+                            key={feature}
+                            feature={feature}
+                            showFeaturesLabel
+                          />
+                        ))
+                        }
+                      </div>
                     </div>
-                  </div>
-                )}
+                  )}
               </div>
             }
           >

web/app/components/header/account-setting/model-provider-page/provider-added-card/model-list-item.tsx

@@ -62,6 +62,8 @@ const ModelListItem = ({ model, provider, isConfigurable, onModifyLoadBalancing
         showModelType
         showMode
         showContextSize
+        showFeatures
+        showFeaturesLabel
       >
       </ModelName>
       <div className='flex shrink-0 items-center'>