feat(chat): add vision input with drag-and-drop & paste images (M9.4)

Drag-and-drop or paste images into chat input. Thumbnail preview above input, max 4 images per message. Images encoded as base64 and sent in Claude vision format (multimodal content arrays). Image attach button in chat input toolbar. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-03 23:25:26 +00:00
parent 0ae497f1db
commit 27aeb2b188
5 changed files with 213 additions and 16 deletions
@@ -1,5 +1,10 @@
 <template>
-  <div class="p-3 md:p-4 relative">
+  <div
+    class="p-3 md:p-4 relative"
+    @dragover.prevent="onDragOver"
+    @dragleave="onDragLeave"
+    @drop.prevent="onDrop"
+  >
    <SearchResults
      v-if="isSearchMode"
      :results="searchResults"
@@ -27,10 +32,57 @@
      </button>
    </div>

+    <!-- Image thumbnails -->
+    <div v-if="images.length > 0" class="mb-2 flex gap-2 flex-wrap animate-fade-up-fast">
+      <div
+        v-for="(img, i) in images"
+        :key="i"
+        class="relative group w-16 h-16 rounded-lg overflow-hidden border border-white/10 bg-white/5"
+      >
+        <img
+          :src="`data:${img.mediaType};base64,${img.data}`"
+          :alt="`Attached image ${i + 1}`"
+          class="w-full h-full object-cover"
+        />
+        <button
+          class="absolute inset-0 flex items-center justify-center bg-black/60 opacity-0 group-hover:opacity-100 transition-opacity"
+          aria-label="Remove image"
+          @click="removeImage(i)"
+        >
+          <svg class="w-4 h-4 text-white/80" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
+          </svg>
+        </button>
+      </div>
+      <span v-if="images.length >= MAX_IMAGES" class="self-center text-[10px] text-white/30">
+        Max {{ MAX_IMAGES }} images
+      </span>
+    </div>
+
+    <!-- Drag overlay -->
+    <div
+      v-if="isDragging"
+      class="absolute inset-0 z-10 flex items-center justify-center rounded-2xl border-2 border-dashed border-accent/50 bg-accent/5 backdrop-blur-sm pointer-events-none"
+    >
+      <p class="text-sm text-accent/80 font-medium">Drop image here</p>
+    </div>
+
    <div
      class="path-glass-bubble rounded-2xl px-4 py-3 flex items-end gap-2 transition-all duration-300"
      :class="focused ? 'border-white/30' : ''"
    >
+      <!-- Image attach button -->
+      <button
+        v-if="!streaming && images.length < MAX_IMAGES"
+        class="shrink-0 w-8 h-8 flex items-center justify-center rounded-lg text-white/40 hover:text-white/70 hover:bg-white/10 transition-all"
+        aria-label="Attach image"
+        @click="openFilePicker"
+      >
+        <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
+        </svg>
+      </button>
+
      <textarea
        ref="textareaRef"
        v-model="text"
@@ -83,6 +135,16 @@
        </button>
      </template>
    </div>
+
+    <!-- Hidden file input -->
+    <input
+      ref="fileInputRef"
+      type="file"
+      accept="image/jpeg,image/png,image/gif,image/webp"
+      multiple
+      class="hidden"
+      @change="onFileSelect"
+    />
  </div>
 </template>

@@ -90,6 +152,10 @@
 import { ref, computed, nextTick, watch } from 'vue'
 import { useFederatedSearch, type SearchResult } from '@/composables/useFederatedSearch'
 import SearchResults from '@/components/ui/SearchResults.vue'
+import type { ImageAttachment } from '@aiui/core/types/message'
+
+const MAX_IMAGES = 4
+const ACCEPTED_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']

 const props = withDefaults(
  defineProps<{
@@ -107,7 +173,7 @@ const props = withDefaults(
 )

 const emit = defineEmits<{
-  send: [text: string]
+  send: [text: string, images: ImageAttachment[]]
  extract: [text: string]
  stop: []
  clearReply: []
@@ -116,14 +182,18 @@ const emit = defineEmits<{
 const text = ref('')
 const focused = ref(false)
 const hasPasted = ref(false)
+const isDragging = ref(false)
 const textareaRef = ref<HTMLTextAreaElement | null>(null)
+const fileInputRef = ref<HTMLInputElement | null>(null)
+const images = ref<ImageAttachment[]>([])

-const canSend = computed(() => text.value.trim().length > 0 && !props.disabled)
+const canSend = computed(() => (text.value.trim().length > 0 || images.value.length > 0) && !props.disabled)

 function send() {
  if (!canSend.value) return
-  emit('send', text.value.trim())
+  emit('send', text.value.trim(), [...images.value])
  text.value = ''
+  images.value = []
  hasPasted.value = false
  nextTick(autoResize)
 }
@@ -136,8 +206,82 @@ function extract() {
  nextTick(autoResize)
 }

-function onPaste() {
-  hasPasted.value = true
+function onPaste(e: ClipboardEvent) {
+  const items = e.clipboardData?.items
+  if (!items) {
+    hasPasted.value = true
+    return
+  }
+
+  let hasImage = false
+  for (const item of items) {
+    if (ACCEPTED_TYPES.includes(item.type)) {
+      hasImage = true
+      const file = item.getAsFile()
+      if (file) addImageFile(file)
+    }
+  }
+
+  if (!hasImage) {
+    hasPasted.value = true
+  }
+}
+
+function onDragOver(e: DragEvent) {
+  if (e.dataTransfer?.types.includes('Files')) {
+    isDragging.value = true
+  }
+}
+
+function onDragLeave() {
+  isDragging.value = false
+}
+
+function onDrop(e: DragEvent) {
+  isDragging.value = false
+  const files = e.dataTransfer?.files
+  if (!files) return
+  for (const file of files) {
+    if (ACCEPTED_TYPES.includes(file.type)) {
+      addImageFile(file)
+    }
+  }
+}
+
+function openFilePicker() {
+  fileInputRef.value?.click()
+}
+
+function onFileSelect(e: Event) {
+  const input = e.target as HTMLInputElement
+  const files = input.files
+  if (!files) return
+  for (const file of files) {
+    if (ACCEPTED_TYPES.includes(file.type)) {
+      addImageFile(file)
+    }
+  }
+  // Reset input so same file can be re-selected
+  input.value = ''
+}
+
+function addImageFile(file: File) {
+  if (images.value.length >= MAX_IMAGES) return
+
+  const reader = new FileReader()
+  reader.onload = () => {
+    const result = reader.result as string
+    // Strip the data:...;base64, prefix
+    const base64 = result.split(',')[1]
+    if (base64) {
+      images.value.push({ data: base64, mediaType: file.type })
+    }
+  }
+  reader.readAsDataURL(file)
+}
+
+function removeImage(index: number) {
+  images.value.splice(index, 1)
 }

 function autoResize() {
@@ -74,13 +74,23 @@

        <!-- Normal display -->
        <template v-else>
+          <!-- Attached images -->
+          <div v-if="message.images && message.images.length > 0" class="flex gap-2 flex-wrap mb-2">
+            <img
+              v-for="(img, i) in message.images"
+              :key="i"
+              :src="`data:${img.mediaType};base64,${img.data}`"
+              :alt="`Attached image ${i + 1}`"
+              class="rounded-lg max-w-[200px] max-h-[200px] object-cover border border-white/10"
+            />
+          </div>
          <div
            v-if="!isUser"
            class="chat-markdown text-sm leading-relaxed break-words text-white/90"
            v-html="renderedMarkdown"
          />
          <p
-            v-else
+            v-else-if="message.content"
            class="text-sm leading-relaxed whitespace-pre-wrap break-words text-white/90"
          >{{ displayText }}</p>
        </template>
@@ -117,7 +117,7 @@ import ChatSearch from './ChatSearch.vue'
 import ContextBar from './ContextBar.vue'
 import ComparisonView from './ComparisonView.vue'
 import ErrorBoundary from '@/components/ui/ErrorBoundary.vue'
-import type { Message } from '@aiui/core/types/message'
+import type { Message, ImageAttachment } from '@aiui/core/types/message'

 withDefaults(
  defineProps<{
@@ -250,7 +250,7 @@ function handleExtract(text: string) {
  updatePanelFromText(text, '', [])
 }

-async function handleSend(text: string) {
+async function handleSend(text: string, images: ImageAttachment[] = []) {
  // Command handling
  const trimmed = text.trim().toLowerCase()
  if (trimmed === '/code') {
@@ -316,13 +316,13 @@ async function handleSend(text: string) {
  // Comparison mode: stream to both models simultaneously
  if (comparison.isComparing.value) {
    const convId = chatStore.activeConversationId ?? chatStore.createConversation()
-    chatStore.addMessage(convId, { role: 'user', content: finalText })
+    chatStore.addMessage(convId, { role: 'user', content: finalText, images: images.length > 0 ? images : undefined })
    const history = chatStore.messages.map(m => ({ role: m.role, content: m.content }))
    await comparison.streamBothModels(streamWithModel, history)
    return
  }

-  await sendMessage(finalText)
+  await sendMessage(finalText, images.length > 0 ? images : undefined)
 }

 function handlePromptSelect(_userMsg: Message, assistantMsg: Message | null) {
@@ -2,6 +2,7 @@ import { ref, computed } from 'vue'
 import { useChatStore } from '@/stores/chat'
 import { searchWeb } from '@/composables/useWebSearch'
 import { getApiKey } from '@/utils/key-vault'
+import type { ImageAttachment } from '@aiui/core/types/message'

 type Provider = 'claude' | 'openrouter' | 'mock'

@@ -110,6 +111,23 @@ function setModel(model: string) {
 interface ChatMessage {
  role: 'user' | 'assistant'
  content: string
+  images?: ImageAttachment[]
+}
+
+/** Build Claude API content array for a message (multimodal when images present) */
+function buildClaudeContent(msg: ChatMessage): string | Array<Record<string, unknown>> {
+  if (!msg.images || msg.images.length === 0) return msg.content
+  const blocks: Array<Record<string, unknown>> = []
+  for (const img of msg.images) {
+    blocks.push({
+      type: 'image',
+      source: { type: 'base64', media_type: img.mediaType, data: img.data },
+    })
+  }
+  if (msg.content) {
+    blocks.push({ type: 'text', text: msg.content })
+  }
+  return blocks
 }

 async function streamMock(
@@ -145,13 +163,19 @@ async function streamClaude(
    headers['x-api-key'] = vaultKey
  }

+  // Build API messages with multimodal content arrays when images present
+  const apiMessages = messages.map(m => ({
+    role: m.role,
+    content: buildClaudeContent(m),
+  }))
+
  const res = await fetch(CLAUDE_PATH, {
    method: 'POST',
    headers,
    body: JSON.stringify({
      model: activeModel.value,
      system: systemPrompt,
-      messages,
+      messages: apiMessages,
      stream: true,
      webSearch,
    }),
@@ -363,7 +387,7 @@ export function useAI() {
    chatStore.isStreaming = false
  }

-  async function sendMessage(userText: string) {
+  async function sendMessage(userText: string, images?: ImageAttachment[]) {
    const provider = activeProvider.value
    currentAbort = new AbortController()
    const signal = currentAbort.signal
@@ -374,7 +398,11 @@ export function useAI() {
    }
    const cid = convId

-    chatStore.addMessage(cid, { role: 'user', content: userText })
+    chatStore.addMessage(cid, {
+      role: 'user',
+      content: userText,
+      images: images && images.length > 0 ? images : undefined,
+    })
    const assistantMsg = chatStore.addMessage(cid, { role: 'assistant', content: '' })
    if (!assistantMsg) return

@@ -399,7 +427,7 @@ export function useAI() {

    const history: ChatMessage[] = chatStore.messages
      .filter((m) => m.id !== assistantMsg.id)
-      .map((m) => ({ role: m.role as 'user' | 'assistant', content: m.content }))
+      .map((m) => ({ role: m.role as 'user' | 'assistant', content: m.content, images: m.images }))

    const onToken = (text: string) => chatStore.appendToLastMessage(cid, text)
    const onError = (err: string) => {
@@ -510,7 +538,7 @@ export function useAI() {

    const history: ChatMessage[] = chatStore.messages
      .filter((m) => m.id !== assistantMsg.id)
-      .map((m) => ({ role: m.role as 'user' | 'assistant', content: m.content }))
+      .map((m) => ({ role: m.role as 'user' | 'assistant', content: m.content, images: m.images }))

    const onToken = (text: string) => chatStore.appendToLastMessage(cid, text)
    const onError = (err: string) => {
@@ -8,6 +8,13 @@ export interface WebSearchResult {
  imgSrc?: string
 }

+export interface ImageAttachment {
+  /** Base64-encoded image data (no data: prefix) */
+  data: string
+  /** MIME type e.g. image/jpeg, image/png, image/gif, image/webp */
+  mediaType: string
+}
+
 export interface Message {
  id: string
  role: 'user' | 'assistant' | 'system'
@@ -23,6 +30,8 @@ export interface Message {
  webResults?: WebSearchResult[]
  /** Timestamp of last edit (set when user edits a sent message) */
  editedAt?: number
+  /** Attached images (base64, max 4) for vision-capable models */
+  images?: ImageAttachment[]
 }

 export interface Reaction {
@@ -39,4 +48,10 @@ export interface Conversation {
  updatedAt: number
  model?: string
  systemPrompt?: string
+  /** ID of parent conversation this was branched from */
+  parentConversationId?: string
+  /** Message ID in parent where this branch forked */
+  branchPoint?: string
+  /** IDs of child branches forked from this conversation */
+  childBranchIds?: string[]
 }