feat(chat): Support Youtube transcripts
This commit is contained in:
+219
-25
@@ -74,6 +74,49 @@
|
||||
overflow-y: auto;
|
||||
padding-right: 0.5rem;
|
||||
}
|
||||
.attachment-card {
|
||||
max-width: 85%;
|
||||
margin: 1rem 0;
|
||||
padding: 0.8rem 1.2rem;
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: 0.8rem;
|
||||
background: var(--bg-secondary);
|
||||
}
|
||||
.attachment-card__title {
|
||||
font-weight: 700;
|
||||
}
|
||||
.attachment-card__meta {
|
||||
color: var(--text-secondary);
|
||||
font-size: 0.9em;
|
||||
}
|
||||
.attachment-card summary {
|
||||
cursor: pointer;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
.attachment-card pre {
|
||||
max-height: 14rem;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
padding: 0.6rem;
|
||||
background: var(--bg-code);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: 0.3rem;
|
||||
}
|
||||
.suggested-questions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.5rem;
|
||||
margin: 0.75rem 0;
|
||||
}
|
||||
.suggested-questions button {
|
||||
cursor: pointer;
|
||||
padding: 0.35rem 0.6rem;
|
||||
color: var(--text-primary);
|
||||
background: var(--bg-primary);
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: 0.3rem;
|
||||
}
|
||||
.message {
|
||||
padding: 0.8rem 1.2rem;
|
||||
border-radius: 0.8rem;
|
||||
@@ -308,6 +351,21 @@
|
||||
<div class="chat-container">
|
||||
<!-- Messages -->
|
||||
<div class="messages" x-ref="msgBox" @click="onClickMessage" @scroll="handleScroll">
|
||||
<section class="attachment-card" x-show="attachment">
|
||||
<div class="attachment-card__title">Transcript attached</div>
|
||||
<div class="attachment-card__meta" x-text="`${attachment?.length.toLocaleString()} characters`"></div>
|
||||
<details>
|
||||
<summary>Show transcript</summary>
|
||||
<pre x-text="attachment?.value"></pre>
|
||||
</details>
|
||||
<template x-if="visibleMessages.length === 0 && !isBusy">
|
||||
<div class="suggested-questions">
|
||||
<button @click="useSuggestedQuestion('What are the key points?')">Key points</button>
|
||||
<button @click="useSuggestedQuestion('What are the most important quotes?')">Notable quotes</button>
|
||||
<button @click="useSuggestedQuestion('What should I challenge or verify?')">What to verify</button>
|
||||
</div>
|
||||
</template>
|
||||
</section>
|
||||
<template x-for="(msg, idx) in visibleMessages" :key="idx">
|
||||
<div :class="msg.role === 'user' ? 'message user-message' : 'message system-message'">
|
||||
<template x-if="msg.role === 'user'">
|
||||
@@ -329,7 +387,7 @@
|
||||
|
||||
<!-- Textarea -->
|
||||
<div class="prompt-wrapper">
|
||||
<div x-show="pendingImages.length > 0" style="display: flex; gap: 0.5rem; flex-wrap: wrap; align-items: flex-start;">
|
||||
<div x-show="!attachment && pendingImages.length > 0" style="display: flex; gap: 0.5rem; flex-wrap: wrap; align-items: flex-start;">
|
||||
<template x-for="(img, i) in pendingImages" :key="img.dataUrl">
|
||||
<div style="position: relative; display: inline-block;">
|
||||
<img :src="img.dataUrl" style="max-height: 72px; max-width: 120px; object-fit: cover; border-radius: 0.3rem; display: block;" />
|
||||
@@ -353,7 +411,7 @@
|
||||
<button class="submit" :class="isBusy && 'busy'" @click="isBusy ? abort() : handleSubmit()" x-text="isBusy ? 'Cancel' : 'Send'"></button>
|
||||
|
||||
<!-- Model Selector -->
|
||||
<details id="model-selector">
|
||||
<details id="model-selector" x-show="!attachment">
|
||||
<summary x-text="'Model: ' + activeModelName"></summary>
|
||||
<template x-for="m in models" :key="m.name">
|
||||
<label class="model-opt__label" style="display: block">
|
||||
@@ -364,7 +422,7 @@
|
||||
</details>
|
||||
|
||||
<!-- History -->
|
||||
<details id="history-panel" x-ref="historyPanel">
|
||||
<details id="history-panel" x-ref="historyPanel" x-show="!attachment">
|
||||
<summary>History</summary>
|
||||
<div class="history-list">
|
||||
<template x-if="historyItems.length === 0">
|
||||
@@ -399,10 +457,60 @@
|
||||
.replace(/^\s+/gm, "")
|
||||
.trim();
|
||||
|
||||
const defaultVideoSummaryPrompt = "Summarize this video or transcript. Cover the key points, necessary context, and notable quotes. Skip promotions.";
|
||||
|
||||
function getTranscriptAttachment(transcript) {
|
||||
const value = typeof transcript === "string" ? transcript.trim() : "";
|
||||
if (value) return { kind: "text", value, length: value.length };
|
||||
return null;
|
||||
}
|
||||
|
||||
function getSystemInstruction(attachment) {
|
||||
if (!attachment) return systemPrompt;
|
||||
|
||||
const attachmentRules = `
|
||||
You are answering questions about the attached transcript.
|
||||
Base your answers on that source. State when the source does not establish an answer, do not invent details, and preserve exact wording when the user asks for quotes.
|
||||
`
|
||||
.replace(/^\s+/gm, "")
|
||||
.trim();
|
||||
|
||||
return `${attachmentRules}\n\nTranscript:\n${attachment.value}`;
|
||||
}
|
||||
|
||||
const initialAttachment = getTranscriptAttachment(window.env.YOUTUBE_TRANSCRIPT);
|
||||
const initialPromptText = window.env.PROMPT ? `${window.env.PROMPT}\n\n` : "";
|
||||
const geminiCachePromises = new Map();
|
||||
|
||||
async function getGeminiAttachmentCache({ model, apiKey, systemInstruction, attachment }) {
|
||||
if (!attachment) return null;
|
||||
const key = `${model}:${attachment.kind}:${attachment.value}`;
|
||||
if (!geminiCachePromises.has(key)) {
|
||||
geminiCachePromises.set(
|
||||
key,
|
||||
fetch(`https://generativelanguage.googleapis.com/v1beta/cachedContents?key=${apiKey}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: `models/${model}`,
|
||||
systemInstruction: { parts: [{ text: systemInstruction }] },
|
||||
contents: [],
|
||||
ttl: "3600s",
|
||||
}),
|
||||
})
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.then((cache) => cache?.name || null)
|
||||
.catch(() => null)
|
||||
);
|
||||
}
|
||||
return geminiCachePromises.get(key);
|
||||
}
|
||||
|
||||
const chatApp = () => ({
|
||||
messages: [{ role: "system", content: systemPrompt }],
|
||||
pendingImages: [],
|
||||
promptText: window.env.PROMPT ? `${window.env.PROMPT}\n\n` : "",
|
||||
attachment: initialAttachment,
|
||||
promptText: initialPromptText,
|
||||
activeModelName: "gemini-3.5-flash-lite",
|
||||
historyItems: [],
|
||||
isBusy: false,
|
||||
@@ -418,8 +526,11 @@
|
||||
|
||||
async init() {
|
||||
this.sessionId = +new Date();
|
||||
this.historyItems = loadHistory();
|
||||
this.historyItems = this.attachment ? [] : loadHistory();
|
||||
setTimeout(() => this.$refs.promptInput.focus(), 0);
|
||||
if (this.attachment && window.env.AUTO_SUBMIT === true) {
|
||||
setTimeout(() => this.useSuggestedQuestion(this.promptText.trim() || defaultVideoSummaryPrompt, true), 0);
|
||||
}
|
||||
},
|
||||
|
||||
async onClickMessage(e) {
|
||||
@@ -459,6 +570,12 @@
|
||||
getExcerpt: (msgs) => msgs.find((m) => m.role === "user")?.content.slice(0, 30) || "Empty chat",
|
||||
|
||||
renderMarkdown,
|
||||
useSuggestedQuestion(question, submit = false) {
|
||||
if (this.isBusy) return;
|
||||
this.promptText = question;
|
||||
this.$nextTick(() => this.$refs.promptInput.focus());
|
||||
if (submit) this.handleSubmit();
|
||||
},
|
||||
async handleSubmit() {
|
||||
const text = this.promptText.trim();
|
||||
if ((!text && !this.pendingImages.length) || this.isBusy) return;
|
||||
@@ -477,11 +594,13 @@
|
||||
this.scrollToBottom();
|
||||
|
||||
try {
|
||||
const modelObj = this.models.find((m) => m.name === this.activeModelName);
|
||||
const modelObj = this.models.find((m) => m.name === (this.attachment ? "gemini-3.5-flash-lite" : this.activeModelName));
|
||||
await modelObj.chat({
|
||||
model: this.activeModelName,
|
||||
model: modelObj.name,
|
||||
apiKey: modelObj.key,
|
||||
messages: this.messages,
|
||||
attachment: this.attachment,
|
||||
systemInstruction: getSystemInstruction(this.attachment),
|
||||
signal: this.abortController.signal,
|
||||
onContent: (chunk) => {
|
||||
fullResponse += chunk;
|
||||
@@ -505,7 +624,7 @@
|
||||
} finally {
|
||||
this.isBusy = false;
|
||||
this.abortController = null;
|
||||
updateSessionHistory(this.sessionId, this.messages);
|
||||
if (!this.attachment) updateSessionHistory(this.sessionId, this.messages);
|
||||
}
|
||||
},
|
||||
|
||||
@@ -514,6 +633,7 @@
|
||||
},
|
||||
|
||||
handlePaste(e) {
|
||||
if (this.attachment) return;
|
||||
const items = [...(e.clipboardData?.items || [])];
|
||||
const imageItems = items.filter((item) => item.type.startsWith("image/"));
|
||||
if (!imageItems.length) return;
|
||||
@@ -533,13 +653,14 @@
|
||||
const idx = this.messages.indexOf(msg);
|
||||
if (idx === -1) return;
|
||||
this.messages = this.messages.slice(0, idx);
|
||||
updateSessionHistory(this.sessionId, this.messages);
|
||||
if (!this.attachment) updateSessionHistory(this.sessionId, this.messages);
|
||||
this.promptText = msg.content;
|
||||
this.pendingImages = msg.images ? JSON.parse(JSON.stringify(msg.images)) : [];
|
||||
this.$nextTick(() => this.$refs.promptInput.focus());
|
||||
},
|
||||
|
||||
restoreConversation(msgs) {
|
||||
if (this.attachment) return;
|
||||
this.messages = JSON.parse(JSON.stringify(msgs));
|
||||
this.userHasScrolledUp = false;
|
||||
this.$refs.historyPanel.removeAttribute("open");
|
||||
@@ -547,6 +668,7 @@
|
||||
},
|
||||
|
||||
clearHistory() {
|
||||
if (this.attachment) return;
|
||||
if (!confirm("Clear all saved conversations?")) return;
|
||||
this.historyItems = clearAllHistory();
|
||||
this.$refs.historyPanel.removeAttribute("open");
|
||||
@@ -572,15 +694,28 @@
|
||||
return $placeholder.innerHTML;
|
||||
}
|
||||
|
||||
async function chatWithOpenAI({ baseUrl = "https://api.openai.com/v1", onContent, onError, messages, signal, apiKey, model, extraHeaders = {}, ...rest }) {
|
||||
async function chatWithOpenAI({ baseUrl = "https://api.openai.com/v1", onContent, onError, messages, signal, apiKey, model, extraHeaders = {}, systemInstruction, attachment, ...rest }) {
|
||||
const lastUserIndex = messages.map((message) => message.role).lastIndexOf("user");
|
||||
const apiMessages = messages.map(({ role, content, images }, index) => {
|
||||
const textPart = {
|
||||
type: "text",
|
||||
text: content,
|
||||
...(index === lastUserIndex ? { prompt_cache_breakpoint: { mode: "explicit" } } : {}),
|
||||
};
|
||||
const parts = [textPart, ...(images || []).map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } }))];
|
||||
return role === "system" || images?.length || index === lastUserIndex ? { role, content: parts } : { role, content };
|
||||
});
|
||||
const response = await fetch(`${baseUrl}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, ...extraHeaders },
|
||||
body: JSON.stringify({ model, messages: messages.map(({ role, content, images }) =>
|
||||
images?.length
|
||||
? { role, content: [{ type: "text", text: content }, ...images.map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } }))] }
|
||||
: { role, content }
|
||||
), stream: true, ...rest }),
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: apiMessages,
|
||||
prompt_cache_key: "chat:index:v1",
|
||||
prompt_cache_options: { mode: "explicit" },
|
||||
stream: true,
|
||||
...rest,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
@@ -609,25 +744,83 @@
|
||||
}
|
||||
|
||||
async function chatWithClaude(args) {
|
||||
return chatWithOpenAI({
|
||||
...args,
|
||||
baseUrl: "https://api.anthropic.com/v1",
|
||||
extraHeaders: {
|
||||
"x-api-key": args.apiKey,
|
||||
const { onContent, onError, messages, signal, apiKey, model } = args;
|
||||
const system = messages
|
||||
.filter((message) => message.role === "system")
|
||||
.map((message) => ({ type: "text", text: message.content }))
|
||||
.filter((part) => part.text);
|
||||
const conversation = messages
|
||||
.filter((message) => message.role !== "system")
|
||||
.map((message) => {
|
||||
const images = (message.images || []).map((image) => ({
|
||||
type: "image",
|
||||
source: { type: "base64", media_type: image.mimeType, data: image.base64 },
|
||||
}));
|
||||
return {
|
||||
role: message.role,
|
||||
content: images.length ? [{ type: "text", text: message.content }, ...images] : message.content,
|
||||
};
|
||||
});
|
||||
|
||||
const response = await fetch("https://api.anthropic.com/v1/messages", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"x-api-key": apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-dangerous-direct-browser-access": "true",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
max_tokens: 4096,
|
||||
system,
|
||||
messages: conversation,
|
||||
cache_control: { type: "ephemeral" },
|
||||
stream: true,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
await onError?.(new Error(`Anthropic API Error ${response.status}: ${text.trim()}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const events = buffer.split("\n\n");
|
||||
buffer = events.pop();
|
||||
for (const event of events) {
|
||||
const data = event
|
||||
.split("\n")
|
||||
.find((line) => line.startsWith("data:"))
|
||||
?.slice(5)
|
||||
.trim();
|
||||
if (!data) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(data);
|
||||
if (parsed.type === "content_block_delta" && parsed.delta?.type === "text_delta") await onContent(parsed.delta.text);
|
||||
} catch (error) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function chatWithGemini({ onContent, onError, messages, signal, apiKey, model, ...rest }) {
|
||||
const contents = messages.map((msg, i, arr) => {
|
||||
async function chatWithGemini({ onContent, onError, messages, signal, apiKey, model, systemInstruction, attachment, ...rest }) {
|
||||
const conversation = messages.filter((msg) => msg.role !== "system");
|
||||
const cachedContent = await getGeminiAttachmentCache({ model, apiKey, systemInstruction, attachment });
|
||||
const contents = conversation.map((msg, i) => {
|
||||
const role = msg.role === "assistant" ? "model" : "user";
|
||||
let parts = [{ text: msg.content }];
|
||||
|
||||
// Add YouTube URLs as file data for the final user message
|
||||
const isLastMessage = i === arr.length - 1;
|
||||
if (isLastMessage && role === "user") {
|
||||
// Preserve the existing ability to send a YouTube URL in a normal chat message.
|
||||
const isLastMessage = i === conversation.length - 1;
|
||||
if (!attachment && isLastMessage && role === "user") {
|
||||
const youtubeUrlPattern = /https?:\/\/(www\.)?youtube\.com\/watch\?v=[\w-]+|https?:\/\/youtu\.be\/[\w-]+/g;
|
||||
const youtubeUrls = msg.content.match(youtubeUrlPattern) || [];
|
||||
const fileParts = youtubeUrls.map((url) => ({ file_data: { file_uri: url } }));
|
||||
@@ -650,6 +843,7 @@
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
...(cachedContent ? { cachedContent } : systemInstruction ? { system_instruction: { parts: [{ text: systemInstruction }] } } : {}),
|
||||
contents,
|
||||
generationConfig: rest,
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user