import ollama from "ollama"; import blessed from "blessed"; import { execSync } from "child_process"; import toml from "toml"; import fs from "fs"; import path from "path"; import NodeWebcam from "node-webcam"; import sharp from "sharp"; if (!fs.existsSync("./config.toml")) { if (fs.existsSync("./config.example.toml")) { fs.copyFileSync("./config.example.toml", "./config.toml"); } else { console.error( "error: config.example.toml not found. git pull from repository", ); process.exit(1); } } const config = toml.parse(fs.readFileSync("./config.toml", "utf-8")); let assistantname = config.assistant.name; let assistantface = config.assistant.assistantface; let assistantmodel = config.assistant.model; let maxtokens = config.advanced.max_tokens; let temperature = config.advanced.temperature; let username = config.user.name; let facefont = config.appearance.facefont; // camera settings let cameraWidth = config.camera?.width || 1280; let cameraHeight = config.camera?.height || 720; let cameraQuality = config.camera?.quality || 100; let cameraDevice = config.camera?.device || false; let systemprompt = config.assistant.system_prompt .replace("${name}", assistantname) .replace("${username}", username); const screen = blessed.screen({ smartCSR: true, title: assistantname, }); const faceBox = blessed.box({ top: 0, left: 1, width: "100%", height: 9, scrollable: false, mouse: true, keys: true, vi: true, tags: true, content: "", }); const chatBox = blessed.box({ top: 9, left: 1, width: "100%", bottom: 3, scrollable: true, alwaysScroll: true, mouse: true, keys: true, vi: true, tags: true, content: `go on, tell ${assistantname} something! esc to tab out of chatbox.`, }); const inputBox = blessed.textbox({ bottom: 0, left: 0, width: "100%", height: 3, border: { type: "line", }, style: { border: { fg: "cyan", }, }, inputOnFocus: true, keys: true, mouse: true, placeholder: `go on, tell ${assistantname} something!`, }); const menuBox = blessed.list({ top: "center", left: "center", width: 60, height: 16, border: { type: "line", }, style: { border: { fg: "cyan", }, bg: "black", selected: { bg: "cyan", fg: "black", }, item: { fg: "white", }, }, keys: true, vi: true, mouse: true, hidden: true, label: ` ${assistantname} menu `, items: ["help & commands", "send picture", "take webcam snapshot", `exit ${assistantname}`], }); const popup = blessed.box({ parent: screen, top: "center", left: "center", width: 40, height: 8, border: { type: "line", }, style: { border: { fg: "cyan", }, bg: "black", }, tags: true, hidden: true, content: "", }); const popupButton = blessed.button({ parent: popup, bottom: 1, left: "center", width: 8, height: 1, content: "OK", // < have no clue how to center this style: { bg: "cyan", fg: "black", focus: { bg: "white", fg: "black", }, }, mouse: true, keys: true, }); screen.append(faceBox); screen.append(chatBox); screen.append(inputBox); screen.append(menuBox); screen.append(popup); inputBox.focus(); let chatHistory = []; let conversationHistory = []; let currentStreamMessage = ""; let menuVisible = false; function addMessage(role, content) { let message; if (role === username) { const padding = Math.max(0, chatBox.width - content.length - 5); message = " ".repeat(padding) + content + " <\n"; } else { message = `> ${assistantname}: ${content}\n`; } chatHistory.push(message); chatBox.setContent(chatHistory.join("")); chatBox.setScrollPerc(100); screen.render(); } function updateStreamMessage(content) { currentStreamMessage = content; // handle the kaomoji let letterIndex = -1; for (let i = 0; i < content.length - 1; i++) { if (/[a-zA-Z]/.test(content[i])) { if (content[i] === "I") { letterIndex = i; break; } if (/[a-zA-Z]/.test(content[i + 1])) { letterIndex = i; break; } } } if (letterIndex !== -1) { const faceContent = content.substring(0, letterIndex).trim(); const messageContent = content.substring(letterIndex).trim(); // only update face if kaomoji is 3+ characters because sometimes the model wont listen if (faceContent.length >= 3) { setFaceBoxContent(faceContent); } const tempHistory = [...chatHistory]; tempHistory.push(`> ${assistantname}: ${messageContent}\n`); chatBox.setContent(tempHistory.join("")); chatBox.setScrollPerc(100); screen.render(); } else { // if no letters found yet, just show the entire thing const tempHistory = [...chatHistory]; tempHistory.push(`> ${assistantname}: ${content}\n`); chatBox.setContent(tempHistory.join("")); chatBox.setScrollPerc(100); screen.render(); } } function finalizeStreamMessage() { if (currentStreamMessage) { let letterIndex = -1; for (let i = 0; i < currentStreamMessage.length - 1; i++) { if (/[a-zA-Z]/.test(currentStreamMessage[i])) { // complicated kaomoji splitting garboleum if (currentStreamMessage[i] === "I") { letterIndex = i; break; } if (/[a-zA-Z]/.test(currentStreamMessage[i + 1])) { letterIndex = i; break; } } } let assistantMessage; if (letterIndex !== -1) { assistantMessage = currentStreamMessage.substring(letterIndex).trim(); addMessage(assistantname, assistantMessage); } else { assistantMessage = currentStreamMessage; addMessage(assistantname, currentStreamMessage); } conversationHistory.push({ role: "assistant", content: assistantMessage, }); currentStreamMessage = ""; } } function setFaceBoxContent(content) { try { const figletOutput = execSync(`figlet -f ${facefont} "${content}"`, { encoding: "utf8", }); faceBox.setContent(figletOutput); } catch (err) { faceBox.setContent(content); } screen.render(); } function clearChatHistory() { chatHistory = []; conversationHistory = []; chatBox.setContent(""); screen.render(); } // webcam configuration const webcamOptions = { width: cameraWidth, height: cameraHeight, quality: cameraQuality, delay: 0, saveShots: true, output: "jpeg", device: cameraDevice, callbackReturn: "location" }; const webcam = NodeWebcam.create(webcamOptions); async function convertImageToBase64(imagePath) { try { // resize image to reasonable size for vision models const buffer = await sharp(imagePath) .resize(800, 600, { fit: 'inside', withoutEnlargement: true }) .jpeg({ quality: 80 }) .toBuffer(); return buffer.toString('base64'); } catch (error) { throw new Error(`Failed to process image: ${error.message}`); } } async function sendImageFile() { return new Promise((resolve) => { // Create a simple input box for file path const fileInput = blessed.textbox({ parent: screen, top: 'center', left: 'center', width: '80%', height: 5, border: { type: 'line' }, style: { border: { fg: 'cyan' }, bg: 'black' }, inputOnFocus: true, keys: true, mouse: true, label: ' Enter Image File Path (ESC to cancel) ', placeholder: 'Enter full path to image file...' }); screen.append(fileInput); fileInput.focus(); screen.render(); fileInput.on('submit', async (filePath) => { fileInput.destroy(); if (!filePath || !filePath.trim()) { resolve(); return; } const trimmedPath = filePath.trim(); // Check if file exists if (!fs.existsSync(trimmedPath)) { showPopup('Error: File does not exist'); resolve(); return; } const ext = path.extname(trimmedPath).toLowerCase(); const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp']; if (!imageExtensions.includes(ext)) { showPopup('Error: Please select a valid image file\n(.jpg, .png, .gif, .bmp, .webp)'); resolve(); return; } try { const base64Image = await convertImageToBase64(trimmedPath); await sendMessageWithImage("Here's an image I'd like you to look at:", base64Image); addMessage(username, `[Sent image: ${path.basename(trimmedPath)}]`); } catch (error) { showPopup(`Error processing image: ${error.message}`); } resolve(); }); fileInput.on('cancel', () => { fileInput.destroy(); resolve(); }); screen.key(['escape'], () => { if (!fileInput.destroyed) { fileInput.destroy(); resolve(); } }); }); } async function takeWebcamSnapshot() { return new Promise((resolve) => { showPopup('Taking webcam snapshot...\nPlease wait...'); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const filename = `webcam-${timestamp}.jpg`; const filepath = path.join(process.cwd(), filename); webcam.capture(filename, async (err, data) => { popup.hide(); if (err) { let errorMsg = 'Webcam error occurred'; if (err.message) { errorMsg = `Webcam error: ${err.message}`; } if (err.message && err.message.includes('No such file or directory')) { errorMsg += '\n\nTip: Make sure you have a webcam connected\nand try installing fswebcam or imagesnap'; } showPopup(errorMsg); resolve(); return; } try { // check if file exists with a small delay await new Promise(r => setTimeout(r, 500)); if (!fs.existsSync(filepath)) { showPopup('Error: Snapshot file was not created\nCheck if webcam is connected and accessible'); resolve(); return; } const base64Image = await convertImageToBase64(filepath); await sendMessageWithImage("I just took this webcam snapshot:", base64Image); addMessage(username, `[Took webcam snapshot: ${filename}]`); // clean up the temporary file try { fs.unlinkSync(filepath); } catch (unlinkErr) { // ignore cleanup errors } } catch (error) { showPopup(`Error processing snapshot: ${error.message}`); // try to clean up file even if there was an error try { if (fs.existsSync(filepath)) { fs.unlinkSync(filepath); } } catch (unlinkErr) { // ignore cleanup errors } } resolve(); }); }); } function showPopup(message) { popup.setContent(`{center}${message}{/center}`); popup.show(); popupButton.focus(); screen.render(); } function showMenu() { menuVisible = true; menuBox.show(); menuBox.focus(); screen.render(); } function hideMenu() { menuVisible = false; menuBox.hide(); inputBox.focus(); screen.render(); } async function handleMenuSelection() { const selectedIndex = menuBox.selected; hideMenu(); switch (selectedIndex) { case 0: addMessage( assistantname, "available commands:\nl!help - if you wanna know what i can do, run this!\nl!clear - clear chat history, if you want me to forget everything, just run this!\nl!face - if you want to force my expression, here you go! not sure i'll be too happy about it though.\nl!prompt - if you want to change how i act, here you go! not sure i'll be too happy about that either.\nl!image or l!img - send an image file (or just l!image to browse)\nl!webcam or l!cam - take and send a webcam snapshot\n\nMenu shortcuts:\nESC - open/close menu\nUp/Down arrows - navigate menu\nEnter - select option\nCtrl+C or q - quit application\n\nMenu options:\n- help & commands - show this help\n- send picture - browse and send an image file\n- take webcam snapshot - capture and send a webcam photo\n- exit - quit the application\n\nNote: Image features require a vision-capable model like llava or bakllava in Ollama!\n", ); break; case 1: await sendImageFile(); break; case 2: await takeWebcamSnapshot(); break; case 3: process.exit(0); break; } } async function sendMessage(message) { if (!message.trim()) return; addMessage(username, message); conversationHistory.push({ role: "user", content: message, }); try { currentStreamMessage = ""; const response = await ollama.chat({ model: assistantmodel, messages: [ { role: "system", content: systemprompt, }, ...conversationHistory, ], stream: true, options: { num_predict: maxtokens, temperature: temperature, }, }); for await (const part of response) { if (part.message && part.message.content) { currentStreamMessage += part.message.content; updateStreamMessage(currentStreamMessage); } } finalizeStreamMessage(); } catch (error) { addMessage(assistantname, `Failed to get response: ${error.message}`); } } async function sendMessageWithImage(message, base64Image) { if (!message.trim()) return; conversationHistory.push({ role: "user", content: message, images: [base64Image] }); try { currentStreamMessage = ""; const response = await ollama.chat({ model: assistantmodel, messages: [ { role: "system", content: systemprompt, }, ...conversationHistory, ], stream: true, options: { num_predict: maxtokens, temperature: temperature, }, }); for await (const part of response) { if (part.message && part.message.content) { currentStreamMessage += part.message.content; updateStreamMessage(currentStreamMessage); } } finalizeStreamMessage(); } catch (error) { addMessage(assistantname, `Failed to get response: ${error.message}`); } } inputBox.on("submit", async (text) => { if (text.trim()) { inputBox.clearValue(); inputBox.focus(); // command trash if (text.trim().startsWith("l!")) { const commandParts = text.trim().slice(2).split(" "); const command = commandParts[0]; const args = commandParts.slice(1).join(" "); switch (command) { case "help": addMessage( assistantname, "available commands:\nl!help - if you wanna know what i can do, run this!\nl!clear - clear chat history, if you want me to forget everything, just run this!\nl!face - if you want to force my expression, here you go! not sure i'll be too happy about it though.\nl!prompt - if you want to change how i act, here you go! not sure i'll be too happy about that either.\nl!image or l!img - send an image file (or just l!image to browse)\nl!webcam or l!cam - take and send a webcam snapshot\n\nMenu shortcuts:\nESC - open/close menu\nUp/Down arrows - navigate menu\nEnter - select option\n\nMenu options:\n- help & commands - show this help\n- send picture - browse and send an image file\n- take webcam snapshot - capture and send a webcam photo\n- exit - quit the application\n", ); break; case "clear": clearChatHistory(); break; case "face": if (args.trim()) { setFaceBoxContent(args.trim()); } else { setFaceBoxContent("=w="); } break; case "prompt": if (args.trim()) { systemprompt = args.trim(); clearChatHistory(); addMessage("system", "SYSTEM: Prompt set."); } else { addMessage("system", "SYSTEM: You didn't provide a prompt."); } break; case "image": case "img": if (args.trim()) { const imagePath = args.trim(); if (!fs.existsSync(imagePath)) { addMessage("system", "SYSTEM: Image file not found."); break; } const ext = path.extname(imagePath).toLowerCase(); const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp']; if (!imageExtensions.includes(ext)) { addMessage("system", "SYSTEM: Invalid image file format."); break; } try { const base64Image = await convertImageToBase64(imagePath); await sendMessageWithImage("Here's an image I'd like you to look at:", base64Image); addMessage(username, `[Sent image: ${path.basename(imagePath)}]`); } catch (error) { addMessage("system", `SYSTEM: Error processing image: ${error.message}`); } } else { await sendImageFile(); } break; case "webcam": case "cam": await takeWebcamSnapshot(); break; default: addMessage(assistantname, `unknown command: ${command}`); break; } } else { // message just send it await sendMessage(text); } } }); screen.key(["q", "C-c"], () => { process.exit(0); // KILL lydia!!!!!!!!!!!!!!!!!!!!!!! }); screen.key(["escape"], () => { if (menuVisible) { hideMenu(); } else { showMenu(); } }); menuBox.on("select", async (item, selected) => { if (menuVisible) { await handleMenuSelection(); } }); popupButton.on("press", () => { popup.hide(); inputBox.focus(); screen.render(); }); screen.key(["enter"], async () => { if (!popup.hidden) { popup.hide(); inputBox.focus(); screen.render(); } else if (menuVisible) { await handleMenuSelection(); } }); screen.on("resize", () => { screen.render(); }); setFaceBoxContent(assistantface);