lydia/lydia.js

import ollama from "ollama";
import blessed from "blessed";
import { execSync } from "child_process";
import toml from "toml";
import fs from "fs";
import path from "path";
import NodeWebcam from "node-webcam";
import sharp from "sharp";

if (!fs.existsSync("./config.toml")) {
  if (fs.existsSync("./config.example.toml")) {
    fs.copyFileSync("./config.example.toml", "./config.toml");
  } else {
    console.error(
      "error: config.example.toml not found. git pull from repository",
    );
    process.exit(1);
  }
}

const config = toml.parse(fs.readFileSync("./config.toml", "utf-8"));

let assistantname = config.assistant.name;
let assistantface = config.assistant.assistantface;
let assistantmodel = config.assistant.model;
let maxtokens = config.advanced.max_tokens;
let temperature = config.advanced.temperature;

let username = config.user.name;

let facefont = config.appearance.facefont;

// camera settings
let cameraWidth = config.camera?.width || 1280;
let cameraHeight = config.camera?.height || 720;
let cameraQuality = config.camera?.quality || 100;
let cameraDevice = config.camera?.device || false;

let systemprompt = config.assistant.system_prompt
  .replace("${name}", assistantname)
  .replace("${username}", username);

const screen = blessed.screen({
  smartCSR: true,
  title: assistantname,
});

const faceBox = blessed.box({
  top: 0,
  left: 1,
  width: "100%",
  height: 9,
  scrollable: false,
  mouse: true,
  keys: true,
  vi: true,
  tags: true,
  content: "",
});

const chatBox = blessed.box({
  top: 9,
  left: 1,
  width: "100%",
  bottom: 3,
  scrollable: true,
  alwaysScroll: true,
  mouse: true,
  keys: true,
  vi: true,
  tags: true,
  content: `go on, tell ${assistantname} something! esc to tab out of chatbox.`,
});

const inputBox = blessed.textbox({
  bottom: 0,
  left: 0,
  width: "100%",
  height: 3,
  border: {
    type: "line",
  },
  style: {
    border: {
      fg: "cyan",
    },
  },
  inputOnFocus: true,
  keys: true,
  mouse: true,
  placeholder: `go on, tell ${assistantname} something!`,
});

const menuBox = blessed.list({
  top: "center",
  left: "center",
  width: 60,
  height: 16,
  border: {
    type: "line",
  },
  style: {
    border: {
      fg: "cyan",
    },
    bg: "black",
    selected: {
      bg: "cyan",
      fg: "black",
    },
    item: {
      fg: "white",
    },
  },
  keys: true,
  vi: true,
  mouse: true,
  hidden: true,
  label: ` ${assistantname} menu `,
  items: ["help & commands", "send picture", "take webcam snapshot", `exit ${assistantname}`],
});

const popup = blessed.box({
  parent: screen,
  top: "center",
  left: "center",
  width: 40,
  height: 8,
  border: {
    type: "line",
  },
  style: {
    border: {
      fg: "cyan",
    },
    bg: "black",
  },
  tags: true,
  hidden: true,
  content: "",
});

const popupButton = blessed.button({
  parent: popup,
  bottom: 1,
  left: "center",
  width: 8,
  height: 1,
  content: "OK", // < have no clue how to center this
  style: {
    bg: "cyan",
    fg: "black",
    focus: {
      bg: "white",
      fg: "black",
    },
  },
  mouse: true,
  keys: true,
});

screen.append(faceBox);
screen.append(chatBox);
screen.append(inputBox);
screen.append(menuBox);
screen.append(popup);

inputBox.focus();

let chatHistory = [];
let conversationHistory = [];
let currentStreamMessage = "";
let menuVisible = false;

function addMessage(role, content) {
  let message;
  if (role === username) {
    const padding = Math.max(0, chatBox.width - content.length - 5);
    message = " ".repeat(padding) + content + " <\n";
  } else {
    message = `> ${assistantname}: ${content}\n`;
  }
  chatHistory.push(message);

  chatBox.setContent(chatHistory.join(""));
  chatBox.setScrollPerc(100);
  screen.render();
}

function updateStreamMessage(content) {
  currentStreamMessage = content;

  // handle the kaomoji
  let letterIndex = -1;
  for (let i = 0; i < content.length - 1; i++) {
    if (/[a-zA-Z]/.test(content[i])) {
      if (content[i] === "I") {
        letterIndex = i;
        break;
      }
      if (/[a-zA-Z]/.test(content[i + 1])) {
        letterIndex = i;
        break;
      }
    }
  }

  if (letterIndex !== -1) {
    const faceContent = content.substring(0, letterIndex).trim();
    const messageContent = content.substring(letterIndex).trim();

    // only update face if kaomoji is 3+ characters because sometimes the model wont listen
    if (faceContent.length >= 3) {
      setFaceBoxContent(faceContent);
    }

    const tempHistory = [...chatHistory];
    tempHistory.push(`> ${assistantname}: ${messageContent}\n`);

    chatBox.setContent(tempHistory.join(""));
    chatBox.setScrollPerc(100);
    screen.render();
  } else {
    // if no letters found yet, just show the entire thing
    const tempHistory = [...chatHistory];
    tempHistory.push(`> ${assistantname}: ${content}\n`);

    chatBox.setContent(tempHistory.join(""));
    chatBox.setScrollPerc(100);
    screen.render();
  }
}

function finalizeStreamMessage() {
  if (currentStreamMessage) {
    let letterIndex = -1;
    for (let i = 0; i < currentStreamMessage.length - 1; i++) {
      if (/[a-zA-Z]/.test(currentStreamMessage[i])) {
        // complicated kaomoji splitting garboleum
        if (currentStreamMessage[i] === "I") {
          letterIndex = i;
          break;
        }
        if (/[a-zA-Z]/.test(currentStreamMessage[i + 1])) {
          letterIndex = i;
          break;
        }
      }
    }

    let assistantMessage;
    if (letterIndex !== -1) {
      assistantMessage = currentStreamMessage.substring(letterIndex).trim();
      addMessage(assistantname, assistantMessage);
    } else {
      assistantMessage = currentStreamMessage;
      addMessage(assistantname, currentStreamMessage);
    }

    conversationHistory.push({
      role: "assistant",
      content: assistantMessage,
    });

    currentStreamMessage = "";
  }
}

function setFaceBoxContent(content) {
  try {
    const figletOutput = execSync(`figlet -f ${facefont} "${content}"`, {
      encoding: "utf8",
    });
    faceBox.setContent(figletOutput);
  } catch (err) {
    faceBox.setContent(content);
  }
  screen.render();
}

function clearChatHistory() {
  chatHistory = [];
  conversationHistory = [];
  chatBox.setContent("");
  screen.render();
}

// webcam configuration
const webcamOptions = {
  width: cameraWidth,
  height: cameraHeight,
  quality: cameraQuality,
  delay: 0,
  saveShots: true,
  output: "jpeg",
  device: cameraDevice,
  callbackReturn: "location"
};

const webcam = NodeWebcam.create(webcamOptions);

async function convertImageToBase64(imagePath) {
  try {
    // resize image to reasonable size for vision models
    const buffer = await sharp(imagePath)
      .resize(800, 600, { fit: 'inside', withoutEnlargement: true })
      .jpeg({ quality: 80 })
      .toBuffer();
    
    return buffer.toString('base64');
  } catch (error) {
    throw new Error(`Failed to process image: ${error.message}`);
  }
}

async function sendImageFile() {
  return new Promise((resolve) => {
    // Create a simple input box for file path
    const fileInput = blessed.textbox({
      parent: screen,
      top: 'center',
      left: 'center',
      width: '80%',
      height: 5,
      border: {
        type: 'line'
      },
      style: {
        border: {
          fg: 'cyan'
        },
        bg: 'black'
      },
      inputOnFocus: true,
      keys: true,
      mouse: true,
      label: ' Enter Image File Path (ESC to cancel) ',
      placeholder: 'Enter full path to image file...'
    });

    screen.append(fileInput);
    fileInput.focus();
    screen.render();

    fileInput.on('submit', async (filePath) => {
      fileInput.destroy();
      
      if (!filePath || !filePath.trim()) {
        resolve();
        return;
      }

      const trimmedPath = filePath.trim();
      
      // Check if file exists
      if (!fs.existsSync(trimmedPath)) {
        showPopup('Error: File does not exist');
        resolve();
        return;
      }

      const ext = path.extname(trimmedPath).toLowerCase();
      const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp'];
      
      if (!imageExtensions.includes(ext)) {
        showPopup('Error: Please select a valid image file\n(.jpg, .png, .gif, .bmp, .webp)');
        resolve();
        return;
      }

      try {
        const base64Image = await convertImageToBase64(trimmedPath);
        await sendMessageWithImage("Here's an image I'd like you to look at:", base64Image);
        addMessage(username, `[Sent image: ${path.basename(trimmedPath)}]`);
      } catch (error) {
        showPopup(`Error processing image: ${error.message}`);
      }

      resolve();
    });

    fileInput.on('cancel', () => {
      fileInput.destroy();
      resolve();
    });

    screen.key(['escape'], () => {
      if (!fileInput.destroyed) {
        fileInput.destroy();
        resolve();
      }
    });
  });
}

async function takeWebcamSnapshot() {
  return new Promise((resolve) => {
    showPopup('Taking webcam snapshot...\nPlease wait...');
    
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const filename = `webcam-${timestamp}.jpg`;
    const filepath = path.join(process.cwd(), filename);

    webcam.capture(filename, async (err, data) => {
      popup.hide();
      
      if (err) {
        let errorMsg = 'Webcam error occurred';
        if (err.message) {
          errorMsg = `Webcam error: ${err.message}`;
        }
        if (err.message && err.message.includes('No such file or directory')) {
          errorMsg += '\n\nTip: Make sure you have a webcam connected\nand try installing fswebcam or imagesnap';
        }
        showPopup(errorMsg);
        resolve();
        return;
      }

      try {
        // check if file exists with a small delay
        await new Promise(r => setTimeout(r, 500));
        
        if (!fs.existsSync(filepath)) {
          showPopup('Error: Snapshot file was not created\nCheck if webcam is connected and accessible');
          resolve();
          return;
        }

        const base64Image = await convertImageToBase64(filepath);
        await sendMessageWithImage("I just took this webcam snapshot:", base64Image);
        addMessage(username, `[Took webcam snapshot: ${filename}]`);
        
        // clean up the temporary file
        try {
          fs.unlinkSync(filepath);
        } catch (unlinkErr) {
          // ignore cleanup errors
        }
      } catch (error) {
        showPopup(`Error processing snapshot: ${error.message}`);
        // try to clean up file even if there was an error
        try {
          if (fs.existsSync(filepath)) {
            fs.unlinkSync(filepath);
          }
        } catch (unlinkErr) {
          // ignore cleanup errors
        }
      }

      resolve();
    });
  });
}

function showPopup(message) {
  popup.setContent(`{center}${message}{/center}`);
  popup.show();
  popupButton.focus();
  screen.render();
}

function showMenu() {
  menuVisible = true;
  menuBox.show();
  menuBox.focus();
  screen.render();
}

function hideMenu() {
  menuVisible = false;
  menuBox.hide();
  inputBox.focus();
  screen.render();
}

async function handleMenuSelection() {
  const selectedIndex = menuBox.selected;
  hideMenu();

  switch (selectedIndex) {
    case 0:
      addMessage(
        assistantname,
        "available commands:\nl!help - if you wanna know what i can do, run this!\nl!clear - clear chat history, if you want me to forget everything, just run this!\nl!face <text> - if you want to force my expression, here you go! not sure i'll be too happy about it though.\nl!prompt <text> - if you want to change how i act, here you go! not sure i'll be too happy about that either.\nl!image <path> or l!img <path> - send an image file (or just l!image to browse)\nl!webcam or l!cam - take and send a webcam snapshot\n\nMenu shortcuts:\nESC - open/close menu\nUp/Down arrows - navigate menu\nEnter - select option\nCtrl+C or q - quit application\n\nMenu options:\n- help & commands - show this help\n- send picture - browse and send an image file\n- take webcam snapshot - capture and send a webcam photo\n- exit - quit the application\n\nNote: Image features require a vision-capable model like llava or bakllava in Ollama!\n",
      );
      break;
    case 1:
      await sendImageFile();
      break;
    case 2:
      await takeWebcamSnapshot();
      break;
    case 3:
      process.exit(0);
      break;
  }
}

async function sendMessage(message) {
  if (!message.trim()) return;

  addMessage(username, message);

  conversationHistory.push({
    role: "user",
    content: message,
  });

  try {
    currentStreamMessage = "";

    const response = await ollama.chat({
      model: assistantmodel,
      messages: [
        {
          role: "system",
          content: systemprompt,
        },
        ...conversationHistory,
      ],
      stream: true,
      options: {
        num_predict: maxtokens,
        temperature: temperature,
      },
    });

    for await (const part of response) {
      if (part.message && part.message.content) {
        currentStreamMessage += part.message.content;
        updateStreamMessage(currentStreamMessage);
      }
    }

    finalizeStreamMessage();
  } catch (error) {
    addMessage(assistantname, `Failed to get response: ${error.message}`);
  }
}

async function sendMessageWithImage(message, base64Image) {
  if (!message.trim()) return;

  conversationHistory.push({
    role: "user",
    content: message,
    images: [base64Image]
  });

  try {
    currentStreamMessage = "";

    const response = await ollama.chat({
      model: assistantmodel,
      messages: [
        {
          role: "system",
          content: systemprompt,
        },
        ...conversationHistory,
      ],
      stream: true,
      options: {
        num_predict: maxtokens,
        temperature: temperature,
      },
    });

    for await (const part of response) {
      if (part.message && part.message.content) {
        currentStreamMessage += part.message.content;
        updateStreamMessage(currentStreamMessage);
      }
    }

    finalizeStreamMessage();
  } catch (error) {
    addMessage(assistantname, `Failed to get response: ${error.message}`);
  }
}

inputBox.on("submit", async (text) => {
  if (text.trim()) {
    inputBox.clearValue();
    inputBox.focus();

    // command trash
    if (text.trim().startsWith("l!")) {
      const commandParts = text.trim().slice(2).split(" ");
      const command = commandParts[0];
      const args = commandParts.slice(1).join(" ");

      switch (command) {
        case "help":
          addMessage(
            assistantname,
            "available commands:\nl!help - if you wanna know what i can do, run this!\nl!clear - clear chat history, if you want me to forget everything, just run this!\nl!face <text> - if you want to force my expression, here you go! not sure i'll be too happy about it though.\nl!prompt <text> - if you want to change how i act, here you go! not sure i'll be too happy about that either.\nl!image <path> or l!img <path> - send an image file (or just l!image to browse)\nl!webcam or l!cam - take and send a webcam snapshot\n\nMenu shortcuts:\nESC - open/close menu\nUp/Down arrows - navigate menu\nEnter - select option\n\nMenu options:\n- help & commands - show this help\n- send picture - browse and send an image file\n- take webcam snapshot - capture and send a webcam photo\n- exit - quit the application\n",
          );
          break;
        case "clear":
          clearChatHistory();
          break;
        case "face":
          if (args.trim()) {
            setFaceBoxContent(args.trim());
          } else {
            setFaceBoxContent("=w=");
          }
          break;
        case "prompt":
          if (args.trim()) {
            systemprompt = args.trim();
            clearChatHistory();
            addMessage("system", "SYSTEM: Prompt set.");
          } else {
            addMessage("system", "SYSTEM: You didn't provide a prompt.");
          }
          break;
        case "image":
        case "img":
          if (args.trim()) {
            const imagePath = args.trim();
            if (!fs.existsSync(imagePath)) {
              addMessage("system", "SYSTEM: Image file not found.");
              break;
            }
            const ext = path.extname(imagePath).toLowerCase();
            const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp'];
            if (!imageExtensions.includes(ext)) {
              addMessage("system", "SYSTEM: Invalid image file format.");
              break;
            }
            try {
              const base64Image = await convertImageToBase64(imagePath);
              await sendMessageWithImage("Here's an image I'd like you to look at:", base64Image);
              addMessage(username, `[Sent image: ${path.basename(imagePath)}]`);
            } catch (error) {
              addMessage("system", `SYSTEM: Error processing image: ${error.message}`);
            }
          } else {
            await sendImageFile();
          }
          break;
        case "webcam":
        case "cam":
          await takeWebcamSnapshot();
          break;
        default:
          addMessage(assistantname, `unknown command: ${command}`);
          break;
      }
    } else {
      // message just send it
      await sendMessage(text);
    }
  }
});

screen.key(["q", "C-c"], () => {
  process.exit(0); // KILL lydia!!!!!!!!!!!!!!!!!!!!!!!
});

screen.key(["escape"], () => {
  if (menuVisible) {
    hideMenu();
  } else {
    showMenu();
  }
});

menuBox.on("select", async (item, selected) => {
  if (menuVisible) {
    await handleMenuSelection();
  }
});

popupButton.on("press", () => {
  popup.hide();
  inputBox.focus();
  screen.render();
});

screen.key(["enter"], async () => {
  if (!popup.hidden) {
    popup.hide();
    inputBox.focus();
    screen.render();
  } else if (menuVisible) {
    await handleMenuSelection();
  }
});

screen.on("resize", () => {
  screen.render();
});

setFaceBoxContent(assistantface);