From 76a1069436218e035255768dbf8b5253be9b00d2 Mon Sep 17 00:00:00 2001 From: Simen Svale Date: Tue, 30 Dec 2025 02:05:52 +0100 Subject: [PATCH] Add visual feedback tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protocol layer: - Add DisplayGet command (0x84) for screen capture - Add PaletteGet command (0x91) for color palette - Implement getDisplay() and getPalette() client methods New tools: - screenshot: Capture display as indexed pixel data with palette - Returns base64-encoded pixel buffer for efficient transfer - Includes display dimensions and visible area bounds - Optionally includes RGB palette values - renderScreen: ASCII art representation of current display - Converts pixel luminance to ASCII shading characters - Configurable output dimensions and character set - Useful for quick visual debugging in text-only contexts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- src/index.ts | 142 +++++++++++++++++++++++++++++++++++++++++ src/protocol/client.ts | 65 +++++++++++++++++++ src/protocol/types.ts | 4 ++ 3 files changed, 211 insertions(+) diff --git a/src/index.ts b/src/index.ts index 13aaf31..ae5c77b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1490,6 +1490,148 @@ Related tools: readVicState, readMemory (for sprite data)`, } ); +// ============================================================================= +// VISUAL FEEDBACK TOOLS - Display capture and rendering +// ============================================================================= + +// Tool: screenshot - Capture display as PNG +server.registerTool( + "screenshot", + { + description: `Capture the current VICE display as image data. + +Returns the raw display buffer with: +- Pixel data (indexed 8-bit palette colors) +- Display dimensions and visible area +- Current palette RGB values + +The data can be used to understand what's currently on screen visually. +For text mode screens, readScreen provides a simpler text representation. + +Options: +- includePalette: Also return the color palette (default: true) + +Related tools: readScreen, readVicState`, + inputSchema: z.object({ + includePalette: z.boolean().optional().describe("Include palette RGB values (default: true)"), + }), + }, + async (args) => { + try { + const display = await client.getDisplay(); + + const response: Record = { + width: display.width, + height: display.height, + bitsPerPixel: display.bitsPerPixel, + visibleArea: { + offsetX: display.offsetX, + offsetY: display.offsetY, + innerWidth: display.innerWidth, + innerHeight: display.innerHeight, + }, + pixelCount: display.pixels.length, + // Return pixels as base64 for efficient transfer + pixelsBase64: display.pixels.toString("base64"), + }; + + if (args.includePalette !== false) { + const palette = await client.getPalette(); + response.palette = palette; + response.paletteCount = palette.length; + } + + response.hint = `Display is ${display.width}x${display.height} (visible: ${display.innerWidth}x${display.innerHeight}). Use readScreen() for text mode content.`; + + return formatResponse(response); + } catch (error) { + return formatError(error as ViceError); + } + } +); + +// Tool: renderScreen - ASCII art screen rendering +server.registerTool( + "renderScreen", + { + description: `Render the current screen as ASCII art representation. + +Creates a visual representation of the screen using ASCII characters +to approximate the colors and content visible on the C64 display. + +This is useful for quick visual debugging without image handling. +For actual screen text, use readScreen instead. + +Options: +- width: Output width in characters (default: 80) +- height: Output height in lines (default: 50) +- charset: Character set to use for shading (default: " .:-=+*#%@") + +Related tools: readScreen, screenshot, readVicState`, + inputSchema: z.object({ + width: z.number().min(20).max(200).optional().describe("Output width in characters (default: 80)"), + height: z.number().min(10).max(100).optional().describe("Output height in lines (default: 50)"), + charset: z.string().optional().describe("Characters for shading from dark to light (default: ' .:-=+*#%@')"), + }), + }, + async (args) => { + try { + const display = await client.getDisplay(); + const palette = await client.getPalette(); + + const outputWidth = args.width || 80; + const outputHeight = args.height || 50; + const charset = args.charset || " .:-=+*#%@"; + + // Calculate luminance for each palette color + const luminance = palette.map((c) => { + // Standard luminance formula + return 0.299 * c.r + 0.587 * c.g + 0.114 * c.b; + }); + + // Sample the display and convert to ASCII + const scaleX = display.innerWidth / outputWidth; + const scaleY = display.innerHeight / outputHeight; + + const lines: string[] = []; + + for (let y = 0; y < outputHeight; y++) { + let line = ""; + for (let x = 0; x < outputWidth; x++) { + // Sample pixel from display + const srcX = Math.floor(display.offsetX + x * scaleX); + const srcY = Math.floor(display.offsetY + y * scaleY); + const pixelIndex = srcY * display.width + srcX; + + if (pixelIndex < display.pixels.length) { + const colorIndex = display.pixels[pixelIndex]; + const lum = colorIndex < luminance.length ? luminance[colorIndex] : 0; + + // Map luminance (0-255) to charset index + const charIndex = Math.floor((lum / 256) * charset.length); + line += charset[Math.min(charIndex, charset.length - 1)]; + } else { + line += " "; + } + } + lines.push(line); + } + + return formatResponse({ + width: outputWidth, + height: outputHeight, + sourceWidth: display.innerWidth, + sourceHeight: display.innerHeight, + charset, + render: lines.join("\n"), + hint: `ASCII rendering of ${display.innerWidth}x${display.innerHeight} display scaled to ${outputWidth}x${outputHeight}`, + }); + } catch (error) { + return formatError(error as ViceError); + } + } +); + async function main() { const transport = new StdioServerTransport(); await server.connect(transport); diff --git a/src/protocol/client.ts b/src/protocol/client.ts index f16d796..cdcc2d7 100644 --- a/src/protocol/client.ts +++ b/src/protocol/client.ts @@ -618,6 +618,71 @@ export class ViceClient { filenameBuffer.copy(body, 4); await this.sendCommand(Command.AutoStart, body); } + + // Get display buffer (screenshot data) + async getDisplay(useVicii = true): Promise<{ + width: number; + height: number; + bitsPerPixel: number; + offsetX: number; + offsetY: number; + innerWidth: number; + innerHeight: number; + pixels: Buffer; + }> { + // Body: useVicii(1) + format(1) + // Format: 0 = indexed 8-bit + const body = Buffer.alloc(2); + body[0] = useVicii ? 1 : 0; + body[1] = 0; // 8-bit indexed + + const response = await this.sendCommand(Command.DisplayGet, body); + + // Parse response + // Response: length(4) + width(4) + height(4) + bpp(1) + offsetX(4) + offsetY(4) + + // innerWidth(4) + innerHeight(4) + pixels... + const dataLength = response.body.readUInt32LE(0); + const width = response.body.readUInt32LE(4); + const height = response.body.readUInt32LE(8); + const bitsPerPixel = response.body[12]; + const offsetX = response.body.readUInt32LE(13); + const offsetY = response.body.readUInt32LE(17); + const innerWidth = response.body.readUInt32LE(21); + const innerHeight = response.body.readUInt32LE(25); + const pixels = response.body.subarray(29, 29 + dataLength); + + return { + width, + height, + bitsPerPixel, + offsetX, + offsetY, + innerWidth, + innerHeight, + pixels, + }; + } + + // Get palette (color table) + async getPalette(): Promise> { + const response = await this.sendCommand(Command.PaletteGet); + + // Parse response + // Response: count(2) + [r(1) + g(1) + b(1)]... + const count = response.body.readUInt16LE(0); + const colors: Array<{ r: number; g: number; b: number }> = []; + + for (let i = 0; i < count; i++) { + const offset = 2 + i * 3; + colors.push({ + r: response.body[offset], + g: response.body[offset + 1], + b: response.body[offset + 2], + }); + } + + return colors; + } } // Singleton instance diff --git a/src/protocol/types.ts b/src/protocol/types.ts index f4d98af..192ec47 100644 --- a/src/protocol/types.ts +++ b/src/protocol/types.ts @@ -39,6 +39,10 @@ export enum Command { // Quit Quit = 0xbb, + // Display + DisplayGet = 0x84, + PaletteGet = 0x91, + // Autostart AutoStart = 0xdd, }