Add visual feedback tools

Protocol layer:
- Add DisplayGet command (0x84) for screen capture
- Add PaletteGet command (0x91) for color palette
- Implement getDisplay() and getPalette() client methods

New tools:
- screenshot: Capture display as indexed pixel data with palette
  - Returns base64-encoded pixel buffer for efficient transfer
  - Includes display dimensions and visible area bounds
  - Optionally includes RGB palette values
- renderScreen: ASCII art representation of current display
  - Converts pixel luminance to ASCII shading characters
  - Configurable output dimensions and character set
  - Useful for quick visual debugging in text-only contexts

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Simen Svale 2025-12-30 02:05:52 +01:00
parent cebe6f4a14
commit 76a1069436
3 changed files with 211 additions and 0 deletions

View file

@ -1490,6 +1490,148 @@ Related tools: readVicState, readMemory (for sprite data)`,
} }
); );
// =============================================================================
// VISUAL FEEDBACK TOOLS - Display capture and rendering
// =============================================================================
// Tool: screenshot - Capture display as PNG
server.registerTool(
"screenshot",
{
description: `Capture the current VICE display as image data.
Returns the raw display buffer with:
- Pixel data (indexed 8-bit palette colors)
- Display dimensions and visible area
- Current palette RGB values
The data can be used to understand what's currently on screen visually.
For text mode screens, readScreen provides a simpler text representation.
Options:
- includePalette: Also return the color palette (default: true)
Related tools: readScreen, readVicState`,
inputSchema: z.object({
includePalette: z.boolean().optional().describe("Include palette RGB values (default: true)"),
}),
},
async (args) => {
try {
const display = await client.getDisplay();
const response: Record<string, unknown> = {
width: display.width,
height: display.height,
bitsPerPixel: display.bitsPerPixel,
visibleArea: {
offsetX: display.offsetX,
offsetY: display.offsetY,
innerWidth: display.innerWidth,
innerHeight: display.innerHeight,
},
pixelCount: display.pixels.length,
// Return pixels as base64 for efficient transfer
pixelsBase64: display.pixels.toString("base64"),
};
if (args.includePalette !== false) {
const palette = await client.getPalette();
response.palette = palette;
response.paletteCount = palette.length;
}
response.hint = `Display is ${display.width}x${display.height} (visible: ${display.innerWidth}x${display.innerHeight}). Use readScreen() for text mode content.`;
return formatResponse(response);
} catch (error) {
return formatError(error as ViceError);
}
}
);
// Tool: renderScreen - ASCII art screen rendering
server.registerTool(
"renderScreen",
{
description: `Render the current screen as ASCII art representation.
Creates a visual representation of the screen using ASCII characters
to approximate the colors and content visible on the C64 display.
This is useful for quick visual debugging without image handling.
For actual screen text, use readScreen instead.
Options:
- width: Output width in characters (default: 80)
- height: Output height in lines (default: 50)
- charset: Character set to use for shading (default: " .:-=+*#%@")
Related tools: readScreen, screenshot, readVicState`,
inputSchema: z.object({
width: z.number().min(20).max(200).optional().describe("Output width in characters (default: 80)"),
height: z.number().min(10).max(100).optional().describe("Output height in lines (default: 50)"),
charset: z.string().optional().describe("Characters for shading from dark to light (default: ' .:-=+*#%@')"),
}),
},
async (args) => {
try {
const display = await client.getDisplay();
const palette = await client.getPalette();
const outputWidth = args.width || 80;
const outputHeight = args.height || 50;
const charset = args.charset || " .:-=+*#%@";
// Calculate luminance for each palette color
const luminance = palette.map((c) => {
// Standard luminance formula
return 0.299 * c.r + 0.587 * c.g + 0.114 * c.b;
});
// Sample the display and convert to ASCII
const scaleX = display.innerWidth / outputWidth;
const scaleY = display.innerHeight / outputHeight;
const lines: string[] = [];
for (let y = 0; y < outputHeight; y++) {
let line = "";
for (let x = 0; x < outputWidth; x++) {
// Sample pixel from display
const srcX = Math.floor(display.offsetX + x * scaleX);
const srcY = Math.floor(display.offsetY + y * scaleY);
const pixelIndex = srcY * display.width + srcX;
if (pixelIndex < display.pixels.length) {
const colorIndex = display.pixels[pixelIndex];
const lum = colorIndex < luminance.length ? luminance[colorIndex] : 0;
// Map luminance (0-255) to charset index
const charIndex = Math.floor((lum / 256) * charset.length);
line += charset[Math.min(charIndex, charset.length - 1)];
} else {
line += " ";
}
}
lines.push(line);
}
return formatResponse({
width: outputWidth,
height: outputHeight,
sourceWidth: display.innerWidth,
sourceHeight: display.innerHeight,
charset,
render: lines.join("\n"),
hint: `ASCII rendering of ${display.innerWidth}x${display.innerHeight} display scaled to ${outputWidth}x${outputHeight}`,
});
} catch (error) {
return formatError(error as ViceError);
}
}
);
async function main() { async function main() {
const transport = new StdioServerTransport(); const transport = new StdioServerTransport();
await server.connect(transport); await server.connect(transport);

View file

@ -618,6 +618,71 @@ export class ViceClient {
filenameBuffer.copy(body, 4); filenameBuffer.copy(body, 4);
await this.sendCommand(Command.AutoStart, body); await this.sendCommand(Command.AutoStart, body);
} }
// Get display buffer (screenshot data)
async getDisplay(useVicii = true): Promise<{
width: number;
height: number;
bitsPerPixel: number;
offsetX: number;
offsetY: number;
innerWidth: number;
innerHeight: number;
pixels: Buffer;
}> {
// Body: useVicii(1) + format(1)
// Format: 0 = indexed 8-bit
const body = Buffer.alloc(2);
body[0] = useVicii ? 1 : 0;
body[1] = 0; // 8-bit indexed
const response = await this.sendCommand(Command.DisplayGet, body);
// Parse response
// Response: length(4) + width(4) + height(4) + bpp(1) + offsetX(4) + offsetY(4) +
// innerWidth(4) + innerHeight(4) + pixels...
const dataLength = response.body.readUInt32LE(0);
const width = response.body.readUInt32LE(4);
const height = response.body.readUInt32LE(8);
const bitsPerPixel = response.body[12];
const offsetX = response.body.readUInt32LE(13);
const offsetY = response.body.readUInt32LE(17);
const innerWidth = response.body.readUInt32LE(21);
const innerHeight = response.body.readUInt32LE(25);
const pixels = response.body.subarray(29, 29 + dataLength);
return {
width,
height,
bitsPerPixel,
offsetX,
offsetY,
innerWidth,
innerHeight,
pixels,
};
}
// Get palette (color table)
async getPalette(): Promise<Array<{ r: number; g: number; b: number }>> {
const response = await this.sendCommand(Command.PaletteGet);
// Parse response
// Response: count(2) + [r(1) + g(1) + b(1)]...
const count = response.body.readUInt16LE(0);
const colors: Array<{ r: number; g: number; b: number }> = [];
for (let i = 0; i < count; i++) {
const offset = 2 + i * 3;
colors.push({
r: response.body[offset],
g: response.body[offset + 1],
b: response.body[offset + 2],
});
}
return colors;
}
} }
// Singleton instance // Singleton instance

View file

@ -39,6 +39,10 @@ export enum Command {
// Quit // Quit
Quit = 0xbb, Quit = 0xbb,
// Display
DisplayGet = 0x84,
PaletteGet = 0x91,
// Autostart // Autostart
AutoStart = 0xdd, AutoStart = 0xdd,
} }