Add visual feedback tools
Protocol layer: - Add DisplayGet command (0x84) for screen capture - Add PaletteGet command (0x91) for color palette - Implement getDisplay() and getPalette() client methods New tools: - screenshot: Capture display as indexed pixel data with palette - Returns base64-encoded pixel buffer for efficient transfer - Includes display dimensions and visible area bounds - Optionally includes RGB palette values - renderScreen: ASCII art representation of current display - Converts pixel luminance to ASCII shading characters - Configurable output dimensions and character set - Useful for quick visual debugging in text-only contexts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
cebe6f4a14
commit
76a1069436
3 changed files with 211 additions and 0 deletions
142
src/index.ts
142
src/index.ts
|
|
@ -1490,6 +1490,148 @@ Related tools: readVicState, readMemory (for sprite data)`,
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// =============================================================================
|
||||||
|
// VISUAL FEEDBACK TOOLS - Display capture and rendering
|
||||||
|
// =============================================================================
|
||||||
|
|
||||||
|
// Tool: screenshot - Capture display as PNG
|
||||||
|
server.registerTool(
|
||||||
|
"screenshot",
|
||||||
|
{
|
||||||
|
description: `Capture the current VICE display as image data.
|
||||||
|
|
||||||
|
Returns the raw display buffer with:
|
||||||
|
- Pixel data (indexed 8-bit palette colors)
|
||||||
|
- Display dimensions and visible area
|
||||||
|
- Current palette RGB values
|
||||||
|
|
||||||
|
The data can be used to understand what's currently on screen visually.
|
||||||
|
For text mode screens, readScreen provides a simpler text representation.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
- includePalette: Also return the color palette (default: true)
|
||||||
|
|
||||||
|
Related tools: readScreen, readVicState`,
|
||||||
|
inputSchema: z.object({
|
||||||
|
includePalette: z.boolean().optional().describe("Include palette RGB values (default: true)"),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
async (args) => {
|
||||||
|
try {
|
||||||
|
const display = await client.getDisplay();
|
||||||
|
|
||||||
|
const response: Record<string, unknown> = {
|
||||||
|
width: display.width,
|
||||||
|
height: display.height,
|
||||||
|
bitsPerPixel: display.bitsPerPixel,
|
||||||
|
visibleArea: {
|
||||||
|
offsetX: display.offsetX,
|
||||||
|
offsetY: display.offsetY,
|
||||||
|
innerWidth: display.innerWidth,
|
||||||
|
innerHeight: display.innerHeight,
|
||||||
|
},
|
||||||
|
pixelCount: display.pixels.length,
|
||||||
|
// Return pixels as base64 for efficient transfer
|
||||||
|
pixelsBase64: display.pixels.toString("base64"),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (args.includePalette !== false) {
|
||||||
|
const palette = await client.getPalette();
|
||||||
|
response.palette = palette;
|
||||||
|
response.paletteCount = palette.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
response.hint = `Display is ${display.width}x${display.height} (visible: ${display.innerWidth}x${display.innerHeight}). Use readScreen() for text mode content.`;
|
||||||
|
|
||||||
|
return formatResponse(response);
|
||||||
|
} catch (error) {
|
||||||
|
return formatError(error as ViceError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// Tool: renderScreen - ASCII art screen rendering
|
||||||
|
server.registerTool(
|
||||||
|
"renderScreen",
|
||||||
|
{
|
||||||
|
description: `Render the current screen as ASCII art representation.
|
||||||
|
|
||||||
|
Creates a visual representation of the screen using ASCII characters
|
||||||
|
to approximate the colors and content visible on the C64 display.
|
||||||
|
|
||||||
|
This is useful for quick visual debugging without image handling.
|
||||||
|
For actual screen text, use readScreen instead.
|
||||||
|
|
||||||
|
Options:
|
||||||
|
- width: Output width in characters (default: 80)
|
||||||
|
- height: Output height in lines (default: 50)
|
||||||
|
- charset: Character set to use for shading (default: " .:-=+*#%@")
|
||||||
|
|
||||||
|
Related tools: readScreen, screenshot, readVicState`,
|
||||||
|
inputSchema: z.object({
|
||||||
|
width: z.number().min(20).max(200).optional().describe("Output width in characters (default: 80)"),
|
||||||
|
height: z.number().min(10).max(100).optional().describe("Output height in lines (default: 50)"),
|
||||||
|
charset: z.string().optional().describe("Characters for shading from dark to light (default: ' .:-=+*#%@')"),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
async (args) => {
|
||||||
|
try {
|
||||||
|
const display = await client.getDisplay();
|
||||||
|
const palette = await client.getPalette();
|
||||||
|
|
||||||
|
const outputWidth = args.width || 80;
|
||||||
|
const outputHeight = args.height || 50;
|
||||||
|
const charset = args.charset || " .:-=+*#%@";
|
||||||
|
|
||||||
|
// Calculate luminance for each palette color
|
||||||
|
const luminance = palette.map((c) => {
|
||||||
|
// Standard luminance formula
|
||||||
|
return 0.299 * c.r + 0.587 * c.g + 0.114 * c.b;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Sample the display and convert to ASCII
|
||||||
|
const scaleX = display.innerWidth / outputWidth;
|
||||||
|
const scaleY = display.innerHeight / outputHeight;
|
||||||
|
|
||||||
|
const lines: string[] = [];
|
||||||
|
|
||||||
|
for (let y = 0; y < outputHeight; y++) {
|
||||||
|
let line = "";
|
||||||
|
for (let x = 0; x < outputWidth; x++) {
|
||||||
|
// Sample pixel from display
|
||||||
|
const srcX = Math.floor(display.offsetX + x * scaleX);
|
||||||
|
const srcY = Math.floor(display.offsetY + y * scaleY);
|
||||||
|
const pixelIndex = srcY * display.width + srcX;
|
||||||
|
|
||||||
|
if (pixelIndex < display.pixels.length) {
|
||||||
|
const colorIndex = display.pixels[pixelIndex];
|
||||||
|
const lum = colorIndex < luminance.length ? luminance[colorIndex] : 0;
|
||||||
|
|
||||||
|
// Map luminance (0-255) to charset index
|
||||||
|
const charIndex = Math.floor((lum / 256) * charset.length);
|
||||||
|
line += charset[Math.min(charIndex, charset.length - 1)];
|
||||||
|
} else {
|
||||||
|
line += " ";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
return formatResponse({
|
||||||
|
width: outputWidth,
|
||||||
|
height: outputHeight,
|
||||||
|
sourceWidth: display.innerWidth,
|
||||||
|
sourceHeight: display.innerHeight,
|
||||||
|
charset,
|
||||||
|
render: lines.join("\n"),
|
||||||
|
hint: `ASCII rendering of ${display.innerWidth}x${display.innerHeight} display scaled to ${outputWidth}x${outputHeight}`,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
return formatError(error as ViceError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
const transport = new StdioServerTransport();
|
const transport = new StdioServerTransport();
|
||||||
await server.connect(transport);
|
await server.connect(transport);
|
||||||
|
|
|
||||||
|
|
@ -618,6 +618,71 @@ export class ViceClient {
|
||||||
filenameBuffer.copy(body, 4);
|
filenameBuffer.copy(body, 4);
|
||||||
await this.sendCommand(Command.AutoStart, body);
|
await this.sendCommand(Command.AutoStart, body);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get display buffer (screenshot data)
|
||||||
|
async getDisplay(useVicii = true): Promise<{
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
bitsPerPixel: number;
|
||||||
|
offsetX: number;
|
||||||
|
offsetY: number;
|
||||||
|
innerWidth: number;
|
||||||
|
innerHeight: number;
|
||||||
|
pixels: Buffer;
|
||||||
|
}> {
|
||||||
|
// Body: useVicii(1) + format(1)
|
||||||
|
// Format: 0 = indexed 8-bit
|
||||||
|
const body = Buffer.alloc(2);
|
||||||
|
body[0] = useVicii ? 1 : 0;
|
||||||
|
body[1] = 0; // 8-bit indexed
|
||||||
|
|
||||||
|
const response = await this.sendCommand(Command.DisplayGet, body);
|
||||||
|
|
||||||
|
// Parse response
|
||||||
|
// Response: length(4) + width(4) + height(4) + bpp(1) + offsetX(4) + offsetY(4) +
|
||||||
|
// innerWidth(4) + innerHeight(4) + pixels...
|
||||||
|
const dataLength = response.body.readUInt32LE(0);
|
||||||
|
const width = response.body.readUInt32LE(4);
|
||||||
|
const height = response.body.readUInt32LE(8);
|
||||||
|
const bitsPerPixel = response.body[12];
|
||||||
|
const offsetX = response.body.readUInt32LE(13);
|
||||||
|
const offsetY = response.body.readUInt32LE(17);
|
||||||
|
const innerWidth = response.body.readUInt32LE(21);
|
||||||
|
const innerHeight = response.body.readUInt32LE(25);
|
||||||
|
const pixels = response.body.subarray(29, 29 + dataLength);
|
||||||
|
|
||||||
|
return {
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
bitsPerPixel,
|
||||||
|
offsetX,
|
||||||
|
offsetY,
|
||||||
|
innerWidth,
|
||||||
|
innerHeight,
|
||||||
|
pixels,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get palette (color table)
|
||||||
|
async getPalette(): Promise<Array<{ r: number; g: number; b: number }>> {
|
||||||
|
const response = await this.sendCommand(Command.PaletteGet);
|
||||||
|
|
||||||
|
// Parse response
|
||||||
|
// Response: count(2) + [r(1) + g(1) + b(1)]...
|
||||||
|
const count = response.body.readUInt16LE(0);
|
||||||
|
const colors: Array<{ r: number; g: number; b: number }> = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const offset = 2 + i * 3;
|
||||||
|
colors.push({
|
||||||
|
r: response.body[offset],
|
||||||
|
g: response.body[offset + 1],
|
||||||
|
b: response.body[offset + 2],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return colors;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Singleton instance
|
// Singleton instance
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,10 @@ export enum Command {
|
||||||
// Quit
|
// Quit
|
||||||
Quit = 0xbb,
|
Quit = 0xbb,
|
||||||
|
|
||||||
|
// Display
|
||||||
|
DisplayGet = 0x84,
|
||||||
|
PaletteGet = 0x91,
|
||||||
|
|
||||||
// Autostart
|
// Autostart
|
||||||
AutoStart = 0xdd,
|
AutoStart = 0xdd,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue