evals mcp server for VS Code
io.github.cyanheads/evals-mcp-server
Author verifiable eval records through a draft→review→revise→submit loop with enforced graders.
client:VS Code
transport:stdio
runtime:npm
Install evals mcp server in VS Code
.vscode/mcp.json
{
"inputs": [
{
"type": "promptString",
"id": "evals-data-dir",
"description": "EVALS_DATA_DIR",
"password": true
},
{
"type": "promptString",
"id": "evals-require-confirmation",
"description": "EVALS_REQUIRE_CONFIRMATION",
"password": true
},
{
"type": "promptString",
"id": "evals-default-license",
"description": "EVALS_DEFAULT_LICENSE",
"password": true
},
{
"type": "promptString",
"id": "evals-capture-dir",
"description": "EVALS_CAPTURE_DIR",
"password": true
},
{
"type": "promptString",
"id": "mcp-log-level",
"description": "MCP_LOG_LEVEL",
"password": true
}
],
"servers": {
"evals-mcp-server": {
"type": "stdio",
"command": "npx",
"args": [
"-y",
"@cyanheads/evals-mcp-server"
],
"env": {
"EVALS_DATA_DIR": "${input:evals-data-dir}",
"EVALS_REQUIRE_CONFIRMATION": "${input:evals-require-confirmation}",
"EVALS_DEFAULT_LICENSE": "${input:evals-default-license}",
"EVALS_CAPTURE_DIR": "${input:evals-capture-dir}",
"MCP_LOG_LEVEL": "${input:mcp-log-level}"
}
}
}
}VS Code prompts for each input the first time the server starts, so no secret is written to the file. VS Code docs