mcp/.vally.yaml at main · microsoft/mcp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
paths:
  skills: []
  evals: ["tools/Azure.Mcp.Tools.*/tests/", ".work/vally/evals/*/"]
  results: ".work/vally/vally-results/"

environments:
  linux:
    mcpServers:
      azmcp:
        type: stdio
        command: ".work/build/Azure.Mcp.Server/linux-x64/azmcp"
        args: ["server", "start"]
        timeout: "2m"
        env:
          AZURE_MCP_COLLECT_TELEMETRY: 'false'
  windows:
    mcpServers:
      azmcp:
        type: stdio
        command: ".work/build/Azure.Mcp.Server/windows-x64/azmcp"
        args: ["server", "start"]
        timeout: "2m"
        env:
          AZURE_MCP_COLLECT_TELEMETRY: 'false'

suites:
  basic:
    description: "Basic set of MCP tools to run evals on."
    evals:
      - ".work/evals/appconfig/"
      - ".work/evals/storage/"
      - ".work/evals/keyvault/"
      - "tools/Azure.Mcp.Tools.AppConfig/tests/"
      - "tools/Azure.Mcp.Tools.KeyVault/tests/"
      - "tools/Azure.Mcp.Tools.Storage/tests/"