blob: 561f281731207afa0845307459847be236af5020 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
[CmdletBinding()]
param(
[Parameter(Mandatory)] [uri]$Url,
[string]$EvidenceDirectory = "artifacts/evidence/mcp-2026-07-28/conformance-tasks"
)
$ErrorActionPreference = "Stop"
if (-not $Url.IsLoopback) { throw "Conformance target must be loopback." }
$repository = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
$packageJson = Join-Path $repository "build/conformance-runner/node_modules/@modelcontextprotocol/conformance/package.json"
if (-not (Test-Path -LiteralPath $packageJson)) { throw "Offline runner is absent. Run scripts/Acquire-McpConformance.ps1 first." }
$metadata = Get-Content -Raw -LiteralPath $packageJson | ConvertFrom-Json
if ($metadata.version -ne "0.2.0-alpha.11") { throw "Unexpected conformance version: $($metadata.version)" }
$cli = Join-Path (Split-Path $packageJson) $metadata.bin.conformance
$evidence = [IO.Path]::GetFullPath($EvidenceDirectory, $repository)
New-Item -ItemType Directory -Force -Path $evidence | Out-Null
$scenarios = @(
"tasks-lifecycle", "tasks-capability-negotiation", "tasks-wire-fields", "tasks-request-state-removal",
"tasks-mrtr-input", "tasks-request-headers", "tasks-dispatch-and-envelope", "tasks-status-notifications",
"tasks-required-task-error", "tasks-mrtr-composition"
)
$applicationFailures = @()
$runnerLimitations = @()
foreach ($scenario in $scenarios) {
$output = Join-Path $evidence $scenario
& node $cli server --url $Url.AbsoluteUri --scenario $scenario --spec-version 2026-07-28 --force --output-dir $output 2>&1 |
Tee-Object -FilePath (Join-Path $evidence "$scenario.txt")
$runnerExitCode = $LASTEXITCODE
$resultDirectory = Get-ChildItem -LiteralPath $output -Directory | Sort-Object LastWriteTimeUtc -Descending | Select-Object -First 1
if ($null -eq $resultDirectory) {
$runnerLimitations += "$scenario`:runner_exit_$runnerExitCode`:no_result_directory"
continue
}
$checksPath = Join-Path $resultDirectory.FullName "checks.json"
if (-not (Test-Path -LiteralPath $checksPath)) {
$runnerLimitations += "$scenario`:runner_exit_$runnerExitCode`:checks_missing"
continue
}
foreach ($check in (Get-Content -Raw -LiteralPath $checksPath | ConvertFrom-Json)) {
if ($check.status -ne "FAILURE") { continue }
$message = [string]$check.errorMessage
if ($check.id -eq "wire-schema-valid" -and $message -match "CallToolResult: must have required property 'content'.*resultType.*task") {
$runnerLimitations += "$scenario`:$($check.id)"
} elseif ($message -match "^Not testable:" -or
$message -match "^Unknown tool: '(greet|slow_compute|failing_job|protocol_error_job|confirm_delete|multi_input|test_tool_with_task)'" -or
$message -match "fixture .* unavailable" -or
$message -match "tools/call for failing_job returned error code -32602") {
$runnerLimitations += "$scenario`:$($check.id) - $message"
} else {
$applicationFailures += "$scenario`:$($check.id) - $message"
}
}
}
[ordered]@{
runner_version = $metadata.version
scenarios = $scenarios
application_failure_count = $applicationFailures.Count
application_failures = $applicationFailures
runner_limitation_count = $runnerLimitations.Count
runner_limitations = $runnerLimitations
} | ConvertTo-Json -Depth 5 | Set-Content -Encoding utf8 -LiteralPath (Join-Path $evidence "summary.json")
if ($applicationFailures.Count -ne 0) { throw "$($applicationFailures.Count) application Tasks conformance check(s) failed." }
|
