summaryrefslogtreecommitdiff
path: root/scripts/Operational-E2E.ps1
blob: 2a32a8ecf72dee37cd2d0e045a43669f050d4c57 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
param(
    [ValidateSet('stdio', 'http')]
    [string]$Transport = 'stdio',
    [ValidateSet('127.0.0.1', '::1')]
    [string]$HttpHost = '127.0.0.1',
    [string]$ReleaseZip = 'build\Release\publish\winagent-sandbox-mcp-v0.1.0-win-x64.zip',
    [string]$PythonPath
)

$ErrorActionPreference = 'Stop'
$OutputEncoding = [Text.UTF8Encoding]::new($false)
[Console]::OutputEncoding = $OutputEncoding
$repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
$zip = (Resolve-Path (Join-Path $repository $ReleaseZip)).Path
$root = Join-Path ([IO.Path]::GetTempPath()) ("agent-mcp-operational-e2e-" + [Guid]::NewGuid().ToString('N'))
$bundleRoot = Join-Path $root 'bundle'
$fixture = Join-Path $root 'fixture'
$alias = 'python-e2e-' + [Guid]::NewGuid().ToString('N').Substring(0, 12)
$passed = $false
$appliedStorePath = $null
$server = $null
New-Item -ItemType Directory -Path $bundleRoot,$fixture | Out-Null
$fixtureAclBefore = (Get-Acl -LiteralPath $fixture).Sddl

try {
    Expand-Archive -LiteralPath $zip -DestinationPath $bundleRoot
    $bundle = (Get-ChildItem -LiteralPath $bundleRoot -Directory | Select-Object -First 1).FullName
    $agent = Join-Path $bundle 'agent-mcp.exe'
    $configPath = Join-Path $bundle 'agent.yaml'
    if (-not $PythonPath) {
        $PythonPath = (& uv python find 3.14).Trim()
        if ($LASTEXITCODE -ne 0 -or -not $PythonPath) { throw 'Python 3.14 could not be resolved with uv.' }
    }
    $pythonLogical = [IO.Path]::GetFullPath($PythonPath)
    $python = (Get-Item -LiteralPath $pythonLogical).Target
    if (-not $python) { $python = $pythonLogical }
    elseif (-not [IO.Path]::IsPathFullyQualified($python)) { $python = [IO.Path]::GetFullPath((Join-Path (Split-Path $pythonLogical) $python)) }
    $python = (Resolve-Path -LiteralPath $python).Path
    $pythonVersion = (& $python --version 2>&1 | Out-String).Trim()
    if ($pythonVersion -notmatch '^Python 3\.14\.') { throw "Operational E2E requires Python 3.14; resolved '$pythonVersion'." }
    $runtimeRoot = Split-Path $python
    $pythonYaml = $pythonLogical.Replace('\','/')
    $runtimeYaml = $runtimeRoot.Replace('\','/')

    Set-Content -LiteralPath (Join-Path $fixture 'README.md') -Encoding utf8NoBOM -Value @'
# Graph metrics fixture

Implement `graph_metrics.py` and `test_graph_metrics.py`. The implementation must validate dependencies,
produce a deterministic topological order, detect cycles, and calculate the weighted critical path.
'@

    $fixtureYaml = $fixture.Replace('\','/')
    $config = @"
schema_version: 2
application:
  language: ja
mcp:
  transport: $Transport
  http:
    host: $HttpHost
    port: 18080
  images:
    max_count: 4
    max_bytes_each: 10485760
    max_total_bytes: 20971520
agent:
  user_system_prompt: prompts/user-system.md
  max_iterations: 32
  same_failure_limit: 5
ollama:
  endpoint: http://127.0.0.1:11434
  model: deepseek-v4-flash:cloud
  think: true
  keep_alive: 5m
  options:
    num_ctx: 32768
    temperature: 0
  extra_options: {}
planner:
  max_tasks: 32
sandbox:
  profile: development
  workspaces:
    - $fixtureYaml
  mounts: []
  execution:
    entries:
      - alias: $alias
        path: $pythonYaml
        runtime_root: $runtimeYaml
  network:
    access: none
    local_endpoints: []
    restricted_endpoints: []
  resources:
    profile: constrained
    execution_timeout_seconds: 300
file_edit:
  memory_backup_limit_mb: 16
limits:
  whole_read_bytes: 1048576
  search_max_results: 1000
  tail_max_lines: 10000
  context_summary_ratio: 0.8
localization:
  directory: config/locales
  fallback: en
skills:
  directory: skills
logging:
  directory: logs
  max_file_bytes: 10485760
  retained_files: 5
"@
    Set-Content -LiteralPath $configPath -Value $config -Encoding utf8NoBOM

    & $agent apply $configPath --yes
    if ($LASTEXITCODE -ne 0) { throw 'Applied Configuration failed.' }
    $normalizedConfig = [IO.Path]::GetFullPath($configPath).ToUpperInvariant()
    $hash = [Convert]::ToHexString([Security.Cryptography.SHA256]::HashData([Text.Encoding]::UTF8.GetBytes($normalizedConfig)))
    $appliedStorePath = Join-Path $env:LOCALAPPDATA "AgentSandboxMcp\applied-configurations\$hash.json"
    & $agent show $configPath | Out-Null
    if ($LASTEXITCODE -ne 0) { throw 'Effective Sandbox Policy show failed.' }

    $request = @"
実作業を行ってください。作業ディレクトリは $fixtureYaml です。
README.md の仕様に従い、外部パッケージを使わず graph_metrics.py と test_graph_metrics.py を作成してください。
テストには、正常なDAG、同順位時の決定的な順序、未知dependency、cycle、weighted critical pathを含めてください。
さらに test_graph_metrics.py 自身が subprocess.run([sys.executable, ...]) で小さな子Pythonを起動して成功を検証するテストを含めてください。
README.md の読取後、2ファイルの作成、最初のテスト実行、失敗時の修正と再実行は、分割せず単一の実装・検証Taskとして計画してください。
最後に process_run で executable=$alias, cwd=$fixtureYaml, args=["-m","unittest","discover","-v"] を実行し、全テスト成功を確認してください。
最終応答は日本語にし、作成ファイルとテスト結果を簡潔に報告してください。
"@
    $clientProject = Join-Path $repository 'scripts\E2EClient\E2EClient.csproj'
    if ($Transport -eq 'stdio') {
        dotnet run --project $clientProject -c Release --no-build -- stdio $agent $configPath $request
        if ($LASTEXITCODE -ne 0) { throw 'E2E stdio client failed.' }
    }
    else {
        $server = Start-Process -FilePath $agent -ArgumentList @($configPath) -WorkingDirectory $bundle -WindowStyle Hidden -PassThru
        $httpAuthority = if ($HttpHost -eq '::1') { '[::1]' } else { $HttpHost }
        $endpoint = "http://${httpAuthority}:18080/mcp"
        $ready = $false
        for ($attempt = 0; $attempt -lt 100; $attempt++) {
            try { $response = Invoke-WebRequest -Uri $endpoint -Method Get -SkipHttpErrorCheck; if ($response.StatusCode -eq 405) { $ready=$true; break } } catch { }
            Start-Sleep -Milliseconds 100
        }
        if (-not $ready) { throw 'HTTP server did not become ready.' }
        dotnet run --project $clientProject -c Release --no-build -- http $endpoint unused $request
        if ($LASTEXITCODE -ne 0) { throw 'E2E HTTP client failed.' }
        if (-not $server.HasExited) { $server.Kill($true); $server.WaitForExit() }
        $server = $null
    }

    & $python -m unittest discover -v -s $fixture
    if ($LASTEXITCODE -ne 0) { throw 'Independent Python verification failed.' }
    foreach ($required in 'graph_metrics.py','test_graph_metrics.py') {
        if (-not (Test-Path -LiteralPath (Join-Path $fixture $required))) { throw "Expected artifact is missing: $required" }
    }
    if ((Get-ChildItem -LiteralPath (Join-Path $bundle 'state\journals') -File -ErrorAction SilentlyContinue | Measure-Object).Count -ne 0) { throw 'Journal residue detected.' }
    if ((Get-ChildItem -LiteralPath (Join-Path $bundle 'state\backups') -Recurse -File -ErrorAction SilentlyContinue | Measure-Object).Count -ne 0) { throw 'Temporary backup residue detected.' }
    if ((Get-Acl -LiteralPath $fixture).Sddl -ne $fixtureAclBefore) { throw 'Fixture ACL was not restored byte-for-byte.' }
    $auditEvents = Get-Content -LiteralPath (Join-Path $bundle 'logs\audit.jsonl') | ForEach-Object { $_ | ConvertFrom-Json }
    $appContainerSid = ($auditEvents | Where-Object Event -eq 'acl.apply' | Select-Object -First 1).Data.sid
    if (-not $appContainerSid) { throw 'Audit is missing the AppContainer SID.' }
    foreach ($path in @($fixture) + @(Get-ChildItem -LiteralPath $fixture -Recurse -Force | Select-Object -ExpandProperty FullName)) {
        if ((Get-Acl -LiteralPath $path).Sddl.Contains($appContainerSid, [StringComparison]::OrdinalIgnoreCase)) {
            throw "AppContainer ACL residue detected: $path"
        }
    }
    if (-not ($auditEvents | Where-Object Event -eq 'process.start')) { throw 'Audit is missing process.start.' }
    if (-not ($auditEvents | Where-Object Event -eq 'process.exit')) { throw 'Audit is missing process.exit.' }
    if (-not ($auditEvents | Where-Object { $_.Event -eq 'network.rule.remove' -and $_.Data.residual_filters -eq '0' })) { throw 'Audit is missing zero-residual network.rule.remove.' }
    $orphan = Get-Process -Name agent-mcp -ErrorAction SilentlyContinue | Where-Object { try { $_.Path -eq $agent } catch { $false } }
    if ($orphan) { throw 'Operational Bundle left an orphan agent-mcp process.' }
    $passed = $true
    Write-Output "Operational E2E $Transport/$HttpHost PASS with $pythonVersion"
}
catch {
    Write-Error "Operational E2E $Transport FAILED. Evidence retained at $root. $($_.Exception.Message)"
    throw
}
finally {
    if ($server -and -not $server.HasExited) { $server.Kill($true); $server.WaitForExit() }
    if ($appliedStorePath -and (Test-Path -LiteralPath $appliedStorePath)) {
        Remove-Item -LiteralPath $appliedStorePath -Force
    }
    if ($passed -and (Test-Path -LiteralPath $root)) { Remove-Item -LiteralPath $root -Recurse -Force }
}