Sol 6 与 Sol 6.1 在自有仓库任务上的对比：更便宜且至少同样好
I ran Sol 6 vs. 6.1 on tasks from my own repo. It's cheaper and at least as good
在自有仓库任务上对比 Sol 6 与 Sol 6.1，结果显示 Sol 6.1 更便宜且至少同样好。该对比基于 stet 的 repair-ai-coverage 流程，涉及 code_review 提示词解析重试、覆盖率修复与评分器重评等环节。
译文尚不完整，完整内容请切换到原文。
…stet runs repair-ai-coverage --out /path/to/h2h-output --model-key model-one --c
292292
293293Use `--parse-retries` when saved `code_review` prompts exist but evaluator
294294responses failed JSON/schema parsing. The command rewrites validation artifacts
295−only after a successful repair and regenerates summaries/task details.
295+only after a successful repair and regenerates summaries/task details. Stored
296+prompts retain their measurement mode and truncation. Missing-prompt repair
297+requires the original task projection, matching artifact digests, persisted test
298+commands, and complete repository context; unavailable provenance stays
299+unresolved. Attempt counts and retry reasons distinguish recovery from an
300+initial successful grade; assertions and unrecoverable runtime errors are not
301+retried.
296302
297303Regrade additive custom or bundled graders from canonical task details:
298304
…const coverageValidationGolden = `{
843843 },
844844 "equiv_rescue_policy": "on",
845845 "equivalence": {
846+ "attempt_count": 1,
846847 "confidence": 0.95,
847848 "evaluator_model": "judge-a",
848849 "issue_type": "stylistic",
…func (p *repairAICoverageProducer) Produce(mctx *MutationContext) (*ProduceResul
274274 var mu sync.Mutex
275275 runWithConcurrency(jobs, min(len(jobs), coverageRepairWorkers), func(job repairJob) {
276276 reporter.recordTaskStart(variant, job.TaskID, job.CurrentWork, time.Now().UTC())
277− result, repairErr := validate.RepairCoverage(ctx, validate.CoverageRepairConfig{
277+ repairCfg := validate.CoverageRepairConfig{
278278 Existing: job.Validation,
279279 AI: ai,
280280 AIModelID: resolvedAIModelID,
281281 RepairEquivalence: repairEquivalence && needsRepairCoverage(job.Validation, TaskGraderEquivalence, resolvedAIModelID),
282282 RepairCodeReview: repairCodeReview && needsRepairCoverage(job.Validation, TaskGraderCodeReview, resolvedAIModelID),
283283 ReviewParseRetries: cfg.ParseRetries,
284− })
284+ }
285+ cleanup, reconstructionIssues := prepareCoverageRepairPrompts(&repairCfg, manifest, NewTaskArtifacts(outputRoot, variant, job.TaskID, ""))
286+ if cleanup != nil {
287+ defer cleanup()
288+ }
289+ result := validate.CoverageRepairResult{Validation: job.Validation}
290+ var repairErr error
291+ if repairCfg.RepairEquivalence || repairCfg.RepairCodeReview {
292+ result, repairErr = validate.RepairCoverage(ctx, repairCfg)
293+ }
294+ result.Unresolved = append(result.Unresolved, reconstructionIssues...)
285295 if repairErr != nil {
286296 reporter.recordTaskEnd(variant, job.TaskID, job.CurrentWork, "failed", time.Now().UTC())
287297 mu.Lock()
…func RunAdaptedReferenceTest(ctx context.Context, ai AIClient, promptCtx PromptC
292292 composedDiff, composeErr := composeAdaptedReferenceTestPatch(runCtx, inputs, referenceTestPatch, canonicalDiff)
293293 if composeErr != nil {
294294 result.Reason = "patch_compose_failed"
295− result.Summary = fmt.Sprintf("compose canonical reference test patch and adapted_diff: %v", composeErr)
295+ result.Summary = fmt.Sprintf("adapted_diff could not be composed with the canonical reference test patch: %v", composeErr)
296296 return result
297297 }
298298 composedPayload := payload
…import (
99 "strings"
1010 "time"
1111
12+ "stet/internal/commandlist"
1213 "stet/internal/filecopy"
1314 "stet/internal/gitops"
1415 "stet/internal/harness"
…func legacyCommandsThatNameAgentTests(commands, agentFiles []string) []string {
192193 files := normalizeAgentTestFiles(agentFiles)
193194 var out []string
194195 for _, command := range normalizeCommands(commands) {
195− if commandHasPotentialNonPathTestSelection(command) || receiptBackedCommandHasShellControl(command) {
196+ if _, _, safe := commandlist.TestInvocation(command); !safe || commandHasPotentialNonPathTestSelection(command) || receiptBackedCommandHasShellControl(command) {
196197 continue
197198 }
198199 for _, file := range files {
…import (
66 "fmt"
77 "os"
88 "path/filepath"
9+ "strconv"
910 "strings"
1011
12+ "stet/internal/commandlist"
1113 "stet/internal/shellcmd"
1214)
1315
…func replaceExactFinalTestPath(command, sourcePath, replacement string) (string,
198200}
199201
200202func certifiedGoReplayCommand(command, sourcePath string) (string, bool) {
201− if !strings.HasSuffix(strings.ToLower(sourcePath), ".go") || !goCommandTargetsAgentTestFiles(command, []string{sourcePath}) {
203+ _, fields, safe := commandlist.TestInvocation(command)
204+ if !safe || len(fields) < 3 || fields[0].Value != "go" || fields[1].Value != "test" || !strings.HasSuffix(strings.ToLower(sourcePath), ".go") {
202205 return "", false
203206 }
204− replay, changed := removeGoRunSelector(command)
205− if !changed || hasRemainingGoTestSelector(replay) {
207+ // Require one explicit package. A recursive selector or additional package
208+ // can credit tests outside the certified source package.
209+ sourceDirectory := filepath.Dir(sourcePath)
210+ allFields, _ := shellcmd.FieldSpans(command)
211+ // Go also consumes selectors from GOFLAGS. Recorded overrides are opaque
212+ // to this bounded rewrite, so they cannot certify an unfiltered replay.
213+ for _, field := range allFields {
214+ if field.Start >= fields[0].Start {
215+ break
216+ }
217+ if strings.HasPrefix(field.Value, "GOFLAGS=") {
218+ return "", false
219+ }
220+ }
221+ if len(allFields) >= 3 && allFields[0].Value == "cd" {
222+ relative, err := filepath.Rel(allFields[1].Value, sourceDirectory)
223+ if err != nil || relative == ".." || strings.HasPrefix(relative, "../") {
224+ return "", false
225+ }
226+ sourceDirectory = relative
227+ }
228+ packagePath := "./" + filepath.ToSlash(sourceDirectory)
229+ if sourceDirectory == "." {
230+ packagePath = "."
231+ }
232+ if strings.TrimSuffix(fields[2].Value, "/") != packagePath {
233+ return "", false
234+ }
235+ start, end := -1, -1
236+ for i := 3; i < len(fields); i++ {
237+ value := fields[i].Value
238+ if value == "-run" || strings.HasPrefix(value, "-run=") {
239+ if start >= 0 {
240+ return "", false
241+ }
242+ start, end = fields[i].Start, fields[i].End
243+ if value == "-run" {
244+ if i+1 >= len(fields) || strings.HasPrefix(fields[i+1].Value, "-") {
245+ return "", false
246+ }
247+ i++
248+ end = fields[i].End
249+ }
250+ continue
251+ }
252+ // Only flags with self-contained values may remain. Positional words,
253+ // passthrough args, and further selectors cannot be proven safe.
254+ name := strings.SplitN(value, "=", 2)[0]
255+ if name == "-count" {
256+ countText, hasValue := strings.CutPrefix(value, "-count=")
257+ count, err := strconv.Atoi(countText)
258+ if !hasValue || err != nil || count <= 0 {
259+ return "", false
260+ }
261+ }
262+ switch name {
263+ case "-v", "-race", "-short", "-count", "-timeout", "-parallel", "-shuffle", "-tags", "-mod":
264+ if !strings.Contains(value, "=") && name != "-v" && name != "-race" && name != "-short" {
265+ return "", false
266+ }
267+ default:
268+ return "", false
269+ }
270+ }
271+ if start < 0 {
206272 return "", false
207273 }
208− return replay, true
274+ return strings.TrimSpace(strings.TrimRight(command[:start], " \t") + command[end:]), true
209275}
210276
211277func normalizeReplayRelativePath(path string) string {
…func TestValidateAgentTestReplayPlanRejectsUnprovenCommand(t *testing.T) {
6161 t.Fatalf("validation error = %v", err)
6262 }
6363}
64+
65+func TestCertifiedGoReplayPreservesOnlyProvenInvocation(t *testing.T) {
66+ command := "cd module && env MODE='test value' timeout 30s go test ./pkg -run 'TestHidden|TestOther' -count=1"
67+ got, ok := certifiedGoReplayCommand(command, "module/pkg/hidden_test.go")
68+ if !ok || got != "cd module && env MODE='test value' timeout 30s go test ./pkg -count=1" {
69+ t.Fatalf("replay = %q, %v", got, ok)
70+ }
71+ for _, unsafe := range []string{
72+ "echo go test ./pkg -run TestHidden", "go test ./pkg -run TestHidden; true",
73+ "go test ./pkg -run TestHidden && go test ./other", "go test ./... -run TestHidden",
74+ "go test ./pkg ./other -run TestHidden", "go test ./pkg -run TestHidden -skip TestOther",
75+ "go test ./pkg -run TestHidden -args payload", "go test ./pkg -run TestHidden -run TestOther",
76+ "go test ./pkg -run TestHidden -count=0", "go test ./pkg -run TestHidden -count=-1",
77+ "go test ./pkg -run TestHidden -count=invalid", "go test ./pkg -run TestHidden -count 0",
78+ "go test ./pkg -run TestHidden -count -1",
79+ "GOFLAGS=-count=0 go test ./pkg -run TestHidden",
80+ "GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden",
81+ "env GOFLAGS=-count=0 go test ./pkg -run TestHidden",
82+ "cd module && env GOFLAGS='-run=Nothing' go test ./pkg -run TestHidden",
83+ } {
84+ if replay, ok := certifiedGoReplayCommand(unsafe, "pkg/hidden_test.go"); ok {
85+ t.Errorf("accepted %q as %q", unsafe, replay)
86+ }
87+ }
88+}
89+
90+func TestLegacyReplayRequiresAnActualTestRunner(t *testing.T) {
91+ commands := legacyCommandsThatNameAgentTests([]string{
92+ "echo tests/agent_test.sh", "cat tests/agent_test.sh", "rm tests/agent_test.sh",
93+ "sh -c tests/agent_test.sh", "'CI=1' sh tests/agent_test.sh", "sh tests/agent_test.sh",
94+ }, []string{"tests/agent_test.sh"})
95+ if len(commands) != 1 || commands[0] != "sh 'tests/agent_test.sh'" {
96+ t.Fatalf("legacy replay commands = %q", commands)
97+ }
98+}
…func buildEquivalenceRetryPrompt(originalPrompt, rawOutput string, parseErr erro
1818 )
1919}
2020
21−func RunEquivalence(ctx context.Context, ai AIClient, promptCtx PromptContext) EquivalenceResult {
21+func RunEquivalence(ctx context.Context, ai AIClient, promptCtx PromptContext) (result EquivalenceResult) {
22+ mode := EquivalencePromptModeStandard
23+ if goldBlindEquivalenceEnabled() {
24+ mode = EquivalencePromptModeGoldBlind
25+ }
26+ history := newGradeAttemptClient(ai)
27+ ai = history
28+ defer func() { result.PromptMode = mode; history.annotateEquivalence(&result) }()
29+
2230 if goldBlindEquivalenceEnabled() {
2331 // No gold reaches the model, so gold truncation cannot have mutilated
2432 // the graded evidence: clear it before the truncation annotation runs.
…
11package validate
22
33import (
4+ "fmt"
45 "os"
56 "strconv"
67 "strings"
…notes: <short explanation tied to the task obligations>
127128// mode. In gold-blind mode the gold patch is dropped from the template data as
128129// well as from the template, so no gold content can reach the model.
129130func renderEquivalencePrompt(promptCtx PromptContext) (prompt string, goldBlind bool, err error) {
130− if !goldBlindEquivalenceEnabled() {
131− rendered, renderErr := renderPrompt(equivalencePromptTemplate, promptCtx.asTemplateData())
132− return rendered, false, renderErr
131+ mode := EquivalencePromptModeStandard
132+ if goldBlindEquivalenceEnabled() {
133+ mode = EquivalencePromptModeGoldBlind
134+ }
135+ rendered, err := RenderEquivalencePromptForMode(promptCtx, mode)
136+ return rendered, mode == EquivalencePromptModeGoldBlind, err
137+}
138+
139+// RenderEquivalencePromptForMode uses recorded measurement authority, never the process environment.
140+func RenderEquivalencePromptForMode(promptCtx PromptContext, mode EquivalencePromptMode) (string, error) {
141+ switch mode {
142+ case EquivalencePromptModeStandard:
143+ return renderPrompt(equivalencePromptTemplate, promptCtx.asTemplateData())
144+ case EquivalencePromptModeGoldBlind:
145+ data := promptCtx.asTemplateData()
146+ delete(data, "GoldPatch")
147+ return renderPrompt(equivalenceGoldBlindPromptTemplate, data)
148+ default:
149+ return "", fmt.Errorf("unknown equivalence prompt mode %q", mode)
133150 }
134− data := promptCtx.asTemplateData()
135− delete(data, "GoldPatch")
136− rendered, renderErr := renderPrompt(equivalenceGoldBlindPromptTemplate, data)
137− return rendered, true, renderErr
138151}
…func runPerCommandTests(
22022202 cmdCfg.Commands = []string{cmd}
22032203 goldRun, err := RunGoldBaselineTests(infraCtx, inputs, cmdCfg)
22042204 gr.EnvCache = goldRun.EnvCache
2205− gr.Result = TestCommandResult{Command: cmd}
2205+ gr.Result = TestCommandResult{Command: cmd, AttemptCount: 1}
22062206 if err != nil {
22072207 gr.Err = err
2208− if mode := inferInfraFailureModeFromError(err); mode != "" {
2208+ if mode := retryableVerifierFailureMode(goldRun, err); mode != "" {
22092209 gr.Result.GoldOutcome = TestOutcomeFail
22102210 gr.Result.GoldFailureMode = mode
22112211 gr.IsInfra = true
…func runPerCommandTests(
22182218 gr.Result.GoldOutcome = TestOutcomeUnknown
22192219 gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline failed for %q: %v", cmd, err))
22202220 }
2221− return
2221+ if !isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) {
2222+ return
2223+ }
22222224 }
22232225 gr.EnvCache = goldRun.EnvCache
22242226 applyGoldRunResult(gr, goldRun)
2225− if gr.Result.GoldOutcome == TestOutcomeFail && isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) && budgetCtx.Err() == nil {
2227+ if isUnsetFailureMode(gr.Result.GoldFailureMode) {
2228+ gr.Result.GoldFailureMode = retryableVerifierFailureMode(goldRun, err)
2229+ }
2230+ if gr.Result.GoldOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(gr.Result.GoldFailureMode) && budgetCtx.Err() == nil {
22262231 beginGoldRetry(&gr.Result, goldRun)
2232+ recordTestRetry(&gr.Result, gr.Result.GoldFailureMode)
22272233 retryCfg := goldRetryAttemptConfig(overrideCfg, gr.Index, 2)
22282234 retryCfg.Commands = []string{cmd}
22292235 retryRun, retryErr := RunGoldBaselineTests(infraCtx, inputs, retryCfg)
…func runPerCommandTests(
22442250 gr.Result.GoldRetried = true
22452251 finishGoldRetry(&gr.Result, retryRun)
22462252 if gr.Result.GoldOutcome == TestOutcomePass {
2253+ gr.Result.RecoveredAfterRetry = true
2254+ gr.Err = nil
2255+ gr.IsInfra = false
22472256 gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline for %q recovered on retry after transient dependency fetch", cmd))
22482257 } else {
22492258 gr.Notes = append(gr.Notes, fmt.Sprintf("gold baseline retry for %q did not recover", cmd))
…func runPerCommandTests(
24842493 cmdCfg := overrideCfg
24852494 cmdCfg.Commands = []string{entry.cmd}
24862495 or.Attempted = true
2496+ or.Result.AttemptCount++
24872497 overrideRun, err := RunOverrideTests(budgetCtx, inputs, cmdCfg)
24882498 or.EnvCache = overrideRun.EnvCache
24892499 if err != nil {
2490− if mode := inferInfraFailureModeFromError(err); mode != "" {
2500+ if mode := retryableVerifierFailureMode(overrideRun, err); mode != "" {
24912501 or.Result.AgentFailureMode = mode
24922502 if mode == "timeout" {
24932503 or.Notes = append(or.Notes, fmt.Sprintf("agent retest timeout for %q", entry.cmd))
…func runPerCommandTests(
24962506 or.Notes = append(or.Notes, fmt.Sprintf("agent retest failed for %q: %v", entry.cmd, err))
24972507 or.Result.AgentOutcome = TestOutcomeUnknown
24982508 or.Err = err
2499− return
2509+ if !isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) {
2510+ return
2511+ }
25002512 }
25012513 or.Result.AgentOutcome = testOutcomeFromTrial(overrideRun.Trial)
25022514 or.Result.AgentTrialName = overrideRun.Trial.TrialName
2503− or.Result.AgentFailureMode = overrideRun.Trial.FailureMode
2515+ or.Result.AgentFailureMode = retryableVerifierFailureMode(overrideRun, err)
25042516
2505− // Retry once if agent failed but gold passed.
2506− if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err() == nil {
2517+ // Retry only a typed transient infrastructure failure; assertions are terminal.
2518+ if or.Result.AgentOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) && or.Result.GoldOutcome == TestOutcomePass && budgetCtx.Err() == nil {
25072519 retryCfg := overrideCfg
25082520 retryCfg.Commands = []string{entry.cmd}
2521+ recordTestRetry(&or.Result, or.Result.AgentFailureMode)
2522+ or.Result.AgentRetried = true
25092523 retryRun, retryErr := RunOverrideTests(budgetCtx, inputs, retryCfg)
25102524 or.RetryEnvCache = &retryRun.EnvCache
2525+ or.Result.AgentOutcome = testOutcomeFromTrial(retryRun.Trial)
2526+ or.Result.AgentTrialName = retryRun.Trial.TrialName
2527+ or.Result.AgentFailureMode = retryableVerifierFailureMode(retryRun, retryErr)
2528+ or.Err = retryErr
25112529 if retryErr == nil {
25122530 retryOutcome := testOutcomeFromTrial(retryRun.Trial)
25132531 if retryOutcome == TestOutcomePass {
…func runPerCommandTests(
25152533 or.Result.AgentTrialName = retryRun.Trial.TrialName
25162534 or.Result.AgentFailureMode = retryRun.Trial.FailureMode
25172535 or.Result.AgentRetried = true
2536+ or.Result.RecoveredAfterRetry = true
2537+ or.Err = nil
25182538 or.Notes = append(or.Notes, fmt.Sprintf("agent retest for %q succeeded on retry", entry.cmd))
25192539 }
25202540 }
…func runGoldBatchSlice(
347347 cleanups := []func(){}
348348 for _, idx := range indexes {
349349 gr := &goldResults[idx]
350+ priorResult := gr.Result
350351 priorRetried, priorEvidence := gr.Result.GoldRetried, gr.Result.GoldRetryEvidence
351352 gr.IsInfra = false
352353 gr.IsGlobalInfra = false
353354 gr.Err = nil
354355 if ctx.Err() != nil {
355356 gr.Result = TestCommandResult{
356− Command: gr.Command,
357− GoldOutcome: TestOutcomeUnknown,
358− GoldRetried: priorRetried,
359− GoldRetryEvidence: priorEvidence,
357+ Command: gr.Command,
358+ GoldOutcome: TestOutcomeUnknown,
359+ GoldRetried: priorRetried,
360+ GoldRetryEvidence: priorEvidence,
361+ AttemptCount: priorResult.AttemptCount,
362+ RetryCount: priorResult.RetryCount,
363+ RetryReasons: priorResult.RetryReasons,
364+ RecoveredAfterRetry: priorResult.RecoveredAfterRetry,
360365 }
361366 gr.Err = ctx.Err()
362367 continue
…func runGoldBatchSlice(
364369 stageRoot, cacheResult, cleanup, err := stageGoldBaselineTask(ctx, inputs, []string{commands[idx]}, false, overrideCfg.OutputDir)
365370 gr.EnvCache = cacheResult
366371 gr.Result = TestCommandResult{
367− Command: gr.Command,
368− GoldRetried: priorRetried,
369− GoldRetryEvidence: priorEvidence,
372+ Command: gr.Command,
373+ GoldRetried: priorRetried,
374+ GoldRetryEvidence: priorEvidence,
375+ AttemptCount: priorResult.AttemptCount,
376+ RetryCount: priorResult.RetryCount,
377+ RetryReasons: priorResult.RetryReasons,
378+ RecoveredAfterRetry: priorResult.RecoveredAfterRetry,
370379 }
371380 if err != nil {
372381 gr.Err = err
…func runGoldBatchSlice(
407416 gr.Result.GoldRetried = true
408417 gr.Result.GoldRetryEvidence = &TestRetryEvidence{First: *gr.FirstRetryEvidence}
409418 }
419+ recordTestRetry(&gr.Result, FailureModeInfraDependencyFetch)
410420 gr.RetryDispatched = true
411421 }
412422 }
413423
424+ for _, item := range items {
425+ var idx int
426+ fmt.Sscanf(item.ProjectionKey, "gold-%d", &idx)
427+ if attempt == 1 {
428+ goldResults[idx].Result.AttemptCount++
429+ }
430+ }
414431 results, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{
415432 OutputDir: filepath.Join(filepath.Dir(items[0].SourceTaskDir), ".tb-runs"),
416433 RunID: fmt.Sprintf("validate-gold-batch-%d", time.Now().UnixNano()),
…func runGoldBatchSlice(
452469 evidence := retainBatchGoldRetryAttemptEvidence(projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt)
453470 gr.FirstRetryEvidence = &evidence
454471 }
472+ if attempt > 1 && gr.Result.GoldOutcome == TestOutcomePass {
473+ gr.Result.RecoveredAfterRetry = true
474+ }
455475 if attempt > 1 && gr.Result.GoldRetryEvidence != nil {
456476 evidence := retainBatchGoldRetryAttemptEvidence(projected, gr.Result.GoldFailureMode, overrideCfg, gr.Index, attempt)
457477 gr.Result.GoldRetryEvidence.Retry = &evidence
…func runOverrideBatchSlice(
564584 return
565585 }
566586
587+ for _, item := range items {
588+ var idx int
589+ fmt.Sscanf(item.ProjectionKey, "override-%d", &idx)
590+ overrideResults[idx-offset].Result.AttemptCount++
591+ }
592+
567593 results, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{
568594 OutputDir: filepath.Join(filepath.Dir(items[0].SourceTaskDir), ".tb-runs"),
569595 RunID: fmt.Sprintf("validate-override-batch-%d", time.Now().UnixNano()),
…func runOverrideBatchSlice(
604630 projected := results[fmt.Sprintf("override-%d", offset+i)]
605631 or.Result.AgentOutcome = testOutcomeFromTrial(projected.Trial)
606632 or.Result.AgentTrialName = projected.Trial.TrialName
607− or.Result.AgentFailureMode = projected.Trial.FailureMode
608− if or.Result.AgentOutcome == TestOutcomeFail && or.Result.GoldOutcome == TestOutcomePass && ctx.Err() == nil {
633+ or.Result.AgentFailureMode = classifyBatchGoldFailureMode(projected)
634+ if or.Result.AgentOutcome != TestOutcomePass && isRetryableGoldInfrastructureFailure(or.Result.AgentFailureMode) && or.Result.GoldOutcome == TestOutcomePass && ctx.Err() == nil {
609635 stageRoot, cacheResult, cleanup, err := stageOverrideTask(ctx, inputs, []string{or.Command}, false, overrideCfg.OutputDir)
610636 if err == nil {
611637 or.RetryEnvCache = &cacheResult
…func runOverrideBatchSlice(
623649 if len(retryItems) == 0 {
624650 return
625651 }
652+ for _, idx := range retryIndexes {
653+ or := &overrideResults[idx]
654+ recordTestRetry(&or.Result, or.Result.AgentFailureMode)
655+ or.Result.AgentRetried = true
656+ }
626657 retryResults, err := batchRunner.RunBatch(ctx, harness.BatchRunConfig{
627658 OutputDir: filepath.Join(filepath.Dir(retryItems[0].SourceTaskDir), ".tb-runs"),
628659 RunID: fmt.Sprintf("validate-override-retry-batch-%d", time.Now().UnixNano()),
…func runOverrideBatchSlice(
634665 HarnessCLICacheMode: overrideCfg.HarnessCLICacheMode,
635666 Items: retryItems,
636667 })
637− if err != nil {
638− return
639− }
640668 for _, idx := range retryIndexes {
641669 or := &overrideResults[idx]
642670 projected := retryResults[fmt.Sprintf("retry-%d", offset+idx)]
643− if testOutcomeFromTrial(projected.Trial) == TestOutcomePass {
644− or.Result.AgentOutcome = TestOutcomePass
645− or.Result.AgentTrialName = projected.Trial.TrialName
646− or.Result.AgentFailureMode = projected.Trial.FailureMode
647− or.Result.AgentRetried = true
671+ or.Result.AgentOutcome = testOutcomeFromTrial(projected.Trial)
672+ or.Result.AgentTrialName = projected.Trial.TrialName
673+ or.Result.AgentFailureMode = classifyBatchGoldFailureMode(projected)
674+ or.Err = err
675+ if err != nil {
676+ if isUnsetFailureMode(or.Result.AgentFailureMode) {
677+ or.Result.AgentFailureMode = inferInfraFailureModeFromError(err)
678+ }
679+ or.Notes = append(or.Notes, fmt.Sprintf("agent retest retry for %q failed: %v", or.Command, err))
680+ }
681+ if err == nil && or.Result.AgentOutcome == TestOutcomePass {
682+ or.Result.RecoveredAfterRetry = true
648683 or.Notes = append(or.Notes, fmt.Sprintf("agent retest for %q succeeded on retry", or.Command))
649684 }
650685 }
…go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/
50215021 t.Fatalf("command results = %d, want 1", len(summary.CommandResults))
50225022 }
50235023 result := summary.CommandResults[0]
5024+ if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || len(result.RetryReasons) != 1 || result.RetryReasons[0] != FailureModeInfraDependencyFetch {
5025+ t.Fatalf("retry metadata = %+v", result)
5026+ }
50245027 if !result.GoldRetried || result.GoldOutcome != TestOutcomePass || result.GoldTrialName != "gold-retry" {
50255028 t.Fatalf("gold retry result = %+v", result)
50265029 }
…go: golang.org/x/sys@v0.42.0: Get "https://proxy.golang.org/golang.org/x/sys/@v/
51115114 t.Fatalf("command results = %d, want 1", len(summary.CommandResults))
51125115 }
51135116 result := summary.CommandResults[0]
5117+ if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || len(result.RetryReasons) != 1 || result.RetryReasons[0] != FailureModeInfraDependencyFetch {
5118+ t.Fatalf("retry metadata = %+v", result)
5119+ }
51145120 if !result.GoldRetried || result.GoldOutcome != TestOutcomePass || result.GoldTrialName != "gold-retry" {
51155121 t.Fatalf("gold retry result = %+v", result)
51165122 }
…func TestRunPerCommandTestsBatchedGoldTimeoutClassifiedAsTimeBudgetExhausted(t *
53825388 }
53835389}
53845390
5385−func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testing.T) {
5391+func TestRunPerCommandTestsBatchedRetriesDependencyFetchOverrideAndMarksRetried(t *testing.T) {
53865392 t.Parallel()
53875393 repo, base, head := createRepoWithCommits(t)
53885394 dataset := t.TempDir()
…func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testin
54185424 runner := &batchValidateRunner{
54195425 results: map[string]TBTrialResult{
54205426 "gold-0": {TaskID: taskID, IsResolved: &pass, TrialName: "gold-a"},
5421− "override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a"},
5427+ "override-0": {TaskID: taskID, IsResolved: &fail, TrialName: "override-a", FailureMode: FailureModeInfraDependencyFetch},
54225428 "retry-0": {TaskID: taskID, IsResolved: &pass, TrialName: "retry-a"},
54235429 },
54245430 }
…func TestRunPerCommandTestsBatchedRetriesFailedOverrideAndMarksRetried(t *testin
54435449 t.Fatalf("expected 1 command result, got %d", len(result.Tests.CommandResults))
54445450 }
54455451 cmd := result.Tests.CommandResults[0]
5452+ if cmd.AttemptCount != 3 || cmd.RetryCount != 1 || !cmd.RecoveredAfterRetry || len(cmd.RetryReasons) != 1 || cmd.RetryReasons[0] != FailureModeInfraDependencyFetch {
5453+ t.Fatalf("retry metadata = %+v", cmd)
5454+ }
54465455 if !cmd.AgentRetried {
54475456 t.Fatal("expected AgentRetried=true")
54485457 }
…
11package validate
22
3−import "fmt"
3+import (
4+ "fmt"
5+ "path/filepath"
6+ "strings"
7+)
48
59// OfflinePromptInputs describes everything needed to reassemble the grader
610// PromptContext for a single already-evaluated cell without re-running the
…type OfflinePromptInputs struct {
1822 // TBResultsRaw is the raw TB results JSON for this cell, or "" when
1923 // unavailable (e.g. no-test-signal corpora).
2024 TBResultsRaw string
25+ // EffectiveTestCommands overrides dataset commands when non-nil. An empty
26+ // non-nil slice preserves a persisted grade with no verifier commands.
27+ EffectiveTestCommands []string
2128 // PromptPatchMaxBytes caps the gold/agent patch bytes embedded in the
2229 // prompt. Values <= 0 fall back to DefaultPromptPatchMaxBytes, matching the
2330 // orchestrator/config default.
…type OfflinePromptInputs struct {
2532 // RepoContextOptions controls repo tree/excerpt extraction. The zero value
2633 // inherits BuildRepoContext defaults, matching the live default path.
2734 RepoContextOptions RepoContextOptions
35+ // RequireRepoContext makes reconstruction fail when the source repository
36+ // cannot be extracted, instead of grading an unavailable-context marker.
37+ RequireRepoContext bool
38+ // ExpectedArtifacts binds reconstruction to the original graded files.
39+ // When supplied, every source path and digest must be present and match.
40+ ExpectedArtifacts *ArtifactManifest
2841}
2942
3043// BuildOfflinePromptContext reassembles the grader PromptContext for one cell
…type OfflinePromptInputs struct {
3447// that removes the extracted repo temp dir; callers should defer it.
3548//
3649// The assembly intentionally mirrors internal/validate/orchestrator.go so the
37−// resulting context is byte-for-byte identical to a live grade except for the
38−// template version: it reuses ReadTaskInputs, truncatePatch, and
39−// BuildRepoContext, and applies the same graceful repo-context fallback.
50+// assembly reuses ReadTaskInputs, truncatePatch, and BuildRepoContext. Callers
51+// can supply persisted effective commands and require usable repository context;
52+// otherwise it applies the live path's graceful repo-context fallback.
4053func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), error) {
4154 inputs, err := ReadTaskInputs(in.DatasetPath, in.TaskID, in.AgentPatchPath)
4255 if err != nil {
…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e
4558 if err := verifyRepoSourceForConsumption(inputs); err != nil {
4659 return PromptContext{}, nil, err
4760 }
61+ if in.ExpectedArtifacts != nil {
62+ current := BuildArtifactManifest(inputs)
63+ for _, pair := range []struct {
64+ name string
65+ want, got ArtifactFileManifest
66+ }{
67+ {"agent patch", in.ExpectedArtifacts.AgentPatch, current.AgentPatch},
68+ {"gold patch", in.ExpectedArtifacts.GoldPatch, current.GoldPatch},
69+ {"repository", in.ExpectedArtifacts.RepoTarball, current.RepoTarball},
70+ } {
71+ if pair.want.Path == "" || pair.want.SHA256 == "" {
72+ return PromptContext{}, nil, fmt.Errorf("missing original %s path or digest", pair.name)
73+ }
74+ if filepath.Clean(pair.want.Path) != filepath.Clean(pair.got.Path) || !strings.EqualFold(strings.TrimPrefix(pair.want.SHA256, "sha256:"), pair.got.SHA256) {
75+ return PromptContext{}, nil, fmt.Errorf("original %s provenance mismatch", pair.name)
76+ }
77+ }
78+ }
4879
4980 maxBytes := in.PromptPatchMaxBytes
5081 if maxBytes <= 0 {
…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e
5283 }
5384 goldPatchPrompt, goldPatchTruncation := truncatePatch(inputs.GoldPatch, maxBytes)
5485 agentPatchPrompt, agentPatchTruncation := truncatePatch(inputs.AgentPatch, maxBytes)
86+ if in.ExpectedArtifacts != nil && (goldPatchTruncation.Truncated || agentPatchTruncation.Truncated) {
87+ return PromptContext{}, nil, fmt.Errorf("reconstructed prompt patches would be truncated; original context budgets are unavailable")
88+ }
89+ testCommands := inputs.TestCommands
90+ if in.EffectiveTestCommands != nil {
91+ testCommands = append([]string{}, in.EffectiveTestCommands...)
92+ }
5593
5694 // Mirror orchestrator.go: on repo-context failure, degrade RepoTree to an
5795 // "unavailable: <err>" marker and continue rather than failing the build.
…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e
5997 repoCtx := RepoContext{}
6098 builtRepoCtx, repoCleanup, repoErr := BuildRepoContext(inputs.RepoTarballPath, in.RepoContextOptions)
6199 if repoErr != nil {
100+ if in.RequireRepoContext {
101+ return PromptContext{}, nil, fmt.Errorf("build required repo context: %w", repoErr)
102+ }
62103 builtRepoCtx = RepoContext{
63104 RepoTree: fmt.Sprintf("unavailable: %v", repoErr),
64105 }
65106 } else {
66107 cleanup = repoCleanup
108+ if in.ExpectedArtifacts != nil && (builtRepoCtx.RepoTreeTruncation.Truncated || builtRepoCtx.RepoExcerptsTruncation.Truncated) {
109+ if cleanup != nil {
110+ cleanup()
111+ }
112+ return PromptContext{}, nil, fmt.Errorf("reconstructed repository context would be truncated; original context budgets are unavailable")
113+ }
67114 }
68115 repoCtx = builtRepoCtx
69116
…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e
71118 TaskYAML: inputs.TaskYAML,
72119 GoldPatch: goldPatchPrompt,
73120 AgentPatch: agentPatchPrompt,
74− TestCommands: inputs.TestCommands,
121+ TestCommands: testCommands,
75122 TBResults: in.TBResultsRaw,
76123 RepoPath: repoCtx.RepoPath,
77124 RepoTree: repoCtx.RepoTree,
…func BuildOfflinePromptContext(in OfflinePromptInputs) (PromptContext, func(), e
80127 GoldPatchTruncated: goldPatchTruncation.Truncated,
81128 }, cleanup, nil
82129}
130+
131+// RenderReviewPrompt renders the built-in review template from supplied evidence.
132+func RenderReviewPrompt(promptCtx PromptContext) (string, error) {
133+ return renderPrompt(reviewPromptTemplate, promptCtx.asTemplateData())
134+}
…func TestBuildOfflinePromptContextDegradesWhenTarballUnextractable(t *testing.T)
164164 t.Fatalf("expected agent patch to be populated, got %q", promptCtx.AgentPatch)
165165 }
166166}
167+
168+func TestBuildOfflinePromptContextRequiresUsableRepositoryWhenRequested(t *testing.T) {
169+ datasetPath, taskID, agentPatchPath := writeOfflineTaskDir(t, false)
170+ _, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{
171+ DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath,
172+ RequireRepoContext: true,
173+ })
174+ if cleanup != nil {
175+ defer cleanup()
176+ }
177+ if err == nil || !strings.Contains(err.Error(), "build required repo context") {
178+ t.Fatalf("expected required repository extraction failure, got %v", err)
179+ }
180+}
181+
182+func TestBuildOfflinePromptContextUsesEffectiveVerifierCommands(t *testing.T) {
183+ datasetPath, taskID, agentPatchPath := writeOfflineTaskDir(t, true)
184+ for _, commands := range [][]string{nil, {}, {"effective verifier"}} {
185+ ctx, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{
186+ DatasetPath: datasetPath, TaskID: taskID, AgentPatchPath: agentPatchPath,
187+ EffectiveTestCommands: commands,
188+ })
189+ if cleanup != nil {
190+ defer cleanup()
191+ }
192+ if err != nil {
193+ t.Fatal(err)
194+ }
195+ want := commands
196+ if commands == nil {
197+ want = []string{"pytest -q"}
198+ }
199+ if !reflect.DeepEqual(ctx.TestCommands, want) {
200+ t.Fatalf("effective commands %v: got %v, want %v", commands, ctx.TestCommands, want)
201+ }
202+ }
203+}
204+
205+func TestBuildOfflinePromptContextRejectsMutatedOriginalArtifacts(t *testing.T) {
206+ for _, artifact := range []string{"agent patch", "gold patch", "repository"} {
207+ t.Run(artifact, func(t *testing.T) {
208+ dataset, taskID, patch := writeOfflineTaskDir(t, true)
209+ inputs, err := ReadTaskInputs(dataset, taskID, patch)
210+ if err != nil {
211+ t.Fatal(err)
212+ }
213+ original := BuildArtifactManifest(inputs)
214+ path := patch
215+ if artifact == "gold patch" {
216+ path = inputs.GoldPatchPath
217+ }
218+ if artifact == "repository" {
219+ path = inputs.RepoTarballPath
220+ }
221+ if err := os.WriteFile(path, []byte("mutated evidence"), 0o644); err != nil {
222+ t.Fatal(err)
223+ }
224+ _, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{
225+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch,
226+ ExpectedArtifacts: &original, RequireRepoContext: true,
227+ })
228+ if cleanup != nil {
229+ defer cleanup()
230+ }
231+ if err == nil || !strings.Contains(err.Error(), artifact+" provenance mismatch") {
232+ t.Fatalf("expected original %s mismatch, got %v", artifact, err)
233+ }
234+ })
235+ }
236+}
237+
238+func TestBuildOfflinePromptContextRejectsAbsentOriginalArtifactDigest(t *testing.T) {
239+ dataset, taskID, patch := writeOfflineTaskDir(t, true)
240+ inputs, err := ReadTaskInputs(dataset, taskID, patch)
241+ if err != nil {
242+ t.Fatal(err)
243+ }
244+ original := BuildArtifactManifest(inputs)
245+ original.AgentPatch.SHA256 = ""
246+ _, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{
247+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original,
248+ })
249+ if cleanup != nil {
250+ defer cleanup()
251+ }
252+ if err == nil || !strings.Contains(err.Error(), "missing original agent patch") {
253+ t.Fatalf("expected absent original provenance failure, got %v", err)
254+ }
255+}
256+
257+func TestBuildOfflinePromptContextRejectsNewTruncationWithOriginalProvenance(t *testing.T) {
258+ dataset, taskID, patch := writeOfflineTaskDir(t, true)
259+ if err := os.WriteFile(patch, []byte(largePatch(4_000)), 0o644); err != nil {
260+ t.Fatal(err)
261+ }
262+ inputs, err := ReadTaskInputs(dataset, taskID, patch)
263+ if err != nil {
264+ t.Fatal(err)
265+ }
266+ original := BuildArtifactManifest(inputs)
267+ _, cleanup, err := BuildOfflinePromptContext(OfflinePromptInputs{
268+ DatasetPath: dataset, TaskID: taskID, AgentPatchPath: patch, ExpectedArtifacts: &original,
269+ PromptPatchMaxBytes: 600,
270+ })
271+ if cleanup != nil {
272+ defer cleanup()
273+ }
274+ if err == nil || !strings.Contains(err.Error(), "would be truncated") {
275+ t.Fatalf("expected new truncation refusal, got %v", err)
276+ }
277+}
…const (
2828var coverageRepairRetryBackoffs = []time.Duration{time.Second, 2 * time.Second}
2929
3030type CoverageRepairConfig struct {
31+ EquivalenceAI AIClient
32+ EquivalencePrompt string
33+ EquivalencePromptContext *PromptContext
34+ EquivalencePromptMode EquivalencePromptMode
35+ ReviewAI AIClient
36+ ReviewPrompt string
37+ ReviewPromptContext *PromptContext
38+
3139 Existing ValidationResult
3240 AI AIClient
3341 AIModelID string
…type CoverageRepairConfig struct {
3745}
3846
3947type CoverageRepairIssue struct {
48+ AttemptCount int `json:"attempt_count,omitempty"`
49+ RetryCount int `json:"retry_count,omitempty"`
50+ RetryReasons []string `json:"retry_reasons,omitempty"`
51+
4052 Dimension string `json:"dimension"`
4153 Reason string `json:"reason"`
4254 Category string `json:"category,omitempty"`
…func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa
6981 updated := false
7082
7183 if cfg.RepairEquivalence {
72− equivalence, repaired, issue := repairEquivalenceFromPrompt(ctx, ai, cfg.Existing.Equivalence, cfg.AIModelID, cfg.ReviewParseRetries)
84+ source := cfg.Existing.Equivalence
85+ if cfg.EquivalencePrompt != "" {
86+ source.Prompt = cfg.EquivalencePrompt
87+ }
88+ mode := source.PromptMode
89+ if cfg.EquivalencePromptMode != "" {
90+ if mode != "" && mode != cfg.EquivalencePromptMode {
91+ return result, errors.New("equivalence repair prompt mode differs from original measurement")
92+ }
93+ mode = cfg.EquivalencePromptMode
94+ }
95+ if cfg.EquivalencePromptContext != nil {
96+ rendered, err := RenderEquivalencePromptForMode(*cfg.EquivalencePromptContext, mode)
97+ if err != nil {
98+ return result, err
99+ }
100+ if cfg.EquivalencePrompt != "" && cfg.EquivalencePrompt != rendered {
101+ return result, errors.New("equivalence repair prompt differs from supplied context")
102+ }
103+ source.Prompt = rendered
104+ }
105+ source.PromptMode = mode
106+ dimensionAI := cfg.EquivalenceAI
107+ if dimensionAI == nil {
108+ dimensionAI = ai
109+ }
110+ history := newGradeAttemptClient(dimensionAI)
111+ equivalence, repaired, issue := repairEquivalenceFromPrompt(ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries)
112+ if repaired {
113+ equivalence.PromptMode = mode
114+ if cfg.EquivalencePromptContext != nil {
115+ pc := *cfg.EquivalencePromptContext
116+ if mode == EquivalencePromptModeGoldBlind {
117+ pc.GoldPatchTruncated = false
118+ }
119+ equivalence = annotateEquivalenceTruncation(equivalence, pc)
120+ }
121+ history.annotateEquivalence(&equivalence)
122+ mergeEquivalenceRepairHistory(&equivalence, source)
123+ }
73124 if issue != nil {
125+ issue.AttemptCount = history.attempts
126+ issue.RetryCount = max(0, history.attempts-1)
127+ issue.RetryReasons = append([]string(nil), history.reasons...)
74128 result.Unresolved = append(result.Unresolved, *issue)
75129 } else {
76130 result.Validation.Equivalence = equivalence
…func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa
84138 if strings.TrimSpace(source.Prompt) == "" && cfg.Existing.Review != nil {
85139 source = *cfg.Existing.Review
86140 }
87− review, repaired, issue := repairReviewFromPrompt(ctx, ai, source, cfg.AIModelID, cfg.ReviewParseRetries)
141+ if cfg.ReviewPrompt != "" {
142+ source.Prompt = cfg.ReviewPrompt
143+ }
144+ if cfg.ReviewPromptContext != nil {
145+ rendered, err := renderPrompt(reviewPromptTemplate, cfg.ReviewPromptContext.asTemplateData())
146+ if err != nil {
147+ return result, err
148+ }
149+ if cfg.ReviewPrompt != "" && cfg.ReviewPrompt != rendered {
150+ return result, errors.New("review repair prompt differs from supplied context")
151+ }
152+ source.Prompt = rendered
153+ }
154+ if source.Prompt != resolvedRepairReview(cfg.Existing).Prompt {
155+ source.Output = ""
156+ }
157+ dimensionAI := cfg.ReviewAI
158+ if dimensionAI == nil {
159+ dimensionAI = ai
160+ }
161+ history := newGradeAttemptClient(dimensionAI)
162+ review, repaired, issue := repairReviewFromPromptContext(ctx, history, source, cfg.AIModelID, cfg.ReviewParseRetries, cfg.ReviewPromptContext)
163+ if repaired && history.attempts > 0 {
164+ history.annotateReview(&review)
165+ mergeReviewRepairHistory(&review, source)
166+ }
167+ if repaired && cfg.ReviewPromptContext != nil {
168+ review = annotateReviewTruncation(review, *cfg.ReviewPromptContext)
169+ }
88170 if issue != nil {
171+ issue.AttemptCount = history.attempts
172+ issue.RetryCount = max(0, history.attempts-1)
173+ issue.RetryReasons = append([]string(nil), history.reasons...)
89174 result.Unresolved = append(result.Unresolved, *issue)
90175 } else {
91176 result.Validation.CodeReview = review
…func RepairCoverage(ctx context.Context, cfg CoverageRepairConfig) (CoverageRepa
104189}
105190
106191func repairEquivalenceFromPrompt(ctx context.Context, ai AIClient, existing EquivalenceResult, overrideModelID string, parseRetries int) (EquivalenceResult, bool, *CoverageRepairIssue) {
107− prompt := strings.TrimSpace(existing.Prompt)
108− if prompt == "" {
192+ prompt := existing.Prompt
193+ if strings.TrimSpace(prompt) == "" {
109194 return existing, false, &CoverageRepairIssue{
110195 Dimension: coverageRepairDimensionEquivalence,
111196 Reason: "missing_prompt",
…func repairEquivalenceFromPrompt(ctx context.Context, ai AIClient, existing Equi
145230
146231 parsed.Status = AIStatusUsed
147232 parsed.EvaluatorModel = resolveRepairEvaluatorModel(existing.EvaluatorModel, overrideModelID)
233+ parsed.PromptMode = existing.PromptMode
234+ parsed.Truncated = existing.Truncated
235+ parsed.TruncationReason = existing.TruncationReason
148236 parsed.Prompt = prompt
149237 parsed.Output = output
150238 return parsed, true, nil
…func reviewVerdictIsGuardSuppressible(result ReviewResult) bool {
159247}
160248
161249func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int) (ReviewResult, bool, *CoverageRepairIssue) {
162− prompt := strings.TrimSpace(existing.Prompt)
163− if prompt == "" {
250+ return repairReviewFromPromptContext(ctx, ai, existing, overrideModelID, parseRetries, nil)
251+}
252+
253+func repairReviewFromPromptContext(ctx context.Context, ai AIClient, existing ReviewResult, overrideModelID string, parseRetries int, promptCtx *PromptContext) (ReviewResult, bool, *CoverageRepairIssue) {
254+ prompt := existing.Prompt
255+ if strings.TrimSpace(prompt) == "" {
164256 return existing, false, &CoverageRepairIssue{
165257 Dimension: coverageRepairDimensionCodeReview,
166258 Reason: "missing_prompt",
…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes
174266 // re-apply that guard, so such verdicts must be resampled, never recovered.
175267 if existing.Status == AIStatusUnavailable && strings.TrimSpace(existing.Output) != "" {
176268 if reparsed, reparseErr := parseReviewOutput(existing.Output); reparseErr == nil &&
177− !reviewVerdictIsGuardSuppressible(reparsed) {
269+ ((promptCtx == nil && !reviewVerdictIsGuardSuppressible(reparsed)) || (promptCtx != nil && validateReviewAgainstPromptContext(reparsed, *promptCtx) == nil)) {
178270 reparsed.Status = AIStatusUsed
179271 reparsed.Prompt = prompt
180272 reparsed.Output = existing.Output
…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes
184276 if reparsed.EvaluatorModel == "" {
185277 reparsed.EvaluatorModel = strings.TrimSpace(overrideModelID)
186278 }
279+ reparsed.AttemptCount = existing.AttemptCount
280+ reparsed.RetryCount = existing.RetryCount
281+ reparsed.RetryReasons = append([]string(nil), existing.RetryReasons...)
282+ reparsed.RecoveredAfterRetry = existing.RecoveredAfterRetry
187283 reparsed.Truncated = existing.Truncated
188284 reparsed.TruncationReason = existing.TruncationReason
189285 return reparsed, true, nil
…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes
196292 }
197293
198294 parsed, parseErr := parseReviewOutput(output)
295+ if parseErr == nil && promptCtx != nil {
296+ parseErr = validateReviewAgainstPromptContext(parsed, *promptCtx)
297+ }
199298 for attempt := 0; parseErr != nil && attempt < normalizedReviewParseRetries(parseRetries); attempt++ {
200299 retryPrompt := buildReviewRetryPrompt(prompt, output, parseErr)
201300 retryOutput, retryErr := runAIWithRetries(ctx, ai, retryPrompt)
…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes
203302 return existing, false, newReviewRuntimeFailureIssue(retryErr)
204303 }
205304 parsed, parseErr = parseReviewOutput(retryOutput)
305+ if parseErr == nil && promptCtx != nil {
306+ parseErr = validateReviewAgainstPromptContext(parsed, *promptCtx)
307+ }
206308 output = retryOutput
207309 }
208310 if parseErr != nil {
209311 return existing, false, classifyReviewParseIssue(parseErr)
210312 }
211313
314+ if promptCtx == nil && reviewVerdictIsGuardSuppressible(parsed) {
315+ return existing, false, classifyReviewParseIssue(errors.New("review repair requires original verifier context for speculative blocker verdict"))
316+ }
317+ parsed.Truncated = existing.Truncated
318+ parsed.TruncationReason = existing.TruncationReason
212319 parsed.Status = AIStatusUsed
213320 parsed.Prompt = prompt
214321 parsed.Output = output
…func repairReviewFromPrompt(ctx context.Context, ai AIClient, existing ReviewRes
217324}
218325
219326func runAIWithRetries(ctx context.Context, ai AIClient, prompt string) (string, error) {
220− var lastErr error
221− for attempt := 0; attempt <= len(coverageRepairRetryBackoffs); attempt++ {
222− output, err := ai.Run(ctx, prompt)
223− if err == nil {
224− return output, nil
225− }
226− lastErr = err
227− if errors.Is(err, ErrAIDisabled) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
228− break
229− }
230− if attempt >= len(coverageRepairRetryBackoffs) {
231− break
232− }
233− select {
234− case <-ctx.Done():
235− return "", ctx.Err()
236− case <-time.After(coverageRepairRetryBackoffs[attempt]):
237− }
327+ if history, ok := ai.(*gradeAttemptClient); ok {
328+ return history.Run(ctx, prompt)
238329 }
239− return "", lastErr
330+ return newGradeAttemptClient(ai).Run(ctx, prompt)
240331}
241332
242333func normalizedReviewParseRetries(parseRetries int) int {
…func appendReviewNote(existing, extra string) string {
5353 }
5454}
5555
56−func RunReview(ctx context.Context, ai AIClient, promptCtx PromptContext) ReviewResult {
56+func RunReview(ctx context.Context, ai AIClient, promptCtx PromptContext) (result ReviewResult) {
57+ history := newGradeAttemptClient(ai)
58+ defer func() { history.annotateReview(&result) }()
59+ ai = history
60+
5761 return annotateReviewTruncation(runReview(ctx, ai, promptCtx), promptCtx)
5862}
5963
…type TestVerifierEvidence struct {
712712}
713713
714714type TestCommandResult struct {
715− Command string `json:"command"`
716− GoldOutcome TestOutcome `json:"gold_outcome,omitempty"`
717− GoldTrialName string `json:"gold_tb_trial_name,omitempty"`
718− GoldFailureMode string `json:"gold_failure_mode,omitempty"`
719− GoldRetried bool `json:"gold_retried,omitempty"`
720− GoldRetryEvidence *TestRetryEvidence `json:"gold_retry_evidence,omitempty"`
721− AgentOutcome TestOutcome `json:"agent_outcome,omitempty"`
722− AgentTrialName string `json:"agent_tb_trial_name,omitempty"`
723− AgentFailureMode string `json:"agent_failure_mode,omitempty"`
724− AgentRetried bool `json:"agent_retried,omitempty"`
715+ // Counts cover dispatched gold and candidate verifier calls for this command.
716+ AttemptCount int `json:"attempt_count,omitempty"`
717+ RetryCount int `json:"retry_count,omitempty"`
718+ RetryReasons []string `json:"retry_reasons,omitempty"`
719+ RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"`
720+ Command string `json:"command"`
721+ GoldOutcome TestOutcome `json:"gold_outcome,omitempty"`
722+ GoldTrialName string `json:"gold_tb_trial_name,omitempty"`
723+ GoldFailureMode string `json:"gold_failure_mode,omitempty"`
724+ GoldRetried bool `json:"gold_retried,omitempty"`
725+ GoldRetryEvidence *TestRetryEvidence `json:"gold_retry_evidence,omitempty"`
726+ AgentOutcome TestOutcome `json:"agent_outcome,omitempty"`
727+ AgentTrialName string `json:"agent_tb_trial_name,omitempty"`
728+ AgentFailureMode string `json:"agent_failure_mode,omitempty"`
729+ AgentRetried bool `json:"agent_retried,omitempty"`
725730}
726731
727732// TestRetryEvidence records the two verifier attempts that authorized and
…type TestRetryAttemptEvidence struct {
742747}
743748
744749type EquivalenceResult struct {
750+ AttemptCount int `json:"attempt_count,omitempty"`
751+ RetryCount int `json:"retry_count,omitempty"`
752+ RetryReasons []string `json:"retry_reasons,omitempty"`
753+ RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"`
754+
745755 Outcome EquivalenceOutcome `json:"outcome"`
746756 Status AIStatus `json:"status"`
747757 // PromptMode records the exact built-in equivalence template variant that
…type EquivalenceObligation struct {
785795}
786796
787797type ReviewResult struct {
798+ AttemptCount int `json:"attempt_count,omitempty"`
799+ RetryCount int `json:"retry_count,omitempty"`
800+ RetryReasons []string `json:"retry_reasons,omitempty"`
801+ RecoveredAfterRetry bool `json:"recovered_after_retry,omitempty"`
802+
788803 Status AIStatus `json:"status"`
789804 SchemaVersion string `json:"schema_version,omitempty"`
790805 ProfileID string `json:"profile_id,omitempty"`
…Recovery rules:
793793 Those repair steps preserve compare-critical metadata and existing custom
794794 grader surfaces so the arm stays compare-compatible while recovering
795795 coverage. Add `--parse-retries N` when saved grader prompts failed
796− JSON/schema parsing. Keep the original custom rubric file path on the regrade
797− command.
796+ JSON/schema parsing. Repair preserves recorded prompt mode, truncation, and
797+ attempt history. Missing prompts require matching original task and artifact
798+ provenance plus complete repository context; ambiguous reconstruction remains
799+ unresolved. Runtime retries cover recognized transient failures only.
800+ Keep the original custom rubric file path on the regrade command.
798801- If `stet runs repair-ai-coverage --cr-only` still leaves `code_review`
799802 unresolved, use the summary's stable `unresolved[].reason` and optional
800803 `category`/`detail` fields to separate `model_output_shape`,
…
1+package commandlist
2+
3+import (
4+ "path"
5+ "regexp"
6+ "strings"
7+
8+ "stet/internal/shellcmd"
9+)
10+
11+var assignment = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*=`)
12+var duration = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+)?[smhd]?$`)
13+
14+// TestInvocation identifies exactly one known test command. The original text
15+// is retained, including literal environment assignments and a relative cwd.
16+// Opaque scripts and shell evaluation are deliberately not interpreted.
17+func TestInvocation(command string) (string, []shellcmd.FieldSpan, bool) {
18+ command = strings.TrimSpace(command)
19+ if command == "" || strings.ContainsAny(command, "\n\r`$") {
20+ return "", nil, false
21+ }
22+ spans, ok := shellcmd.FieldSpans(command)
23+ if !ok || len(spans) == 0 {
24+ return "", nil, false
25+ }
26+ i := 0
27+ if spans[0].Value == "cd" {
28+ if len(spans) < 4 || spans[2].Value != "&&" || !relativeDirectory(spans[1].Value) {
29+ return "", nil, false
30+ }
31+ if unsafeShellControl(command[:spans[2].Start]) {
32+ return "", nil, false
33+ }
34+ i = 3
35+ }
36+ if i >= len(spans) || unsafeShellControl(command[spans[i].Start:]) {
37+ return "", nil, false
38+ }
39+ // The sole supported control operator belongs to the proven cd prefix.
40+ for _, span := range spans[i:] {
41+ if span.Value == "&&" || span.Value == "&" {
42+ return "", nil, false
43+ }
44+ }
45+ for i < len(spans) && assignment.MatchString(spans[i].Value) {
46+ if !assignment.MatchString(command[spans[i].Start:spans[i].End]) {
47+ return "", nil, false
48+ }
49+ i++
50+ }
51+ if i < len(spans) && spans[i].Value == "env" {
52+ i++
53+ for i < len(spans) && assignment.MatchString(spans[i].Value) {
54+ i++
55+ }
56+ }
57+ if i < len(spans) && spans[i].Value == "timeout" {
58+ i++
59+ if i >= len(spans) || !duration.MatchString(spans[i].Value) {
60+ return "", nil, false
61+ }
62+ i++
63+ }
64+ if i >= len(spans) {
65+ return "", nil, false
66+ }
67+ fields := spans[i:]
68+ value := func(n int) string {
69+ if n >= len(fields) {
70+ return ""
71+ }
72+ return fields[n].Value
73+ }
74+ switch value(0) {
75+ case "go", "cargo":
76+ ok = value(1) == "test"
77+ case "sh", "bash":
78+ ok = len(fields) == 2 && strings.HasSuffix(value(1), ".sh") && relativeDirectory(value(1))
79+ case "pytest", "pytest-3", "vitest", "jest", "phpunit":
80+ ok = true
81+ case "python", "python3":
82+ ok = value(1) == "-m" && value(2) == "pytest"
83+ case "npm", "pnpm", "yarn":
84+ ok = value(1) == "test" || (value(1) == "run" && value(2) == "test")
85+ case "npx":
86+ ok = value(1) == "vitest" || value(1) == "jest"
87+ default:
88+ ok = false
89+ }
90+ if !ok {
91+ return "", nil, false
92+ }
93+ return command, fields, true
94+}
95+
96+func relativeDirectory(directory string) bool {
97+ if directory == "" || strings.HasPrefix(directory, "-") || strings.Contains(directory, "\\") || path.IsAbs(directory) {
98+ return false
99+ }
100+ clean := path.Clean(directory)
101+ return clean != ".." && !strings.HasPrefix(clean, "../")
102+}
103+
104+func unsafeShellControl(command string) bool {
105+ var quote rune
106+ escaped := false
107+ for _, r := range command {
108+ if escaped {
109+ escaped = false
110+ continue
111+ }
112+ if quote == '\'' {
113+ if r == '\'' {
114+ quote = 0
115+ }
116+ continue
117+ }
118+ if r == '\\' {
119+ escaped = true
120+ continue
121+ }
122+ if quote == '"' {
123+ if r == '"' {
124+ quote = 0
125+ }
126+ continue
127+ }
128+ if r == '\'' || r == '"' {
129+ quote = r
130+ continue
131+ }
132+ if strings.ContainsRune(";|&<>#()", r) {
133+ return true
134+ }
135+ }
136+ return false
137+}
…
1+package commandlist
2+
3+import "testing"
4+
5+func TestTestInvocationPreservesProvenExecutionContext(t *testing.T) {
6+ for _, command := range []string{
7+ "go test ./pkg -run 'TestOne|TestTwo' -count=1",
8+ "cd packages/api && FOO='a b' env BAR=2 timeout 30s python3 -m pytest tests/test_api.py -q",
9+ "npm run test -- --runInBand", "env CI=1 cargo test --locked", "pytest 'tests/test space.py'",
10+ } {
11+ got, fields, ok := TestInvocation(command)
12+ if !ok || got != command || len(fields) == 0 {
13+ t.Errorf("TestInvocation(%q) = %q, %v", command, got, ok)
14+ }
15+ }
16+}
17+
18+func TestTestInvocationRejectsAmbiguousAndUnsafeWrappers(t *testing.T) {
19+ for _, command := range []string{
20+ "echo go test ./pkg", "sh -c 'go test ./pkg'", "go test ./pkg && pytest tests",
21+ "go test ./pkg; true", "go test ./pkg || true", "cd ../outside && go test ./pkg",
22+ "env -i go test ./pkg", "timeout --signal=KILL 30 go test ./pkg", "FOO=$(touch sentinel) go test ./pkg",
23+ "go test ./pkg > result", "cd pkg&&go test ./pkg", "go test ./pkg &", "make test", "\"CI=1\" go test ./pkg", "F\"OO\"=x go test ./pkg",
24+ "cd pkg;touch && go test ./pkg", "echo tests/agent.sh", "cat tests/agent.sh", "rm tests/agent.sh", "sh -c tests/agent.sh",
25+ } {
26+ if _, _, ok := TestInvocation(command); ok {
27+ t.Errorf("accepted %q", command)
28+ }
29+ }
30+}
…
1+package h2h
2+
3+import (
4+ "fmt"
5+ "path/filepath"
6+ "strings"
7+
8+ "stet/internal/validate"
9+)
10+
11+// prepareCoverageRepairPrompts reconstructs only absent prompts. Stored prompts
12+// retain their original evidence; reconstructed prompts use the sealed task
13+// source and keep its extracted repository alive through grading.
14+func prepareCoverageRepairPrompts(cfg *validate.CoverageRepairConfig, manifest *Manifest, artifacts TaskArtifacts) (func(), []validate.CoverageRepairIssue) {
15+ missingEq := cfg.RepairEquivalence && strings.TrimSpace(cfg.Existing.Equivalence.Prompt) == ""
16+ review := cfg.Existing.CodeReview
17+ if strings.TrimSpace(review.Prompt) == "" && cfg.Existing.Review != nil {
18+ review = *cfg.Existing.Review
19+ }
20+ missingReview := cfg.RepairCodeReview && strings.TrimSpace(review.Prompt) == ""
21+ if !missingEq && !missingReview {
22+ return nil, nil
23+ }
24+ var issues []validate.CoverageRepairIssue
25+ fail := func(dimension string, err error) {
26+ detail := "prompt reconstruction failed: " + err.Error()
27+ if len(detail) > 240 {
28+ detail = detail[:240]
29+ }
30+ issues = append(issues, validate.CoverageRepairIssue{Dimension: dimension, Reason: "missing_prompt", Detail: detail, Cause: err})
31+ if dimension == TaskGraderEquivalence {
32+ cfg.RepairEquivalence = false
33+ } else {
34+ cfg.RepairCodeReview = false
35+ }
36+ }
37+ mode := cfg.Existing.Equivalence.PromptMode
38+ if missingEq && mode != validate.EquivalencePromptModeStandard && mode != validate.EquivalencePromptModeGoldBlind {
39+ fail(TaskGraderEquivalence, fmt.Errorf("missing or unsupported equivalence prompt mode %q", mode))
40+ missingEq = false
41+ }
42+ if !missingEq && !missingReview {
43+ return nil, issues
44+ }
45+ truncation := cfg.Existing.PromptTruncation
46+ if truncation.AgentPatch.Truncated || truncation.GoldPatch.Truncated || truncation.RepoTree.Truncated || truncation.RepoExcerpts.Truncated || cfg.Existing.Equivalence.Truncated || review.Truncated {
47+ err := fmt.Errorf("original prompt was truncated; original context budgets are unavailable")
48+ if missingEq {
49+ fail(TaskGraderEquivalence, err)
50+ }
51+ if missingReview {
52+ fail(TaskGraderCodeReview, err)
53+ }
54+ return nil, issues
55+ }
56+ dataset := manifest.Dataset
57+ goldPath := strings.TrimSpace(cfg.Existing.ArtifactsManifest.GoldPatch.Path)
58+ if goldPath != "" {
59+ taskDir := filepath.Dir(goldPath)
60+ if filepath.Base(taskDir) != artifacts.TaskID {
61+ err := fmt.Errorf("gold patch directory does not match task %q", artifacts.TaskID)
62+ if missingEq {
63+ fail(TaskGraderEquivalence, err)
64+ }
65+ if missingReview {
66+ fail(TaskGraderCodeReview, err)
67+ }
68+ return nil, issues
69+ }
70+ dataset = filepath.Dir(taskDir)
71+ }
72+ if err := verifyCoverageRepairTaskAuthority(manifest, artifacts, dataset); err != nil {
73+ if missingEq {
74+ fail(TaskGraderEquivalence, err)
75+ }
76+ if missingReview {
77+ fail(TaskGraderCodeReview, err)
78+ }
79+ return nil, issues
80+ }
81+ resolved, _, _, err := resolveTaskArtifactsFromResults(artifacts)
82+ if err == nil {
83+ artifacts = resolved
84+ }
85+ patchPath := strings.TrimSpace(cfg.Existing.ArtifactsManifest.AgentPatch.Path)
86+ if patchPath == "" {
87+ patchPath = artifacts.ResolveAgentPatchPath()
88+ }
89+ promptCtx, cleanup, err := validate.BuildOfflinePromptContext(validate.OfflinePromptInputs{
90+ DatasetPath: dataset, TaskID: artifacts.TaskID, AgentPatchPath: patchPath,
91+ TBResultsRaw: validate.ReadTBResultsRaw(artifacts.ResultsPath(), artifacts.TaskID), RequireRepoContext: true,
92+ EffectiveTestCommands: append([]string{}, cfg.Existing.Tests.Commands...),
93+ ExpectedArtifacts: &cfg.Existing.ArtifactsManifest,
94+ RepoContextOptions: validate.RepoContextOptions{ExcludePrefixes: cfg.Existing.ContextExcludePaths},
95+ })
96+ if err != nil {
97+ if missingEq {
98+ fail(TaskGraderEquivalence, err)
99+ }
100+ if missingReview {
101+ fail(TaskGraderCodeReview, err)
102+ }
103+ return nil, issues
104+ }
105+ if strings.TrimSpace(promptCtx.AgentPatch) == "" || strings.TrimSpace(promptCtx.TaskYAML) == "" {
106+ err := fmt.Errorf("task YAML and agent patch are required for prompt reconstruction")
107+ if missingEq {
108+ fail(TaskGraderEquivalence, err)
109+ }
110+ if missingReview {
111+ fail(TaskGraderCodeReview, err)
112+ }
113+ return cleanup, issues
114+ }
115+ client := cfg.AI
116+ if shell, ok := client.(validate.ShellAIClient); ok {
117+ shell.Dir = promptCtx.RepoPath
118+ client = shell
119+ }
120+ if missingEq {
121+ prompt, err := validate.RenderEquivalencePromptForMode(promptCtx, mode)
122+ if err != nil {
123+ fail(TaskGraderEquivalence, err)
124+ } else {
125+ cfg.EquivalencePrompt, cfg.EquivalencePromptMode = prompt, mode
126+ cfg.EquivalencePromptContext, cfg.EquivalenceAI = &promptCtx, client
127+ }
128+ }
129+ if missingReview {
130+ prompt, err := validate.RenderReviewPrompt(promptCtx)
131+ if err != nil {
132+ fail(TaskGraderCodeReview, err)
133+ } else {
134+ cfg.ReviewPrompt, cfg.ReviewPromptContext, cfg.ReviewAI = prompt, &promptCtx, client
135+ }
136+ }
137+ return cleanup, issues
138+}
139+
140+func verifyCoverageRepairTaskAuthority(manifest *Manifest, artifacts TaskArtifacts, dataset string) error {
141+ if manifest.TaskSelection == nil || manifest.TaskSelection.PromptProjectionReceipt == "" {
142+ return fmt.Errorf("missing original task prompt projection authority")
143+ }
144+ receipt, err := LoadTaskPromptProjectionReceipt(artifacts.OutputRoot)
145+ if err != nil {
146+ return err
147+ }
148+ if err := validateTaskSelectionProjection(manifest.TaskSelection, receipt); err != nil {
149+ return err
150+ }
151+ digest, err := digestTaskContentTree(filepath.Join(dataset, artifacts.TaskID))
152+ if err != nil {
153+ return err
154+ }
155+ for _, row := range receipt.Tasks {
156+ if row.TaskID == artifacts.TaskID {
157+ if row.TaskContentSHA256 != digest {
158+ return fmt.Errorf("original task content provenance mismatch for %q", artifacts.TaskID)
159+ }
160+ return nil
161+ }
162+ }
163+ return fmt.Errorf("missing original task content authority for %q", artifacts.TaskID)
164+}
…
1+package h2h
2+
3+import (
4+ "bytes"
5+ "context"
6+ "fmt"
7+ "os"
8+ "path/filepath"
9+ "strings"
10+ "testing"
11+ "time"
12+
13+ "stet/internal/validate"
14+)
15+
16+func TestPrepareCoverageRepairPromptsPreservesStoredPrompts(t *testing.T) {
17+ cfg := validate.CoverageRepairConfig{
18+ Existing: validate.ValidationResult{
19+ Equivalence: validate.EquivalenceResult{Prompt: "stored equivalence"},
20+ Review: &validate.ReviewResult{Prompt: "legacy stored review"},
21+ }, RepairEquivalence: true, RepairCodeReview: true,
22+ }
23+ cleanup, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})
24+ if cleanup != nil || len(issues) != 0 || cfg.EquivalencePromptContext != nil || cfg.ReviewPromptContext != nil {
25+ t.Fatalf("stored prompts should require no reconstruction: %+v", cfg)
26+ }
27+}
28+
29+func TestPrepareCoverageRepairPromptsRejectsUnknownModeWithoutBlockingStoredReview(t *testing.T) {
30+ cfg := validate.CoverageRepairConfig{
31+ Existing: validate.ValidationResult{CodeReview: validate.ReviewResult{Prompt: "stored review"}},
32+ RepairEquivalence: true, RepairCodeReview: true,
33+ }
34+ _, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})
35+ if len(issues) != 1 || issues[0].Dimension != TaskGraderEquivalence || cfg.RepairEquivalence || !cfg.RepairCodeReview {
36+ t.Fatalf("unexpected dimension selection: %+v, issues %+v", cfg, issues)
37+ }
38+}
39+
40+func TestPrepareCoverageRepairPromptsRejectsUnknownOriginalTruncationBudget(t *testing.T) {
41+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true}
42+ cfg.Existing.PromptTruncation.AgentPatch.Truncated = true
43+ _, issues := prepareCoverageRepairPrompts(&cfg, &Manifest{Dataset: "missing"}, TaskArtifacts{TaskID: "task-a"})
44+ if len(issues) != 1 || cfg.RepairCodeReview || !strings.Contains(issues[0].Detail, "original context budgets") {
45+ t.Fatalf("unknown original truncation must prevent reconstruction: %+v", issues)
46+ }
47+}
48+
49+func TestPrepareCoverageRepairPromptsReconstructsPersistedModeAndVerifierContext(t *testing.T) {
50+ t.Parallel()
51+ root := t.TempDir()
52+ taskDir := filepath.Join(root, "dataset", "task-a")
53+ mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\ninstruction: fix greeting\n")
54+ mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold reference secret\n")
55+ mustWriteFile(t, filepath.Join(taskDir, "tests", "test_outputs.py"), "COMMANDS = [\"dataset test\"]\n")
56+ patch := filepath.Join(root, "agent.patch")
57+ mustWriteFile(t, patch, "diff --git a/hello.txt b/hello.txt\n+hello\n")
58+ writeAnswerSimilarityTar(t, filepath.Join(taskDir, "repo.tar.gz"), map[string]string{"hello.txt": "old\n"}, nil)
59+ existing := validate.ValidationResult{
60+ Equivalence: validate.EquivalenceResult{PromptMode: validate.EquivalencePromptModeGoldBlind},
61+ Tests: validate.TestResult{Commands: []string{"effective verifier"}},
62+ }
63+ inputs, err := validate.ReadTaskInputs(filepath.Dir(taskDir), "task-a", patch)
64+ if err != nil {
65+ t.Fatal(err)
66+ }
67+ existing.ArtifactsManifest = validate.BuildArtifactManifest(inputs)
68+ manifest := sealCoverageRepairTestTask(t, root, filepath.Dir(taskDir))
69+ cfg := validate.CoverageRepairConfig{Existing: existing, AI: validate.ShellAIClient{}, RepairEquivalence: true, RepairCodeReview: true}
70+ cleanup, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a", ModelKey: "model-a"}, "task-a", ""))
71+ if cleanup != nil {
72+ defer cleanup()
73+ }
74+ if len(issues) != 0 {
75+ t.Fatalf("reconstruction failed: %+v", issues)
76+ }
77+ if cfg.EquivalencePromptMode != validate.EquivalencePromptModeGoldBlind || strings.Contains(cfg.EquivalencePrompt, "gold reference secret") {
78+ t.Fatalf("gold-blind provenance was lost: %s", cfg.EquivalencePrompt)
79+ }
80+ if !strings.Contains(cfg.ReviewPrompt, "effective verifier") || strings.Contains(cfg.ReviewPrompt, "dataset test") {
81+ t.Fatalf("effective verifier context was lost: %s", cfg.ReviewPrompt)
82+ }
83+ for _, client := range []validate.AIClient{cfg.EquivalenceAI, cfg.ReviewAI} {
84+ shell := client.(validate.ShellAIClient)
85+ if shell.Dir == "" || shell.Dir != cfg.ReviewPromptContext.RepoPath {
86+ t.Fatalf("missing grading repository: %+v", shell)
87+ }
88+ if _, err := os.Stat(filepath.Join(shell.Dir, "hello.txt")); err != nil {
89+ t.Fatal(err)
90+ }
91+ }
92+}
93+
94+func TestPrepareCoverageRepairPromptsRejectsUnavailableRepository(t *testing.T) {
95+ root := t.TempDir()
96+ taskDir := filepath.Join(root, "dataset", "task-a")
97+ mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\n")
98+ mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold\n")
99+ mustWriteFile(t, filepath.Join(taskDir, "repo.tar.gz"), "invalid gzip")
100+ patch := filepath.Join(root, "agent.patch")
101+ mustWriteFile(t, patch, "agent\n")
102+ manifest := sealCoverageRepairTestTask(t, root, filepath.Dir(taskDir))
103+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true}
104+ inputs, err := validate.ReadTaskInputs(filepath.Dir(taskDir), "task-a", patch)
105+ if err != nil {
106+ t.Fatal(err)
107+ }
108+ cfg.Existing.ArtifactsManifest = validate.BuildArtifactManifest(inputs)
109+ cleanup, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a"}, "task-a", ""))
110+ if cleanup != nil {
111+ defer cleanup()
112+ }
113+ if len(issues) != 1 || cfg.RepairCodeReview || cfg.ReviewPrompt != "" || !strings.Contains(issues[0].Detail, "required repo context") {
114+ t.Fatalf("repository failure should prevent reconstructed grade: %+v, issues %+v", cfg, issues)
115+ }
116+}
117+
118+func sealCoverageRepairTestTask(t *testing.T, root, dataset string) *Manifest {
119+ t.Helper()
120+ selection := &TaskSelectionRecord{RealizedTaskIDs: []string{"task-a"}}
121+ receipt, err := BuildTaskPromptProjectionReceipt(dataset, root, selection, nil, "dataset", map[string]string{"task-a": "fix greeting"}, time.Time{})
122+ if err != nil {
123+ t.Fatal(err)
124+ }
125+ if _, err := WriteTaskPromptProjectionReceipt(root, receipt); err != nil {
126+ t.Fatal(err)
127+ }
128+ return &Manifest{Dataset: dataset, TaskSelection: ApplyTaskPromptProjectionReceipt(selection, receipt)}
129+}
130+
131+func TestPrepareCoverageRepairPromptsRejectsMutatedTaskYAML(t *testing.T) {
132+ root := t.TempDir()
133+ dataset := filepath.Join(root, "dataset")
134+ mustWriteFile(t, filepath.Join(dataset, "task-a", "task.yaml"), "id: task-a\ninstruction: original\n")
135+ manifest := sealCoverageRepairTestTask(t, root, dataset)
136+ mustWriteFile(t, filepath.Join(dataset, "task-a", "task.yaml"), "id: task-a\ninstruction: mutated\n")
137+ cfg := validate.CoverageRepairConfig{RepairCodeReview: true}
138+ _, issues := prepareCoverageRepairPrompts(&cfg, manifest, NewTaskArtifacts(root, VariantSpec{RunID: "run-a"}, "task-a", ""))
139+ if len(issues) != 1 || cfg.RepairCodeReview || !strings.Contains(issues[0].Detail, "task content provenance mismatch") {
140+ t.Fatalf("changed task must not reconstruct: %+v", issues)
141+ }
142+}
143+
144+func TestRunRepairAICoverageReconstructsBoundSourceAtSubprocessBoundary(t *testing.T) {
145+ for _, mutate := range []bool{false, true} {
146+ t.Run(fmt.Sprintf("mutate_%t", mutate), func(t *testing.T) {
147+ root := t.TempDir()
148+ dataset := filepath.Join(root, "dataset")
149+ taskDir := filepath.Join(dataset, "task-a")
150+ mustWriteFile(t, filepath.Join(taskDir, "task.yaml"), "id: task-a\ninstruction: fix greeting\n")
151+ mustWriteFile(t, filepath.Join(taskDir, "gold.patch"), "gold reference secret\n")
152+ writeAnswerSimilarityTar(t, filepath.Join(taskDir, "repo.tar.gz"), map[string]string{"hello.txt": "old\n"}, nil)
153+ patch := filepath.Join(root, "agent.patch")
154+ mustWriteFile(t, patch, "diff --git a/hello.txt b/hello.txt\n--- a/hello.txt\n+++ b/hello.txt\n@@ -1 +1 @@\n-old\n+hello\n")
155+ manifest := sealCoverageRepairTestTask(t, root, dataset)
156+ manifest.Version, manifest.OutputRoot = ManifestVersion, root
157+ manifest.Models = []string{"model/a"}
158+ manifest.ModelKeys, manifest.RunIDs = map[string]string{"model/a": "model-a"}, map[string]string{"model/a": "run-a"}
159+ if err := SaveManifest(filepath.Join(root, "manifest.json"), manifest); err != nil {
160+ t.Fatal(err)
161+ }
162+ inputs, err := validate.ReadTaskInputs(dataset, "task-a", patch)
163+ if err != nil {
164+ t.Fatal(err)
165+ }
166+ original := validate.ValidationResult{
167+ TaskID: "task-a", PrimarySignal: validate.PrimarySignalTests, Status: validate.StatusPass, MatrixStatus: validate.MatrixPass,
168+ Guardrails: validate.GuardrailResult{Status: validate.StatusPass},
169+ Tests: validate.TestResult{Outcome: validate.TestOutcomePass, Strength: validate.TestStrengthStrong, Commands: []string{"effective verifier"}},
170+ Equivalence: validate.EquivalenceResult{Status: validate.AIStatusUnavailable, Outcome: validate.EquivalenceUnknown, PromptMode: validate.EquivalencePromptModeGoldBlind},
171+ CodeReview: validate.ReviewResult{Status: validate.AIStatusUnavailable, Signal: validate.ReviewSignalUnsure},
172+ ArtifactsManifest: validate.BuildArtifactManifest(inputs),
173+ }
174+ validationRoot := filepath.Join(root, "validation", "model-a")
175+ mustWriteRepairValidation(t, validationRoot, "task-a", original)
176+ if mutate {
177+ mustWriteFile(t, patch, "mutated agent evidence\n")
178+ }
179+ log := filepath.Join(root, "calls")
180+ script := filepath.Join(root, "judge.py")
181+ mustWriteFile(t, script, `import json, pathlib, sys
182+prompt = sys.stdin.read()
183+assert pathlib.Path('hello.txt').read_text() == 'old\n'
184+with open(sys.argv[1], 'a') as log: log.write('call\n')
185+if 'You are reviewing' in prompt:
186+ assert 'effective verifier' in prompt
187+ scores = {name: {'score_0_4': 4, 'confidence_0_1': .95, 'rationale': 'ok', 'citations': [{'path':'hello.txt','side':'new','start_line':1,'end_line':1,'anchor_kind':'diff','head_sha':'abc1234'}]} for name in ['correctness','introduced_bug_risk','edge_case_handling','maintainability_idioms']}
188+ print(json.dumps({'schema_version':'stet.review.v1','dimension_scores':scores,'findings':[],'overall':{'confidence_overall':.95,'summary':'repair passed'}}))
189+else:
190+ assert 'gold reference secret' not in prompt
191+ print('obligations:\n- fix greeting => MET\n\nequivalent: YES\nconfidence: 0.95\nissue_type: stylistic\nnotes: repaired')
192+`)
193+ var stdout, stderr bytes.Buffer
194+ err = RunRepairAICoverage(context.Background(), []string{"--out", root, "--model-key", "model-a", "--ai-cmd", "python3 " + shellQuote(script) + " " + shellQuote(log), "--ai-model-id", "judge-a"}, &stdout, &stderr)
195+ if mutate {
196+ if err == nil {
197+ t.Fatal("mutated artifact repair must fail")
198+ }
199+ if _, err := os.Stat(log); !os.IsNotExist(err) {
200+ t.Fatalf("mutated source invoked grader: %v", err)
201+ }
202+ if !strings.Contains(stdout.String(), "agent patch provenance mismatch") {
203+ t.Fatalf("missing mutation reason: %s", stdout.String())
204+ }
205+ return
206+ }
207+ if err != nil {
208+ t.Fatalf("repair failed: %v\nstdout %s\nstderr %s", err, stdout.String(), stderr.String())
209+ }
210+ updated := mustReadRepairValidation(t, filepath.Join(validationRoot, "task-a", "validation.json"))
211+ if updated.Equivalence.PromptMode != validate.EquivalencePromptModeGoldBlind || updated.Equivalence.Status != validate.AIStatusUsed || updated.CodeReview.Status != validate.AIStatusUsed || updated.CodeReview.AttemptCount != 1 {
212+ t.Fatalf("repair lost mode or attempt metadata: %+v", updated)
213+ }
214+ if updated.ArtifactsManifest.AgentPatch.SHA256 != original.ArtifactsManifest.AgentPatch.SHA256 {
215+ t.Fatal("repair changed original source authority")
216+ }
217+ calls, err := os.ReadFile(log)
218+ if err != nil || strings.Count(string(calls), "call") != 2 {
219+ t.Fatalf("unexpected calls: %s, %v", calls, err)
220+ }
221+ })
222+ }
223+}
…
1+package validate
2+
3+import (
4+ "context"
5+ "errors"
6+ "net"
7+ "strings"
8+ "time"
9+
10+ "stet/internal/runtimefailure"
11+)
12+
13+// gradeAttemptClient retains every dispatch, including runtime and parse retries.
14+// One instance belongs to one measurement dimension and must not be shared.
15+type gradeAttemptClient struct {
16+ ai AIClient
17+ attempts int
18+ reasons []string
19+ previousSucceeded bool
20+}
21+
22+func newGradeAttemptClient(ai AIClient) *gradeAttemptClient {
23+ if ai == nil {
24+ ai = NoopAIClient{}
25+ }
26+ return &gradeAttemptClient{ai: ai}
27+}
28+
29+func (a *gradeAttemptClient) Run(ctx context.Context, prompt string) (string, error) {
30+ if a.previousSucceeded {
31+ a.reasons = append(a.reasons, "model_output_shape")
32+ }
33+ a.previousSucceeded = false
34+ for attempt := 0; ; attempt++ {
35+ if err := ctx.Err(); err != nil {
36+ return "", err
37+ }
38+ a.attempts++
39+ output, err := a.ai.Run(ctx, prompt)
40+ if err == nil {
41+ a.previousSucceeded = true
42+ return output, nil
43+ }
44+ reason := recoverableGradeFailure(err)
45+ if reason == "" || attempt >= len(coverageRepairRetryBackoffs) {
46+ return "", err
47+ }
48+ select {
49+ case <-ctx.Done():
50+ return "", ctx.Err()
51+ case <-time.After(coverageRepairRetryBackoffs[attempt]):
52+ }
53+ a.reasons = append(a.reasons, reason)
54+ }
55+}
56+
57+func recoverableGradeFailure(err error) string {
58+ if errors.Is(err, ErrAIDisabled) || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
59+ return ""
60+ }
61+ c := runtimefailure.ClassifyError(err)
62+ if c.Kind != runtimefailure.KindUnknown {
63+ if c.Kind == runtimefailure.KindQuota && (c.Reason == "rate_limit" || c.Reason == "overloaded_error") {
64+ return c.Reason
65+ }
66+ return ""
67+ }
68+ var network net.Error
69+ if errors.As(err, &network) && (network.Timeout() || network.Temporary()) {
70+ return "transport_failure"
71+ }
72+ text := strings.ToLower(err.Error())
73+ for _, marker := range []string{"connection reset", "transport reset", "connection refused", "tls handshake timeout", "temporary failure in name resolution"} {
74+ if strings.Contains(text, marker) {
75+ return "transport_failure"
76+ }
77+ }
78+ return ""
79+}
80+
81+func (a *gradeAttemptClient) annotateEquivalence(r *EquivalenceResult) {
82+ r.AttemptCount = a.attempts
83+ r.RetryCount = max(0, a.attempts-1)
84+ r.RetryReasons = append([]string(nil), a.reasons...)
85+ r.RecoveredAfterRetry = r.Status == AIStatusUsed && r.RetryCount > 0
86+}
87+func (a *gradeAttemptClient) annotateReview(r *ReviewResult) {
88+ r.AttemptCount = a.attempts
89+ r.RetryCount = max(0, a.attempts-1)
90+ r.RetryReasons = append([]string(nil), a.reasons...)
91+ r.RecoveredAfterRetry = r.Status == AIStatusUsed && r.RetryCount > 0
92+}
93+
94+// A later repair continues the original measurement history. Reparse-only
95+// recovery never calls these helpers because it dispatches no evaluator.
96+func mergeEquivalenceRepairHistory(result *EquivalenceResult, original EquivalenceResult) {
97+ if original.AttemptCount == 0 {
98+ return
99+ }
100+ result.AttemptCount += original.AttemptCount
101+ result.RetryCount = result.AttemptCount - 1
102+ reasons := append([]string(nil), original.RetryReasons...)
103+ reasons = append(reasons, "coverage_repair")
104+ result.RetryReasons = append(reasons, result.RetryReasons...)
105+ result.RecoveredAfterRetry = result.Status == AIStatusUsed && result.RetryCount > 0
106+}
107+func mergeReviewRepairHistory(result *ReviewResult, original ReviewResult) {
108+ if original.AttemptCount == 0 {
109+ return
110+ }
111+ result.AttemptCount += original.AttemptCount
112+ result.RetryCount = result.AttemptCount - 1
113+ reasons := append([]string(nil), original.RetryReasons...)
114+ reasons = append(reasons, "coverage_repair")
115+ result.RetryReasons = append(reasons, result.RetryReasons...)
116+ result.RecoveredAfterRetry = result.Status == AIStatusUsed && result.RetryCount > 0
117+}
…
1+package validate
2+
3+import (
4+ "context"
5+ "errors"
6+ "testing"
7+)
8+
9+func TestRepairEquivalenceKeepsMeasurementModeAndTruncation(t *testing.T) {
10+ t.Setenv(GoldBlindEquivalenceEnv, "false")
11+ ai := &scriptedCoverageAI{outputs: []string{"bad", "obligations:\n- preserve behavior => MET\nequivalent: YES\nconfidence: 0.9\nissue_type: stylistic\nnotes: ok"}}
12+ result, err := RepairCoverage(context.Background(), CoverageRepairConfig{
13+ Existing: ValidationResult{Equivalence: EquivalenceResult{Prompt: "original", PromptMode: EquivalencePromptModeGoldBlind, Truncated: true}},
14+ EquivalenceAI: ai, RepairEquivalence: true,
15+ })
16+ if err != nil {
17+ t.Fatal(err)
18+ }
19+ got := result.Validation.Equivalence
20+ if !result.RepairedEquivalence || got.PromptMode != EquivalencePromptModeGoldBlind || !got.Truncated || got.AttemptCount != 2 || got.RetryCount != 1 || !got.RecoveredAfterRetry || len(got.RetryReasons) != 1 {
21+ t.Fatalf("measurement provenance lost: %+v", got)
22+ }
23+}
24+
25+func TestRepairReviewReappliesVerifierGuard(t *testing.T) {
26+ pc := PromptContext{AgentPatch: "patch", TBResults: `{"is_resolved":true}`}
27+ ai := &scriptedCoverageAI{outputs: []string{guardSuppressedFailReviewOutput(), guardSuppressedFailReviewOutput()}}
28+ result, err := RepairCoverage(context.Background(), CoverageRepairConfig{
29+ Existing: ValidationResult{CodeReview: ReviewResult{Status: AIStatusUnavailable}},
30+ ReviewAI: ai, ReviewPromptContext: &pc, RepairCodeReview: true,
31+ })
32+ if err != nil {
33+ t.Fatal(err)
34+ }
35+ if result.RepairedCodeReview || len(result.Unresolved) != 1 || len(ai.prompts) != 2 {
36+ t.Fatalf("guard bypass: %+v", result)
37+ }
38+}
39+
40+func TestRenderEquivalenceModeDoesNotUseEnvironment(t *testing.T) {
41+ t.Setenv(GoldBlindEquivalenceEnv, "true")
42+ pc := PromptContext{GoldPatch: "UNIQUE_GOLD"}
43+ standard, err := RenderEquivalencePromptForMode(pc, EquivalencePromptModeStandard)
44+ if err != nil {
45+ t.Fatal(err)
46+ }
47+ blind, err := RenderEquivalencePromptForMode(pc, EquivalencePromptModeGoldBlind)
48+ if err != nil {
49+ t.Fatal(err)
50+ }
51+ if standard == blind {
52+ t.Fatal("explicit modes ignored")
53+ }
54+ if _, err := RenderEquivalencePromptForMode(pc, ""); err == nil {
55+ t.Fatal("ambiguous mode accepted")
56+ }
57+}
58+
59+func TestUnrecoverableGradeFailureIsNotRetried(t *testing.T) {
60+ for _, failure := range []error{errors.New("invalid API key"), errors.New("unrecognized failure"), context.Canceled} {
61+ ai := &scriptedCoverageAI{outputs: []string{""}, errs: []error{failure}}
62+ history := newGradeAttemptClient(ai)
63+ _, err := history.Run(context.Background(), "prompt")
64+ if !errors.Is(err, failure) || history.attempts != 1 {
65+ t.Fatalf("unrecoverable retry: %v, %+v", err, history)
66+ }
67+ }
68+}
69+
70+func TestRepairHistoryContinuesOriginalAttempts(t *testing.T) {
71+ ai := &scriptedCoverageAI{outputs: []string{"obligations:\n- preserve behavior => MET\nequivalent: YES\nconfidence: 0.9\nissue_type: stylistic\nnotes: ok"}}
72+ original := EquivalenceResult{Prompt: "original", AttemptCount: 2, RetryCount: 1, RetryReasons: []string{"model_output_shape"}}
73+ result, err := RepairCoverage(context.Background(), CoverageRepairConfig{Existing: ValidationResult{Equivalence: original}, EquivalenceAI: ai, RepairEquivalence: true})
74+ if err != nil {
75+ t.Fatal(err)
76+ }
77+ got := result.Validation.Equivalence
78+ if got.AttemptCount != 3 || got.RetryCount != 2 || len(got.RetryReasons) != 2 || got.RetryReasons[0] != "model_output_shape" || !got.RecoveredAfterRetry {
79+ t.Fatalf("lost original history: %+v", got)
80+ }
81+}
82+
83+func TestFailedRepairRetainsDispatchHistoryInIssue(t *testing.T) {
84+ failure := errors.New("provider authentication expired")
85+ ai := &scriptedCoverageAI{outputs: []string{""}, errs: []error{failure}}
86+ original := EquivalenceResult{Prompt: "original", AttemptCount: 2}
87+ result, err := RepairCoverage(context.Background(), CoverageRepairConfig{Existing: ValidationResult{Equivalence: original}, EquivalenceAI: ai, RepairEquivalence: true})
88+ if err != nil {
89+ t.Fatal(err)
90+ }
91+ if result.Validation.Equivalence.AttemptCount != 2 || len(result.Unresolved) != 1 || result.Unresolved[0].AttemptCount != 1 || !errors.Is(result.Unresolved[0].Cause, failure) {
92+ t.Fatalf("lost failed repair history: %+v", result)
93+ }
94+}
…
1+package validate
2+
3+// recordTestRetry records a dispatched extra verifier call. Recovery is set
4+// separately only after that call produces a pass; a failed retry remains
5+// visible even when the original failure is kept as the decision evidence.
6+func recordTestRetry(result *TestCommandResult, reason string) {
7+ result.AttemptCount++
8+ result.RetryCount++
9+ result.RetryReasons = append(result.RetryReasons, normalizeFailureMode(reason))
10+}
11+
12+func retryableVerifierFailureMode(run OverrideRunResult, err error) string {
13+ if !isUnsetFailureMode(run.Trial.FailureMode) {
14+ return run.Trial.FailureMode
15+ }
16+ if mode := classifyVerifierEnvironmentFailureMode(run.VerifierStdout, run.VerifierStderr); mode != "" {
17+ return mode
18+ }
19+ if mode := inferInfraFailureModeFromError(err); mode != "" {
20+ return mode
21+ }
22+ return run.Trial.FailureMode
23+}
…
1+package validate
2+
3+import (
4+ "context"
5+ "errors"
6+ "os"
7+ "path/filepath"
8+ "testing"
9+ "time"
10+
11+ "stet/internal/harness"
12+)
13+
14+func TestCandidateExecutionRetriesOnlyRecoverableFailures(t *testing.T) {
15+ for _, batched := range []bool{false, true} {
16+ for _, mode := range []string{"", "assertion_failure", FailureModeInfraToolchain, FailureModeInfraDependencyFetch} {
17+ t.Run(mode+map[bool]string{false: "/sequential", true: "/batch"}[batched], func(t *testing.T) {
18+ inputs := retryTestInputs(t)
19+ pass, fail := true, false
20+ cfg := Config{TBCmd: []string{"not-harbor"}, CommandWorkers: 1, TimeBudget: time.Minute}
21+ var summary perCommandRunSummary
22+ if batched {
23+ runner := &batchValidateRunner{results: map[string]TBTrialResult{
24+ "gold-0": {IsResolved: &pass}, "override-0": {IsResolved: &fail, FailureMode: mode}, "retry-0": {IsResolved: &pass},
25+ }}
26+ summary = runPerCommandTestsBatched(context.Background(), time.Now(), cfg, inputs, []string{"cmd-a"}, nil, runner)
27+ } else {
28+ runner := &verifierOutputSequenceTBRunner{goldResults: []TBTrialResult{{IsResolved: &pass}}, overrideResults: []TBTrialResult{{IsResolved: &fail, FailureMode: mode}, {IsResolved: &pass}}}
29+ cfg.TestGenRunner = runner
30+ summary = runPerCommandTests(context.Background(), time.Now(), cfg, inputs, []string{"cmd-a"}, nil)
31+ }
32+ if len(summary.CommandResults) != 1 {
33+ t.Fatalf("results = %+v", summary)
34+ }
35+ result := summary.CommandResults[0]
36+ if mode == FailureModeInfraDependencyFetch {
37+ if result.AttemptCount != 3 || result.RetryCount != 1 || !result.RecoveredAfterRetry || !result.AgentRetried || result.AgentOutcome != TestOutcomePass {
38+ t.Fatalf("recovery = %+v", result)
39+ }
40+ } else if result.AttemptCount != 2 || result.RetryCount != 0 || result.AgentRetried || result.RecoveredAfterRetry || result.AgentOutcome != TestOutcomeFail {
41+ t.Fatalf("unrecoverable failure retried: %+v", result)
42+ }
43+ })
44+ }
45+ }
46+}
47+
48+type failedRetryBatchRunner struct{ batchValidateRunner }
49+
50+func (runner *failedRetryBatchRunner) RunBatch(ctx context.Context, cfg harness.BatchRunConfig) (map[string]harness.BatchRunResult, error) {
51+ if cfg.Items[0].PhaseTag == "validate-override-retry" {
52+ return nil, errors.New("terminal transport error")
53+ }
54+ return runner.batchValidateRunner.RunBatch(ctx, cfg)
55+}
56+
57+func TestCandidateBatchRetryRecordsTransportFailure(t *testing.T) {
58+ fail := false
59+ runner := &failedRetryBatchRunner{batchValidateRunner: batchValidateRunner{results: map[string]TBTrialResult{
60+ "override-0": {IsResolved: &fail, FailureMode: FailureModeInfraDependencyFetch, TrialName: "first"},
61+ }}}
62+ results := []commandOverrideResult{{Command: "cmd-a", Result: TestCommandResult{GoldOutcome: TestOutcomePass, AttemptCount: 1}}}
63+ runOverrideBatchSlice(context.Background(), retryTestInputs(t), OverrideConfig{}, runner, results, 0)
64+ result := results[0]
65+ if result.Err == nil || result.Err.Error() != "terminal transport error" || result.Result.AttemptCount != 3 || result.Result.RetryCount != 1 || result.Result.RecoveredAfterRetry || result.Result.AgentOutcome != TestOutcomeUnknown || result.Result.AgentTrialName != "" || result.Result.AgentFailureMode == FailureModeInfraDependencyFetch {
66+ t.Fatalf("batch terminal error = %+v", result)
67+ }
68+}
69+
70+func retryTestInputs(t *testing.T) TaskInputs {
71+ t.Helper()
72+ repo, base, head := createRepoWithCommits(t)
73+ dataset := t.TempDir()
74+ taskID := "retry-task"
75+ taskDir := filepath.Join(dataset, taskID)
76+ if err := os.MkdirAll(filepath.Join(taskDir, "tests"), 0o755); err != nil {
77+ t.Fatal(err)
78+ }
79+ if err := os.WriteFile(filepath.Join(taskDir, "task.yaml"), []byte("instruction: test"), 0o644); err != nil {
80+ t.Fatal(err)
81+ }
82+ writeGoldAndTarball(t, repo, base, head, taskDir)
83+ patch, err := os.ReadFile(filepath.Join(taskDir, "gold.patch"))
84+ if err != nil {
85+ t.Fatal(err)
86+ }
87+ agentPatch := filepath.Join(t.TempDir(), "agent.patch")
88+ if err := os.WriteFile(agentPatch, patch, 0o644); err != nil {
89+ t.Fatal(err)
90+ }
91+ inputs, err := ReadTaskInputs(dataset, taskID, agentPatch)
92+ if err != nil {
93+ t.Fatal(err)
94+ }
95+ return inputs
96+}
97+
98+func TestCandidateExecutionStopsAfterOneFailedRecovery(t *testing.T) {
99+ for _, batched := range []bool{false, true} {
100+ t.Run(map[bool]string{false: "sequential", true: "batch"}[batched], func(t *testing.T) {
101+ inputs := retryTestInputs(t)
102+ pass, fail := true, false
103+ cfg := Config{TBCmd: []string{"not-harbor"}, CommandWorkers: 1, TimeBudget: time.Minute}
104+ var summary perCommandRunSummary
105+ var calls int
106+ if batched {
107+ runner := &batchValidateRunner{results: map[string]TBTrialResult{
108+ "gold-0": {IsResolved: &pass},
109+ "override-0": {IsResolved: &fail, FailureMode: FailureModeInfraDependencyFetch},
110+ "retry-0": {IsResolved: &fail, FailureMode: "assertion_failure", TrialName: "terminal"},
111+ }}
112+ summary = runPerCommandTestsBatched(context.Background(), time.Now(), cfg, inputs, []string{"cmd-a"}, nil, runner)
113+ calls = len(runner.batchCalls)
114+ } else {
115+ runner := &verifierOutputSequenceTBRunner{goldResults: []TBTrialResult{{IsResolved: &pass}}, overrideResults: []TBTrialResult{
116+ {IsResolved: &fail, FailureMode: FailureModeInfraDependencyFetch},
117+ {IsResolved: &fail, FailureMode: "assertion_failure", TrialName: "terminal"},
118+ {IsResolved: &pass},
119+ }}
120+ cfg.TestGenRunner = runner
121+ summary = runPerCommandTests(context.Background(), time.Now(), cfg, inputs, []string{"cmd-a"}, nil)
122+ calls = len(runner.calls)
123+ }
124+ result := summary.CommandResults[0]
125+ if calls != 3 || result.AttemptCount != 3 || result.RetryCount != 1 || !result.AgentRetried || result.RecoveredAfterRetry || result.AgentFailureMode != "assertion_failure" || result.AgentTrialName != "terminal" || result.AgentOutcome != TestOutcomeFail {
126+ t.Fatalf("failed recovery receipt = %+v, calls = %d", result, calls)
127+ }
128+ })
129+ }
130+}
来源：Hacker News · stet.sh