The Oracle's Rubric: Defining Agent Success Criteria and Signals

# .github/workflows/check-task-completion.yml
name: Check Agent Task Completion

on:
  pull_request:
    types: [opened, edited, ready_for_review, synchronize]
  workflow_run:
    workflows: ["Run Tests"]
    types: [completed]

jobs:
  check-completion:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Check all success signals
        id: check_signals
        uses: actions/github-script@v7
        with:
          script: |
            // Resolve the PR for both 'pull_request' and 'workflow_run' triggers.
            // workflow_run fires after a workflow completes; payload.pull_request is
            // null, but workflow_run.pull_requests carries the associated PR(s).
            let pr = context.payload.pull_request;
            if (!pr && context.payload.workflow_run) {
              pr = context.payload.workflow_run.pull_requests?.[0];
            }
            if (!pr) {
              console.log('No PR associated with this event — skipping');
              return;
            }
            
            const signals = [];
            let allPassed = true;
            
            // Signal 1: PR body references an issue
            const closesPattern = /closes\s+#\d+/i;
            const refersPattern = /issue\s+#\d+/i;
            const hasRef = closesPattern.test(pr.body || '') || refersPattern.test(pr.body || '');
            signals.push({ name: 'Issue reference in PR body', passed: hasRef });
            if (!hasRef) allPassed = false;
            
            // Signal 2: PR is not a draft
            const notDraft = !pr.draft;
            signals.push({ name: 'PR is ready for review (not draft)', passed: notDraft });
            if (!notDraft) allPassed = false;
            
            // Signal 3: Check required status checks
            const { data: checks } = await github.rest.checks.listForRef({
              owner: context.repo.owner,
              repo: context.repo.repo,
              ref: pr.head.sha,
            });
            
            const testCheck = checks.check_runs.find(c => c.name === 'test');
            const testPassed = testCheck?.conclusion === 'success';
            signals.push({ name: 'Test suite passes', passed: testPassed ?? false });
            if (!testPassed) allPassed = false;
            
            // Report results
            const statusLines = signals.map(s => 
              `${s.passed ? '✅' : '❌'} ${s.name}`
            ).join('\n');
            
            const comment = allPassed
              ? `## 🎉 Task Complete\n\nAll success signals verified:\n${statusLines}`
              : `## ⏳ Task In Progress\n\nSuccess signal status:\n${statusLines}\n\n_Checks will re-run automatically as the task progresses._`;
            
            // Add or update completion comment
            const comments = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: pr.number,
            });
            
            const existing = comments.data.find(c => 
              c.body.includes('Task Complete') || c.body.includes('Task In Progress')
            );
            
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body: comment,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: pr.number,
                body: comment,
              });
            }
            
            core.setOutput('all_passed', allPassed);