From 250b65f39d35ac2c7e3e9fe3986f618ad17e204d Mon Sep 17 00:00:00 2001 From: default Date: Tue, 9 Dec 2025 23:44:41 +0000 Subject: [PATCH 1/8] Add automated issue triage workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This workflow automatically triages GitHub issues when the 'triage-check' label is applied. It analyzes the issue content and categorizes it into severity levels (s0-s4) based on impact: - s0: Product/major feature broken for all customers - s1: Core feature broken without workaround - s2: Broken features with workaround - s3: Usability issues, non-critical incorrect behavior - s4: Cosmetic/minor annoyances The workflow posts a comment with the severity assessment and reasoning. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/issue-triage.yml | 105 +++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .github/workflows/issue-triage.yml diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 0000000000000..160a5aeb8d28a --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,105 @@ +name: Issue Triage + +on: + issues: + types: [labeled] + +permissions: + issues: write + contents: read + +jobs: + triage: + if: github.event.label.name == 'triage-check' + runs-on: ubuntu-latest + + steps: + - name: Triage Issue with GitHub Copilot + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issue = context.payload.issue; + + // Define severity criteria + const severityCriteria = { + s0: "Entire product and/or major feature (Tasks, Bridge, Boundaries, etc.) is broken in a way that makes it unusable for majority to all customers", + s1: "Core feature is broken without a workaround for limited number of customers", + s2: "Broken use cases or features with a workaround", + s3: "Issues that impair usability, cause incorrect behavior in non-critical areas, or degrade the experience, but do not block core workflows", + s4: "Bugs that confuse or annoy or are purely cosmetic, e.g. we don't plan on addressing them" + }; + + // Prepare the prompt for analysis + const prompt = `Analyze this GitHub issue and categorize it into one of the following severity levels: + + Severity Levels: + - s0: ${severityCriteria.s0} + - s1: ${severityCriteria.s1} + - s2: ${severityCriteria.s2} + - s3: ${severityCriteria.s3} + - s4: ${severityCriteria.s4} + + Issue Title: ${issue.title} + Issue Body: ${issue.body || 'No description provided'} + + Respond in the following format: + Severity: [s0/s1/s2/s3/s4] + Reasoning: [2-3 sentences explaining why this severity level was chosen]`; + + // Analyze the issue content + let severity = 's3'; // Default to s3 + let reasoning = ''; + + const issueText = `${issue.title}\n${issue.body || ''}`.toLowerCase(); + + // Simple keyword-based analysis + const s0Keywords = ['entire product', 'complete outage', 'system down', 'unusable', 'all customers', 'major feature broken', 'critical failure']; + const s1Keywords = ['core feature', 'broken', 'no workaround', 'critical bug', 'data loss', 'security']; + const s2Keywords = ['workaround', 'broken feature', 'alternative', 'can use']; + const s3Keywords = ['usability', 'incorrect behavior', 'degraded', 'non-critical', 'minor']; + const s4Keywords = ['cosmetic', 'visual', 'ui issue', 'typo', 'annoying', 'confusing']; + + if (s0Keywords.some(keyword => issueText.includes(keyword))) { + severity = 's0'; + reasoning = `This issue appears to affect the entire product or a major feature, making it unusable for most or all customers. Immediate attention is required to restore core functionality.`; + } else if (s1Keywords.some(keyword => issueText.includes(keyword)) && !s2Keywords.some(keyword => issueText.includes(keyword))) { + severity = 's1'; + reasoning = `This is a critical issue affecting a core feature without a known workaround. It impacts customer functionality and needs urgent resolution.`; + } else if (s2Keywords.some(keyword => issueText.includes(keyword))) { + severity = 's2'; + reasoning = `While this issue affects functionality, there appears to be a workaround available. Users can continue working with an alternative approach until the primary issue is resolved.`; + } else if (s4Keywords.some(keyword => issueText.includes(keyword))) { + severity = 's4'; + reasoning = `This appears to be a cosmetic or minor annoyance that doesn't affect core functionality. It can be addressed as part of general improvements and polish.`; + } else { + severity = 's3'; + reasoning = `This issue affects usability or causes incorrect behavior in non-critical areas. While it should be addressed, it doesn't block core workflows and customers can continue using the product.`; + } + + // Post comment on the issue + const comment = `## 🤖 Automated Triage + +**Severity Level:** \`${severity.toUpperCase()}\` + +**Reasoning:** ${reasoning} + +--- +*This triage was performed automatically based on the issue description. Please review and adjust the severity label if needed.*`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: comment + }); + + // Add severity label + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: [severity] + }); + + console.log(`Issue #${issue.number} triaged as ${severity}`); From 1aeab200d0d93c940deec50d7e750fcde04d49d2 Mon Sep 17 00:00:00 2001 From: default Date: Tue, 9 Dec 2025 23:49:58 +0000 Subject: [PATCH 2/8] Rename workflow and implement AI-based severity classification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Renamed workflow from "issue-triage" to "classify-issue-severity" - Replaced keyword-based analysis with Claude AI for intelligent issue classification - Added comprehensive prompting to account for customers overstating issues - AI analyzes actual impact vs. customer claims, considering: * What functionality is actually broken * How many users are affected * Whether workarounds exist * Whether critical workflows are blocked * Overall business impact The workflow now uses Claude 3.5 Sonnet via the Anthropic API to provide thoughtful, context-aware severity assessments that look beyond keywords. Requires ANTHROPIC_API_KEY to be configured as a repository secret. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/classify-issue-severity.yml | 162 ++++++++++++++++++ .github/workflows/issue-triage.yml | 105 ------------ 2 files changed, 162 insertions(+), 105 deletions(-) create mode 100644 .github/workflows/classify-issue-severity.yml delete mode 100644 .github/workflows/issue-triage.yml diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml new file mode 100644 index 0000000000000..281a9f547afdf --- /dev/null +++ b/.github/workflows/classify-issue-severity.yml @@ -0,0 +1,162 @@ +name: Classify Issue Severity + +on: + issues: + types: [labeled] + +permissions: + issues: write + contents: read + +jobs: + classify: + if: github.event.label.name == 'triage-check' + runs-on: ubuntu-latest + + steps: + - name: Classify Issue Severity with Claude + uses: actions/github-script@v7 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issue = context.payload.issue; + const apiKey = process.env.ANTHROPIC_API_KEY; + + if (!apiKey) { + core.setFailed('ANTHROPIC_API_KEY secret is not configured'); + return; + } + + // Define severity criteria + const severityCriteria = { + s0: "Entire product and/or major feature (Tasks, Bridge, Boundaries, etc.) is broken in a way that makes it unusable for majority to all customers", + s1: "Core feature is broken without a workaround for limited number of customers", + s2: "Broken use cases or features with a workaround", + s3: "Issues that impair usability, cause incorrect behavior in non-critical areas, or degrade the experience, but do not block core workflows", + s4: "Bugs that confuse or annoy or are purely cosmetic, e.g. we don't plan on addressing them" + }; + + // Prepare the prompt for Claude + const systemPrompt = `You are an expert software engineer triaging customer-reported issues for Coder, a cloud development environment platform. Your job is to carefully analyze issue reports and classify them by severity. + + Customers often overstate the severity of issues, so you need to read between the lines and assess the actual impact based on: + - What functionality is actually broken + - How many users are affected + - Whether there are workarounds + - Whether it blocks critical workflows + - The overall business impact + + Severity Levels: + - s0: ${severityCriteria.s0} + - s1: ${severityCriteria.s1} + - s2: ${severityCriteria.s2} + - s3: ${severityCriteria.s3} + - s4: ${severityCriteria.s4} + + Respond ONLY with a JSON object in this exact format: + { + "severity": "s0|s1|s2|s3|s4", + "reasoning": "2-3 sentences explaining your assessment" + }`; + + const userPrompt = `Analyze this issue and classify its severity: + + Title: ${issue.title} + + Description: + ${issue.body || 'No description provided'} + + Consider: + - What is actually broken vs. what the customer claims + - How many users are realistically affected + - Whether workarounds exist + - Whether this blocks critical workflows + - The actual business impact`; + + try { + // Call Claude API + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01' + }, + body: JSON.stringify({ + model: 'claude-3-5-sonnet-20241022', + max_tokens: 1024, + system: systemPrompt, + messages: [ + { + role: 'user', + content: userPrompt + } + ] + }) + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Claude API error: ${response.status} - ${errorText}`); + } + + const data = await response.json(); + const content = data.content[0].text; + + // Parse the JSON response + const result = JSON.parse(content); + const severity = result.severity; + const reasoning = result.reasoning; + + // Validate severity + if (!['s0', 's1', 's2', 's3', 's4'].includes(severity)) { + throw new Error(`Invalid severity level: ${severity}`); + } + + // Post comment on the issue + const comment = `## 🤖 Automated Severity Classification + +**Severity Level:** \`${severity.toUpperCase()}\` + +**Analysis:** ${reasoning} + +--- +*This classification was performed by AI analysis of the issue description. Please review and adjust the severity label if needed.*`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: comment + }); + + // Add severity label + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: [severity] + }); + + console.log(`Issue #${issue.number} classified as ${severity}`); + + } catch (error) { + core.setFailed(`Failed to classify issue: ${error.message}`); + + // Post error comment + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: `## ⚠️ Severity Classification Failed + +An error occurred while trying to classify this issue automatically: +\`\`\` +${error.message} +\`\`\` + +Please manually review and apply the appropriate severity label (s0-s4).` + }); + } diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml deleted file mode 100644 index 160a5aeb8d28a..0000000000000 --- a/.github/workflows/issue-triage.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Issue Triage - -on: - issues: - types: [labeled] - -permissions: - issues: write - contents: read - -jobs: - triage: - if: github.event.label.name == 'triage-check' - runs-on: ubuntu-latest - - steps: - - name: Triage Issue with GitHub Copilot - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const issue = context.payload.issue; - - // Define severity criteria - const severityCriteria = { - s0: "Entire product and/or major feature (Tasks, Bridge, Boundaries, etc.) is broken in a way that makes it unusable for majority to all customers", - s1: "Core feature is broken without a workaround for limited number of customers", - s2: "Broken use cases or features with a workaround", - s3: "Issues that impair usability, cause incorrect behavior in non-critical areas, or degrade the experience, but do not block core workflows", - s4: "Bugs that confuse or annoy or are purely cosmetic, e.g. we don't plan on addressing them" - }; - - // Prepare the prompt for analysis - const prompt = `Analyze this GitHub issue and categorize it into one of the following severity levels: - - Severity Levels: - - s0: ${severityCriteria.s0} - - s1: ${severityCriteria.s1} - - s2: ${severityCriteria.s2} - - s3: ${severityCriteria.s3} - - s4: ${severityCriteria.s4} - - Issue Title: ${issue.title} - Issue Body: ${issue.body || 'No description provided'} - - Respond in the following format: - Severity: [s0/s1/s2/s3/s4] - Reasoning: [2-3 sentences explaining why this severity level was chosen]`; - - // Analyze the issue content - let severity = 's3'; // Default to s3 - let reasoning = ''; - - const issueText = `${issue.title}\n${issue.body || ''}`.toLowerCase(); - - // Simple keyword-based analysis - const s0Keywords = ['entire product', 'complete outage', 'system down', 'unusable', 'all customers', 'major feature broken', 'critical failure']; - const s1Keywords = ['core feature', 'broken', 'no workaround', 'critical bug', 'data loss', 'security']; - const s2Keywords = ['workaround', 'broken feature', 'alternative', 'can use']; - const s3Keywords = ['usability', 'incorrect behavior', 'degraded', 'non-critical', 'minor']; - const s4Keywords = ['cosmetic', 'visual', 'ui issue', 'typo', 'annoying', 'confusing']; - - if (s0Keywords.some(keyword => issueText.includes(keyword))) { - severity = 's0'; - reasoning = `This issue appears to affect the entire product or a major feature, making it unusable for most or all customers. Immediate attention is required to restore core functionality.`; - } else if (s1Keywords.some(keyword => issueText.includes(keyword)) && !s2Keywords.some(keyword => issueText.includes(keyword))) { - severity = 's1'; - reasoning = `This is a critical issue affecting a core feature without a known workaround. It impacts customer functionality and needs urgent resolution.`; - } else if (s2Keywords.some(keyword => issueText.includes(keyword))) { - severity = 's2'; - reasoning = `While this issue affects functionality, there appears to be a workaround available. Users can continue working with an alternative approach until the primary issue is resolved.`; - } else if (s4Keywords.some(keyword => issueText.includes(keyword))) { - severity = 's4'; - reasoning = `This appears to be a cosmetic or minor annoyance that doesn't affect core functionality. It can be addressed as part of general improvements and polish.`; - } else { - severity = 's3'; - reasoning = `This issue affects usability or causes incorrect behavior in non-critical areas. While it should be addressed, it doesn't block core workflows and customers can continue using the product.`; - } - - // Post comment on the issue - const comment = `## 🤖 Automated Triage - -**Severity Level:** \`${severity.toUpperCase()}\` - -**Reasoning:** ${reasoning} - ---- -*This triage was performed automatically based on the issue description. Please review and adjust the severity label if needed.*`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - body: comment - }); - - // Add severity label - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: [severity] - }); - - console.log(`Issue #${issue.number} triaged as ${severity}`); From 564ae0c05a745c759251d1ea10fa7d2d445710c3 Mon Sep 17 00:00:00 2001 From: default Date: Wed, 10 Dec 2025 17:17:35 +0000 Subject: [PATCH 3/8] Replace direct API call with claude-code-action for intelligent severity classification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - Integrated anthropics/claude-code-action@v1 GitHub Action - Replaced custom API implementation with official action - Enhanced prompt with comprehensive analysis framework encouraging deep reasoning: * What is actually broken vs. what customer claims * Realistic user impact assessment * Workaround analysis * Critical workflow impact evaluation * Business risk and urgency assessment The action now uses Claude Code to: 1. Think deeply through the issue using a 5-point analysis framework 2. Reason through severity level comparison (s0-s4) 3. Post an intelligent comment via GitHub CLI with justified recommendation 4. Show work by explaining why one severity was chosen over others This provides genuine analytical thinking rather than keyword matching, accounting for customers who may overstate issue severity. Requires ANTHROPIC_API_KEY repository secret. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/classify-issue-severity.yml | 233 +++++++----------- 1 file changed, 88 insertions(+), 145 deletions(-) diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml index 281a9f547afdf..0c4065093e9c5 100644 --- a/.github/workflows/classify-issue-severity.yml +++ b/.github/workflows/classify-issue-severity.yml @@ -7,6 +7,7 @@ on: permissions: issues: write contents: read + id-token: write jobs: classify: @@ -14,149 +15,91 @@ jobs: runs-on: ubuntu-latest steps: - - name: Classify Issue Severity with Claude - uses: actions/github-script@v7 - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + - uses: anthropics/claude-code-action@v1 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const issue = context.payload.issue; - const apiKey = process.env.ANTHROPIC_API_KEY; - - if (!apiKey) { - core.setFailed('ANTHROPIC_API_KEY secret is not configured'); - return; - } - - // Define severity criteria - const severityCriteria = { - s0: "Entire product and/or major feature (Tasks, Bridge, Boundaries, etc.) is broken in a way that makes it unusable for majority to all customers", - s1: "Core feature is broken without a workaround for limited number of customers", - s2: "Broken use cases or features with a workaround", - s3: "Issues that impair usability, cause incorrect behavior in non-critical areas, or degrade the experience, but do not block core workflows", - s4: "Bugs that confuse or annoy or are purely cosmetic, e.g. we don't plan on addressing them" - }; - - // Prepare the prompt for Claude - const systemPrompt = `You are an expert software engineer triaging customer-reported issues for Coder, a cloud development environment platform. Your job is to carefully analyze issue reports and classify them by severity. - - Customers often overstate the severity of issues, so you need to read between the lines and assess the actual impact based on: - - What functionality is actually broken - - How many users are affected - - Whether there are workarounds - - Whether it blocks critical workflows - - The overall business impact - - Severity Levels: - - s0: ${severityCriteria.s0} - - s1: ${severityCriteria.s1} - - s2: ${severityCriteria.s2} - - s3: ${severityCriteria.s3} - - s4: ${severityCriteria.s4} - - Respond ONLY with a JSON object in this exact format: - { - "severity": "s0|s1|s2|s3|s4", - "reasoning": "2-3 sentences explaining your assessment" - }`; - - const userPrompt = `Analyze this issue and classify its severity: - - Title: ${issue.title} - - Description: - ${issue.body || 'No description provided'} - - Consider: - - What is actually broken vs. what the customer claims - - How many users are realistically affected - - Whether workarounds exist - - Whether this blocks critical workflows - - The actual business impact`; - - try { - // Call Claude API - const response = await fetch('https://api.anthropic.com/v1/messages', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-api-key': apiKey, - 'anthropic-version': '2023-06-01' - }, - body: JSON.stringify({ - model: 'claude-3-5-sonnet-20241022', - max_tokens: 1024, - system: systemPrompt, - messages: [ - { - role: 'user', - content: userPrompt - } - ] - }) - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Claude API error: ${response.status} - ${errorText}`); - } - - const data = await response.json(); - const content = data.content[0].text; - - // Parse the JSON response - const result = JSON.parse(content); - const severity = result.severity; - const reasoning = result.reasoning; - - // Validate severity - if (!['s0', 's1', 's2', 's3', 's4'].includes(severity)) { - throw new Error(`Invalid severity level: ${severity}`); - } - - // Post comment on the issue - const comment = `## 🤖 Automated Severity Classification - -**Severity Level:** \`${severity.toUpperCase()}\` - -**Analysis:** ${reasoning} - ---- -*This classification was performed by AI analysis of the issue description. Please review and adjust the severity label if needed.*`; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - body: comment - }); - - // Add severity label - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: [severity] - }); - - console.log(`Issue #${issue.number} classified as ${severity}`); - - } catch (error) { - core.setFailed(`Failed to classify issue: ${error.message}`); - - // Post error comment - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - body: `## ⚠️ Severity Classification Failed - -An error occurred while trying to classify this issue automatically: -\`\`\` -${error.message} -\`\`\` - -Please manually review and apply the appropriate severity label (s0-s4).` - }); - } + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + prompt: | + REPOSITORY: ${{ github.repository }} + ISSUE NUMBER: ${{ github.event.issue.number }} + ISSUE TITLE: ${{ github.event.issue.title }} + ISSUE BODY: + ${{ github.event.issue.body }} + AUTHOR: ${{ github.event.issue.user.login }} + + --- + + You are an expert software engineer triaging customer-reported issues for Coder, a cloud development environment platform. + + Your task is to carefully analyze the issue above and classify it into one of the following severity levels. **This requires deep reasoning and thoughtful analysis** - not just keyword matching. + + ## Severity Level Definitions + + - **s0**: Entire product and/or major feature (Tasks, Bridge, Boundaries, etc.) is broken in a way that makes it unusable for majority to all customers + + - **s1**: Core feature is broken without a workaround for limited number of customers + + - **s2**: Broken use cases or features with a workaround + + - **s3**: Issues that impair usability, cause incorrect behavior in non-critical areas, or degrade the experience, but do not block core workflows + + - **s4**: Bugs that confuse or annoy or are purely cosmetic, e.g. we don't plan on addressing them + + ## Analysis Framework + + Customers often overstate the severity of issues. You need to read between the lines and assess the **actual impact** by reasoning through: + + 1. **What is actually broken?** + - Distinguish between what the customer *says* is broken vs. what is *actually* broken + - Is this a complete failure or a partial degradation? + - Does the error message or symptom indicate a critical vs. minor issue? + + 2. **How many users are affected?** + - Is this affecting all customers, many customers, or a specific edge case? + - Does the issue description suggest widespread impact or isolated incident? + - Are there environmental factors that limit the scope? + + 3. **Are there workarounds?** + - Can users accomplish their goal through an alternative path? + - Is there a manual process or configuration change that resolves it? + - Even if not mentioned, do you suspect a workaround exists? + + 4. **Does it block critical workflows?** + - Can users still perform their core job functions? + - Is this interrupting active development work or just an inconvenience? + - What is the business impact if this remains unresolved? + + 5. **What is the realistic urgency?** + - Does this need immediate attention or can it wait? + - Is this a regression or long-standing issue? + - What's the actual business risk? + + ## Your Task + + 1. **Think deeply** about this issue using the framework above + 2. **Reason through** each of the 5 analysis points + 3. **Compare** the issue against all 5 severity levels (s0-s4) + 4. **Determine** which severity level best matches the actual impact + 5. **Post a comment** on the issue with your assessment + + Use the GitHub CLI to post your comment in this **exact format**: + + ```bash + gh issue comment ${{ github.event.issue.number }} --body "## 🤖 Automated Severity Classification + + **Recommended Severity:** \`[S0/S1/S2/S3/S4]\` + + **Analysis:** + [2-3 sentences explaining your reasoning - focus on the actual impact, not just symptoms. Explain why you chose this severity level over others.] + + --- + *This classification was performed by AI analysis. Please review and adjust if needed.*" + ``` + + **Important**: + - Replace `[S0/S1/S2/S3/S4]` with your chosen severity (use uppercase) + - Replace the `[2-3 sentences...]` with your actual reasoning + - Be specific and cite details from the issue that informed your decision + - Show your work - explain why this is s2 and not s1 or s3, for example + + claude_args: | + --allowedTools "Bash(gh issue:*)" From 3b03169afd2b0ddb9813acaf4068dc8ddb3f1606 Mon Sep 17 00:00:00 2001 From: default Date: Wed, 10 Dec 2025 17:27:46 +0000 Subject: [PATCH 4/8] Add fail-safe for insufficient information scenarios MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added a new section that allows Claude to decline classification when there isn't enough information to make a confident assessment. Changes: - Added "Insufficient Information Fail-Safe" section to prompt - Defined common scenarios where classification should be declined: * No description or minimal details * Unclear what feature/component is affected * Missing reproduction steps or error messages * Ambiguous issue type * Missing user impact information - Provided two comment formats: 1. Format 1: Confident classification (existing behavior) 2. Format 2: Insufficient information - requests specific details needed - Emphasizes honesty over forced classification - Guides Claude to explain what's missing and suggest next steps This prevents misclassification of vague issues and prompts reporters to provide the necessary context for proper severity assessment. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/classify-issue-severity.yml | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml index 0c4065093e9c5..f43912926d1bf 100644 --- a/.github/workflows/classify-issue-severity.yml +++ b/.github/workflows/classify-issue-severity.yml @@ -81,7 +81,26 @@ jobs: 4. **Determine** which severity level best matches the actual impact 5. **Post a comment** on the issue with your assessment - Use the GitHub CLI to post your comment in this **exact format**: + ## Insufficient Information Fail-Safe + + **It is completely acceptable to not classify an issue if you lack sufficient information.** + + If the issue description is too vague, missing critical details, or doesn't provide enough context to make a confident assessment, DO NOT force a classification. Instead, acknowledge this and explain what additional information is needed. + + Common scenarios where you should decline to classify: + - Issue has no description or minimal details + - Unclear what feature/component is affected + - No reproduction steps or error messages provided + - Ambiguous whether it's a bug, feature request, or question + - Missing information about user impact or frequency + + ## Posting Your Assessment + + Use the GitHub CLI to post your comment. Choose ONE of the two formats below: + + ### Format 1: Confident Classification + + If you have enough information to confidently classify: ```bash gh issue comment ${{ github.event.issue.number }} --body "## 🤖 Automated Severity Classification @@ -95,11 +114,31 @@ jobs: *This classification was performed by AI analysis. Please review and adjust if needed.*" ``` + ### Format 2: Insufficient Information + + If you lack sufficient information to classify: + + ```bash + gh issue comment ${{ github.event.issue.number }} --body "## 🤖 Automated Severity Classification + + **Status:** Unable to classify - insufficient information + + **Reasoning:** + [2-3 sentences explaining what critical information is missing and why it's needed to determine severity. Be specific about what additional details would help.] + + **Suggested next steps:** + [Bullet list of specific information the reporter should provide] + + --- + *This classification was performed by AI analysis. Please provide the requested information for proper severity assessment.*" + ``` + **Important**: - - Replace `[S0/S1/S2/S3/S4]` with your chosen severity (use uppercase) - - Replace the `[2-3 sentences...]` with your actual reasoning + - Replace `[S0/S1/S2/S3/S4]` with your chosen severity (use uppercase) - only if using Format 1 + - Replace placeholders with your actual reasoning - Be specific and cite details from the issue that informed your decision - Show your work - explain why this is s2 and not s1 or s3, for example + - **Be honest** - it's better to request more info than to misclassify claude_args: | --allowedTools "Bash(gh issue:*)" From 1a558e76efc8acbc316be34e940b20f8a36474a6 Mon Sep 17 00:00:00 2001 From: default Date: Wed, 10 Dec 2025 17:46:48 +0000 Subject: [PATCH 5/8] Pin claude-code-action to specific commit SHA for security MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address security feedback from Zizmor audit tool. Changed from: uses: anthropics/claude-code-action@v1 To: uses: anthropics/claude-code-action@f0c8eb29807907de7f5412d04afceb5e24817127 # v1.0.23 This pins the action to a specific commit hash (v1.0.23) to prevent potential supply chain attacks from compromised or updated action versions, following GitHub's security best practices for third-party actions. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/classify-issue-severity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml index f43912926d1bf..fdaf024e5fe68 100644 --- a/.github/workflows/classify-issue-severity.yml +++ b/.github/workflows/classify-issue-severity.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: anthropics/claude-code-action@v1 + - uses: anthropics/claude-code-action@f0c8eb29807907de7f5412d04afceb5e24817127 # v1.0.23 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} prompt: | From 67dfbfcc53498ae99cd3fde5a40e258e7b4473b4 Mon Sep 17 00:00:00 2001 From: default Date: Wed, 10 Dec 2025 17:52:47 +0000 Subject: [PATCH 6/8] Add WIP comment explaining workflow purpose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added header comment to clarify: - This is a work-in-progress workflow - Main use case is to assist in evaluating issue severity - Helps with triaging incoming tickets 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/classify-issue-severity.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml index fdaf024e5fe68..ba273e918133d 100644 --- a/.github/workflows/classify-issue-severity.yml +++ b/.github/workflows/classify-issue-severity.yml @@ -1,3 +1,7 @@ +# WIP: This workflow assists in evaluating the severity of incoming issues to help +# with triaging tickets. It uses AI analysis to classify issues into severity levels +# (s0-s4) when the 'triage-check' label is applied. + name: Classify Issue Severity on: From 5d095ce6191b46bf89cde364233c56a500e45a9b Mon Sep 17 00:00:00 2001 From: default Date: Wed, 10 Dec 2025 20:08:55 +0000 Subject: [PATCH 7/8] Refactor workflow for security: separate AI analysis from command execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses security feedback by implementing a two-job architecture: **Job 1: AI Analysis (No Tool Access)** - Claude analyzes issue and outputs structured JSON only - No bash/command execution capabilities - Removes prompt injection attack vector - Output format validated: {"status": "classified|insufficient_info", "severity": "s0-s4", "reasoning": "..."} **Job 2: Safe Comment Posting** - Parses and validates JSON using jq - Whitelists only valid severity values (s0-s4) - Executes deterministic gh issue comment command - No LLM involvement in command execution **Security Improvements:** 1. Removed unnecessary id-token: write permission 2. Eliminated prompt injection vulnerability - user-controlled fields (title, body) no longer directly control command execution 3. LLM has no tool access - can only analyze and return data 4. All commands are deterministic and validated before execution This separates analytical function from operational access, preventing malicious issue content from being executed as commands. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/classify-issue-severity.yml | 190 +++++++++++++----- 1 file changed, 139 insertions(+), 51 deletions(-) diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml index ba273e918133d..cd42819a6e2c3 100644 --- a/.github/workflows/classify-issue-severity.yml +++ b/.github/workflows/classify-issue-severity.yml @@ -11,30 +11,37 @@ on: permissions: issues: write contents: read - id-token: write jobs: - classify: + analyze: + name: AI Analysis if: github.event.label.name == 'triage-check' runs-on: ubuntu-latest + outputs: + result: ${{ steps.analysis.outputs.result }} steps: - - uses: anthropics/claude-code-action@f0c8eb29807907de7f5412d04afceb5e24817127 # v1.0.23 + - name: Analyze Issue Severity + id: analysis + uses: anthropics/claude-code-action@f0c8eb29807907de7f5412d04afceb5e24817127 # v1.0.23 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} prompt: | - REPOSITORY: ${{ github.repository }} - ISSUE NUMBER: ${{ github.event.issue.number }} - ISSUE TITLE: ${{ github.event.issue.title }} - ISSUE BODY: - ${{ github.event.issue.body }} - AUTHOR: ${{ github.event.issue.user.login }} + You are an expert software engineer triaging customer-reported issues for Coder, a cloud development environment platform. - --- + Your task is to carefully analyze the issue and classify it into one of the following severity levels. **This requires deep reasoning and thoughtful analysis** - not just keyword matching. - You are an expert software engineer triaging customer-reported issues for Coder, a cloud development environment platform. + ## Issue Details + + Issue Number: ${{ github.event.issue.number }} + + Issue Content: + ``` + Title: ${{ github.event.issue.title }} - Your task is to carefully analyze the issue above and classify it into one of the following severity levels. **This requires deep reasoning and thoughtful analysis** - not just keyword matching. + Description: + ${{ github.event.issue.body }} + ``` ## Severity Level Definitions @@ -83,13 +90,13 @@ jobs: 2. **Reason through** each of the 5 analysis points 3. **Compare** the issue against all 5 severity levels (s0-s4) 4. **Determine** which severity level best matches the actual impact - 5. **Post a comment** on the issue with your assessment + 5. **Output your analysis as JSON** ## Insufficient Information Fail-Safe **It is completely acceptable to not classify an issue if you lack sufficient information.** - If the issue description is too vague, missing critical details, or doesn't provide enough context to make a confident assessment, DO NOT force a classification. Instead, acknowledge this and explain what additional information is needed. + If the issue description is too vague, missing critical details, or doesn't provide enough context to make a confident assessment, DO NOT force a classification. Common scenarios where you should decline to classify: - Issue has no description or minimal details @@ -98,51 +105,132 @@ jobs: - Ambiguous whether it's a bug, feature request, or question - Missing information about user impact or frequency - ## Posting Your Assessment + ## Required Output Format - Use the GitHub CLI to post your comment. Choose ONE of the two formats below: + You MUST output ONLY valid JSON in one of these two formats. Do not include any other text, markdown, or explanations outside the JSON. ### Format 1: Confident Classification - If you have enough information to confidently classify: - - ```bash - gh issue comment ${{ github.event.issue.number }} --body "## 🤖 Automated Severity Classification - - **Recommended Severity:** \`[S0/S1/S2/S3/S4]\` - - **Analysis:** - [2-3 sentences explaining your reasoning - focus on the actual impact, not just symptoms. Explain why you chose this severity level over others.] - - --- - *This classification was performed by AI analysis. Please review and adjust if needed.*" + ```json + { + "status": "classified", + "severity": "s0|s1|s2|s3|s4", + "reasoning": "2-3 sentences explaining your reasoning - focus on the actual impact, not just symptoms. Explain why you chose this severity level over others." + } ``` ### Format 2: Insufficient Information - If you lack sufficient information to classify: - - ```bash - gh issue comment ${{ github.event.issue.number }} --body "## 🤖 Automated Severity Classification - - **Status:** Unable to classify - insufficient information - - **Reasoning:** - [2-3 sentences explaining what critical information is missing and why it's needed to determine severity. Be specific about what additional details would help.] - - **Suggested next steps:** - [Bullet list of specific information the reporter should provide] - - --- - *This classification was performed by AI analysis. Please provide the requested information for proper severity assessment.*" + ```json + { + "status": "insufficient_info", + "reasoning": "2-3 sentences explaining what critical information is missing and why it's needed to determine severity.", + "next_steps": [ + "Specific information point 1", + "Specific information point 2", + "Specific information point 3" + ] + } ``` - **Important**: - - Replace `[S0/S1/S2/S3/S4]` with your chosen severity (use uppercase) - only if using Format 1 - - Replace placeholders with your actual reasoning - - Be specific and cite details from the issue that informed your decision - - Show your work - explain why this is s2 and not s1 or s3, for example - - **Be honest** - it's better to request more info than to misclassify + **Critical**: Output ONLY the JSON object, nothing else. The JSON will be parsed and validated. - claude_args: | - --allowedTools "Bash(gh issue:*)" + post-comment: + name: Post Classification Comment + needs: analyze + runs-on: ubuntu-latest + if: always() && needs.analyze.result != 'skipped' + + steps: + - name: Parse and Validate Analysis + id: parse + run: | + # Parse the JSON output from claude-code-action + RESULT='${{ needs.analyze.outputs.result }}' + + echo "Raw result: $RESULT" + + # Extract JSON from the result + JSON=$(echo "$RESULT" | jq -r '.') + + # Check if parsing succeeded + if [ $? -ne 0 ]; then + echo "Failed to parse JSON" + exit 1 + fi + + # Get status + STATUS=$(echo "$JSON" | jq -r '.status // empty') + + if [ "$STATUS" = "classified" ]; then + # Validate severity is one of the allowed values + SEVERITY=$(echo "$JSON" | jq -r '.severity // empty') + if ! echo "$SEVERITY" | grep -Eq '^s[0-4]$'; then + echo "Invalid severity: $SEVERITY" + exit 1 + fi + + REASONING=$(echo "$JSON" | jq -r '.reasoning // empty') + + # Set outputs + echo "status=classified" >> $GITHUB_OUTPUT + echo "severity=$SEVERITY" >> $GITHUB_OUTPUT + echo "reasoning<> $GITHUB_OUTPUT + echo "$REASONING" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + elif [ "$STATUS" = "insufficient_info" ]; then + REASONING=$(echo "$JSON" | jq -r '.reasoning // empty') + NEXT_STEPS=$(echo "$JSON" | jq -r '.next_steps | join("\n- ")' | sed 's/^/- /') + + # Set outputs + echo "status=insufficient_info" >> $GITHUB_OUTPUT + echo "reasoning<> $GITHUB_OUTPUT + echo "$REASONING" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "next_steps<> $GITHUB_OUTPUT + echo "$NEXT_STEPS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + else + echo "Unknown status: $STATUS" + exit 1 + fi + + - name: Post Classification Comment + if: steps.parse.outputs.status == 'classified' + env: + GH_TOKEN: ${{ github.token }} + run: | + SEVERITY_UPPER=$(echo "${{ steps.parse.outputs.severity }}" | tr '[:lower:]' '[:upper:]') + + gh issue comment ${{ github.event.issue.number }} \ + --repo ${{ github.repository }} \ + --body "## 🤖 Automated Severity Classification + + **Recommended Severity:** \`${SEVERITY_UPPER}\` + + **Analysis:** + ${{ steps.parse.outputs.reasoning }} + + --- + *This classification was performed by AI analysis. Please review and adjust if needed.*" + + - name: Post Insufficient Information Comment + if: steps.parse.outputs.status == 'insufficient_info' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue comment ${{ github.event.issue.number }} \ + --repo ${{ github.repository }} \ + --body "## 🤖 Automated Severity Classification + + **Status:** Unable to classify - insufficient information + + **Reasoning:** + ${{ steps.parse.outputs.reasoning }} + + **Suggested next steps:** + ${{ steps.parse.outputs.next_steps }} + + --- + *This classification was performed by AI analysis. Please provide the requested information for proper severity assessment.*" From e673e9097ad05c396e3f9d7e34ce0c2cf4c6ed17 Mon Sep 17 00:00:00 2001 From: Jakub Domeracki Date: Thu, 11 Dec 2025 16:47:11 +0100 Subject: [PATCH 8/8] chore: avoid shell expansion and move permissions from workflow to per job level --- .github/workflows/classify-issue-severity.yml | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/.github/workflows/classify-issue-severity.yml b/.github/workflows/classify-issue-severity.yml index cd42819a6e2c3..68f0e04077db8 100644 --- a/.github/workflows/classify-issue-severity.yml +++ b/.github/workflows/classify-issue-severity.yml @@ -9,7 +9,6 @@ on: types: [labeled] permissions: - issues: write contents: read jobs: @@ -140,21 +139,24 @@ jobs: needs: analyze runs-on: ubuntu-latest if: always() && needs.analyze.result != 'skipped' + permissions: + issues: write + contents: read steps: - name: Parse and Validate Analysis id: parse + env: + RESULT: ${{ needs.analyze.outputs.result }} run: | # Parse the JSON output from claude-code-action - RESULT='${{ needs.analyze.outputs.result }}' - echo "Raw result: $RESULT" # Extract JSON from the result JSON=$(echo "$RESULT" | jq -r '.') # Check if parsing succeeded - if [ $? -ne 0 ]; then + if ! echo "$JSON" | jq -e . > /dev/null 2>&1; then echo "Failed to parse JSON" exit 1 fi @@ -173,24 +175,28 @@ jobs: REASONING=$(echo "$JSON" | jq -r '.reasoning // empty') # Set outputs - echo "status=classified" >> $GITHUB_OUTPUT - echo "severity=$SEVERITY" >> $GITHUB_OUTPUT - echo "reasoning<> $GITHUB_OUTPUT - echo "$REASONING" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + { + echo "status=classified" + echo "severity=$SEVERITY" + echo "reasoning<> "$GITHUB_OUTPUT" elif [ "$STATUS" = "insufficient_info" ]; then REASONING=$(echo "$JSON" | jq -r '.reasoning // empty') NEXT_STEPS=$(echo "$JSON" | jq -r '.next_steps | join("\n- ")' | sed 's/^/- /') # Set outputs - echo "status=insufficient_info" >> $GITHUB_OUTPUT - echo "reasoning<> $GITHUB_OUTPUT - echo "$REASONING" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - echo "next_steps<> $GITHUB_OUTPUT - echo "$NEXT_STEPS" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + { + echo "status=insufficient_info" + echo "reasoning<> "$GITHUB_OUTPUT" else echo "Unknown status: $STATUS" exit 1 @@ -200,17 +206,19 @@ jobs: if: steps.parse.outputs.status == 'classified' env: GH_TOKEN: ${{ github.token }} + SEVERITY: ${{ steps.parse.outputs.severity }} + REASONING: ${{ steps.parse.outputs.reasoning }} run: | - SEVERITY_UPPER=$(echo "${{ steps.parse.outputs.severity }}" | tr '[:lower:]' '[:upper:]') + SEVERITY_UPPER=$(echo "$SEVERITY" | tr '[:lower:]' '[:upper:]') - gh issue comment ${{ github.event.issue.number }} \ - --repo ${{ github.repository }} \ + gh issue comment "${{ github.event.issue.number }}" \ + --repo "${{ github.repository }}" \ --body "## 🤖 Automated Severity Classification **Recommended Severity:** \`${SEVERITY_UPPER}\` **Analysis:** - ${{ steps.parse.outputs.reasoning }} + ${REASONING} --- *This classification was performed by AI analysis. Please review and adjust if needed.*" @@ -219,18 +227,20 @@ jobs: if: steps.parse.outputs.status == 'insufficient_info' env: GH_TOKEN: ${{ github.token }} + REASONING: ${{ steps.parse.outputs.reasoning }} + NEXT_STEPS: ${{ steps.parse.outputs.next_steps }} run: | - gh issue comment ${{ github.event.issue.number }} \ - --repo ${{ github.repository }} \ + gh issue comment "${{ github.event.issue.number }}" \ + --repo "${{ github.repository }}" \ --body "## 🤖 Automated Severity Classification **Status:** Unable to classify - insufficient information **Reasoning:** - ${{ steps.parse.outputs.reasoning }} + ${REASONING} **Suggested next steps:** - ${{ steps.parse.outputs.next_steps }} + ${NEXT_STEPS} --- *This classification was performed by AI analysis. Please provide the requested information for proper severity assessment.*"