chore(tests): add scheduled sweeper for leaked e2e infrastructure

.github/scripts/report_e2e_sweep.js

@@ -0,0 +1,100 @@
+const INCIDENT_TITLE = 'E2E stale stack cleanup failing';
+const MAX_SECTION_ENTRIES = 50;
+const MAX_BODY_LENGTH = 60_000;
+
+/** Find the open incident issue by its exact title. */
+const findIncident = async ({ github, owner, repo }) => {
+  const { data: issues } = await github.rest.issues.listForRepo({
+    owner,
+    repo,
+    state: 'open',
+    per_page: 100,
+  });
+
+  return issues.find(
+    (issue) =>
+      issue.title === INCIDENT_TITLE && issue.pull_request === undefined
+  );
+};
+
+/** Render one bounded report section as Markdown. */
+const formatSection = (title, items, format, runUrl) => {
+  if (items.length === 0) return `### ${title}\n\n_None_`;
+  const visibleItems = items.slice(0, MAX_SECTION_ENTRIES);
+  const lines = visibleItems.map((item) => `- ${format(item)}`);
+  const hiddenCount = items.length - visibleItems.length;
+  if (hiddenCount > 0) {
+    lines.push(
+      `- …and ${hiddenCount} more — see the [workflow run](${runUrl})`
+    );
+  }
+  return `### ${title}\n\n${lines.join('\n')}`;
+};
+
+/** Render a bounded sweep report for job summaries and incident issues. */
+const formatReport = ({ report, runUrl }) => {
+  const body = [
+    '# E2E stale stack sweep',
+    `- Mode: **${report.dryRun ? 'dry run' : 'cleanup'}**\n- Timestamp: ${report.timestamp}\n- Discovery failed: ${report.discoveryFailed ? 'yes' : 'no'}`,
+    formatSection(
+      'Candidates',
+      report.candidates,
+      (item) =>
+        `\`${item.stackName}\` — ${item.status}; ${item.ageHours} hours old; reason: ${item.reason}`,
+      runUrl
+    ),
+    formatSection('Deleted', report.deleted, (name) => `\`${name}\``, runUrl),
+    formatSection(
+      'Skipped in progress',
+      report.skippedInProgress,
+      (item) => `\`${item.stackName}\` — ${item.status}`,
+      runUrl
+    ),
+    formatSection(
+      'Unresolved',
+      report.unresolved,
+      (item) =>
+        `\`${item.stackName}\` — ${item.status}; reason: ${item.reason}`,
+      runUrl
+    ),
+    `[View workflow run](${runUrl})`,
+  ].join('\n\n');
+
+  if (body.length <= MAX_BODY_LENGTH) return body;
+  const suffix = `\n\n_Report truncated — see the [workflow run](${runUrl})._`;
+  return `${body.slice(0, MAX_BODY_LENGTH - suffix.length)}${suffix}`;
+};
+
+/** Create or update the stale-stack incident issue after a failed sweep. */
+const reportE2eSweep = async ({ github, context, core, report }) => {
+  const { owner, repo } = context.repo;
+  const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
+  if (report.ok === true) {
+    core.info('Sweep completed without an incident');
+    return;
+  }
+
+  const incident = await findIncident({ github, owner, repo });
+  const body = formatReport({ report, runUrl });
+  if (incident === undefined) {
+    const { data: issue } = await github.rest.issues.create({
+      owner,
+      repo,
+      title: INCIDENT_TITLE,
+      body,
+    });
+    core.info(`Created incident issue #${issue.number}`);
+    return;
+  }
+
+  await github.rest.issues.createComment({
+    owner,
+    repo,
+    issue_number: incident.number,
+    body,
+  });
+  core.info(`Updated incident issue #${incident.number}`);
+};
+
+module.exports = reportE2eSweep;
+module.exports.formatReport = formatReport;

.github/workflows/sweep-stale-e2e-stacks.yml

@@ -0,0 +1,148 @@
+name: Sweep stale e2e stacks
+
+# STALE E2E STACK SWEEPER
+#
+# Nightly safety net for issue #5519. Deletes leaked e2e CloudFormation stacks
+# older than 12 hours when selected by either:
+# - the exact `Service=Powertools-for-AWS-e2e-tests` tag
+# - an `LmiShared-*` stack name
+#
+# Triggers:
+# - Nightly cron at 02:00 UTC; always performs a real cleanup run.
+# - Manual `workflow_dispatch`; `dry_run` defaults to true. Dry runs only
+#   discover stacks and write a job summary, deleting nothing and creating no issue.
+#
+# Failure behavior:
+# - When stacks remain unresolved after two dependency-ordered passes, the notify
+#   job creates or comments on a single incident issue, deduplicated by exact title,
+#   and the run fails.
+# - The incident stays open until a maintainer closes it.
+#
+# Authentication:
+# - Assumes the E2E OIDC role via the `e2e-tests` environment in eu-west-1;
+#   requires no standing infrastructure.
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Discover stale stacks without deleting them
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: sweep-stale-e2e-stacks
+  cancel-in-progress: false
+
+jobs:
+  sweep:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    environment: e2e-tests
+    permissions:
+      id-token: write # needed to interact with GitHub's OIDC Token endpoint.
+      contents: read
+    outputs:
+      dry_run: ${{ steps.mode.outputs.dry_run }}
+      report: ${{ steps.report.outputs.report }}
+      sweep_outcome: ${{ steps.sweep.outcome }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 24
+      - name: Setup dependencies
+        uses: aws-powertools/actions/.github/actions/cached-node-modules@3b5b8e2e58b7af07994be982e83584a94e8c76c5 # v1.5.0
+        with:
+          node-version: 24
+      - name: Setup AWS credentials
+        uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+        with:
+          role-to-assume: ${{ secrets.E2E_IAM_ROLE_ARN }}
+          aws-region: eu-west-1
+          mask-aws-account-id: true
+      - name: Select sweep mode
+        id: mode
+        env:
+          DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run != false }}
+        run: echo "dry_run=$DRY_RUN" >> "$GITHUB_OUTPUT"
+      - name: Sweep stale stacks
+        id: sweep
+        continue-on-error: true
+        env:
+          DRY_RUN: ${{ steps.mode.outputs.dry_run }}
+        run: npm run --silent e2e:sweep -w packages/testing > "$RUNNER_TEMP/report.json"
+      - name: Record report
+        id: report
+        if: always()
+        run: |
+          report=$(node -e '
+            const fs = require("node:fs");
+            const path = process.env.REPORT_PATH;
+            let report;
+            try {
+              report = JSON.parse(fs.readFileSync(path, "utf8"));
+            } catch {
+              report = {
+                dryRun: process.env.DRY_RUN === "true",
+                timestamp: new Date().toISOString(),
+                candidates: [],
+                deleted: [],
+                skippedInProgress: [],
+                unresolved: [],
+                discoveryFailed: true,
+                ok: false,
+              };
+            }
+            process.stdout.write(JSON.stringify(report));
+          ')
+          echo "report=$report" >> "$GITHUB_OUTPUT"
+        env:
+          REPORT_PATH: ${{ runner.temp }}/report.json
+          DRY_RUN: ${{ steps.mode.outputs.dry_run }}
+      - name: Add sweep summary
+        if: always()
+        env:
+          REPORT_JSON: ${{ steps.report.outputs.report }}
+        run: |
+          node <<'NODE' >> "$GITHUB_STEP_SUMMARY"
+          const { formatReport } = require('./.github/scripts/report_e2e_sweep.js');
+          const report = JSON.parse(process.env.REPORT_JSON);
+          const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+          console.log(formatReport({ report, runUrl }));
+          NODE
+      - name: Fail after unsuccessful dry run
+        if: always() && steps.mode.outputs.dry_run == 'true' && steps.sweep.outcome != 'success'
+        run: exit 1
+
+  notify:
+    needs: sweep
+    if: always() && needs.sweep.outputs.dry_run == 'false'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Report sweep status
+        id: notification
+        continue-on-error: true
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          REPORT_JSON: ${{ needs.sweep.outputs.report }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const report_e2e_sweep = require('./.github/scripts/report_e2e_sweep.js')
+            await report_e2e_sweep({github, context, core, report: JSON.parse(process.env.REPORT_JSON)})
+      - name: Fail after unsuccessful cleanup
+        if: always() && (needs.sweep.outputs.sweep_outcome != 'success' || steps.notification.outcome != 'success')
+        run: exit 1

packages/testing/package.json

@@ -13,6 +13,7 @@
     "test:e2e": "echo 'Not implemented'",
     "lmi:deploy": "node lib/esm/lmi/deploySharedCapacityProvider.js",
     "lmi:destroy": "node lib/esm/lmi/destroySharedCapacityProvider.js",
+    "e2e:sweep": "node lib/esm/e2e/sweepStaleStacksCli.js",
     "build:cjs": "tsc --build tsconfig.cjs.json && echo '{ \"type\": \"commonjs\" }' > lib/cjs/package.json",
     "build:esm": "tsc --build tsconfig.json && echo '{ \"type\": \"module\" }' > lib/esm/package.json",
     "build:tests": "tsc --noEmit -p tests/tsconfig.json",
@@ -129,6 +130,7 @@
   },
   "devDependencies": {
     "@types/promise-retry": "^1.1.6",
+    "aws-sdk-client-mock": "^4.1.0",
     "aws-sdk-client-mock-vitest": "^7.1.0"
   }
 }

packages/testing/src/TestStack.ts

@@ -7,7 +7,7 @@ import {
   StackSelectionStrategy,
   Toolkit,
 } from '@aws-cdk/toolkit-lib';
-import { App, Stack } from 'aws-cdk-lib';
+import { App, Stack, Tags } from 'aws-cdk-lib';
 import { generateTestUniqueName } from './helpers.js';
 import type { TestStackProps } from './types.js';
 
@@ -68,13 +68,8 @@ class TestStack {
       testPrefix: stackNameProps.stackNamePrefix,
     });
     this.app = app ?? new App();
-    this.stack =
-      stack ??
-      new Stack(this.app, this.testName, {
-        tags: {
-          Service: 'Powertools-for-AWS-e2e-tests',
-        },
-      });
+    this.stack = stack ?? new Stack(this.app, this.testName);
+    Tags.of(this.stack).add('Service', 'Powertools-for-AWS-e2e-tests');
     let lastCreateLog = 0;
     let lastDestroyLog = 0;
     const creationDeleteLogFrequency = 10000; // 10 seconds