test: add a retry for the merge queue (#762)

The tests need to pass checks before being added to the merge queue so
we can avoid merge queue failures in case there are flaky tests.

Author: OrKoN

.github/workflows/run-tests.yml

@@ -59,7 +59,8 @@ jobs:
 
       - name: Run tests
         shell: bash
-        run: npm run test:no-build
+        # Retry tests if they fail in the merge queue.
+        run: npm run test:no-build -- ${{ github.event_name == 'merge_group' && '--retry' || '' }}
 
   # Gating job for branch protection.
   test-success:

package.json

@@ -16,7 +16,7 @@
     "docs:generate": "node --experimental-strip-types scripts/generate-docs.ts",
     "start": "npm run build && node build/src/index.js",
     "start-debug": "DEBUG=mcp:* DEBUG_COLORS=false npm run build && node build/src/index.js",
-    "test": "npm run build && npm run test:no-build",
+    "test": "npm run build && node scripts/test.mjs",
     "test:no-build": "node scripts/test.mjs",
     "test:only": "npm run build && node scripts/test.mjs --test-only",
     "test:update-snapshots": "npm run build && node scripts/test.mjs --test-update-snapshots",

scripts/test.mjs

@@ -17,6 +17,13 @@ const flags = args.filter(arg => arg.startsWith('-'));
 
 const files = [];
 
+let shouldRetry = false;
+const retryIndex = flags.indexOf('--retry');
+if (retryIndex !== -1) {
+  shouldRetry = true;
+  flags.splice(retryIndex, 1);
+}
+
 if (userArgs.length > 0) {
   for (const arg of userArgs) {
     // Map .ts files to build/ .js files
@@ -50,10 +57,29 @@ const nodeArgs = [
   ...files,
 ];
 
-const child = spawn('node', nodeArgs, {
-  stdio: 'inherit',
-});
+async function runTests(attempt) {
+  if (attempt > 1) {
+    console.log(`\nRun attempt ${attempt}...\n`);
+  }
+  return new Promise(resolve => {
+    const child = spawn('node', nodeArgs, {
+      stdio: 'inherit',
+    });
+
+    child.on('close', code => {
+      resolve(code);
+    });
+  });
+}
+
+const maxAttempts = shouldRetry ? 3 : 1;
+let exitCode = 1;
+
+for (let i = 1; i <= maxAttempts; i++) {
+  exitCode = await runTests(i);
+  if (exitCode === 0) {
+    break;
+  }
+}
 
-child.on('close', code => {
-  process.exit(code ?? 1);
-});
+process.exit(exitCode ?? 1);