fix(checks): tighten checkToolErrorSpan to require non-ok span status (#141)

Simplify the tool error span check to focus on what matters: the span
status field must be set and must not be "ok". Previously the check
looked for a grab-bag of error indicators (data.error, data.exception,
gen_ai.tool.error, tags.error) in addition to status, which was overly
lenient — a span could pass by having any random error-like data key
even if the status itself was wrong.

The new check requires that span.status is present and is not "ok"
(e.g. "internal_error", "error"). This catches frameworks like
Python LangGraph where the tool span status is unset despite the tool
raising an exception.

Author: constantinius

src/test-cases/agents/tool-error.ts

@@ -53,21 +53,16 @@ const checkToolErrorSpan: Check = {
       );
     }
 
-    // Check for error indicators
-    const hasError =
-      toolSpan.status === "error" ||
-      toolSpan.status === "internal_error" ||
-      toolSpan.data?.["error"] !== undefined ||
-      toolSpan.data?.["exception"] !== undefined ||
-      toolSpan.data?.["gen_ai.tool.error"] !== undefined ||
-      (toolSpan.tags && toolSpan.tags["error"] === true);
-
-    if (!hasError) {
+    // The tool raised an error, so the span status must not be "ok" or unset.
+    // Valid error statuses: "internal_error", "error", etc.
+    const status = toolSpan.status;
+    if (!status || status === "ok") {
       throw new CheckError(
-        "Tool span should have an error indicator (status=error, data.error, data.exception, gen_ai.tool.error, or tags.error)",
+        `Tool span status should indicate an error (e.g. "internal_error") but got "${status ?? "undefined"}"`,
         [{
           spanId: toolSpan.span_id,
-          message: `Tool span has status="${toolSpan.status}" with no error indicators`,
+          attribute: "status",
+          message: `Tool span status is "${status ?? "undefined"}", expected a non-ok error status`,
         }],
       );
     }