[Security Rules] Integrate security_detection_engine OOM testing pipeline (#15829)

**Partially addresses:** elastic/kibana#188090

## Summary

This PR integrates [Prebuilt Rules OOM testing Buildkite pipeline](https://buildkite.com/elastic/appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing) into the Pull Request Buildkite pipeline.

## Details

Pull Request Builkite pipeline script have been extended in a generic way to support custom package checker scripts located under `<repo-root>/.buildkite/scripts/packages/<package-name>.sh`.  It allows to run any custom verification and testing logic specific to a package.

This PR adds `.buildkite/scripts/packages/security_detection_engine.sh` script file. This script runs only for **security_detection_engine** package and triggers the [Prebuilt Rules Out-Of-Memory testing pipeline](https://buildkite.com/elastic/appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing). The triggered pipeline performs e2e testing to reveal potential blockers due to Kibana Out-Of-Memory instance failures when performing actions upon the package (installing the package, review prebuilt rules available in the package, installing prebuilt rules from the package etc.).

### Tested stack versions

For now `.buildkite/scripts/packages/security_detection_engine.sh` triggers [Prebuilt Rules OOM testing Buildkite pipeline](https://buildkite.com/elastic/appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing) against compatible minor versions under development. The decision is made based on Kibana's [versions.json](https://github.com/raw-content/elastic/kibana/main/versions.json). While compatibility is determined via `conditions.kibana.version` field in the package's `manifest.yml`.

For example `conditions.kibana.version` has `^9.2.0` restriction and we have `9.2.2` and `9.3.0` under development. It means the OOM tests will run against  `9.2.2-SNAPSHOT` and `9.3.0-SNAPSHOT`.

We consider extending the testing surface to the latest release patch versions after collecting more data in the CI runs.

## Affected teams

@elastic/threat-research-and-detection-engineering,

FYI this PR will affect **security_detection_engine** package release process. Every PR containing changes to the **security_detection_engine** package will trigger [Prebuilt Rules OOM testing ECH Buildkite pipeline](https://buildkite.com/elastic/appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing).

## Further improvements

- Pushing commits to this repo in a quick succession may lead to leaving rouge resources in the cloud. It happens due to `cancel_intermediate_builds: true` configuration at the Integrations PR Buildkite build. Pushing a fresh commit cancels the currently running PR build leading to cancelling the triggered build. Eventually the clean up steps in the triggered build can't execute and clean up resources in the cloud. 
- We may speed up the build by using an **elastic-package** Docker container published to `docker.elastic.co`. **elastic-package** installation is a complex process requiring a chain on installations GVM -> Go -> elastic-package. And it takes in average **3 minutes** per each integration (integrations build in parallel). On top of that [Prebuilt Rules OOM testing Buildkite pipeline](https://buildkite.com/elastic/appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing) has to install **elastic-package** as well. It sums up to **6 minutes** which could be reduced.

Author: maximpn

.buildkite/scripts/common.sh

@@ -728,7 +728,7 @@ is_pr_affected() {
             return 1
         fi
         if ! is_supported_capability ; then
-            echo "[${package}] PR is not affected: capabilities not mached with the project (${SERVERLESS_PROJECT})"
+            echo "[${package}] PR is not affected: capabilities not matched with the project (${SERVERLESS_PROJECT})"
             return 1
         fi
         if [[ "${package}" == "fleet_server" ]]; then
@@ -763,10 +763,19 @@ is_pr_affected() {
     # Example:
     # https://buildkite.com/elastic/integrations/builds/25606
     # https://github.com/elastic/integrations/pull/13810
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|CODE_OF_CONDUCT\.md|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml))' > /dev/null; then
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE|workflows/)|CODE_OF_CONDUCT\.md|README\.md|docs/|catalog-info\.yaml|\.buildkite/(pull-requests\.json|pipeline\.schedule-daily\.yml|pipeline\.schedule-weekly\.yml|pipeline\.backport\.yml|scripts/packages/.+\.sh))' > /dev/null; then
         echo "[${package}] PR is affected: found non-package files"
         return 0
     fi
+    echoerr "[${package}] git-diff: check custom package checker script file (${commit_merge}..${to})"
+    # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
+    # Example:
+    # https://buildkite.com/elastic/integrations/builds/25606
+    # https://github.com/elastic/integrations/pull/13810
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E "^\.buildkite/scripts/packages/${package}.sh" > /dev/null; then
+        echo "[${package}] PR is affected: found package checker script changes"
+        return 0
+    fi
     echoerr "[${package}] git-diff: check package files (${commit_merge}..${to})"
     # Avoid using "-q" in grep in this pipe, it could cause that some files updated are not detected due to SIGPIPE errors when "set -o pipefail"
     # Example:

.buildkite/scripts/packages/security_detection_engine.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+  exit 0
+fi
+
+# Fetch active Kibana versions
+ACTIVE_KIBANA_VERSIONS=$(curl -sL https://github.com/raw-content/elastic/kibana/main/versions.json | yq '.versions[].version' | xargs)
+echo "Active Kibana versions: $ACTIVE_KIBANA_VERSIONS"
+
+# Extract version spec from the manifest
+KIBANA_REQ=$(yq .conditions.kibana.version ./packages/security_detection_engine/manifest.yml)
+echo "Kibana requirement from the security_detection_engine manifest: $KIBANA_REQ"
+
+# Dump a trivial Go program to filter by semver constrains
+TEMP_DIR=$(mktemp -d)
+SEMVER_FILTER_PATH="$TEMP_DIR/semver.go"
+
+cat <<'GO' > "$SEMVER_FILTER_PATH"
+package main
+
+import (
+  "strings"
+	"fmt"
+	"os"
+	"github.com/Masterminds/semver/v3"
+)
+
+func main() {
+	c, err := semver.NewConstraint(os.Args[1])
+  if err != nil {
+		panic(err)
+	}
+
+	for _, s := range strings.Split(os.Args[2], " ") {
+		if v, _ := semver.NewVersion(s); c.Check(v) {
+			fmt.Println(s + "-SNAPSHOT")
+		}
+	}
+}
+GO
+
+# Capture the "returned" array in STACK_VERSIONS
+read -r -a STACK_VERSIONS <<< "$(go run "${SEMVER_FILTER_PATH}" "${KIBANA_REQ}" "${ACTIVE_KIBANA_VERSIONS}" | xargs)"
+
+if [[ ! -n "${STACK_VERSIONS+x}" ]]; then
+	echo "There are no active versions satisfying the constraint ${KIBANA_REQ}."
+  exit 0
+fi
+
+# Trigger OOM testing pipeline for each stack version
+for STACK_VERSION in "${STACK_VERSIONS[@]}"
+do
+  echo "--- [security_detection_engine] Trigger OOM testing pipeline against $STACK_VERSION ECH"
+
+  cat <<YAML | buildkite-agent pipeline upload
+steps:
+  - key: 'run-oom-testing-$(echo "$STACK_VERSION" | sed 's/\./_/g')$BUILDKITE_BUILD_NUMBER'
+    label: ":elastic-cloud::bar_chart: [security_detection_engine] Test for OOM issues against $STACK_VERSION ECH"
+    trigger: "appex-qa-stateful-security-prebuilt-rules-ftr-oom-testing"
+    async: false
+    build:
+      message: "Test security_detection_engine package against $STACK_VERSION ($GITHUB_PR_BASE_OWNER/$GITHUB_PR_BASE_REPO, branch: $GITHUB_PR_BRANCH, commit: $BUILDKITE_COMMIT)"
+      env:
+        STACK_VERSION: $STACK_VERSION
+        ELASTIC_INTEGRATIONS_REPO_COMMIT: $BUILDKITE_COMMIT
+YAML
+done

.buildkite/scripts/test_one_package.sh

@@ -35,4 +35,13 @@ if ! process_package "${package}" ; then
 fi
 popd > /dev/null
 
-exit "${exit_code}"
+if [ "${exit_code}" -ne 0 ] ; then
+  exit "${exit_code}"
+fi
+
+custom_package_checker_script_path="${SCRIPTS_BUILDKITE_PATH}/packages/${package}.sh"
+
+if [ -x "$custom_package_checker_script_path" ]; then
+  echo "--- [${package}] Run individual package checker"
+  "$custom_package_checker_script_path"
+fi