[e2e] Fix flaky user-metrics test

The flake is not reproducible locally
This is an attempt to fix the the observed issue, with a new approach
to how we load a panel.

Bug: 444182703
No-Try: True
Change-Id: I74df16aae8fae86349e5425c9b9067314a575217
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6983466
Auto-Submit: Nikolay Vitkov <nvitkov@chromium.org>
Commit-Queue: Nikolay Vitkov <nvitkov@chromium.org>
Reviewed-by: Alex Rudenko <alexrudenko@chromium.org>

Author: Lightning00Blade

test/e2e_non_hosted/conductor/state-provider.ts

@@ -137,5 +137,6 @@ export function mergeSettings(s1: E2E.SuiteSettings, s2: E2E.HarnessSettings): E
     disabledDevToolsExperiments: mergeAsSet(s1.disabledDevToolsExperiments, s2.disabledDevToolsExperiments),
     devToolsSettings: {...(s2.devToolsSettings ?? {}), ...(s1.devToolsSettings ?? {})},
     dockingMode: s1.dockingMode ?? s2.dockingMode,
+    panel: s1.panel ?? s2.panel,
   };
 }

test/e2e_non_hosted/host/user-metrics_test.ts

@@ -79,19 +79,22 @@ async function setupInspectorFrontendHostStub(devToolsPage: DevToolsPage) {
   await devToolsPage.reload();
 }
 
-function retrieveRecordedHistogramEvents(page: puppeteer.Page): Promise<EnumHistogramEvent[]> {
+async function retrieveRecordedHistogramEvents(devToolsPage: DevToolsPage): Promise<EnumHistogramEvent[]> {
   // @ts-expect-error
-  return page.evaluate(() => window.InspectorFrontendHost.recordedEnumeratedHistograms);
+  return await devToolsPage.page.evaluate(() => window.InspectorFrontendHost.recordedEnumeratedHistograms);
 }
 
-function retrieveRecordedPerformanceHistogramEvents(page: puppeteer.Page): Promise<PerformanceHistogramEvent[]> {
+async function retrieveRecordedPerformanceHistogramEvents(devToolsPage: DevToolsPage):
+    Promise<PerformanceHistogramEvent[]> {
+  // The reporting happens in a raf
+  await devToolsPage.drainTaskQueue();
   // @ts-expect-error
-  return page.evaluate(() => window.InspectorFrontendHost.recordedPerformanceHistograms);
+  return await devToolsPage.page.evaluate(() => window.InspectorFrontendHost.recordedPerformanceHistograms);
 }
 
 async function assertHistogramEventsInclude(expected: EnumHistogramEvent[], devToolsPage: DevToolsPage) {
   await devToolsPage.waitForFunction(async () => {
-    const events = await retrieveRecordedHistogramEvents(devToolsPage.page);
+    const events = await retrieveRecordedHistogramEvents(devToolsPage);
     try {
       assert.includeDeepMembers(events, expected);
       return true;
@@ -101,10 +104,24 @@ async function assertHistogramEventsInclude(expected: EnumHistogramEvent[], devT
   });
 }
 
+async function assertHistogramPerformanceEventsInclude(expected: string[], devToolsPage: DevToolsPage) {
+  await devToolsPage.waitForFunction(async () => {
+    const events = await retrieveRecordedPerformanceHistogramEvents(devToolsPage);
+    try {
+      for (const expect of expected) {
+        assert.include(events.map(e => e.histogramName), expect);
+      }
+      return true;
+    } catch {
+      return false;
+    }
+  });
+}
+
 async function waitForHistogramEvent(
     expected: EnumHistogramEventWithOptionalCode, expectedCount = 1, devToolsPage: DevToolsPage) {
   await devToolsPage.waitForFunction(async () => {
-    const events = await retrieveRecordedHistogramEvents(devToolsPage.page);
+    const events = await retrieveRecordedHistogramEvents(devToolsPage);
     const matchedEvents = events.filter(
         e => e.actionName === expected.actionName &&
             (!('actionCode' in expected) || e.actionCode === expected.actionCode));
@@ -400,11 +417,7 @@ describe('User Metrics', () => {
 });
 
 describe('User Metric with deferent initial panel', () => {
-  setup({
-    devToolsSettings: {
-      'panel-selected-tab': 'network',
-    },
-  });
+  setup({panel: 'network'});
 
   it('dispatches events for view shown at launch', async ({devToolsPage}) => {
     await setupInspectorFrontendHostStub(devToolsPage);
@@ -424,11 +437,9 @@ describe('User Metric with deferent initial panel', () => {
     // request that the resetPages helper sets the network as the target panel, and
     // we wait for the network in the test. This means, in turn, we get the PanelLoaded
     // event.
-    await devToolsPage.waitFor('.network');
-
-    const events = await retrieveRecordedPerformanceHistogramEvents(devToolsPage.page);
+    await devToolsPage.waitFor('.network .network-toolbar-container');
 
-    assert.include(events.map(e => e.histogramName), 'DevTools.Launch.Network');
+    await assertHistogramPerformanceEventsInclude(['DevTools.Launch.Network'], devToolsPage);
   });
 });
 

test/e2e_non_hosted/shared/frontend-helper.ts

@@ -123,8 +123,15 @@ export class DevToolsPage extends PageWrapper {
    * @param panel Mocks DevTools URL search params to make it open a specific panel on load.
    * @param canDock Mocks DevTools URL search params to make it think whether it can dock or not.
    * This does not control whether or not the panel can actually dock or not.
+   * @param persistReloads If this is true running {@link DevToolsPage.reload} will reloading with
+   * the provided options
    */
-  async reloadWithParams({panel, canDock}: DevToolsReloadParams) {
+  async reloadWithParams({panel, canDock}: DevToolsReloadParams, persistReloads = false) {
+    if (!panel && !canDock) {
+      await this.reload();
+      return;
+    }
+
     // evaluateOnNewDocument is ran before all other JS is loaded
     // ES Modules are only resolved once and always resolved asynchronously
     // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Modules#other_differences_between_modules_and_classic_scripts
@@ -141,7 +148,9 @@ export class DevToolsPage extends PageWrapper {
     }, panel, canDock);
 
     await this.reload();
-    await this.page.removeScriptToEvaluateOnNewDocument(token.identifier);
+    if (!persistReloads) {
+      await this.page.removeScriptToEvaluateOnNewDocument(token.identifier);
+    }
     if (panel) {
       await this.waitFor(`.panel.${panel}`);
     }
@@ -758,8 +767,18 @@ export interface DevtoolsSettings {
   enabledDevToolsExperiments: string[];
   disabledDevToolsExperiments: string[];
   devToolsSettings: Record<string, unknown>;
-  // front_end/ui/legacy/DockController.ts DockState
+  /**
+   * Defined in front_end/ui/legacy/DockController.ts DockState
+   */
   dockingMode: 'bottom'|'right'|'left'|'undocked';
+  // DevTools panel to open on load
+  /**
+   * The name of the panel to be loaded initially
+   * This persist after {@link DevToolsPage.reload}
+   *
+   * To reload into a panel use {@link DevToolsPage.reloadWithParams}
+   */
+  panel: string|undefined;
 }
 
 export const DEFAULT_DEVTOOLS_SETTINGS: DevtoolsSettings = {
@@ -769,6 +788,7 @@ export const DEFAULT_DEVTOOLS_SETTINGS: DevtoolsSettings = {
     veLogsTestMode: true,
   },
   dockingMode: 'right',
+  panel: undefined
 };
 
 /**
@@ -871,7 +891,8 @@ export async function setupDevToolsPage(
     setDevToolsExperiments(devToolsPage, settings.enabledDevToolsExperiments),
     setDisabledDevToolsExperiments(devToolsPage, settings.disabledDevToolsExperiments),
   ]);
-  await devToolsPage.reload();
+
+  await devToolsPage.reloadWithParams({panel: settings.panel}, true);
 
   await Promise.all([
     devToolsPage.throttleCPUIfRequired(),