Add OTel metrics for edit acceptance and code survival telemetry (#4792)

Co-authored-by: Harald Kirschner <digitarald@gmail.com>

Author: digitarald

src/extension/conversation/vscode-node/userActions.ts

@@ -10,6 +10,9 @@ import { EditSurvivalResult } from '../../../platform/editSurvivalTracking/commo
 import { ILanguageDiagnosticsService } from '../../../platform/languages/common/languageDiagnosticsService';
 import { IMultiFileEditInternalTelemetryService } from '../../../platform/multiFileEdit/common/multiFileEditQualityTelemetry';
 import { INotebookService } from '../../../platform/notebook/common/notebookService';
+import { GenAiMetrics } from '../../../platform/otel/common/genAiMetrics';
+import type { EditOutcome } from '../../../platform/otel/common/genAiAttributes';
+import { IOTelService } from '../../../platform/otel/common/otelService';
 import { ISurveyService } from '../../../platform/survey/common/surveyService';
 import { ITelemetryService, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../platform/telemetry/common/telemetry';
 import { isNotebookCellOrNotebookChatInput } from '../../../util/common/notebooks';
@@ -44,7 +47,8 @@ export class UserFeedbackService implements IUserFeedbackService {
 		@ISurveyService private readonly surveyService: ISurveyService,
 		@ILanguageDiagnosticsService private readonly languageDiagnosticsService: ILanguageDiagnosticsService,
 		@IMultiFileEditInternalTelemetryService private readonly multiFileEditTelemetryService: IMultiFileEditInternalTelemetryService,
-		@INotebookService private readonly notebookService: INotebookService
+		@INotebookService private readonly notebookService: INotebookService,
+		@IOTelService private readonly otelService: IOTelService
 	) { }
 
 	handleUserAction(e: vscode.ChatUserActionEvent, agentId: string): void {
@@ -200,6 +204,8 @@ export class UserFeedbackService implements IUserFeedbackService {
 						isNotebookCell: e.action.uri.scheme === Schemas.vscodeNotebookCell ? 1 : 0
 					});
 
+					GenAiMetrics.recordChatEditOutcome(this.otelService, 'chat_editing', outcomes.get(e.action.outcome) ?? 'unknown', document?.languageId, e.action.hasRemainingEdits);
+
 					if (result.metadata?.responseId
 						&& (e.action.outcome === vscode.ChatEditingSessionActionOutcome.Accepted
 							|| e.action.outcome === vscode.ChatEditingSessionActionOutcome.Rejected)
@@ -234,6 +240,7 @@ export class UserFeedbackService implements IUserFeedbackService {
 						measurements,
 						'edit.hunk.action'
 					);
+					GenAiMetrics.recordEditAcceptance(this.otelService, 'chat_editing_hunk', outcome, document?.languageId);
 				}
 				break;
 			}
@@ -438,7 +445,7 @@ export class UserFeedbackService implements IUserFeedbackService {
 		};
 
 		if (kind === InteractiveEditorResponseFeedbackKind.Accepted && response.editSurvivalTracker) {
-			response.editSurvivalTracker.startReporter(res => reportInlineEditSurvivalEvent(res, sharedProps, sharedMeasures));
+			response.editSurvivalTracker.startReporter(res => reportInlineEditSurvivalEvent(res, sharedProps, sharedMeasures, this.otelService));
 		}
 		(response as any).editSurvivalTracker = undefined; // TODO@jrieken
 
@@ -467,6 +474,10 @@ export class UserFeedbackService implements IUserFeedbackService {
 		this.telemetryService.sendMSFTTelemetryEvent('inline.done', sharedProps, {
 			...sharedMeasures, accepted
 		});
+		this.telemetryService.sendGHTelemetryEvent('inline.done', sharedProps, {
+			...sharedMeasures, accepted
+		});
+		GenAiMetrics.recordEditAcceptance(this.otelService, 'inline_chat', accepted ? 'accepted' : 'rejected', languageId);
 
 		this.telemetryService.sendInternalMSFTTelemetryEvent('interactiveSessionDone', {
 			language: languageId,
@@ -501,7 +512,14 @@ export class UserFeedbackService implements IUserFeedbackService {
 	}
 }
 
-function reportInlineEditSurvivalEvent(res: EditSurvivalResult, sharedProps: TelemetryEventProperties | undefined, sharedMeasures: TelemetryEventMeasurements | undefined) {
+function reportInlineEditSurvivalEvent(res: EditSurvivalResult, sharedProps: TelemetryEventProperties | undefined, sharedMeasures: TelemetryEventMeasurements | undefined, otelService: IOTelService) {
+	const survivalMeasures = {
+		...sharedMeasures,
+		survivalRateFourGram: res.fourGram,
+		survivalRateNoRevert: res.noRevert,
+		timeDelayMs: res.timeDelayMs,
+		didBranchChange: res.didBranchChange ? 1 : 0,
+	};
 	/* __GDPR__
 		"inline.trackEditSurvival" : {
 			"owner": "hediet",
@@ -526,16 +544,13 @@ function reportInlineEditSurvivalEvent(res: EditSurvivalResult, sharedProps: Tel
 			"isNotebook": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Whether the document is a notebook" }
 		}
 	*/
-	res.telemetryService.sendMSFTTelemetryEvent('inline.trackEditSurvival', sharedProps, {
-		...sharedMeasures,
-		survivalRateFourGram: res.fourGram,
-		survivalRateNoRevert: res.noRevert,
-		timeDelayMs: res.timeDelayMs,
-		didBranchChange: res.didBranchChange ? 1 : 0,
-	});
+	res.telemetryService.sendMSFTTelemetryEvent('inline.trackEditSurvival', sharedProps, survivalMeasures);
+	res.telemetryService.sendGHTelemetryEvent('inline.trackEditSurvival', sharedProps, survivalMeasures);
+	GenAiMetrics.recordEditSurvivalFourGram(otelService, 'inline_chat', res.fourGram, res.timeDelayMs);
+	GenAiMetrics.recordEditSurvivalNoRevert(otelService, 'inline_chat', res.noRevert, res.timeDelayMs);
 }
 
-const outcomes = new Map([
+const outcomes = new Map<vscode.ChatEditingSessionActionOutcome, EditOutcome>([
 	[vscode.ChatEditingSessionActionOutcome.Accepted, 'accepted'],
 	[vscode.ChatEditingSessionActionOutcome.Rejected, 'rejected'],
 	[vscode.ChatEditingSessionActionOutcome.Saved, 'saved']

src/platform/otel/common/genAiAttributes.ts

@@ -129,8 +129,21 @@ export const CopilotChatAttr = {
 	DEBUG_LOG_LABEL: 'copilot_chat.debug_log_label',
 	/** Markdown content for standalone content events */
 	MARKDOWN_CONTENT: 'copilot_chat.markdown_content',
+	/** Edit source: inline_chat, chat_editing, chat_editing_hunk */
+	EDIT_SOURCE: 'copilot_chat.edit.source',
+	/** Edit outcome: accepted, rejected, saved, unknown */
+	EDIT_OUTCOME: 'copilot_chat.edit.outcome',
+	/** Language identifier of the document */
+	LANGUAGE_ID: 'copilot_chat.language_id',
+	/** Time delay in milliseconds between acceptance and measurement */
+	TIME_DELAY_MS: 'copilot_chat.time_delay_ms',
+	/** Whether additional unactioned edits remain */
+	HAS_REMAINING_EDITS: 'copilot_chat.has_remaining_edits',
 } as const;
 
+export type EditSource = 'inline_chat' | 'chat_editing' | 'chat_editing_hunk';
+export type EditOutcome = 'accepted' | 'rejected' | 'saved' | 'unknown';
+
 /**
  * Standard OTel attributes used alongside GenAI attributes.
  */

src/platform/otel/common/genAiMetrics.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { GenAiAttr, StdAttr } from './genAiAttributes';
+import { CopilotChatAttr, type EditOutcome, type EditSource, GenAiAttr, StdAttr } from './genAiAttributes';
 import type { IOTelService } from './otelService';
 
 /**
@@ -96,4 +96,37 @@ export class GenAiMetrics {
 	static incrementSessionCount(otel: IOTelService): void {
 		otel.incrementCounter('copilot_chat.session.count');
 	}
+
+	// ── Edit Acceptance & Survival Metrics ──
+
+	static recordEditAcceptance(otel: IOTelService, source: EditSource, outcome: EditOutcome, languageId?: string): void {
+		otel.incrementCounter('copilot_chat.edit.acceptance.count', 1, {
+			[CopilotChatAttr.EDIT_SOURCE]: source,
+			[CopilotChatAttr.EDIT_OUTCOME]: outcome,
+			...(languageId ? { [CopilotChatAttr.LANGUAGE_ID]: languageId } : {}),
+		});
+	}
+
+	static recordEditSurvivalFourGram(otel: IOTelService, source: EditSource, score: number, timeDelayMs: number): void {
+		otel.recordMetric('copilot_chat.edit.survival.four_gram', score, {
+			[CopilotChatAttr.EDIT_SOURCE]: source,
+			[CopilotChatAttr.TIME_DELAY_MS]: timeDelayMs,
+		});
+	}
+
+	static recordEditSurvivalNoRevert(otel: IOTelService, source: EditSource, score: number, timeDelayMs: number): void {
+		otel.recordMetric('copilot_chat.edit.survival.no_revert', score, {
+			[CopilotChatAttr.EDIT_SOURCE]: source,
+			[CopilotChatAttr.TIME_DELAY_MS]: timeDelayMs,
+		});
+	}
+
+	static recordChatEditOutcome(otel: IOTelService, source: EditSource, outcome: EditOutcome, languageId?: string, hasRemainingEdits?: boolean): void {
+		otel.incrementCounter('copilot_chat.chat_edit.outcome.count', 1, {
+			[CopilotChatAttr.EDIT_SOURCE]: source,
+			[CopilotChatAttr.EDIT_OUTCOME]: outcome,
+			...(languageId ? { [CopilotChatAttr.LANGUAGE_ID]: languageId } : {}),
+			...(hasRemainingEdits !== undefined ? { [CopilotChatAttr.HAS_REMAINING_EDITS]: hasRemainingEdits } : {}),
+		});
+	}
 }

src/platform/otel/common/test/agentTraceHierarchy.spec.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { describe, expect, it } from 'vitest';
-import { GenAiAttr, GenAiOperationName, GenAiProviderName } from '../genAiAttributes';
+import { CopilotChatAttr, GenAiAttr, GenAiOperationName, GenAiProviderName } from '../genAiAttributes';
 import { emitAgentTurnEvent, emitSessionStartEvent } from '../genAiEvents';
 import { GenAiMetrics } from '../genAiMetrics';
 import { SpanKind, SpanStatusCode } from '../otelService';
@@ -223,4 +223,51 @@ describe('Agent Trace Hierarchy', () => {
 		expect(otel.metrics[2].name).toBe('gen_ai.client.token.usage');
 		expect(otel.metrics[2].value).toBe(250);
 	});
+
+	it('records edit acceptance and survival metrics', () => {
+		const otel = new CapturingOTelService();
+
+		GenAiMetrics.recordEditAcceptance(otel, 'inline_chat', 'accepted', 'typescript');
+		GenAiMetrics.recordEditAcceptance(otel, 'chat_editing_hunk', 'rejected', 'python');
+		GenAiMetrics.recordEditSurvivalFourGram(otel, 'inline_chat', 0.85, 30000);
+		GenAiMetrics.recordEditSurvivalNoRevert(otel, 'inline_chat', 0.92, 30000);
+		GenAiMetrics.recordChatEditOutcome(otel, 'chat_editing', 'accepted', 'typescript', false);
+
+		// Acceptance counters
+		expect(otel.counters).toHaveLength(3);
+		expect(otel.counters[0].name).toBe('copilot_chat.edit.acceptance.count');
+		expect(otel.counters[0].attributes?.[CopilotChatAttr.EDIT_SOURCE]).toBe('inline_chat');
+		expect(otel.counters[0].attributes?.[CopilotChatAttr.EDIT_OUTCOME]).toBe('accepted');
+		expect(otel.counters[0].attributes?.[CopilotChatAttr.LANGUAGE_ID]).toBe('typescript');
+
+		expect(otel.counters[1].name).toBe('copilot_chat.edit.acceptance.count');
+		expect(otel.counters[1].attributes?.[CopilotChatAttr.EDIT_OUTCOME]).toBe('rejected');
+
+		// Chat edit outcome counter
+		expect(otel.counters[2].name).toBe('copilot_chat.chat_edit.outcome.count');
+		expect(otel.counters[2].attributes?.[CopilotChatAttr.EDIT_SOURCE]).toBe('chat_editing');
+		expect(otel.counters[2].attributes?.[CopilotChatAttr.EDIT_OUTCOME]).toBe('accepted');
+		expect(otel.counters[2].attributes?.[CopilotChatAttr.HAS_REMAINING_EDITS]).toBe(false);
+
+		// Survival histograms
+		expect(otel.metrics).toHaveLength(2);
+		expect(otel.metrics[0].name).toBe('copilot_chat.edit.survival.four_gram');
+		expect(otel.metrics[0].value).toBe(0.85);
+		expect(otel.metrics[0].attributes?.[CopilotChatAttr.EDIT_SOURCE]).toBe('inline_chat');
+		expect(otel.metrics[0].attributes?.[CopilotChatAttr.TIME_DELAY_MS]).toBe(30000);
+
+		expect(otel.metrics[1].name).toBe('copilot_chat.edit.survival.no_revert');
+		expect(otel.metrics[1].value).toBe(0.92);
+	});
+
+	it('omits optional attributes when undefined', () => {
+		const otel = new CapturingOTelService();
+
+		GenAiMetrics.recordEditAcceptance(otel, 'inline_chat', 'accepted', undefined);
+		GenAiMetrics.recordChatEditOutcome(otel, 'chat_editing', 'rejected', undefined, undefined);
+
+		expect(otel.counters[0].attributes?.[CopilotChatAttr.LANGUAGE_ID]).toBeUndefined();
+		expect(otel.counters[1].attributes?.[CopilotChatAttr.LANGUAGE_ID]).toBeUndefined();
+		expect(otel.counters[1].attributes?.[CopilotChatAttr.HAS_REMAINING_EDITS]).toBeUndefined();
+	});
 });