豆豆友情提示：这是一个非官方 GitHub 代理镜像，主要用于网络测试或访问加速。请勿在此进行登录、注册或处理任何敏感信息。进行这些操作请务必访问官方网站 github.com。 Raw 内容也通过此代理提供。
Skip to content

Commit 9187ca7

Browse files
matthew29tangcopybara-github
matthew29tang
authored and
copybara-github
committed
feat: Add Image Grounding support to GoogleSearch tool
PiperOrigin-RevId: 875338036
1 parent 2247f18 commit 9187ca7

File tree

9 files changed

+348
-131
lines changed

9 files changed

+348
-131
lines changed

‎api-report/genai-node.api.md‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1705,6 +1705,7 @@ export interface GoogleRpcStatus {
17051705
export interface GoogleSearch {
17061706
blockingConfidence?: PhishBlockThreshold;
17071707
excludeDomains?: string[];
1708+
searchTypes?: SearchTypes;
17081709
timeRangeFilter?: Interval;
17091710
}
17101711

@@ -1722,11 +1723,20 @@ export interface GoogleTypeDate {
17221723

17231724
// @public
17241725
export interface GroundingChunk {
1726+
image?: GroundingChunkImage;
17251727
maps?: GroundingChunkMaps;
17261728
retrievedContext?: GroundingChunkRetrievedContext;
17271729
web?: GroundingChunkWeb;
17281730
}
17291731

1732+
// @public
1733+
export interface GroundingChunkImage {
1734+
domain?: string;
1735+
imageUri?: string;
1736+
sourceUri?: string;
1737+
title?: string;
1738+
}
1739+
17301740
// @public
17311741
export interface GroundingChunkMaps {
17321742
placeAnswerSources?: GroundingChunkMapsPlaceAnswerSources;
@@ -1776,11 +1786,12 @@ export interface GroundingChunkWeb {
17761786
uri?: string;
17771787
}
17781788

1779-
// @public
1789+
// @public (undocumented)
17801790
export interface GroundingMetadata {
17811791
googleMapsWidgetContextToken?: string;
17821792
groundingChunks?: GroundingChunk[];
17831793
groundingSupports?: GroundingSupport[];
1794+
imageSearchQueries?: string[];
17841795
retrievalMetadata?: RetrievalMetadata;
17851796
retrievalQueries?: string[];
17861797
searchEntryPoint?: SearchEntryPoint;
@@ -1902,6 +1913,7 @@ export interface ImageConfig {
19021913
outputCompressionQuality?: number;
19031914
outputMimeType?: string;
19041915
personGeneration?: string;
1916+
prominentPeople?: ProminentPeople;
19051917
}
19061918

19071919
// @public
@@ -1916,6 +1928,10 @@ export enum ImagePromptLanguage {
19161928
zh = "zh"
19171929
}
19181930

1931+
// @public
1932+
export interface ImageSearch {
1933+
}
1934+
19191935
// @public
19201936
export interface ImportFileConfig {
19211937
abortSignal?: AbortSignal;
@@ -2848,6 +2864,13 @@ export interface ProductImage {
28482864
productImage?: Image_2;
28492865
}
28502866

2867+
// @public
2868+
export enum ProminentPeople {
2869+
ALLOW_PROMINENT_PEOPLE = "ALLOW_PROMINENT_PEOPLE",
2870+
BLOCK_PROMINENT_PEOPLE = "BLOCK_PROMINENT_PEOPLE",
2871+
PROMINENT_PEOPLE_UNSPECIFIED = "PROMINENT_PEOPLE_UNSPECIFIED"
2872+
}
2873+
28512874
// @public
28522875
export interface RagChunk {
28532876
pageSpan?: RagChunkPageSpan;
@@ -3133,6 +3156,12 @@ export interface SearchEntryPoint {
31333156
sdkBlob?: string;
31343157
}
31353158

3159+
// @public
3160+
export interface SearchTypes {
3161+
imageSearch?: ImageSearch;
3162+
webSearch?: WebSearch;
3163+
}
3164+
31363165
// @public
31373166
export interface Segment {
31383167
endIndex?: number;
@@ -3874,6 +3903,10 @@ export interface VoiceConfig {
38743903
replicatedVoiceConfig?: ReplicatedVoiceConfig;
38753904
}
38763905

3906+
// @public
3907+
export interface WebSearch {
3908+
}
3909+
38773910
// @public
38783911
export interface WeightedPrompt {
38793912
text?: string;

‎api-report/genai-web.api.md‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1705,6 +1705,7 @@ export interface GoogleRpcStatus {
17051705
export interface GoogleSearch {
17061706
blockingConfidence?: PhishBlockThreshold;
17071707
excludeDomains?: string[];
1708+
searchTypes?: SearchTypes;
17081709
timeRangeFilter?: Interval;
17091710
}
17101711

@@ -1722,11 +1723,20 @@ export interface GoogleTypeDate {
17221723

17231724
// @public
17241725
export interface GroundingChunk {
1726+
image?: GroundingChunkImage;
17251727
maps?: GroundingChunkMaps;
17261728
retrievedContext?: GroundingChunkRetrievedContext;
17271729
web?: GroundingChunkWeb;
17281730
}
17291731

1732+
// @public
1733+
export interface GroundingChunkImage {
1734+
domain?: string;
1735+
imageUri?: string;
1736+
sourceUri?: string;
1737+
title?: string;
1738+
}
1739+
17301740
// @public
17311741
export interface GroundingChunkMaps {
17321742
placeAnswerSources?: GroundingChunkMapsPlaceAnswerSources;
@@ -1776,11 +1786,12 @@ export interface GroundingChunkWeb {
17761786
uri?: string;
17771787
}
17781788

1779-
// @public
1789+
// @public (undocumented)
17801790
export interface GroundingMetadata {
17811791
googleMapsWidgetContextToken?: string;
17821792
groundingChunks?: GroundingChunk[];
17831793
groundingSupports?: GroundingSupport[];
1794+
imageSearchQueries?: string[];
17841795
retrievalMetadata?: RetrievalMetadata;
17851796
retrievalQueries?: string[];
17861797
searchEntryPoint?: SearchEntryPoint;
@@ -1902,6 +1913,7 @@ export interface ImageConfig {
19021913
outputCompressionQuality?: number;
19031914
outputMimeType?: string;
19041915
personGeneration?: string;
1916+
prominentPeople?: ProminentPeople;
19051917
}
19061918

19071919
// @public
@@ -1916,6 +1928,10 @@ export enum ImagePromptLanguage {
19161928
zh = "zh"
19171929
}
19181930

1931+
// @public
1932+
export interface ImageSearch {
1933+
}
1934+
19191935
// @public
19201936
export interface ImportFileConfig {
19211937
abortSignal?: AbortSignal;
@@ -2848,6 +2864,13 @@ export interface ProductImage {
28482864
productImage?: Image_2;
28492865
}
28502866

2867+
// @public
2868+
export enum ProminentPeople {
2869+
ALLOW_PROMINENT_PEOPLE = "ALLOW_PROMINENT_PEOPLE",
2870+
BLOCK_PROMINENT_PEOPLE = "BLOCK_PROMINENT_PEOPLE",
2871+
PROMINENT_PEOPLE_UNSPECIFIED = "PROMINENT_PEOPLE_UNSPECIFIED"
2872+
}
2873+
28512874
// @public
28522875
export interface RagChunk {
28532876
pageSpan?: RagChunkPageSpan;
@@ -3133,6 +3156,12 @@ export interface SearchEntryPoint {
31333156
sdkBlob?: string;
31343157
}
31353158

3159+
// @public
3160+
export interface SearchTypes {
3161+
imageSearch?: ImageSearch;
3162+
webSearch?: WebSearch;
3163+
}
3164+
31363165
// @public
31373166
export interface Segment {
31383167
endIndex?: number;
@@ -3874,6 +3903,10 @@ export interface VoiceConfig {
38743903
replicatedVoiceConfig?: ReplicatedVoiceConfig;
38753904
}
38763905

3906+
// @public
3907+
export interface WebSearch {
3908+
}
3909+
38773910
// @public
38783911
export interface WeightedPrompt {
38793912
text?: string;

‎api-report/genai.api.md‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1705,6 +1705,7 @@ export interface GoogleRpcStatus {
17051705
export interface GoogleSearch {
17061706
blockingConfidence?: PhishBlockThreshold;
17071707
excludeDomains?: string[];
1708+
searchTypes?: SearchTypes;
17081709
timeRangeFilter?: Interval;
17091710
}
17101711

@@ -1722,11 +1723,20 @@ export interface GoogleTypeDate {
17221723

17231724
// @public
17241725
export interface GroundingChunk {
1726+
image?: GroundingChunkImage;
17251727
maps?: GroundingChunkMaps;
17261728
retrievedContext?: GroundingChunkRetrievedContext;
17271729
web?: GroundingChunkWeb;
17281730
}
17291731

1732+
// @public
1733+
export interface GroundingChunkImage {
1734+
domain?: string;
1735+
imageUri?: string;
1736+
sourceUri?: string;
1737+
title?: string;
1738+
}
1739+
17301740
// @public
17311741
export interface GroundingChunkMaps {
17321742
placeAnswerSources?: GroundingChunkMapsPlaceAnswerSources;
@@ -1776,11 +1786,12 @@ export interface GroundingChunkWeb {
17761786
uri?: string;
17771787
}
17781788

1779-
// @public
1789+
// @public (undocumented)
17801790
export interface GroundingMetadata {
17811791
googleMapsWidgetContextToken?: string;
17821792
groundingChunks?: GroundingChunk[];
17831793
groundingSupports?: GroundingSupport[];
1794+
imageSearchQueries?: string[];
17841795
retrievalMetadata?: RetrievalMetadata;
17851796
retrievalQueries?: string[];
17861797
searchEntryPoint?: SearchEntryPoint;
@@ -1902,6 +1913,7 @@ export interface ImageConfig {
19021913
outputCompressionQuality?: number;
19031914
outputMimeType?: string;
19041915
personGeneration?: string;
1916+
prominentPeople?: ProminentPeople;
19051917
}
19061918

19071919
// @public
@@ -1916,6 +1928,10 @@ export enum ImagePromptLanguage {
19161928
zh = "zh"
19171929
}
19181930

1931+
// @public
1932+
export interface ImageSearch {
1933+
}
1934+
19191935
// @public
19201936
export interface ImportFileConfig {
19211937
abortSignal?: AbortSignal;
@@ -2848,6 +2864,13 @@ export interface ProductImage {
28482864
productImage?: Image_2;
28492865
}
28502866

2867+
// @public
2868+
export enum ProminentPeople {
2869+
ALLOW_PROMINENT_PEOPLE = "ALLOW_PROMINENT_PEOPLE",
2870+
BLOCK_PROMINENT_PEOPLE = "BLOCK_PROMINENT_PEOPLE",
2871+
PROMINENT_PEOPLE_UNSPECIFIED = "PROMINENT_PEOPLE_UNSPECIFIED"
2872+
}
2873+
28512874
// @public
28522875
export interface RagChunk {
28532876
pageSpan?: RagChunkPageSpan;
@@ -3133,6 +3156,12 @@ export interface SearchEntryPoint {
31333156
sdkBlob?: string;
31343157
}
31353158

3159+
// @public
3160+
export interface SearchTypes {
3161+
imageSearch?: ImageSearch;
3162+
webSearch?: WebSearch;
3163+
}
3164+
31363165
// @public
31373166
export interface Segment {
31383167
endIndex?: number;
@@ -3874,6 +3903,10 @@ export interface VoiceConfig {
38743903
replicatedVoiceConfig?: ReplicatedVoiceConfig;
38753904
}
38763905

3906+
// @public
3907+
export interface WebSearch {
3908+
}
3909+
38773910
// @public
38783911
export interface WeightedPrompt {
38793912
text?: string;

‎src/converters/_batches_converters.ts‎

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -460,11 +460,6 @@ export function candidateFromMldev(
460460
common.setValueByPath(toObject, ['finishReason'], fromFinishReason);
461461
}
462462

463-
const fromAvgLogprobs = common.getValueByPath(fromObject, ['avgLogprobs']);
464-
if (fromAvgLogprobs != null) {
465-
common.setValueByPath(toObject, ['avgLogprobs'], fromAvgLogprobs);
466-
}
467-
468463
const fromGroundingMetadata = common.getValueByPath(fromObject, [
469464
'groundingMetadata',
470465
]);
@@ -476,6 +471,11 @@ export function candidateFromMldev(
476471
);
477472
}
478473

474+
const fromAvgLogprobs = common.getValueByPath(fromObject, ['avgLogprobs']);
475+
if (fromAvgLogprobs != null) {
476+
common.setValueByPath(toObject, ['avgLogprobs'], fromAvgLogprobs);
477+
}
478+
479479
const fromIndex = common.getValueByPath(fromObject, ['index']);
480480
if (fromIndex != null) {
481481
common.setValueByPath(toObject, ['index'], fromIndex);
@@ -1345,6 +1345,11 @@ export function googleSearchToMldev(
13451345
): Record<string, unknown> {
13461346
const toObject: Record<string, unknown> = {};
13471347

1348+
const fromSearchTypes = common.getValueByPath(fromObject, ['searchTypes']);
1349+
if (fromSearchTypes != null) {
1350+
common.setValueByPath(toObject, ['searchTypes'], fromSearchTypes);
1351+
}
1352+
13481353
if (common.getValueByPath(fromObject, ['excludeDomains']) !== undefined) {
13491354
throw new Error('excludeDomains parameter is not supported in Gemini API.');
13501355
}
@@ -1386,6 +1391,12 @@ export function imageConfigToMldev(
13861391
);
13871392
}
13881393

1394+
if (common.getValueByPath(fromObject, ['prominentPeople']) !== undefined) {
1395+
throw new Error(
1396+
'prominentPeople parameter is not supported in Gemini API.',
1397+
);
1398+
}
1399+
13891400
if (common.getValueByPath(fromObject, ['outputMimeType']) !== undefined) {
13901401
throw new Error('outputMimeType parameter is not supported in Gemini API.');
13911402
}
@@ -1776,6 +1787,15 @@ export function toolToMldev(fromObject: types.Tool): Record<string, unknown> {
17761787
common.setValueByPath(toObject, ['fileSearch'], fromFileSearch);
17771788
}
17781789

1790+
const fromGoogleSearch = common.getValueByPath(fromObject, ['googleSearch']);
1791+
if (fromGoogleSearch != null) {
1792+
common.setValueByPath(
1793+
toObject,
1794+
['googleSearch'],
1795+
googleSearchToMldev(fromGoogleSearch),
1796+
);
1797+
}
1798+
17791799
const fromCodeExecution = common.getValueByPath(fromObject, [
17801800
'codeExecution',
17811801
]);
@@ -1813,15 +1833,6 @@ export function toolToMldev(fromObject: types.Tool): Record<string, unknown> {
18131833
);
18141834
}
18151835

1816-
const fromGoogleSearch = common.getValueByPath(fromObject, ['googleSearch']);
1817-
if (fromGoogleSearch != null) {
1818-
common.setValueByPath(
1819-
toObject,
1820-
['googleSearch'],
1821-
googleSearchToMldev(fromGoogleSearch),
1822-
);
1823-
}
1824-
18251836
const fromGoogleSearchRetrieval = common.getValueByPath(fromObject, [
18261837
'googleSearchRetrieval',
18271838
]);

0 commit comments

Comments
 (0)