From 8ccf402388360077c3e572bfe3718b7655bdc6b3 Mon Sep 17 00:00:00 2001 From: jinpeng2700-tech <263869970+jinpeng2700-tech@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:31:06 +0800 Subject: [PATCH 1/2] fix: accept MP4 interaction video with omitted MIME enum --- docs/protocol.md | 2 + internal/aistudio/interaction.go | 26 +++++-- internal/aistudio/interaction_media_test.go | 86 +++++++++++++++++++++ 3 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 internal/aistudio/interaction_media_test.go diff --git a/docs/protocol.md b/docs/protocol.md index 4ffe5684..8a672843 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1064,6 +1064,8 @@ thinking level 取值 MINIMAL=1、LOW=2、MEDIUM=3、HIGH=4，由 `reasoning_eff 响应为 `[[...], ?]`。事件索引 10 的 content delta 中 field 1 为正文、field 5 为视频 `[1,""]`、field 6 为思考摘要，分别映射为 text、`video/mp4` 媒体与 reasoning 事件；索引 19 的最终 interaction 状态 `3` 产生 usage（输入、输出、思考、总 token 位于 usage 索引 0、4、8、9）与 `stop` 终态，状态 `4`、`5` 返回错误。事件列表后的 google.rpc 状态按 code 映射 HTTP 状态，额度 code 8 为 429；首个正文事件之前的错误进入换号与冷却，之后的错误以流内 error 结束。 +视频 field 5 也可能返回 `[null,""]`，省略 MIME 枚举；此时解码 Base64 后检查文件类型，仅接受可识别为 `video/mp4` 的数据。未知的非空枚举仍返回协议错误，图片 MIME 枚举的处理不变。 + ### 后台 Interaction 任务 模型目录 field 79 为后台任务的模型（`deep-research-preview-04-2026`、`deep-research-max-preview-04-2026`）经两个 RPC 生成：`CreateInteraction` 创建任务，`GetInteractionStream` 按 interaction ID 读取任务的增量与终态。两个 RPC 的 WAA proof 都位于 field 5，`CreateInteraction` 的 binding 为发送的全部文本以空格连接，`GetInteractionStream` 的 binding 为 interaction ID： diff --git a/internal/aistudio/interaction.go b/internal/aistudio/interaction.go index 46c51494..6018897a 100644 --- a/internal/aistudio/interaction.go +++ b/internal/aistudio/interaction.go @@ -745,13 +745,19 @@ func decodeInteractionMedia(method string, value json.RawMessage, index int, mim if err != nil || len(media) < 2 { return nil, &ProtocolEvidenceError{Method: method, Path: path, Detail: "媒体 delta 无效", Raw: cloneRaw(raw)} } - code, err := rawInt64(media[0], path+"[0]", raw) - if err != nil { - return nil, withMethod(err, method) - } - mimeType, ok := mimes[code] - if !ok { - return nil, &ProtocolEvidenceError{Method: method, Path: path + "[0]", Detail: fmt.Sprintf("未识别的媒体 MIME 枚举 %d", code), Raw: cloneRaw(raw)} + // 视频响应可能省略 MIME 枚举；仅在数据可识别为 MP4 时接受。 + missingVideoMIME := index == 4 && isJSONNull(media[0]) + var mimeType string + if !missingVideoMIME { + code, err := rawInt64(media[0], path+"[0]", raw) + if err != nil { + return nil, withMethod(err, method) + } + var ok bool + mimeType, ok = mimes[code] + if !ok { + return nil, &ProtocolEvidenceError{Method: method, Path: path + "[0]", Detail: fmt.Sprintf("未识别的媒体 MIME 枚举 %d", code), Raw: cloneRaw(raw)} + } } encoded, err := rawString(media[1], path+"[1]", raw) if err != nil { @@ -761,5 +767,11 @@ func decodeInteractionMedia(method string, value json.RawMessage, index int, mim if err != nil { return nil, &ProtocolEvidenceError{Method: method, Path: path + "[1]", Detail: "媒体数据不是 Base64", Raw: cloneRaw(raw)} } + if missingVideoMIME { + mimeType = http.DetectContentType(data) + if mimeType != "video/mp4" { + return nil, &ProtocolEvidenceError{Method: method, Path: path, Detail: "视频 MIME 枚举缺失且数据不是 MP4", Raw: cloneRaw(raw)} + } + } return &Blob{MIME: mimeType, Data: data}, nil } diff --git a/internal/aistudio/interaction_media_test.go b/internal/aistudio/interaction_media_test.go new file mode 100644 index 00000000..55a0a1ee --- /dev/null +++ b/internal/aistudio/interaction_media_test.go @@ -0,0 +1,86 @@ +package aistudio + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "strings" + "testing" +) + +func TestDecodeInteractionMedia(t *testing.T) { + // Minimal ftyp box with an MP4 major brand. + mp4 := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") + encoded := base64.StdEncoding.EncodeToString(mp4) + for _, tc := range []struct { + name string + wire string + index int + wantMIME string + wantErr bool + }{ + {"omitted video enum", `[null,"` + encoded + `"]`, 4, "video/mp4", false}, + {"numeric video enum", `[1,"` + encoded + `"]`, 4, "video/mp4", false}, + {"numeric image enum", `[1,"aW1hZ2U="]`, 1, "image/png", false}, + {"missing image enum", `[null,"` + encoded + `"]`, 1, "", true}, + {"unknown video enum", `[9,"` + encoded + `"]`, 4, "", true}, + {"string MIME", `["video/mp4","` + encoded + `"]`, 4, "", true}, + {"bad base64", `[null,"***"]`, 4, "", true}, + {"wrong content", `[null,"aW1hZ2U="]`, 4, "", true}, + {"empty content", `[null,""]`, 4, "", true}, + {"short media", `[null]`, 4, "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + mimes := interactionVideoMIME + if tc.index == 1 { + mimes = interactionImageMIME + } + blob, err := decodeInteractionMedia("CreateInteractionStream", json.RawMessage(tc.wire), tc.index, mimes, json.RawMessage(tc.wire)) + if tc.wantErr { + var evidence *ProtocolEvidenceError + if !errors.As(err, &evidence) { + t.Fatalf("expected protocol error, got %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if blob.MIME != tc.wantMIME { + t.Fatalf("MIME = %q", blob.MIME) + } + if tc.index == 4 && !bytes.Equal(blob.Data, mp4) { + t.Fatal("video bytes changed") + } + }) + } +} + +func TestDecodeInteractionStreamVideoWithoutMIME(t *testing.T) { + mp4 := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") + content := make([]any, 5) + content[4] = []any{nil, base64.StdEncoding.EncodeToString(mp4)} + delta := make([]any, 11) + delta[10] = []any{0, content} + final := make([]any, 20) + final[19] = []any{[]any{nil, 3}} + wire, err := json.Marshal([]any{[]any{delta, final}, []any{0}}) + if err != nil { + t.Fatal(err) + } + var events []Event + err = DecodeInteractionStream("CreateInteractionStream", strings.NewReader(string(wire)), func(event Event) error { + events = append(events, event) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(events) != 2 || events[0].Kind != EventMedia || events[1].Kind != EventFinish { + t.Fatalf("unexpected events: %#v", events) + } + if events[0].InlineData.MIME != "video/mp4" || !bytes.Equal(events[0].InlineData.Data, mp4) { + t.Fatal("video corrupted") + } +} From 45e33b43cdf7b7c57fc13090dc109bc4a15b022b Mon Sep 17 00:00:00 2001 From: Mag1cFall Date: Sun, 1 Nov 2026 10:26:01 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=E6=8C=89=E5=AE=98=E7=BD=91=E6=9E=9A?= =?UTF-8?q?=E4=B8=BE=E8=A1=A8=E4=B8=8E=E7=BC=BA=E7=9C=81=E5=80=BC=E8=A7=A3?= =?UTF-8?q?=E7=A0=81=20Interaction=20=E5=9B=BE=E7=89=87=E3=80=81=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E3=80=81=E6=96=87=E6=A1=A3=E4=B8=8E=E8=A7=86=E9=A2=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/protocol.md | 11 +- internal/aistudio/interaction.go | 55 +++++----- internal/aistudio/interaction_media_test.go | 107 +++++++------------- 3 files changed, 71 insertions(+), 102 deletions(-) diff --git a/docs/protocol.md b/docs/protocol.md index 8a672843..c4255064 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1062,9 +1062,16 @@ thinking level 取值 MINIMAL=1、LOW=2、MEDIUM=3、HIGH=4，由 `reasoning_eff 视频设置来自 Interactions `response_format` 的 `{"type":"video","aspect_ratio","duration","resolution"}` 与 `generation_config.video_config.task`，或 Gemini API `generationConfig.responseFormat` 的 `video.aspectRatio`、`video.duration`、`video.resolution` 与扩展字段 `generationConfig.videoConfig.task`；Interactions `response_modalities` 的 `video` 与 Gemini API `generationConfig.responseModalities` 的 `VIDEO` 等同于不带设置的视频输出格式；`duration` 取 `"4s"` 或秒数并四舍五入为整秒，宽高比与分辨率换成最接近的上游取值，无法解析的取值与未知任务返回 400。 -响应为 `[[...], ?]`。事件索引 10 的 content delta 中 field 1 为正文、field 5 为视频 `[1,""]`、field 6 为思考摘要，分别映射为 text、`video/mp4` 媒体与 reasoning 事件；索引 19 的最终 interaction 状态 `3` 产生 usage（输入、输出、思考、总 token 位于 usage 索引 0、4、8、9）与 `stop` 终态，状态 `4`、`5` 返回错误。事件列表后的 google.rpc 状态按 code 映射 HTTP 状态，额度 code 8 为 429；首个正文事件之前的错误进入换号与冷却，之后的错误以流内 error 结束。 +响应为 `[[...], ?]`。事件索引 10 的 content delta 中 field 1 为正文、field 2 至 5 为图片、音频、文档与视频 `[,""]`、field 6 为思考摘要，分别映射为 text、媒体与 reasoning 事件；索引 19 的最终 interaction 状态 `3` 产生 usage（输入、输出、思考、总 token 位于 usage 索引 0、4、8、9）与 `stop` 终态，状态 `4`、`5` 返回错误。事件列表后的 google.rpc 状态按 code 映射 HTTP 状态，额度 code 8 为 429；首个正文事件之前的错误进入换号与冷却，之后的错误以流内 error 结束。 -视频 field 5 也可能返回 `[null,""]`，省略 MIME 枚举；此时解码 Base64 后检查文件类型，仅接受可识别为 `video/mp4` 的数据。未知的非空枚举仍返回协议错误，图片 MIME 枚举的处理不变。 +媒体 MIME 枚举与官网前端相同，枚举为 `null` 或不在表中时取该类的默认 MIME： + +| 字段 | 枚举 | 默认 | +| --- | --- | --- | +| 图片 field 2 | 1 png、2 jpeg、3 webp、4 heic、5 heif、7 gif、8 bmp、9 tiff | `image/jpeg` | +| 音频 field 3 | 1 wav、2 mp3、3 aiff、4 aac、5 ogg、6 flac、7 mpeg、8 m4a、9 l16、10 s16le、11 opus、12 alaw、13 mulaw | `audio/wav` | +| 文档 field 4 | 1 pdf | `application/pdf` | +| 视频 field 5 | 1 mp4、2 mpeg、3 mpg、4 mov、5 avi、6 x-flv、7 webm、8 wmv、9 3gpp | `video/mp4` | ### 后台 Interaction 任务 diff --git a/internal/aistudio/interaction.go b/internal/aistudio/interaction.go index 6018897a..4333b460 100644 --- a/internal/aistudio/interaction.go +++ b/internal/aistudio/interaction.go @@ -21,11 +21,19 @@ var interactionThinkingLevels = map[int64]int64{4: 1, 1: 2, 2: 3, 3: 4} // interactionCancelTimeout 是取消后台任务的期限，包含等待账户请求槽位的时间 const interactionCancelTimeout = time.Minute -// interactionVideoMIME 表示 Interaction 视频内容的 MIME 枚举 -var interactionVideoMIME = map[int64]string{1: "video/mp4"} +// interactionMediaMIME 是 content delta 各媒体字段的 MIME 枚举表，与官网前端相同 +var interactionMediaMIME = map[int]interactionMIME{ + 1: {codes: map[int64]string{1: "image/png", 2: "image/jpeg", 3: "image/webp", 4: "image/heic", 5: "image/heif", 7: "image/gif", 8: "image/bmp", 9: "image/tiff"}, unlisted: "image/jpeg"}, + 2: {codes: map[int64]string{1: "audio/wav", 2: "audio/mp3", 3: "audio/aiff", 4: "audio/aac", 5: "audio/ogg", 6: "audio/flac", 7: "audio/mpeg", 8: "audio/m4a", 9: "audio/l16", 10: "audio/s16le", 11: "audio/opus", 12: "audio/alaw", 13: "audio/mulaw"}, unlisted: "audio/wav"}, + 3: {codes: map[int64]string{1: "application/pdf"}, unlisted: "application/pdf"}, + 4: {codes: map[int64]string{1: "video/mp4", 2: "video/mpeg", 3: "video/mpg", 4: "video/mov", 5: "video/avi", 6: "video/x-flv", 7: "video/webm", 8: "video/wmv", 9: "video/3gpp"}, unlisted: "video/mp4"}, +} -// interactionImageMIME 表示 Interaction 图片内容的 MIME 枚举 -var interactionImageMIME = map[int64]string{1: "image/png"} +// interactionMIME 是一类媒体的 MIME 枚举与枚举缺省或不在表中时的 MIME +type interactionMIME struct { + codes map[int64]string + unlisted string +} // interactionStatusHTTP 把流尾 google.rpc 状态码换算为 HTTP 状态 var interactionStatusHTTP = map[int64]int{ @@ -485,7 +493,7 @@ func decodeInteractionEvent(method string, raw json.RawMessage, emit func(Event) return true, emit(Event{Kind: EventFinish, FinishReason: "stop"}) } -// decodeInteractionDelta 解码 content delta：field 1 文本、field 2 图片、field 5 视频、field 6 思考摘要、field 7 思考签名、field 10 引用 +// decodeInteractionDelta 解码 content delta：field 1 文本、field 2 图片、field 3 音频、field 4 文档、field 5 视频、field 6 思考摘要、field 7 思考签名、field 10 引用 func decodeInteractionDelta(method string, raw json.RawMessage, emit func(Event) error) error { delta, err := rawArray(raw, "$delta", raw) if err != nil || len(delta) < 2 { @@ -512,12 +520,8 @@ func decodeInteractionDelta(method string, raw json.RawMessage, emit func(Event) if err := emit(Event{Kind: EventText, Text: decoded}); err != nil { return err } - case 1, 4: - mimes := interactionImageMIME - if index == 4 { - mimes = interactionVideoMIME - } - media, err := decodeInteractionMedia(method, value, index, mimes, raw) + case 1, 2, 3, 4: + media, err := decodeInteractionMedia(method, value, index, raw) if err != nil { return err } @@ -738,26 +742,23 @@ func decodeInteractionCitations(method string, value json.RawMessage, raw json.R return nil } -// decodeInteractionMedia 解码图片或视频 content：MIME 枚举与 Base64 数据 -func decodeInteractionMedia(method string, value json.RawMessage, index int, mimes map[int64]string, raw json.RawMessage) (*Blob, error) { +// decodeInteractionMedia 解码媒体 content：MIME 枚举与 Base64 数据，枚举缺省或不在表中时按官网取该类媒体的默认 MIME +func decodeInteractionMedia(method string, value json.RawMessage, index int, raw json.RawMessage) (*Blob, error) { path := fmt.Sprintf("$delta[1][%d]", index) media, err := rawArray(value, path, raw) if err != nil || len(media) < 2 { return nil, &ProtocolEvidenceError{Method: method, Path: path, Detail: "媒体 delta 无效", Raw: cloneRaw(raw)} } - // 视频响应可能省略 MIME 枚举；仅在数据可识别为 MP4 时接受。 - missingVideoMIME := index == 4 && isJSONNull(media[0]) - var mimeType string - if !missingVideoMIME { - code, err := rawInt64(media[0], path+"[0]", raw) - if err != nil { + var code int64 + if !isJSONNull(media[0]) { + if code, err = rawInt64(media[0], path+"[0]", raw); err != nil { return nil, withMethod(err, method) } - var ok bool - mimeType, ok = mimes[code] - if !ok { - return nil, &ProtocolEvidenceError{Method: method, Path: path + "[0]", Detail: fmt.Sprintf("未识别的媒体 MIME 枚举 %d", code), Raw: cloneRaw(raw)} - } + } + kind := interactionMediaMIME[index] + mimeType, ok := kind.codes[code] + if !ok { + mimeType = kind.unlisted } encoded, err := rawString(media[1], path+"[1]", raw) if err != nil { @@ -767,11 +768,5 @@ func decodeInteractionMedia(method string, value json.RawMessage, index int, mim if err != nil { return nil, &ProtocolEvidenceError{Method: method, Path: path + "[1]", Detail: "媒体数据不是 Base64", Raw: cloneRaw(raw)} } - if missingVideoMIME { - mimeType = http.DetectContentType(data) - if mimeType != "video/mp4" { - return nil, &ProtocolEvidenceError{Method: method, Path: path, Detail: "视频 MIME 枚举缺失且数据不是 MP4", Raw: cloneRaw(raw)} - } - } return &Blob{MIME: mimeType, Data: data}, nil } diff --git a/internal/aistudio/interaction_media_test.go b/internal/aistudio/interaction_media_test.go index 55a0a1ee..76a8b3a6 100644 --- a/internal/aistudio/interaction_media_test.go +++ b/internal/aistudio/interaction_media_test.go @@ -2,85 +2,52 @@ package aistudio import ( "bytes" - "encoding/base64" "encoding/json" - "errors" "strings" "testing" ) -func TestDecodeInteractionMedia(t *testing.T) { - // Minimal ftyp box with an MP4 major brand. - mp4 := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") - encoded := base64.StdEncoding.EncodeToString(mp4) +// TestDecodeInteractionMediaMIME 核对各类媒体 delta 按官网枚举表取 MIME，枚举缺省或不在表中时取该类的默认 MIME +func TestDecodeInteractionMediaMIME(t *testing.T) { + data := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") for _, tc := range []struct { - name string - wire string - index int - wantMIME string - wantErr bool + field int + code any + mime string }{ - {"omitted video enum", `[null,"` + encoded + `"]`, 4, "video/mp4", false}, - {"numeric video enum", `[1,"` + encoded + `"]`, 4, "video/mp4", false}, - {"numeric image enum", `[1,"aW1hZ2U="]`, 1, "image/png", false}, - {"missing image enum", `[null,"` + encoded + `"]`, 1, "", true}, - {"unknown video enum", `[9,"` + encoded + `"]`, 4, "", true}, - {"string MIME", `["video/mp4","` + encoded + `"]`, 4, "", true}, - {"bad base64", `[null,"***"]`, 4, "", true}, - {"wrong content", `[null,"aW1hZ2U="]`, 4, "", true}, - {"empty content", `[null,""]`, 4, "", true}, - {"short media", `[null]`, 4, "", true}, + {4, nil, "video/mp4"}, + {4, 1, "video/mp4"}, + {4, 7, "video/webm"}, + {4, 42, "video/mp4"}, + {1, nil, "image/jpeg"}, + {1, 1, "image/png"}, + {1, 3, "image/webp"}, + {2, nil, "audio/wav"}, + {2, 11, "audio/opus"}, + {3, nil, "application/pdf"}, } { - t.Run(tc.name, func(t *testing.T) { - mimes := interactionVideoMIME - if tc.index == 1 { - mimes = interactionImageMIME - } - blob, err := decodeInteractionMedia("CreateInteractionStream", json.RawMessage(tc.wire), tc.index, mimes, json.RawMessage(tc.wire)) - if tc.wantErr { - var evidence *ProtocolEvidenceError - if !errors.As(err, &evidence) { - t.Fatalf("expected protocol error, got %v", err) - } - return - } - if err != nil { - t.Fatal(err) - } - if blob.MIME != tc.wantMIME { - t.Fatalf("MIME = %q", blob.MIME) - } - if tc.index == 4 && !bytes.Equal(blob.Data, mp4) { - t.Fatal("video bytes changed") + content := make([]any, tc.field+1) + content[tc.field] = []any{tc.code, data} + delta := make([]any, 11) + delta[10] = []any{0, content} + final := make([]any, 20) + final[19] = []any{[]any{nil, 3}} + wire, err := json.Marshal([]any{[]any{delta, final}, []any{0}}) + if err != nil { + t.Fatal(err) + } + var media *Blob + err = DecodeInteractionStream("CreateInteractionStream", strings.NewReader(string(wire)), func(event Event) error { + if event.Kind == EventMedia { + media = event.InlineData } + return nil }) - } -} - -func TestDecodeInteractionStreamVideoWithoutMIME(t *testing.T) { - mp4 := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") - content := make([]any, 5) - content[4] = []any{nil, base64.StdEncoding.EncodeToString(mp4)} - delta := make([]any, 11) - delta[10] = []any{0, content} - final := make([]any, 20) - final[19] = []any{[]any{nil, 3}} - wire, err := json.Marshal([]any{[]any{delta, final}, []any{0}}) - if err != nil { - t.Fatal(err) - } - var events []Event - err = DecodeInteractionStream("CreateInteractionStream", strings.NewReader(string(wire)), func(event Event) error { - events = append(events, event) - return nil - }) - if err != nil { - t.Fatal(err) - } - if len(events) != 2 || events[0].Kind != EventMedia || events[1].Kind != EventFinish { - t.Fatalf("unexpected events: %#v", events) - } - if events[0].InlineData.MIME != "video/mp4" || !bytes.Equal(events[0].InlineData.Data, mp4) { - t.Fatal("video corrupted") + if err != nil { + t.Fatalf("field %d code %v: %v", tc.field, tc.code, err) + } + if media == nil || media.MIME != tc.mime || !bytes.Equal(media.Data, data) { + t.Fatalf("field %d code %v: media=%+v", tc.field, tc.code, media) + } } }