diff --git a/docs/protocol.md b/docs/protocol.md index 4ffe5684..c4255064 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1062,7 +1062,16 @@ thinking level 取值 MINIMAL=1、LOW=2、MEDIUM=3、HIGH=4,由 `reasoning_eff 视频设置来自 Interactions `response_format` 的 `{"type":"video","aspect_ratio","duration","resolution"}` 与 `generation_config.video_config.task`,或 Gemini API `generationConfig.responseFormat` 的 `video.aspectRatio`、`video.duration`、`video.resolution` 与扩展字段 `generationConfig.videoConfig.task`;Interactions `response_modalities` 的 `video` 与 Gemini API `generationConfig.responseModalities` 的 `VIDEO` 等同于不带设置的视频输出格式;`duration` 取 `"4s"` 或秒数并四舍五入为整秒,宽高比与分辨率换成最接近的上游取值,无法解析的取值与未知任务返回 400。 -响应为 `[[...], ?]`。事件索引 10 的 content delta 中 field 1 为正文、field 5 为视频 `[1,""]`、field 6 为思考摘要,分别映射为 text、`video/mp4` 媒体与 reasoning 事件;索引 19 的最终 interaction 状态 `3` 产生 usage(输入、输出、思考、总 token 位于 usage 索引 0、4、8、9)与 `stop` 终态,状态 `4`、`5` 返回错误。事件列表后的 google.rpc 状态按 code 映射 HTTP 状态,额度 code 8 为 429;首个正文事件之前的错误进入换号与冷却,之后的错误以流内 error 结束。 +响应为 `[[...], ?]`。事件索引 10 的 content delta 中 field 1 为正文、field 2 至 5 为图片、音频、文档与视频 `[,""]`、field 6 为思考摘要,分别映射为 text、媒体与 reasoning 事件;索引 19 的最终 interaction 状态 `3` 产生 usage(输入、输出、思考、总 token 位于 usage 索引 0、4、8、9)与 `stop` 终态,状态 `4`、`5` 返回错误。事件列表后的 google.rpc 状态按 code 映射 HTTP 状态,额度 code 8 为 429;首个正文事件之前的错误进入换号与冷却,之后的错误以流内 error 结束。 + +媒体 MIME 枚举与官网前端相同,枚举为 `null` 或不在表中时取该类的默认 MIME: + +| 字段 | 枚举 | 默认 | +| --- | --- | --- | +| 图片 field 2 | 1 png、2 jpeg、3 webp、4 heic、5 heif、7 gif、8 bmp、9 tiff | `image/jpeg` | +| 音频 field 3 | 1 wav、2 mp3、3 aiff、4 aac、5 ogg、6 flac、7 mpeg、8 m4a、9 l16、10 s16le、11 opus、12 alaw、13 mulaw | `audio/wav` | +| 文档 field 4 | 1 pdf | `application/pdf` | +| 视频 field 5 | 1 mp4、2 mpeg、3 mpg、4 mov、5 avi、6 x-flv、7 webm、8 wmv、9 3gpp | `video/mp4` | ### 后台 Interaction 任务 diff --git a/internal/aistudio/interaction.go b/internal/aistudio/interaction.go index 46c51494..4333b460 100644 --- a/internal/aistudio/interaction.go +++ b/internal/aistudio/interaction.go @@ -21,11 +21,19 @@ var interactionThinkingLevels = map[int64]int64{4: 1, 1: 2, 2: 3, 3: 4} // interactionCancelTimeout 是取消后台任务的期限,包含等待账户请求槽位的时间 const interactionCancelTimeout = time.Minute -// interactionVideoMIME 表示 Interaction 视频内容的 MIME 枚举 -var interactionVideoMIME = map[int64]string{1: "video/mp4"} +// interactionMediaMIME 是 content delta 各媒体字段的 MIME 枚举表,与官网前端相同 +var interactionMediaMIME = map[int]interactionMIME{ + 1: {codes: map[int64]string{1: "image/png", 2: "image/jpeg", 3: "image/webp", 4: "image/heic", 5: "image/heif", 7: "image/gif", 8: "image/bmp", 9: "image/tiff"}, unlisted: "image/jpeg"}, + 2: {codes: map[int64]string{1: "audio/wav", 2: "audio/mp3", 3: "audio/aiff", 4: "audio/aac", 5: "audio/ogg", 6: "audio/flac", 7: "audio/mpeg", 8: "audio/m4a", 9: "audio/l16", 10: "audio/s16le", 11: "audio/opus", 12: "audio/alaw", 13: "audio/mulaw"}, unlisted: "audio/wav"}, + 3: {codes: map[int64]string{1: "application/pdf"}, unlisted: "application/pdf"}, + 4: {codes: map[int64]string{1: "video/mp4", 2: "video/mpeg", 3: "video/mpg", 4: "video/mov", 5: "video/avi", 6: "video/x-flv", 7: "video/webm", 8: "video/wmv", 9: "video/3gpp"}, unlisted: "video/mp4"}, +} -// interactionImageMIME 表示 Interaction 图片内容的 MIME 枚举 -var interactionImageMIME = map[int64]string{1: "image/png"} +// interactionMIME 是一类媒体的 MIME 枚举与枚举缺省或不在表中时的 MIME +type interactionMIME struct { + codes map[int64]string + unlisted string +} // interactionStatusHTTP 把流尾 google.rpc 状态码换算为 HTTP 状态 var interactionStatusHTTP = map[int64]int{ @@ -485,7 +493,7 @@ func decodeInteractionEvent(method string, raw json.RawMessage, emit func(Event) return true, emit(Event{Kind: EventFinish, FinishReason: "stop"}) } -// decodeInteractionDelta 解码 content delta:field 1 文本、field 2 图片、field 5 视频、field 6 思考摘要、field 7 思考签名、field 10 引用 +// decodeInteractionDelta 解码 content delta:field 1 文本、field 2 图片、field 3 音频、field 4 文档、field 5 视频、field 6 思考摘要、field 7 思考签名、field 10 引用 func decodeInteractionDelta(method string, raw json.RawMessage, emit func(Event) error) error { delta, err := rawArray(raw, "$delta", raw) if err != nil || len(delta) < 2 { @@ -512,12 +520,8 @@ func decodeInteractionDelta(method string, raw json.RawMessage, emit func(Event) if err := emit(Event{Kind: EventText, Text: decoded}); err != nil { return err } - case 1, 4: - mimes := interactionImageMIME - if index == 4 { - mimes = interactionVideoMIME - } - media, err := decodeInteractionMedia(method, value, index, mimes, raw) + case 1, 2, 3, 4: + media, err := decodeInteractionMedia(method, value, index, raw) if err != nil { return err } @@ -738,20 +742,23 @@ func decodeInteractionCitations(method string, value json.RawMessage, raw json.R return nil } -// decodeInteractionMedia 解码图片或视频 content:MIME 枚举与 Base64 数据 -func decodeInteractionMedia(method string, value json.RawMessage, index int, mimes map[int64]string, raw json.RawMessage) (*Blob, error) { +// decodeInteractionMedia 解码媒体 content:MIME 枚举与 Base64 数据,枚举缺省或不在表中时按官网取该类媒体的默认 MIME +func decodeInteractionMedia(method string, value json.RawMessage, index int, raw json.RawMessage) (*Blob, error) { path := fmt.Sprintf("$delta[1][%d]", index) media, err := rawArray(value, path, raw) if err != nil || len(media) < 2 { return nil, &ProtocolEvidenceError{Method: method, Path: path, Detail: "媒体 delta 无效", Raw: cloneRaw(raw)} } - code, err := rawInt64(media[0], path+"[0]", raw) - if err != nil { - return nil, withMethod(err, method) + var code int64 + if !isJSONNull(media[0]) { + if code, err = rawInt64(media[0], path+"[0]", raw); err != nil { + return nil, withMethod(err, method) + } } - mimeType, ok := mimes[code] + kind := interactionMediaMIME[index] + mimeType, ok := kind.codes[code] if !ok { - return nil, &ProtocolEvidenceError{Method: method, Path: path + "[0]", Detail: fmt.Sprintf("未识别的媒体 MIME 枚举 %d", code), Raw: cloneRaw(raw)} + mimeType = kind.unlisted } encoded, err := rawString(media[1], path+"[1]", raw) if err != nil { diff --git a/internal/aistudio/interaction_media_test.go b/internal/aistudio/interaction_media_test.go new file mode 100644 index 00000000..76a8b3a6 --- /dev/null +++ b/internal/aistudio/interaction_media_test.go @@ -0,0 +1,53 @@ +package aistudio + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +// TestDecodeInteractionMediaMIME 核对各类媒体 delta 按官网枚举表取 MIME,枚举缺省或不在表中时取该类的默认 MIME +func TestDecodeInteractionMediaMIME(t *testing.T) { + data := []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom") + for _, tc := range []struct { + field int + code any + mime string + }{ + {4, nil, "video/mp4"}, + {4, 1, "video/mp4"}, + {4, 7, "video/webm"}, + {4, 42, "video/mp4"}, + {1, nil, "image/jpeg"}, + {1, 1, "image/png"}, + {1, 3, "image/webp"}, + {2, nil, "audio/wav"}, + {2, 11, "audio/opus"}, + {3, nil, "application/pdf"}, + } { + content := make([]any, tc.field+1) + content[tc.field] = []any{tc.code, data} + delta := make([]any, 11) + delta[10] = []any{0, content} + final := make([]any, 20) + final[19] = []any{[]any{nil, 3}} + wire, err := json.Marshal([]any{[]any{delta, final}, []any{0}}) + if err != nil { + t.Fatal(err) + } + var media *Blob + err = DecodeInteractionStream("CreateInteractionStream", strings.NewReader(string(wire)), func(event Event) error { + if event.Kind == EventMedia { + media = event.InlineData + } + return nil + }) + if err != nil { + t.Fatalf("field %d code %v: %v", tc.field, tc.code, err) + } + if media == nil || media.MIME != tc.mime || !bytes.Equal(media.Data, data) { + t.Fatalf("field %d code %v: media=%+v", tc.field, tc.code, media) + } + } +}