diff --git a/Makefile b/Makefile index 626e2f3..fefd578 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -locale-json: src/locales/en.json src/locales/zh-cn.json +locale-json: + node_modules/.bin/tsx scripts/generate-locale-json.ts locale-json-watch: watch make locale-json @@ -9,11 +10,4 @@ format: build: .PHONY npm run build -src/locales/en.json: src/locales/messages.yaml - node_modules/.bin/tsx scripts/generate-locale-json.ts - -src/locales/zh-cn.json: src/locales/messages.yaml - node_modules/.bin/tsx scripts/generate-locale-json.ts - - .PHONY: diff --git a/package-lock.json b/package-lock.json index 065c3db..e3d977b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,8 @@ "@fortawesome/react-fontawesome": "^0.1.19", "@jokester/ts-commonutil": "^0.6.1", "@reduxjs/toolkit": "^1.9.7", + "@xsai-ext/providers-cloud": "^0.4.0-beta.2", + "@xsai/generate-object": "^0.4.0-beta.2", "@zip.js/zip.js": "^2.7.60", "antd": "^4.24.16", "antd-img-crop": "^3.16.0", @@ -49,7 +51,10 @@ "redux-saga": "^1.3.0", "store": "^2.0.12", "use-debounce": "^10.0.4", - "uuid": "^7.0.3" + "uuid": "^7.0.3", + "xsai": "^0.4.0-beta.2", + "zod": "^3.25.76", + "zod-to-json-schema": "^3.24.6" }, "devDependencies": { "@tsconfig/strictest": "^2.0.5", @@ -2944,6 +2949,156 @@ "resolved": "https://registry.npmjs.org/@xobotyi/scrollbar-width/-/scrollbar-width-1.9.5.tgz", "integrity": "sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==" }, + "node_modules/@xsai-ext/providers-cloud": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai-ext/providers-cloud/-/providers-cloud-0.4.0-beta.2.tgz", + "integrity": "sha512-kquc/gLHZzBevdSbpRIlLr6jBHToVbvVIhjeUtqMuGcL613l9A9CJ2CSlnHmrgxHZQSSPIBeUAl0WEG4sWsP9g==", + "license": "MIT", + "dependencies": { + "@xsai-ext/shared-providers": "~0.4.0-beta.2", + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai-ext/shared-providers": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai-ext/shared-providers/-/shared-providers-0.4.0-beta.2.tgz", + "integrity": "sha512-+GEct6b9Q1/4o9NpoL8+aviZPiV5EU/r100F6gNlSk9QweWCqlfAUCnS9eAIWbGJ3fZcBADGxWyQE6Sk6a3LGQ==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/embed": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/embed/-/embed-0.4.0-beta.2.tgz", + "integrity": "sha512-9tl8WZvIbqjMidOvtDTeGMoeK0d8i6Wz7T6NEHwFuWt4ZLeFn3PXjx7Sm5F/607ByBs1mp6p7P4KRA0kR3ma4Q==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/generate-image": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/generate-image/-/generate-image-0.4.0-beta.2.tgz", + "integrity": "sha512-pxpiWW7NqBQkzREKByADM9l5Q+15an/K4RW5zorM2D2koqnK09pNH7jxMOJZwsjbTQE1+h38MwhEeJXdstokEw==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/generate-object": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/generate-object/-/generate-object-0.4.0-beta.2.tgz", + "integrity": "sha512-nkcY2Mn01s7p0SiNhYUlsrrrrOUgEQZtnGpfTfefAi0bynxXLVg//MEpm3tS4WZUpQvcZZRjTgMU91tdEyHxmQ==", + "license": "MIT", + "dependencies": { + "@xsai/generate-text": "~0.4.0-beta.2", + "xsschema": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/generate-speech": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/generate-speech/-/generate-speech-0.4.0-beta.2.tgz", + "integrity": "sha512-DitmNQYkTbz6a4btBFDZOlNxs2tU0JuE60r4FjaNDU1kpI5X2Ah49kfcCQya9i+3RnXDcgPMUzAd1zgOmFEkGw==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/generate-text": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/generate-text/-/generate-text-0.4.0-beta.2.tgz", + "integrity": "sha512-H0Fq8+O/8zJpNiwW4+PjUYQfrZlfh0DFvUmPg3wPFdQULZICKhMbxP/adZTIkXw+w7hXwd2Uho8aYjhzrEIfUg==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2", + "@xsai/shared-chat": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/generate-transcription": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/generate-transcription/-/generate-transcription-0.4.0-beta.2.tgz", + "integrity": "sha512-LVUM5Ew7GEuSUn5H9Gvz14YLHn/T2Dc/RngdEvYg+HNAm9CsLq51A5T6aqEpkkK2csAOsMvtaHEoEIYSKISDAQ==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/model": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/model/-/model-0.4.0-beta.2.tgz", + "integrity": "sha512-gNfCbfdYw3mCi9OUMe4OGVZ3I752QveOndMVT/99VymC0c8albdKLBGT8UgRLrW6bKTl1Vx9Vy0kausZWxo6jw==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/shared": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/shared/-/shared-0.4.0-beta.2.tgz", + "integrity": "sha512-nKdT+/gon1FxkEqv1iKfS2QRWnmYY/2o7Wl+Bcfot45qACE0sK9E4nw2BeLI/MeRYD6w7bTLP2J2U8373aHdYA==", + "license": "MIT" + }, + "node_modules/@xsai/shared-chat": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/shared-chat/-/shared-chat-0.4.0-beta.2.tgz", + "integrity": "sha512-2+HX5XEiC4x17NvtlIGTA/aOH9/EyJ2bD/gS+nmbiU8zuPykffS5EJ7CwuBt8rWTQUXEDHFA6hRpNLG7Q64EfA==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/stream-object": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/stream-object/-/stream-object-0.4.0-beta.2.tgz", + "integrity": "sha512-FBjVEVs6HMS5U7RMXgzx3h+3p7M6dnDVP2dzL4oP8RznFYvat1tU/tZWEER0pdLa/ny7b3thzH36z07DS0wzeQ==", + "license": "MIT", + "dependencies": { + "@xsai/stream-text": "~0.4.0-beta.2", + "xsschema": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/stream-text": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/stream-text/-/stream-text-0.4.0-beta.2.tgz", + "integrity": "sha512-16jQfXZ6RTw5JsN6zxeJ4At1CMsCrZUGJLvVsRmdfMRVjhxEXGGSvK1meMnWPKA4xtf+UEQCvwtJW6t8YQlK/w==", + "license": "MIT", + "dependencies": { + "@xsai/shared-chat": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/tool": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/tool/-/tool-0.4.0-beta.2.tgz", + "integrity": "sha512-yp+nD6/l6pHwr8LYYckEub/+ZDz2NkSRVwguU9Uv1nlIPPi5OR4txiF8LnjC35msJptCqKioFki+FkWiYd32WA==", + "license": "MIT", + "dependencies": { + "@xsai/shared": "~0.4.0-beta.2", + "@xsai/shared-chat": "~0.4.0-beta.2", + "xsschema": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/utils-chat": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/utils-chat/-/utils-chat-0.4.0-beta.2.tgz", + "integrity": "sha512-VgB9ohysQFUA6mogvP0e3R24Tr+RH0VKZ28Yx0Y372ptzo3hXjFjlftnpXCdNOtY1usiujVuC17WuryhV041aQ==", + "license": "MIT", + "dependencies": { + "@xsai/shared-chat": "~0.4.0-beta.2" + } + }, + "node_modules/@xsai/utils-reasoning": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/utils-reasoning/-/utils-reasoning-0.4.0-beta.2.tgz", + "integrity": "sha512-JFzgRVppyEPadqkNIbsrdViwZxY7/BGuWvFdb9zmaPiGLrOB1O6vCRs2GHMKAcJHvz1j7klDEc2JoMOcJtJXmQ==", + "license": "MIT" + }, + "node_modules/@xsai/utils-stream": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/@xsai/utils-stream/-/utils-stream-0.4.0-beta.2.tgz", + "integrity": "sha512-4+ecBLGZ7LMPHEvz6QSFVkdZLLlgieycDtPSJgvZxy6sNLekuhFzTDdhJ7Q717zZt8oCyYnBSG2m+moBwwe/2g==", + "license": "MIT" + }, "node_modules/@zip.js/zip.js": { "version": "2.7.60", "resolved": "https://registry.npmjs.org/@zip.js/zip.js/-/zip.js-2.7.60.tgz", @@ -13111,6 +13266,63 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/xsai": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/xsai/-/xsai-0.4.0-beta.2.tgz", + "integrity": "sha512-AXThpC7TkdA1vcZ0+xEwsp5WK6bt5S0UY9N0bh8EHVZS0lWzjrLk3AYGhTnHNRSc8okQekXDBXEWN1jj0fet1A==", + "license": "MIT", + "dependencies": { + "@xsai/embed": "~0.4.0-beta.2", + "@xsai/generate-image": "~0.4.0-beta.2", + "@xsai/generate-object": "~0.4.0-beta.2", + "@xsai/generate-speech": "~0.4.0-beta.2", + "@xsai/generate-text": "~0.4.0-beta.2", + "@xsai/generate-transcription": "~0.4.0-beta.2", + "@xsai/model": "~0.4.0-beta.2", + "@xsai/shared": "~0.4.0-beta.2", + "@xsai/shared-chat": "~0.4.0-beta.2", + "@xsai/stream-object": "~0.4.0-beta.2", + "@xsai/stream-text": "~0.4.0-beta.2", + "@xsai/tool": "~0.4.0-beta.2", + "@xsai/utils-chat": "~0.4.0-beta.2", + "@xsai/utils-reasoning": "~0.4.0-beta.2", + "@xsai/utils-stream": "~0.4.0-beta.2" + } + }, + "node_modules/xsschema": { + "version": "0.4.0-beta.2", + "resolved": "https://registry.npmjs.org/xsschema/-/xsschema-0.4.0-beta.2.tgz", + "integrity": "sha512-bzwAHTao5dcEy+GM/mVPbrWuFslUCPizvMjrLqV0PTDNw1jIUuYc+eNdSVl7+vp5RtA5DfWAf+qdTCJitQoCSw==", + "license": "MIT", + "peerDependencies": { + "@valibot/to-json-schema": "^1.0.0", + "arktype": "^2.1.20", + "effect": "^3.16.0", + "sury": "^10.0.0", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.24.5" + }, + "peerDependenciesMeta": { + "@valibot/to-json-schema": { + "optional": true + }, + "arktype": { + "optional": true + }, + "effect": { + "optional": true + }, + "sury": { + "optional": true + }, + "zod": { + "optional": true + }, + "zod-to-json-schema": { + "optional": true + } + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -13183,6 +13395,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.24.6", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", + "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.24.1" + } + }, "node_modules/zscroller": { "version": "0.4.8", "resolved": "https://registry.npmjs.org/zscroller/-/zscroller-0.4.8.tgz", diff --git a/package.json b/package.json index 85e9fe8..ea800b6 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "@fortawesome/react-fontawesome": "^0.1.19", "@jokester/ts-commonutil": "^0.6.1", "@reduxjs/toolkit": "^1.9.7", + "@xsai-ext/providers-cloud": "^0.4.0-beta.2", + "@xsai/generate-object": "^0.4.0-beta.2", "@zip.js/zip.js": "^2.7.60", "antd": "^4.24.16", "antd-img-crop": "^3.16.0", @@ -43,7 +45,10 @@ "redux-saga": "^1.3.0", "store": "^2.0.12", "use-debounce": "^10.0.4", - "uuid": "^7.0.3" + "uuid": "^7.0.3", + "xsai": "^0.4.0-beta.2", + "zod": "^3.25.76", + "zod-to-json-schema": "^3.24.6" }, "scripts": { "build": "vite build", diff --git a/scripts/generate-locale-json.ts b/scripts/generate-locale-json.ts index 854385b..d6e1bc7 100644 --- a/scripts/generate-locale-json.ts +++ b/scripts/generate-locale-json.ts @@ -5,6 +5,26 @@ import yaml from 'js-yaml'; const assetDir = path.join(__dirname, '../src/locales'); const messageYaml = path.join(assetDir, 'messages.yaml'); +/** + * yield [path, message] pairs + */ +function* extractPathedMessages(obj: object, locale: string, pathPrefix: readonly string[] = []): Generator<[string, string]> { + for (const [key, value] of Object.entries(obj)) { + if (typeof value === 'object' && value) { + yield* extractPathedMessages(value, locale, [...pathPrefix, key]); + } else if (typeof value === 'string') { + if (key === locale) yield [pathPrefix.join('.'), value]; + } else { + throw new Error(`unexpected value type at ${[...pathPrefix, key].join('.')}: ${typeof value}`); + } + } +} + +const lang2Basename = Object.entries({ + zhCn: 'zh-cn.json', + en: 'en.json', +}) + setTimeout(async function main() { /** * key => locale => message @@ -13,22 +33,16 @@ setTimeout(async function main() { await fsp.readFile(messageYaml, { encoding: 'utf-8' }), ) as Record>; - for (const [locale, basename] of Object.entries({ - zhCn: 'zh-cn.json', - en: 'en.json', - })) { + const path2count: Record = {}; + for (const [locale, basename] of lang2Basename ) { /** * key => message */ const value: Record = {}; - Object.keys(messages).forEach((messageKey) => { - const msg = (value[messageKey] = messages[messageKey][locale]); - if (!msg) { - throw new Error( - `translated message not found for key=${messageKey} / locale=${locale}`, - ); - } - }); + for(const [path, msg] of extractPathedMessages(messages, locale)) { + path2count[path] = (path2count[path] ?? 0) + 1; + value[path] = msg; + } const dest = path.join(assetDir, basename); await fsp.writeFile(dest, JSON.stringify(value, null, 2)); console.info(`written to ${dest}`); diff --git a/src/components/ai/BatchTranslateModal.tsx b/src/components/ai/BatchTranslateModal.tsx new file mode 100644 index 0000000..a7ad4be --- /dev/null +++ b/src/components/ai/BatchTranslateModal.tsx @@ -0,0 +1,271 @@ +import { FC } from 'react'; +import { File as MFile } from '@/interfaces'; +import { Target } from '@/interfaces'; +import { useIntl } from 'react-intl'; +import { useState } from 'react'; +import { ResourcePool } from '@jokester/ts-commonutil/lib/concurrency/resource-pool-basic'; +import { getCancelToken } from '@/utils/api'; +import { useAsyncEffect } from '@jokester/ts-commonutil/lib/react/hook/use-async-effect'; +import { createDebugLogger } from '@/utils/debug-logger'; +import { api, resultTypes } from '@/apis'; +import { toLowerCamelCase } from '@/utils'; +import { + llmTranslateImage, + LLMConf, + FilePreprocessResult, +} from '@/services/ai/llm_preprocess'; +import { ModalHandle } from '.'; +import { Icon } from '../icon'; + +const debugLogger = createDebugLogger('components:ai:BatchTranslateModal'); +interface FileProgress { + file: MFile; + icon: React.ReactNode | string; + message?: React.ReactNode | string; +} + +function clipTo01(x: number) { + return Math.max(0, Math.min(1, x)); +} + +const stateIcons = { + waiting: , + working: , + skip: , + fail: , + success: , +} as const; + +export const BatchTranslateModalContent: FC<{ + llmConf: LLMConf; + files: MFile[]; + target: Target; + onFileSaved?(f: MFile): void; + getHandle(): ModalHandle; +}> = ({ files, target, getHandle, llmConf, onFileSaved }) => { + const { formatMessage } = useIntl(); + const [fileStates, setFileStates] = useState(() => + files.map( + (file): FileProgress => ({ + file, + icon: stateIcons.waiting, + message: formatMessage({ + id: 'fileList.aiTranslate.fileMessage.waiting', + }), + }), + ), + ); + + useAsyncEffect(async (running, released) => { + const [cancelToken, fillCancelToken] = getCancelToken(); + const fileLimiter = ResourcePool.multiple([1, 2]); + const moeflowApiLimiter = ResourcePool.multiple([1, 2, 3, 4]); + const abort = new AbortController(); + released.then(() => fillCancelToken('unmounted')); + released.then(() => abort.abort('unmounted')); + + if (!running.current) { + debugLogger('canceled'); + return; + } + released = released.then(() => { + debugLogger('released'); + }); + const tasksEnded = Promise.allSettled( + files.map((f) => fileLimiter.use(() => translateFile(f))), + ); + const cancelled = await Promise.race([ + released.then(() => true), + tasksEnded.then(() => false), + ]); + debugLogger('cancelled', cancelled); + if (!cancelled) { + const handle = getHandle(); + handle.update({ okButtonProps: { disabled: false } }); + } + return; + + function setFileState(f: MFile, message: string, icon: React.ReactNode) { + debugLogger('setFileState', f.id, message); + setFileStates((prev) => + prev.map((state) => + state.file === f ? { ...state, message, icon } : state, + ), + ); + } + + async function translateFile(f: MFile) { + setFileState( + f, + formatMessage({ id: 'fileList.aiTranslate.fileMessage.sendingImage' }), + stateIcons.working, + ); + if (![undefined, null, 'success'].includes(f.uploadState)) { + setFileState( + f, + formatMessage({ + id: 'fileList.aiTranslate.fileMessage.uploadNotFinished', + }), + stateIcons.skip, + ); + return; + } + const refetchRes = await api.file + .getFile({ fileID: f.id, configs: { cancelToken } }) + .catch(() => null); + if (refetchRes?.type !== resultTypes.SUCCESS) { + setFileState( + f, + formatMessage({ + id: 'fileList.aiTranslate.fileMessage.failFetchingImage', + }), + stateIcons.fail, + ); + return; + } + const resData = toLowerCamelCase(refetchRes.data); + if (resData.sourceCount) { + setFileState( + f, + formatMessage({ + id: 'fileList.aiTranslate.fileMessage.textAlreadyExist', + }), + stateIcons.skip, + ); + return; + } + const imgBlob = await fetch(resData.url!, { signal: abort.signal }).then( + (r) => r.blob(), + () => null, + ); + if (!imgBlob) { + setFileState( + f, + formatMessage({ + id: 'fileList.aiTranslate.fileMessage.failFetchingImage', + }), + stateIcons.fail, + ); + return; + } + + setFileState( + f, + formatMessage({ id: 'fileList.aiTranslate.fileMessage.translating' }), + stateIcons.working, + ); + + const result = await llmTranslateImage( + llmConf, + target.language.enName, + imgBlob, + ).catch((e: unknown) => { + debugLogger('translate failed', e); + return null; + }); + debugLogger('translate result', result); + if (!running.current) { + return; + } + + if (result) { + await saveTranslations(f, result); + } else { + setFileState( + f, + formatMessage({ + id: 'fileList.aiTranslate.fileMessage.translateFailed', + }), + stateIcons.fail, + ); + } + } + + async function saveTextBlock( + f: MFile, + tf: FilePreprocessResult, + tb: FilePreprocessResult['texts'][number], + ) { + const src = await api.source.createSource({ + fileID: f.id, + data: { + x: clipTo01((tb.left + tb.width / 2) / tf.imageW), + y: clipTo01((tb.top + tb.height / 2) / tf.imageH), + content: tb.text, + }, + configs: { cancelToken }, + }); + await api.translation.createTranslation({ + sourceID: src.data.id, + data: { + content: tb.translated, + targetID: target.id, + }, + // not using the cancel token, to make the saving operation closer to atomic + // configs: { cancelToken }, + }); + } + + async function saveTranslations(f: MFile, r: FilePreprocessResult) { + if (r.texts.length === 0) { + setFileState( + f, + formatMessage({ + id: 'fileList.aiTranslate.fileMessage.noTextDetected', + }), + stateIcons.skip, + ); + } + setFileState( + f, + formatMessage({ id: 'fileList.aiTranslate.fileMessage.saving' }), + stateIcons.working, + ); + try { + await Promise.all( + r.texts.map((tb) => + moeflowApiLimiter.use(() => saveTextBlock(f, r, tb)), + ), + ); + setFileState( + f, + formatMessage( + { id: 'fileList.aiTranslate.fileMessage.success' }, + { count: r.texts.length }, + ), + stateIcons.success, + ); + onFileSaved?.({ + ...f, + sourceCount: r.texts.length, + translatedSourceCount: r.texts.length, + }); + } catch (e) { + debugLogger('save text block failed', e); + setFileState( + f, + formatMessage({ id: 'fileList.aiTranslate.fileMessage.failSaving' }), + stateIcons.fail, + ); + } + } + }, []); + return ( +
+

+ {formatMessage( + { id: 'fileList.aiTranslate.workingModal.content' }, + { fileCount: files.length }, + )} +

+
    + {fileStates.map((state) => ( +
  • + {state.icon} + {state.file.name} - {state.message} +
    • + ))} +
+
+ ); +}; diff --git a/src/components/ai/ModelConfigForm.tsx b/src/components/ai/ModelConfigForm.tsx new file mode 100644 index 0000000..6bcde04 --- /dev/null +++ b/src/components/ai/ModelConfigForm.tsx @@ -0,0 +1,212 @@ +import React, { useEffect } from 'react'; +import { Form, Input, Select, Divider, Typography } from 'antd'; +import * as LlmService from '@/services/ai/llm_preprocess'; +import { useIntl } from 'react-intl'; + +interface ModelConfigFormProps { + initialValue?: LlmService.LLMConf; + onChange?: (config: LlmService.LLMConf) => void; +} + +export const ModelConfigForm: React.FC = ({ + initialValue, + onChange, +}) => { + const { formatMessage } = useIntl(); + const [form] = Form.useForm(); + + // Find matching preset index for initial value + const findPresetIndex = (config: LlmService.LLMConf): number => { + const index = LlmService.llmPresets.findIndex( + (preset) => + preset.model === config.model && preset.baseUrl === config.baseUrl, + ); + return index >= 0 ? index : -1; // -1 for custom + }; + + useEffect(() => { + if (initialValue) { + const presetIndex = findPresetIndex(initialValue); + form.setFieldsValue({ + preset: presetIndex, + model: initialValue.model, + baseUrl: initialValue.baseUrl, + apiKey: initialValue.apiKey, + }); + onChange?.(initialValue); + } + }, [initialValue, form, onChange]); + + // Handle preset selection change + const handlePresetChange = (presetIndex: number) => { + if (presetIndex >= 0 && presetIndex < LlmService.llmPresets.length) { + const preset = LlmService.llmPresets[presetIndex]; + const patch = { + model: preset.model, + baseUrl: preset.baseUrl, + apiKey: preset.apiKey || '', + }; + form.setFieldsValue(patch); + handleFormChange(patch, form.getFieldsValue()); + } + // For custom preset (index -1), don't auto-fill fields + }; + + // Handle form values change + const handleFormChange = (changedValues: any, allValues: any) => { + // Check if model or baseUrl was changed and update preset accordingly + if ( + changedValues.model !== undefined || + changedValues.baseUrl !== undefined + ) { + const currentModel = allValues.model || changedValues.model; + const currentBaseUrl = allValues.baseUrl || changedValues.baseUrl; + + // Find matching preset + const matchingPresetIndex = LlmService.llmPresets.findIndex( + (preset) => + preset.model === currentModel && preset.baseUrl === currentBaseUrl, + ); + + // Update preset to match the current values + if (matchingPresetIndex >= 0) { + // Found a matching preset, switch to it + if (allValues.preset !== matchingPresetIndex) { + form.setFieldValue('preset', matchingPresetIndex); + } + } else { + // No preset matches, set to custom (-1) + if (allValues.preset !== -1) { + form.setFieldValue('preset', -1); + } + } + } + + const values = form.getFieldsValue(); + // Get provider from selected preset if available + let provider = ''; + if (values.preset >= 0 && values.preset < LlmService.llmPresets.length) { + provider = LlmService.llmPresets[values.preset].provider; + } + + const config: LlmService.LLMConf = { + provider, + model: values.model, + baseUrl: values.baseUrl, + apiKey: values.apiKey, + }; + onChange?.(config); + }; + return ( +
+ + {formatMessage({ id: 'fileList.aiTranslate.configModal.title' })} + +

+ {formatMessage({ id: 'fileList.aiTranslate.configModal.modelDesc' })} +

+

+ {formatMessage({ + id: 'fileList.aiTranslate.configModal.modelRequirements', + })} +

+

+ {formatMessage({ + id: 'fileList.aiTranslate.configModal.configsAreLocal', + })} +

+
+ + + + + + + + + + + + + + + +
+ + +
+ ); +}; diff --git a/src/components/ai/index.tsx b/src/components/ai/index.tsx new file mode 100644 index 0000000..301b412 --- /dev/null +++ b/src/components/ai/index.tsx @@ -0,0 +1,105 @@ +import { Modal } from 'antd'; +import { File as MFile, Target } from '@/interfaces'; +import { createDebugLogger } from '@/utils/debug-logger'; +import { ModalStaticFunctions } from 'antd/lib/modal/confirm'; + +import { ModelConfigForm } from './ModelConfigForm'; +import { BatchTranslateModalContent } from './BatchTranslateModal'; +import { useMemo } from 'react'; +import { LLMConf, llmPresets } from '@/services/ai/llm_preprocess'; +import { llmConfStorage } from '@/utils/storage'; +import { IntlShape, useIntl } from 'react-intl'; + +const debugLogger = createDebugLogger('components:project:FileListAiTranslate'); + +export type ModalHandle = ReturnType; + +interface TranslationCallbacks { + onFileSaved?(f: MFile): void; +} + +interface TranslatorApi { + start(callbacks: TranslationCallbacks): Promise; + testModel?(modelConf: LLMConf): Promise<{ worked: boolean; message: string }>; +} +function bind( + files: MFile[], + target: Target, + modal: ModalStaticFunctions, + { formatMessage }: IntlShape, +): TranslatorApi { + return { + start, + // testModel, + }; + async function start(callbacks: TranslationCallbacks) { + const llmConf = await new Promise((resolve, reject) => { + let confValue: LLMConf = llmConfStorage.load() ?? { + ...llmPresets.at(0)!, + }; + const onChange = (conf: LLMConf) => { + debugLogger('model configured', conf); + confValue = conf; + if (confValue.model && confValue.baseUrl && confValue.apiKey) { + handle.update({ okButtonProps: {} }); + } + }; + const handle = modal.confirm({ + icon: null, + content: ( + + ), + okText: formatMessage({ id: 'fileList.aiTranslate.startTranslate' }), + okButtonProps: { disabled: true }, + onOk: () => { + resolve(confValue); + }, + onCancel: () => { + resolve(null); + }, + }); + }); + if (!llmConf) { + return; + } + llmConfStorage.save(llmConf); + + await new Promise((resolve) => { + const handle = modal.confirm({ + icon: null, + content: ( + handle as ModalHandle} + /> + ), + okButtonProps: { disabled: true }, + onOk: () => { + resolve(true); + }, + onCancel: () => { + resolve(false); + }, + }); + }); + } +} + +export function useAiTranslate( + files: MFile[], + target: Target, +): [true, TranslatorApi, React.ReactNode] | [false, null, null] { + const [modal, contextHolder] = Modal.useModal(); + const intl = useIntl(); + + const api = useMemo( + () => bind(files, target, modal as ModalStaticFunctions, intl), + // eslint-disable-next-line react-hooks/exhaustive-deps + [target.id, files.map((file) => file.id).join('|')], + ); + + return [true, api, contextHolder]; +} diff --git a/src/components/icon/index.ts b/src/components/icon/index.ts index 0cd3d7d..f845488 100644 --- a/src/components/icon/index.ts +++ b/src/components/icon/index.ts @@ -1 +1,2 @@ +/** */ export { FontAwesomeIcon as Icon } from '@fortawesome/react-fontawesome'; diff --git a/src/components/project/FileList.tsx b/src/components/project/FileList.tsx index 0cedc7a..8129be9 100644 --- a/src/components/project/FileList.tsx +++ b/src/components/project/FileList.tsx @@ -29,6 +29,7 @@ import { routes } from '@/pages/routes'; import { ListPageSpec } from '@/components/shared/List'; import { FilePondFile } from 'filepond'; import { createDebugLogger } from '@/utils/debug-logger'; +import { useAiTranslate } from '@/components/ai'; /** 文件列表的属性接口 */ interface FileListProps { @@ -64,7 +65,6 @@ export const FileList: FC = ({ const [outputDrawerVisible, setOutputDrawerVisible] = useState(false); const coverWidth = IMAGE_COVER.WIDTH; const coverHeight = IMAGE_COVER.HEIGHT; - // const [aiTranslateAvailable, startAiTranslate, modalContextHolder] = useMoeflowCompanionAiTranslate(); const [items, setItems] = useState([]); const [spinningIDs, setSpinningIDs] = useState([]); // 删除请求中 @@ -83,6 +83,12 @@ export const FileList: FC = ({ const selectedFileIds = useSelector( (state: AppState) => state.file.filesState.selectedFileIds, ); + const [aiEnabled, aiTranslateApi, aiModalHolder] = useAiTranslate( + [...new Set(selectedFileIds)] + .map((id) => items.find((item) => item.id === id)) + .filter(Boolean) as MFile[], + target, + ); const openInTranslator = (file: MFile) => { history.push(routes.imageTranslator.build(file.id, target.id)); @@ -378,14 +384,23 @@ export const FileList: FC = ({ ? formatMessage({ id: 'project.changeTarget' }) + ' - ' : '') + target?.language.i18nName} - {false && ( + {aiEnabled && aiTranslateApi && ( )} {/* {can(team, TEAM_PERMISSION.USE_OCR_QUOTA) && ( @@ -579,6 +594,7 @@ export const FileList: FC = ({ selectedFileIds={selectedFileIds} /> + {aiModalHolder} ); }; diff --git a/src/components/project/FileListAiTranslate.tsx b/src/components/project/FileListAiTranslate.tsx deleted file mode 100644 index 42bbad8..0000000 --- a/src/components/project/FileListAiTranslate.tsx +++ /dev/null @@ -1,235 +0,0 @@ -import { Modal } from 'antd'; -import { FC, File as MFile, Target } from '@/interfaces'; -import { - useMoeflowCompanion, - moeflowCompanionServiceState, - MoeflowCompanionService, - TranslatedFile, -} from '@/services/ai/use_moeflow_companion'; -import { useAsyncEffect } from '@jokester/ts-commonutil/lib/react/hook/use-async-effect'; -import { createDebugLogger } from '@/utils/debug-logger'; -import { api, resultTypes } from '@/apis'; -import { useIntl } from 'react-intl'; -import { ModalStaticFunctions } from 'antd/lib/modal/confirm'; -import { useState } from 'react'; -import { ResourcePool } from '@jokester/ts-commonutil/lib/concurrency/resource-pool-basic'; -import { getCancelToken } from '@/utils/api'; -import { toLowerCamelCase } from '@/utils'; - -const debugLogger = createDebugLogger('components:project:FileListAiTranslate'); - -type ModalHandle = ReturnType; - -interface TranslatorFunc { - (files: MFile[], target: Target): void; -} -function openTranslateModal( - files: MFile[], - target: Target, - service: MoeflowCompanionService, - modal: ModalStaticFunctions, -) { - const handle = modal.confirm({ - content: ( - handle} - /> - ), - okButtonProps: { disabled: true }, - onOk: () => { - console.log('ok'); - }, - onCancel: () => { - console.log('cancel'); - }, - }); -} - -export function useMoeflowCompanionAiTranslate(): - | [true, TranslatorFunc, React.ReactNode] - | [false, null, null] { - const [serviceState, service] = useMoeflowCompanion(); - const [modal, contextHolder] = Modal.useModal(); - - debugLogger('service', serviceState, service); - if (serviceState !== moeflowCompanionServiceState.connected) { - return [false, null, null]; - } - - return [ - true, - (files, target) => - openTranslateModal( - files, - target, - service!, - modal as ModalStaticFunctions, - ), - contextHolder, - ]; -} - -interface TranslateTaskState { - file: MFile; - status: string; -} - -function clipTo01(x: number) { - return Math.max(0, Math.min(1, x)); -} - -const ModalContent: FC<{ - service: MoeflowCompanionService; - files: MFile[]; - target: Target; - getHandle(): ModalHandle; -}> = ({ - service: { client, serviceConf, multimodalTranslate }, - files, - target, - getHandle, -}) => { - const intl = useIntl(); - const [fileStates, setFileStates] = useState(() => - files.map((file) => ({ file, status: 'waiting' })), - ); - useAsyncEffect(async (running, released) => { - const [cancelToken, fillCancelToken] = getCancelToken(); - const fileLimiter = ResourcePool.multiple([1, 2]); - const moeflowApiLimiter = ResourcePool.multiple([1, 2, 3, 4]); - const abort = new AbortController(); - released.then(() => fillCancelToken('unmounted')); - released.then(() => abort.abort('unmounted')); - - if (!running.current) { - debugLogger('canceled'); - return; - } - const tasksEnded = Promise.allSettled([ - files.map((f, idx) => fileLimiter.use(() => translateFile(f, idx))), - ]); - const cancelled = await Promise.race([ - released.then(() => true), - tasksEnded.then(() => false), - ]); - if (!cancelled) { - const handle = getHandle(); - handle.update({ okButtonProps: { disabled: false } }); - } - return; - - function setFileState(f: MFile, status: string) { - setFileStates((prev) => - prev.map((state) => (state.file === f ? { ...state, status } : state)), - ); - } - - async function translateFile(f: MFile, idx: number) { - setFileState(f, 'working'); - if (![undefined, null, 'success'].includes(f.uploadState)) { - setFileState(f, 'skip: upload not finished'); - return; - } - const refetchRes = await api.file - .getFile({ fileID: f.id }) - .catch(() => null); - if (refetchRes?.type !== resultTypes.SUCCESS) { - setFileState(f, 'skip: fetch file failed'); - return; - } - const resData = toLowerCamelCase(refetchRes.data); - if (resData.sourceCount) { - setFileState(f, 'skip: source count not 0'); - } - const imgBlob = await fetch(resData.url!, { - // mode: 'no-cors', - }).then( - (r) => r.blob(), - () => null, - ); - if (!imgBlob) { - setFileState(f, 'skip: fetch image blob failed'); - return; - } - - const result = await multimodalTranslate( - client, - [imgBlob], - target.language.enName, - serviceConf!.defaultMultimodalModel!, - ).catch((e) => { - debugLogger('translate failed', e); - return []; - }); - debugLogger('translate result', result); - - const [r] = result; - - if (r) { - await saveTranslations(f, r); - } else { - setFileState(f, 'error: translate failed'); - } - } - - async function saveTextBlock( - f: MFile, - tf: TranslatedFile, - tb: TranslatedFile['text_blocks'][number], - ) { - const src = await api.source.createSource({ - fileID: f.id, - data: { - x: clipTo01((tb.left + tb.right) / 2 / tf.image_w), - y: clipTo01((tb.top + tb.bottom) / 2 / tf.image_h), - content: tb.source, - }, - configs: { cancelToken }, - }); - await api.translation.createTranslation({ - sourceID: src.data.id, - data: { - content: tb.translated, - targetID: target.id, - }, - configs: { cancelToken }, - }); - } - - async function saveTranslations(f: MFile, r: TranslatedFile) { - if (r.text_blocks.length === 0) { - setFileState(f, 'done: no text blocks'); - } - setFileState(f, 'saving'); - try { - await Promise.all( - r.text_blocks.map((tb) => - moeflowApiLimiter.use(() => saveTextBlock(f, r, tb)), - ), - ); - setFileState( - f, - `success: translated ${r.text_blocks.length} text marks`, - ); - } catch (e) { - debugLogger('save text block failed', e); - setFileState(f, 'save file failed'); - } - } - }, []); - return ( -
- {files.length} files to translate -
    - {fileStates.map((state) => ( -
  • - {state.file.name} - {state.status} -
    • - ))} -
-
- ); -}; diff --git a/src/locales/en.json b/src/locales/en.json index 792ab6d..bb42d9a 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -263,8 +263,35 @@ "fileList.changeMode": "Toggle Display Mode", "file.parseNotStart": "Auto Mark as Not Started", "fileList.ocrButtonTip": "Image Auto Tagging", - "fileList.aiTranslate": "Auto Translate", - "fileList.aiTranslateTip": "Detect marks and Translate", + "fileList.aiTranslate.buttonText": "Auto Translate", + "fileList.aiTranslate.buttonTip": "Detect text and Translate with LLM", + "fileList.aiTranslate.startTranslate": "Start Translate", + "fileList.aiTranslate.configModal.title": "Configure LLM", + "fileList.aiTranslate.configModal.modelDesc": "Please provide the LLM API configuration to translate the images.", + "fileList.aiTranslate.configModal.modelRequirements": "The LLM API should use the OpenAI-compatible format. Multimodal (image input) and tool calling capabilities are required. The models in presets are tested to work.", + "fileList.aiTranslate.configModal.configsAreLocal": "The last used configuration will be stored automatically. LLM configuration is only used and stored in your local browser.", + "fileList.aiTranslate.configModal.presets.label": "Presets", + "fileList.aiTranslate.configModal.presets.placeholder": "Please pick a preset", + "fileList.aiTranslate.configModal.presets.custom": "Custom", + "fileList.aiTranslate.configModal.model.label": "Model", + "fileList.aiTranslate.configModal.model.required": "Please enter the model ID", + "fileList.aiTranslate.configModal.baseUrl.label": "API Base URL", + "fileList.aiTranslate.configModal.baseUrl.required": "Please enter the base URL", + "fileList.aiTranslate.configModal.baseUrl.invalidUrl": "Please enter a valid URL", + "fileList.aiTranslate.configModal.apiKey.label": "API Key", + "fileList.aiTranslate.configModal.apiKey.required": "Please enter your API key", + "fileList.aiTranslate.workingModal.content": "Translating {fileCount} files with LLM. Closing this dialog will stop the work.", + "fileList.aiTranslate.fileMessage.sendingImage": "Sending image...", + "fileList.aiTranslate.fileMessage.uploadNotFinished": "Upload not finished, please try again later", + "fileList.aiTranslate.fileMessage.waiting": "Waiting in queue...", + "fileList.aiTranslate.fileMessage.failFetchingImage": "Failed to fetch image", + "fileList.aiTranslate.fileMessage.translating": "Translating...", + "fileList.aiTranslate.fileMessage.translateFailed": "Failed to translate", + "fileList.aiTranslate.fileMessage.textAlreadyExist": "Skipped: text marks already existed", + "fileList.aiTranslate.fileMessage.noTextDetected": "No text detected", + "fileList.aiTranslate.fileMessage.saving": "Saving...", + "fileList.aiTranslate.fileMessage.failSaving": "Failed to save", + "fileList.aiTranslate.fileMessage.success": "Recognized and translated {count} text blocks", "project.startOCR": "Start Auto Tagging", "project.startOCRTip": "Do you want to start auto-tagging images? (Only images marked as \"Not Started\" or \"Tagging Failed\" will be auto-tagged, and the quota will only be deducted upon successful tagging)", "site.ocrQuota": "Auto Tagging Limit", diff --git a/src/locales/messages.yaml b/src/locales/messages.yaml index 8902bc0..09c0e45 100644 --- a/src/locales/messages.yaml +++ b/src/locales/messages.yaml @@ -805,11 +805,100 @@ fileList.ocrButtonTip: zhCn: 图片自动标记 en: Image Auto Tagging fileList.aiTranslate: - zhCn: 自动翻译 - en: Auto Translate -fileList.aiTranslateTip: - zhCn: 自动识别文字并翻译 - en: Detect marks and Translate + buttonText: + zhCn: 自动翻译 + en: Auto Translate + buttonTip: + zhCn: 用LLM自动识别文字并翻译 + en: Detect text and Translate with LLM + startTranslate: + zhCn: 开始翻译 + en: Start Translate + configModal: + title: + zhCn: 配置LLM + en: Configure LLM + modelDesc: + zhCn: 请提供用于翻译图片的LLM API配置。 + en: Please provide the LLM API configuration to translate the images. + modelRequirements: + zhCn: LLM API 需使用OpenAI兼容格式。模型需支持图像输入和工具调用。建议使用预设配置中经过测试的模型。 + en: The LLM API should use the OpenAI-compatible format. Multimodal (image input) and tool calling capabilities are required. The models in presets are tested to work. + configsAreLocal: + zhCn: 前一次使用的 LLM 配置会被自动保存。LLM 配置仅在您的本地浏览器中使用和保存。 + en: The last used configuration will be stored automatically. LLM configuration is only used and stored in your local browser. + presets: + label: + zhCn: 预设配置 + en: Presets + placeholder: + zhCn: 请选择预设 + en: Please pick a preset + custom: + zhCn: 自定义 + en: Custom + model: + label: + zhCn: 模型ID + en: Model + required: + zhCn: 请输入模型 ID + en: Please enter the model ID + baseUrl: + label: + zhCn: API URL + en: API Base URL + required: + zhCn: 请输入 API URL + en: Please enter the base URL + invalidUrl: + zhCn: 请输入有效的URL + en: Please enter a valid URL + apiKey: + label: + zhCn: API 密钥 + en: API Key + required: + zhCn: 请输入您的 API 密钥 + en: Please enter your API key + workingModal: + content: + en: Translating {fileCount} files with LLM. Closing this dialog will stop the work. + zhCn: 正在用LLM翻译 {fileCount} 个文件，关闭此对话框将中断翻译。 + fileMessage: + sendingImage: + zhCn: 发送图片... + en: Sending image... + uploadNotFinished: + zhCn: 此图片上传未完成，请稍后再试 + en: Upload not finished, please try again later + waiting: + zhCn: 排队中... + en: Waiting in queue... + failFetchingImage: + zhCn: 获取图片失败 + en: Failed to fetch image + translating: + zhCn: 翻译中... + en: Translating... + translateFailed: + zhCn: 翻译失败 + en: Failed to translate + textAlreadyExist: + zhCn: 已有文字标记，不再翻译 + en: 'Skipped: text marks already existed' + noTextDetected: + zhCn: 未检测到文字 + en: No text detected + saving: + zhCn: 保存中... + en: Saving... + failSaving: + zhCn: 保存失败 + en: Failed to save + success: + zhCn: 识别并翻译了 {count} 处文字 + en: Recognized and translated {count} text blocks project.startOCR: zhCn: 开始自动标记 en: Start Auto Tagging @@ -1201,5 +1290,3 @@ admin.imageModeration: admin.captchas: zhCn: 验证码 en: Captchas - - diff --git a/src/locales/zh-cn.json b/src/locales/zh-cn.json index 788302f..beed32d 100644 --- a/src/locales/zh-cn.json +++ b/src/locales/zh-cn.json @@ -263,8 +263,35 @@ "fileList.changeMode": "切换显示模式", "file.parseNotStart": "自动标记未开始", "fileList.ocrButtonTip": "图片自动标记", - "fileList.aiTranslate": "自动翻译", - "fileList.aiTranslateTip": "自动识别文字并翻译", + "fileList.aiTranslate.buttonText": "自动翻译", + "fileList.aiTranslate.buttonTip": "用LLM自动识别文字并翻译", + "fileList.aiTranslate.startTranslate": "开始翻译", + "fileList.aiTranslate.configModal.title": "配置LLM", + "fileList.aiTranslate.configModal.modelDesc": "请提供用于翻译图片的LLM API配置。", + "fileList.aiTranslate.configModal.modelRequirements": "LLM API 需使用OpenAI兼容格式。模型需支持图像输入和工具调用。建议使用预设配置中经过测试的模型。", + "fileList.aiTranslate.configModal.configsAreLocal": "前一次使用的 LLM 配置会被自动保存。LLM 配置仅在您的本地浏览器中使用和保存。", + "fileList.aiTranslate.configModal.presets.label": "预设配置", + "fileList.aiTranslate.configModal.presets.placeholder": "请选择预设", + "fileList.aiTranslate.configModal.presets.custom": "自定义", + "fileList.aiTranslate.configModal.model.label": "模型ID", + "fileList.aiTranslate.configModal.model.required": "请输入模型 ID", + "fileList.aiTranslate.configModal.baseUrl.label": "API URL", + "fileList.aiTranslate.configModal.baseUrl.required": "请输入 API URL", + "fileList.aiTranslate.configModal.baseUrl.invalidUrl": "请输入有效的URL", + "fileList.aiTranslate.configModal.apiKey.label": "API 密钥", + "fileList.aiTranslate.configModal.apiKey.required": "请输入您的 API 密钥", + "fileList.aiTranslate.workingModal.content": "正在用LLM翻译 {fileCount} 个文件，关闭此对话框将中断翻译。", + "fileList.aiTranslate.fileMessage.sendingImage": "发送图片...", + "fileList.aiTranslate.fileMessage.uploadNotFinished": "此图片上传未完成，请稍后再试", + "fileList.aiTranslate.fileMessage.waiting": "排队中...", + "fileList.aiTranslate.fileMessage.failFetchingImage": "获取图片失败", + "fileList.aiTranslate.fileMessage.translating": "翻译中...", + "fileList.aiTranslate.fileMessage.translateFailed": "翻译失败", + "fileList.aiTranslate.fileMessage.textAlreadyExist": "已有文字标记，不再翻译", + "fileList.aiTranslate.fileMessage.noTextDetected": "未检测到文字", + "fileList.aiTranslate.fileMessage.saving": "保存中...", + "fileList.aiTranslate.fileMessage.failSaving": "保存失败", + "fileList.aiTranslate.fileMessage.success": "识别并翻译了 {count} 处文字", "project.startOCR": "开始自动标记", "project.startOCRTip": "您要开始图片自动标记吗？（仅自动标记“未开始”、“标记失败”的图片，且仅成功时扣减限额）", "site.ocrQuota": "自动标记限额", diff --git a/src/services/ai/TranslateCompanion.tsx b/src/services/ai/TranslateCompanion.tsx deleted file mode 100644 index 7688434..0000000 --- a/src/services/ai/TranslateCompanion.tsx +++ /dev/null @@ -1,320 +0,0 @@ -import { FC } from '@/interfaces'; -import { RefObject, useRef, useState } from 'react'; -import { FilePond } from 'react-filepond'; -import { css } from '@emotion/core'; -import { Button } from '@/components/shared/Button'; -import { createMoeflowProjectZip, LPFile } from '../labelplus_packager'; -import { FailureResults } from '@/apis'; -import { measureImgSize } from '@jokester/ts-commonutil/lib/web/measure-img'; -import { clamp } from 'lodash-es'; -import { BBox, mitPreprocess, TextQuad } from './mit_preprocess'; -import { ResourcePool } from '@jokester/ts-commonutil/lib/concurrency/resource-pool'; - -const MAX_FILE_COUNT = 30; - -function getQuadCenter(q: TextQuad) { - const xs = q.pts.flatMap((pt) => pt.map((p) => p[0])); - const ys = q.pts.flatMap((pt) => pt.map((p) => p[1])); - const minX = Math.min(...xs); - const maxX = Math.max(...xs); - const minY = Math.min(...ys); - const maxY = Math.max(...ys); - return { - x: (minX + maxX) / 2, - y: (minY + maxY) / 2, - }; -} - -function buildLpFile( - img: File, - size: { width: number; height: number }, - textQuads: TextQuad[], -): LPFile { - const labels = textQuads - .sort((a, b) => { - // sort : top=>bottom , right=>left - const ca = getQuadCenter(a); - const cb = getQuadCenter(b); - return Math.sign(ca.y - cb.y) || Math.sign(cb.x - ca.x); - }) - .map((q) => { - const { x, y } = getQuadCenter(q); - return { - x: clamp(x / size.width, 0, 1), - y: clamp(y / size.height, 0, 1), - position_type: 1, - translation: `${q.raw_text}\n${q.translated}`, - }; - }); - console.debug('labels', labels); - return { - file_name: img.name, - labels, - }; -} - -async function translateWithTask( - text: string, - targetLang = 'CHT', -): Promise { - const task = await mitPreprocess.createTranslateTask({ - query: text, - target_lang: targetLang, - translator: 'gpt4', - }); - const result = await mitPreprocess.waitTranslateTask(task.data.task_id); - return result[0] || ''; -} - -async function* startTranslateFile( - image: File, - running: RefObject, -): AsyncGenerator<{ - progress?: string; - failed?: FailureResults; - detectTextResult?: unknown; - ocrResult?: unknown; - translateResult?: unknown; - result?: LPFile; -}> { - let uploaded; - yield { progress: 'uploading' }; - try { - uploaded = await mitPreprocess.uploadImg(image); - } catch (e: unknown) { - yield { - failed: e as FailureResults, - }; - return; - } - yield { progress: 'extracting text lines' }; - const { filename } = uploaded.data; - - let detectTextResult; - try { - const task = await mitPreprocess.createImgTask( - filename, - 'mit_detect_text', - {}, - ); - detectTextResult = await mitPreprocess.waitImgTask<{ - textlines: { - prob: number; - pts: BBox[]; - text: string; - // textlines: any[]; // FIXME why did server return this? - }[]; - }>(task.data.task_id); - } catch (e: unknown) { - yield { - failed: e as FailureResults, - }; - return; - } - - yield { progress: 'recognizing text lines' }; - let ocrResult; - try { - const created = await mitPreprocess.createImgTask(filename, 'mit_ocr', { - regions: detectTextResult.textlines, - }); - ocrResult = await mitPreprocess.waitImgTask< - { - pts: BBox[]; - text: string; - textlines: string[]; - }[] - >(created.data.task_id); - console.debug('ocrResult', ocrResult); - } catch (e: unknown) { - yield { - failed: e as FailureResults, - }; - return; - } - - yield { progress: 'translating' }; - let translateResult: string[]; - try { - const limiter = ResourcePool.multiple([1, 2, 3, 4]); - translateResult = await Promise.all( - ocrResult.map((textBlock) => - limiter.use(() => translateWithTask(textBlock.text)), - ), - ); - } catch (e: unknown) { - yield { - failed: e as FailureResults, - }; - return; - } - - const textQuads: TextQuad[] = ocrResult.map((textBlock, i) => ({ - pts: textBlock.pts, - raw_text: textBlock.text, - translated: translateResult[i] ?? '', - })); - - const lpFile = buildLpFile(image, await measureImgSize(image), textQuads); - - yield { - result: lpFile, - }; -} - -async function translateFile(image: File, imageIndex: number): Promise { - try { - for await (const fileProgress of startTranslateFile(image, { - current: true, - })) { - console.debug( - `translating file #${imageIndex} / ${image.name}`, - 'step', - fileProgress, - ); - if (fileProgress.result) { - return fileProgress.result; - } else if (fileProgress.failed) { - throw fileProgress.failed; - } // else: continue - } - } catch (e) { - console.error(`failed translating file #${imageIndex} / ${image.name}`, e); - return { - file_name: image.name, - labels: [], - }; - } - throw new Error(`should not be here`); -} - -async function startOcr( - files: File[], - onProgress?: (finished: number, total: number) => void, -): Promise { - const limiter = ResourcePool.multiple([1, 2]); - - const translations = await Promise.all( - files.map((f, i) => - limiter.use(async () => { - const lpFile = await translateFile(f, i); - onProgress?.(i + 1, files.length); - return lpFile; - }), - ), - ); - const zipBlob = await createMoeflowProjectZip( - { - name: `${files[0]!.name}`, - intro: `这是由<萌翻+Mit demo>生成的项目. https://moeflow-mit-poc.voxscape.io/temp/mit-preprocess`, - default_role: 'supporter', - allow_apply_type: 3, - application_check_type: 1, - is_need_check_application: true, - source_language: 'ja', - output_language: 'zh-TW', - }, - translations.map((lp, i) => ({ lp, image: files[i] })), - ); - return new File( - [zipBlob], - `moeflow-project-${Date.now()}-${files[0]!.name}.zip`, - ); -} - -interface DemoWorkingState { - nonce: string; - numPages: number; - finished: number; -} - -export const DemoOcrFiles: FC<{}> = (props) => { - const [working, setWorking] = useState(null); - const [origFiles, setOrigFiles] = useState(() => []); - const [error, setError] = useState(null); - const [translated, setTranslated] = useState(null); - const filePondRef = useRef(null); - - const onStartOcr = async (files: File[]) => { - try { - const initState = { - nonce: `${Math.random()}`, - numPages: files.length, - finished: 0, - }; - setWorking(initState); - setTranslated( - await startOcr(files, (finished, total) => - setWorking((s) => - s?.nonce === initState.nonce - ? { - ...s, - finished: Math.max(s.finished, finished), - numPages: total, - } - : s, - ), - ), - ); - } catch (e: any) { - alert(e?.message || 'error'); - console.error(e); - } finally { - setWorking(null); - } - }; - return ( -
- 0} - ref={(value) => (filePondRef.current = value)} - css={css` - display: none; - `} - allowMultiple - acceptedFileTypes={['image/*', '.png', '.jpg']} - onupdatefiles={(_files) => { - const files = _files.map((f) => f.file) as File[]; - console.debug('onaddfile', files); - if (!(files.length > 0 && files.length <= MAX_FILE_COUNT)) { - setError(`一次最多只能上传${MAX_FILE_COUNT}张图片`); - setOrigFiles([]); - filePondRef.current!.removeFiles(); - } else { - setOrigFiles(files); - setError(null); - } - }} - /> - - - -
- ); -}; diff --git a/src/services/ai/llm_preprocess.ts b/src/services/ai/llm_preprocess.ts new file mode 100644 index 0000000..a5e5c90 --- /dev/null +++ b/src/services/ai/llm_preprocess.ts @@ -0,0 +1,155 @@ +import { z } from 'zod'; +import { + generateText, + GenerateTextOptions, + SystemMessage, + UserMessage, +} from 'xsai'; +import { tool } from '@xsai/tool'; +import { createDebugLogger } from '@/utils/debug-logger'; + +const debugLogger = createDebugLogger('services:ai:llm_preprocess'); + +export interface LLMConf { + provider: string; + model: string; + baseUrl: string; + apiKey?: string; + extraPrompt?: string; +} + +export const llmPresets: readonly Readonly[] = [ + // gemini: + // see https://ai.google.dev/gemini-api/docs/openai + ...['gemini-2.5-flash', 'gemini-2.5-pro'].map((model) => ({ + provider: 'Google', + model, + baseUrl: 'https://generativelanguage.googleapis.com/v1beta/openai/', + })), + // OpenAI models: see https://platform.openai.com/docs/models + ...['gpt-5-mini', 'gpt-4o', 'gpt-5'].map((model) => ({ + provider: 'OpenAI', + model, + baseUrl: 'https://api.openai.com/v1/', + })), + // Anthropic models in OpenAI compatible format: https://docs.claude.com/en/api/openai-sdk + ...['claude-sonnet-4-20250514', 'claude-3-7-sonnet-latest'].map((model) => ({ + provider: 'Anthropic', + model, + baseUrl: 'https://api.anthropic.com/v1/', + })), +]; + +const filePreprocessResultSchema = z.object({ + imageW: z.number({ message: 'the width of the image in PX' }), + imageH: z.number({ message: 'the height of the image in PX' }), + texts: z.array( + z.object({ + left: z + .number() + .describe('left coordinate of the text in PX, in the whole image'), + top: z + .number() + .describe('top coordinate of the text in PX, in the whole image'), + width: z.number().describe('width of the text in PX'), + height: z.number().describe('height of the text in PX'), + textLines: z.array(z.string()).describe('the text lines'), + text: z.string().describe('concatenated original text'), + translated: z.string().describe('translated text'), + comment: z + .string() + .describe('additional comment of the text, or the translation'), + }), + ), +}); + +export type FilePreprocessResult = z.infer; + +export async function llmTranslateImage( + llmConf: LLMConf, + targetLang: string, + imgBlob: Blob, + abortSignal?: AbortSignal, +): Promise { + const userMessage: UserMessage = { + role: 'user', + content: [ + { + type: 'text', + text: `Please translate the image to ${targetLang}. ${llmConf.extraPrompt || ''}`, + }, + { + type: 'image_url', + image_url: { + url: await img2dataurl(imgBlob), + detail: 'high', + }, + }, + ], + }; + + const messages: (UserMessage | SystemMessage)[] = [ + { + content: + 'You are a helpful assistant. Please do as user instructs. The extracted text and translations should be submitted using the provided tool.', + role: 'system', + }, + userMessage, + ]; + + let ret = await callModelWithTools(); + if (llmConf.model?.toLowerCase().includes('gemini-')) { + debugLogger('gemini workaround: set coords to 1000 scale'); + ret = { + ...ret, + // gemini-only workaround: gemini returns coords in [0, 1000] scale + // see https://ai.google.dev/gemini-api/docs/image-understanding + imageH: 1000, + imageW: 1000, + }; + } + return ret; + + // + async function callModelWithTools(): Promise { + let submittedResult: FilePreprocessResult | null = null; + const submitTool = await tool({ + execute: (_result) => { + submittedResult = _result; + return 'saved'; + }, + parameters: filePreprocessResultSchema, + name: 'submit', + description: 'Submit the result of preprocessing the image', + }); + + const generateConf: GenerateTextOptions = { + messages, + headers: { + // Anthropic-only workaround, to call API from browser (otherwise it rejects with CORS error). + ...(llmConf.model.toLowerCase().includes('claude-') && { + 'anthropic-dangerous-direct-browser-access': 'true', + }), + }, + tools: [submitTool], + baseURL: llmConf.baseUrl, + model: llmConf.model, + apiKey: llmConf.apiKey, + abortSignal, + }; + await generateText(generateConf); + + if (!submittedResult) { + throw new Error('LLM did not submit the result using the tool.'); + } + return submittedResult; + } +} + +async function img2dataurl(img: Blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result as string); + reader.readAsDataURL(img); + }); +} diff --git a/src/services/ai/mit_preprocess.ts b/src/services/ai/mit_preprocess.ts deleted file mode 100644 index f1574a5..0000000 --- a/src/services/ai/mit_preprocess.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { request } from '../../apis'; -import { uploadRequest } from '../../apis/_request'; -import { wait } from '@jokester/ts-commonutil/lib/concurrency/timing'; - -const mitApiPrefix = `/v1/mit`; - -export type CoordPair = [number, number]; // x, y in non-normalized pixels -export type BBox = [CoordPair, CoordPair, CoordPair, CoordPair]; // left-top, right-top, right-bottom, left-bottom - -export interface TextQuad { - pts: BBox[]; - raw_text: string; - translated: string; -} - -async function uploadImg(file: File) { - const formData = new FormData(); - formData.append('file', file); - - return uploadRequest<{ filename: string }>(formData, { - method: 'POST', - url: `${mitApiPrefix}/images`, - }); -} - -async function createImgTask( - filename: string, - taskName: 'mit_ocr' | 'mit_detect_text', - payload: object, -) { - return request<{ task_id: string }>({ - method: 'POST', - url: `${mitApiPrefix}/image-tasks`, - data: { - task_name: taskName, - filename, - ...payload, - }, - }); -} - -interface TaskState { - task_id: string; - status: 'success' | 'pending' | 'fail'; - result?: Result; - message?: string; -} - -async function waitImgTask(taskId: string) { - while (true) { - const r = await request>({ - method: 'GET', - url: `${mitApiPrefix}/image-tasks/${taskId}`, - }); - if (r.data.status === 'success') { - return r.data.result!; - } else if (r.data.status === 'pending') { - await wait(2e3); - } else { - throw new Error(`task failed: ${r.data.message ?? 'unknown'}`); - } - } -} - -async function createTranslateTask(payload: object) { - return request<{ task_id: string }>({ - method: 'POST', - url: `${mitApiPrefix}/translate-tasks`, - data: { - ...payload, - }, - }); -} - -async function waitTranslateTask(taskId: string) { - while (true) { - const r = await request>({ - method: 'GET', - url: `${mitApiPrefix}/translate-tasks/${taskId}`, - }); - if (r.data.status === 'success') { - return r.data.result!; - } else if (r.data.status === 'pending') { - await wait(1e3); - } else { - throw new Error(`task failed: ${r.data.message ?? 'unknown'}`); - } - } -} - -export const mitPreprocess = { - uploadImg, - createImgTask, - waitImgTask, - createTranslateTask, - waitTranslateTask, -} as const; diff --git a/src/services/ai/multimodal_recognize.ts b/src/services/ai/multimodal_recognize.ts deleted file mode 100644 index 46bf401..0000000 --- a/src/services/ai/multimodal_recognize.ts +++ /dev/null @@ -1,22 +0,0 @@ -interface MultimodalModelConf { - provider: string; - model: string; - baseUrl: string; -} - -export const multimodalPresets: readonly MultimodalModelConf[] = [ - // gemini: - // see https://ai.google.dev/gemini-api/docs/openai - { - provider: 'gemini', - model: 'gemini-2.5-flash', - baseUrl: 'https://generativelanguage.googleapis.com/v1beta/openai/', - }, - { - provider: 'gemini', - model: 'gemini-2.5-pro', - baseUrl: 'https://generativelanguage.googleapis.com/v1beta/openai/', - }, -]; - -export async function x(); diff --git a/src/services/ai/use_moeflow_companion.ts b/src/services/ai/use_moeflow_companion.ts deleted file mode 100644 index 582b798..0000000 --- a/src/services/ai/use_moeflow_companion.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { useState, useRef } from 'react'; -import { Client } from '@gradio/client'; -import { useAsyncEffect } from '@jokester/ts-commonutil/lib/react/hook/use-async-effect'; -import { useSelector } from 'react-redux'; -import { AppState } from '@/store'; -import { createDebugLogger } from '@/utils/debug-logger'; -import { RuntimeConfig } from '@/configs'; - -export const moeflowCompanionServiceState = { - disabled: 'disabled', - connecting: 'connecting', - connected: 'connected', - disconnected: 'disconnected', -} as const; - -const debugLogger = createDebugLogger('service:moeflow_companion'); - -export interface MoeflowCompanionService { - client: Client; - serviceConf: RuntimeConfig['moeflowCompanion']; - multimodalTranslate: typeof multimodalTranslate; -} - -export function useMoeflowCompanion(): [ - string, - MoeflowCompanionService | null, -] { - const serviceRef = useRef(null); - const [clientState, setClientState] = useState( - moeflowCompanionServiceState.connecting, - ); - const serviceConf = useSelector( - (s: AppState) => s.site.runtimeConfig.moeflowCompanion, - ); - - useAsyncEffect( - async (_, released) => { - if ( - !( - serviceConf && - serviceConf.gradioUrl && - serviceConf.defaultMultimodalModel - ) - ) { - serviceRef.current = null; - setClientState(moeflowCompanionServiceState.disabled); - return; - } - try { - const client = await Client.connect(serviceConf.gradioUrl); - serviceRef.current = { - client, - multimodalTranslate, - serviceConf, - }; - setClientState(moeflowCompanionServiceState.connected); - released.then(() => client.close()); - } catch (e) { - debugLogger('error connecting', e, serviceConf.gradioUrl); - serviceRef.current = null; - setClientState(moeflowCompanionServiceState.disconnected); - } - }, - [serviceConf], - ); - return [clientState, serviceRef.current] as const; -} - -async function multimodalTranslate( - client: Client, - files: Blob[], - targetLang: string, - model: string, -): Promise { - // const uploadRes = await client.upload_files(hfSpaceUrl, files) - // files.forEach(file => formData.append('files[]', file)); - // debugLogger('Upload response:', uploadRes); - const predictRes = await client.predict( - '/multimodal_llm_translate_file_api', - { - gradio_temp_files: files, // uploadRes.files!.map(handle_file), - model, - target_language: targetLang, - }, - ); - const [{ files: translated }] = predictRes.data as MoeflowMultimodalResData; - - debugLogger('Predict response:', predictRes, translated); - return translated; -} -export interface TranslatedFile { - local_path: string; - image_w: number; - image_h: number; - text_blocks: Array<{ - left: number; - top: number; - right: number; - bottom: number; - source: string; - translated: string; - }>; -} -/** - * the type in gradio https://github.com/moeflow-com/manga-image-translator/blob/moeflow-companion-main/moeflow_companion/gradio/multimodal.py#L62 - */ -type MoeflowMultimodalResData = [{ files: TranslatedFile[] }]; diff --git a/src/services/labelplus_packager.ts b/src/services/labelplus_packager.ts deleted file mode 100644 index 5da4389..0000000 --- a/src/services/labelplus_packager.ts +++ /dev/null @@ -1,80 +0,0 @@ -import * as zip from '@zip.js/zip.js'; - -export interface LPLabel { - x: number; // normalized - y: number; // normalized - position_type: number; // int , always 1 ? - translation: string; // singleline -} - -export interface LPFile { - file_name: string; // img filename (basename) - labels: LPLabel[]; -} - -function serializeIntoLabelplusFormat(files: LPFile[]): string[] { - return files.flatMap((file) => [ - `>>>>[${file.file_name}]<<<<`, - ...file.labels.flatMap((l, labelIndex) => [ - `----[${labelIndex}]----[${l.x},${l.y},${l.position_type}]`, - l.translation, - ]), - ]); -} - -type LANG_CODE = 'ja' | 'en' | 'zh-CN' | 'zh-TW'; - -interface MoeflowProjectMeta { - name: string; - intro: string; - default_role: 'supporter'; - allow_apply_type: 3; - application_check_type: 1; - is_need_check_application: boolean; - // create_time: string; - // edit_time: string; - source_language: 'ja'; - // target_languages: LANG_CODE[]; - // output_id: string; - output_language: LANG_CODE; -} - -export interface MoeflowImageFile { - lp: LPFile; - image: Blob; -} - -/** - * see moeflow-backend "TeamProjectImportAPI" - * @return a zip file for importing into moeflow-backend - */ -export async function createMoeflowProjectZip( - meta: MoeflowProjectMeta, - files: MoeflowImageFile[], -): Promise { - const zipWriter = new zip.ZipWriter(new zip.BlobWriter('application/zip'), { - bufferedWrite: true, - level: 9, - }); - - { - const translationsTxt = - serializeIntoLabelplusFormat(files.map((f) => f.lp)).join('\n') + '\n'; - const blob = new Blob([translationsTxt], { type: 'text/plain' }); - await zipWriter.add('translations.txt', new zip.BlobReader(blob)); - } - - for (const f of files) { - await zipWriter.add( - `images/${f.lp.file_name}`, - new zip.BlobReader(f.image), - ); - } - - await zipWriter.add( - 'project.json', - new zip.TextReader(JSON.stringify(meta, null, 2)), - ); - - return zipWriter.close(); -} diff --git a/src/store/user/sagas.ts b/src/store/user/sagas.ts index 105a062..45b64e6 100644 --- a/src/store/user/sagas.ts +++ b/src/store/user/sagas.ts @@ -11,6 +11,12 @@ function* getUserInfoAsync(action: ReturnType) { const token = action.payload.token; const instance: Axios = yield api.getAxiosInstance(); if (token === '') { + if (import.meta.env.DEV) { + // do nothing in dev: vite hot reloading may create APIClient multiple times, + // causing 401 and an empty token being set + console.debug('[user/sagas] Ignored empty token in DEV due to HMR'); + return; + } // 清除 Axios Authorization 头 delete instance.defaults.headers.common['Authorization']; // 清除 Cookie token diff --git a/src/utils/storage.ts b/src/utils/storage.ts index 33c5018..4735f0c 100644 --- a/src/utils/storage.ts +++ b/src/utils/storage.ts @@ -1,6 +1,7 @@ import store from 'store'; import { HotKeyOption } from '../components/HotKey/interfaces'; import { HotKeyState } from '../store/hotKey/slice'; +import { LLMConf } from '@/services/ai/llm_preprocess'; interface DefaultTarget { projectID: string; @@ -75,3 +76,16 @@ export const loadHotKey = ({ ); return options[index] ? options[index] : null; }; + +export const llmConfStorage = { + load(): LLMConf | null { + return store.get('llmConf', null); + }, + save(conf: LLMConf | null) { + if (conf) { + store.set('llmConf', conf); + } else { + store.remove('llmConf'); + } + }, +} as const;