Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion controller/relay.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,16 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) {
contains, words := service.CheckSensitiveText(meta.CombineText)
if contains {
logger.LogWarn(c, fmt.Sprintf("user sensitive words detected: %s", strings.Join(words, ", ")))
newAPIError = types.NewError(err, types.ErrorCodeSensitiveWordsDetected)
if service.WriteSensitiveRefusal(c, relayFormat, relayInfo, request) {
return
}
// fallback: unsupported format, return explicit 400 refusal
newAPIError = types.NewErrorWithStatusCode(
errors.New(service.GetSensitiveRefusalText()),
types.ErrorCodeSensitiveWordsDetected,
http.StatusBadRequest,
types.ErrOptionWithSkipRetry(),
)
return
}
}
Expand Down
3 changes: 3 additions & 0 deletions model/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ func InitOptionMap() {
common.OptionMap["CheckSensitiveOnPromptEnabled"] = strconv.FormatBool(setting.CheckSensitiveOnPromptEnabled)
common.OptionMap["StopOnSensitiveEnabled"] = strconv.FormatBool(setting.StopOnSensitiveEnabled)
common.OptionMap["SensitiveWords"] = setting.SensitiveWordsToString()
common.OptionMap["SensitiveRefusalText"] = setting.SensitiveRefusalText
common.OptionMap["StreamCacheQueueLength"] = strconv.Itoa(setting.StreamCacheQueueLength)
common.OptionMap["AutomaticDisableKeywords"] = operation_setting.AutomaticDisableKeywordsToString()
common.OptionMap["AutomaticDisableStatusCodes"] = operation_setting.AutomaticDisableStatusCodesToString()
Expand Down Expand Up @@ -629,6 +630,8 @@ func updateOptionMap(key string, value string) (err error) {
common.QuotaPerUnit, _ = strconv.ParseFloat(value, 64)
case "SensitiveWords":
setting.SensitiveWordsFromString(value)
case "SensitiveRefusalText":
setting.SensitiveRefusalText = value
case "AutomaticDisableKeywords":
operation_setting.AutomaticDisableKeywordsFromString(value)
case "AutomaticDisableStatusCodes":
Expand Down
223 changes: 223 additions & 0 deletions service/sensitive_refusal.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
package service

import (
"fmt"
"net/http"
"time"

"github.com/gin-gonic/gin"

"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/dto"
relaycommon "github.com/QuantumNous/new-api/relay/common"
relayhelper "github.com/QuantumNous/new-api/relay/helper"
"github.com/QuantumNous/new-api/setting"
"github.com/QuantumNous/new-api/types"
)

const defaultSensitiveRefusalText = "很抱歉,你输入的内容含有敏感信息,我无法回答。"

func GetSensitiveRefusalText() string {
if setting.SensitiveRefusalText != "" {
return setting.SensitiveRefusalText
}
return defaultSensitiveRefusalText
}

// WriteSensitiveRefusal writes a well-formed refusal response (200 OK) when sensitive
// words are detected. Returns true when the response was written successfully; false
// means the caller should fall back to returning a 400 error instead.
func WriteSensitiveRefusal(c *gin.Context, relayFormat types.RelayFormat, relayInfo *relaycommon.RelayInfo, request dto.Request) bool {
text := GetSensitiveRefusalText()

switch relayFormat {
case types.RelayFormatOpenAI:
// Only chat/completions requests have Messages; /v1/completions (Prompt) and
// /v1/moderations (Input) share the same RelayFormat but have different schemas,
// so fall back to 400 for those.
r, ok := request.(*dto.GeneralOpenAIRequest)
if !ok || len(r.Messages) == 0 {
return false
}
if relayInfo != nil && relayInfo.IsStream {
return writeOpenAIStreamRefusal(c, text, relayInfo)
}
return writeOpenAIRefusal(c, text, relayInfo)

case types.RelayFormatClaude:
if relayInfo != nil && relayInfo.IsStream {
return writeClaudeStreamRefusal(c, text, relayInfo)
}
return writeClaudeRefusal(c, text, relayInfo)

default:
// RelayFormatOpenAIResponses, RelayFormatOpenAIResponsesCompaction,
// RelayFormatGemini, RelayFormatEmbedding, RelayFormatRerank,
// RelayFormatOpenAIAudio, RelayFormatOpenAIImage,
// RelayFormatTask, RelayFormatMjProxy, RelayFormatOpenAIRealtime
// — these have endpoint-specific schemas; fall back to 400 error.
return false
}
}

func writeOpenAIRefusal(c *gin.Context, text string, relayInfo *relaycommon.RelayInfo) bool {
model := ""
if relayInfo != nil {
model = relayInfo.OriginModelName
}
resp := dto.OpenAITextResponse{
Id: fmt.Sprintf("chatcmpl-sensitive-%d", time.Now().UnixNano()),
Object: "chat.completion",
Created: time.Now().Unix(),
Model: model,
Choices: []dto.OpenAITextResponseChoice{
{
Index: 0,
Message: dto.Message{
Role: "assistant",
Content: text,
},
FinishReason: "stop",
},
},
Usage: dto.Usage{},
}
c.JSON(http.StatusOK, resp)
return true
}

func writeOpenAIStreamRefusal(c *gin.Context, text string, relayInfo *relaycommon.RelayInfo) bool {
model := ""
if relayInfo != nil {
model = relayInfo.OriginModelName
}
id := fmt.Sprintf("chatcmpl-sensitive-%d", time.Now().UnixNano())
created := time.Now().Unix()
stop := "stop"

content := text
delta := dto.ChatCompletionsStreamResponse{
Id: id,
Object: "chat.completion.chunk",
Created: created,
Model: model,
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Index: 0,
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
Role: "assistant",
Content: &content,
},
},
},
}
finishChunk := dto.ChatCompletionsStreamResponse{
Id: id,
Object: "chat.completion.chunk",
Created: created,
Model: model,
Choices: []dto.ChatCompletionsStreamResponseChoice{
{
Index: 0,
Delta: dto.ChatCompletionsStreamResponseChoiceDelta{},
FinishReason: &stop,
},
},
}

deltaJson, err1 := common.Marshal(delta)
finishJson, err2 := common.Marshal(finishChunk)
if err1 != nil || err2 != nil {
return false
}

c.Writer.Header().Set("Content-Type", "text/event-stream")
c.Writer.Header().Set("Cache-Control", "no-cache")
c.Writer.Header().Set("X-Accel-Buffering", "no")
c.Writer.WriteHeader(http.StatusOK)

c.Render(-1, common.CustomEvent{Data: "data: " + string(deltaJson)})
c.Render(-1, common.CustomEvent{Data: "data: " + string(finishJson)})
c.Render(-1, common.CustomEvent{Data: "data: [DONE]"})
_ = relayhelper.FlushWriter(c)
return true
}

func writeClaudeRefusal(c *gin.Context, text string, relayInfo *relaycommon.RelayInfo) bool {
model := ""
if relayInfo != nil {
model = relayInfo.OriginModelName
}
textCopy := text
resp := dto.ClaudeResponse{
Id: fmt.Sprintf("msg_sensitive_%d", time.Now().UnixNano()),
Type: "message",
Role: "assistant",
Model: model,
Content: []dto.ClaudeMediaMessage{
{Type: "text", Text: &textCopy},
},
StopReason: "end_turn",
Usage: &dto.ClaudeUsage{},
}
c.JSON(http.StatusOK, resp)
return true
}

func writeClaudeStreamRefusal(c *gin.Context, text string, relayInfo *relaycommon.RelayInfo) bool {
model := ""
if relayInfo != nil {
model = relayInfo.OriginModelName
}
msgId := fmt.Sprintf("msg_sensitive_%d", time.Now().UnixNano())
textCopy := text
index := 0
emptyStr := ""
stopReason := "end_turn"

events := []dto.ClaudeResponse{
{
Type: "message_start",
Message: &dto.ClaudeMediaMessage{
Id: msgId,
Type: "message",
Role: "assistant",
Model: model,
Usage: &dto.ClaudeUsage{},
},
},
{
Type: "content_block_start",
Index: &index,
ContentBlock: &dto.ClaudeMediaMessage{Type: "text", Text: &emptyStr},
},
{
Type: "content_block_delta",
Index: &index,
Delta: &dto.ClaudeMediaMessage{Type: "text_delta", Text: &textCopy},
},
{
Type: "content_block_stop",
Index: &index,
},
{
// stop_reason goes inside delta per Anthropic SSE spec
Type: "message_delta",
Delta: &dto.ClaudeMediaMessage{StopReason: &stopReason},
Usage: &dto.ClaudeUsage{},
},
{
Type: "message_stop",
},
}

c.Writer.Header().Set("Content-Type", "text/event-stream")
c.Writer.Header().Set("Cache-Control", "no-cache")
c.Writer.Header().Set("X-Accel-Buffering", "no")
c.Writer.WriteHeader(http.StatusOK)

for _, ev := range events {
_ = relayhelper.ClaudeData(c, ev)
}
return true
}
3 changes: 3 additions & 0 deletions setting/sensitive.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ var StopOnSensitiveEnabled = true
// StreamCacheQueueLength 流模式缓存队列长度,0表示无缓存
var StreamCacheQueueLength = 0

// SensitiveRefusalText 敏感词命中时返回给用户的拒绝文案,空则使用默认值
var SensitiveRefusalText = ""

// SensitiveWords 敏感词
// var SensitiveWords []string
var SensitiveWords = []string{
Expand Down
1 change: 1 addition & 0 deletions web/classic/src/components/settings/OperationSetting.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ const OperationSetting = () => {
CheckSensitiveEnabled: false,
CheckSensitiveOnPromptEnabled: false,
SensitiveWords: '',
SensitiveRefusalText: '',

/* 日志设置 */
LogConsumeEnabled: false,
Expand Down
5 changes: 3 additions & 2 deletions web/classic/src/i18n/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -4038,6 +4038,7 @@
"该记录未上传企业认证图片": "This record has no enterprise certification images",
"确认重置该认证?用户认证状态将清空,需重新提交所有信息(含营业执照与法人证件图片)。": "Reset this certification? The user's status will be cleared and all info (including the business license and legal-rep ID images) must be resubmitted.",
"企业认证状态": "Enterprise Certification Status",
"企业认证审核管理": "Enterprise certification review management"
"企业认证审核管理": "Enterprise certification review management",
"留空则使用内置默认拒绝文案": "Leave blank to use the built-in default refusal message"
}
}
}
5 changes: 3 additions & 2 deletions web/classic/src/i18n/locales/fr.json
Original file line number Diff line number Diff line change
Expand Up @@ -3851,6 +3851,7 @@
"该记录未上传企业认证图片": "Cet enregistrement n'a pas d'images de certification d'entreprise",
"确认重置该认证?用户认证状态将清空,需重新提交所有信息(含营业执照与法人证件图片)。": "Réinitialiser cette certification ? Le statut de l'utilisateur sera effacé et toutes les informations (y compris la licence commerciale et les pièces d'identité du représentant légal) devront être resoumises.",
"企业认证状态": "Statut de certification d'entreprise",
"企业认证审核管理": "Gestion de l'examen de la certification d'entreprise"
"企业认证审核管理": "Gestion de l'examen de la certification d'entreprise",
"留空则使用内置默认拒绝文案": "Laisser vide pour utiliser le message de refus par défaut intégré"
}
}
}
5 changes: 3 additions & 2 deletions web/classic/src/i18n/locales/ja.json
Original file line number Diff line number Diff line change
Expand Up @@ -3820,6 +3820,7 @@
"该记录未上传企业认证图片": "このレコードには企業認証画像がありません",
"确认重置该认证?用户认证状态将清空,需重新提交所有信息(含营业执照与法人证件图片)。": "この認証をリセットしますか?ユーザーの状態がクリアされ、すべての情報(営業許可証および法人代表者の身分証画像を含む)を再提出する必要があります。",
"企业认证状态": "企業認証ステータス",
"企业认证审核管理": "企業認証審査管理"
"企业认证审核管理": "企業認証審査管理",
"留空则使用内置默认拒绝文案": "空白にするとデフォルトの拒否メッセージを使用します"
}
}
}
5 changes: 3 additions & 2 deletions web/classic/src/i18n/locales/ru.json
Original file line number Diff line number Diff line change
Expand Up @@ -3871,6 +3871,7 @@
"该记录未上传企业认证图片": "К этой записи не прикреплены изображения корпоративной верификации",
"确认重置该认证?用户认证状态将清空,需重新提交所有信息(含营业执照与法人证件图片)。": "Сбросить эту верификацию? Статус пользователя будет очищен, и все данные (включая лицензию и удостоверение представителя) нужно будет отправить заново.",
"企业认证状态": "Статус корпоративной верификации",
"企业认证审核管理": "Управление проверкой корпоративной верификации"
"企业认证审核管理": "Управление проверкой корпоративной верификации",
"留空则使用内置默认拒绝文案": "Оставьте пустым для использования встроенного сообщения об отказе по умолчанию"
}
}
}
5 changes: 3 additions & 2 deletions web/classic/src/i18n/locales/vi.json
Original file line number Diff line number Diff line change
Expand Up @@ -4379,6 +4379,7 @@
"该记录未上传企业认证图片": "Bản ghi này chưa tải lên hình ảnh xác minh doanh nghiệp",
"确认重置该认证?用户认证状态将清空,需重新提交所有信息(含营业执照与法人证件图片)。": "Đặt lại xác minh này? Trạng thái người dùng sẽ bị xóa và phải gửi lại toàn bộ thông tin (bao gồm giấy phép kinh doanh và ảnh CMND/CCCD người đại diện).",
"企业认证状态": "Trạng thái xác minh doanh nghiệp",
"企业认证审核管理": "Quản lý xét duyệt xác minh doanh nghiệp"
"企业认证审核管理": "Quản lý xét duyệt xác minh doanh nghiệp",
"留空则使用内置默认拒绝文案": "Để trống để sử dụng thông báo từ chối mặc định tích hợp"
}
}
}
1 change: 1 addition & 0 deletions web/classic/src/i18n/locales/zh-CN.json
Original file line number Diff line number Diff line change
Expand Up @@ -1599,6 +1599,7 @@
"拉取模型": "拉取模型",
"拉取进度": "拉取进度",
"拒绝提示模板(可选)": "拒绝提示模板(可选)",
"留空则使用内置默认拒绝文案": "留空则使用内置默认拒绝文案",
"拦截原因": "拦截原因",
"按K显示单位": "按K显示单位",
"按价格设置": "按价格设置",
Expand Down
5 changes: 3 additions & 2 deletions web/classic/src/i18n/locales/zh-TW.json
Original file line number Diff line number Diff line change
Expand Up @@ -3844,6 +3844,7 @@
"该记录未上传企业认证图片": "該記錄未上傳企業認證圖片",
"确认重置该认证?用户认证状态将清空,需重新提交所有信息(含营业执照与法人证件图片)。": "確認重置該認證?使用者認證狀態將清空,需重新提交所有資訊(含營業執照與法人證件圖片)。",
"企业认证状态": "企業認證狀態",
"企业认证审核管理": "企業認證審核管理"
"企业认证审核管理": "企業認證審核管理",
"留空则使用内置默认拒绝文案": ""
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export default function SettingsSensitiveWords(props) {
CheckSensitiveEnabled: false,
CheckSensitiveOnPromptEnabled: false,
SensitiveWords: '',
SensitiveRefusalText: '',
});
const refForm = useRef();
const [inputsRow, setInputsRow] = useState(inputs);
Expand Down Expand Up @@ -144,6 +145,23 @@ export default function SettingsSensitiveWords(props) {
/>
</Col>
</Row>
<Row>
<Col xs={24} sm={12} md={8} lg={8} xl={8}>
<Form.TextArea
label={t('拒绝提示模板(可选)')}
extraText={t('留空则使用内置默认拒绝文案')}
placeholder={t('留空则使用内置默认拒绝文案')}
field={'SensitiveRefusalText'}
onChange={(value) =>
setInputs({
...inputs,
SensitiveRefusalText: value,
})
}
autosize={{ minRows: 2, maxRows: 6 }}
/>
</Col>
</Row>
<Row>
<Button size='default' onClick={onSubmit}>
{t('保存屏蔽词过滤设置')}
Expand Down