diff --git a/.gitignore b/.gitignore index 7308b9ee3..61a1340af 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,4 @@ docker-compose.local.yml .DS_Store .idea .pnpm-store +.worktrees/ diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 203a88184..660041080 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -1360,6 +1360,287 @@ const docTemplate = `{ } } }, + "/admin/content-moderation/config": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Get content moderation config", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationConfigResponseDoc" + } + } + } + }, + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Update content moderation config", + "parameters": [ + { + "description": "Content moderation configuration", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/ContentModerationUpdateConfigRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationConfigUpdateResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/events": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "List content moderation events", + "parameters": [ + { + "type": "integer", + "description": "Page number", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size", + "name": "pageSize", + "in": "query" + }, + { + "type": "string", + "description": "Result filter", + "name": "result", + "in": "query" + }, + { + "type": "string", + "description": "Direction filter", + "name": "direction", + "in": "query" + }, + { + "type": "string", + "description": "Modality filter", + "name": "modality", + "in": "query" + }, + { + "type": "string", + "description": "Category filter", + "name": "category", + "in": "query" + }, + { + "type": "integer", + "description": "User ID", + "name": "userId", + "in": "query" + }, + { + "type": "string", + "description": "Run ID", + "name": "runId", + "in": "query" + }, + { + "type": "string", + "description": "Start time (RFC3339)", + "name": "from", + "in": "query" + }, + { + "type": "string", + "description": "End time (RFC3339)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationEventListResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/events/{eventID}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Get content moderation event detail", + "parameters": [ + { + "type": "string", + "description": "Moderation event ID", + "name": "eventID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationEventDetailResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/events/{eventID}/images/{index}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/octet-stream" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Stream a isolated moderation image", + "parameters": [ + { + "type": "string", + "description": "Moderation event ID", + "name": "eventID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Image index", + "name": "index", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + } + } + } + }, + "/admin/content-moderation/probe": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Probe content moderation service", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationProbeResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/stats": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Get content moderation daily stats", + "parameters": [ + { + "type": "string", + "description": "Start time (RFC3339)", + "name": "from", + "in": "query" + }, + { + "type": "string", + "description": "End time (RFC3339)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationStatsResponseDoc" + } + } + } + } + }, "/admin/conversation-events": { "get": { "security": [ @@ -13934,37 +14215,592 @@ const docTemplate = `{ } } }, - "ContextArtifactResponse": { + "ContentModerationCategoryCatalogResponse": { "type": "object", "required": [ - "content", - "createdAt", - "id", - "kind", - "messageID", - "metadataJSON", - "runID", - "score", - "sourceID", - "sourceTitle", - "sourceType", - "tokenEstimate" + "image", + "text" ], "properties": { - "content": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "expiresAt": { - "type": "string" - }, - "id": { - "type": "integer" - }, - "kind": { - "type": "string" + "image": { + "type": "array", + "items": { + "type": "string" + } + }, + "text": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ContentModerationConfigDataResponse": { + "type": "object", + "required": [ + "categories", + "config" + ], + "properties": { + "categories": { + "$ref": "#/definitions/ContentModerationCategoryCatalogResponse" + }, + "config": { + "$ref": "#/definitions/ContentModerationServiceConfigResponse" + } + } + }, + "ContentModerationConfigResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationConfigDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationConfigUpdateDataResponse": { + "type": "object", + "required": [ + "config" + ], + "properties": { + "config": { + "$ref": "#/definitions/ContentModerationServiceConfigResponse" + } + } + }, + "ContentModerationConfigUpdateResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationConfigUpdateDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationDailyStatResponse": { + "type": "object", + "required": [ + "category", + "checkCount", + "contentItems", + "direction", + "failureCount", + "hitCount", + "latencyCount", + "latencySumMS", + "modality", + "result", + "statDate" + ], + "properties": { + "category": { + "type": "string" + }, + "checkCount": { + "type": "integer" + }, + "contentItems": { + "type": "integer" + }, + "direction": { + "type": "string" + }, + "failureCount": { + "type": "integer" + }, + "hitCount": { + "type": "integer" + }, + "latencyCount": { + "type": "integer" + }, + "latencySumMS": { + "type": "integer" + }, + "modality": { + "type": "string" + }, + "result": { + "type": "string" + }, + "statDate": { + "type": "string" + } + } + }, + "ContentModerationEventDetailResponse": { + "type": "object", + "required": [ + "categoryScores", + "event", + "images", + "imagesAvailable", + "textAvailable" + ], + "properties": { + "categoryScores": { + "type": "object", + "additionalProperties": { + "type": "number", + "format": "float64" + } + }, + "decryptedText": { + "type": "string" + }, + "event": { + "$ref": "#/definitions/ContentModerationEventResponse" + }, + "images": { + "type": "array", + "items": { + "$ref": "#/definitions/ContentModerationIsolatedImageResponse" + } + }, + "imagesAvailable": { + "type": "boolean" + }, + "textAvailable": { + "type": "boolean" + } + } + }, + "ContentModerationEventDetailResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationEventDetailResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationEventListDataResponse": { + "type": "object", + "required": [ + "items", + "page", + "pageSize", + "total" + ], + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/ContentModerationEventResponse" + } + }, + "page": { + "type": "integer" + }, + "pageSize": { + "type": "integer" + }, + "total": { + "type": "integer" + } + } + }, + "ContentModerationEventListResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationEventListDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationEventResponse": { + "type": "object", + "required": [ + "categories", + "contentSummary", + "conversationID", + "createdAt", + "direction", + "errorCode", + "errorMessage", + "latencyMS", + "messagePublicID", + "modality", + "model", + "policyVersion", + "publicID", + "result", + "runID", + "userID" + ], + "properties": { + "categories": { + "type": "array", + "items": { + "type": "string" + } + }, + "contentSummary": { + "type": "string" + }, + "conversationID": { + "type": "integer" + }, + "createdAt": { + "type": "string" + }, + "direction": { + "type": "string" + }, + "errorCode": { + "type": "string" + }, + "errorMessage": { + "type": "string" + }, + "latencyMS": { + "type": "integer" + }, + "messagePublicID": { + "type": "string" + }, + "modality": { + "type": "string" + }, + "model": { + "type": "string" + }, + "policyVersion": { + "type": "integer" + }, + "publicID": { + "type": "string" + }, + "result": { + "type": "string" + }, + "runID": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "userLabel": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "ContentModerationIsolatedImageResponse": { + "type": "object", + "required": [ + "index", + "mimeType", + "sha256", + "sizeBytes" + ], + "properties": { + "index": { + "type": "integer" + }, + "mimeType": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "sizeBytes": { + "type": "integer" + }, + "sourceFileID": { + "type": "string" + } + } + }, + "ContentModerationPolicyRequest": { + "type": "object", + "required": [ + "inputImageCategories", + "inputTextCategories", + "outputImageCategories", + "outputTextCategories" + ], + "properties": { + "inputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "inputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ContentModerationPolicyResponse": { + "type": "object", + "required": [ + "inputImageCategories", + "inputTextCategories", + "outputImageCategories", + "outputTextCategories", + "version" + ], + "properties": { + "inputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "inputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "version": { + "type": "integer" + } + } + }, + "ContentModerationProbeResponse": { + "type": "object", + "required": [ + "image", + "text" + ], + "properties": { + "image": { + "$ref": "#/definitions/ContentModerationProbeResultResponse" + }, + "text": { + "$ref": "#/definitions/ContentModerationProbeResultResponse" + } + } + }, + "ContentModerationProbeResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationProbeResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationProbeResultResponse": { + "type": "object", + "required": [ + "latencyMS", + "valid" + ], + "properties": { + "error": { + "type": "string" + }, + "latencyMS": { + "type": "integer" + }, + "model": { + "type": "string" + }, + "valid": { + "type": "boolean" + } + } + }, + "ContentModerationServiceConfigResponse": { + "type": "object", + "required": [ + "baseUrl", + "enabled", + "hasAPIKey", + "maxConcurrency", + "model", + "policy", + "queueCapacity", + "timeoutSeconds" + ], + "properties": { + "apiKeyMasked": { + "type": "string" + }, + "baseUrl": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hasAPIKey": { + "type": "boolean" + }, + "maxConcurrency": { + "type": "integer" + }, + "model": { + "type": "string" + }, + "policy": { + "$ref": "#/definitions/ContentModerationPolicyResponse" + }, + "queueCapacity": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } + } + }, + "ContentModerationStatsDataResponse": { + "type": "object", + "required": [ + "items" + ], + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/ContentModerationDailyStatResponse" + } + } + } + }, + "ContentModerationStatsResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationStatsDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationUpdateConfigRequest": { + "type": "object", + "properties": { + "apiKey": { + "type": "string" + }, + "baseUrl": { + "type": "string" + }, + "clearAPIKey": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "maxConcurrency": { + "type": "integer" + }, + "model": { + "type": "string" + }, + "policy": { + "$ref": "#/definitions/ContentModerationPolicyRequest" + }, + "queueCapacity": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } + } + }, + "ContextArtifactResponse": { + "type": "object", + "required": [ + "content", + "createdAt", + "id", + "kind", + "messageID", + "metadataJSON", + "runID", + "score", + "sourceID", + "sourceTitle", + "sourceType", + "tokenEstimate" + ], + "properties": { + "content": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "expiresAt": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "kind": { + "type": "string" }, "messageID": { "type": "integer" @@ -16715,6 +17551,26 @@ const docTemplate = `{ } } }, + "MessageModerationResponse": { + "type": "object", + "properties": { + "categories": { + "type": "array", + "items": { + "type": "string" + } + }, + "direction": { + "type": "string" + }, + "eventID": { + "type": "string" + }, + "state": { + "type": "string" + } + } + }, "MessageProcessTraceResponse": { "type": "object", "required": [ @@ -16949,6 +17805,9 @@ const docTemplate = `{ "modelVendor": { "type": "string" }, + "moderation": { + "$ref": "#/definitions/MessageModerationResponse" + }, "myFeedback": { "type": "string" }, diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index c35f19c51..f6f6888b2 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -1353,6 +1353,287 @@ } } }, + "/admin/content-moderation/config": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Get content moderation config", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationConfigResponseDoc" + } + } + } + }, + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Update content moderation config", + "parameters": [ + { + "description": "Content moderation configuration", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/ContentModerationUpdateConfigRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationConfigUpdateResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/events": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "List content moderation events", + "parameters": [ + { + "type": "integer", + "description": "Page number", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Page size", + "name": "pageSize", + "in": "query" + }, + { + "type": "string", + "description": "Result filter", + "name": "result", + "in": "query" + }, + { + "type": "string", + "description": "Direction filter", + "name": "direction", + "in": "query" + }, + { + "type": "string", + "description": "Modality filter", + "name": "modality", + "in": "query" + }, + { + "type": "string", + "description": "Category filter", + "name": "category", + "in": "query" + }, + { + "type": "integer", + "description": "User ID", + "name": "userId", + "in": "query" + }, + { + "type": "string", + "description": "Run ID", + "name": "runId", + "in": "query" + }, + { + "type": "string", + "description": "Start time (RFC3339)", + "name": "from", + "in": "query" + }, + { + "type": "string", + "description": "End time (RFC3339)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationEventListResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/events/{eventID}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Get content moderation event detail", + "parameters": [ + { + "type": "string", + "description": "Moderation event ID", + "name": "eventID", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationEventDetailResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/events/{eventID}/images/{index}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/octet-stream" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Stream a isolated moderation image", + "parameters": [ + { + "type": "string", + "description": "Moderation event ID", + "name": "eventID", + "in": "path", + "required": true + }, + { + "type": "integer", + "description": "Image index", + "name": "index", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + } + } + } + }, + "/admin/content-moderation/probe": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Probe content moderation service", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationProbeResponseDoc" + } + } + } + } + }, + "/admin/content-moderation/stats": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "admin-content-moderation" + ], + "summary": "Get content moderation daily stats", + "parameters": [ + { + "type": "string", + "description": "Start time (RFC3339)", + "name": "from", + "in": "query" + }, + { + "type": "string", + "description": "End time (RFC3339)", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/ContentModerationStatsResponseDoc" + } + } + } + } + }, "/admin/conversation-events": { "get": { "security": [ @@ -13927,37 +14208,592 @@ } } }, - "ContextArtifactResponse": { + "ContentModerationCategoryCatalogResponse": { "type": "object", "required": [ - "content", - "createdAt", - "id", - "kind", - "messageID", - "metadataJSON", - "runID", - "score", - "sourceID", - "sourceTitle", - "sourceType", - "tokenEstimate" + "image", + "text" ], "properties": { - "content": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "expiresAt": { - "type": "string" - }, - "id": { - "type": "integer" - }, - "kind": { - "type": "string" + "image": { + "type": "array", + "items": { + "type": "string" + } + }, + "text": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ContentModerationConfigDataResponse": { + "type": "object", + "required": [ + "categories", + "config" + ], + "properties": { + "categories": { + "$ref": "#/definitions/ContentModerationCategoryCatalogResponse" + }, + "config": { + "$ref": "#/definitions/ContentModerationServiceConfigResponse" + } + } + }, + "ContentModerationConfigResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationConfigDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationConfigUpdateDataResponse": { + "type": "object", + "required": [ + "config" + ], + "properties": { + "config": { + "$ref": "#/definitions/ContentModerationServiceConfigResponse" + } + } + }, + "ContentModerationConfigUpdateResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationConfigUpdateDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationDailyStatResponse": { + "type": "object", + "required": [ + "category", + "checkCount", + "contentItems", + "direction", + "failureCount", + "hitCount", + "latencyCount", + "latencySumMS", + "modality", + "result", + "statDate" + ], + "properties": { + "category": { + "type": "string" + }, + "checkCount": { + "type": "integer" + }, + "contentItems": { + "type": "integer" + }, + "direction": { + "type": "string" + }, + "failureCount": { + "type": "integer" + }, + "hitCount": { + "type": "integer" + }, + "latencyCount": { + "type": "integer" + }, + "latencySumMS": { + "type": "integer" + }, + "modality": { + "type": "string" + }, + "result": { + "type": "string" + }, + "statDate": { + "type": "string" + } + } + }, + "ContentModerationEventDetailResponse": { + "type": "object", + "required": [ + "categoryScores", + "event", + "images", + "imagesAvailable", + "textAvailable" + ], + "properties": { + "categoryScores": { + "type": "object", + "additionalProperties": { + "type": "number", + "format": "float64" + } + }, + "decryptedText": { + "type": "string" + }, + "event": { + "$ref": "#/definitions/ContentModerationEventResponse" + }, + "images": { + "type": "array", + "items": { + "$ref": "#/definitions/ContentModerationIsolatedImageResponse" + } + }, + "imagesAvailable": { + "type": "boolean" + }, + "textAvailable": { + "type": "boolean" + } + } + }, + "ContentModerationEventDetailResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationEventDetailResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationEventListDataResponse": { + "type": "object", + "required": [ + "items", + "page", + "pageSize", + "total" + ], + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/ContentModerationEventResponse" + } + }, + "page": { + "type": "integer" + }, + "pageSize": { + "type": "integer" + }, + "total": { + "type": "integer" + } + } + }, + "ContentModerationEventListResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationEventListDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationEventResponse": { + "type": "object", + "required": [ + "categories", + "contentSummary", + "conversationID", + "createdAt", + "direction", + "errorCode", + "errorMessage", + "latencyMS", + "messagePublicID", + "modality", + "model", + "policyVersion", + "publicID", + "result", + "runID", + "userID" + ], + "properties": { + "categories": { + "type": "array", + "items": { + "type": "string" + } + }, + "contentSummary": { + "type": "string" + }, + "conversationID": { + "type": "integer" + }, + "createdAt": { + "type": "string" + }, + "direction": { + "type": "string" + }, + "errorCode": { + "type": "string" + }, + "errorMessage": { + "type": "string" + }, + "latencyMS": { + "type": "integer" + }, + "messagePublicID": { + "type": "string" + }, + "modality": { + "type": "string" + }, + "model": { + "type": "string" + }, + "policyVersion": { + "type": "integer" + }, + "publicID": { + "type": "string" + }, + "result": { + "type": "string" + }, + "runID": { + "type": "string" + }, + "userID": { + "type": "integer" + }, + "userLabel": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "ContentModerationIsolatedImageResponse": { + "type": "object", + "required": [ + "index", + "mimeType", + "sha256", + "sizeBytes" + ], + "properties": { + "index": { + "type": "integer" + }, + "mimeType": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "sizeBytes": { + "type": "integer" + }, + "sourceFileID": { + "type": "string" + } + } + }, + "ContentModerationPolicyRequest": { + "type": "object", + "required": [ + "inputImageCategories", + "inputTextCategories", + "outputImageCategories", + "outputTextCategories" + ], + "properties": { + "inputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "inputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ContentModerationPolicyResponse": { + "type": "object", + "required": [ + "inputImageCategories", + "inputTextCategories", + "outputImageCategories", + "outputTextCategories", + "version" + ], + "properties": { + "inputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "inputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputImageCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputTextCategories": { + "type": "array", + "items": { + "type": "string" + } + }, + "version": { + "type": "integer" + } + } + }, + "ContentModerationProbeResponse": { + "type": "object", + "required": [ + "image", + "text" + ], + "properties": { + "image": { + "$ref": "#/definitions/ContentModerationProbeResultResponse" + }, + "text": { + "$ref": "#/definitions/ContentModerationProbeResultResponse" + } + } + }, + "ContentModerationProbeResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationProbeResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationProbeResultResponse": { + "type": "object", + "required": [ + "latencyMS", + "valid" + ], + "properties": { + "error": { + "type": "string" + }, + "latencyMS": { + "type": "integer" + }, + "model": { + "type": "string" + }, + "valid": { + "type": "boolean" + } + } + }, + "ContentModerationServiceConfigResponse": { + "type": "object", + "required": [ + "baseUrl", + "enabled", + "hasAPIKey", + "maxConcurrency", + "model", + "policy", + "queueCapacity", + "timeoutSeconds" + ], + "properties": { + "apiKeyMasked": { + "type": "string" + }, + "baseUrl": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hasAPIKey": { + "type": "boolean" + }, + "maxConcurrency": { + "type": "integer" + }, + "model": { + "type": "string" + }, + "policy": { + "$ref": "#/definitions/ContentModerationPolicyResponse" + }, + "queueCapacity": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } + } + }, + "ContentModerationStatsDataResponse": { + "type": "object", + "required": [ + "items" + ], + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/ContentModerationDailyStatResponse" + } + } + } + }, + "ContentModerationStatsResponseDoc": { + "type": "object", + "required": [ + "data", + "errorMsg" + ], + "properties": { + "data": { + "$ref": "#/definitions/ContentModerationStatsDataResponse" + }, + "errorMsg": { + "type": "string" + } + } + }, + "ContentModerationUpdateConfigRequest": { + "type": "object", + "properties": { + "apiKey": { + "type": "string" + }, + "baseUrl": { + "type": "string" + }, + "clearAPIKey": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "maxConcurrency": { + "type": "integer" + }, + "model": { + "type": "string" + }, + "policy": { + "$ref": "#/definitions/ContentModerationPolicyRequest" + }, + "queueCapacity": { + "type": "integer" + }, + "timeoutSeconds": { + "type": "integer" + } + } + }, + "ContextArtifactResponse": { + "type": "object", + "required": [ + "content", + "createdAt", + "id", + "kind", + "messageID", + "metadataJSON", + "runID", + "score", + "sourceID", + "sourceTitle", + "sourceType", + "tokenEstimate" + ], + "properties": { + "content": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "expiresAt": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "kind": { + "type": "string" }, "messageID": { "type": "integer" @@ -16708,6 +17544,26 @@ } } }, + "MessageModerationResponse": { + "type": "object", + "properties": { + "categories": { + "type": "array", + "items": { + "type": "string" + } + }, + "direction": { + "type": "string" + }, + "eventID": { + "type": "string" + }, + "state": { + "type": "string" + } + } + }, "MessageProcessTraceResponse": { "type": "object", "required": [ @@ -16942,6 +17798,9 @@ "modelVendor": { "type": "string" }, + "moderation": { + "$ref": "#/definitions/MessageModerationResponse" + }, "myFeedback": { "type": "string" }, diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index bb8df980e..bf2efa89a 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -1436,6 +1436,389 @@ definitions: - data - errorMsg type: object + ContentModerationCategoryCatalogResponse: + properties: + image: + items: + type: string + type: array + text: + items: + type: string + type: array + required: + - image + - text + type: object + ContentModerationConfigDataResponse: + properties: + categories: + $ref: '#/definitions/ContentModerationCategoryCatalogResponse' + config: + $ref: '#/definitions/ContentModerationServiceConfigResponse' + required: + - categories + - config + type: object + ContentModerationConfigResponseDoc: + properties: + data: + $ref: '#/definitions/ContentModerationConfigDataResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object + ContentModerationConfigUpdateDataResponse: + properties: + config: + $ref: '#/definitions/ContentModerationServiceConfigResponse' + required: + - config + type: object + ContentModerationConfigUpdateResponseDoc: + properties: + data: + $ref: '#/definitions/ContentModerationConfigUpdateDataResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object + ContentModerationDailyStatResponse: + properties: + category: + type: string + checkCount: + type: integer + contentItems: + type: integer + direction: + type: string + failureCount: + type: integer + hitCount: + type: integer + latencyCount: + type: integer + latencySumMS: + type: integer + modality: + type: string + result: + type: string + statDate: + type: string + required: + - category + - checkCount + - contentItems + - direction + - failureCount + - hitCount + - latencyCount + - latencySumMS + - modality + - result + - statDate + type: object + ContentModerationEventDetailResponse: + properties: + categoryScores: + additionalProperties: + format: float64 + type: number + type: object + decryptedText: + type: string + event: + $ref: '#/definitions/ContentModerationEventResponse' + images: + items: + $ref: '#/definitions/ContentModerationIsolatedImageResponse' + type: array + imagesAvailable: + type: boolean + textAvailable: + type: boolean + required: + - categoryScores + - event + - images + - imagesAvailable + - textAvailable + type: object + ContentModerationEventDetailResponseDoc: + properties: + data: + $ref: '#/definitions/ContentModerationEventDetailResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object + ContentModerationEventListDataResponse: + properties: + items: + items: + $ref: '#/definitions/ContentModerationEventResponse' + type: array + page: + type: integer + pageSize: + type: integer + total: + type: integer + required: + - items + - page + - pageSize + - total + type: object + ContentModerationEventListResponseDoc: + properties: + data: + $ref: '#/definitions/ContentModerationEventListDataResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object + ContentModerationEventResponse: + properties: + categories: + items: + type: string + type: array + contentSummary: + type: string + conversationID: + type: integer + createdAt: + type: string + direction: + type: string + errorCode: + type: string + errorMessage: + type: string + latencyMS: + type: integer + messagePublicID: + type: string + modality: + type: string + model: + type: string + policyVersion: + type: integer + publicID: + type: string + result: + type: string + runID: + type: string + userID: + type: integer + userLabel: + type: string + username: + type: string + required: + - categories + - contentSummary + - conversationID + - createdAt + - direction + - errorCode + - errorMessage + - latencyMS + - messagePublicID + - modality + - model + - policyVersion + - publicID + - result + - runID + - userID + type: object + ContentModerationIsolatedImageResponse: + properties: + index: + type: integer + mimeType: + type: string + sha256: + type: string + sizeBytes: + type: integer + sourceFileID: + type: string + required: + - index + - mimeType + - sha256 + - sizeBytes + type: object + ContentModerationPolicyRequest: + properties: + inputImageCategories: + items: + type: string + type: array + inputTextCategories: + items: + type: string + type: array + outputImageCategories: + items: + type: string + type: array + outputTextCategories: + items: + type: string + type: array + required: + - inputImageCategories + - inputTextCategories + - outputImageCategories + - outputTextCategories + type: object + ContentModerationPolicyResponse: + properties: + inputImageCategories: + items: + type: string + type: array + inputTextCategories: + items: + type: string + type: array + outputImageCategories: + items: + type: string + type: array + outputTextCategories: + items: + type: string + type: array + version: + type: integer + required: + - inputImageCategories + - inputTextCategories + - outputImageCategories + - outputTextCategories + - version + type: object + ContentModerationProbeResponse: + properties: + image: + $ref: '#/definitions/ContentModerationProbeResultResponse' + text: + $ref: '#/definitions/ContentModerationProbeResultResponse' + required: + - image + - text + type: object + ContentModerationProbeResponseDoc: + properties: + data: + $ref: '#/definitions/ContentModerationProbeResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object + ContentModerationProbeResultResponse: + properties: + error: + type: string + latencyMS: + type: integer + model: + type: string + valid: + type: boolean + required: + - latencyMS + - valid + type: object + ContentModerationServiceConfigResponse: + properties: + apiKeyMasked: + type: string + baseUrl: + type: string + enabled: + type: boolean + hasAPIKey: + type: boolean + maxConcurrency: + type: integer + model: + type: string + policy: + $ref: '#/definitions/ContentModerationPolicyResponse' + queueCapacity: + type: integer + timeoutSeconds: + type: integer + required: + - baseUrl + - enabled + - hasAPIKey + - maxConcurrency + - model + - policy + - queueCapacity + - timeoutSeconds + type: object + ContentModerationStatsDataResponse: + properties: + items: + items: + $ref: '#/definitions/ContentModerationDailyStatResponse' + type: array + required: + - items + type: object + ContentModerationStatsResponseDoc: + properties: + data: + $ref: '#/definitions/ContentModerationStatsDataResponse' + errorMsg: + type: string + required: + - data + - errorMsg + type: object + ContentModerationUpdateConfigRequest: + properties: + apiKey: + type: string + baseUrl: + type: string + clearAPIKey: + type: boolean + enabled: + type: boolean + maxConcurrency: + type: integer + model: + type: string + policy: + $ref: '#/definitions/ContentModerationPolicyRequest' + queueCapacity: + type: integer + timeoutSeconds: + type: integer + type: object ContextArtifactResponse: properties: content: @@ -3392,6 +3775,19 @@ definitions: - data - errorMsg type: object + MessageModerationResponse: + properties: + categories: + items: + type: string + type: array + direction: + type: string + eventID: + type: string + state: + type: string + type: object MessageProcessTraceResponse: properties: enabled: @@ -3529,6 +3925,8 @@ definitions: type: string modelVendor: type: string + moderation: + $ref: '#/definitions/MessageModerationResponse' myFeedback: type: string outputTokens: @@ -9146,6 +9544,179 @@ paths: summary: 管理员查询模型调用日志 tags: - admin + /admin/content-moderation/config: + get: + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/ContentModerationConfigResponseDoc' + security: + - BearerAuth: [] + summary: Get content moderation config + tags: + - admin-content-moderation + put: + consumes: + - application/json + parameters: + - description: Content moderation configuration + in: body + name: body + required: true + schema: + $ref: '#/definitions/ContentModerationUpdateConfigRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/ContentModerationConfigUpdateResponseDoc' + security: + - BearerAuth: [] + summary: Update content moderation config + tags: + - admin-content-moderation + /admin/content-moderation/events: + get: + parameters: + - description: Page number + in: query + name: page + type: integer + - description: Page size + in: query + name: pageSize + type: integer + - description: Result filter + in: query + name: result + type: string + - description: Direction filter + in: query + name: direction + type: string + - description: Modality filter + in: query + name: modality + type: string + - description: Category filter + in: query + name: category + type: string + - description: User ID + in: query + name: userId + type: integer + - description: Run ID + in: query + name: runId + type: string + - description: Start time (RFC3339) + in: query + name: from + type: string + - description: End time (RFC3339) + in: query + name: to + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/ContentModerationEventListResponseDoc' + security: + - BearerAuth: [] + summary: List content moderation events + tags: + - admin-content-moderation + /admin/content-moderation/events/{eventID}: + get: + parameters: + - description: Moderation event ID + in: path + name: eventID + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/ContentModerationEventDetailResponseDoc' + security: + - BearerAuth: [] + summary: Get content moderation event detail + tags: + - admin-content-moderation + /admin/content-moderation/events/{eventID}/images/{index}: + get: + parameters: + - description: Moderation event ID + in: path + name: eventID + required: true + type: string + - description: Image index + in: path + name: index + required: true + type: integer + produces: + - application/octet-stream + responses: + "200": + description: OK + schema: + type: file + security: + - BearerAuth: [] + summary: Stream a isolated moderation image + tags: + - admin-content-moderation + /admin/content-moderation/probe: + post: + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/ContentModerationProbeResponseDoc' + security: + - BearerAuth: [] + summary: Probe content moderation service + tags: + - admin-content-moderation + /admin/content-moderation/stats: + get: + parameters: + - description: Start time (RFC3339) + in: query + name: from + type: string + - description: End time (RFC3339) + in: query + name: to + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/ContentModerationStatsResponseDoc' + security: + - BearerAuth: [] + summary: Get content moderation daily stats + tags: + - admin-content-moderation /admin/conversation-events: get: consumes: diff --git a/backend/internal/app/app.go b/backend/internal/app/app.go index fd763d53f..e6d324b29 100644 --- a/backend/internal/app/app.go +++ b/backend/internal/app/app.go @@ -16,6 +16,7 @@ import ( "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/billing" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/channel" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/compact" + appcontentmoderation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/conversation" appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/extraction" @@ -33,6 +34,7 @@ import ( "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/user" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/usersettings" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + moderationclient "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/contentmoderation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/embedding" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/geoip" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/identityprovider" @@ -49,6 +51,7 @@ import ( auditrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/audit" billingrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/billing" channelrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/channel" + contentmoderationrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/contentmoderation" conversationrepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/conversation" logcleanuprepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/logcleanup" mcprepo "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/postgres/mcp" @@ -66,6 +69,7 @@ import ( authhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/auth" billinghttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/billing" channelhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/channel" + contentmoderationhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/contentmoderation" conversationhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/conversation" mcphttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/mcp" memoryhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/memory" @@ -93,6 +97,7 @@ type App struct { mcpClient *mcp.Client embeddingClient *embedding.Client mediaArtifactClient *mediaartifact.Client + moderationClient *moderationclient.Client backgroundCancel context.CancelFunc } @@ -293,6 +298,14 @@ func NewApp() (*App, error) { conversationService.SetAuditWriter(auditService) conversationService.SetObjectStoreProvider(objectStoreProvider) conversationService.SetMCPRepository(mcpRepo) + contentModerationRepo := contentmoderationrepo.NewRepo(db) + contentModerationService := appcontentmoderation.NewService(settingsRepo, contentModerationRepo, cfg.DataEncryptionKey, log) + moderationClient := moderationclient.New(trustedOutboundPolicy) + contentModerationService.SetProvider(moderationClient) + contentModerationService.SetAuditWriter(auditService) + conversationService.SetModerationService(contentModerationService) + contentModerationHandler := contentmoderationhttp.NewHandler(contentModerationService) + contentModerationModule := contentmoderationhttp.NewModule(contentModerationHandler) userService.SetAvatarContentOpener(avatarContentOpener{conversationService: conversationService}) userService.SetAvatarFileValidator(conversationService) authService.SetAvatarFileValidator(conversationService) @@ -321,6 +334,7 @@ func NewApp() (*App, error) { adminHandler := adminhttp.NewHandler(adminService) adminHandler.SetConversationExporter(conversationService) adminModule := adminhttp.NewModule(adminHandler) + contentModerationHandler.SetUserLabelResolver(adminService) userSettingsRepo := usersettingsrepo.NewRepo(db) userSettingsService := usersettings.NewService(userSettingsRepo) userSettingsHandler := usersettingshttp.NewHandler(userSettingsService) @@ -344,20 +358,21 @@ func NewApp() (*App, error) { hc := newHealthChecker(db, cfg.CacheDriver, redisClient) rateLimiter := buildRateLimiter(cfg, redisClient, memoryCache) engine, err := platformhttp.NewEngine(runtimeCfg, log, platformhttp.Modules{ - Auth: authModule, - AuthService: authService, - Channel: channelModule, - Conversation: conversationModule, - MCP: mcpModule, - Memory: memoryModule, - Billing: billingModule, - Admin: adminModule, - Announcement: announcementModule, - PromptPreset: promptPresetModule, - Skill: skillModule, - Settings: settingsModule, - UserSettings: userSettingsModule, - User: userModule, + Auth: authModule, + AuthService: authService, + Channel: channelModule, + Conversation: conversationModule, + MCP: mcpModule, + Memory: memoryModule, + Billing: billingModule, + Admin: adminModule, + ContentModeration: contentModerationModule, + Announcement: announcementModule, + PromptPreset: promptPresetModule, + Skill: skillModule, + Settings: settingsModule, + UserSettings: userSettingsModule, + User: userModule, StartupLog: func(log *zap.Logger) { if log == nil || bootstrapSuperAdmin == nil { return @@ -374,6 +389,7 @@ func NewApp() (*App, error) { backgroundCtx, backgroundCancel := context.WithCancel(context.Background()) conversationService.StartBackgroundWorkers(backgroundCtx) + contentModerationService.StartBackgroundWorkers(backgroundCtx) return &App{ cfg: runtimeCfg.Snapshot(), @@ -387,6 +403,7 @@ func NewApp() (*App, error) { mcpClient: mcpClient, embeddingClient: embedClient, mediaArtifactClient: mediaArtifactClient, + moderationClient: moderationClient, backgroundCancel: backgroundCancel, }, nil } @@ -475,6 +492,9 @@ func (a *App) Close() { if a.mediaArtifactClient != nil { a.mediaArtifactClient.CloseIdleConnections() } + if a.moderationClient != nil { + a.moderationClient.CloseIdleConnections() + } if a.db != nil { if sqlDB, err := a.db.DB(); err == nil { _ = sqlDB.Close() diff --git a/backend/internal/application/contentmoderation/admin_api.go b/backend/internal/application/contentmoderation/admin_api.go new file mode 100644 index 000000000..3d3d40f84 --- /dev/null +++ b/backend/internal/application/contentmoderation/admin_api.go @@ -0,0 +1,201 @@ +package contentmoderation + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +// StatsFilter bounds the admin stats query. +type StatsFilter struct { + From *time.Time + To *time.Time +} + +// EventListInput is the super-admin events query. +type EventListInput struct { + Direction string + Modality string + Result string + Category string + UserID uint + RunID string + From *time.Time + To *time.Time + Page int + PageSize int +} + +// EventDetail is the super-admin detail payload (may include decrypted text). +type EventDetail struct { + Event domaincm.Event + CategoryScores map[string]float64 + DecryptedText string + TextAvailable bool + ImagesAvailable bool + Images []domaincm.IsolatedImageMeta +} + +// GetStats returns anonymous aggregates for the last 90 days (admin+). +func (s *Service) GetStats(ctx context.Context, actorRole string, filter StatsFilter) ([]domaincm.DailyStat, error) { + if !isAdminRole(actorRole) { + return nil, ErrAdminRequired + } + now := time.Now().UTC() + to := now + from := now.Add(-metadataRetention) + if filter.To != nil && !filter.To.IsZero() { + to = filter.To.UTC() + } + if filter.From != nil && !filter.From.IsZero() { + from = filter.From.UTC() + } + minFrom := now.Add(-metadataRetention) + if from.Before(minFrom) { + from = minFrom + } + if to.Before(from) { + return nil, ErrInvalidEventFilter + } + return s.repo.ListDailyStats(ctx, from, to) +} + +// ListEvents lists hit/fail metadata for super-admin. +func (s *Service) ListEvents(ctx context.Context, actorRole string, input EventListInput) ([]domaincm.Event, int64, error) { + if !isSuperAdmin(actorRole) { + return nil, 0, ErrSuperAdminRequired + } + direction := strings.TrimSpace(input.Direction) + if direction != "" && direction != domaincm.DirectionInput && direction != domaincm.DirectionOutput { + return nil, 0, ErrInvalidEventFilter + } + modality := strings.TrimSpace(input.Modality) + if modality != "" && modality != domaincm.ModalityText && modality != domaincm.ModalityImage { + return nil, 0, ErrInvalidEventFilter + } + result := strings.TrimSpace(input.Result) + if result != "" && result != domaincm.ResultHit && result != domaincm.ResultFailedOpen && result != domaincm.ResultPassed { + return nil, 0, ErrInvalidEventFilter + } + category := strings.TrimSpace(input.Category) + if category != "" && !IsKnownCategory(category) { + return nil, 0, ErrInvalidEventFilter + } + if input.From != nil && input.To != nil && input.To.Before(*input.From) { + return nil, 0, ErrInvalidEventFilter + } + page := input.Page + if page < 1 { + page = 1 + } + pageSize := input.PageSize + if pageSize < 1 { + pageSize = 20 + } + if pageSize > 100 { + pageSize = 100 + } + return s.repo.ListEvents(ctx, domaincm.EventListFilter{ + Direction: direction, + Modality: modality, + Result: result, + Category: category, + UserID: input.UserID, + RunID: strings.TrimSpace(input.RunID), + From: input.From, + To: input.To, + Offset: (page - 1) * pageSize, + Limit: pageSize, + }) +} + +// GetEventDetail returns decrypted text when still retained. +func (s *Service) GetEventDetail( + ctx context.Context, + actorRole string, + eventID string, +) (*EventDetail, error) { + if !isSuperAdmin(actorRole) { + return nil, ErrSuperAdminRequired + } + event, err := s.repo.GetEventByPublicID(ctx, strings.TrimSpace(eventID)) + if errors.Is(err, repository.ErrNotFound) || (err == nil && event == nil) { + return nil, ErrEventNotFound + } + if err != nil { + return nil, err + } + detail := &EventDetail{Event: *event} + _ = json.Unmarshal([]byte(event.CategoryScoresJSON), &detail.CategoryScores) + detail.Images = unmarshalIsolatedImageMetadata(event.ImageMetaJSON) + + if event.Result == domaincm.ResultHit && + event.Modality == domaincm.ModalityText && + strings.TrimSpace(event.EncryptedText) != "" && + time.Now().Before(event.ContentExpiresAt) { + if plain, decErr := s.decryptText(event.EncryptedText); decErr == nil { + detail.DecryptedText = plain + detail.TextAvailable = true + } + } + if event.Modality == domaincm.ModalityImage && len(detail.Images) > 0 && time.Now().Before(event.ContentExpiresAt) { + detail.ImagesAvailable = true + } + return detail, nil +} + +// OpenEventImage decrypts an isolated image for super-admin streaming. +func (s *Service) OpenEventImage( + ctx context.Context, + actorRole string, + eventID string, + index int, +) (data []byte, mimeType string, err error) { + if !isSuperAdmin(actorRole) { + return nil, "", ErrSuperAdminRequired + } + event, err := s.repo.GetEventByPublicID(ctx, strings.TrimSpace(eventID)) + if errors.Is(err, repository.ErrNotFound) || (err == nil && event == nil) { + return nil, "", ErrEventNotFound + } + if err != nil { + return nil, "", err + } + if time.Now().After(event.ContentExpiresAt) { + return nil, "", ErrEventNotFound + } + images := unmarshalIsolatedImageMetadata(event.ImageMetaJSON) + var meta *domaincm.IsolatedImageMeta + for i := range images { + if images[i].Index == index { + meta = &images[i] + break + } + } + if meta == nil || s.objectStore == nil { + return nil, "", ErrEventNotFound + } + raw, err := s.objectStore.Open(ctx, meta.StoragePath) + if err != nil { + return nil, "", err + } + // Isolated images are stored as encryptBytes payloads (v1: base64), not UTF-8 text. + plain, err := s.decryptBytes(string(raw)) + if err != nil { + return nil, "", err + } + return plain, firstNonEmpty(meta.MimeType, "image/png"), nil +} + +// CategoryCatalog returns category lists for the admin UI. +func CategoryCatalog() map[string][]string { + return map[string][]string{ + "text": AllTextCategories(), + "image": ImageCategories(), + } +} diff --git a/backend/internal/application/contentmoderation/admin_api_test.go b/backend/internal/application/contentmoderation/admin_api_test.go new file mode 100644 index 000000000..e236f3e1a --- /dev/null +++ b/backend/internal/application/contentmoderation/admin_api_test.go @@ -0,0 +1,77 @@ +package contentmoderation + +import ( + "context" + "errors" + "testing" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +func TestGetEventDetailOnlyMapsRepositoryNotFound(t *testing.T) { + storageErr := errors.New("database unavailable") + tests := []struct { + name string + repo *coordinatorTestRepo + wantError error + }{ + { + name: "not found", + repo: &coordinatorTestRepo{getEventErr: repository.ErrNotFound}, + wantError: ErrEventNotFound, + }, + { + name: "storage failure", + repo: &coordinatorTestRepo{getEventErr: storageErr}, + wantError: storageErr, + }, + { + name: "event", + repo: &coordinatorTestRepo{getEvent: &domaincm.Event{ + PublicID: "cme_1", + CategoriesJSON: "[]", + CategoryScoresJSON: "{}", + }}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + service := NewService(nil, test.repo, "test-key", nil) + detail, err := service.GetEventDetail(context.Background(), "superadmin", "cme_1") + if test.wantError != nil { + if !errors.Is(err, test.wantError) { + t.Fatalf("error = %v, want %v", err, test.wantError) + } + return + } + if err != nil || detail == nil { + t.Fatalf("detail = %#v, error = %v", detail, err) + } + }) + } +} + +func TestOpenEventImagePreservesRepositoryFailure(t *testing.T) { + storageErr := errors.New("database unavailable") + service := NewService(nil, &coordinatorTestRepo{getEventErr: storageErr}, "test-key", nil) + _, _, err := service.OpenEventImage(context.Background(), "superadmin", "cme_1", 0) + if !errors.Is(err, storageErr) { + t.Fatalf("error = %v, want storage error", err) + } +} + +func TestListEventsRejectsInvalidFilters(t *testing.T) { + service := NewService(nil, &coordinatorTestRepo{}, "test-key", nil) + tests := []EventListInput{ + {Direction: "sideways"}, + {Modality: "video"}, + {Result: "blocked"}, + {Category: "%"}, + } + for _, input := range tests { + if _, _, err := service.ListEvents(context.Background(), "superadmin", input); !errors.Is(err, ErrInvalidEventFilter) { + t.Fatalf("input %#v returned %v, want invalid filter", input, err) + } + } +} diff --git a/backend/internal/application/contentmoderation/audit.go b/backend/internal/application/contentmoderation/audit.go new file mode 100644 index 000000000..a902de7bc --- /dev/null +++ b/backend/internal/application/contentmoderation/audit.go @@ -0,0 +1,39 @@ +package contentmoderation + +import ( + "context" + "strings" +) + +type auditWriter interface { + Write(ctx context.Context, requestID string, actorUserID uint, action string, resource string, resourceID string, ip string, userAgent string, detail interface{}) +} + +// ReviewAuditInput contains request metadata for a privileged retained-content read. +type ReviewAuditInput struct { + ActorUserID uint + RequestID string + Action string + EventID string + ClientIP string + UserAgent string + Detail interface{} +} + +// RecordReviewAudit records which administrator viewed retained moderation content. +func (s *Service) RecordReviewAudit(ctx context.Context, input ReviewAuditInput) { + if s == nil || s.auditWriter == nil { + return + } + s.auditWriter.Write( + ctx, + strings.TrimSpace(input.RequestID), + input.ActorUserID, + strings.TrimSpace(input.Action), + "content_moderation_event", + strings.TrimSpace(input.EventID), + strings.TrimSpace(input.ClientIP), + strings.TrimSpace(input.UserAgent), + input.Detail, + ) +} diff --git a/backend/internal/application/contentmoderation/audit_test.go b/backend/internal/application/contentmoderation/audit_test.go new file mode 100644 index 000000000..08d0c4ed6 --- /dev/null +++ b/backend/internal/application/contentmoderation/audit_test.go @@ -0,0 +1,55 @@ +package contentmoderation + +import ( + "context" + "testing" +) + +type reviewAuditCall struct { + requestID string + actorUserID uint + action string + resource string + resourceID string +} + +type reviewAuditWriter struct { + call reviewAuditCall +} + +func (writer *reviewAuditWriter) Write( + _ context.Context, + requestID string, + actorUserID uint, + action string, + resource string, + resourceID string, + _ string, + _ string, + _ interface{}, +) { + writer.call = reviewAuditCall{ + requestID: requestID, + actorUserID: actorUserID, + action: action, + resource: resource, + resourceID: resourceID, + } +} + +func TestRecordReviewAuditIdentifiesActorAndEvent(t *testing.T) { + writer := &reviewAuditWriter{} + service := NewService(nil, nil, "", nil) + service.SetAuditWriter(writer) + service.RecordReviewAudit(context.Background(), ReviewAuditInput{ + ActorUserID: 42, + RequestID: " req_1 ", + Action: " content_moderation.event.view ", + EventID: " cme_1 ", + }) + if writer.call.actorUserID != 42 || writer.call.requestID != "req_1" || + writer.call.action != "content_moderation.event.view" || + writer.call.resource != "content_moderation_event" || writer.call.resourceID != "cme_1" { + t.Fatalf("unexpected audit call: %#v", writer.call) + } +} diff --git a/backend/internal/application/contentmoderation/categories.go b/backend/internal/application/contentmoderation/categories.go new file mode 100644 index 000000000..6b16d19e4 --- /dev/null +++ b/backend/internal/application/contentmoderation/categories.go @@ -0,0 +1,85 @@ +package contentmoderation + +import domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + +// Direction / modality aliases for application code. +const ( + DirectionInput = domaincm.DirectionInput + DirectionOutput = domaincm.DirectionOutput + ModalityText = domaincm.ModalityText + ModalityImage = domaincm.ModalityImage +) + +// Official Omni Moderation categories (13 total). +var allTextCategories = []string{ + "hate", + "hate/threatening", + "harassment", + "harassment/threatening", + "self-harm", + "self-harm/intent", + "self-harm/instructions", + "sexual", + "sexual/minors", + "violence", + "violence/graphic", + "illicit", + "illicit/violent", +} + +// Image-applicable categories (excludes official text-only categories). +var imageCategories = []string{ + "self-harm", + "self-harm/intent", + "self-harm/instructions", + "sexual", + "violence", + "violence/graphic", +} + +// textOnlyCategories are not shown/configurable on image policy pages. +var textOnlyCategories = map[string]struct{}{ + "hate": {}, + "hate/threatening": {}, + "harassment": {}, + "harassment/threatening": {}, + "sexual/minors": {}, + "illicit": {}, + "illicit/violent": {}, +} + +// AllTextCategories returns the full official set for admin UI defaults. +func AllTextCategories() []string { + return append([]string(nil), allTextCategories...) +} + +// ImageCategories returns categories valid for image moderation. +func ImageCategories() []string { + return append([]string(nil), imageCategories...) +} + +// IsTextOnlyCategory reports whether a category is text-only. +func IsTextOnlyCategory(category string) bool { + _, ok := textOnlyCategories[category] + return ok +} + +// IsKnownCategory reports whether category is in the official list. +func IsKnownCategory(category string) bool { + for _, item := range allTextCategories { + if item == category { + return true + } + } + return false +} + +// IsImageCategory reports whether category applies to images. +func IsImageCategory(category string) bool { + for _, item := range imageCategories { + if item == category { + return true + } + } + return false +} diff --git a/backend/internal/application/contentmoderation/cleanup.go b/backend/internal/application/contentmoderation/cleanup.go new file mode 100644 index 000000000..7631c8dfb --- /dev/null +++ b/backend/internal/application/contentmoderation/cleanup.go @@ -0,0 +1,255 @@ +package contentmoderation + +import ( + "context" + "encoding/json" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "go.uber.org/zap" +) + +const blockRecoveryInterval = 10 * time.Second + +func (s *Service) cleanupLoop(ctx context.Context) { + defer s.wg.Done() + cleanupTicker := time.NewTicker(cleanupInterval) + recoveryTicker := time.NewTicker(blockRecoveryInterval) + defer cleanupTicker.Stop() + defer recoveryTicker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-s.stopCh: + return + case <-cleanupTicker.C: + bg, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + s.runCleanup(bg) + s.recoverPendingBlocks(bg) + s.recoverStaleRuns(bg) + cancel() + case <-recoveryTicker.C: + bg, cancel := context.WithTimeout(context.Background(), 45*time.Second) + s.recoverPendingBlocks(bg) + s.recoverStaleRuns(bg) + s.retryBlockedGeneratedFileDeletes(bg, 200) + cancel() + } + } +} + +func (s *Service) runCleanup(ctx context.Context) { + if s.repo == nil { + return + } + now := time.Now() + // Only clear metadata for events whose isolated objects were deleted successfully. + // Loop until no more expired content rows remain (or a safety cap is hit). + for pass := 0; pass < 50; pass++ { + events, err := s.repo.ListExpiredContentEvents(ctx, now, 200) + if err != nil { + s.logWarn("content_moderation_list_expired_content_failed", zap.Error(err)) + break + } + if len(events) == 0 { + break + } + clearedIDs := make([]string, 0, len(events)) + for _, event := range events { + // Text-only hits have no isolated images; always clearable after expiry. + // Image hits require successful object delete first. + if s.deleteIsolatedImages(ctx, event) { + clearedIDs = append(clearedIDs, event.PublicID) + } + } + if len(clearedIDs) > 0 { + if n, err := s.repo.ClearExpiredContentByPublicIDs(ctx, clearedIDs); err != nil { + s.logWarn("content_moderation_clear_content_failed", zap.Error(err)) + } else if n > 0 { + s.logWarn("content_moderation_content_cleared", zap.Int64("count", n)) + } + } + // If nothing could be cleared this pass (all deletes failed), stop to retry later. + if len(clearedIDs) == 0 { + break + } + } + if n, err := s.repo.DeleteExpiredMetadata(ctx, now); err != nil { + s.logWarn("content_moderation_delete_metadata_failed", zap.Error(err)) + } else if n > 0 { + s.logWarn("content_moderation_metadata_deleted", zap.Int64("count", n)) + } + cutoff := now.Add(-metadataRetention) + if n, err := s.repo.DeleteDailyStatsBefore(ctx, cutoff); err != nil { + s.logWarn("content_moderation_delete_stats_failed", zap.Error(err)) + } else if n > 0 { + s.logWarn("content_moderation_stats_deleted", zap.Int64("count", n)) + } + s.retryBlockedGeneratedFileDeletes(ctx, 200) +} + +func (s *Service) retryBlockedGeneratedFileDeletes(ctx context.Context, limit int) { + if s.fileAccess == nil { + return + } + if n, err := s.fileAccess.RetryBlockedGeneratedFileDeletes(ctx, limit); err != nil { + s.logWarn("content_moderation_blocked_file_cleanup_failed", zap.Error(err)) + } else if n > 0 { + s.logWarn("content_moderation_blocked_files_deleted", zap.Int("count", n)) + } +} + +// deleteIsolatedImages removes encrypted image copies. Returns true only when all deletes succeed +// (or there were no images), so callers can safely clear metadata paths. +func (s *Service) deleteIsolatedImages(ctx context.Context, event domaincm.Event) bool { + if event.ImageMetaJSON == "" || event.ImageMetaJSON == "[]" { + return true + } + images := unmarshalIsolatedImageMetadata(event.ImageMetaJSON) + if images == nil { + return false + } + if len(images) == 0 { + return true + } + if s.objectStore == nil { + return false + } + for _, img := range images { + if img.StoragePath == "" { + continue + } + if err := s.objectStore.Delete(ctx, img.StoragePath); err != nil { + s.logWarn("content_moderation_delete_isolated_image_failed", + zap.String("event_id", event.PublicID), + zap.String("path", img.StoragePath), + zap.Error(err), + ) + return false + } + } + return true +} + +func (s *Service) recoverStaleRuns(ctx context.Context) { + if s.repo == nil { + return + } + olderThan := time.Now().Add(-2 * time.Minute) + runIDs, err := s.repo.ListStaleModeratingRuns(ctx, olderThan, 100) + if err != nil { + return + } + for _, runID := range runIDs { + if s.HasActiveCoordinator(runID) { + continue + } + if s.recoverKnownHit(ctx, runID) { + continue + } + s.recordFailedOpen(ctx, RunMeta{RunID: runID}, domaincm.DirectionOutput, domaincm.ModalityText, domaincm.ErrorCodeWorkerLost, ErrWorkerLost.Error(), 0) + if err := s.repo.UpdateRunModeration(ctx, runID, domaincm.ModerationStateFailedOpen, "", "[]"); err != nil { + s.logWarn("content_moderation_recover_mark_failed_open_failed", zap.String("run_id", runID), zap.Error(err)) + } + } +} + +func (s *Service) recoverKnownHit(ctx context.Context, runID string) bool { + alreadyNotified := s.hasPendingBlock(runID) + event, err := s.repo.GetLatestHitEventByRunID(ctx, runID) + if err != nil { + // A repository read failure must not convert a potentially known hit into failed-open. + s.logWarn("content_moderation_recover_hit_lookup_failed", zap.String("run_id", runID), zap.Error(err)) + return true + } + if event == nil { + return false + } + var categories []string + _ = json.Unmarshal([]byte(event.CategoriesJSON), &categories) + info := BlockInfo{EventID: event.PublicID, Direction: event.Direction, Categories: categories} + fileIDs, err := s.repo.ApplyRunBlock(ctx, runID, event.Direction == domaincm.DirectionInput, event.PublicID, event.CategoriesJSON) + if err != nil { + s.registerPendingBlock(RunMeta{RunID: runID}, info) + s.logWarn("content_moderation_recover_hit_apply_failed", zap.String("run_id", runID), zap.Error(err)) + return true + } + s.removePendingBlock(runID) + s.deleteBlockedOutputFiles(fileIDs) + if !alreadyNotified { + s.notifyBlockedRecovery(runID, info) + } + return true +} + +func (s *Service) recoverPendingBlocks(ctx context.Context) { + for _, item := range s.pendingBlockSnapshot() { + fileIDs, err := s.repo.ApplyRunBlock( + ctx, + item.meta.RunID, + item.info.Direction == domaincm.DirectionInput, + item.info.EventID, + mustJSON(item.info.Categories), + ) + if err != nil { + s.logWarn("content_moderation_pending_block_retry_failed", zap.String("run_id", item.meta.RunID), zap.Error(err)) + continue + } + s.removePendingBlock(item.meta.RunID) + s.deleteBlockedOutputFiles(fileIDs) + } +} + +func (s *Service) handleLateBlock(meta RunMeta, info BlockInfo) { + if s == nil || s.repo == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + fileIDs, err := s.repo.ApplyRunBlock( + ctx, + meta.RunID, + info.Direction == domaincm.DirectionInput, + info.EventID, + mustJSON(info.Categories), + ) + if err != nil { + s.registerPendingBlock(meta, info) + if stateErr := s.repo.UpdateRunModeration(ctx, meta.RunID, domaincm.ModerationStateModerating, info.EventID, mustJSON(info.Categories)); stateErr != nil { + s.logWarn("content_moderation_late_block_mark_pending_failed", zap.String("run_id", meta.RunID), zap.Error(stateErr)) + } + s.logWarn("content_moderation_late_block_apply_failed", zap.String("run_id", meta.RunID), zap.Error(err)) + } else { + s.removePendingBlock(meta.RunID) + s.deleteBlockedOutputFiles(fileIDs) + } + cancel() + s.notifyBlockedRecovery(meta.RunID, info) +} + +func (s *Service) notifyBlockedRecovery(runID string, info BlockInfo) { + if s.onBlocked != nil { + s.onBlocked(runID, info) + } + if s.emitEvent != nil { + s.emitEvent(runID, "moderation_blocked", map[string]interface{}{ + "type": "moderation_blocked", + "eventID": info.EventID, + "direction": info.Direction, + "categories": info.Categories, + }) + } +} + +func (s *Service) deleteBlockedOutputFiles(fileIDs []string) { + if s == nil || s.fileAccess == nil || len(fileIDs) == 0 { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for _, fileID := range fileIDs { + if err := s.fileAccess.DeleteGeneratedFileArtifacts(ctx, fileID); err != nil { + s.logWarn("content_moderation_delete_blocked_output_failed", zap.String("file_id", fileID), zap.Error(err)) + } + } +} diff --git a/backend/internal/application/contentmoderation/config.go b/backend/internal/application/contentmoderation/config.go new file mode 100644 index 000000000..dbfbe5237 --- /dev/null +++ b/backend/internal/application/contentmoderation/config.go @@ -0,0 +1,356 @@ +package contentmoderation + +import ( + "context" + "encoding/json" + "strconv" + "strings" + "time" + + domainsettings "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/settings" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/secretbox" +) + +const ( + settingsNamespace = "content_moderation" + + keyEnabled = "enabled" + keyBaseURL = "base_url" + keyAPIKey = "api_key" + keyModel = "model" + keyTimeoutSeconds = "timeout_seconds" + keyMaxConcurrency = "max_concurrency" + keyQueueCapacity = "queue_capacity" + keyPolicyJSON = "policy_json" + keyPolicyVersion = "policy_version" + + defaultBaseURL = "https://api.openai.com/v1" + defaultModel = "omni-moderation-latest" + defaultTimeoutSeconds = 10 + defaultMaxConcurrency = 4 + defaultQueueCapacity = 256 +) + +// ServiceConfig is the saved moderation service configuration. +type ServiceConfig struct { + Enabled bool + BaseURL string + APIKeyMasked string + HasAPIKey bool + Model string + TimeoutSeconds int + MaxConcurrency int + QueueCapacity int + Policy Policy +} + +// UpdateConfigInput is the super-admin PUT body. +type UpdateConfigInput struct { + Enabled *bool + BaseURL *string + APIKey *string + ClearAPIKey bool + Model *string + TimeoutSeconds *int + MaxConcurrency *int + QueueCapacity *int + Policy *Policy +} + +type runtimeConfig struct { + Enabled bool + BaseURL string + APIKey string + Model string + Timeout time.Duration + MaxConcurrency int + QueueCapacity int + Policy Policy +} + +func (s *Service) loadRuntimeConfig(ctx context.Context) (runtimeConfig, error) { + s.configMu.RLock() + if s.cachedConfig != nil && time.Since(s.cachedAt) < 2*time.Second { + cfg := *s.cachedConfig + s.configMu.RUnlock() + return cfg, nil + } + s.configMu.RUnlock() + + cfg, err := s.readRuntimeConfig(ctx) + if err != nil { + return runtimeConfig{}, err + } + s.configMu.Lock() + s.cachedConfig = &cfg + s.cachedAt = time.Now() + s.configMu.Unlock() + return cfg, nil +} + +func (s *Service) invalidateConfigCache() { + s.configMu.Lock() + s.cachedConfig = nil + s.configMu.Unlock() +} + +func (s *Service) readRuntimeConfig(ctx context.Context) (runtimeConfig, error) { + items, err := s.settingsRepo.ListByNamespace(ctx, settingsNamespace) + if err != nil { + return runtimeConfig{}, err + } + values := make(map[string]string, len(items)) + for _, item := range items { + values[item.Key] = item.Value + } + + apiKey := "" + if encrypted := strings.TrimSpace(values[keyAPIKey]); encrypted != "" { + decrypted, decErr := secretbox.DecryptString(s.dataEncryptionKey, encrypted) + if decErr != nil { + return runtimeConfig{}, decErr + } + apiKey = decrypted + } + + policy := Policy{} + if raw := strings.TrimSpace(values[keyPolicyJSON]); raw != "" { + var policyDocument policyJSON + if err := json.Unmarshal([]byte(raw), &policyDocument); err != nil { + return runtimeConfig{}, err + } + policy = policyDocument.toPolicy() + } + policy.Version = parseInt64(values[keyPolicyVersion], 0) + policy, err = NormalizePolicy(policy) + if err != nil { + return runtimeConfig{}, err + } + enabled := false + if raw, exists := values[keyEnabled]; exists { + if parsed, parseErr := strconv.ParseBool(strings.TrimSpace(raw)); parseErr == nil { + enabled = parsed + } + } + + timeoutSec := parseInt(values[keyTimeoutSeconds], defaultTimeoutSeconds) + if timeoutSec < 1 { + timeoutSec = defaultTimeoutSeconds + } + maxConc := parseInt(values[keyMaxConcurrency], defaultMaxConcurrency) + if maxConc < 1 { + maxConc = defaultMaxConcurrency + } + queueCap := parseInt(values[keyQueueCapacity], defaultQueueCapacity) + if queueCap < 1 { + queueCap = defaultQueueCapacity + } + baseURL := strings.TrimSpace(values[keyBaseURL]) + if baseURL == "" { + baseURL = defaultBaseURL + } + model := strings.TrimSpace(values[keyModel]) + if model == "" { + model = defaultModel + } + + return runtimeConfig{ + Enabled: enabled, + BaseURL: baseURL, + APIKey: apiKey, + Model: model, + Timeout: time.Duration(timeoutSec) * time.Second, + MaxConcurrency: maxConc, + QueueCapacity: queueCap, + Policy: policy, + }, nil +} + +// GetConfig returns masked config for super-admin UI. +func (s *Service) GetConfig(ctx context.Context, actorRole string) (*ServiceConfig, error) { + if !isSuperAdmin(actorRole) { + return nil, ErrSuperAdminRequired + } + cfg, err := s.readRuntimeConfig(ctx) + if err != nil { + return nil, err + } + return toServiceConfig(cfg), nil +} + +// UpdateConfig atomically saves configuration. +func (s *Service) UpdateConfig(ctx context.Context, actorRole string, input UpdateConfigInput) (*ServiceConfig, error) { + if !isSuperAdmin(actorRole) { + return nil, ErrSuperAdminRequired + } + current, err := s.readRuntimeConfig(ctx) + if err != nil { + return nil, err + } + next := current + if input.Enabled != nil { + next.Enabled = *input.Enabled + } + if input.BaseURL != nil { + base := strings.TrimSpace(*input.BaseURL) + if base == "" { + base = defaultBaseURL + } + if s.provider == nil || s.provider.ValidateBaseURL(base) != nil { + return nil, ErrInvalidBaseURL + } + next.BaseURL = base + } + if input.ClearAPIKey { + next.APIKey = "" + } else if input.APIKey != nil { + next.APIKey = strings.TrimSpace(*input.APIKey) + } + if input.Model != nil { + model := strings.TrimSpace(*input.Model) + if model == "" { + return nil, ErrInvalidModel + } + next.Model = model + } + if input.TimeoutSeconds != nil { + if *input.TimeoutSeconds < 1 || *input.TimeoutSeconds > 60 { + return nil, ErrInvalidTimeout + } + next.Timeout = time.Duration(*input.TimeoutSeconds) * time.Second + } + if input.MaxConcurrency != nil { + if *input.MaxConcurrency < 1 || *input.MaxConcurrency > 64 { + return nil, ErrInvalidConcurrency + } + next.MaxConcurrency = *input.MaxConcurrency + } + if input.QueueCapacity != nil { + if *input.QueueCapacity < 1 || *input.QueueCapacity > 4096 { + return nil, ErrInvalidQueueCapacity + } + next.QueueCapacity = *input.QueueCapacity + } + if input.Policy != nil { + policy, normErr := NormalizePolicy(*input.Policy) + if normErr != nil { + return nil, normErr + } + if !policyEqualCategories(current.Policy, policy) { + policy.Version = current.Policy.Version + 1 + } else { + policy.Version = current.Policy.Version + } + next.Policy = policy + } + if next.Enabled { + if !next.Policy.Enabled() || strings.TrimSpace(next.BaseURL) == "" || strings.TrimSpace(next.Model) == "" || strings.TrimSpace(next.APIKey) == "" { + return nil, ErrServiceConfigRequired + } + if s.provider == nil || s.provider.ValidateBaseURL(next.BaseURL) != nil { + return nil, ErrInvalidBaseURL + } + } + + items, err := buildSettingItems(next, s.dataEncryptionKey) + if err != nil { + return nil, err + } + if err := s.settingsRepo.Upsert(ctx, items); err != nil { + return nil, err + } + s.invalidateConfigCache() + s.resizeWorker(next.MaxConcurrency, next.QueueCapacity) + return toServiceConfig(next), nil +} + +func policyEqualCategories(a, b Policy) bool { + return stringSlicesEqual(a.InputTextCategories, b.InputTextCategories) && + stringSlicesEqual(a.OutputTextCategories, b.OutputTextCategories) && + stringSlicesEqual(a.InputImageCategories, b.InputImageCategories) && + stringSlicesEqual(a.OutputImageCategories, b.OutputImageCategories) +} + +func stringSlicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func toServiceConfig(cfg runtimeConfig) *ServiceConfig { + return &ServiceConfig{ + Enabled: cfg.Enabled, + BaseURL: cfg.BaseURL, + APIKeyMasked: maskAPIKey(cfg.APIKey), + HasAPIKey: strings.TrimSpace(cfg.APIKey) != "", + Model: cfg.Model, + TimeoutSeconds: int(cfg.Timeout / time.Second), + MaxConcurrency: cfg.MaxConcurrency, + QueueCapacity: cfg.QueueCapacity, + Policy: cfg.Policy, + } +} + +func buildSettingItems(cfg runtimeConfig, encryptionKey string) ([]domainsettings.SystemSetting, error) { + policyJSON, err := json.Marshal(newPolicyJSON(cfg.Policy)) + if err != nil { + return nil, err + } + encryptedKey := "" + if strings.TrimSpace(cfg.APIKey) != "" { + encryptedKey, err = secretbox.EncryptString(encryptionKey, cfg.APIKey) + if err != nil { + return nil, err + } + } + return []domainsettings.SystemSetting{ + {Namespace: settingsNamespace, Key: keyEnabled, Value: strconv.FormatBool(cfg.Enabled), ValueType: "bool"}, + {Namespace: settingsNamespace, Key: keyBaseURL, Value: cfg.BaseURL, ValueType: "string"}, + {Namespace: settingsNamespace, Key: keyAPIKey, Value: encryptedKey, ValueType: "string"}, + {Namespace: settingsNamespace, Key: keyModel, Value: cfg.Model, ValueType: "string"}, + {Namespace: settingsNamespace, Key: keyTimeoutSeconds, Value: strconv.Itoa(int(cfg.Timeout / time.Second)), ValueType: "int"}, + {Namespace: settingsNamespace, Key: keyMaxConcurrency, Value: strconv.Itoa(cfg.MaxConcurrency), ValueType: "int"}, + {Namespace: settingsNamespace, Key: keyQueueCapacity, Value: strconv.Itoa(cfg.QueueCapacity), ValueType: "int"}, + {Namespace: settingsNamespace, Key: keyPolicyJSON, Value: string(policyJSON), ValueType: "json"}, + {Namespace: settingsNamespace, Key: keyPolicyVersion, Value: strconv.FormatInt(cfg.Policy.Version, 10), ValueType: "int"}, + }, nil +} + +func parseInt(raw string, fallback int) int { + value := strings.TrimSpace(raw) + if value == "" { + return fallback + } + n, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return n +} + +func parseInt64(raw string, fallback int64) int64 { + value := strings.TrimSpace(raw) + if value == "" { + return fallback + } + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fallback + } + return n +} + +func isSuperAdmin(role string) bool { + return strings.EqualFold(strings.TrimSpace(role), "superadmin") +} + +func isAdminRole(role string) bool { + r := strings.ToLower(strings.TrimSpace(role)) + return r == "admin" || r == "superadmin" +} diff --git a/backend/internal/application/contentmoderation/config_test.go b/backend/internal/application/contentmoderation/config_test.go new file mode 100644 index 000000000..56aff30d6 --- /dev/null +++ b/backend/internal/application/contentmoderation/config_test.go @@ -0,0 +1,203 @@ +package contentmoderation + +import ( + "context" + "errors" + "testing" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + domainsettings "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/settings" + "go.uber.org/zap" +) + +type configTestSettingsRepo struct { + items []domainsettings.SystemSetting + err error +} + +type configTestProvider struct{} + +func (configTestProvider) ValidateBaseURL(string) error { return nil } + +func (configTestProvider) ModerateText( + context.Context, + ProviderConfig, + string, + []string, + string, +) (*Response, error) { + return nil, nil +} + +func (configTestProvider) ModerateImages( + context.Context, + ProviderConfig, + []ProviderImage, + []string, + string, +) (*Response, error) { + return nil, nil +} + +func (r *configTestSettingsRepo) ListAll(context.Context) ([]domainsettings.SystemSetting, error) { + return append([]domainsettings.SystemSetting(nil), r.items...), nil +} + +func (r *configTestSettingsRepo) ListByNamespace(_ context.Context, namespace string) ([]domainsettings.SystemSetting, error) { + if r.err != nil { + return nil, r.err + } + items := make([]domainsettings.SystemSetting, 0, len(r.items)) + for _, item := range r.items { + if item.Namespace == namespace { + items = append(items, item) + } + } + return items, nil +} + +func TestBeginRunRecordsConfigurationFailureAsFailedOpen(t *testing.T) { + settingsRepo := &configTestSettingsRepo{err: errors.New("database unavailable: sensitive detail")} + moderationRepo := &coordinatorTestRepo{} + service := NewService(settingsRepo, moderationRepo, "test-data-encryption-key", zap.NewNop()) + + coordinator := service.BeginRun(context.Background(), RunMeta{RunID: "run_config_failure", UserID: 42}) + if coordinator == nil { + t.Fatal("configuration failure must return a failed-open coordinator") + } + result := coordinator.WaitInputOnly(context.Background()) + if result.State != domaincm.ModerationStateFailedOpen { + t.Fatalf("state = %q, want failed_open", result.State) + } + + moderationRepo.mu.Lock() + defer moderationRepo.mu.Unlock() + if len(moderationRepo.events) != 1 { + t.Fatalf("events = %d, want 1", len(moderationRepo.events)) + } + event := moderationRepo.events[0] + if event.ErrorCode != domaincm.ErrorCodeConfigMissing { + t.Fatalf("error code = %q", event.ErrorCode) + } + if event.ErrorMessage != "content moderation configuration unavailable" { + t.Fatalf("unsafe or unexpected persisted error = %q", event.ErrorMessage) + } + if moderationRepo.runState != domaincm.ModerationStateFailedOpen { + t.Fatalf("run state = %q, want failed_open", moderationRepo.runState) + } +} + +func (r *configTestSettingsRepo) Upsert(_ context.Context, items []domainsettings.SystemSetting) error { + for _, next := range items { + replaced := false + for index := range r.items { + if r.items[index].Namespace == next.Namespace && r.items[index].Key == next.Key { + r.items[index] = next + replaced = true + break + } + } + if !replaced { + r.items = append(r.items, next) + } + } + return nil +} + +func (r *configTestSettingsRepo) UpsertWithDescription(ctx context.Context, items []domainsettings.SystemSetting) error { + return r.Upsert(ctx, items) +} + +func (r *configTestSettingsRepo) Delete(_ context.Context, namespace, key string) error { + for index, item := range r.items { + if item.Namespace == namespace && item.Key == key { + r.items = append(r.items[:index], r.items[index+1:]...) + break + } + } + return nil +} + +func TestGetConfigRequiresExplicitEnabledState(t *testing.T) { + repo := &configTestSettingsRepo{items: []domainsettings.SystemSetting{ + {Namespace: settingsNamespace, Key: keyPolicyJSON, Value: `{"inputTextCategories":["hate"]}`}, + {Namespace: settingsNamespace, Key: keyPolicyVersion, Value: "3"}, + }} + service := NewService(repo, nil, "test-data-encryption-key", zap.NewNop()) + + config, err := service.GetConfig(context.Background(), "superadmin") + if err != nil { + t.Fatalf("get config: %v", err) + } + if config.Enabled { + t.Fatal("expected missing enabled setting to default to disabled") + } + if config.Policy.Version != 3 { + t.Fatalf("expected policy version 3, got %d", config.Policy.Version) + } +} + +func TestGetConfigHonorsExplicitDisabledStateWithRetainedPolicy(t *testing.T) { + repo := &configTestSettingsRepo{items: []domainsettings.SystemSetting{ + {Namespace: settingsNamespace, Key: keyEnabled, Value: "false"}, + {Namespace: settingsNamespace, Key: keyPolicyJSON, Value: `{"inputTextCategories":["hate"]}`}, + }} + service := NewService(repo, nil, "test-data-encryption-key", zap.NewNop()) + + config, err := service.GetConfig(context.Background(), "superadmin") + if err != nil { + t.Fatalf("get config: %v", err) + } + if config.Enabled { + t.Fatal("expected explicit disabled state to override the retained policy") + } + if len(config.Policy.InputTextCategories) != 1 { + t.Fatalf("expected retained policy, got %#v", config.Policy.InputTextCategories) + } +} + +func TestUpdateConfigDisablesWithoutClearingPolicy(t *testing.T) { + repo := &configTestSettingsRepo{items: []domainsettings.SystemSetting{ + {Namespace: settingsNamespace, Key: keyEnabled, Value: "true"}, + {Namespace: settingsNamespace, Key: keyPolicyJSON, Value: `{"inputTextCategories":["hate"]}`}, + }} + service := NewService(repo, nil, "test-data-encryption-key", zap.NewNop()) + enabled := false + + config, err := service.UpdateConfig(context.Background(), "superadmin", UpdateConfigInput{Enabled: &enabled}) + if err != nil { + t.Fatalf("disable config: %v", err) + } + if config.Enabled { + t.Fatal("expected moderation to be disabled") + } + if len(config.Policy.InputTextCategories) != 1 || config.Policy.InputTextCategories[0] != "hate" { + t.Fatalf("expected policy to be retained, got %#v", config.Policy.InputTextCategories) + } +} + +func TestUpdateConfigRequiresServiceAndPolicyWhenEnabled(t *testing.T) { + repo := &configTestSettingsRepo{} + service := NewService(repo, nil, "test-data-encryption-key", zap.NewNop()) + service.SetProvider(configTestProvider{}) + enabled := true + + _, err := service.UpdateConfig(context.Background(), "superadmin", UpdateConfigInput{Enabled: &enabled}) + if !errors.Is(err, ErrServiceConfigRequired) { + t.Fatalf("expected ErrServiceConfigRequired, got %v", err) + } + + apiKey := "test-api-key" + policy := Policy{InputTextCategories: []string{"hate"}} + config, err := service.UpdateConfig(context.Background(), "superadmin", UpdateConfigInput{ + Enabled: &enabled, + APIKey: &apiKey, + Policy: &policy, + }) + if err != nil { + t.Fatalf("enable complete config: %v", err) + } + if !config.Enabled || !config.HasAPIKey { + t.Fatalf("expected enabled config with API key, got %#v", config) + } +} diff --git a/backend/internal/application/contentmoderation/coordinator.go b/backend/internal/application/contentmoderation/coordinator.go new file mode 100644 index 000000000..2e59856f5 --- /dev/null +++ b/backend/internal/application/contentmoderation/coordinator.go @@ -0,0 +1,549 @@ +package contentmoderation + +import ( + "context" + "errors" + "strings" + "sync" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "go.uber.org/zap" +) + +// RunMeta identifies the moderated conversation turn. +type RunMeta struct { + UserID uint + ConversationID uint + RunID string + MessageID uint + MessagePublicID string + AssistantMessageID uint + UserMessageID uint +} + +// BlockInfo is returned when a round is blocked. +type BlockInfo struct { + EventID string + Direction string + Categories []string +} + +// BarrierResult is the post-generation / input-only barrier outcome. +type BarrierResult struct { + // Block is non-nil whenever a known hit must be hidden from the caller. Durable + // persistence may converge asynchronously when the primary transaction is unavailable. + Block *BlockInfo + // State is the caller-visible moderation target (passed|failed_open|blocked). + State string + // TerminalEmitted is true when moderation_blocked was published to live/recovery streams. + TerminalEmitted bool +} + +// LiveEmitter delivers events to the active HTTP stream (in addition to recovery storage). +type LiveEmitter func(eventType string, payload map[string]interface{}) + +// RunCoordinator tracks moderation tasks for a single chat/media run. +type RunCoordinator struct { + service *Service + meta RunMeta + cfg runtimeConfig + liveEmit LiveEmitter + + mu sync.Mutex + pending int + blocked bool + blockInfo BlockInfo + failedOpen bool + allDone chan struct{} + allClosed bool + cancelOnce sync.Once + finished bool + settled bool + blockHandled bool + outputEnqueued bool +} + +func newRunCoordinator(service *Service, meta RunMeta, cfg runtimeConfig) *RunCoordinator { + return &RunCoordinator{ + service: service, + meta: meta, + cfg: cfg, + allDone: make(chan struct{}), + } +} + +// SetLiveEmitter wires the active stream sink for moderation_checking / moderation_blocked. +func (c *RunCoordinator) SetLiveEmitter(emit LiveEmitter) { + if c == nil { + return + } + c.liveEmit = emit +} + +// EnqueueInputText queues input text moderation if policy requires it. +func (c *RunCoordinator) EnqueueInputText(text string) { + if c == nil { + return + } + selected := c.cfg.Policy.CategoriesFor(domaincm.DirectionInput, domaincm.ModalityText) + if len(selected) == 0 || strings.TrimSpace(text) == "" { + return + } + c.startTask(&moderationTask{ + Coord: c, + Direction: domaincm.DirectionInput, + Modality: domaincm.ModalityText, + Text: text, + Selected: selected, + Location: domaincm.ContentLocation{Field: "user_message"}, + }) +} + +// EnqueueInputImages queues input image moderation for used attachments. +func (c *RunCoordinator) EnqueueInputImages(ctx context.Context, fileIDs []string) { + if c == nil { + return + } + selected := c.cfg.Policy.CategoriesFor(domaincm.DirectionInput, domaincm.ModalityImage) + if len(selected) == 0 || len(fileIDs) == 0 { + return + } + if c.service == nil || c.service.imageLoader == nil { + c.recordSurfaceFailure(domaincm.DirectionInput, domaincm.ModalityImage, "", ErrModerationService) + return + } + seenSHA := make(map[string]struct{}) + raw := make([]OutputImageSource, 0, len(fileIDs)) + keptFiles := make([]string, 0, len(fileIDs)) + for _, fileID := range fileIDs { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + continue + } + prepared, err := c.service.imageLoader(ctx, c.meta.UserID, fileID) + if errors.Is(err, ErrNonImageAttachment) { + continue + } + if err != nil || len(prepared.Data) == 0 { + if err == nil { + err = ErrModerationInvalidResp + } + c.recordSurfaceFailure(domaincm.DirectionInput, domaincm.ModalityImage, fileID, err) + continue + } + sha := prepared.SHA256 + if sha != "" { + if _, ok := seenSHA[sha]; ok { + continue + } + seenSHA[sha] = struct{}{} + } + // Isolated copy for review only; user originals are not deleted. + raw = append(raw, OutputImageSource{ + FileID: fileID, + Data: prepared.Data, + MimeType: prepared.Mime, + SHA256: sha, + }) + keptFiles = append(keptFiles, fileID) + } + if len(raw) == 0 { + return + } + c.startTask(&moderationTask{ + Coord: c, + Direction: domaincm.DirectionInput, + Modality: domaincm.ModalityImage, + RawImages: raw, + FileIDs: keptFiles, + Selected: selected, + Location: domaincm.ContentLocation{Field: "user_attachments"}, + // Input hits must isolate but not revoke user library files. + IsolateOnly: true, + }) +} + +// AfterGeneration runs the post-generation barrier. +func (c *RunCoordinator) AfterGeneration(ctx context.Context, outputText string, outputImages []OutputImageSource) BarrierResult { + if c == nil { + return BarrierResult{State: domaincm.ModerationStatePassed} + } + if err := c.service.repo.UpdateRunModeration(ctx, c.meta.RunID, domaincm.ModerationStateModerating, "", "[]"); err != nil { + c.service.logWarn("content_moderation_mark_moderating_failed", zap.String("run_id", c.meta.RunID), zap.Error(err)) + } + c.emit("moderation_checking", map[string]interface{}{ + "type": "moderation_checking", + }) + + c.enqueueOutputText(outputText) + c.enqueueOutputImages(outputImages) + c.markOutputsEnqueued() + c.waitAll(ctx) + + blocked, info, failedOpen := c.settle() + + if blocked { + emitted, err := c.applyBlock(info) + if err != nil { + c.service.registerPendingBlock(c.meta, info) + emitted = c.notifyBlocked(info) + } + c.finish() + return BarrierResult{ + Block: &info, + State: domaincm.ModerationStateBlocked, + TerminalEmitted: emitted, + } + } + state := domaincm.ModerationStatePassed + if failedOpen { + state = domaincm.ModerationStateFailedOpen + } + c.updateRunState(state, "", "[]") + c.finish() + return BarrierResult{State: state} +} + +// WaitInputOnly continues input checks after generation errors/cancels. +func (c *RunCoordinator) WaitInputOnly(ctx context.Context) BarrierResult { + if c == nil { + return BarrierResult{State: domaincm.ModerationStatePassed} + } + c.markOutputsEnqueued() + c.waitAll(ctx) + blocked, info, failedOpen := c.settle() + if blocked { + emitted, err := c.applyBlock(info) + if err != nil { + c.service.registerPendingBlock(c.meta, info) + emitted = c.notifyBlocked(info) + } + c.finish() + return BarrierResult{ + Block: &info, + State: domaincm.ModerationStateBlocked, + TerminalEmitted: emitted, + } + } + state := domaincm.ModerationStatePassed + if failedOpen { + state = domaincm.ModerationStateFailedOpen + } + c.updateRunState(state, "", "[]") + c.finish() + return BarrierResult{State: state} +} + +func (c *RunCoordinator) settle() (bool, BlockInfo, bool) { + c.mu.Lock() + defer c.mu.Unlock() + c.settled = true + if c.blocked { + c.blockHandled = true + } + return c.blocked, c.blockInfo, c.failedOpen +} + +func (c *RunCoordinator) updateRunState(state, eventID, categoriesJSON string) { + if c == nil || c.service == nil || c.service.repo == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := c.service.repo.UpdateRunModeration(ctx, c.meta.RunID, state, eventID, categoriesJSON); err != nil { + c.service.logWarn("content_moderation_update_run_state_failed", zap.String("run_id", c.meta.RunID), zap.String("state", state), zap.Error(err)) + } +} + +// RecordOutputImageFailure records an expected model-output image that could not be loaded. +// A missing image must never be treated as a clean moderation pass. +func (c *RunCoordinator) RecordOutputImageFailure(fileID string, loadErr error) { + c.recordSurfaceFailure(domaincm.DirectionOutput, domaincm.ModalityImage, fileID, loadErr) +} + +func (c *RunCoordinator) recordSurfaceFailure(direction, modality, fileID string, surfaceErr error) { + if c == nil || c.service == nil || len(c.cfg.Policy.CategoriesFor(direction, modality)) == 0 { + return + } + c.mu.Lock() + if c.settled || c.finished { + c.mu.Unlock() + return + } + c.failedOpen = true + c.mu.Unlock() + + message := errString(surfaceErr) + if strings.TrimSpace(message) == "" { + message = "content unavailable for moderation" + } + if fileID = strings.TrimSpace(fileID); fileID != "" { + message += " (file_id=" + fileID + ")" + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c.service.recordFailedOpen(ctx, c.meta, direction, modality, domaincm.ErrorCodeServiceError, message, 0) + c.service.bumpDailyStat(ctx, direction, modality, domaincm.ResultFailedOpen, "", 1, 1, 0, 1, 0) +} + +func (c *RunCoordinator) enqueueOutputText(text string) { + selected := c.cfg.Policy.CategoriesFor(domaincm.DirectionOutput, domaincm.ModalityText) + if len(selected) == 0 || strings.TrimSpace(text) == "" { + return + } + c.startTask(&moderationTask{ + Coord: c, + Direction: domaincm.DirectionOutput, + Modality: domaincm.ModalityText, + Text: text, + Selected: selected, + Location: domaincm.ContentLocation{Field: "assistant_message"}, + }) +} + +func (c *RunCoordinator) enqueueOutputImages(images []OutputImageSource) { + selected := c.cfg.Policy.CategoriesFor(domaincm.DirectionOutput, domaincm.ModalityImage) + if len(selected) == 0 || len(images) == 0 { + return + } + seen := make(map[string]struct{}) + raw := make([]OutputImageSource, 0, len(images)) + for _, img := range images { + if len(img.Data) == 0 { + continue + } + sha := img.SHA256 + if sha == "" { + sha = sha256Hex(img.Data) + img.SHA256 = sha + } + if _, ok := seen[sha]; ok { + continue + } + seen[sha] = struct{}{} + img.MimeType = firstNonEmpty(img.MimeType, "image/png") + raw = append(raw, img) + } + if len(raw) == 0 { + return + } + c.startTask(&moderationTask{ + Coord: c, + Direction: domaincm.DirectionOutput, + Modality: domaincm.ModalityImage, + RawImages: raw, + Selected: selected, + Location: domaincm.ContentLocation{Field: "assistant_images"}, + }) +} + +func (c *RunCoordinator) startTask(task *moderationTask) { + c.mu.Lock() + if c.finished { + c.mu.Unlock() + return + } + c.pending++ + c.mu.Unlock() + + // Worker (or enqueue-full path) calls onTaskResult directly — no Done-wait goroutine. + if err := c.service.enqueue(task); err != nil { + c.onTaskResult(task, taskResult{Err: err, ErrorCode: domaincm.ErrorCodeQueueFull}) + } +} + +func (c *RunCoordinator) onTaskResult(task *moderationTask, result taskResult) *BlockInfo { + c.mu.Lock() + direction := "" + if task != nil { + direction = task.Direction + } + if c.pending > 0 { + c.pending-- + } + // Clear task payloads after processing to reduce retained sensitive memory. + if task != nil { + task.Text = "" + task.RawImages = nil + } + if c.settled || c.finished { + var lateBlock *BlockInfo + if result.Hit && (!c.blockHandled || preferBlockDirection(direction, c.blockInfo.Direction)) { + c.blockHandled = true + info := BlockInfo{ + EventID: result.EventID, + Direction: direction, + Categories: append([]string(nil), result.Categories...), + } + c.blocked = true + c.blockInfo = info + lateBlock = &info + } + if c.pending == 0 && c.outputEnqueued { + c.closeAllLocked() + } + c.mu.Unlock() + return lateBlock + } + cancelInput := false + if result.Hit { + if !c.blocked || preferBlockDirection(direction, c.blockInfo.Direction) { + c.blocked = true + c.blockInfo = BlockInfo{ + EventID: result.EventID, + Direction: direction, + Categories: append([]string(nil), result.Categories...), + } + } + if direction == domaincm.DirectionInput { + cancelInput = true + } + } else if result.Err != nil { + c.failedOpen = true + } + if c.pending == 0 && c.outputEnqueued { + c.closeAllLocked() + } + c.mu.Unlock() + if cancelInput { + c.cancelOnce.Do(func() { + if c.service.cancelRun != nil { + c.service.cancelRun(c.meta.RunID) + } + }) + } + return nil +} + +func (c *RunCoordinator) markOutputsEnqueued() { + c.mu.Lock() + defer c.mu.Unlock() + c.outputEnqueued = true + if c.pending == 0 { + c.closeAllLocked() + } +} + +func (c *RunCoordinator) closeAllLocked() { + if !c.allClosed { + close(c.allDone) + c.allClosed = true + } +} + +func preferBlockDirection(candidate, current string) bool { + return candidate == domaincm.DirectionInput && current != domaincm.DirectionInput +} + +func (c *RunCoordinator) waitAll(ctx context.Context) { + // Bound wait to remaining policy timeout so stream cannot hang forever. + timeout := c.cfg.Timeout + if timeout <= 0 { + timeout = 10 * time.Second + } + // Allow queued + multi-surface work up to 2x single-check budget, capped at 60s. + deadline := timeout * 2 + if deadline > 60*time.Second { + deadline = 60 * time.Second + } + timer := time.NewTimer(deadline) + defer timer.Stop() + select { + case <-c.allDone: + case <-ctx.Done(): + select { + case <-c.allDone: + case <-time.After(100 * time.Millisecond): + c.mu.Lock() + if !c.blocked { + c.failedOpen = true + } + c.mu.Unlock() + } + case <-timer.C: + c.mu.Lock() + if !c.blocked { + c.failedOpen = true + } + c.mu.Unlock() + } +} + +// applyBlock persists withdrawal then emits the terminal stream event. +func (c *RunCoordinator) applyBlock(info BlockInfo) (bool, error) { + // Client disconnect cancels the request context; persistence must survive that. + persistCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + includeUser := info.Direction == domaincm.DirectionInput + categoriesJSON := mustJSON(info.Categories) + + // Single transactional write path — no sequential fallback. + fileIDs, err := c.service.repo.ApplyRunBlock(persistCtx, c.meta.RunID, includeUser, info.EventID, categoriesJSON) + if err != nil { + c.service.logWarn("content_moderation_apply_block_failed", + zap.String("run_id", c.meta.RunID), + zap.Error(err), + ) + return false, err + } + c.service.removePendingBlock(c.meta.RunID) + c.service.deleteBlockedOutputFiles(fileIDs) + return c.notifyBlocked(info), nil +} + +func (c *RunCoordinator) notifyBlocked(info BlockInfo) bool { + if c.service.onBlocked != nil { + c.service.onBlocked(c.meta.RunID, info) + } + c.emit("moderation_blocked", map[string]interface{}{ + "type": "moderation_blocked", + "eventID": info.EventID, + "direction": info.Direction, + "categories": info.Categories, + }) + return true +} + +func (c *RunCoordinator) emit(eventType string, payload map[string]interface{}) { + if payload == nil { + payload = map[string]interface{}{"type": eventType} + } else if _, ok := payload["type"]; !ok { + payload["type"] = eventType + } + // Prefer live sink (handler flushStreamEvent already persists + writes NDJSON). + // Fall back to recovery-only emitter when no live connection is bound. + if c.liveEmit != nil { + c.liveEmit(eventType, payload) + return + } + if c.service != nil && c.service.emitEvent != nil { + c.service.emitEvent(c.meta.RunID, eventType, payload) + } +} + +func (c *RunCoordinator) finish() { + c.mu.Lock() + c.finished = true + c.mu.Unlock() + if c.service != nil { + c.service.releaseCoordinator(c.meta.RunID) + } +} + +// IsBlocked returns whether a hit has already been recorded. +func (c *RunCoordinator) IsBlocked() (bool, BlockInfo) { + if c == nil { + return false, BlockInfo{} + } + c.mu.Lock() + defer c.mu.Unlock() + return c.blocked, c.blockInfo +} + +func errString(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/backend/internal/application/contentmoderation/coordinator_test.go b/backend/internal/application/contentmoderation/coordinator_test.go new file mode 100644 index 000000000..62764f420 --- /dev/null +++ b/backend/internal/application/contentmoderation/coordinator_test.go @@ -0,0 +1,351 @@ +package contentmoderation + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +type coordinatorTestRepo struct { + mu sync.Mutex + createErr error + applyErr error + applyCalls int + applyInputs []bool + events []domaincm.Event + stats []repository.DailyStatIncrement + latestHit *domaincm.Event + getEvent *domaincm.Event + getEventErr error + runState string + staleRunIDs []string +} + +func (r *coordinatorTestRepo) CreateEvent(_ context.Context, event *domaincm.Event) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.createErr != nil { + return r.createErr + } + if event != nil { + r.events = append(r.events, *event) + } + return nil +} + +type coordinatorTestObjectStore struct { + mu sync.Mutex + put []string + deleted []string +} + +func (s *coordinatorTestObjectStore) Put(_ context.Context, path string, _ []byte, _ string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.put = append(s.put, path) + return nil +} + +func (s *coordinatorTestObjectStore) Open(context.Context, string) ([]byte, error) { + return nil, nil +} + +func (s *coordinatorTestObjectStore) Delete(_ context.Context, path string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.deleted = append(s.deleted, path) + return nil +} + +func (r *coordinatorTestRepo) GetEventByPublicID(context.Context, string) (*domaincm.Event, error) { + return r.getEvent, r.getEventErr +} + +func (r *coordinatorTestRepo) GetLatestHitEventByRunID(context.Context, string) (*domaincm.Event, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.latestHit, nil +} + +func (r *coordinatorTestRepo) ListEvents(context.Context, domaincm.EventListFilter) ([]domaincm.Event, int64, error) { + return nil, 0, nil +} + +func (r *coordinatorTestRepo) ClearExpiredContentByPublicIDs(context.Context, []string) (int64, error) { + return 0, nil +} + +func (r *coordinatorTestRepo) ListExpiredContentEvents(context.Context, time.Time, int) ([]domaincm.Event, error) { + return nil, nil +} + +func (r *coordinatorTestRepo) DeleteExpiredMetadata(context.Context, time.Time) (int64, error) { + return 0, nil +} + +func (r *coordinatorTestRepo) IncrementDailyStat(_ context.Context, input repository.DailyStatIncrement) error { + r.mu.Lock() + defer r.mu.Unlock() + r.stats = append(r.stats, input) + return nil +} + +func (r *coordinatorTestRepo) ListDailyStats(context.Context, time.Time, time.Time) ([]domaincm.DailyStat, error) { + return nil, nil +} + +func (r *coordinatorTestRepo) DeleteDailyStatsBefore(context.Context, time.Time) (int64, error) { + return 0, nil +} + +func (r *coordinatorTestRepo) UpdateRunModeration(_ context.Context, _ string, state string, _ string, _ string) error { + r.mu.Lock() + r.runState = state + r.mu.Unlock() + return nil +} + +func (r *coordinatorTestRepo) ApplyRunBlock(_ context.Context, _ string, includeUser bool, _ string, _ string) ([]string, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.applyCalls++ + r.applyInputs = append(r.applyInputs, includeUser) + return nil, r.applyErr +} + +func (r *coordinatorTestRepo) GetRunModerationState(context.Context, string) (string, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.runState, nil +} + +func (r *coordinatorTestRepo) ListStaleModeratingRuns(context.Context, time.Time, int) ([]string, error) { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.staleRunIDs...), nil +} + +func TestKnownHitRemainsBlockedWhenDurableApplyFails(t *testing.T) { + repo := &coordinatorTestRepo{applyErr: errors.New("database unavailable")} + service := NewService(nil, repo, "", nil) + coord := newRunCoordinator(service, RunMeta{RunID: "run_known_hit"}, runtimeConfig{Timeout: time.Second}) + coord.blocked = true + coord.blockInfo = BlockInfo{EventID: "cme_hit", Direction: domaincm.DirectionOutput, Categories: []string{"violence"}} + + var emitted string + coord.SetLiveEmitter(func(eventType string, _ map[string]interface{}) { + emitted = eventType + }) + + result := coord.AfterGeneration(context.Background(), "", nil) + if result.Block == nil || result.State != domaincm.ModerationStateBlocked { + t.Fatalf("known hit must remain blocked, got %#v", result) + } + if !result.TerminalEmitted || emitted != "moderation_blocked" { + t.Fatalf("expected moderation_blocked terminal event, emitted=%q result=%#v", emitted, result) + } + if !service.hasPendingBlock("run_known_hit") { + t.Fatal("failed durable apply must be registered for compensation") + } + repo.mu.Lock() + repo.applyErr = nil + repo.mu.Unlock() + service.recoverPendingBlocks(context.Background()) + if service.hasPendingBlock("run_known_hit") { + t.Fatal("successful compensation must remove the pending block") + } +} + +func TestLateHitRunsFullBlockCompensation(t *testing.T) { + repo := &coordinatorTestRepo{} + service := NewService(nil, repo, "", nil) + coord := newRunCoordinator(service, RunMeta{RunID: "run_late_hit"}, runtimeConfig{}) + coord.pending = 1 + coord.outputEnqueued = true + coord.settled = true + + var emitted string + service.SetEventEmitter(func(_ string, eventType string, _ map[string]interface{}) { + emitted = eventType + }) + task := &moderationTask{Coord: coord, Direction: domaincm.DirectionOutput, Modality: domaincm.ModalityText} + lateBlock := coord.onTaskResult(task, taskResult{Hit: true, EventID: "cme_late", Categories: []string{"violence"}}) + if lateBlock == nil { + t.Fatal("late hit must request background block compensation") + } + service.handleLateBlock(coord.meta, *lateBlock) + + repo.mu.Lock() + applyCalls := repo.applyCalls + repo.mu.Unlock() + if applyCalls != 1 { + t.Fatalf("expected one durable block apply, got %d", applyCalls) + } + if emitted != "moderation_blocked" { + t.Fatalf("expected recovery moderation_blocked event, got %q", emitted) + } +} + +func TestInputHitTakesPrecedenceOverOutputHit(t *testing.T) { + repo := &coordinatorTestRepo{} + service := NewService(nil, repo, "", nil) + cancelCalls := 0 + service.SetCancelRun(func(string) { cancelCalls++ }) + coord := newRunCoordinator(service, RunMeta{RunID: "run_input_priority"}, runtimeConfig{}) + coord.pending = 2 + coord.outputEnqueued = true + + outputTask := &moderationTask{Coord: coord, Direction: domaincm.DirectionOutput, Modality: domaincm.ModalityText} + coord.onTaskResult(outputTask, taskResult{Hit: true, EventID: "cme_output", Categories: []string{"violence"}}) + inputTask := &moderationTask{Coord: coord, Direction: domaincm.DirectionInput, Modality: domaincm.ModalityText} + coord.onTaskResult(inputTask, taskResult{Hit: true, EventID: "cme_input", Categories: []string{"hate"}}) + + blocked, info, _ := coord.settle() + if !blocked || info.Direction != domaincm.DirectionInput || info.EventID != "cme_input" { + t.Fatalf("input hit must win, got blocked=%v info=%#v", blocked, info) + } + if cancelCalls != 1 { + t.Fatalf("input hit must cancel generation once, got %d", cancelCalls) + } +} + +func TestLateInputHitUpgradesHandledOutputBlock(t *testing.T) { + repo := &coordinatorTestRepo{} + service := NewService(nil, repo, "", nil) + coord := newRunCoordinator(service, RunMeta{RunID: "run_late_input"}, runtimeConfig{}) + coord.pending = 1 + coord.outputEnqueued = true + coord.settled = true + coord.blocked = true + coord.blockHandled = true + coord.blockInfo = BlockInfo{EventID: "cme_output", Direction: domaincm.DirectionOutput} + + inputTask := &moderationTask{Coord: coord, Direction: domaincm.DirectionInput, Modality: domaincm.ModalityText} + lateBlock := coord.onTaskResult(inputTask, taskResult{Hit: true, EventID: "cme_input", Categories: []string{"hate"}}) + if lateBlock == nil || lateBlock.Direction != domaincm.DirectionInput { + t.Fatalf("late input hit must request an upgraded block, got %#v", lateBlock) + } + service.handleLateBlock(coord.meta, *lateBlock) + + repo.mu.Lock() + defer repo.mu.Unlock() + if len(repo.applyInputs) != 1 || !repo.applyInputs[0] { + t.Fatalf("late input block must include the user message, calls=%#v", repo.applyInputs) + } +} + +func TestWorkerPrefetchDoesNotBypassQueueCapacity(t *testing.T) { + repo := &coordinatorTestRepo{} + service := NewService(nil, repo, "", nil) + service.maxConcurrency = 1 + service.queueCapacity = 1 + service.activeWorkers = 1 // keep the worker waiting for a logical slot + + ctx, cancel := context.WithCancel(context.Background()) + service.wg.Add(1) + go service.workerLoop(ctx) + + coord := newRunCoordinator(service, RunMeta{RunID: "run_queue"}, runtimeConfig{}) + first := &moderationTask{Coord: coord, Direction: domaincm.DirectionOutput, Modality: domaincm.ModalityText} + if err := service.enqueue(first); err != nil { + t.Fatalf("enqueue first task: %v", err) + } + deadline := time.Now().Add(time.Second) + for len(service.taskQueue) != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if len(service.taskQueue) != 0 { + t.Fatal("worker did not prefetch first task") + } + + second := &moderationTask{Direction: domaincm.DirectionOutput, Modality: domaincm.ModalityText} + if err := service.enqueue(second); !errors.Is(err, ErrQueueFull) { + t.Fatalf("prefetched waiting task must still consume queue capacity, got %v", err) + } + cancel() + service.wg.Wait() +} + +func TestOutputImageLoadFailureIsAuditedAsFailedOpen(t *testing.T) { + repo := &coordinatorTestRepo{} + service := NewService(nil, repo, "", nil) + cfg := runtimeConfig{ + Policy: Policy{OutputImageCategories: []string{"violence"}}, + } + service.cachedConfig = &cfg + service.cachedAt = time.Now() + coord := newRunCoordinator(service, RunMeta{RunID: "run_image_load"}, cfg) + + coord.RecordOutputImageFailure("file_123", errors.New("object missing")) + + coord.mu.Lock() + failedOpen := coord.failedOpen + coord.mu.Unlock() + repo.mu.Lock() + defer repo.mu.Unlock() + if !failedOpen { + t.Fatal("missing output image must mark the run failed-open") + } + if len(repo.events) != 1 || repo.events[0].Result != domaincm.ResultFailedOpen { + t.Fatalf("expected one failed-open audit event, got %#v", repo.events) + } + if len(repo.stats) != 1 || repo.stats[0].FailureCount != 1 { + t.Fatalf("expected failed-open statistics, got %#v", repo.stats) + } +} + +func TestInputImageModerationSkipsNonImageAttachments(t *testing.T) { + repo := &coordinatorTestRepo{} + service := NewService(nil, repo, "", nil) + service.SetImageLoader(func(context.Context, uint, string) (PreparedImage, error) { + return PreparedImage{}, ErrNonImageAttachment + }) + cfg := runtimeConfig{Policy: Policy{InputImageCategories: []string{"violence"}}} + coord := newRunCoordinator(service, RunMeta{RunID: "run_document_attachment"}, cfg) + + coord.EnqueueInputImages(context.Background(), []string{"file_pdf"}) + + coord.mu.Lock() + failedOpen := coord.failedOpen + pending := coord.pending + coord.mu.Unlock() + repo.mu.Lock() + eventCount := len(repo.events) + repo.mu.Unlock() + if failedOpen || pending != 0 || eventCount != 0 { + t.Fatalf("non-image attachment must be ignored: failedOpen=%v pending=%d events=%d", failedOpen, pending, eventCount) + } +} + +func TestRecordHitRollsBackIsolatedImagesWhenEventCreateFails(t *testing.T) { + repo := &coordinatorTestRepo{createErr: errors.New("database unavailable")} + store := &coordinatorTestObjectStore{} + service := NewService(nil, repo, "test-encryption-key", nil) + service.SetObjectStore(store) + coord := newRunCoordinator(service, RunMeta{RunID: "run_rollback", UserID: 42}, runtimeConfig{}) + task := &moderationTask{ + Coord: coord, + Direction: domaincm.DirectionOutput, + Modality: domaincm.ModalityImage, + RawImages: []OutputImageSource{{FileID: "file_1", Data: []byte("image-bytes"), MimeType: "image/png"}}, + } + + if _, err := service.recordHit(context.Background(), task, HitEvaluation{ + Hit: true, + Categories: []string{"violence"}, + Scores: map[string]float64{"violence": 0.99}, + }, 10, nil); !errors.Is(err, repo.createErr) { + t.Fatalf("record hit error=%v, want %v", err, repo.createErr) + } + + store.mu.Lock() + defer store.mu.Unlock() + if len(store.put) != 1 || len(store.deleted) != 1 || store.put[0] != store.deleted[0] { + t.Fatalf("isolated image was not compensated: put=%#v deleted=%#v", store.put, store.deleted) + } +} diff --git a/backend/internal/application/contentmoderation/errs.go b/backend/internal/application/contentmoderation/errs.go new file mode 100644 index 000000000..62f52ff97 --- /dev/null +++ b/backend/internal/application/contentmoderation/errs.go @@ -0,0 +1,34 @@ +package contentmoderation + +import ( + "errors" + + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +var ( + ErrSuperAdminRequired = errors.New("superadmin permission required") + ErrAdminRequired = errors.New("admin permission required") + ErrInvalidConfig = errors.New("invalid content moderation config") + ErrServiceConfigRequired = errors.New("content moderation service config and policy are required when enabled") + ErrInvalidBaseURL = repository.ErrContentModerationInvalidBaseURL + ErrInvalidModel = errors.New("invalid content moderation model") + ErrInvalidTimeout = errors.New("content moderation timeout must be between 1 and 60 seconds") + ErrInvalidConcurrency = errors.New("content moderation max concurrency must be between 1 and 64") + ErrInvalidQueueCapacity = errors.New("content moderation queue capacity must be between 1 and 4096") + ErrInvalidCategories = errors.New("invalid content moderation categories") + ErrInvalidEventFilter = errors.New("invalid content moderation event filter") + ErrImageTextOnlyCategory = errors.New("text-only categories cannot be selected for image policies") + ErrEventNotFound = errors.New("content moderation event not found") + ErrProbeFailed = errors.New("content moderation probe failed") + ErrQueueFull = errors.New("content moderation queue is full") + ErrModerationTimeout = repository.ErrContentModerationTimeout + ErrModerationService = repository.ErrContentModerationService + ErrModerationRateLimited = repository.ErrContentModerationRateLimited + ErrModerationInvalidResp = repository.ErrContentModerationInvalidResp + ErrModerationNetwork = repository.ErrContentModerationNetwork + ErrWorkerLost = errors.New("content moderation worker lost") + // ErrNonImageAttachment lets image loaders skip ordinary files without + // turning an inapplicable image policy into a failed-open check. + ErrNonImageAttachment = errors.New("attachment is not an image") +) diff --git a/backend/internal/application/contentmoderation/policy.go b/backend/internal/application/contentmoderation/policy.go new file mode 100644 index 000000000..5eab0c209 --- /dev/null +++ b/backend/internal/application/contentmoderation/policy.go @@ -0,0 +1,123 @@ +package contentmoderation + +import ( + "sort" + "strings" +) + +// Policy holds the four category arrays (empty array = skip that surface). +type Policy struct { + InputTextCategories []string + OutputTextCategories []string + InputImageCategories []string + OutputImageCategories []string + Version int64 +} + +type policyJSON struct { + InputTextCategories []string `json:"inputTextCategories"` + OutputTextCategories []string `json:"outputTextCategories"` + InputImageCategories []string `json:"inputImageCategories"` + OutputImageCategories []string `json:"outputImageCategories"` + Version int64 `json:"version"` +} + +func newPolicyJSON(policy Policy) policyJSON { + return policyJSON{ + InputTextCategories: policy.InputTextCategories, + OutputTextCategories: policy.OutputTextCategories, + InputImageCategories: policy.InputImageCategories, + OutputImageCategories: policy.OutputImageCategories, + Version: policy.Version, + } +} + +func (document policyJSON) toPolicy() Policy { + return Policy{ + InputTextCategories: document.InputTextCategories, + OutputTextCategories: document.OutputTextCategories, + InputImageCategories: document.InputImageCategories, + OutputImageCategories: document.OutputImageCategories, + Version: document.Version, + } +} + +// Enabled reports whether any surface has categories selected. +func (p Policy) Enabled() bool { + return len(p.InputTextCategories) > 0 || + len(p.OutputTextCategories) > 0 || + len(p.InputImageCategories) > 0 || + len(p.OutputImageCategories) > 0 +} + +// CategoriesFor returns selected categories for a direction+modality surface. +func (p Policy) CategoriesFor(direction, modality string) []string { + switch { + case direction == DirectionInput && modality == ModalityText: + return append([]string(nil), p.InputTextCategories...) + case direction == DirectionOutput && modality == ModalityText: + return append([]string(nil), p.OutputTextCategories...) + case direction == DirectionInput && modality == ModalityImage: + return append([]string(nil), p.InputImageCategories...) + case direction == DirectionOutput && modality == ModalityImage: + return append([]string(nil), p.OutputImageCategories...) + default: + return nil + } +} + +// NormalizePolicy validates and normalizes category selections. +func NormalizePolicy(p Policy) (Policy, error) { + out := Policy{Version: p.Version} + var err error + if out.InputTextCategories, err = normalizeTextCategories(p.InputTextCategories); err != nil { + return Policy{}, err + } + if out.OutputTextCategories, err = normalizeTextCategories(p.OutputTextCategories); err != nil { + return Policy{}, err + } + if out.InputImageCategories, err = normalizeImageCategories(p.InputImageCategories); err != nil { + return Policy{}, err + } + if out.OutputImageCategories, err = normalizeImageCategories(p.OutputImageCategories); err != nil { + return Policy{}, err + } + return out, nil +} + +func normalizeTextCategories(raw []string) ([]string, error) { + return normalizeCategories(raw, false) +} + +func normalizeImageCategories(raw []string) ([]string, error) { + return normalizeCategories(raw, true) +} + +func normalizeCategories(raw []string, imageOnly bool) ([]string, error) { + if len(raw) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(raw)) + out := make([]string, 0, len(raw)) + for _, item := range raw { + cat := strings.TrimSpace(item) + if cat == "" { + continue + } + if !IsKnownCategory(cat) { + return nil, ErrInvalidCategories + } + if imageOnly { + if IsTextOnlyCategory(cat) || !IsImageCategory(cat) { + return nil, ErrImageTextOnlyCategory + } + } + if _, ok := seen[cat]; ok { + continue + } + seen[cat] = struct{}{} + out = append(out, cat) + } + sort.Strings(out) + return out, nil +} diff --git a/backend/internal/application/contentmoderation/probe.go b/backend/internal/application/contentmoderation/probe.go new file mode 100644 index 000000000..7c2a90bd1 --- /dev/null +++ b/backend/internal/application/contentmoderation/probe.go @@ -0,0 +1,95 @@ +package contentmoderation + +import ( + "context" + "encoding/base64" + "strings" + "time" +) + +// ProbeResult is the super-admin probe response for one modality. +type ProbeResult struct { + Valid bool + Model string + Latency int64 + Error string +} + +// ProbeResponse covers text and image probes. +type ProbeResponse struct { + Text ProbeResult + Image ProbeResult +} + +// 1x1 transparent PNG. +var probePNG, _ = base64.StdEncoding.DecodeString( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==", +) + +// Probe validates the saved config against built-in harmless samples. +func (s *Service) Probe(ctx context.Context, actorRole string) (*ProbeResponse, error) { + if !isSuperAdmin(actorRole) { + return nil, ErrSuperAdminRequired + } + cfg, err := s.readRuntimeConfig(ctx) + if err != nil { + return nil, err + } + out := &ProbeResponse{} + if s.provider == nil { + return nil, ErrModerationService + } + providerConfig := providerConfigFromRuntime(cfg) + + // Text probe + { + started := time.Now() + resp, err := s.provider.ModerateText(ctx, providerConfig, "hello", nil, ModalityText) + out.Text.Latency = time.Since(started).Milliseconds() + if err != nil { + out.Text.Error = err.Error() + } else if resp == nil || len(resp.Results) == 0 || resp.Results[0].Categories == nil { + out.Text.Error = ErrModerationInvalidResp.Error() + } else { + out.Text.Valid = true + out.Text.Model = firstNonEmpty(resp.Model, cfg.Model) + } + } + + // Image probe + { + started := time.Now() + resp, err := s.provider.ModerateImages(ctx, providerConfig, []ProviderImage{{Data: probePNG, MimeType: "image/png"}}, nil, ModalityImage) + out.Image.Latency = time.Since(started).Milliseconds() + if err != nil { + out.Image.Error = err.Error() + } else if resp == nil || len(resp.Results) == 0 || resp.Results[0].Categories == nil { + out.Image.Error = ErrModerationInvalidResp.Error() + } else { + // Official Omni responses include category_applied_input_types; require image proof. + applied := resp.Results[0].CategoryAppliedInputTypes + foundImage := false + if applied != nil { + for _, types := range applied { + for _, t := range types { + if strings.EqualFold(strings.TrimSpace(t), "image") { + foundImage = true + break + } + } + if foundImage { + break + } + } + } + if !foundImage { + out.Image.Valid = false + out.Image.Error = "moderation response missing image category_applied_input_types" + } else { + out.Image.Valid = true + out.Image.Model = firstNonEmpty(resp.Model, cfg.Model) + } + } + } + return out, nil +} diff --git a/backend/internal/application/contentmoderation/provider.go b/backend/internal/application/contentmoderation/provider.go new file mode 100644 index 000000000..830d1865f --- /dev/null +++ b/backend/internal/application/contentmoderation/provider.go @@ -0,0 +1,30 @@ +package contentmoderation + +import ( + "strings" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +type Provider = repository.ContentModerationProvider +type ProviderConfig = domaincm.ProviderConfig +type ProviderImage = domaincm.ProviderImage +type CategoryResult = domaincm.CategoryResult +type Response = domaincm.ProviderResponse +type HitEvaluation = domaincm.HitEvaluation + +func EvaluateHit(response *Response, selected []string, expectedModality string) HitEvaluation { + return domaincm.EvaluateHit(response, selected, expectedModality) +} + +func maskAPIKey(key string) string { + value := strings.TrimSpace(key) + if value == "" { + return "" + } + if len(value) <= 8 { + return "****" + } + return value[:4] + "..." + value[len(value)-4:] +} diff --git a/backend/internal/application/contentmoderation/provider_test.go b/backend/internal/application/contentmoderation/provider_test.go new file mode 100644 index 000000000..8e70b392b --- /dev/null +++ b/backend/internal/application/contentmoderation/provider_test.go @@ -0,0 +1,41 @@ +package contentmoderation + +import "testing" + +func TestEvaluateHitIgnoresTopLevelFlagged(t *testing.T) { + resp := &Response{Results: []CategoryResult{{ + Flagged: true, + Categories: map[string]bool{ + "hate": false, + "violence": false, + }, + CategoryScores: map[string]float64{"hate": 0.9}, + }}} + if eval := EvaluateHit(resp, []string{"hate", "violence"}, ModalityText); eval.Hit { + t.Fatal("expected no hit when selected categories are false") + } +} + +func TestEvaluateHitSelectedCategory(t *testing.T) { + resp := &Response{Results: []CategoryResult{{ + Categories: map[string]bool{"violence": true, "hate": true}, + CategoryScores: map[string]float64{ + "violence": 0.8, + "hate": 0.7, + }, + CategoryAppliedInputTypes: map[string][]string{ + "violence": {"text"}, + "hate": {"text"}, + }, + }}} + eval := EvaluateHit(resp, []string{"violence"}, ModalityText) + if !eval.Hit || len(eval.Categories) != 1 || eval.Categories[0] != "violence" { + t.Fatalf("unexpected evaluation: %#v", eval) + } +} + +func TestNormalizeImageCategoriesRejectsTextOnly(t *testing.T) { + if _, err := NormalizePolicy(Policy{InputImageCategories: []string{"hate"}}); err == nil { + t.Fatal("expected text-only category rejection") + } +} diff --git a/backend/internal/application/contentmoderation/service.go b/backend/internal/application/contentmoderation/service.go new file mode 100644 index 000000000..2cf4536b8 --- /dev/null +++ b/backend/internal/application/contentmoderation/service.go @@ -0,0 +1,446 @@ +package contentmoderation + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/secretbox" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "github.com/google/uuid" + "go.uber.org/zap" +) + +const ( + contentRetention = 30 * 24 * time.Hour + metadataRetention = 90 * 24 * time.Hour + cleanupInterval = 6 * time.Hour +) + +// EventEmitter publishes recovery-stream events for a run (optional). +type EventEmitter func(runID string, eventType string, payload map[string]interface{}) + +// CancelRun cancels in-flight upstream generation for a run. +type CancelRun func(runID string) + +// OnBlocked is invoked after a run is marked blocked (e.g. sanitize recovery stream). +type OnBlocked func(runID string, info BlockInfo) + +// PreparedImage is a resized moderation-ready image. +type PreparedImage struct { + Data []byte + SHA256 string + Mime string + Size int64 + FileID string +} + +// ImageLoader loads and prepares an image for moderation. +type ImageLoader func(ctx context.Context, userID uint, fileID string) (PreparedImage, error) + +// OutputImageSource provides final generated image bytes for isolation. +type OutputImageSource struct { + FileID string + Data []byte + MimeType string + SHA256 string +} + +// ObjectStore abstracts isolated image storage. +type ObjectStore interface { + Put(ctx context.Context, path string, data []byte, contentType string) error + Open(ctx context.Context, path string) ([]byte, error) + Delete(ctx context.Context, path string) error +} + +// FileAccessController marks ordinary generated files inaccessible after a hit. +type FileAccessController interface { + RevokeGeneratedFile(ctx context.Context, fileID string) error + DeleteGeneratedFileArtifacts(ctx context.Context, fileID string) error + RetryBlockedGeneratedFileDeletes(ctx context.Context, limit int) (int, error) +} + +type pendingBlock struct { + meta RunMeta + info BlockInfo +} + +// Service orchestrates config, workers, events, and run coordinators. +type Service struct { + settingsRepo repository.SettingsRepository + repo repository.ContentModerationRepository + dataEncryptionKey string + logger *zap.Logger + objectStore ObjectStore + fileAccess FileAccessController + imageLoader ImageLoader + emitEvent EventEmitter + cancelRun CancelRun + onBlocked OnBlocked + provider Provider + auditWriter auditWriter + + configMu sync.RWMutex + cachedConfig *runtimeConfig + cachedAt time.Time + + workerMu sync.Mutex + taskQueue chan *moderationTask + workerSem chan struct{} // fixed capacity maxPhysicalConcurrency; never replaced + workerWake chan struct{} // wakes workers waiting for a logical concurrency slot + maxConcurrency int + queueCapacity int + queuedCount int // logical admission counter (paired with queueCapacity) + activeWorkers int // logical concurrency counter (paired with maxConcurrency) + workerCount int + workerCtx context.Context + stopCh chan struct{} + wg sync.WaitGroup + + coordMu sync.Mutex + coordinators map[string]*RunCoordinator + + pendingBlockMu sync.Mutex + pendingBlocks map[string]pendingBlock +} + +// NewService creates the content moderation service. +func NewService( + settingsRepo repository.SettingsRepository, + repo repository.ContentModerationRepository, + dataEncryptionKey string, + logger *zap.Logger, +) *Service { + s := &Service{ + settingsRepo: settingsRepo, + repo: repo, + dataEncryptionKey: dataEncryptionKey, + logger: logger, + coordinators: make(map[string]*RunCoordinator), + pendingBlocks: make(map[string]pendingBlock), + stopCh: make(chan struct{}), + maxConcurrency: defaultMaxConcurrency, + queueCapacity: defaultQueueCapacity, + } + s.taskQueue = make(chan *moderationTask, maxPhysicalQueueCapacity) + // Fixed physical concurrency ceiling; logical maxConcurrency is enforced via activeWorkers. + s.workerSem = make(chan struct{}, maxPhysicalConcurrency) + s.workerWake = make(chan struct{}, maxPhysicalConcurrency) + return s +} + +func (s *Service) SetObjectStore(store ObjectStore) { s.objectStore = store } +func (s *Service) SetFileAccessController(c FileAccessController) { s.fileAccess = c } +func (s *Service) SetImageLoader(loader ImageLoader) { s.imageLoader = loader } +func (s *Service) SetEventEmitter(emit EventEmitter) { s.emitEvent = emit } +func (s *Service) SetCancelRun(cancel CancelRun) { s.cancelRun = cancel } +func (s *Service) SetOnBlocked(fn OnBlocked) { s.onBlocked = fn } + +// SetProvider injects the infrastructure adapter used for moderation calls. +func (s *Service) SetProvider(provider Provider) { + if s == nil { + return + } + s.provider = provider +} + +// SetAuditWriter injects the operation-audit sink used for privileged review reads. +func (s *Service) SetAuditWriter(writer auditWriter) { + if s != nil { + s.auditWriter = writer + } +} + +// StartBackgroundWorkers starts the worker pool and cleanup loop. +func (s *Service) StartBackgroundWorkers(ctx context.Context) { + s.workerCtx = ctx + if cfg, err := s.readRuntimeConfig(ctx); err == nil { + s.resizeWorker(cfg.MaxConcurrency, cfg.QueueCapacity) + } else { + s.ensureWorkers(ctx, s.maxConcurrency) + } + s.wg.Add(1) + go s.cleanupLoop(ctx) + go func() { + bg, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + s.runCleanup(bg) + s.recoverPendingBlocks(bg) + s.recoverStaleRuns(bg) + }() +} + +// Stop stops workers. +func (s *Service) Stop() { + select { + case <-s.stopCh: + default: + close(s.stopCh) + } + s.wg.Wait() +} + +// maxPhysicalQueueCapacity is the fixed channel buffer. Configured queueCapacity +// is enforced as a logical limit so resize never swaps channels under workers. +const maxPhysicalQueueCapacity = 4096 + +// maxPhysicalConcurrency is the fixed workerSem capacity. Logical maxConcurrency +// is enforced with activeWorkers so resize never replaces the semaphore channel. +const maxPhysicalConcurrency = 64 + +func (s *Service) resizeWorker(maxConcurrency, queueCapacity int) { + s.workerMu.Lock() + defer s.workerMu.Unlock() + if maxConcurrency < 1 { + maxConcurrency = defaultMaxConcurrency + } + if maxConcurrency > maxPhysicalConcurrency { + maxConcurrency = maxPhysicalConcurrency + } + if queueCapacity < 1 { + queueCapacity = defaultQueueCapacity + } + if queueCapacity > maxPhysicalQueueCapacity { + queueCapacity = maxPhysicalQueueCapacity + } + + // Logical limits only - never replace taskQueue or workerSem. + previousConcurrency := s.maxConcurrency + s.queueCapacity = queueCapacity + s.maxConcurrency = maxConcurrency + + if s.taskQueue == nil { + s.taskQueue = make(chan *moderationTask, maxPhysicalQueueCapacity) + } + if s.workerSem == nil { + s.workerSem = make(chan struct{}, maxPhysicalConcurrency) + } + if s.workerWake == nil { + s.workerWake = make(chan struct{}, maxPhysicalConcurrency) + } + if maxConcurrency > previousConcurrency { + for i := previousConcurrency; i < maxConcurrency; i++ { + select { + case s.workerWake <- struct{}{}: + default: + } + } + } + + ctx := s.workerCtx + if ctx == nil { + ctx = context.Background() + } + // Grow worker loops with configured demand. Existing loops are retained when + // the limit decreases, while the logical gate enforces the new lower limit. + for s.workerCount < maxConcurrency { + s.workerCount++ + s.wg.Add(1) + go s.workerLoop(ctx) + } +} + +func (s *Service) ensureWorkers(ctx context.Context, count int) { + s.workerMu.Lock() + defer s.workerMu.Unlock() + if count < 1 { + count = 1 + } + for s.workerCount < count { + s.workerCount++ + s.wg.Add(1) + go s.workerLoop(ctx) + } +} + +// BeginRun registers a per-run coordinator. Returns nil if moderation is fully disabled. +// Callers must ensure a conversation_runs row exists (EnsureConversationRun) before or +// immediately after BeginRun so UpdateRunModeration is not a silent no-op; SyncRunPending +// re-applies pending after the row is ensured. +func (s *Service) BeginRun(ctx context.Context, meta RunMeta) *RunCoordinator { + cfg, err := s.loadRuntimeConfig(ctx) + if err != nil { + // Configuration/storage failures are fail-open, but must remain observable. + // Return a coordinator so the conversation run is durably settled as + // failed_open instead of becoming indistinguishable from an intentionally + // disabled policy. + coord := newRunCoordinator(s, meta, runtimeConfig{Timeout: defaultTimeoutSeconds * time.Second}) + coord.failedOpen = true + s.coordMu.Lock() + s.coordinators[meta.RunID] = coord + s.coordMu.Unlock() + s.recordFailedOpen( + ctx, + meta, + domaincm.DirectionInput, + domaincm.ModalityText, + domaincm.ErrorCodeConfigMissing, + "content moderation configuration unavailable", + 0, + ) + s.bumpDailyStat(ctx, domaincm.DirectionInput, domaincm.ModalityText, domaincm.ResultFailedOpen, "", 1, 1, 0, 1, 0) + s.logWarn("content_moderation_config_load_failed", zap.String("run_id", meta.RunID), zap.Error(err)) + return coord + } + if !cfg.Enabled || !cfg.Policy.Enabled() { + return nil + } + coord := newRunCoordinator(s, meta, cfg) + s.coordMu.Lock() + s.coordinators[meta.RunID] = coord + s.coordMu.Unlock() + if err := s.repo.UpdateRunModeration(ctx, meta.RunID, domaincm.ModerationStatePending, "", "[]"); err != nil { + s.logWarn("content_moderation_mark_pending_failed", zap.String("run_id", meta.RunID), zap.Error(err)) + } + return coord +} + +// SyncRunPending marks a run pending after its conversation_runs row has been ensured. +func (s *Service) SyncRunPending(ctx context.Context, runID string) { + if s == nil || s.repo == nil { + return + } + if err := s.repo.UpdateRunModeration(ctx, strings.TrimSpace(runID), domaincm.ModerationStatePending, "", "[]"); err != nil { + s.logWarn("content_moderation_sync_pending_failed", zap.String("run_id", runID), zap.Error(err)) + } +} + +// GetCoordinator returns an active coordinator if present. +func (s *Service) GetCoordinator(runID string) *RunCoordinator { + s.coordMu.Lock() + defer s.coordMu.Unlock() + return s.coordinators[strings.TrimSpace(runID)] +} + +func (s *Service) releaseCoordinator(runID string) { + s.coordMu.Lock() + delete(s.coordinators, strings.TrimSpace(runID)) + s.coordMu.Unlock() +} + +func (s *Service) registerPendingBlock(meta RunMeta, info BlockInfo) { + if s == nil || strings.TrimSpace(meta.RunID) == "" { + return + } + s.pendingBlockMu.Lock() + s.pendingBlocks[strings.TrimSpace(meta.RunID)] = pendingBlock{meta: meta, info: info} + s.pendingBlockMu.Unlock() +} + +func (s *Service) removePendingBlock(runID string) { + if s == nil { + return + } + s.pendingBlockMu.Lock() + delete(s.pendingBlocks, strings.TrimSpace(runID)) + s.pendingBlockMu.Unlock() +} + +func (s *Service) hasPendingBlock(runID string) bool { + if s == nil { + return false + } + s.pendingBlockMu.Lock() + defer s.pendingBlockMu.Unlock() + _, ok := s.pendingBlocks[strings.TrimSpace(runID)] + return ok +} + +func (s *Service) pendingBlockSnapshot() []pendingBlock { + if s == nil { + return nil + } + s.pendingBlockMu.Lock() + defer s.pendingBlockMu.Unlock() + items := make([]pendingBlock, 0, len(s.pendingBlocks)) + for _, item := range s.pendingBlocks { + items = append(items, item) + } + return items +} + +// HasActiveCoordinator reports whether a run still has an in-memory coordinator. +func (s *Service) HasActiveCoordinator(runID string) bool { + return s.GetCoordinator(runID) != nil +} + +// RecoverRunIfStale fail-opens a moderating run with no live coordinator. +func (s *Service) RecoverRunIfStale(ctx context.Context, runID string) { + state, err := s.repo.GetRunModerationState(ctx, runID) + if err != nil { + return + } + if state != domaincm.ModerationStateModerating && state != domaincm.ModerationStatePending { + return + } + if s.HasActiveCoordinator(runID) { + return + } + if s.recoverKnownHit(ctx, runID) { + return + } + s.recordFailedOpen(ctx, RunMeta{RunID: runID}, domaincm.DirectionOutput, domaincm.ModalityText, domaincm.ErrorCodeWorkerLost, ErrWorkerLost.Error(), 0) + if err := s.repo.UpdateRunModeration(ctx, runID, domaincm.ModerationStateFailedOpen, "", "[]"); err != nil { + s.logWarn("content_moderation_recover_run_mark_failed_open_failed", zap.String("run_id", runID), zap.Error(err)) + } +} + +func providerConfigFromRuntime(cfg runtimeConfig) ProviderConfig { + return ProviderConfig{ + BaseURL: cfg.BaseURL, + APIKey: cfg.APIKey, + Model: cfg.Model, + Timeout: cfg.Timeout, + } +} + +func (s *Service) encryptText(plaintext string) (string, error) { + return secretbox.EncryptString(s.dataEncryptionKey, plaintext) +} + +func (s *Service) decryptText(ciphertext string) (string, error) { + return secretbox.DecryptString(s.dataEncryptionKey, ciphertext) +} + +// encryptBytes encrypts arbitrary binary (isolated images) as a v1: base64 payload string. +func (s *Service) encryptBytes(plaintext []byte) (string, error) { + return secretbox.Encrypt(s.dataEncryptionKey, plaintext) +} + +// decryptBytes decrypts a payload produced by encryptBytes. +func (s *Service) decryptBytes(ciphertext string) ([]byte, error) { + return secretbox.Decrypt(s.dataEncryptionKey, ciphertext) +} + +func newPublicEventID() string { + return "cme_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:24] +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func mustJSON(v interface{}) string { + raw, err := json.Marshal(v) + if err != nil { + return "{}" + } + return string(raw) +} + +func (s *Service) logWarn(msg string, fields ...zap.Field) { + if s.logger != nil { + s.logger.Warn(msg, fields...) + } +} + +func isolatedImagePath(eventPublicID string, index int, sha string) string { + return fmt.Sprintf("moderation-isolated/%s/%d_%s.bin", eventPublicID, index, sha[:min(16, len(sha))]) +} diff --git a/backend/internal/application/contentmoderation/storage_json.go b/backend/internal/application/contentmoderation/storage_json.go new file mode 100644 index 000000000..64f6c329f --- /dev/null +++ b/backend/internal/application/contentmoderation/storage_json.go @@ -0,0 +1,74 @@ +package contentmoderation + +import ( + "encoding/json" + "strings" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" +) + +// These private documents preserve the stored JSON schema without coupling domain types +// to a transport or persistence protocol. +type contentLocationJSON struct { + Field string `json:"field,omitempty"` + FileID string `json:"fileID,omitempty"` + Attachment int `json:"attachment,omitempty"` + ChunkIndex int `json:"chunkIndex,omitempty"` + ChunkCount int `json:"chunkCount,omitempty"` +} + +type isolatedImageMetaJSON struct { + Index int `json:"index"` + SHA256 string `json:"sha256"` + MimeType string `json:"mimeType"` + SizeBytes int64 `json:"sizeBytes"` + StoragePath string `json:"storagePath"` + SourceFileID string `json:"sourceFileID,omitempty"` +} + +func marshalContentLocation(location domaincm.ContentLocation) string { + return mustJSON(contentLocationJSON{ + Field: location.Field, + FileID: location.FileID, + Attachment: location.Attachment, + ChunkIndex: location.ChunkIndex, + ChunkCount: location.ChunkCount, + }) +} + +func marshalIsolatedImageMetadata(images []domaincm.IsolatedImageMeta) string { + documents := make([]isolatedImageMetaJSON, 0, len(images)) + for _, image := range images { + documents = append(documents, isolatedImageMetaJSON{ + Index: image.Index, + SHA256: image.SHA256, + MimeType: image.MimeType, + SizeBytes: image.SizeBytes, + StoragePath: image.StoragePath, + SourceFileID: image.SourceFileID, + }) + } + return mustJSON(documents) +} + +func unmarshalIsolatedImageMetadata(raw string) []domaincm.IsolatedImageMeta { + if strings.TrimSpace(raw) == "" { + return nil + } + var documents []isolatedImageMetaJSON + if err := json.Unmarshal([]byte(raw), &documents); err != nil { + return nil + } + images := make([]domaincm.IsolatedImageMeta, 0, len(documents)) + for _, document := range documents { + images = append(images, domaincm.IsolatedImageMeta{ + Index: document.Index, + SHA256: document.SHA256, + MimeType: document.MimeType, + SizeBytes: document.SizeBytes, + StoragePath: document.StoragePath, + SourceFileID: document.SourceFileID, + }) + } + return images +} diff --git a/backend/internal/application/contentmoderation/storage_json_test.go b/backend/internal/application/contentmoderation/storage_json_test.go new file mode 100644 index 000000000..d95b94016 --- /dev/null +++ b/backend/internal/application/contentmoderation/storage_json_test.go @@ -0,0 +1,61 @@ +package contentmoderation + +import ( + "encoding/json" + "reflect" + "testing" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" +) + +func TestStorageJSONPreservesExistingFieldNames(t *testing.T) { + location := domaincm.ContentLocation{ + Field: "assistant_images", + FileID: "file-1", + Attachment: 2, + ChunkIndex: 3, + ChunkCount: 4, + } + if got, want := marshalContentLocation(location), `{"field":"assistant_images","fileID":"file-1","attachment":2,"chunkIndex":3,"chunkCount":4}`; got != want { + t.Fatalf("location JSON = %s, want %s", got, want) + } + + images := []domaincm.IsolatedImageMeta{{ + Index: 1, + SHA256: "abc", + MimeType: "image/png", + SizeBytes: 42, + StoragePath: "moderation-isolated/event/1.bin", + SourceFileID: "file-1", + }} + raw := marshalIsolatedImageMetadata(images) + if got := unmarshalIsolatedImageMetadata(raw); !reflect.DeepEqual(got, images) { + t.Fatalf("image metadata round trip = %#v, want %#v", got, images) + } + if raw == "" || raw[0] != '[' { + t.Fatalf("unexpected image metadata JSON: %q", raw) + } +} + +func TestPolicyJSONPreservesExistingFieldNames(t *testing.T) { + policy := Policy{ + InputTextCategories: []string{"hate"}, + OutputTextCategories: []string{}, + InputImageCategories: []string{"violence"}, + OutputImageCategories: []string{}, + Version: 7, + } + raw, err := json.Marshal(newPolicyJSON(policy)) + if err != nil { + t.Fatal(err) + } + var document map[string]interface{} + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatal(err) + } + for _, key := range []string{"inputTextCategories", "outputTextCategories", "inputImageCategories", "outputImageCategories", "version"} { + if _, ok := document[key]; !ok { + t.Fatalf("policy JSON missing %q: %s", key, raw) + } + } +} diff --git a/backend/internal/application/contentmoderation/worker.go b/backend/internal/application/contentmoderation/worker.go new file mode 100644 index 000000000..4a41c4660 --- /dev/null +++ b/backend/internal/application/contentmoderation/worker.go @@ -0,0 +1,554 @@ +package contentmoderation + +import ( + "context" + "errors" + "strings" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "go.uber.org/zap" +) + +type moderationTask struct { + Coord *RunCoordinator + Direction string + Modality string + Text string + FileIDs []string + Selected []string + Location domaincm.ContentLocation + RawImages []OutputImageSource + IsolateOnly bool // input images: isolate copy only, do not revoke user files +} + +type taskResult struct { + Hit bool + Categories []string + Scores map[string]float64 + EventID string + LatencyMS int64 + Err error + ErrorCode string +} + +func (s *Service) workerLoop(ctx context.Context) { + defer s.wg.Done() + for { + select { + case <-ctx.Done(): + return + case <-s.stopCh: + return + case task, ok := <-s.taskQueue: + if !ok || task == nil { + return + } + + // Physical token (fixed channel) + logical activeWorkers gate. + select { + case <-ctx.Done(): + s.markTaskDequeued() + if task.Coord != nil { + task.Coord.onTaskResult(task, taskResult{Err: ErrModerationTimeout, ErrorCode: domaincm.ErrorCodeTimeout}) + } + return + case <-s.stopCh: + s.markTaskDequeued() + if task.Coord != nil { + task.Coord.onTaskResult(task, taskResult{Err: ErrWorkerLost, ErrorCode: domaincm.ErrorCodeWorkerLost}) + } + return + case s.workerSem <- struct{}{}: + } + if !s.waitLogicalSlot(ctx) { + s.markTaskDequeued() + if task.Coord != nil { + task.Coord.onTaskResult(task, taskResult{Err: ErrWorkerLost, ErrorCode: domaincm.ErrorCodeWorkerLost}) + } + <-s.workerSem + continue + } + s.markTaskDequeued() + s.executeTask(ctx, task) + s.releaseLogicalSlot() + <-s.workerSem + } + } +} + +func (s *Service) markTaskDequeued() { + s.workerMu.Lock() + if s.queuedCount > 0 { + s.queuedCount-- + } + s.workerMu.Unlock() +} + +// waitLogicalSlot blocks until activeWorkers < maxConcurrency. +// Returns false if the worker is shutting down before a slot is acquired. +func (s *Service) waitLogicalSlot(ctx context.Context) bool { + for { + s.workerMu.Lock() + limit := s.maxConcurrency + if limit < 1 { + limit = defaultMaxConcurrency + } + if s.activeWorkers < limit { + s.activeWorkers++ + s.workerMu.Unlock() + return true + } + wake := s.workerWake + s.workerMu.Unlock() + select { + case <-ctx.Done(): + return false + case <-s.stopCh: + return false + case <-wake: + } + } +} + +func (s *Service) releaseLogicalSlot() { + s.workerMu.Lock() + if s.activeWorkers > 0 { + s.activeWorkers-- + } + wake := s.workerWake + s.workerMu.Unlock() + if wake != nil { + select { + case wake <- struct{}{}: + default: + } + } +} + +func (s *Service) enqueue(task *moderationTask) error { + if task == nil { + return nil + } + s.workerMu.Lock() + limit := s.queueCapacity + if limit < 1 { + limit = defaultQueueCapacity + } + if s.queuedCount >= limit { + s.workerMu.Unlock() + if task.Coord != nil { + bg, cancel := context.WithTimeout(context.Background(), 5*time.Second) + s.recordFailedOpen(bg, task.Coord.meta, task.Direction, task.Modality, domaincm.ErrorCodeQueueFull, ErrQueueFull.Error(), 0) + s.bumpDailyStat(bg, task.Direction, task.Modality, domaincm.ResultFailedOpen, "", 1, contentItemCount(task), 0, 1, 0) + cancel() + } + return ErrQueueFull + } + queue := s.taskQueue + select { + case queue <- task: + s.queuedCount++ + s.workerMu.Unlock() + return nil + default: + s.workerMu.Unlock() + if task.Coord != nil { + bg, cancel := context.WithTimeout(context.Background(), 5*time.Second) + s.recordFailedOpen(bg, task.Coord.meta, task.Direction, task.Modality, domaincm.ErrorCodeQueueFull, ErrQueueFull.Error(), 0) + s.bumpDailyStat(bg, task.Direction, task.Modality, domaincm.ResultFailedOpen, "", 1, contentItemCount(task), 0, 1, 0) + cancel() + } + return ErrQueueFull + } +} + +func (s *Service) executeTask(parent context.Context, task *moderationTask) { + started := time.Now() + cfg := task.Coord.cfg + timeout := cfg.Timeout + if timeout <= 0 { + timeout = 10 * time.Second + } + // Parent may be cancelled; moderation work and persistence use a detached budget. + workParent := context.WithoutCancel(parent) + ctx, cancel := context.WithTimeout(workParent, timeout) + defer cancel() + + selected := task.Selected + if len(selected) == 0 { + selected = cfg.Policy.CategoriesFor(task.Direction, task.Modality) + } + + var ( + resp *Response + err error + ) + if s.provider == nil { + err = ErrModerationService + } else { + providerConfig := providerConfigFromRuntime(cfg) + switch task.Modality { + case domaincm.ModalityImage: + images := make([]ProviderImage, 0, len(task.RawImages)) + for _, image := range task.RawImages { + if len(image.Data) == 0 { + continue + } + images = append(images, ProviderImage{Data: image.Data, MimeType: image.MimeType}) + } + resp, err = s.provider.ModerateImages(ctx, providerConfig, images, selected, task.Modality) + default: + resp, err = s.provider.ModerateText(ctx, providerConfig, task.Text, selected, task.Modality) + } + } + latency := time.Since(started).Milliseconds() + // Start a fresh persistence budget only after the upstream request finishes. + // Slow moderation requests must not consume the time reserved for recording + // their result and statistics. + persistCtx, persistCancel := context.WithTimeout(context.WithoutCancel(parent), 10*time.Second) + defer persistCancel() + + if err != nil { + code := classifyErrorCode(err) + s.recordFailedOpen(persistCtx, task.Coord.meta, task.Direction, task.Modality, code, err.Error(), latency) + s.bumpDailyStat(persistCtx, task.Direction, task.Modality, domaincm.ResultFailedOpen, "", 1, contentItemCount(task), 0, 1, latency) + task.Coord.onTaskResult(task, taskResult{ + Err: err, + ErrorCode: code, + LatencyMS: latency, + }) + return + } + + eval := EvaluateHit(resp, selected, task.Modality) + if !eval.Hit { + if _, recordErr := s.recordPass(persistCtx, task, latency, resp); recordErr != nil { + s.logWarn("content_moderation_record_pass_failed", zap.Error(recordErr)) + } + s.bumpDailyStat(persistCtx, task.Direction, task.Modality, domaincm.ResultPassed, "", 1, contentItemCount(task), 0, 0, latency) + task.Coord.onTaskResult(task, taskResult{LatencyMS: latency}) + return + } + + eventID, recordErr := s.recordHit(persistCtx, task, eval, latency, resp) + if recordErr != nil { + s.logWarn("content_moderation_record_hit_failed", zap.Error(recordErr)) + } + s.bumpDailyStat(persistCtx, task.Direction, task.Modality, domaincm.ResultHit, "", 1, contentItemCount(task), 1, 0, latency) + for _, cat := range eval.Categories { + s.bumpDailyStat(persistCtx, task.Direction, task.Modality, domaincm.ResultHit, cat, 0, 0, 1, 0, 0) + } + lateBlock := task.Coord.onTaskResult(task, taskResult{ + Hit: true, + Categories: eval.Categories, + Scores: eval.Scores, + EventID: eventID, + LatencyMS: latency, + }) + if lateBlock != nil { + s.handleLateBlock(task.Coord.meta, *lateBlock) + } +} + +func contentItemCount(task *moderationTask) int64 { + if task == nil { + return 0 + } + if task.Modality == domaincm.ModalityImage { + n := len(task.RawImages) + if n == 0 { + return 1 + } + return int64(n) + } + return 1 +} + +func classifyErrorCode(err error) string { + switch { + case err == nil: + return "" + case errors.Is(err, ErrQueueFull): + return domaincm.ErrorCodeQueueFull + case errors.Is(err, ErrModerationTimeout): + return domaincm.ErrorCodeTimeout + case errors.Is(err, ErrModerationRateLimited): + return domaincm.ErrorCodeRateLimited + case errors.Is(err, ErrModerationInvalidResp): + return domaincm.ErrorCodeInvalidResp + case errors.Is(err, ErrModerationNetwork): + return domaincm.ErrorCodeNetworkError + case errors.Is(err, ErrWorkerLost): + return domaincm.ErrorCodeWorkerLost + default: + return domaincm.ErrorCodeServiceError + } +} + +func (s *Service) recordFailedOpen( + ctx context.Context, + meta RunMeta, + direction, modality, errorCode, errorMessage string, + latencyMS int64, +) { + if s.repo == nil { + return + } + now := time.Now() + event := &domaincm.Event{ + PublicID: newPublicEventID(), + UserID: meta.UserID, + ConversationID: meta.ConversationID, + RunID: meta.RunID, + MessageID: meta.MessageID, + MessagePublicID: meta.MessagePublicID, + Direction: direction, + Modality: modality, + Result: domaincm.ResultFailedOpen, + CategoriesJSON: "[]", + CategoryScoresJSON: "{}", + LatencyMS: latencyMS, + ErrorCode: errorCode, + ErrorMessage: truncate(errorMessage, 255), + ContentLocationJSON: "{}", + ContentSummary: "", + ImageMetaJSON: "[]", + ContentExpiresAt: now.Add(contentRetention), + MetadataExpiresAt: now.Add(metadataRetention), + } + if cfg, err := s.loadRuntimeConfig(ctx); err == nil { + event.Model = cfg.Model + event.PolicyVersion = cfg.Policy.Version + } + if err := s.repo.CreateEvent(ctx, event); err != nil { + s.logWarn("content_moderation_failed_open_event_failed", zap.Error(err)) + } + s.logWarn("content_moderation_failed_open", + zap.String("run_id", meta.RunID), + zap.String("direction", direction), + zap.String("modality", modality), + zap.String("error_code", errorCode), + ) +} + +func (s *Service) recordPass(ctx context.Context, task *moderationTask, latencyMS int64, resp *Response) (string, error) { + if s.repo == nil || task == nil || task.Coord == nil { + return "", nil + } + now := time.Now() + publicID := newPublicEventID() + modelName := task.Coord.cfg.Model + policyVersion := task.Coord.cfg.Policy.Version + if resp != nil && strings.TrimSpace(resp.Model) != "" { + modelName = resp.Model + } + summary := "image_pass" + if task.Modality == domaincm.ModalityText { + if task.Direction == domaincm.DirectionInput { + summary = "input_text_pass" + } else { + summary = "output_text_pass" + } + } + // Pass events keep metadata only — do not retain encrypted content payloads. + event := &domaincm.Event{ + PublicID: publicID, + UserID: task.Coord.meta.UserID, + ConversationID: task.Coord.meta.ConversationID, + RunID: task.Coord.meta.RunID, + MessageID: task.Coord.meta.MessageID, + MessagePublicID: task.Coord.meta.MessagePublicID, + Direction: task.Direction, + Modality: task.Modality, + Model: modelName, + PolicyVersion: policyVersion, + Result: domaincm.ResultPassed, + CategoriesJSON: "[]", + CategoryScoresJSON: "{}", + LatencyMS: latencyMS, + ContentLocationJSON: marshalContentLocation(task.Location), + ContentSummary: summary, + ImageMetaJSON: "[]", + ContentExpiresAt: now, + MetadataExpiresAt: now.Add(metadataRetention), + } + if err := s.repo.CreateEvent(ctx, event); err != nil { + return publicID, err + } + return publicID, nil +} + +func (s *Service) deleteUntrackedIsolatedImages(ctx context.Context, eventID string, images []domaincm.IsolatedImageMeta) { + if s == nil || s.objectStore == nil { + return + } + for _, image := range images { + path := strings.TrimSpace(image.StoragePath) + if path == "" { + continue + } + if err := s.objectStore.Delete(ctx, path); err != nil { + s.logWarn( + "content_moderation_rollback_isolated_image_failed", + zap.String("event_id", eventID), + zap.String("path", path), + zap.Error(err), + ) + } + } +} + +func (s *Service) recordHit(ctx context.Context, task *moderationTask, eval HitEvaluation, latencyMS int64, resp *Response) (string, error) { + now := time.Now() + publicID := newPublicEventID() + modelName := "" + policyVersion := int64(0) + if task.Coord != nil { + modelName = task.Coord.cfg.Model + policyVersion = task.Coord.cfg.Policy.Version + } + if resp != nil && strings.TrimSpace(resp.Model) != "" { + modelName = resp.Model + } + + encryptedText := "" + // Opaque summary only — never store plaintext snippets in the list metadata. + summary := "image_hit" + if task.Modality == domaincm.ModalityText { + if enc, err := s.encryptText(task.Text); err == nil { + encryptedText = enc + } + if task.Direction == domaincm.DirectionInput { + summary = "input_text_hit" + } else { + summary = "output_text_hit" + } + } + + imageMeta := make([]domaincm.IsolatedImageMeta, 0) + if task.Modality == domaincm.ModalityImage && len(task.RawImages) > 0 { + for i, img := range task.RawImages { + data := img.Data + if len(data) == 0 { + // Still revoke/delete output images even when payload bytes are missing. + if !task.IsolateOnly && s.fileAccess != nil && strings.TrimSpace(img.FileID) != "" { + if err := s.fileAccess.RevokeGeneratedFile(ctx, img.FileID); err != nil { + s.logWarn("content_moderation_revoke_file_failed", zap.String("file_id", img.FileID), zap.Error(err)) + } + if err := s.fileAccess.DeleteGeneratedFileArtifacts(ctx, img.FileID); err != nil { + s.logWarn("content_moderation_delete_file_artifacts_failed", zap.String("file_id", img.FileID), zap.Error(err)) + } + } + continue + } + sha := img.SHA256 + if sha == "" { + sha = sha256Hex(data) + } + // Attempt isolation copy; output revoke/delete always runs regardless of isolation success. + if s.objectStore != nil { + encStr, err := s.encryptBytes(data) + if err != nil { + s.logWarn("content_moderation_encrypt_image_failed", zap.Error(err)) + } else { + path := isolatedImagePath(publicID, i, sha) + if err := s.objectStore.Put(ctx, path, []byte(encStr), "application/octet-stream"); err != nil { + s.logWarn("content_moderation_isolate_image_failed", zap.Error(err)) + } else { + imageMeta = append(imageMeta, domaincm.IsolatedImageMeta{ + Index: i, + SHA256: sha, + MimeType: firstNonEmpty(img.MimeType, "image/png"), + SizeBytes: int64(len(data)), + StoragePath: path, + SourceFileID: img.FileID, + }) + } + } + } + if !task.IsolateOnly && s.fileAccess != nil && strings.TrimSpace(img.FileID) != "" { + if err := s.fileAccess.RevokeGeneratedFile(ctx, img.FileID); err != nil { + s.logWarn("content_moderation_revoke_file_failed", zap.String("file_id", img.FileID), zap.Error(err)) + } + if err := s.fileAccess.DeleteGeneratedFileArtifacts(ctx, img.FileID); err != nil { + s.logWarn("content_moderation_delete_file_artifacts_failed", zap.String("file_id", img.FileID), zap.Error(err)) + } + } + } + } + + event := &domaincm.Event{ + PublicID: publicID, + UserID: task.Coord.meta.UserID, + ConversationID: task.Coord.meta.ConversationID, + RunID: task.Coord.meta.RunID, + MessageID: task.Coord.meta.MessageID, + MessagePublicID: task.Coord.meta.MessagePublicID, + Direction: task.Direction, + Modality: task.Modality, + Model: modelName, + PolicyVersion: policyVersion, + Result: domaincm.ResultHit, + CategoriesJSON: mustJSON(eval.Categories), + CategoryScoresJSON: mustJSON(eval.Scores), + LatencyMS: latencyMS, + ContentLocationJSON: marshalContentLocation(task.Location), + ContentSummary: summary, + EncryptedText: encryptedText, + ImageCount: len(imageMeta), + ImageMetaJSON: marshalIsolatedImageMetadata(imageMeta), + ContentExpiresAt: now.Add(contentRetention), + MetadataExpiresAt: now.Add(metadataRetention), + } + if err := s.repo.CreateEvent(ctx, event); err != nil { + rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + s.deleteUntrackedIsolatedImages(rollbackCtx, publicID, imageMeta) + cancel() + return publicID, err + } + return publicID, nil +} + +func (s *Service) bumpDailyStat( + ctx context.Context, + direction, modality, result, category string, + checkCount, contentItems, hitCount, failureCount, latencyMS int64, +) { + if s.repo == nil { + return + } + day := time.Now().UTC().Truncate(24 * time.Hour) + if err := s.repo.IncrementDailyStat(ctx, repository.DailyStatIncrement{ + StatDate: day, + Direction: direction, + Modality: modality, + Result: result, + Category: category, + CheckCount: checkCount, + ContentItems: contentItems, + HitCount: hitCount, + FailureCount: failureCount, + LatencyMS: latencyMS, + }); err != nil { + s.logWarn("content_moderation_increment_daily_stat_failed", zap.Error(err)) + } +} + +func truncate(value string, max int) string { + value = strings.TrimSpace(value) + if max <= 0 || len(value) <= max { + return value + } + return value[:max] +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} diff --git a/backend/internal/application/conversation/generation_stream.go b/backend/internal/application/conversation/generation_stream.go index defeda2ff..e7b634219 100644 --- a/backend/internal/application/conversation/generation_stream.go +++ b/backend/internal/application/conversation/generation_stream.go @@ -223,11 +223,29 @@ func (r *generationStreamRegistry) cancel(ctx context.Context, userID uint, runI if !r.authorized(ctx, r.store, runID, userID) { return false } + return r.cancelActive(ctx, userID, runID, false) +} + +// cancelForced cancels a run without owner checks (internal system paths such as moderation). +func (r *generationStreamRegistry) cancelForced(ctx context.Context, runID string) bool { + if runID == "" { + return false + } + return r.cancelActive(ctx, 0, runID, true) +} + +func (r *generationStreamRegistry) cancelActive(ctx context.Context, userID uint, runID string, force bool) bool { if r.store != nil { _ = r.store.RequestGenerationStreamCancel(ctx, runID, r.options.Retention) } - active, ok := r.deleteActive(userID, runID) + var active *activeGeneration + var ok bool + if force { + active, ok = r.deleteActiveAny(runID) + } else { + active, ok = r.deleteActive(userID, runID) + } if ok { stopActiveGeneration(active) } @@ -238,6 +256,20 @@ func (r *generationStreamRegistry) cancel(ctx context.Context, userID uint, runI return true } +func (r *generationStreamRegistry) deleteActiveAny(runID string) (*activeGeneration, bool) { + if runID == "" { + return nil, false + } + r.mu.Lock() + defer r.mu.Unlock() + active, ok := r.active[runID] + if !ok { + return nil, false + } + delete(r.active, runID) + return active, true +} + func (r *generationStreamRegistry) isCanceled(ctx context.Context, runID string) bool { if runID == "" { return false @@ -274,6 +306,14 @@ func (r *generationStreamRegistry) publish(ctx context.Context, runID string, pa return actual } +// resetEvents clears retained stream events so blocked content cannot be replayed. +func (r *generationStreamRegistry) resetEvents(ctx context.Context, runID string) { + if runID == "" || r.store == nil { + return + } + _ = r.store.ResetGenerationStreamEvents(ctx, runID) +} + func (r *generationStreamRegistry) append(ctx context.Context, store repository.GenerationStreamCacheRepository, runID string, payloadJSON string) (repository.GenerationStreamMessage, error) { if store == nil { return repository.GenerationStreamMessage{}, nil @@ -710,7 +750,7 @@ func cloneStreamPayload(payload map[string]interface{}) map[string]interface{} { func isTerminalStreamPayload(payload map[string]interface{}) bool { eventType, _ := payload["type"].(string) - return eventType == "completed" || eventType == "error" + return eventType == "completed" || eventType == "error" || eventType == "moderation_blocked" } func int64FromPayload(raw interface{}) int64 { diff --git a/backend/internal/application/conversation/generation_stream_test.go b/backend/internal/application/conversation/generation_stream_test.go index 6661251fc..290753dbd 100644 --- a/backend/internal/application/conversation/generation_stream_test.go +++ b/backend/internal/application/conversation/generation_stream_test.go @@ -419,6 +419,15 @@ func (s *testGenerationStreamStore) ReadGenerationStreamEvents(ctx context.Conte return results, nil } +func (s *testGenerationStreamStore) ResetGenerationStreamEvents(_ context.Context, runID string) error { + s.mu.Lock() + defer s.mu.Unlock() + if item, ok := s.items[runID]; ok { + item.events = nil + } + return nil +} + func (s *testGenerationStreamStore) ExpireGenerationStream(_ context.Context, runID string, ttl time.Duration) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/backend/internal/application/conversation/prompt_scope.go b/backend/internal/application/conversation/prompt_scope.go index 1583afc65..8508d25de 100644 --- a/backend/internal/application/conversation/prompt_scope.go +++ b/backend/internal/application/conversation/prompt_scope.go @@ -1,12 +1,18 @@ package conversation import ( + "strings" + appcompact "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/compact" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" ) +func stringsEqualFold(a, b string) bool { + return strings.EqualFold(strings.TrimSpace(a), strings.TrimSpace(b)) +} + type promptScope struct { FullBranchMessages []model.Message CoveredMessages []model.Message @@ -75,6 +81,9 @@ func historyMessagesFromDomain(messages []model.Message, options historyMessageO if item.Role != "user" && item.Role != "assistant" && item.Role != "system" { continue } + if stringsEqualFold(item.Status, "blocked") { + continue + } message := llm.Message{ Role: item.Role, Content: item.Content, diff --git a/backend/internal/application/conversation/service.go b/backend/internal/application/conversation/service.go index a84bb62e2..4faa616f9 100644 --- a/backend/internal/application/conversation/service.go +++ b/backend/internal/application/conversation/service.go @@ -9,6 +9,7 @@ import ( appbilling "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/billing" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/channel" appcompact "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/compact" + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" appembedding "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/embedding" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/extraction" appstorage "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/objectstorage" @@ -113,6 +114,7 @@ type Service struct { auditWriter auditWriter storeProvider appstorage.Provider logger *zap.Logger + moderationSvc *appcm.Service toolLimiters sync.Map generationStreams *generationStreamRegistry snapshotCache sync.Map // conversationID (uint) → *cachedSnapshot @@ -187,27 +189,29 @@ func (s *Service) SetSkillResolver(resolver skillResolver) { // SendMessageResult 返回用户消息与 AI 消息。 type SendMessageResult struct { - UserMessage model.Message - AssistantMessage model.Message - MetadataRefreshHint string - Billable bool - UpstreamID uint - UpstreamName string - PlatformModelName string - RoutedBindingCode string - UpstreamModelName string - UpstreamProtocol string - EffectiveOptions map[string]interface{} - UsageSpeed string - UsageServiceTier string - UsageSource string - RawUsageJSON string - CacheWrite5mTokens int64 - CacheWrite1hTokens int64 - ServerSideToolUsage map[string]int64 - LatencyMS int64 - DurationSeconds int64 - StartedAt time.Time + UserMessage model.Message + AssistantMessage model.Message + MetadataRefreshHint string + Billable bool + UpstreamID uint + UpstreamName string + PlatformModelName string + RoutedBindingCode string + UpstreamModelName string + UpstreamProtocol string + EffectiveOptions map[string]interface{} + UsageSpeed string + UsageServiceTier string + UsageSource string + RawUsageJSON string + CacheWrite5mTokens int64 + CacheWrite1hTokens int64 + ServerSideToolUsage map[string]int64 + LatencyMS int64 + DurationSeconds int64 + StartedAt time.Time + // Moderation is set when a soft-moderation barrier ran; Blocked means withdrawn. + Moderation *MessageModerationOutcome postBillingCompaction *postBillingCompactionTask } diff --git a/backend/internal/application/conversation/service_media_generation.go b/backend/internal/application/conversation/service_media_generation.go index b5900496f..6c884fe15 100644 --- a/backend/internal/application/conversation/service_media_generation.go +++ b/backend/internal/application/conversation/service_media_generation.go @@ -12,6 +12,7 @@ import ( "time" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/channel" + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" @@ -176,23 +177,51 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( StartedAt: startedAt, } var retErr error + var moderationCoord *appcm.RunCoordinator + var result *SendMessageResult + var userMessage *model.Message + var assistantMessage *model.Message defer func() { + // On generation failure, still finish input-only moderation when active. + if retErr != nil && moderationCoord != nil { + if result == nil && userMessage != nil && assistantMessage != nil { + result = &SendMessageResult{ + UserMessage: *userMessage, + AssistantMessage: *assistantMessage, + Billable: false, + StartedAt: startedAt, + } + } + s.completeModerationAfterFailure(context.WithoutCancel(ctx), moderationCoord, result) + } endedAt := time.Now() run.EndedAt = &endedAt run.TotalLatencyMS = endedAt.Sub(startedAt).Milliseconds() - if retErr == nil { + switch { + case result != nil && result.IsModerationBlocked(): + applyBlockedRunFields(run, result) + case retErr == nil: run.Status = "success" - } else if errors.Is(retErr, ErrMessageGenerationCanceled) { + if result != nil { + applyModerationRunState(run, result) + } + case errors.Is(retErr, ErrMessageGenerationCanceled): run.Status = "canceled" run.ErrorCode = classifyRunErrorCode(retErr) run.ErrorMessage = truncateError(messageErrorSummary(retErr), 255) - } else { + if result != nil { + applyModerationRunState(run, result) + } + default: run.Status = "error" run.ErrorCode = classifyRunErrorCode(retErr) run.ErrorMessage = truncateError(messageErrorSummary(retErr), 255) + if result != nil { + applyModerationRunState(run, result) + } } - if err := s.repo.CreateConversationRun(context.WithoutCancel(ctx), run); err != nil && s.logger != nil { - s.logger.Error("create_media_conversation_run_failed", + if err := s.repo.UpsertConversationRun(context.WithoutCancel(ctx), run); err != nil && s.logger != nil { + s.logger.Error("upsert_media_conversation_run_failed", zap.String("trace_id", traceid.FromContext(ctx)), zap.String("run_id", run.RunID), zap.Error(err), @@ -203,7 +232,7 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( ctx = cancelCtx s.generationStreams.register(ctx, runID, input.UserID, cancel) - assistantMessage := &model.Message{ + assistantMessage = &model.Message{ ConversationID: input.ConversationID, UserID: input.UserID, PublicID: normalizePublicID(uuid.NewString()), @@ -215,7 +244,6 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( Status: "pending", Attachments: "[]", } - var userMessage *model.Message if reuseUserMessage { reused := *branchState.ReuseUserMessage userMessage = &reused @@ -276,11 +304,29 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( } traceRecorder := newMessageTraceRecorder(s, ctx, assistantMessage, input.OnEvent) defer func() { - if retErr != nil && traceRecorder != nil { + if retErr != nil && (result == nil || !result.IsModerationBlocked()) && traceRecorder != nil { traceRecorder.fail(retErr) traceRecorder.attachToMessage(assistantMessage) } }() + // Prefer explicit FileIDs; fall back to resolved edit attachments. + moderationFileIDs := append([]string{}, input.FileIDs...) + if len(moderationFileIDs) == 0 { + for _, item := range resolvedAttachments { + if id := strings.TrimSpace(item.FileID); id != "" { + moderationFileIDs = append(moderationFileIDs, id) + } + } + } + moderationCoord = s.startModerationRun(ctx, SendMessageInput{ + UserID: input.UserID, + ConversationID: input.ConversationID, + RequestID: input.RequestID, + Content: strings.TrimSpace(input.Prompt), + FileIDs: moderationFileIDs, + ClientRunID: runID, + OnEvent: input.OnEvent, + }, runID, userMessage, assistantMessage, run) emitMediaEvent(input.OnEvent, "queued", "image task queued") cfg := s.cfg.Snapshot() @@ -409,6 +455,7 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( } uploaded := make([]model.FileObject, 0, len(output.GeneratedImages)) attachmentRows := make([]model.Attachment, 0, len(output.GeneratedImages)) + generatedBytesByFileID := make(map[string][]byte, len(output.GeneratedImages)) now := time.Now() for i, image := range output.GeneratedImages { data, mimeType, readErr := s.readGeneratedImage(ctx, image, route.BaseURL) @@ -432,6 +479,7 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( } file := uploadResult.File uploaded = append(uploaded, file) + generatedBytesByFileID[file.FileID] = data attachmentRows = append(attachmentRows, model.Attachment{ ConversationID: input.ConversationID, MessageID: assistantMessage.ID, @@ -526,7 +574,7 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( run.CacheWriteTokens = usage.CacheWriteTokens run.ReasoningTokens = usage.ReasoningTokens - return &SendMessageResult{ + result = &SendMessageResult{ UserMessage: *userMessage, AssistantMessage: *assistantMessage, MetadataRefreshHint: s.resolveConversationMetadataRefreshHint(ctx, *conversation, *userMessage), @@ -545,7 +593,23 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( CacheWrite1hTokens: usage.CacheWrite1hTokens, LatencyMS: latencyMS, StartedAt: startedAt, - }, nil + } + if moderationCoord != nil { + outputImages := loadOutputImagesFromFiles(moderationCoord, uploaded, generatedBytesByFileID) + s.completeModerationAfterSuccess( + ctx, + moderationCoord, + result, + moderationOutputText(output.Text, traceRecorder.upstreamThinkContent()), + outputImages, + SendMessageInput{ + UserID: input.UserID, + ConversationID: input.ConversationID, + }, + reuseUserMessage, + ) + } + return result, nil } // mediaOutputUsage 安全提取允许为空的媒体响应 usage。 diff --git a/backend/internal/application/conversation/service_media_video.go b/backend/internal/application/conversation/service_media_video.go index 2676be115..ea170beef 100644 --- a/backend/internal/application/conversation/service_media_video.go +++ b/backend/internal/application/conversation/service_media_video.go @@ -11,6 +11,7 @@ import ( "time" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/channel" + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" appupload "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/upload" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" @@ -133,23 +134,50 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( StartedAt: startedAt, } var retErr error + var moderationCoord *appcm.RunCoordinator + var result *SendMessageResult + var userMessage *model.Message + var assistantMessage *model.Message defer func() { + if retErr != nil && moderationCoord != nil { + if result == nil && userMessage != nil && assistantMessage != nil { + result = &SendMessageResult{ + UserMessage: *userMessage, + AssistantMessage: *assistantMessage, + Billable: false, + StartedAt: startedAt, + } + } + s.completeModerationAfterFailure(context.WithoutCancel(ctx), moderationCoord, result) + } endedAt := time.Now() run.EndedAt = &endedAt run.TotalLatencyMS = endedAt.Sub(startedAt).Milliseconds() - if retErr == nil { + switch { + case result != nil && result.IsModerationBlocked(): + applyBlockedRunFields(run, result) + case retErr == nil: run.Status = "success" - } else if errors.Is(retErr, ErrMessageGenerationCanceled) { + if result != nil { + applyModerationRunState(run, result) + } + case errors.Is(retErr, ErrMessageGenerationCanceled): run.Status = "canceled" run.ErrorCode = classifyRunErrorCode(retErr) run.ErrorMessage = truncateError(messageErrorSummary(retErr), 255) - } else { + if result != nil { + applyModerationRunState(run, result) + } + default: run.Status = "error" run.ErrorCode = classifyRunErrorCode(retErr) run.ErrorMessage = truncateError(messageErrorSummary(retErr), 255) + if result != nil { + applyModerationRunState(run, result) + } } - if err := s.repo.CreateConversationRun(context.WithoutCancel(ctx), run); err != nil && s.logger != nil { - s.logger.Error("create_video_conversation_run_failed", + if err := s.repo.UpsertConversationRun(context.WithoutCancel(ctx), run); err != nil && s.logger != nil { + s.logger.Error("upsert_video_conversation_run_failed", zap.String("trace_id", traceid.FromContext(ctx)), zap.String("run_id", run.RunID), zap.Error(err), @@ -160,7 +188,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( ctx = cancelCtx s.generationStreams.register(ctx, runID, input.UserID, cancel) - assistantMessage := &model.Message{ + assistantMessage = &model.Message{ ConversationID: input.ConversationID, UserID: input.UserID, PublicID: normalizePublicID(uuid.NewString()), @@ -172,7 +200,6 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( Status: "pending", Attachments: "[]", } - var userMessage *model.Message if reuseUserMessage { reused := *branchState.ReuseUserMessage userMessage = &reused @@ -217,6 +244,21 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( traceRecorder.attachToMessage(assistantMessage) } }() + moderationFileIDs := make([]string, 0, len(resolvedAttachments)) + for _, item := range resolvedAttachments { + if fileID := strings.TrimSpace(item.FileID); fileID != "" { + moderationFileIDs = append(moderationFileIDs, fileID) + } + } + moderationCoord = s.startModerationRun(ctx, SendMessageInput{ + UserID: input.UserID, + ConversationID: input.ConversationID, + RequestID: input.RequestID, + Content: strings.TrimSpace(input.Prompt), + FileIDs: moderationFileIDs, + ClientRunID: runID, + OnEvent: input.OnEvent, + }, runID, userMessage, assistantMessage, run) emitMediaEvent(input.OnEvent, "queued", "video task queued", "video") cfg := s.cfg.Snapshot() @@ -282,7 +324,8 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( if err != nil { if s.isCanceledMediaGeneration(ctx, runID, err) { retErr = ErrMessageGenerationCanceled - result, cancelErr := s.completeCanceledMediaGeneration(canceledMediaGenerationInput{ + var cancelErr error + result, cancelErr = s.completeCanceledMediaGeneration(canceledMediaGenerationInput{ Context: ctx, Conversation: conversation, UserMessage: userMessage, @@ -432,7 +475,7 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( run.CacheWriteTokens = usage.CacheWriteTokens run.ReasoningTokens = usage.ReasoningTokens - return &SendMessageResult{ + result = &SendMessageResult{ UserMessage: *userMessage, AssistantMessage: *assistantMessage, MetadataRefreshHint: s.resolveConversationMetadataRefreshHint(ctx, *conversation, *userMessage), @@ -452,7 +495,21 @@ func (s *Service) StreamMediaVideo(ctx context.Context, input MediaVideoInput) ( StartedAt: startedAt, LatencyMS: latencyMS, DurationSeconds: durationSeconds, - }, nil + } + if moderationCoord != nil { + // Omni Moderation has no video modality. The prompt and optional input + // image still participate in the same barrier as other media tasks. + s.completeModerationAfterSuccess( + ctx, + moderationCoord, + result, + "", + nil, + SendMessageInput{UserID: input.UserID, ConversationID: input.ConversationID}, + reuseUserMessage, + ) + } + return result, nil } func mediaVideoUserContentType(hasInputs bool) string { diff --git a/backend/internal/application/conversation/service_message_completion.go b/backend/internal/application/conversation/service_message_completion.go index 2b3e2637a..f0da89a12 100644 --- a/backend/internal/application/conversation/service_message_completion.go +++ b/backend/internal/application/conversation/service_message_completion.go @@ -34,6 +34,8 @@ type persistMessageGenerationInput struct { PersistedToolCallKeys map[string]struct{} Route *channel.ResolvedRoute ReuseUserMessage bool + // SkipEmbed defers message embedding until the moderation barrier passes. + SkipEmbed bool } type persistInterruptedMessageGenerationInput struct { @@ -289,6 +291,9 @@ func (s *Service) finishSuccessfulMessageGeneration(ctx context.Context, input p if normalizeBranchReason(input.SendInput.BranchReason) == "default" { s.updateStatefulResponseAsync(input.SendInput.ConversationID, input.ResponseID, input.StatefulPromptFingerprint) } + if input.SkipEmbed { + return nil + } if input.ReuseUserMessage { s.embedMessagePairAsync(input.SendInput, nil, input.AssistantMessage) } else { @@ -300,6 +305,8 @@ func (s *Service) finishSuccessfulMessageGeneration(ctx context.Context, input p // persistInterruptedMessageGeneration 在模型调用已经产生可见内容或工具轨迹后失败时,保留本轮 assistant 消息。 // 显式取消由取消流程单独处理,避免把用户主动停止误标为异常中断。 +// Partial outputs from cancel/interrupt/upstream errors remain subject to the +// moderation barrier after persistence. func (s *Service) persistInterruptedMessageGeneration(ctx context.Context, input persistInterruptedMessageGenerationInput) *SendMessageResult { if !shouldPersistInterruptedMessageGeneration(input) { return nil @@ -402,7 +409,11 @@ func shouldPersistInterruptedMessageGeneration(input persistInterruptedMessageGe hasEstimatedCanceledInput := errors.Is(input.Error, ErrMessageGenerationCanceled) && input.UpstreamCallStarted && input.EstimatedInputTokens > 0 - return strings.TrimSpace(input.AssistantText) != "" || hasRetainedToolTrace || hasObservedUsage || hasEstimatedCanceledInput + return strings.TrimSpace(input.AssistantText) != "" || + strings.TrimSpace(input.AssistantReasoningText) != "" || + hasRetainedToolTrace || + hasObservedUsage || + hasEstimatedCanceledInput } // resolveInterruptedMessageGenerationMetrics 统一处理中断消息的真实 usage 与估算兜底。 diff --git a/backend/internal/application/conversation/service_message_completion_test.go b/backend/internal/application/conversation/service_message_completion_test.go index 575411ab1..ae57d883f 100644 --- a/backend/internal/application/conversation/service_message_completion_test.go +++ b/backend/internal/application/conversation/service_message_completion_test.go @@ -89,6 +89,9 @@ func TestCanceledGenerationEstimatesVisibleReasoningUsage(t *testing.T) { Error: ErrMessageGenerationCanceled, StartedAt: time.Now(), } + if !shouldPersistInterruptedMessageGeneration(input) { + t.Fatal("reasoning-only visible output must be retained for moderation") + } metrics := resolveInterruptedMessageGenerationMetrics(input) if metrics.OutputTokens != 0 || metrics.ReasoningTokens != estimateTokens(reasoningText) { @@ -181,3 +184,11 @@ func TestInterruptedGenerationRetainsReasoningContent(t *testing.T) { t.Fatalf("expected trimmed reasoning to be retained, got %q", assistant.ReasoningContent) } } + +func TestModerationOutputTextIncludesVisibleReasoningWithoutDuplicates(t *testing.T) { + got := moderationOutputText("assistant answer", "visible reasoning", "visible reasoning") + want := "assistant answer\n\nvisible reasoning" + if got != want { + t.Fatalf("moderation output=%q, want %q", got, want) + } +} diff --git a/backend/internal/application/conversation/service_message_send.go b/backend/internal/application/conversation/service_message_send.go index 44863ec66..dd4b8a49d 100644 --- a/backend/internal/application/conversation/service_message_send.go +++ b/backend/internal/application/conversation/service_message_send.go @@ -9,6 +9,7 @@ import ( "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/channel" appcompact "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/compact" + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" apprag "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/rag" model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" domainmemory "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/memory" @@ -192,6 +193,7 @@ func (s *Service) sendMessageInternal( if runID == "" { runID = "run_" + normalizePublicID(uuid.NewString()) } + var moderationCoord *appcm.RunCoordinator conversation, err := s.repo.GetConversationByUser(ctx, input.ConversationID, input.UserID) if err != nil { @@ -244,6 +246,7 @@ func (s *Service) sendMessageInternal( runState.bind(&userMessage, &assistantMessage, &traceRecorder, &result, ctx) defer func() { if retErr != nil { + retainedOutput := false if errors.Is(retErr, ErrMessageGenerationCanceled) || llm.RequestWasAccepted(retErr) { if usage, ok := s.recoverOpenAIResponsesBackgroundUsage(responsesBackgroundRouteConfig, responsesBackgroundRecovery); ok { responsesBackgroundUsageRecovered = true @@ -274,8 +277,31 @@ func (s *Service) sendMessageInternal( ReuseUserMessage: reuseUserMessage, }); retained != nil { result = retained + retainedOutput = true applyRetainedGenerationRunUsage(run, retained, len(toolCallRows), startedAt) } + // Input checks and any retained visible output continue after + // cancel/interrupt/error; either surface may still block the turn. + if moderationCoord != nil { + if result == nil && userMessage != nil && assistantMessage != nil { + result = &SendMessageResult{ + UserMessage: *userMessage, + AssistantMessage: *assistantMessage, + Billable: false, + StartedAt: startedAt, + } + } + if result != nil && retainedOutput { + s.completeModerationAfterInterruption( + context.Background(), + moderationCoord, + result, + moderationOutputText(streamedText.String(), traceRecorder.upstreamThinkContent()), + ) + } else { + s.completeModerationAfterFailure(context.Background(), moderationCoord, result) + } + } } runState.finalize(ctx, retErr) if retErr != nil && result == nil && userMessage != nil && assistantMessage != nil { @@ -316,6 +342,7 @@ func (s *Service) sendMessageInternal( assistantMessage = pair.assistant s.persistInitialConversationFallbackTitle(ctx, *conversation, *userMessage) traceRecorder = newMessageTraceRecorder(s, ctx, assistantMessage, input.OnEvent) + moderationCoord = s.startModerationRun(ctx, input, runID, userMessage, assistantMessage, run) if s.routeResolver == nil || s.llmClient == nil { retErr = ErrModelRouteNotConfigured @@ -376,7 +403,7 @@ func (s *Service) sendMessageInternal( } // 构建完整活跃分支路径;压缩裁剪先于模型预算截断,避免摘要和全量历史重复发送。 - contextMessages := buildBranchMessagePath(branchState, userMessage) + contextMessages := filterBlockedMessages(buildBranchMessagePath(branchState, userMessage)) cfg := s.cfg.Snapshot() compactPolicy := s.resolveContextCompactionPolicy(ctx, cfg, input.UserID) @@ -1528,6 +1555,7 @@ func (s *Service) sendMessageInternal( PersistedToolCallKeys: persistedToolCallKeys, Route: resolvedRoute, ReuseUserMessage: reuseUserMessage, + SkipEmbed: moderationCoord != nil, }) platformtracing.RecordError(persistSpan, err) persistSpan.End() @@ -1588,7 +1616,7 @@ func (s *Service) sendMessageInternal( } } - return &SendMessageResult{ + result = &SendMessageResult{ UserMessage: *userMessage, AssistantMessage: *assistantMessage, MetadataRefreshHint: s.resolveConversationMetadataRefreshHint(ctx, *conversation, *userMessage), @@ -1609,5 +1637,19 @@ func (s *Service) sendMessageInternal( LatencyMS: time.Since(startedAt).Milliseconds(), StartedAt: startedAt, postBillingCompaction: postBillingCompaction, - }, nil + } + // Soft moderation barrier: show checking, then block or pass. + if moderationCoord != nil { + outputImages := s.loadOutputImagesForModeration(ctx, moderationCoord, input.UserID, assistantMessage.Attachments) + s.completeModerationAfterSuccess( + ctx, + moderationCoord, + result, + moderationOutputText(assistantText, assistantReasoningContent, traceRecorder.upstreamThinkContent()), + outputImages, + input, + reuseUserMessage, + ) + } + return result, nil } diff --git a/backend/internal/application/conversation/service_moderation.go b/backend/internal/application/conversation/service_moderation.go new file mode 100644 index 000000000..3f4dde210 --- /dev/null +++ b/backend/internal/application/conversation/service_moderation.go @@ -0,0 +1,576 @@ +package conversation + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "time" + + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" + appstorage "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/objectstorage" + model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/config" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/objectstore" + "go.uber.org/zap" +) + +// MessageModerationOutcome is the soft-moderation end state for a turn. +// Nil means moderation was not required (policy off / no coordinator). +type MessageModerationOutcome struct { + Blocked bool + EventID string + Direction string + Categories []string + // State is the moderation target: passed | failed_open | blocked. Known blocks may + // converge durably through the moderation compensation loop. + State string + // TerminalEmitted is true when moderation_blocked was already pushed to the stream. + TerminalEmitted bool +} + +// IsModerationBlocked reports whether the turn was blocked after a safety check. +func (r *SendMessageResult) IsModerationBlocked() bool { + return r != nil && r.Moderation != nil && r.Moderation.Blocked +} + +// ModerationTerminalEmitted reports whether moderation_blocked already went out on the stream. +func (r *SendMessageResult) ModerationTerminalEmitted() bool { + return r != nil && r.Moderation != nil && r.Moderation.TerminalEmitted +} + +// SetModerationService injects the optional content moderation orchestrator. +func (s *Service) SetModerationService(svc *appcm.Service) { + s.moderationSvc = svc + if svc == nil { + return + } + svc.SetEventEmitter(func(runID string, eventType string, payload map[string]interface{}) { + if payload == nil { + payload = map[string]interface{}{"type": eventType} + } else if _, ok := payload["type"]; !ok { + payload["type"] = eventType + } + s.PublishMessageGenerationEvent(runID, payload) + }) + svc.SetCancelRun(func(runID string) { + if s.generationStreams != nil { + s.generationStreams.cancelForced(context.Background(), normalizeRunID(runID)) + } + }) + svc.SetOnBlocked(func(runID string, _ appcm.BlockInfo) { + // Drop retained deltas/media so reconnect cannot replay withdrawn content. + // The following emit of moderation_blocked re-seeds a safe terminal event. + s.resetGenerationStreamEvents(runID) + }) + svc.SetImageLoader(s.loadImageForModeration) + svc.SetObjectStore(&moderationObjectStoreAdapter{service: s}) + svc.SetFileAccessController(&moderationFileAccessAdapter{service: s}) +} + +// startModerationRun begins per-turn moderation when policy is enabled. +// Live events use the existing OnEvent path (set by HTTP handlers) — no side channel. +// seedRun, when non-nil, is ensured in DB so mid-flight moderation_state updates have a row. +func (s *Service) startModerationRun( + ctx context.Context, + input SendMessageInput, + runID string, + userMessage *model.Message, + assistantMessage *model.Message, + seedRun *model.Run, +) *appcm.RunCoordinator { + if s == nil || s.moderationSvc == nil || userMessage == nil { + return nil + } + meta := appcm.RunMeta{ + UserID: input.UserID, + ConversationID: input.ConversationID, + RunID: runID, + MessageID: userMessage.ID, + MessagePublicID: userMessage.PublicID, + UserMessageID: userMessage.ID, + AssistantMessageID: 0, + } + if assistantMessage != nil { + meta.AssistantMessageID = assistantMessage.ID + } + coord := s.moderationSvc.BeginRun(ctx, meta) + if coord == nil { + return nil + } + // Durable run row must exist before UpdateRunModeration / ApplyRunBlock. + s.ensureConversationRunForModeration(ctx, seedRun, input, runID) + // BeginRun may have updated before the row existed; re-apply pending now. + s.moderationSvc.SyncRunPending(ctx, runID) + if input.OnEvent != nil { + coord.SetLiveEmitter(func(eventType string, payload map[string]interface{}) { + _ = input.OnEvent(eventType, payload) + }) + } + coord.EnqueueInputText(input.Content) + if len(input.FileIDs) > 0 { + coord.EnqueueInputImages(ctx, input.FileIDs) + } + return coord +} + +// ensureConversationRunForModeration inserts a mid-flight run so barrier state updates are not no-ops. +func (s *Service) ensureConversationRunForModeration( + ctx context.Context, + seedRun *model.Run, + input SendMessageInput, + runID string, +) { + if s == nil || s.repo == nil { + return + } + var run model.Run + if seedRun != nil { + run = *seedRun + } else { + run = model.Run{ + RunID: runID, + RequestID: strings.TrimSpace(input.RequestID), + UserID: input.UserID, + ConversationID: input.ConversationID, + TaskType: "chat", + Status: "running", + StartedAt: time.Now(), + } + } + if strings.TrimSpace(run.RunID) == "" { + run.RunID = runID + } + if strings.TrimSpace(run.Status) == "" || run.Status == "error" { + run.Status = "running" + } + run.ModerationState = "pending" + run.EndedAt = nil + if err := s.repo.EnsureConversationRun(ctx, &run); err != nil && s.logger != nil { + s.logger.Warn("ensure_conversation_run_for_moderation_failed", + zap.String("run_id", run.RunID), + zap.Error(err), + ) + } +} + +// completeModerationAfterSuccess runs the post-generation barrier. +// On block it mutates result into a blocked snapshot and sets result.Moderation. +// Callers branch on result.IsModerationBlocked(); embed only runs on pass/fail-open. +func (s *Service) completeModerationAfterSuccess( + ctx context.Context, + coord *appcm.RunCoordinator, + result *SendMessageResult, + outputText string, + outputImages []appcm.OutputImageSource, + embedInput SendMessageInput, + reuseUserMessage bool, +) { + if coord == nil || result == nil { + return + } + barrier := coord.AfterGeneration(ctx, outputText, outputImages) + applyBarrierOutcome(result, barrier) + if result.IsModerationBlocked() { + return + } + // Pass / fail-open: embed now (persist path skipped embed while barrier was active). + if reuseUserMessage { + s.embedMessagePairAsync(embedInput, nil, &result.AssistantMessage) + } else { + s.embedMessagePairAsync(embedInput, &result.UserMessage, &result.AssistantMessage) + } +} + +// completeModerationAfterInterruption moderates content that was already visible +// and retained after a cancel or upstream failure, without embedding a partial reply. +func (s *Service) completeModerationAfterInterruption( + ctx context.Context, + coord *appcm.RunCoordinator, + result *SendMessageResult, + outputText string, +) { + if coord == nil || result == nil { + return + } + barrier := coord.AfterGeneration(ctx, outputText, nil) + applyBarrierOutcome(result, barrier) +} + +// completeModerationAfterFailure continues input-only checks (no output moderation). +func (s *Service) completeModerationAfterFailure( + ctx context.Context, + coord *appcm.RunCoordinator, + result *SendMessageResult, +) { + if coord == nil { + return + } + barrier := coord.WaitInputOnly(ctx) + if result == nil { + return + } + applyBarrierOutcome(result, barrier) +} + +func applyBarrierOutcome(result *SendMessageResult, barrier appcm.BarrierResult) { + if result == nil { + return + } + if barrier.Block == nil { + result.Moderation = &MessageModerationOutcome{ + Blocked: false, + State: firstNonEmptyString(barrier.State, "passed"), + } + return + } + result.postBillingCompaction = nil + result.MetadataRefreshHint = conversationMetadataRefreshNotNeeded + applyBlockedSnapshot(result, *barrier.Block, barrier.TerminalEmitted) +} + +func applyBlockedSnapshot(result *SendMessageResult, block appcm.BlockInfo, terminalEmitted bool) { + if result == nil { + return + } + if block.Direction == appcm.DirectionInput { + result.UserMessage.Status = "blocked" + result.UserMessage.ModerationEventID = block.EventID + result.UserMessage.ModerationCategoriesJSON = mustJSONArray(block.Categories) + result.UserMessage.ErrorCode = "content_moderation.blocked" + result.UserMessage.ErrorMessage = "content blocked by moderation" + } + result.AssistantMessage.Status = "blocked" + result.AssistantMessage.Content = "" + result.AssistantMessage.ReasoningContent = "" + result.AssistantMessage.Attachments = "[]" + result.AssistantMessage.ProcessTrace = nil + result.AssistantMessage.ModerationEventID = block.EventID + result.AssistantMessage.ModerationCategoriesJSON = mustJSONArray(block.Categories) + result.AssistantMessage.ErrorCode = "content_moderation.blocked" + result.AssistantMessage.ErrorMessage = "content blocked by moderation" + result.Moderation = &MessageModerationOutcome{ + Blocked: true, + EventID: block.EventID, + Direction: block.Direction, + Categories: append([]string(nil), block.Categories...), + State: "blocked", + TerminalEmitted: terminalEmitted, + } +} + +// applyBlockedRunFields copies soft-block outcome onto a conversation run for finalize/upsert. +func applyBlockedRunFields(run *model.Run, result *SendMessageResult) { + if run == nil || result == nil || !result.IsModerationBlocked() { + return + } + run.Status = "blocked" + run.ErrorCode = "content_moderation.blocked" + run.ErrorMessage = "content blocked by moderation" + run.ModerationState = "blocked" + if result.Moderation != nil { + run.ModerationEventID = result.Moderation.EventID + run.ModerationCategoriesJSON = mustJSONArray(result.Moderation.Categories) + } +} + +// applyModerationRunState copies non-blocked barrier state onto the run for upsert. +func applyModerationRunState(run *model.Run, result *SendMessageResult) { + if run == nil || result == nil || result.Moderation == nil { + return + } + if result.Moderation.Blocked { + applyBlockedRunFields(run, result) + return + } + if state := strings.TrimSpace(result.Moderation.State); state != "" { + run.ModerationState = state + } +} + +func mustJSONArray(items []string) string { + if len(items) == 0 { + return "[]" + } + raw, err := json.Marshal(items) + if err != nil { + return "[]" + } + return string(raw) +} + +func moderationOutputText(parts ...string) string { + seen := make(map[string]struct{}, len(parts)) + kept := make([]string, 0, len(parts)) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if _, exists := seen[part]; exists { + continue + } + seen[part] = struct{}{} + kept = append(kept, part) + } + return strings.Join(kept, "\n\n") +} + +func (s *Service) loadImageForModeration(ctx context.Context, userID uint, fileID string) (appcm.PreparedImage, error) { + empty := appcm.PreparedImage{} + file, err := s.repo.GetActiveFileObjectByID(ctx, userID, strings.TrimSpace(fileID)) + if err != nil || file == nil { + return empty, err + } + declaredMIME := firstNonEmptyString(file.DetectedMIME, file.MimeType) + if normalizeAttachmentKind("", declaredMIME) != "image" { + return empty, appcm.ErrNonImageAttachment + } + cfg := s.cfg.Snapshot() + storeProvider := s.storeProvider + if storeProvider == nil { + storeProvider = appstorage.NewRuntimeProvider(config.NewRuntime(cfg), nil) + } + store, err := storeProvider.Open(ctx) + if err != nil { + return empty, err + } + reader, _, err := store.Open(ctx, strings.TrimSpace(file.StoragePath)) + if err != nil { + return empty, err + } + data, readErr := io.ReadAll(io.LimitReader(reader, maxConversationImageSourceBytes+1)) + _ = reader.Close() + if readErr != nil { + return empty, readErr + } + if len(data) == 0 { + return empty, errEmptyModerationImage + } + if len(data) > 20*1024*1024 { + return empty, errModerationImageTooLarge + } + detectedMIME := detectGeneratedImageMIME(data) + if detectedMIME == "" { + return empty, errUnsupportedModerationImage + } + maxDim := cfg.ImageMaxDimension + if maxDim <= 0 { + maxDim = 1024 + } + resized, actualMIME := resizeImageIfNeeded(data, detectedMIME, maxDim) + return appcm.PreparedImage{ + Data: resized, + SHA256: file.SHA256, + Mime: actualMIME, + Size: int64(len(resized)), + FileID: file.FileID, + }, nil +} + +var ( + errEmptyModerationImage = errString("empty image") + errModerationImageTooLarge = errString("image exceeds 20MB") + errUnsupportedModerationImage = errString("unsupported moderation image") +) + +type stringError string + +func (e stringError) Error() string { return string(e) } +func errString(s string) error { return stringError(s) } + +// loadOutputImagesForModeration loads final assistant image attachments for output checks. +func (s *Service) loadOutputImagesForModeration(ctx context.Context, coord *appcm.RunCoordinator, userID uint, attachmentsJSON string) []appcm.OutputImageSource { + refs := parseAttachmentSnapshotRefs(attachmentsJSON) + if len(refs) == 0 { + return nil + } + out := make([]appcm.OutputImageSource, 0, len(refs)) + for _, ref := range refs { + fileID := strings.TrimSpace(ref.FileID) + if fileID == "" { + continue + } + kind := normalizeAttachmentKind(ref.Kind, firstNonEmptyString(ref.DetectedMIME, ref.MimeType)) + if kind != "image" { + continue + } + prepared, err := s.loadImageForModeration(ctx, userID, fileID) + if err != nil || len(prepared.Data) == 0 { + if err == nil { + err = errEmptyModerationImage + } + coord.RecordOutputImageFailure(fileID, err) + continue + } + out = append(out, appcm.OutputImageSource{ + FileID: fileID, + Data: prepared.Data, + MimeType: prepared.Mime, + SHA256: prepared.SHA256, + }) + } + return out +} + +func loadOutputImagesFromFiles(coord *appcm.RunCoordinator, files []model.FileObject, dataByFileID map[string][]byte) []appcm.OutputImageSource { + out := make([]appcm.OutputImageSource, 0, len(files)) + for _, file := range files { + data := dataByFileID[file.FileID] + if len(data) == 0 { + coord.RecordOutputImageFailure(file.FileID, errEmptyModerationImage) + continue + } + out = append(out, appcm.OutputImageSource{ + FileID: file.FileID, + Data: data, + MimeType: firstNonEmptyString(file.DetectedMIME, file.MimeType, "image/png"), + SHA256: file.SHA256, + }) + } + return out +} + +func (s *Service) resetGenerationStreamEvents(runID string) { + runID = normalizeRunID(runID) + if runID == "" || s == nil || s.generationStreams == nil { + return + } + s.generationStreams.resetEvents(context.Background(), runID) +} + +type moderationObjectStoreAdapter struct { + service *Service +} + +func (a *moderationObjectStoreAdapter) Put(ctx context.Context, path string, data []byte, contentType string) error { + store, err := a.open(ctx) + if err != nil { + return err + } + _, err = store.Put(ctx, path, bytes.NewReader(data), objectstore.PutOptions{ContentType: contentType}) + return err +} + +func (a *moderationObjectStoreAdapter) Open(ctx context.Context, path string) ([]byte, error) { + store, err := a.open(ctx) + if err != nil { + return nil, err + } + reader, _, err := store.Open(ctx, path) + if err != nil { + return nil, err + } + defer reader.Close() + return io.ReadAll(reader) +} + +func (a *moderationObjectStoreAdapter) Delete(ctx context.Context, path string) error { + store, err := a.open(ctx) + if err != nil { + return err + } + return store.Delete(ctx, path) +} + +func (a *moderationObjectStoreAdapter) open(ctx context.Context) (objectstore.Store, error) { + provider := a.service.storeProvider + if provider == nil { + provider = appstorage.NewRuntimeProvider(a.service.cfg, nil) + } + return provider.Open(ctx) +} + +type moderationFileAccessAdapter struct { + service *Service +} + +var _ appcm.FileAccessController = (*moderationFileAccessAdapter)(nil) + +type moderationBlockedFileLister interface { + ListModerationBlockedFileIDsForCleanup(ctx context.Context, limit int) ([]string, error) +} + +func (a *moderationFileAccessAdapter) RevokeGeneratedFile(ctx context.Context, fileID string) error { + if a.service == nil || a.service.repo == nil { + return nil + } + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return nil + } + return a.service.repo.RevokeGeneratedFileForModeration(ctx, fileID) +} + +func (a *moderationFileAccessAdapter) DeleteGeneratedFileArtifacts(ctx context.Context, fileID string) error { + if a.service == nil || a.service.repo == nil { + return nil + } + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return nil + } + var storagePath string + if file, err := a.service.repo.GetFileObjectByFileIDAnyStatus(ctx, fileID); err == nil && file != nil { + storagePath = strings.TrimSpace(file.StoragePath) + } + if err := a.service.repo.DeleteGeneratedFileArtifactsForModeration(ctx, fileID); err != nil { + return err + } + if storagePath == "" { + return nil + } + storeProvider := a.service.storeProvider + if storeProvider == nil { + storeProvider = appstorage.NewRuntimeProvider(a.service.cfg, nil) + } + store, err := storeProvider.Open(ctx) + if err != nil { + return err + } + if err := store.Delete(ctx, storagePath); err != nil { + return err + } + return a.service.repo.ClearGeneratedFileStoragePath(ctx, fileID) +} + +func (a *moderationFileAccessAdapter) RetryBlockedGeneratedFileDeletes(ctx context.Context, limit int) (int, error) { + if a.service == nil || a.service.repo == nil { + return 0, nil + } + lister, ok := a.service.repo.(moderationBlockedFileLister) + if !ok { + return 0, errors.New("conversation repository does not support moderation file cleanup") + } + fileIDs, err := lister.ListModerationBlockedFileIDsForCleanup(ctx, limit) + if err != nil { + return 0, err + } + deleted := 0 + var cleanupErr error + for _, fileID := range fileIDs { + if err := a.DeleteGeneratedFileArtifacts(ctx, fileID); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + continue + } + deleted++ + } + return deleted, cleanupErr +} + +// filterBlockedMessages excludes blocked messages from model context. +func filterBlockedMessages(messages []model.Message) []model.Message { + if len(messages) == 0 { + return messages + } + out := make([]model.Message, 0, len(messages)) + for _, item := range messages { + if strings.EqualFold(strings.TrimSpace(item.Status), "blocked") { + continue + } + out = append(out, item) + } + return out +} diff --git a/backend/internal/application/conversation/service_run.go b/backend/internal/application/conversation/service_run.go index a3f67b4a9..f81eaa2f4 100644 --- a/backend/internal/application/conversation/service_run.go +++ b/backend/internal/application/conversation/service_run.go @@ -115,6 +115,10 @@ func (r *messageSendRunState) finalizeRun(retErr error) { if r.run.TotalLatencyMS < 0 { r.run.TotalLatencyMS = 0 } + if result := r.currentResult(); result != nil && result.IsModerationBlocked() { + applyBlockedRunFields(r.run, result) + return + } switch { case retErr == nil: r.run.Status = "success" @@ -131,6 +135,10 @@ func (r *messageSendRunState) finalizeRun(retErr error) { r.run.ErrorCode = classifyRunErrorCode(retErr) r.run.ErrorMessage = truncateError(retErr.Error(), 255) } + // Preserve barrier pass/fail-open state written mid-flight (do not default to not_required). + if result := r.currentResult(); result != nil { + applyModerationRunState(r.run, result) + } } func (r *messageSendRunState) finalizeUserMessage(ctx context.Context, retErr error) { @@ -141,6 +149,10 @@ func (r *messageSendRunState) finalizeUserMessage(ctx context.Context, retErr er if userMessage == nil { return } + if result := r.currentResult(); result != nil && result.IsModerationBlocked() { + // Block path already wrote message moderation state; do not overwrite. + return + } messageStatus := "success" messageErrorCode := "" messageErrorMessage := "" @@ -172,6 +184,10 @@ func (r *messageSendRunState) finalizeAssistantMessage(ctx context.Context, retE if retErr == nil { return } + if result := r.currentResult(); result != nil && result.IsModerationBlocked() { + // Block path already wrote assistant moderation state; do not overwrite. + return + } assistantMessage := r.currentAssistantMessage() if assistantMessage == nil { return @@ -205,8 +221,9 @@ func (r *messageSendRunState) finalizeAssistantMessage(ctx context.Context, retE } func (r *messageSendRunState) createRun(ctx context.Context) { - if err := r.service.repo.CreateConversationRun(ctx, r.run); err != nil { - r.service.logger.Error("create_conversation_run_failed", + // Upsert: mid-flight EnsureConversationRun may have already inserted the row. + if err := r.service.repo.UpsertConversationRun(ctx, r.run); err != nil { + r.service.logger.Error("upsert_conversation_run_failed", zap.String("trace_id", traceid.FromContext(r.traceContext)), zap.String("run_id", r.run.RunID), zap.Error(err), diff --git a/backend/internal/application/settings/sensitive.go b/backend/internal/application/settings/sensitive.go index 459427fac..f5b762110 100644 --- a/backend/internal/application/settings/sensitive.go +++ b/backend/internal/application/settings/sensitive.go @@ -24,6 +24,7 @@ var sensitiveSettingKeys = map[string]struct{}{ "extract:mineru_auth_token": {}, "extract:llm_ocr_auth_token": {}, "file:embedding_key": {}, + "content_moderation:api_key": {}, } func isSensitiveSetting(namespace string, key string) bool { diff --git a/backend/internal/application/upload/service_test.go b/backend/internal/application/upload/service_test.go index 1ad6b6641..4f2b8f0de 100644 --- a/backend/internal/application/upload/service_test.go +++ b/backend/internal/application/upload/service_test.go @@ -527,6 +527,41 @@ func (r *uploadTestRepo) TouchFileObjectLastAccessedAt(_ context.Context, userID return repository.ErrNotFound } +func (r *uploadTestRepo) RevokeGeneratedFileForModeration(_ context.Context, fileID string) error { + for i := range r.files { + if r.files[i].FileID == fileID { + r.files[i].Status = "moderation_blocked" + r.files[i].UserID = 0 + return nil + } + } + return repository.ErrNotFound +} + +func (r *uploadTestRepo) DeleteGeneratedFileArtifactsForModeration(context.Context, string) error { + return nil +} + +func (r *uploadTestRepo) ClearGeneratedFileStoragePath(_ context.Context, fileID string) error { + for i := range r.files { + if r.files[i].FileID == fileID { + r.files[i].StoragePath = "" + return nil + } + } + return repository.ErrNotFound +} + +func (r *uploadTestRepo) GetFileObjectByFileIDAnyStatus(_ context.Context, fileID string) (*domainconversation.FileObject, error) { + for i := range r.files { + if r.files[i].FileID == fileID { + result := r.files[i] + return &result, nil + } + } + return nil, repository.ErrNotFound +} + func (r *uploadTestRepo) GetUserByID(context.Context, uint) (*domainuser.User, error) { result := r.user return &result, nil diff --git a/backend/internal/domain/contentmoderation/types.go b/backend/internal/domain/contentmoderation/types.go new file mode 100644 index 000000000..8d9046763 --- /dev/null +++ b/backend/internal/domain/contentmoderation/types.go @@ -0,0 +1,224 @@ +package contentmoderation + +import ( + "sort" + "strings" + "time" +) + +// Direction indicates whether content is user input or model output. +const ( + DirectionInput = "input" + DirectionOutput = "output" +) + +// Modality indicates text or image content. +const ( + ModalityText = "text" + ModalityImage = "image" +) + +// Result values for events and daily stats. +const ( + ResultHit = "hit" + ResultFailedOpen = "failed_open" + ResultPassed = "passed" +) + +// Run moderation_state values. +const ( + ModerationStateNotRequired = "not_required" + ModerationStatePending = "pending" + ModerationStateModerating = "moderating" + ModerationStatePassed = "passed" + ModerationStateBlocked = "blocked" + ModerationStateFailedOpen = "failed_open" +) + +// Message/run status for blocked rounds. +const ( + StatusBlocked = "blocked" +) + +// Error codes stored on failure events. +const ( + ErrorCodeTimeout = "timeout" + ErrorCodeRateLimited = "rate_limited" + ErrorCodeQueueFull = "queue_full" + ErrorCodeServiceError = "service_error" + ErrorCodeInvalidResp = "invalid_response" + ErrorCodeWorkerLost = "worker_lost" + ErrorCodeNetworkError = "network_error" + ErrorCodeConfigMissing = "config_missing" +) + +// Event is a moderation check record (pass, hit, or failed-open). +type Event struct { + ID uint + PublicID string + UserID uint + ConversationID uint + RunID string + MessageID uint + MessagePublicID string + Direction string + Modality string + Model string + PolicyVersion int64 + Result string + CategoriesJSON string + CategoryScoresJSON string + LatencyMS int64 + ErrorCode string + ErrorMessage string + ContentLocationJSON string + ContentSummary string + EncryptedText string + ImageCount int + ImageMetaJSON string + ContentExpiresAt time.Time + MetadataExpiresAt time.Time + CreatedAt time.Time + UpdatedAt time.Time +} + +// DailyStat aggregates anonymous counters for a calendar day. +type DailyStat struct { + ID uint + StatDate time.Time + Direction string + Modality string + Result string + Category string + CheckCount int64 + ContentItems int64 + HitCount int64 + FailureCount int64 + LatencySumMS int64 + LatencyCount int64 + CreatedAt time.Time + UpdatedAt time.Time +} + +// EventListFilter filters super-admin event queries. +type EventListFilter struct { + Direction string + Modality string + Result string + Category string + UserID uint + RunID string + From *time.Time + To *time.Time + Offset int + Limit int +} + +// IsolatedImageMeta describes one encrypted image copy held for review. +type IsolatedImageMeta struct { + Index int + SHA256 string + MimeType string + SizeBytes int64 + StoragePath string + SourceFileID string +} + +// ContentLocation describes where moderated content originated. +type ContentLocation struct { + Field string + FileID string + Attachment int + ChunkIndex int + ChunkCount int +} + +// ProviderConfig contains runtime values needed by a moderation provider. +type ProviderConfig struct { + BaseURL string + APIKey string + Model string + Timeout time.Duration +} + +// ProviderImage is an image submitted to a moderation provider. +type ProviderImage struct { + Data []byte + MimeType string +} + +// CategoryResult is a provider-neutral category result. +type CategoryResult struct { + Flagged bool + Categories map[string]bool + CategoryScores map[string]float64 + CategoryAppliedInputTypes map[string][]string +} + +// ProviderResponse is a provider-neutral moderation result. +type ProviderResponse struct { + ID string + Model string + Results []CategoryResult +} + +// HitEvaluation is the policy-aware decision for one provider response. +type HitEvaluation struct { + Hit bool + Categories []string + Scores map[string]float64 +} + +// EvaluateHit decides a block using only selected categories that apply to the expected modality. +func EvaluateHit(response *ProviderResponse, selected []string, expectedModality string) HitEvaluation { + evaluation := HitEvaluation{ + Categories: make([]string, 0), + Scores: make(map[string]float64), + } + if response == nil || len(selected) == 0 { + return evaluation + } + selectedSet := make(map[string]struct{}, len(selected)) + for _, category := range selected { + selectedSet[category] = struct{}{} + } + expected := "text" + if strings.TrimSpace(expectedModality) == ModalityImage { + expected = "image" + } + for _, item := range response.Results { + for category := range selectedSet { + if !item.Categories[category] || !categoryAppliesToModality(item.CategoryAppliedInputTypes, category, expected) { + continue + } + if _, exists := evaluation.Scores[category]; exists { + continue + } + evaluation.Categories = append(evaluation.Categories, category) + if item.CategoryScores != nil { + evaluation.Scores[category] = item.CategoryScores[category] + } + } + } + if len(evaluation.Categories) > 0 { + sort.Strings(evaluation.Categories) + evaluation.Hit = true + } + return evaluation +} + +func categoryAppliesToModality(applied map[string][]string, category, expected string) bool { + if applied == nil { + return true + } + types, ok := applied[category] + if !ok || len(types) == 0 { + return true + } + for _, inputType := range types { + if strings.EqualFold(strings.TrimSpace(inputType), expected) { + return true + } + } + return false +} diff --git a/backend/internal/domain/conversation/types.go b/backend/internal/domain/conversation/types.go index 845ad4e3d..2c8978419 100644 --- a/backend/internal/domain/conversation/types.go +++ b/backend/internal/domain/conversation/types.go @@ -166,41 +166,43 @@ type MessagePromptTrace struct { // Message 表示会话消息。 type Message struct { - ID uint - ConversationID uint - UserID uint - PublicID string - ParentMessageID *uint - RunID string - Role string - ContentType string - Content string - ReasoningContent string - BranchReason string - SourceMessageID *uint - TokenUsage int64 - InputTokens int64 - OutputTokens int64 - CacheReadTokens int64 - CacheWriteTokens int64 - ReasoningTokens int64 - LatencyMS int64 - BilledCurrency string - BilledNanousd int64 - PricingSnapshot string - Status string - ErrorCode string - ErrorMessage string - Attachments string - ParentPublicID string - SourcePublicID string - MyFeedback string - ThumbsUpCount int64 - ThumbsDownCount int64 - ProcessTrace *MessageProcessTrace - EditedAt *time.Time - CreatedAt time.Time - UpdatedAt time.Time + ID uint + ConversationID uint + UserID uint + PublicID string + ParentMessageID *uint + RunID string + Role string + ContentType string + Content string + ReasoningContent string + BranchReason string + SourceMessageID *uint + TokenUsage int64 + InputTokens int64 + OutputTokens int64 + CacheReadTokens int64 + CacheWriteTokens int64 + ReasoningTokens int64 + LatencyMS int64 + BilledCurrency string + BilledNanousd int64 + PricingSnapshot string + Status string + ErrorCode string + ErrorMessage string + ModerationEventID string + ModerationCategoriesJSON string + Attachments string + ParentPublicID string + SourcePublicID string + MyFeedback string + ThumbsUpCount int64 + ThumbsDownCount int64 + ProcessTrace *MessageProcessTrace + EditedAt *time.Time + CreatedAt time.Time + UpdatedAt time.Time } // MessageFeedback 表示消息反馈。 @@ -336,39 +338,42 @@ type StorageQuota struct { // Run 表示对话运行日志。 type Run struct { - ID uint - RunID string - RequestID string - UserID uint - ConversationID uint - TaskType string - Endpoint string - Provider string - ProviderProtocol string - UpstreamID uint - UpstreamModelID uint - UpstreamName string - RequestedModelName string - PlatformModelName string - RoutedBindingCode string - ModelVendor string - ModelIcon string - UpstreamModelName string - InputTokens int64 - OutputTokens int64 - CacheReadTokens int64 - CacheWriteTokens int64 - ReasoningTokens int64 - ToolCallsCount int - FirstTokenLatencyMS int64 - TotalLatencyMS int64 - Status string - ErrorCode string - ErrorMessage string - StartedAt time.Time - EndedAt *time.Time - CreatedAt time.Time - UpdatedAt time.Time + ID uint + RunID string + RequestID string + UserID uint + ConversationID uint + TaskType string + Endpoint string + Provider string + ProviderProtocol string + UpstreamID uint + UpstreamModelID uint + UpstreamName string + RequestedModelName string + PlatformModelName string + RoutedBindingCode string + ModelVendor string + ModelIcon string + UpstreamModelName string + InputTokens int64 + OutputTokens int64 + CacheReadTokens int64 + CacheWriteTokens int64 + ReasoningTokens int64 + ToolCallsCount int + FirstTokenLatencyMS int64 + TotalLatencyMS int64 + Status string + ErrorCode string + ErrorMessage string + ModerationState string + ModerationEventID string + ModerationCategoriesJSON string + StartedAt time.Time + EndedAt *time.Time + CreatedAt time.Time + UpdatedAt time.Time } // MessageTrace 表示消息处理轨迹。 diff --git a/backend/internal/infra/cache/memory/generation_stream.go b/backend/internal/infra/cache/memory/generation_stream.go index fb73fe138..7dd6be71d 100644 --- a/backend/internal/infra/cache/memory/generation_stream.go +++ b/backend/internal/infra/cache/memory/generation_stream.go @@ -157,6 +157,19 @@ func (c *Cache) ReadGenerationStreamEvents(ctx context.Context, runID string, af } } +func (c *Cache) ResetGenerationStreamEvents(ctx context.Context, runID string) error { + c.mu.Lock() + defer c.mu.Unlock() + stream := c.streams[strings.TrimSpace(runID)] + if stream == nil { + return nil + } + stream.events = nil + // Keep seq monotonic so any in-flight afterSeq cursors stay valid. + stream.notifyLocked() + return nil +} + func (c *Cache) ExpireGenerationStream(ctx context.Context, runID string, ttl time.Duration) error { c.mu.Lock() defer c.mu.Unlock() diff --git a/backend/internal/infra/cache/redis/conversation_cache.go b/backend/internal/infra/cache/redis/conversation_cache.go index a5125faf2..a370a86b7 100644 --- a/backend/internal/infra/cache/redis/conversation_cache.go +++ b/backend/internal/infra/cache/redis/conversation_cache.go @@ -438,6 +438,19 @@ func (c *conversationCache) ReadGenerationStreamEvents(ctx context.Context, runI return results, nil } +// ResetGenerationStreamEvents 清空恢复流事件,阻止撤回内容在重连时被回放。 +func (c *conversationCache) ResetGenerationStreamEvents(ctx context.Context, runID string) error { + if c.client == nil { + return nil + } + runID = strings.TrimSpace(runID) + if runID == "" { + return nil + } + // Keep seq key so subsequent appends stay monotonic for reconnect cursors. + return c.client.Del(ctx, generationStreamEventsKey(runID)).Err() +} + // ExpireGenerationStream 设置生成流相关键的过期时间。 func (c *conversationCache) ExpireGenerationStream(ctx context.Context, runID string, ttl time.Duration) error { if c.client == nil || ttl <= 0 { diff --git a/backend/internal/infra/contentmoderation/client.go b/backend/internal/infra/contentmoderation/client.go new file mode 100644 index 000000000..505b2506b --- /dev/null +++ b/backend/internal/infra/contentmoderation/client.go @@ -0,0 +1,62 @@ +// Package contentmoderation provides the managed outbound HTTP boundary for +// administrator-configured moderation services. +package contentmoderation + +import ( + "fmt" + "net/http" + "time" + + platformtracing "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/observability/tracing" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/outboundhttp" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/security" +) + +const moderationRequestTimeout = 30 * time.Second + +// Client reuses origin-scoped HTTP transports for moderation endpoints. +type Client struct { + pool *outboundhttp.Pool + doRequest func(request *http.Request, configuredEndpoint string) (*http.Response, error) +} + +// New creates a managed client under the injected outbound policy. +func New(outboundPolicy security.OutboundPolicy) *Client { + return &Client{pool: outboundhttp.NewPool( + outboundPolicy, + outboundhttp.DefaultCacheLimit, + func(policy security.OutboundPolicy, trustedOrigin string, _ string) (outboundhttp.ManagedClient, error) { + client := security.NewOutboundHTTPClient(policy, moderationRequestTimeout) + transport, ok := client.Transport.(*http.Transport) + if !ok { + return outboundhttp.ManagedClient{}, fmt.Errorf("moderation HTTP transport is not reusable") + } + client.Transport = platformtracing.NewHTTPTransport(transport) + if trustedOrigin != "" { + client.CheckRedirect = outboundhttp.NewRedirectPolicy(outboundPolicy, trustedOrigin, "content moderation request") + } + return outboundhttp.ManagedClient{ + Client: client, + CloseIdleConnections: transport.CloseIdleConnections, + }, nil + }, + )} +} + +// Do executes a request only against the exact origin configured by an administrator. +func (c *Client) Do(request *http.Request, configuredEndpoint string) (*http.Response, error) { + if c != nil && c.doRequest != nil { + return c.doRequest(request, configuredEndpoint) + } + if c == nil || c.pool == nil { + return nil, fmt.Errorf("content moderation HTTP client is not configured") + } + return c.pool.Do(request, configuredEndpoint, "") +} + +// CloseIdleConnections releases pooled transports during application shutdown. +func (c *Client) CloseIdleConnections() { + if c != nil && c.pool != nil { + c.pool.CloseIdleConnections() + } +} diff --git a/backend/internal/infra/contentmoderation/provider.go b/backend/internal/infra/contentmoderation/provider.go new file mode 100644 index 000000000..2e3205a8d --- /dev/null +++ b/backend/internal/infra/contentmoderation/provider.go @@ -0,0 +1,431 @@ +package contentmoderation + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "syscall" + "time" + "unicode/utf8" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +const ( + defaultModerationPath = "/moderations" + maxTextChunkBytes = 12 * 1024 + maxImageSourceBytes = 20 * 1024 * 1024 + maxImageBatchBytes = 20 * 1024 * 1024 + maxRetryAfter = 30 * time.Second + maxResponseBytes = 4 << 20 +) + +var _ repository.ContentModerationProvider = (*Client)(nil) + +type moderationInput struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *moderationImageURL `json:"image_url,omitempty"` +} + +type moderationImageURL struct { + URL string `json:"url"` +} + +type moderationRequest struct { + Model string `json:"model"` + Input interface{} `json:"input"` +} + +type moderationCategoryResult struct { + Flagged bool `json:"flagged"` + Categories map[string]bool `json:"categories"` + CategoryScores map[string]float64 `json:"category_scores"` + CategoryAppliedInputTypes map[string][]string `json:"category_applied_input_types"` +} + +type moderationResponse struct { + ID string `json:"id"` + Model string `json:"model"` + Results []moderationCategoryResult `json:"results"` +} + +func (document moderationResponse) toDomainResponse() *domaincm.ProviderResponse { + results := make([]domaincm.CategoryResult, 0, len(document.Results)) + for _, result := range document.Results { + results = append(results, domaincm.CategoryResult{ + Flagged: result.Flagged, + Categories: result.Categories, + CategoryScores: result.CategoryScores, + CategoryAppliedInputTypes: result.CategoryAppliedInputTypes, + }) + } + return &domaincm.ProviderResponse{ID: document.ID, Model: document.Model, Results: results} +} + +// ValidateBaseURL validates and normalizes the supported OpenAI-compatible endpoint shape. +func (c *Client) ValidateBaseURL(raw string) error { + _, err := normalizeBaseURL(raw) + return err +} + +// ModerateText chunks UTF-8 text and submits each chunk within one shared timeout. +func (c *Client) ModerateText( + ctx context.Context, + config domaincm.ProviderConfig, + text string, + selected []string, + modality string, +) (*domaincm.ProviderResponse, error) { + chunks := splitTextChunks(text) + if len(chunks) == 0 { + return emptyResponse(), nil + } + if modality == "" { + modality = domaincm.ModalityText + } + deadline := providerDeadline(config.Timeout) + var merged *domaincm.ProviderResponse + for _, chunk := range chunks { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil, repository.ErrContentModerationTimeout + } + requestCtx, cancel := context.WithTimeout(ctx, remaining) + response, err := c.moderate(requestCtx, config, buildTextInput(chunk)) + cancel() + if err != nil { + return nil, err + } + merged = mergeResponses(merged, response) + if domaincm.EvaluateHit(response, selected, modality).Hit { + return merged, nil + } + } + return merged, nil +} + +// ModerateImages batches encoded image inputs within one shared timeout. +func (c *Client) ModerateImages( + ctx context.Context, + config domaincm.ProviderConfig, + images []domaincm.ProviderImage, + selected []string, + modality string, +) (*domaincm.ProviderResponse, error) { + dataURLs := buildImageDataURLs(images) + if len(dataURLs) == 0 { + return emptyResponse(), nil + } + if modality == "" { + modality = domaincm.ModalityImage + } + deadline := providerDeadline(config.Timeout) + var merged *domaincm.ProviderResponse + for _, batch := range batchImageDataURLs(dataURLs) { + remaining := time.Until(deadline) + if remaining <= 0 { + return nil, repository.ErrContentModerationTimeout + } + requestCtx, cancel := context.WithTimeout(ctx, remaining) + response, err := c.moderate(requestCtx, config, buildImageInputs(batch)) + cancel() + if err != nil { + return nil, err + } + merged = mergeResponses(merged, response) + if domaincm.EvaluateHit(response, selected, modality).Hit { + return merged, nil + } + } + return merged, nil +} + +func (c *Client) moderate(ctx context.Context, config domaincm.ProviderConfig, input interface{}) (*domaincm.ProviderResponse, error) { + endpoint, err := normalizeBaseURL(config.BaseURL) + if err != nil { + return nil, err + } + model := strings.TrimSpace(config.Model) + if model == "" { + model = "omni-moderation-latest" + } + body, err := json.Marshal(moderationRequest{Model: model, Input: input}) + if err != nil { + return nil, fmt.Errorf("marshal moderation request: %w", err) + } + + var lastErr error + for attempt := 0; attempt < 2; attempt++ { + if err := ctx.Err(); err != nil { + return nil, mapContextError(err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("build moderation request: %w", err) + } + request.Header.Set("Content-Type", "application/json") + if key := strings.TrimSpace(config.APIKey); key != "" { + request.Header.Set("Authorization", "Bearer "+key) + } + + response, err := c.Do(request, config.BaseURL) + if err != nil { + lastErr = fmt.Errorf("%w: %v", repository.ErrContentModerationNetwork, err) + if attempt == 0 && shouldRetryNetwork(err) { + continue + } + return nil, lastErr + } + payload, readErr := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes)) + _ = response.Body.Close() + if readErr != nil { + lastErr = fmt.Errorf("%w: read response", repository.ErrContentModerationNetwork) + if attempt == 0 && shouldRetryNetwork(readErr) { + continue + } + return nil, lastErr + } + + if response.StatusCode == http.StatusTooManyRequests || response.StatusCode >= 500 { + lastErr = mapHTTPStatus(response.StatusCode) + if attempt == 0 { + if err := waitForRetry(ctx, parseRetryAfter(response.Header.Get("Retry-After"))); err != nil { + return nil, err + } + continue + } + return nil, lastErr + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, mapHTTPStatus(response.StatusCode) + } + + var document moderationResponse + if err := json.Unmarshal(payload, &document); err != nil { + return nil, fmt.Errorf("%w: malformed JSON", repository.ErrContentModerationInvalidResp) + } + if len(document.Results) == 0 { + return nil, fmt.Errorf("%w: empty results", repository.ErrContentModerationInvalidResp) + } + for _, result := range document.Results { + if result.Categories == nil { + return nil, fmt.Errorf("%w: missing categories", repository.ErrContentModerationInvalidResp) + } + } + return document.toDomainResponse(), nil + } + if lastErr != nil { + return nil, lastErr + } + return nil, repository.ErrContentModerationService +} + +func normalizeBaseURL(raw string) (string, error) { + value := strings.TrimSpace(raw) + if value == "" { + return "", repository.ErrContentModerationInvalidBaseURL + } + if !strings.Contains(value, "://") { + value = "https://" + value + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" || parsed.User != nil { + return "", repository.ErrContentModerationInvalidBaseURL + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", repository.ErrContentModerationInvalidBaseURL + } + path := strings.TrimRight(parsed.Path, "/") + switch lower := strings.ToLower(path); { + case strings.HasSuffix(lower, "/moderations"): + case strings.HasSuffix(lower, "/v1"): + path += defaultModerationPath + case path == "" || path == "/": + path = "/v1" + defaultModerationPath + default: + path += "/v1" + defaultModerationPath + } + parsed.Path = path + parsed.RawQuery = "" + parsed.Fragment = "" + return parsed.String(), nil +} + +func splitTextChunks(text string) []string { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + raw := []byte(text) + chunks := make([]string, 0, (len(raw)/maxTextChunkBytes)+1) + for len(raw) > 0 { + limit := min(maxTextChunkBytes, len(raw)) + for limit > 0 && !utf8.Valid(raw[:limit]) { + limit-- + } + if limit == 0 { + _, size := utf8.DecodeRune(raw) + limit = max(size, 1) + } + chunks = append(chunks, string(raw[:limit])) + raw = raw[limit:] + } + return chunks +} + +func buildTextInput(text string) []moderationInput { + return []moderationInput{{Type: "text", Text: text}} +} + +func buildImageDataURLs(images []domaincm.ProviderImage) []string { + items := make([]string, 0, len(images)) + for _, image := range images { + if len(image.Data) == 0 { + continue + } + mimeType := strings.TrimSpace(image.MimeType) + if mimeType == "" { + mimeType = "image/png" + } + items = append(items, "data:"+mimeType+";base64,"+base64.StdEncoding.EncodeToString(image.Data)) + } + return items +} + +func buildImageInputs(dataURLs []string) []moderationInput { + items := make([]moderationInput, 0, len(dataURLs)) + for _, raw := range dataURLs { + if value := strings.TrimSpace(raw); value != "" { + items = append(items, moderationInput{Type: "image_url", ImageURL: &moderationImageURL{URL: value}}) + } + } + return items +} + +func batchImageDataURLs(dataURLs []string) [][]string { + batches := make([][]string, 0) + current := make([]string, 0) + currentSize := 0 + for _, item := range dataURLs { + size := len(item) + if size > maxImageSourceBytes { + if len(current) > 0 { + batches = append(batches, current) + current = nil + currentSize = 0 + } + batches = append(batches, []string{item}) + continue + } + if len(current) > 0 && currentSize+size > maxImageBatchBytes { + batches = append(batches, current) + current = nil + currentSize = 0 + } + current = append(current, item) + currentSize += size + } + if len(current) > 0 { + batches = append(batches, current) + } + return batches +} + +func mergeResponses(base, next *domaincm.ProviderResponse) *domaincm.ProviderResponse { + if base == nil { + return next + } + if next == nil || len(next.Results) == 0 { + return base + } + base.Results = append(base.Results, next.Results...) + if strings.TrimSpace(next.Model) != "" { + base.Model = next.Model + } + return base +} + +func emptyResponse() *domaincm.ProviderResponse { + return &domaincm.ProviderResponse{Results: []domaincm.CategoryResult{{ + Categories: map[string]bool{}, + CategoryScores: map[string]float64{}, + CategoryAppliedInputTypes: map[string][]string{}, + }}} +} + +func providerDeadline(timeout time.Duration) time.Time { + if timeout <= 0 { + timeout = 10 * time.Second + } + return time.Now().Add(timeout) +} + +func mapHTTPStatus(status int) error { + // Never include provider response bodies: compatible services may echo + // moderated content or credentials and this error can be persisted. + if status == http.StatusTooManyRequests { + return fmt.Errorf("%w: status %d", repository.ErrContentModerationRateLimited, status) + } + return fmt.Errorf("%w: status %d", repository.ErrContentModerationService, status) +} + +func mapContextError(err error) error { + if errors.Is(err, context.DeadlineExceeded) { + return repository.ErrContentModerationTimeout + } + return fmt.Errorf("%w: %v", repository.ErrContentModerationNetwork, err) +} + +func shouldRetryNetwork(err error) bool { + if err == nil { + return false + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNREFUSED) { + return true + } + var networkError net.Error + return errors.As(err, &networkError) && networkError.Timeout() +} + +func waitForRetry(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + return nil + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return mapContextError(ctx.Err()) + case <-timer.C: + return nil + } +} + +func parseRetryAfter(raw string) time.Duration { + value := strings.TrimSpace(raw) + if value == "" { + return 0 + } + if seconds, err := strconv.Atoi(value); err == nil && seconds > 0 { + return min(time.Duration(seconds)*time.Second, maxRetryAfter) + } + if parsed, err := http.ParseTime(value); err == nil { + delay := time.Until(parsed) + if delay > 0 { + return min(delay, maxRetryAfter) + } + } + return 0 +} diff --git a/backend/internal/infra/contentmoderation/provider_test.go b/backend/internal/infra/contentmoderation/provider_test.go new file mode 100644 index 000000000..553d54387 --- /dev/null +++ b/backend/internal/infra/contentmoderation/provider_test.go @@ -0,0 +1,148 @@ +package contentmoderation + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "testing" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" +) + +func TestNormalizeBaseURL(t *testing.T) { + tests := map[string]string{ + "https://api.openai.com": "https://api.openai.com/v1/moderations", + "https://api.openai.com/v1": "https://api.openai.com/v1/moderations", + "https://api.openai.com/v1/moderations": "https://api.openai.com/v1/moderations", + "http://localhost:8080/proxy": "http://localhost:8080/proxy/v1/moderations", + } + for input, expected := range tests { + actual, err := normalizeBaseURL(input) + if err != nil { + t.Fatalf("normalize %q: %v", input, err) + } + if actual != expected { + t.Fatalf("normalize %q = %q, want %q", input, actual, expected) + } + } +} + +func TestNormalizeBaseURLRejectsEmbeddedCredentials(t *testing.T) { + if _, err := normalizeBaseURL("https://user:secret@moderation.example/v1"); !errors.Is(err, repository.ErrContentModerationInvalidBaseURL) { + t.Fatalf("error = %v, want invalid base URL", err) + } +} + +func TestModerateTextUsesProviderWireContract(t *testing.T) { + client := &Client{doRequest: func(request *http.Request, configuredEndpoint string) (*http.Response, error) { + if configuredEndpoint != "https://moderation.example/v1" { + t.Fatalf("configured endpoint = %q", configuredEndpoint) + } + if request.URL.String() != "https://moderation.example/v1/moderations" { + t.Fatalf("request URL = %q", request.URL.String()) + } + if request.Header.Get("Authorization") != "Bearer secret" { + t.Fatalf("authorization header not set") + } + var body moderationRequest + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatalf("decode request: %v", err) + } + if body.Model != "omni-moderation-latest" { + t.Fatalf("model = %q", body.Model) + } + payload := `{"id":"modr_1","model":"omni-moderation-latest","results":[{"flagged":false,"categories":{"hate":false},"category_scores":{"hate":0.01},"category_applied_input_types":{"hate":["text"]}}]}` + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(payload)), + }, nil + }} + + response, err := client.ModerateText(context.Background(), domaincm.ProviderConfig{ + BaseURL: "https://moderation.example/v1", + APIKey: "secret", + Model: "omni-moderation-latest", + Timeout: time.Second, + }, "hello", []string{"hate"}, domaincm.ModalityText) + if err != nil { + t.Fatalf("moderate text: %v", err) + } + if response.Model != "omni-moderation-latest" || len(response.Results) != 1 { + t.Fatalf("unexpected response: %#v", response) + } +} + +func TestHTTPErrorNeverIncludesProviderBody(t *testing.T) { + errorValue := mapHTTPStatus(http.StatusInternalServerError) + if !errors.Is(errorValue, repository.ErrContentModerationService) { + t.Fatalf("unexpected error classification: %v", errorValue) + } + if strings.Contains(errorValue.Error(), "sensitive echoed prompt") { + t.Fatalf("provider body leaked into error: %v", errorValue) + } +} + +func TestSplitTextChunksPreservesUTF8(t *testing.T) { + var input strings.Builder + for input.Len() < maxTextChunkBytes+100 { + input.WriteString("你好世界") + } + chunks := splitTextChunks(input.String()) + if len(chunks) < 2 { + t.Fatalf("expected multiple chunks, got %d", len(chunks)) + } + for _, chunk := range chunks { + if len([]byte(chunk)) > maxTextChunkBytes { + t.Fatalf("chunk exceeds limit: %d", len([]byte(chunk))) + } + } +} + +func TestModerateTextContinuesAfterUnselectedHit(t *testing.T) { + calls := 0 + client := &Client{doRequest: func(*http.Request, string) (*http.Response, error) { + calls++ + categories := `{"hate":true,"violence":false}` + if calls > 1 { + categories = `{"hate":false,"violence":true}` + } + payload := `{"model":"omni-moderation-latest","results":[{"categories":` + categories + `,"category_scores":{"hate":0.9,"violence":0.9},"category_applied_input_types":{"hate":["text"],"violence":["text"]}}]}` + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(payload)), + }, nil + }} + input := strings.Repeat("abcdefghij", (maxTextChunkBytes/10)+20) + response, err := client.ModerateText(context.Background(), domaincm.ProviderConfig{ + BaseURL: "https://moderation.example/v1", + Timeout: time.Second, + }, input, []string{"violence"}, domaincm.ModalityText) + if err != nil { + t.Fatalf("moderate text: %v", err) + } + if calls < 2 { + t.Fatalf("calls = %d, want at least 2", calls) + } + if evaluation := domaincm.EvaluateHit(response, []string{"violence"}, domaincm.ModalityText); !evaluation.Hit { + t.Fatalf("selected hit was lost: %#v", evaluation) + } +} + +func TestMergeResponsesPreservesEveryResult(t *testing.T) { + base := &domaincm.ProviderResponse{Results: []domaincm.CategoryResult{{Categories: map[string]bool{"violence": false}}}} + next := &domaincm.ProviderResponse{Results: []domaincm.CategoryResult{ + {Categories: map[string]bool{"violence": false}}, + {Categories: map[string]bool{"violence": true}}, + }} + merged := mergeResponses(base, next) + if len(merged.Results) != 3 { + t.Fatalf("results = %d, want 3", len(merged.Results)) + } +} diff --git a/backend/internal/infra/persistence/models/chat.go b/backend/internal/infra/persistence/models/chat.go index e1899a935..21c744261 100644 --- a/backend/internal/infra/persistence/models/chat.go +++ b/backend/internal/infra/persistence/models/chat.go @@ -97,38 +97,40 @@ func (ConversationShare) TableName() string { // Message 存储会话内消息。 type Message struct { BaseModel - ConversationID uint `gorm:"not null;index:idx_chat_messages_conversation_id;comment:会话ID"` - UserID uint `gorm:"not null;index:idx_chat_messages_user_id;comment:用户ID"` - PublicID string `gorm:"size:32;not null;default:'';uniqueIndex:idx_chat_messages_public_id;comment:公开消息ID"` - ParentMessageID *uint `gorm:"index:idx_chat_messages_parent_message_id;comment:父消息ID"` - RunID string `gorm:"size:64;not null;default:'';index:idx_chat_messages_run_id;comment:会话运行ID"` - Role string `gorm:"size:32;not null;default:'';index:idx_chat_messages_role;comment:消息角色(user/assistant/system/tool)"` - ContentType string `gorm:"size:32;not null;default:'';comment:消息内容类型"` - Content string `gorm:"type:text;not null;default:'';comment:消息内容"` - ReasoningContent string `gorm:"type:text;not null;default:'';comment:上游推理内容回灌上下文"` - BranchReason string `gorm:"size:32;not null;default:'default';index:idx_chat_messages_branch_reason;comment:分支来源(default/retry/edit)"` - SourceMessageID *uint `gorm:"index:idx_chat_messages_source_message_id;comment:来源消息ID(重试/编辑源)"` - TokenUsage int64 `gorm:"not null;default:0;comment:token总消耗"` - InputTokens int64 `gorm:"not null;default:0;comment:输入Token"` - OutputTokens int64 `gorm:"not null;default:0;comment:输出Token"` - CacheReadTokens int64 `gorm:"not null;default:0;comment:缓存读取Token"` - CacheWriteTokens int64 `gorm:"not null;default:0;comment:缓存写入Token"` - ReasoningTokens int64 `gorm:"not null;default:0;comment:推理Token"` - LatencyMS int64 `gorm:"not null;default:0;comment:消息处理时长毫秒"` - BilledCurrency string `gorm:"size:16;not null;default:'USD';comment:消息计费币种"` - BilledNanousd int64 `gorm:"not null;default:0;comment:消息计费金额(纳美元)"` - PricingSnapshot string `gorm:"type:text;not null;default:'';comment:消息计费快照JSON"` - Status string `gorm:"size:32;not null;default:'';index:idx_chat_messages_status;comment:消息处理状态"` - ErrorCode string `gorm:"size:64;not null;default:'';comment:错误码"` - ErrorMessage string `gorm:"size:255;not null;default:'';comment:错误信息"` - IsCompacted bool `gorm:"not null;default:false;index:idx_chat_messages_is_compacted;comment:预留未使用(祖先链未按此过滤,无读写方)"` - EditedAt *time.Time `gorm:"index:idx_chat_messages_edited_at;comment:用户编辑时间"` - ParentPublicID string `gorm:"-"` - SourcePublicID string `gorm:"-"` - Attachments string `gorm:"-"` - MyFeedback string `gorm:"-"` - ThumbsUpCount int64 `gorm:"-"` - ThumbsDownCount int64 `gorm:"-"` + ConversationID uint `gorm:"not null;index:idx_chat_messages_conversation_id;comment:会话ID"` + UserID uint `gorm:"not null;index:idx_chat_messages_user_id;comment:用户ID"` + PublicID string `gorm:"size:32;not null;default:'';uniqueIndex:idx_chat_messages_public_id;comment:公开消息ID"` + ParentMessageID *uint `gorm:"index:idx_chat_messages_parent_message_id;comment:父消息ID"` + RunID string `gorm:"size:64;not null;default:'';index:idx_chat_messages_run_id;comment:会话运行ID"` + Role string `gorm:"size:32;not null;default:'';index:idx_chat_messages_role;comment:消息角色(user/assistant/system/tool)"` + ContentType string `gorm:"size:32;not null;default:'';comment:消息内容类型"` + Content string `gorm:"type:text;not null;default:'';comment:消息内容"` + ReasoningContent string `gorm:"type:text;not null;default:'';comment:上游推理内容回灌上下文"` + BranchReason string `gorm:"size:32;not null;default:'default';index:idx_chat_messages_branch_reason;comment:分支来源(default/retry/edit)"` + SourceMessageID *uint `gorm:"index:idx_chat_messages_source_message_id;comment:来源消息ID(重试/编辑源)"` + TokenUsage int64 `gorm:"not null;default:0;comment:token总消耗"` + InputTokens int64 `gorm:"not null;default:0;comment:输入Token"` + OutputTokens int64 `gorm:"not null;default:0;comment:输出Token"` + CacheReadTokens int64 `gorm:"not null;default:0;comment:缓存读取Token"` + CacheWriteTokens int64 `gorm:"not null;default:0;comment:缓存写入Token"` + ReasoningTokens int64 `gorm:"not null;default:0;comment:推理Token"` + LatencyMS int64 `gorm:"not null;default:0;comment:消息处理时长毫秒"` + BilledCurrency string `gorm:"size:16;not null;default:'USD';comment:消息计费币种"` + BilledNanousd int64 `gorm:"not null;default:0;comment:消息计费金额(纳美元)"` + PricingSnapshot string `gorm:"type:text;not null;default:'';comment:消息计费快照JSON"` + Status string `gorm:"size:32;not null;default:'';index:idx_chat_messages_status;comment:消息处理状态"` + ErrorCode string `gorm:"size:64;not null;default:'';comment:错误码"` + ErrorMessage string `gorm:"size:255;not null;default:'';comment:错误信息"` + ModerationEventID string `gorm:"size:40;not null;default:'';index:idx_chat_messages_moderation_event_id;comment:内容审核事件编号"` + ModerationCategoriesJSON string `gorm:"type:text;not null;default:'[]';comment:内容审核命中分类JSON"` + IsCompacted bool `gorm:"not null;default:false;index:idx_chat_messages_is_compacted;comment:预留未使用(祖先链未按此过滤,无读写方)"` + EditedAt *time.Time `gorm:"index:idx_chat_messages_edited_at;comment:用户编辑时间"` + ParentPublicID string `gorm:"-"` + SourcePublicID string `gorm:"-"` + Attachments string `gorm:"-"` + MyFeedback string `gorm:"-"` + ThumbsUpCount int64 `gorm:"-"` + ThumbsDownCount int64 `gorm:"-"` } // TableName 指定表名。 @@ -257,36 +259,39 @@ func (UserStorageQuota) TableName() string { // ConversationRun 存储每轮对话运行日志。 type ConversationRun struct { BaseModel - RunID string `gorm:"size:64;not null;default:'';uniqueIndex:idx_chat_runs_run_id;comment:运行ID"` - RequestID string `gorm:"size:64;not null;default:'';index:idx_chat_runs_request_id;comment:请求ID"` - UserID uint `gorm:"not null;default:0;index:idx_chat_runs_user_id;comment:用户ID"` - ConversationID uint `gorm:"not null;default:0;index:idx_chat_runs_conversation_id;comment:会话ID"` - TaskType string `gorm:"size:32;not null;default:'chat';index:idx_chat_runs_task_type;comment:任务类型"` - Endpoint string `gorm:"size:32;not null;default:'';index:idx_chat_runs_endpoint;comment:调用端点"` - Provider string `gorm:"size:32;not null;default:'';index:idx_chat_runs_provider;comment:模型提供商"` - ProviderProtocol string `gorm:"size:64;not null;default:'';index:idx_chat_runs_provider_protocol;comment:协议适配器快照"` - UpstreamID uint `gorm:"not null;default:0;index:idx_chat_runs_upstream_id;comment:上游ID"` - UpstreamModelID uint `gorm:"not null;default:0;index:idx_chat_runs_upstream_model_id;comment:上游真实模型ID"` - UpstreamName string `gorm:"size:128;not null;default:'';comment:上游名称快照"` - RequestedModelName string `gorm:"size:128;not null;default:'';index:idx_chat_runs_requested_model_name;comment:请求平台模型名"` - PlatformModelName string `gorm:"size:128;not null;default:'';index:idx_chat_runs_platform_model_name;comment:路由命中平台模型名"` - RoutedBindingCode string `gorm:"size:64;not null;default:'';index:idx_chat_runs_routed_binding_code;comment:实际路由上游模型绑定编码"` - ModelVendor string `gorm:"size:64;not null;default:'';comment:平台模型厂商快照"` - ModelIcon string `gorm:"size:64;not null;default:'';comment:平台模型图标快照"` - UpstreamModelName string `gorm:"size:256;not null;default:'';comment:上游真实模型名称"` - InputTokens int64 `gorm:"not null;default:0;comment:输入Token"` - OutputTokens int64 `gorm:"not null;default:0;comment:输出Token"` - CacheReadTokens int64 `gorm:"not null;default:0;comment:缓存读取Token"` - CacheWriteTokens int64 `gorm:"not null;default:0;comment:缓存写入Token"` - ReasoningTokens int64 `gorm:"not null;default:0;comment:推理Token"` - ToolCallsCount int `gorm:"not null;default:0;comment:工具调用次数"` - FirstTokenLatencyMS int64 `gorm:"not null;default:0;comment:首Token时延毫秒"` - TotalLatencyMS int64 `gorm:"not null;default:0;comment:总时长毫秒"` - Status string `gorm:"size:32;not null;default:'';index:idx_chat_runs_status;comment:运行状态"` - ErrorCode string `gorm:"size:64;not null;default:'';comment:错误码"` - ErrorMessage string `gorm:"size:255;not null;default:'';comment:错误信息"` - StartedAt time.Time `gorm:"not null;comment:开始时间"` - EndedAt *time.Time `gorm:"comment:结束时间"` + RunID string `gorm:"size:64;not null;default:'';uniqueIndex:idx_chat_runs_run_id;comment:运行ID"` + RequestID string `gorm:"size:64;not null;default:'';index:idx_chat_runs_request_id;comment:请求ID"` + UserID uint `gorm:"not null;default:0;index:idx_chat_runs_user_id;comment:用户ID"` + ConversationID uint `gorm:"not null;default:0;index:idx_chat_runs_conversation_id;comment:会话ID"` + TaskType string `gorm:"size:32;not null;default:'chat';index:idx_chat_runs_task_type;comment:任务类型"` + Endpoint string `gorm:"size:32;not null;default:'';index:idx_chat_runs_endpoint;comment:调用端点"` + Provider string `gorm:"size:32;not null;default:'';index:idx_chat_runs_provider;comment:模型提供商"` + ProviderProtocol string `gorm:"size:64;not null;default:'';index:idx_chat_runs_provider_protocol;comment:协议适配器快照"` + UpstreamID uint `gorm:"not null;default:0;index:idx_chat_runs_upstream_id;comment:上游ID"` + UpstreamModelID uint `gorm:"not null;default:0;index:idx_chat_runs_upstream_model_id;comment:上游真实模型ID"` + UpstreamName string `gorm:"size:128;not null;default:'';comment:上游名称快照"` + RequestedModelName string `gorm:"size:128;not null;default:'';index:idx_chat_runs_requested_model_name;comment:请求平台模型名"` + PlatformModelName string `gorm:"size:128;not null;default:'';index:idx_chat_runs_platform_model_name;comment:路由命中平台模型名"` + RoutedBindingCode string `gorm:"size:64;not null;default:'';index:idx_chat_runs_routed_binding_code;comment:实际路由上游模型绑定编码"` + ModelVendor string `gorm:"size:64;not null;default:'';comment:平台模型厂商快照"` + ModelIcon string `gorm:"size:64;not null;default:'';comment:平台模型图标快照"` + UpstreamModelName string `gorm:"size:256;not null;default:'';comment:上游真实模型名称"` + InputTokens int64 `gorm:"not null;default:0;comment:输入Token"` + OutputTokens int64 `gorm:"not null;default:0;comment:输出Token"` + CacheReadTokens int64 `gorm:"not null;default:0;comment:缓存读取Token"` + CacheWriteTokens int64 `gorm:"not null;default:0;comment:缓存写入Token"` + ReasoningTokens int64 `gorm:"not null;default:0;comment:推理Token"` + ToolCallsCount int `gorm:"not null;default:0;comment:工具调用次数"` + FirstTokenLatencyMS int64 `gorm:"not null;default:0;comment:首Token时延毫秒"` + TotalLatencyMS int64 `gorm:"not null;default:0;comment:总时长毫秒"` + Status string `gorm:"size:32;not null;default:'';index:idx_chat_runs_status;comment:运行状态"` + ErrorCode string `gorm:"size:64;not null;default:'';comment:错误码"` + ErrorMessage string `gorm:"size:255;not null;default:'';comment:错误信息"` + ModerationState string `gorm:"size:32;not null;default:'not_required';index:idx_chat_runs_moderation_state;comment:内容审核状态"` + ModerationEventID string `gorm:"size:40;not null;default:'';index:idx_chat_runs_moderation_event_id;comment:主审核事件编号"` + ModerationCategoriesJSON string `gorm:"type:text;not null;default:'[]';comment:审核命中分类摘要JSON"` + StartedAt time.Time `gorm:"not null;comment:开始时间"` + EndedAt *time.Time `gorm:"comment:结束时间"` } // TableName 指定表名。 diff --git a/backend/internal/infra/persistence/models/content_moderation.go b/backend/internal/infra/persistence/models/content_moderation.go new file mode 100644 index 000000000..1c3647762 --- /dev/null +++ b/backend/internal/infra/persistence/models/content_moderation.go @@ -0,0 +1,57 @@ +package model + +import "time" + +// ContentModerationEvent stores hit and failed-open moderation records only. +type ContentModerationEvent struct { + BaseModel + PublicID string `gorm:"size:40;not null;default:'';uniqueIndex:idx_content_moderation_events_public_id;comment:公开事件编号"` + UserID uint `gorm:"not null;default:0;index:idx_content_moderation_events_user_id;comment:用户ID"` + ConversationID uint `gorm:"not null;default:0;index:idx_content_moderation_events_conversation_id;comment:会话ID"` + RunID string `gorm:"size:64;not null;default:'';index:idx_content_moderation_events_run_id;comment:运行ID"` + MessageID uint `gorm:"not null;default:0;index:idx_content_moderation_events_message_id;comment:消息ID"` + MessagePublicID string `gorm:"size:32;not null;default:'';index:idx_content_moderation_events_message_public_id;comment:消息公开ID"` + Direction string `gorm:"size:16;not null;default:'';index:idx_content_moderation_events_direction;comment:方向(input/output)"` + Modality string `gorm:"size:16;not null;default:'';index:idx_content_moderation_events_modality;comment:模态(text/image)"` + Model string `gorm:"size:128;not null;default:'';comment:审核模型"` + PolicyVersion int64 `gorm:"not null;default:0;comment:策略版本"` + Result string `gorm:"size:32;not null;default:'';index:idx_content_moderation_events_result;comment:结果(passed/hit/failed_open)"` + CategoriesJSON string `gorm:"type:text;not null;default:'[]';comment:命中分类JSON"` + CategoryScoresJSON string `gorm:"type:text;not null;default:'{}';comment:分类分数JSON"` + LatencyMS int64 `gorm:"not null;default:0;comment:审核延迟毫秒"` + ErrorCode string `gorm:"size:64;not null;default:'';index:idx_content_moderation_events_error_code;comment:错误码"` + ErrorMessage string `gorm:"size:255;not null;default:'';comment:错误信息"` + ContentLocationJSON string `gorm:"type:text;not null;default:'{}';comment:内容位置JSON"` + ContentSummary string `gorm:"size:255;not null;default:'';comment:内容摘要"` + EncryptedText string `gorm:"type:text;not null;default:'';comment:命中文本AES-GCM密文"` + ImageCount int `gorm:"not null;default:0;comment:隔离图片数量"` + ImageMetaJSON string `gorm:"type:text;not null;default:'[]';comment:隔离图片元数据JSON"` + ContentExpiresAt time.Time `gorm:"not null;index:idx_content_moderation_events_content_expires_at;comment:原文密文过期时间"` + MetadataExpiresAt time.Time `gorm:"not null;index:idx_content_moderation_events_metadata_expires_at;comment:元数据过期时间"` +} + +// TableName 指定表名。 +func (ContentModerationEvent) TableName() string { + return "content_moderation_events" +} + +// ContentModerationDailyStat stores anonymous daily aggregates. +type ContentModerationDailyStat struct { + BaseModel + StatDate time.Time `gorm:"type:date;not null;uniqueIndex:uk_content_moderation_daily_stats,priority:1;comment:统计日期"` + Direction string `gorm:"size:16;not null;default:'';uniqueIndex:uk_content_moderation_daily_stats,priority:2;comment:方向"` + Modality string `gorm:"size:16;not null;default:'';uniqueIndex:uk_content_moderation_daily_stats,priority:3;comment:模态"` + Result string `gorm:"size:32;not null;default:'';uniqueIndex:uk_content_moderation_daily_stats,priority:4;comment:结果"` + Category string `gorm:"size:64;not null;default:'';uniqueIndex:uk_content_moderation_daily_stats,priority:5;comment:分类(空表示汇总)"` + CheckCount int64 `gorm:"not null;default:0;comment:检查次数"` + ContentItems int64 `gorm:"not null;default:0;comment:内容项数"` + HitCount int64 `gorm:"not null;default:0;comment:命中次数"` + FailureCount int64 `gorm:"not null;default:0;comment:失败开放次数"` + LatencySumMS int64 `gorm:"not null;default:0;comment:延迟合计毫秒"` + LatencyCount int64 `gorm:"not null;default:0;comment:延迟样本数"` +} + +// TableName 指定表名。 +func (ContentModerationDailyStat) TableName() string { + return "content_moderation_daily_stats" +} diff --git a/backend/internal/infra/persistence/postgres/contentmoderation/repository.go b/backend/internal/infra/persistence/postgres/contentmoderation/repository.go new file mode 100644 index 000000000..a72f96f7d --- /dev/null +++ b/backend/internal/infra/persistence/postgres/contentmoderation/repository.go @@ -0,0 +1,458 @@ +package contentmoderation + +import ( + "context" + "fmt" + "strings" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/dberror" + model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/models" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// Repo implements repository.ContentModerationRepository. +type Repo struct { + db *gorm.DB +} + +var _ repository.ContentModerationRepository = (*Repo)(nil) + +// NewRepo creates a content moderation repository. +func NewRepo(db *gorm.DB) *Repo { + return &Repo{db: db} +} + +func translateError(err error) error { + if dberror.IsRecordNotFound(err) { + return repository.ErrNotFound + } + return err +} + +func (r *Repo) CreateEvent(ctx context.Context, event *domaincm.Event) error { + if event == nil { + return nil + } + row := toModelEvent(*event) + if err := r.db.WithContext(ctx).Create(&row).Error; err != nil { + return translateError(err) + } + event.ID = row.ID + event.CreatedAt = row.CreatedAt + event.UpdatedAt = row.UpdatedAt + return nil +} + +func (r *Repo) GetEventByPublicID(ctx context.Context, publicID string) (*domaincm.Event, error) { + var row model.ContentModerationEvent + if err := r.db.WithContext(ctx).Where("public_id = ?", strings.TrimSpace(publicID)).First(&row).Error; err != nil { + return nil, translateError(err) + } + item := toDomainEvent(row) + return &item, nil +} + +func (r *Repo) GetLatestHitEventByRunID(ctx context.Context, runID string) (*domaincm.Event, error) { + runID = strings.TrimSpace(runID) + if runID == "" { + return nil, nil + } + var row model.ContentModerationEvent + err := r.db.WithContext(ctx). + Where("run_id = ? AND result = ?", runID, domaincm.ResultHit). + Order("id DESC"). + First(&row).Error + if err != nil { + if dberror.IsRecordNotFound(err) || err == gorm.ErrRecordNotFound { + return nil, nil + } + return nil, translateError(err) + } + item := toDomainEvent(row) + return &item, nil +} + +func (r *Repo) ListEvents(ctx context.Context, filter domaincm.EventListFilter) ([]domaincm.Event, int64, error) { + q := r.db.WithContext(ctx).Model(&model.ContentModerationEvent{}) + if v := strings.TrimSpace(filter.Direction); v != "" { + q = q.Where("direction = ?", v) + } + if v := strings.TrimSpace(filter.Modality); v != "" { + q = q.Where("modality = ?", v) + } + if v := strings.TrimSpace(filter.Result); v != "" { + q = q.Where("result = ?", v) + } + if v := strings.TrimSpace(filter.Category); v != "" { + q = q.Where("categories_json LIKE ?", "%\""+v+"\"%") + } + if filter.UserID > 0 { + q = q.Where("user_id = ?", filter.UserID) + } + if v := strings.TrimSpace(filter.RunID); v != "" { + q = q.Where("run_id = ?", v) + } + if filter.From != nil { + q = q.Where("created_at >= ?", *filter.From) + } + if filter.To != nil { + q = q.Where("created_at <= ?", *filter.To) + } + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, translateError(err) + } + limit := filter.Limit + if limit <= 0 { + limit = 20 + } + var rows []model.ContentModerationEvent + if err := q.Order("id desc").Offset(filter.Offset).Limit(limit).Find(&rows).Error; err != nil { + return nil, 0, translateError(err) + } + items := make([]domaincm.Event, 0, len(rows)) + for _, row := range rows { + items = append(items, toDomainEvent(row)) + } + return items, total, nil +} + +func (r *Repo) ClearExpiredContentByPublicIDs(ctx context.Context, publicIDs []string) (int64, error) { + if len(publicIDs) == 0 { + return 0, nil + } + res := r.db.WithContext(ctx).Model(&model.ContentModerationEvent{}). + Where("public_id IN ?", publicIDs). + Updates(map[string]interface{}{ + "encrypted_text": "", + "image_count": 0, + "image_meta_json": "[]", + "content_summary": "", + }) + return res.RowsAffected, translateError(res.Error) +} + +func (r *Repo) ListExpiredContentEvents(ctx context.Context, before time.Time, limit int) ([]domaincm.Event, error) { + if limit <= 0 { + limit = 100 + } + var rows []model.ContentModerationEvent + // Include text ciphertext and image isolation metadata so pure-text hits expire too. + if err := r.db.WithContext(ctx). + Where("content_expires_at <= ? AND (encrypted_text <> '' OR image_count > 0 OR (image_meta_json <> '' AND image_meta_json <> '[]'))", before). + Limit(limit). + Find(&rows).Error; err != nil { + return nil, translateError(err) + } + items := make([]domaincm.Event, 0, len(rows)) + for _, row := range rows { + items = append(items, toDomainEvent(row)) + } + return items, nil +} + +func (r *Repo) DeleteExpiredMetadata(ctx context.Context, before time.Time) (int64, error) { + // Physical delete: retention policy requires rows to disappear, not soft-delete. + res := r.db.WithContext(ctx). + Unscoped(). + Where( + "metadata_expires_at <= ? AND encrypted_text = '' AND image_count = 0 AND (image_meta_json = '' OR image_meta_json = '[]')", + before, + ). + Delete(&model.ContentModerationEvent{}) + return res.RowsAffected, translateError(res.Error) +} + +func (r *Repo) IncrementDailyStat(ctx context.Context, input repository.DailyStatIncrement) error { + day := input.StatDate.UTC().Truncate(24 * time.Hour) + row := model.ContentModerationDailyStat{ + StatDate: day, + Direction: input.Direction, + Modality: input.Modality, + Result: input.Result, + Category: input.Category, + CheckCount: input.CheckCount, + ContentItems: input.ContentItems, + HitCount: input.HitCount, + FailureCount: input.FailureCount, + LatencySumMS: input.LatencyMS, + LatencyCount: 0, + } + if input.LatencyMS > 0 { + row.LatencyCount = 1 + } + return translateError(r.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "stat_date"}, + {Name: "direction"}, + {Name: "modality"}, + {Name: "result"}, + {Name: "category"}, + }, + DoUpdates: clause.Assignments(map[string]interface{}{ + "check_count": gorm.Expr("check_count + ?", input.CheckCount), + "content_items": gorm.Expr("content_items + ?", input.ContentItems), + "hit_count": gorm.Expr("hit_count + ?", input.HitCount), + "failure_count": gorm.Expr("failure_count + ?", input.FailureCount), + "latency_sum_ms": gorm.Expr("latency_sum_ms + ?", input.LatencyMS), + "latency_count": gorm.Expr("latency_count + ?", row.LatencyCount), + "updated_at": time.Now(), + }), + }).Create(&row).Error) +} + +func (r *Repo) ListDailyStats(ctx context.Context, from, to time.Time) ([]domaincm.DailyStat, error) { + var rows []model.ContentModerationDailyStat + if err := r.db.WithContext(ctx). + Where("stat_date >= ? AND stat_date <= ?", from.UTC().Truncate(24*time.Hour), to.UTC().Truncate(24*time.Hour)). + Order("stat_date asc, direction, modality, result, category"). + Find(&rows).Error; err != nil { + return nil, translateError(err) + } + items := make([]domaincm.DailyStat, 0, len(rows)) + for _, row := range rows { + items = append(items, toDomainStat(row)) + } + return items, nil +} + +func (r *Repo) DeleteDailyStatsBefore(ctx context.Context, before time.Time) (int64, error) { + res := r.db.WithContext(ctx). + Unscoped(). + Where("stat_date < ?", before.UTC().Truncate(24*time.Hour)). + Delete(&model.ContentModerationDailyStat{}) + return res.RowsAffected, translateError(res.Error) +} + +func (r *Repo) UpdateRunModeration(ctx context.Context, runID string, state string, eventPublicID string, categoriesJSON string) error { + runID = strings.TrimSpace(runID) + if runID == "" { + return nil + } + updates := map[string]interface{}{ + "moderation_state": strings.TrimSpace(state), + } + if eventPublicID != "" { + updates["moderation_event_id"] = eventPublicID + } + if categoriesJSON != "" { + updates["moderation_categories_json"] = categoriesJSON + } + if state == domaincm.ModerationStateBlocked { + updates["status"] = domaincm.StatusBlocked + } + return translateError(r.db.WithContext(ctx).Model(&model.ConversationRun{}). + Where("run_id = ?", runID). + Updates(updates).Error) +} + +// ApplyRunBlock writes blocked message state, revokes assistant attachments, clears +// assistant text/process traces, and marks the run blocked in a single transaction. +func (r *Repo) ApplyRunBlock(ctx context.Context, runID string, includeUser bool, eventPublicID string, categoriesJSON string) ([]string, error) { + runID = strings.TrimSpace(runID) + if runID == "" { + return nil, nil + } + fileIDs := make([]string, 0) + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var assistantMessageIDs []uint + if err := tx.Model(&model.Message{}). + Where("run_id = ? AND role = ?", runID, "assistant"). + Pluck("id", &assistantMessageIDs).Error; err != nil { + return err + } + if len(assistantMessageIDs) > 0 { + var attachmentFileIDs []string + if err := tx.Model(&model.Attachment{}). + Where("message_id IN ? AND status <> ? AND file_id <> ''", assistantMessageIDs, "deleted"). + Pluck("file_id", &attachmentFileIDs).Error; err != nil { + return err + } + seen := make(map[string]struct{}, len(attachmentFileIDs)) + for _, fileID := range attachmentFileIDs { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + continue + } + if _, ok := seen[fileID]; ok { + continue + } + seen[fileID] = struct{}{} + fileIDs = append(fileIDs, fileID) + } + if err := tx.Model(&model.Attachment{}). + Where("message_id IN ? AND status <> ?", assistantMessageIDs, "deleted"). + Update("status", "deleted").Error; err != nil { + return err + } + if len(fileIDs) > 0 { + if err := tx.Model(&model.FileObject{}). + Where("file_id IN ?", fileIDs). + Updates(map[string]interface{}{ + "status": "moderation_blocked", + "user_id": 0, + }).Error; err != nil { + return err + } + } + } + + msgUpdates := map[string]interface{}{ + "status": domaincm.StatusBlocked, + "moderation_event_id": eventPublicID, + "moderation_categories_json": categoriesJSON, + "error_code": "content_moderation.blocked", + "error_message": "content blocked by moderation", + } + msgQ := tx.Model(&model.Message{}).Where("run_id = ?", runID) + if !includeUser { + msgQ = msgQ.Where("role = ?", "assistant") + } + if err := msgQ.Updates(msgUpdates).Error; err != nil { + return err + } + if err := tx.Model(&model.Message{}). + Where("run_id = ? AND role = ?", runID, "assistant"). + Updates(map[string]interface{}{ + "content": "", + "reasoning_content": "", + "content_type": "text", + }).Error; err != nil { + return err + } + // Drop user-visible process traces / upstream-think so history cannot rehydrate withdrawn content. + if err := tx.Where("run_id = ? AND event_scope IN ?", runID, []string{"trace_block", "trace_event"}). + Delete(&model.ChatRunEvent{}).Error; err != nil { + return err + } + runUpdates := map[string]interface{}{ + "moderation_state": domaincm.ModerationStateBlocked, + "moderation_event_id": eventPublicID, + "moderation_categories_json": categoriesJSON, + "status": domaincm.StatusBlocked, + } + res := tx.Model(&model.ConversationRun{}). + Where("run_id = ?", runID). + Updates(runUpdates) + if res.Error != nil { + return res.Error + } + if res.RowsAffected == 0 { + return fmt.Errorf("content moderation apply block: run %s not found", runID) + } + return nil + }) + if err != nil { + return nil, translateError(err) + } + return fileIDs, nil +} + +func (r *Repo) GetRunModerationState(ctx context.Context, runID string) (string, error) { + var state string + err := r.db.WithContext(ctx).Model(&model.ConversationRun{}). + Select("moderation_state"). + Where("run_id = ?", strings.TrimSpace(runID)). + Limit(1). + Scan(&state).Error + return state, translateError(err) +} + +func (r *Repo) ListStaleModeratingRuns(ctx context.Context, olderThan time.Time, limit int) ([]string, error) { + if limit <= 0 { + limit = 100 + } + var runIDs []string + err := r.db.WithContext(ctx).Model(&model.ConversationRun{}). + Select("run_id"). + Where("moderation_state IN ? AND updated_at < ?", []string{ + domaincm.ModerationStateModerating, + domaincm.ModerationStatePending, + }, olderThan). + Limit(limit). + Pluck("run_id", &runIDs).Error + return runIDs, translateError(err) +} + +func toModelEvent(item domaincm.Event) model.ContentModerationEvent { + return model.ContentModerationEvent{ + BaseModel: model.BaseModel{ID: item.ID, CreatedAt: item.CreatedAt, UpdatedAt: item.UpdatedAt}, + PublicID: item.PublicID, + UserID: item.UserID, + ConversationID: item.ConversationID, + RunID: item.RunID, + MessageID: item.MessageID, + MessagePublicID: item.MessagePublicID, + Direction: item.Direction, + Modality: item.Modality, + Model: item.Model, + PolicyVersion: item.PolicyVersion, + Result: item.Result, + CategoriesJSON: item.CategoriesJSON, + CategoryScoresJSON: item.CategoryScoresJSON, + LatencyMS: item.LatencyMS, + ErrorCode: item.ErrorCode, + ErrorMessage: item.ErrorMessage, + ContentLocationJSON: item.ContentLocationJSON, + ContentSummary: item.ContentSummary, + EncryptedText: item.EncryptedText, + ImageCount: item.ImageCount, + ImageMetaJSON: item.ImageMetaJSON, + ContentExpiresAt: item.ContentExpiresAt, + MetadataExpiresAt: item.MetadataExpiresAt, + } +} + +func toDomainEvent(row model.ContentModerationEvent) domaincm.Event { + return domaincm.Event{ + ID: row.ID, + PublicID: row.PublicID, + UserID: row.UserID, + ConversationID: row.ConversationID, + RunID: row.RunID, + MessageID: row.MessageID, + MessagePublicID: row.MessagePublicID, + Direction: row.Direction, + Modality: row.Modality, + Model: row.Model, + PolicyVersion: row.PolicyVersion, + Result: row.Result, + CategoriesJSON: row.CategoriesJSON, + CategoryScoresJSON: row.CategoryScoresJSON, + LatencyMS: row.LatencyMS, + ErrorCode: row.ErrorCode, + ErrorMessage: row.ErrorMessage, + ContentLocationJSON: row.ContentLocationJSON, + ContentSummary: row.ContentSummary, + EncryptedText: row.EncryptedText, + ImageCount: row.ImageCount, + ImageMetaJSON: row.ImageMetaJSON, + ContentExpiresAt: row.ContentExpiresAt, + MetadataExpiresAt: row.MetadataExpiresAt, + CreatedAt: row.CreatedAt, + UpdatedAt: row.UpdatedAt, + } +} + +func toDomainStat(row model.ContentModerationDailyStat) domaincm.DailyStat { + return domaincm.DailyStat{ + ID: row.ID, + StatDate: row.StatDate, + Direction: row.Direction, + Modality: row.Modality, + Result: row.Result, + Category: row.Category, + CheckCount: row.CheckCount, + ContentItems: row.ContentItems, + HitCount: row.HitCount, + FailureCount: row.FailureCount, + LatencySumMS: row.LatencySumMS, + LatencyCount: row.LatencyCount, + CreatedAt: row.CreatedAt, + UpdatedAt: row.UpdatedAt, + } +} diff --git a/backend/internal/infra/persistence/postgres/contentmoderation/repository_test.go b/backend/internal/infra/persistence/postgres/contentmoderation/repository_test.go new file mode 100644 index 000000000..1955ff753 --- /dev/null +++ b/backend/internal/infra/persistence/postgres/contentmoderation/repository_test.go @@ -0,0 +1,135 @@ +package contentmoderation + +import ( + "context" + "testing" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" + model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/models" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestApplyRunBlockWithdrawsAssistantAttachments(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:content_moderation_apply_block?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + if err := db.AutoMigrate( + &model.Message{}, + &model.Attachment{}, + &model.FileObject{}, + &model.ConversationRun{}, + &model.ChatRunEvent{}, + ); err != nil { + t.Fatalf("migrate: %v", err) + } + + run := model.ConversationRun{RunID: "run_with_attachment", Status: "success", ModerationState: domaincm.ModerationStateModerating, StartedAt: time.Now()} + if err := db.Create(&run).Error; err != nil { + t.Fatalf("create run: %v", err) + } + message := model.Message{RunID: run.RunID, Role: "assistant", ContentType: "image", Content: "unsafe output", ReasoningContent: "unsafe reasoning", Status: "success"} + if err := db.Create(&message).Error; err != nil { + t.Fatalf("create message: %v", err) + } + file := model.FileObject{FileID: "file_generated", UserID: 42, StoragePath: "generated/file.png", Status: "active"} + if err := db.Create(&file).Error; err != nil { + t.Fatalf("create file: %v", err) + } + attachment := model.Attachment{MessageID: message.ID, UserID: 42, FileID: file.FileID, Kind: "image", Status: "active", UploadedAt: time.Now()} + if err := db.Create(&attachment).Error; err != nil { + t.Fatalf("create attachment: %v", err) + } + trace := model.ChatRunEvent{RunID: run.RunID, EventScope: "trace_event", EventID: "trace_1", StartedAt: time.Now()} + if err := db.Create(&trace).Error; err != nil { + t.Fatalf("create trace: %v", err) + } + + repo := NewRepo(db) + fileIDs, err := repo.ApplyRunBlock(context.Background(), run.RunID, false, "cme_hit", `["violence"]`) + if err != nil { + t.Fatalf("apply block: %v", err) + } + if len(fileIDs) != 1 || fileIDs[0] != file.FileID { + t.Fatalf("unexpected output file IDs: %#v", fileIDs) + } + + if err := db.First(&message, message.ID).Error; err != nil { + t.Fatalf("reload message: %v", err) + } + if message.Status != domaincm.StatusBlocked || message.Content != "" || message.ReasoningContent != "" { + t.Fatalf("assistant content was not withdrawn: %#v", message) + } + if err := db.First(&attachment, attachment.ID).Error; err != nil { + t.Fatalf("reload attachment: %v", err) + } + if attachment.Status != "deleted" { + t.Fatalf("attachment remains accessible: %#v", attachment) + } + if err := db.First(&file, file.ID).Error; err != nil { + t.Fatalf("reload file: %v", err) + } + if file.Status != "moderation_blocked" || file.UserID != 0 || file.StoragePath == "" { + t.Fatalf("file was not revoked while retaining its cleanup path: %#v", file) + } + if err := db.First(&run, run.ID).Error; err != nil { + t.Fatalf("reload run: %v", err) + } + if run.Status != domaincm.StatusBlocked || run.ModerationState != domaincm.ModerationStateBlocked { + t.Fatalf("run was not blocked: %#v", run) + } + var traceCount int64 + if err := db.Model(&model.ChatRunEvent{}).Where("run_id = ?", run.RunID).Count(&traceCount).Error; err != nil { + t.Fatalf("count traces: %v", err) + } + if traceCount != 0 { + t.Fatalf("blocked trace remains visible, count=%d", traceCount) + } +} + +func TestDeleteExpiredMetadataKeepsRowsWithUnclearedContent(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:content_moderation_retention?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + if err := db.AutoMigrate(&model.ContentModerationEvent{}); err != nil { + t.Fatalf("migrate: %v", err) + } + + expired := time.Now().Add(-time.Hour) + pendingCleanup := model.ContentModerationEvent{ + PublicID: "cme_pending_cleanup", + ImageCount: 1, + ImageMetaJSON: `[{"storage_path":"moderation/pending"}]`, + ContentExpiresAt: expired, + MetadataExpiresAt: expired, + } + cleared := model.ContentModerationEvent{ + PublicID: "cme_cleared", + ImageMetaJSON: "[]", + ContentExpiresAt: expired, + MetadataExpiresAt: expired, + } + if err := db.Create(&[]model.ContentModerationEvent{pendingCleanup, cleared}).Error; err != nil { + t.Fatalf("create events: %v", err) + } + + repo := NewRepo(db) + deleted, err := repo.DeleteExpiredMetadata(context.Background(), time.Now()) + if err != nil { + t.Fatalf("delete expired metadata: %v", err) + } + if deleted != 1 { + t.Fatalf("deleted=%d, want 1", deleted) + } + + var remaining []model.ContentModerationEvent + if err := db.Find(&remaining).Error; err != nil { + t.Fatalf("list remaining events: %v", err) + } + if len(remaining) != 1 || remaining[0].PublicID != pendingCleanup.PublicID { + t.Fatalf("uncleared isolation metadata must remain retryable: %#v", remaining) + } +} diff --git a/backend/internal/infra/persistence/postgres/conversation/repository.go b/backend/internal/infra/persistence/postgres/conversation/repository.go index b33a8761b..ee03f6bcd 100644 --- a/backend/internal/infra/persistence/postgres/conversation/repository.go +++ b/backend/internal/infra/persistence/postgres/conversation/repository.go @@ -1697,6 +1697,73 @@ func (r *Repo) CreateConversationRun(ctx context.Context, item *domainconversati return nil } +// EnsureConversationRun inserts a run row when missing so mid-flight moderation updates have a target. +func (r *Repo) EnsureConversationRun(ctx context.Context, item *domainconversation.Run) error { + if item == nil || strings.TrimSpace(item.RunID) == "" { + return nil + } + entity := toConversationRunModel(item) + return translateError(r.db.WithContext(ctx). + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "run_id"}}, + DoNothing: true, + }). + Create(&entity).Error) +} + +// UpsertConversationRun writes the final run snapshot (create or full update by run_id). +func (r *Repo) UpsertConversationRun(ctx context.Context, item *domainconversation.Run) error { + if item == nil || strings.TrimSpace(item.RunID) == "" { + return nil + } + entity := toConversationRunModel(item) + err := r.db.WithContext(ctx). + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "run_id"}}, + DoUpdates: clause.AssignmentColumns([]string{ + "request_id", + "user_id", + "conversation_id", + "task_type", + "endpoint", + "provider", + "provider_protocol", + "upstream_id", + "upstream_model_id", + "upstream_name", + "requested_model_name", + "platform_model_name", + "routed_binding_code", + "model_vendor", + "model_icon", + "upstream_model_name", + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "reasoning_tokens", + "tool_calls_count", + "first_token_latency_ms", + "total_latency_ms", + "status", + "error_code", + "error_message", + "moderation_state", + "moderation_event_id", + "moderation_categories_json", + "started_at", + "ended_at", + "updated_at", + }), + }). + Create(&entity).Error + if err != nil { + return translateError(err) + } + *item = toConversationRunDomain(entity) + return nil +} + // UpsertConversationMessageTrace 写入或更新消息轨迹。 func (r *Repo) UpsertConversationMessageTrace(ctx context.Context, item *domainconversation.MessageTrace) error { if item == nil { @@ -3573,40 +3640,42 @@ func toUserDomain(item models.User) domainuser.User { func toMessageDomain(item models.Message) domainconversation.Message { return domainconversation.Message{ - ID: item.ID, - ConversationID: item.ConversationID, - UserID: item.UserID, - PublicID: item.PublicID, - ParentMessageID: item.ParentMessageID, - RunID: item.RunID, - Role: item.Role, - ContentType: item.ContentType, - Content: item.Content, - ReasoningContent: item.ReasoningContent, - BranchReason: item.BranchReason, - SourceMessageID: item.SourceMessageID, - TokenUsage: item.TokenUsage, - InputTokens: item.InputTokens, - OutputTokens: item.OutputTokens, - CacheReadTokens: item.CacheReadTokens, - CacheWriteTokens: item.CacheWriteTokens, - ReasoningTokens: item.ReasoningTokens, - LatencyMS: item.LatencyMS, - BilledCurrency: item.BilledCurrency, - BilledNanousd: item.BilledNanousd, - PricingSnapshot: item.PricingSnapshot, - Status: item.Status, - ErrorCode: item.ErrorCode, - ErrorMessage: item.ErrorMessage, - Attachments: item.Attachments, - ParentPublicID: item.ParentPublicID, - SourcePublicID: item.SourcePublicID, - MyFeedback: item.MyFeedback, - ThumbsUpCount: item.ThumbsUpCount, - ThumbsDownCount: item.ThumbsDownCount, - EditedAt: item.EditedAt, - CreatedAt: item.CreatedAt, - UpdatedAt: item.UpdatedAt, + ID: item.ID, + ConversationID: item.ConversationID, + UserID: item.UserID, + PublicID: item.PublicID, + ParentMessageID: item.ParentMessageID, + RunID: item.RunID, + Role: item.Role, + ContentType: item.ContentType, + Content: item.Content, + ReasoningContent: item.ReasoningContent, + BranchReason: item.BranchReason, + SourceMessageID: item.SourceMessageID, + TokenUsage: item.TokenUsage, + InputTokens: item.InputTokens, + OutputTokens: item.OutputTokens, + CacheReadTokens: item.CacheReadTokens, + CacheWriteTokens: item.CacheWriteTokens, + ReasoningTokens: item.ReasoningTokens, + LatencyMS: item.LatencyMS, + BilledCurrency: item.BilledCurrency, + BilledNanousd: item.BilledNanousd, + PricingSnapshot: item.PricingSnapshot, + Status: item.Status, + ErrorCode: item.ErrorCode, + ErrorMessage: item.ErrorMessage, + ModerationEventID: item.ModerationEventID, + ModerationCategoriesJSON: item.ModerationCategoriesJSON, + Attachments: item.Attachments, + ParentPublicID: item.ParentPublicID, + SourcePublicID: item.SourcePublicID, + MyFeedback: item.MyFeedback, + ThumbsUpCount: item.ThumbsUpCount, + ThumbsDownCount: item.ThumbsDownCount, + EditedAt: item.EditedAt, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, } } @@ -3623,31 +3692,33 @@ func toMessageModel(item *domainconversation.Message) models.Message { return models.Message{} } return models.Message{ - ConversationID: item.ConversationID, - UserID: item.UserID, - PublicID: item.PublicID, - ParentMessageID: item.ParentMessageID, - RunID: item.RunID, - Role: item.Role, - ContentType: item.ContentType, - Content: item.Content, - ReasoningContent: item.ReasoningContent, - BranchReason: item.BranchReason, - SourceMessageID: item.SourceMessageID, - TokenUsage: item.TokenUsage, - InputTokens: item.InputTokens, - OutputTokens: item.OutputTokens, - CacheReadTokens: item.CacheReadTokens, - CacheWriteTokens: item.CacheWriteTokens, - ReasoningTokens: item.ReasoningTokens, - LatencyMS: item.LatencyMS, - BilledCurrency: item.BilledCurrency, - BilledNanousd: item.BilledNanousd, - PricingSnapshot: item.PricingSnapshot, - Status: item.Status, - ErrorCode: item.ErrorCode, - ErrorMessage: item.ErrorMessage, - EditedAt: item.EditedAt, + ConversationID: item.ConversationID, + UserID: item.UserID, + PublicID: item.PublicID, + ParentMessageID: item.ParentMessageID, + RunID: item.RunID, + Role: item.Role, + ContentType: item.ContentType, + Content: item.Content, + ReasoningContent: item.ReasoningContent, + BranchReason: item.BranchReason, + SourceMessageID: item.SourceMessageID, + TokenUsage: item.TokenUsage, + InputTokens: item.InputTokens, + OutputTokens: item.OutputTokens, + CacheReadTokens: item.CacheReadTokens, + CacheWriteTokens: item.CacheWriteTokens, + ReasoningTokens: item.ReasoningTokens, + LatencyMS: item.LatencyMS, + BilledCurrency: item.BilledCurrency, + BilledNanousd: item.BilledNanousd, + PricingSnapshot: item.PricingSnapshot, + Status: item.Status, + ErrorCode: item.ErrorCode, + ErrorMessage: item.ErrorMessage, + ModerationEventID: item.ModerationEventID, + ModerationCategoriesJSON: item.ModerationCategoriesJSON, + EditedAt: item.EditedAt, } } @@ -3686,39 +3757,42 @@ func toAttachmentModel(item *domainconversation.Attachment) models.Attachment { func toConversationRunDomain(item models.ConversationRun) domainconversation.Run { return domainconversation.Run{ - ID: item.ID, - RunID: item.RunID, - RequestID: item.RequestID, - UserID: item.UserID, - ConversationID: item.ConversationID, - TaskType: item.TaskType, - Endpoint: item.Endpoint, - Provider: item.Provider, - ProviderProtocol: item.ProviderProtocol, - UpstreamID: item.UpstreamID, - UpstreamModelID: item.UpstreamModelID, - UpstreamName: item.UpstreamName, - RequestedModelName: item.RequestedModelName, - PlatformModelName: item.PlatformModelName, - RoutedBindingCode: item.RoutedBindingCode, - ModelVendor: item.ModelVendor, - ModelIcon: item.ModelIcon, - UpstreamModelName: item.UpstreamModelName, - InputTokens: item.InputTokens, - OutputTokens: item.OutputTokens, - CacheReadTokens: item.CacheReadTokens, - CacheWriteTokens: item.CacheWriteTokens, - ReasoningTokens: item.ReasoningTokens, - ToolCallsCount: item.ToolCallsCount, - FirstTokenLatencyMS: item.FirstTokenLatencyMS, - TotalLatencyMS: item.TotalLatencyMS, - Status: item.Status, - ErrorCode: item.ErrorCode, - ErrorMessage: item.ErrorMessage, - StartedAt: item.StartedAt, - EndedAt: item.EndedAt, - CreatedAt: item.CreatedAt, - UpdatedAt: item.UpdatedAt, + ID: item.ID, + RunID: item.RunID, + RequestID: item.RequestID, + UserID: item.UserID, + ConversationID: item.ConversationID, + TaskType: item.TaskType, + Endpoint: item.Endpoint, + Provider: item.Provider, + ProviderProtocol: item.ProviderProtocol, + UpstreamID: item.UpstreamID, + UpstreamModelID: item.UpstreamModelID, + UpstreamName: item.UpstreamName, + RequestedModelName: item.RequestedModelName, + PlatformModelName: item.PlatformModelName, + RoutedBindingCode: item.RoutedBindingCode, + ModelVendor: item.ModelVendor, + ModelIcon: item.ModelIcon, + UpstreamModelName: item.UpstreamModelName, + InputTokens: item.InputTokens, + OutputTokens: item.OutputTokens, + CacheReadTokens: item.CacheReadTokens, + CacheWriteTokens: item.CacheWriteTokens, + ReasoningTokens: item.ReasoningTokens, + ToolCallsCount: item.ToolCallsCount, + FirstTokenLatencyMS: item.FirstTokenLatencyMS, + TotalLatencyMS: item.TotalLatencyMS, + Status: item.Status, + ErrorCode: item.ErrorCode, + ErrorMessage: item.ErrorMessage, + ModerationState: item.ModerationState, + ModerationEventID: item.ModerationEventID, + ModerationCategoriesJSON: item.ModerationCategoriesJSON, + StartedAt: item.StartedAt, + EndedAt: item.EndedAt, + CreatedAt: item.CreatedAt, + UpdatedAt: item.UpdatedAt, } } @@ -3774,37 +3848,54 @@ func toConversationRunModel(item *domainconversation.Run) models.ConversationRun return models.ConversationRun{} } return models.ConversationRun{ - RunID: item.RunID, - RequestID: item.RequestID, - UserID: item.UserID, - ConversationID: item.ConversationID, - TaskType: item.TaskType, - Endpoint: item.Endpoint, - Provider: item.Provider, - ProviderProtocol: item.ProviderProtocol, - UpstreamID: item.UpstreamID, - UpstreamModelID: item.UpstreamModelID, - UpstreamName: item.UpstreamName, - RequestedModelName: item.RequestedModelName, - PlatformModelName: item.PlatformModelName, - RoutedBindingCode: item.RoutedBindingCode, - ModelVendor: item.ModelVendor, - ModelIcon: item.ModelIcon, - UpstreamModelName: item.UpstreamModelName, - InputTokens: item.InputTokens, - OutputTokens: item.OutputTokens, - CacheReadTokens: item.CacheReadTokens, - CacheWriteTokens: item.CacheWriteTokens, - ReasoningTokens: item.ReasoningTokens, - ToolCallsCount: item.ToolCallsCount, - FirstTokenLatencyMS: item.FirstTokenLatencyMS, - TotalLatencyMS: item.TotalLatencyMS, - Status: item.Status, - ErrorCode: item.ErrorCode, - ErrorMessage: item.ErrorMessage, - StartedAt: item.StartedAt, - EndedAt: item.EndedAt, + RunID: item.RunID, + RequestID: item.RequestID, + UserID: item.UserID, + ConversationID: item.ConversationID, + TaskType: item.TaskType, + Endpoint: item.Endpoint, + Provider: item.Provider, + ProviderProtocol: item.ProviderProtocol, + UpstreamID: item.UpstreamID, + UpstreamModelID: item.UpstreamModelID, + UpstreamName: item.UpstreamName, + RequestedModelName: item.RequestedModelName, + PlatformModelName: item.PlatformModelName, + RoutedBindingCode: item.RoutedBindingCode, + ModelVendor: item.ModelVendor, + ModelIcon: item.ModelIcon, + UpstreamModelName: item.UpstreamModelName, + InputTokens: item.InputTokens, + OutputTokens: item.OutputTokens, + CacheReadTokens: item.CacheReadTokens, + CacheWriteTokens: item.CacheWriteTokens, + ReasoningTokens: item.ReasoningTokens, + ToolCallsCount: item.ToolCallsCount, + FirstTokenLatencyMS: item.FirstTokenLatencyMS, + TotalLatencyMS: item.TotalLatencyMS, + Status: item.Status, + ErrorCode: item.ErrorCode, + ErrorMessage: item.ErrorMessage, + ModerationState: defaultModerationState(item.ModerationState), + ModerationEventID: item.ModerationEventID, + ModerationCategoriesJSON: defaultJSONArray(item.ModerationCategoriesJSON), + StartedAt: item.StartedAt, + EndedAt: item.EndedAt, + } +} + +func defaultModerationState(value string) string { + if strings.TrimSpace(value) == "" { + return "not_required" + } + return value +} + +func defaultJSONArray(value string) string { + if strings.TrimSpace(value) == "" { + return "[]" } + return value } func toConversationMessageTraceDomains(items []models.ChatRunEvent) []domainconversation.MessageTrace { diff --git a/backend/internal/infra/persistence/postgres/conversation/repository_moderation_files.go b/backend/internal/infra/persistence/postgres/conversation/repository_moderation_files.go new file mode 100644 index 000000000..ccc1ba0ca --- /dev/null +++ b/backend/internal/infra/persistence/postgres/conversation/repository_moderation_files.go @@ -0,0 +1,107 @@ +package conversation + +import ( + "context" + "strings" + + domainconversation "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/dberror" + models "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/persistence/models" + "gorm.io/gorm" +) + +// GetFileObjectByFileIDAnyStatus loads a file row regardless of ownership/status. +func (r *Repo) GetFileObjectByFileIDAnyStatus(ctx context.Context, fileID string) (*domainconversation.FileObject, error) { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return nil, nil + } + var file models.FileObject + if err := r.db.WithContext(ctx).Where("file_id = ?", fileID).First(&file).Error; err != nil { + if dberror.IsRecordNotFound(err) || err == gorm.ErrRecordNotFound { + return nil, nil + } + return nil, translateError(err) + } + item := toFileObjectDomain(file) + return &item, nil +} + +// ListModerationBlockedFileIDsForCleanup returns revoked files whose physical objects +// still need deletion. storage_path is cleared only after object-store deletion succeeds. +func (r *Repo) ListModerationBlockedFileIDsForCleanup(ctx context.Context, limit int) ([]string, error) { + if limit <= 0 { + limit = 100 + } + var fileIDs []string + err := r.db.WithContext(ctx).Model(&models.FileObject{}). + Where("status = ? AND storage_path <> ''", "moderation_blocked"). + Order("id ASC"). + Limit(limit). + Pluck("file_id", &fileIDs).Error + return fileIDs, translateError(err) +} + +// RevokeGeneratedFileForModeration marks a generated file inaccessible and unlinks user ownership. +func (r *Repo) RevokeGeneratedFileForModeration(ctx context.Context, fileID string) error { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return nil + } + err := r.db.WithContext(ctx).Model(&models.FileObject{}). + Where("file_id = ? AND status = ?", fileID, "active"). + Updates(map[string]interface{}{ + "status": "moderation_blocked", + "user_id": 0, + }).Error + if dberror.IsRecordNotFound(err) { + return nil + } + return translateError(err) +} + +// DeleteGeneratedFileArtifactsForModeration marks attachments deleted and returns storage path +// for physical deletion. storage_path is NOT cleared here — callers clear it only after a +// successful object-store delete so failed deletes remain retryable. +func (r *Repo) DeleteGeneratedFileArtifactsForModeration(ctx context.Context, fileID string) error { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return nil + } + var file models.FileObject + err := r.db.WithContext(ctx). + Where("file_id = ?", fileID). + First(&file).Error + if err != nil { + if dberror.IsRecordNotFound(err) || err == gorm.ErrRecordNotFound { + return nil + } + return translateError(err) + } + // Soft-delete attachments that still reference this file. + if err := r.db.WithContext(ctx).Model(&models.Attachment{}). + Where("file_id = ? AND status <> ?", fileID, "deleted"). + Update("status", "deleted").Error; err != nil { + return translateError(err) + } + // Keep status blocked; leave storage_path intact for retryable physical cleanup. + if err := r.db.WithContext(ctx).Model(&models.FileObject{}). + Where("id = ?", file.ID). + Updates(map[string]interface{}{ + "status": "moderation_blocked", + }).Error; err != nil { + return translateError(err) + } + return nil +} + +// ClearGeneratedFileStoragePath clears the storage path only after physical delete succeeds. +func (r *Repo) ClearGeneratedFileStoragePath(ctx context.Context, fileID string) error { + fileID = strings.TrimSpace(fileID) + if fileID == "" { + return nil + } + return translateError(r.db.WithContext(ctx).Model(&models.FileObject{}). + Where("file_id = ?", fileID). + Update("storage_path", "").Error) +} diff --git a/backend/internal/infra/persistence/schema/schema.go b/backend/internal/infra/persistence/schema/schema.go index f3c586173..7f92b971d 100644 --- a/backend/internal/infra/persistence/schema/schema.go +++ b/backend/internal/infra/persistence/schema/schema.go @@ -37,6 +37,8 @@ func Models() []interface{} { &model.FileObject{}, &model.UserStorageQuota{}, &model.ConversationRun{}, + &model.ContentModerationEvent{}, + &model.ContentModerationDailyStat{}, &model.ChatRunEvent{}, &model.ChatContextRecord{}, &model.UserMemory{}, diff --git a/backend/internal/pkg/secretbox/secretbox.go b/backend/internal/pkg/secretbox/secretbox.go index 2de99ed11..a3e2aff85 100644 --- a/backend/internal/pkg/secretbox/secretbox.go +++ b/backend/internal/pkg/secretbox/secretbox.go @@ -19,6 +19,23 @@ func EncryptString(secret string, plaintext string) (string, error) { if value == "" { return "", nil } + return Encrypt(secret, []byte(value)) +} + +// DecryptString 解密 AES-GCM 字符串。 +func DecryptString(secret string, encrypted string) (string, error) { + raw, err := Decrypt(secret, encrypted) + if err != nil { + return "", err + } + return string(raw), nil +} + +// Encrypt 使用 AES-GCM 加密任意字节,返回带 v1: 前缀的 base64 载荷。 +func Encrypt(secret string, plaintext []byte) (string, error) { + if len(plaintext) == 0 { + return "", nil + } block, err := aes.NewCipher(key(secret)) if err != nil { return "", err @@ -31,42 +48,42 @@ func EncryptString(secret string, plaintext string) (string, error) { if _, err = io.ReadFull(rand.Reader, nonce); err != nil { return "", err } - ciphertext := gcm.Seal(nil, nonce, []byte(value), nil) + ciphertext := gcm.Seal(nil, nonce, plaintext, nil) payload := append(nonce, ciphertext...) return prefix + base64.StdEncoding.EncodeToString(payload), nil } -// DecryptString 解密 AES-GCM 字符串。 -func DecryptString(secret string, encrypted string) (string, error) { +// Decrypt 解密 Encrypt 产生的载荷。 +func Decrypt(secret string, encrypted string) ([]byte, error) { value := strings.TrimSpace(encrypted) if value == "" { - return "", nil + return nil, nil } if !strings.HasPrefix(value, prefix) { - return "", errors.New("invalid encrypted payload") + return nil, errors.New("invalid encrypted payload") } raw, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(value, prefix)) if err != nil { - return "", err + return nil, err } block, err := aes.NewCipher(key(secret)) if err != nil { - return "", err + return nil, err } gcm, err := cipher.NewGCM(block) if err != nil { - return "", err + return nil, err } if len(raw) < gcm.NonceSize() { - return "", errors.New("invalid encrypted payload") + return nil, errors.New("invalid encrypted payload") } nonce := raw[:gcm.NonceSize()] ciphertext := raw[gcm.NonceSize():] plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) if err != nil { - return "", err + return nil, err } - return string(plaintext), nil + return plaintext, nil } func key(secret string) []byte { diff --git a/backend/internal/pkg/secretbox/secretbox_test.go b/backend/internal/pkg/secretbox/secretbox_test.go index 8def7c3ba..e00073eae 100644 --- a/backend/internal/pkg/secretbox/secretbox_test.go +++ b/backend/internal/pkg/secretbox/secretbox_test.go @@ -25,3 +25,26 @@ func TestDecryptStringRejectsPlaintext(t *testing.T) { t.Fatal("DecryptString accepted plaintext") } } + +func TestEncryptBytesRoundTrip(t *testing.T) { + plaintext := []byte{0x00, 0xff, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a} + encrypted, err := Encrypt("test-data-encryption-key", plaintext) + if err != nil { + t.Fatalf("Encrypt returned error: %v", err) + } + if encrypted == string(plaintext) { + t.Fatal("Encrypt returned plaintext") + } + decrypted, err := Decrypt("test-data-encryption-key", encrypted) + if err != nil { + t.Fatalf("Decrypt returned error: %v", err) + } + if len(decrypted) != len(plaintext) { + t.Fatalf("length mismatch: got %d want %d", len(decrypted), len(plaintext)) + } + for i := range plaintext { + if decrypted[i] != plaintext[i] { + t.Fatalf("byte %d: got %x want %x", i, decrypted[i], plaintext[i]) + } + } +} diff --git a/backend/internal/repository/content_moderation.go b/backend/internal/repository/content_moderation.go new file mode 100644 index 000000000..e181be883 --- /dev/null +++ b/backend/internal/repository/content_moderation.go @@ -0,0 +1,43 @@ +package repository + +import ( + "context" + "time" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" +) + +// ContentModerationRepository persists moderation events and daily stats. +type ContentModerationRepository interface { + CreateEvent(ctx context.Context, event *domaincm.Event) error + GetEventByPublicID(ctx context.Context, publicID string) (*domaincm.Event, error) + GetLatestHitEventByRunID(ctx context.Context, runID string) (*domaincm.Event, error) + ListEvents(ctx context.Context, filter domaincm.EventListFilter) ([]domaincm.Event, int64, error) + // ClearExpiredContentByPublicIDs clears payloads only for events whose isolated objects were deleted. + ClearExpiredContentByPublicIDs(ctx context.Context, publicIDs []string) (int64, error) + ListExpiredContentEvents(ctx context.Context, before time.Time, limit int) ([]domaincm.Event, error) + DeleteExpiredMetadata(ctx context.Context, before time.Time) (int64, error) + IncrementDailyStat(ctx context.Context, input DailyStatIncrement) error + ListDailyStats(ctx context.Context, from, to time.Time) ([]domaincm.DailyStat, error) + DeleteDailyStatsBefore(ctx context.Context, before time.Time) (int64, error) + UpdateRunModeration(ctx context.Context, runID string, state string, eventPublicID string, categoriesJSON string) error + // ApplyRunBlock atomically marks messages/output files blocked, clears assistant content/traces, + // and updates run state. It returns output file IDs that need physical object cleanup. + ApplyRunBlock(ctx context.Context, runID string, includeUser bool, eventPublicID string, categoriesJSON string) ([]string, error) + GetRunModerationState(ctx context.Context, runID string) (state string, err error) + ListStaleModeratingRuns(ctx context.Context, olderThan time.Time, limit int) ([]string, error) +} + +// DailyStatIncrement updates anonymous daily counters. +type DailyStatIncrement struct { + StatDate time.Time + Direction string + Modality string + Result string + Category string + CheckCount int64 + ContentItems int64 + HitCount int64 + FailureCount int64 + LatencyMS int64 +} diff --git a/backend/internal/repository/content_moderation_provider.go b/backend/internal/repository/content_moderation_provider.go new file mode 100644 index 000000000..c894d72c4 --- /dev/null +++ b/backend/internal/repository/content_moderation_provider.go @@ -0,0 +1,36 @@ +package repository + +import ( + "context" + "errors" + + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" +) + +var ( + ErrContentModerationInvalidBaseURL = errors.New("invalid content moderation base url") + ErrContentModerationTimeout = errors.New("content moderation timed out") + ErrContentModerationService = errors.New("content moderation service error") + ErrContentModerationRateLimited = errors.New("content moderation rate limited") + ErrContentModerationInvalidResp = errors.New("content moderation invalid response") + ErrContentModerationNetwork = errors.New("content moderation network error") +) + +// ContentModerationProvider is the outbound moderation port implemented by infrastructure. +type ContentModerationProvider interface { + ValidateBaseURL(raw string) error + ModerateText( + ctx context.Context, + config domaincm.ProviderConfig, + text string, + selected []string, + modality string, + ) (*domaincm.ProviderResponse, error) + ModerateImages( + ctx context.Context, + config domaincm.ProviderConfig, + images []domaincm.ProviderImage, + selected []string, + modality string, + ) (*domaincm.ProviderResponse, error) +} diff --git a/backend/internal/repository/conversation.go b/backend/internal/repository/conversation.go index 3fe3882a7..bd3b63089 100644 --- a/backend/internal/repository/conversation.go +++ b/backend/internal/repository/conversation.go @@ -11,6 +11,7 @@ type ConversationRepository interface { MessageEmbeddingRepository FileListingRepository FileLookupRepository + ModerationFileRepository FileBatchRepository UploadRepository FileEmbeddingArtifactsRepository diff --git a/backend/internal/repository/conversation_cache.go b/backend/internal/repository/conversation_cache.go index 9614358e6..db6f48010 100644 --- a/backend/internal/repository/conversation_cache.go +++ b/backend/internal/repository/conversation_cache.go @@ -53,6 +53,9 @@ type GenerationStreamCacheRepository interface { AppendGenerationStreamEvent(ctx context.Context, runID string, payloadJSON string, maxEvents int64, ttl time.Duration) (GenerationStreamMessage, error) ListGenerationStreamEvents(ctx context.Context, runID string, limit int64) ([]GenerationStreamMessage, error) ReadGenerationStreamEvents(ctx context.Context, runID string, afterID string, block time.Duration, limit int64) ([]GenerationStreamMessage, error) + // ResetGenerationStreamEvents clears retained events while keeping owner metadata so + // blocked rounds cannot be replayed with withdrawn content on reconnect. + ResetGenerationStreamEvents(ctx context.Context, runID string) error ExpireGenerationStream(ctx context.Context, runID string, ttl time.Duration) error } diff --git a/backend/internal/repository/conversation_core.go b/backend/internal/repository/conversation_core.go index d1f4e2d48..4da2bd519 100644 --- a/backend/internal/repository/conversation_core.go +++ b/backend/internal/repository/conversation_core.go @@ -118,6 +118,10 @@ type MessageFeedbackRepository interface { type ConversationTraceRepository interface { CreateAttachments(ctx context.Context, items []domainconversation.Attachment) error CreateConversationRun(ctx context.Context, item *domainconversation.Run) error + // EnsureConversationRun inserts a mid-flight run row if absent (moderation / recovery). + EnsureConversationRun(ctx context.Context, item *domainconversation.Run) error + // UpsertConversationRun creates or updates the final run snapshot by run_id. + UpsertConversationRun(ctx context.Context, item *domainconversation.Run) error UpsertConversationMessageTrace(ctx context.Context, item *domainconversation.MessageTrace) error ListConversationMessageTracesByMessageIDs(ctx context.Context, messageIDs []uint) ([]domainconversation.MessageTrace, error) UpsertConversationMessageTraceEvent(ctx context.Context, item *domainconversation.MessageTraceEventRow) error diff --git a/backend/internal/repository/conversation_file.go b/backend/internal/repository/conversation_file.go index 0fb741d7f..a1c6002a4 100644 --- a/backend/internal/repository/conversation_file.go +++ b/backend/internal/repository/conversation_file.go @@ -22,6 +22,19 @@ type FileLookupRepository interface { TouchFileObjectLastAccessedAt(ctx context.Context, userID uint, fileID string, accessedAt time.Time) error } +// ModerationFileRepository 封装内容审核清理所需的文件操作能力。 +// 与 FileLookupRepository 隔离,避免上传模块及其测试 mock 依赖审核专用方法。 +type ModerationFileRepository interface { + // RevokeGeneratedFileForModeration marks a generated file inaccessible and unlinks user ownership. + RevokeGeneratedFileForModeration(ctx context.Context, fileID string) error + // DeleteGeneratedFileArtifactsForModeration marks attachments deleted (keeps storage_path for retry). + DeleteGeneratedFileArtifactsForModeration(ctx context.Context, fileID string) error + // ClearGeneratedFileStoragePath clears storage_path after a successful physical delete. + ClearGeneratedFileStoragePath(ctx context.Context, fileID string) error + // GetFileObjectByFileIDAnyStatus loads a file regardless of status (for moderation cleanup). + GetFileObjectByFileIDAnyStatus(ctx context.Context, fileID string) (*domainconversation.FileObject, error) +} + // FileBatchRepository 封装批量读取文件能力。 type FileBatchRepository interface { GetActiveFileObjectsByIDs(ctx context.Context, userID uint, fileIDs []string) ([]domainconversation.FileObject, error) diff --git a/backend/internal/shared/response/error_code.go b/backend/internal/shared/response/error_code.go index 8d033e548..e01d31523 100644 --- a/backend/internal/shared/response/error_code.go +++ b/backend/internal/shared/response/error_code.go @@ -292,6 +292,14 @@ var exactErrorSpecs = map[string]errorSpec{ "too many refresh attempts": {Code: "rate_limit.refresh_exceeded", Message: "too many refresh attempts"}, "too many authentication attempts": {Code: "rate_limit.authentication_exceeded", Message: "too many authentication attempts"}, + "content moderation event not found": {Code: "content_moderation.event_not_found", Message: "content moderation event not found"}, + "content moderation service config and policy are required when enabled": {Code: "content_moderation.config_required", Message: "content moderation service config and policy are required when enabled"}, + "invalid content moderation config": {Code: "content_moderation.invalid_config", Message: "invalid content moderation config"}, + "invalid content moderation base url": {Code: "content_moderation.invalid_config", Message: "invalid content moderation base url"}, + "invalid content moderation model": {Code: "content_moderation.invalid_config", Message: "invalid content moderation model"}, + "content moderation probe failed": {Code: "content_moderation.probe_failed", Message: "content moderation probe failed"}, + "content blocked by moderation": {Code: "content_moderation.blocked", Message: "content blocked by moderation"}, + "deleting this identity provider would remove the only login method for some users": {Code: "identity_provider.delete_conflict", Message: "deleting this identity provider would remove the only login method for some users"}, } @@ -587,8 +595,8 @@ var fallbackMessages = map[string]string{ "llm.system_prompt_too_long": "system prompt too long", "llm.platform_model_name_required": "platform model name is required", "llm.protocol_required": "protocol is required", - "media.artifact_unavailable": "generated media artifact is temporarily unavailable", - "media.image_stream_unsupported": "upstream may not support image streaming; disable image.stream for this model", + "media.artifact_unavailable": "generated media artifact is temporarily unavailable", + "media.image_stream_unsupported": "upstream may not support image streaming; disable image.stream for this model", "billing.period_credit_exceeded": "period usage credit exceeded", "billing.invalid_subscription_tier": "invalid subscription tier", "billing.subscription_expiry_required": "subscription expiry required", diff --git a/backend/internal/transport/http/admin/handler.go b/backend/internal/transport/http/admin/handler.go index 6f6db9fdc..c188dcde4 100644 --- a/backend/internal/transport/http/admin/handler.go +++ b/backend/internal/transport/http/admin/handler.go @@ -1273,7 +1273,7 @@ func (h *Handler) ExportConversations(c *gin.Context) { failedIDs = append(failedIDs, conversations[i].ID) continue } - if err := encoder.Encode(conversationhttp.ToConversationExportResponse(result)); err != nil { + if err := encoder.Encode(conversationhttp.ToAdminConversationExportResponse(result)); err != nil { return } exported++ diff --git a/backend/internal/transport/http/contentmoderation/dto.go b/backend/internal/transport/http/contentmoderation/dto.go new file mode 100644 index 000000000..c381103bc --- /dev/null +++ b/backend/internal/transport/http/contentmoderation/dto.go @@ -0,0 +1,329 @@ +package contentmoderation + +import ( + "encoding/json" + "time" + + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" +) + +// ContentModerationPolicyRequest configures the categories enabled for each surface. +type ContentModerationPolicyRequest struct { + InputTextCategories []string `json:"inputTextCategories"` + OutputTextCategories []string `json:"outputTextCategories"` + InputImageCategories []string `json:"inputImageCategories"` + OutputImageCategories []string `json:"outputImageCategories"` +} + +// ContentModerationUpdateConfigRequest updates the moderation service and policy. +// Pointer fields preserve the difference between omission and explicit zero values. +type ContentModerationUpdateConfigRequest struct { + Enabled *bool `json:"enabled,omitempty"` + BaseURL *string `json:"baseUrl,omitempty"` + APIKey *string `json:"apiKey,omitempty"` + ClearAPIKey *bool `json:"clearAPIKey,omitempty"` + Model *string `json:"model,omitempty"` + TimeoutSeconds *int `json:"timeoutSeconds,omitempty"` + MaxConcurrency *int `json:"maxConcurrency,omitempty"` + QueueCapacity *int `json:"queueCapacity,omitempty"` + Policy *ContentModerationPolicyRequest `json:"policy,omitempty"` +} + +// ContentModerationPolicyResponse is the normalized saved policy. +type ContentModerationPolicyResponse struct { + InputTextCategories []string `json:"inputTextCategories"` + OutputTextCategories []string `json:"outputTextCategories"` + InputImageCategories []string `json:"inputImageCategories"` + OutputImageCategories []string `json:"outputImageCategories"` + Version int64 `json:"version"` +} + +// ContentModerationServiceConfigResponse is the masked moderation configuration. +type ContentModerationServiceConfigResponse struct { + Enabled bool `json:"enabled"` + BaseURL string `json:"baseUrl"` + APIKeyMasked string `json:"apiKeyMasked,omitempty"` + HasAPIKey bool `json:"hasAPIKey"` + Model string `json:"model"` + TimeoutSeconds int `json:"timeoutSeconds"` + MaxConcurrency int `json:"maxConcurrency"` + QueueCapacity int `json:"queueCapacity"` + Policy ContentModerationPolicyResponse `json:"policy"` +} + +// ContentModerationCategoryCatalogResponse lists categories supported by modality. +type ContentModerationCategoryCatalogResponse struct { + Text []string `json:"text"` + Image []string `json:"image"` +} + +// ContentModerationConfigDataResponse is the GET config response payload. +type ContentModerationConfigDataResponse struct { + Config ContentModerationServiceConfigResponse `json:"config"` + Categories ContentModerationCategoryCatalogResponse `json:"categories"` +} + +// ContentModerationConfigResponseDoc documents the standard config response envelope. +type ContentModerationConfigResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data ContentModerationConfigDataResponse `json:"data"` +} + +// ContentModerationConfigUpdateDataResponse is the PUT config response payload. +type ContentModerationConfigUpdateDataResponse struct { + Config ContentModerationServiceConfigResponse `json:"config"` +} + +// ContentModerationConfigUpdateResponseDoc documents the standard update response envelope. +type ContentModerationConfigUpdateResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data ContentModerationConfigUpdateDataResponse `json:"data"` +} + +// ContentModerationProbeResultResponse describes one probe surface. +type ContentModerationProbeResultResponse struct { + Valid bool `json:"valid"` + Model string `json:"model,omitempty"` + LatencyMS int64 `json:"latencyMS"` + Error string `json:"error,omitempty"` +} + +// ContentModerationProbeResponse is the probe response payload. +type ContentModerationProbeResponse struct { + Text ContentModerationProbeResultResponse `json:"text"` + Image ContentModerationProbeResultResponse `json:"image"` +} + +// ContentModerationProbeResponseDoc documents the standard probe response envelope. +type ContentModerationProbeResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data ContentModerationProbeResponse `json:"data"` +} + +// ContentModerationDailyStatResponse is an anonymous aggregate statistics row. +type ContentModerationDailyStatResponse struct { + StatDate time.Time `json:"statDate"` + Direction string `json:"direction"` + Modality string `json:"modality"` + Result string `json:"result"` + Category string `json:"category"` + CheckCount int64 `json:"checkCount"` + ContentItems int64 `json:"contentItems"` + HitCount int64 `json:"hitCount"` + FailureCount int64 `json:"failureCount"` + LatencySumMS int64 `json:"latencySumMS"` + LatencyCount int64 `json:"latencyCount"` +} + +// ContentModerationStatsDataResponse is the statistics response payload. +type ContentModerationStatsDataResponse struct { + Items []ContentModerationDailyStatResponse `json:"items"` +} + +// ContentModerationStatsResponseDoc documents the standard statistics response envelope. +type ContentModerationStatsResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data ContentModerationStatsDataResponse `json:"data"` +} + +// ContentModerationEventResponse is a retained event metadata row. +type ContentModerationEventResponse struct { + PublicID string `json:"publicID"` + UserID uint `json:"userID"` + UserLabel string `json:"userLabel,omitempty"` + Username string `json:"username,omitempty"` + ConversationID uint `json:"conversationID"` + RunID string `json:"runID"` + MessagePublicID string `json:"messagePublicID"` + Direction string `json:"direction"` + Modality string `json:"modality"` + Model string `json:"model"` + PolicyVersion int64 `json:"policyVersion"` + Result string `json:"result"` + Categories []string `json:"categories"` + LatencyMS int64 `json:"latencyMS"` + ErrorCode string `json:"errorCode"` + ErrorMessage string `json:"errorMessage"` + ContentSummary string `json:"contentSummary"` + CreatedAt time.Time `json:"createdAt"` +} + +// ContentModerationEventListDataResponse is the paginated event list payload. +type ContentModerationEventListDataResponse struct { + Items []ContentModerationEventResponse `json:"items"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"pageSize"` +} + +// ContentModerationEventListResponseDoc documents the standard event list response envelope. +type ContentModerationEventListResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data ContentModerationEventListDataResponse `json:"data"` +} + +// ContentModerationIsolatedImageResponse exposes review metadata without storage paths. +type ContentModerationIsolatedImageResponse struct { + Index int `json:"index"` + SHA256 string `json:"sha256"` + MimeType string `json:"mimeType"` + SizeBytes int64 `json:"sizeBytes"` + SourceFileID string `json:"sourceFileID,omitempty"` +} + +// ContentModerationEventDetailResponse is the super-admin event detail payload. +type ContentModerationEventDetailResponse struct { + Event ContentModerationEventResponse `json:"event"` + CategoryScores map[string]float64 `json:"categoryScores"` + DecryptedText string `json:"decryptedText,omitempty"` + TextAvailable bool `json:"textAvailable"` + ImagesAvailable bool `json:"imagesAvailable"` + Images []ContentModerationIsolatedImageResponse `json:"images"` +} + +// ContentModerationEventDetailResponseDoc documents the standard event detail response envelope. +type ContentModerationEventDetailResponseDoc struct { + ErrorMsg string `json:"errorMsg"` + Data ContentModerationEventDetailResponse `json:"data"` +} + +func (request ContentModerationUpdateConfigRequest) toApplicationInput() appcm.UpdateConfigInput { + input := appcm.UpdateConfigInput{ + Enabled: request.Enabled, + BaseURL: request.BaseURL, + APIKey: request.APIKey, + Model: request.Model, + TimeoutSeconds: request.TimeoutSeconds, + MaxConcurrency: request.MaxConcurrency, + QueueCapacity: request.QueueCapacity, + } + if request.ClearAPIKey != nil { + input.ClearAPIKey = *request.ClearAPIKey + } + if request.Policy != nil { + input.Policy = &appcm.Policy{ + InputTextCategories: request.Policy.InputTextCategories, + OutputTextCategories: request.Policy.OutputTextCategories, + InputImageCategories: request.Policy.InputImageCategories, + OutputImageCategories: request.Policy.OutputImageCategories, + } + } + return input +} + +func toConfigResponse(config *appcm.ServiceConfig) ContentModerationServiceConfigResponse { + if config == nil { + return ContentModerationServiceConfigResponse{} + } + return ContentModerationServiceConfigResponse{ + Enabled: config.Enabled, + BaseURL: config.BaseURL, + APIKeyMasked: config.APIKeyMasked, + HasAPIKey: config.HasAPIKey, + Model: config.Model, + TimeoutSeconds: config.TimeoutSeconds, + MaxConcurrency: config.MaxConcurrency, + QueueCapacity: config.QueueCapacity, + Policy: ContentModerationPolicyResponse{ + InputTextCategories: config.Policy.InputTextCategories, + OutputTextCategories: config.Policy.OutputTextCategories, + InputImageCategories: config.Policy.InputImageCategories, + OutputImageCategories: config.Policy.OutputImageCategories, + Version: config.Policy.Version, + }, + } +} + +func toProbeResponse(result *appcm.ProbeResponse) ContentModerationProbeResponse { + if result == nil { + return ContentModerationProbeResponse{} + } + return ContentModerationProbeResponse{ + Text: ContentModerationProbeResultResponse{ + Valid: result.Text.Valid, + Model: result.Text.Model, + LatencyMS: result.Text.Latency, + Error: result.Text.Error, + }, + Image: ContentModerationProbeResultResponse{ + Valid: result.Image.Valid, + Model: result.Image.Model, + LatencyMS: result.Image.Latency, + Error: result.Image.Error, + }, + } +} + +func toDailyStatResponse(item domaincm.DailyStat) ContentModerationDailyStatResponse { + return ContentModerationDailyStatResponse{ + StatDate: item.StatDate, + Direction: item.Direction, + Modality: item.Modality, + Result: item.Result, + Category: item.Category, + CheckCount: item.CheckCount, + ContentItems: item.ContentItems, + HitCount: item.HitCount, + FailureCount: item.FailureCount, + LatencySumMS: item.LatencySumMS, + LatencyCount: item.LatencyCount, + } +} + +func toEventResponse(item domaincm.Event, label string, username string) ContentModerationEventResponse { + categories := make([]string, 0) + _ = json.Unmarshal([]byte(item.CategoriesJSON), &categories) + return ContentModerationEventResponse{ + PublicID: item.PublicID, + UserID: item.UserID, + UserLabel: label, + Username: username, + ConversationID: item.ConversationID, + RunID: item.RunID, + MessagePublicID: item.MessagePublicID, + Direction: item.Direction, + Modality: item.Modality, + Model: item.Model, + PolicyVersion: item.PolicyVersion, + Result: item.Result, + Categories: categories, + LatencyMS: item.LatencyMS, + ErrorCode: item.ErrorCode, + ErrorMessage: item.ErrorMessage, + ContentSummary: item.ContentSummary, + CreatedAt: item.CreatedAt, + } +} + +func toEventDetailResponse( + detail *appcm.EventDetail, + userLabel string, + username string, +) ContentModerationEventDetailResponse { + if detail == nil { + return ContentModerationEventDetailResponse{} + } + categoryScores := detail.CategoryScores + if categoryScores == nil { + categoryScores = map[string]float64{} + } + images := make([]ContentModerationIsolatedImageResponse, 0, len(detail.Images)) + for _, image := range detail.Images { + images = append(images, ContentModerationIsolatedImageResponse{ + Index: image.Index, + SHA256: image.SHA256, + MimeType: image.MimeType, + SizeBytes: image.SizeBytes, + SourceFileID: image.SourceFileID, + }) + } + return ContentModerationEventDetailResponse{ + Event: toEventResponse(detail.Event, userLabel, username), + CategoryScores: categoryScores, + DecryptedText: detail.DecryptedText, + TextAvailable: detail.TextAvailable, + ImagesAvailable: detail.ImagesAvailable, + Images: images, + } +} diff --git a/backend/internal/transport/http/contentmoderation/dto_test.go b/backend/internal/transport/http/contentmoderation/dto_test.go new file mode 100644 index 000000000..cd65dd3e7 --- /dev/null +++ b/backend/internal/transport/http/contentmoderation/dto_test.go @@ -0,0 +1,90 @@ +package contentmoderation + +import ( + "encoding/json" + "strings" + "testing" + + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" + domaincm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/contentmoderation" +) + +func TestServiceConfigResponseJSONContract(t *testing.T) { + response := toConfigResponse(&appcm.ServiceConfig{ + Enabled: true, + BaseURL: "https://api.openai.com/v1", + APIKeyMasked: "sk-a...mnop", + HasAPIKey: true, + Model: "omni-moderation-latest", + TimeoutSeconds: 10, + MaxConcurrency: 4, + QueueCapacity: 256, + Policy: appcm.Policy{ + InputTextCategories: []string{"hate"}, + Version: 2, + }, + }) + raw, err := json.Marshal(response) + if err != nil { + t.Fatal(err) + } + text := string(raw) + for _, field := range []string{`"enabled":true`, `"baseUrl"`, `"apiKeyMasked"`, `"inputTextCategories"`, `"version":2`} { + if !strings.Contains(text, field) { + t.Fatalf("config JSON missing %s: %s", field, text) + } + } + if strings.Contains(text, `"policyVersion"`) || strings.Contains(text, `"BaseURL"`) || strings.Contains(text, `"APIKey"`) { + t.Fatalf("config JSON contains an internal or duplicate field: %s", text) + } +} + +func TestUpdateConfigRequestMapsToApplicationInput(t *testing.T) { + var request ContentModerationUpdateConfigRequest + if err := json.Unmarshal([]byte(`{"enabled":false,"baseUrl":"https://example.com/v1","clearAPIKey":true,"policy":{"inputTextCategories":["hate"]}}`), &request); err != nil { + t.Fatal(err) + } + input := request.toApplicationInput() + if input.Enabled == nil || *input.Enabled { + t.Fatalf("expected explicit false enabled value, got %#v", input.Enabled) + } + if input.BaseURL == nil || *input.BaseURL != "https://example.com/v1" || !input.ClearAPIKey { + t.Fatalf("unexpected config input: %#v", input) + } + if input.Policy == nil || len(input.Policy.InputTextCategories) != 1 { + t.Fatalf("unexpected policy input: %#v", input.Policy) + } +} + +func TestEventDetailResponseHidesInternalFields(t *testing.T) { + response := toEventDetailResponse(&appcm.EventDetail{ + Event: domaincm.Event{ + PublicID: "evt-1", + EncryptedText: "secret-ciphertext", + }, + DecryptedText: "reviewable text", + Images: []domaincm.IsolatedImageMeta{{ + Index: 0, + StoragePath: "moderation/private/object", + }}, + }, "User 1", "user1") + raw, err := json.Marshal(response) + if err != nil { + t.Fatal(err) + } + text := string(raw) + for _, forbidden := range []string{"secret-ciphertext", "EncryptedText", "storagePath", "moderation/private/object"} { + if strings.Contains(text, forbidden) { + t.Fatalf("event detail leaked %q: %s", forbidden, text) + } + } + if !strings.Contains(text, `"decryptedText":"reviewable text"`) { + t.Fatalf("event detail contract changed: %s", text) + } + if response.Event.Categories == nil || response.CategoryScores == nil || response.Images == nil { + t.Fatalf("required collections must not serialize as null: %#v", response) + } + if strings.Contains(text, `"categoriesJSON"`) || strings.Count(text, `"userLabel"`) > 1 || strings.Count(text, `"username"`) > 1 { + t.Fatalf("event detail contains storage-shaped or duplicate fields: %s", text) + } +} diff --git a/backend/internal/transport/http/contentmoderation/handler.go b/backend/internal/transport/http/contentmoderation/handler.go new file mode 100644 index 000000000..df5e88133 --- /dev/null +++ b/backend/internal/transport/http/contentmoderation/handler.go @@ -0,0 +1,359 @@ +package contentmoderation + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + appadmin "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/admin" + appcm "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/application/contentmoderation" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/response" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/middleware" + "github.com/gin-gonic/gin" +) + +type userLabelResolver interface { + ResolveUserLabels(ctx context.Context, userIDs []uint) map[uint]appadmin.UserLabel +} + +// Handler exposes admin content-moderation APIs. +type Handler struct { + service *appcm.Service + userLabelResolver userLabelResolver +} + +// NewHandler creates the HTTP handler. +func NewHandler(service *appcm.Service) *Handler { + return &Handler{service: service} +} + +// SetUserLabelResolver injects batch user-label resolution for event lists/details. +func (h *Handler) SetUserLabelResolver(resolver userLabelResolver) { + h.userLabelResolver = resolver +} + +func (h *Handler) resolveUserLabels(ctx context.Context, userIDs []uint) map[uint]appadmin.UserLabel { + if h.userLabelResolver == nil { + return map[uint]appadmin.UserLabel{} + } + return h.userLabelResolver.ResolveUserLabels(ctx, userIDs) +} + +func parseOptionalRFC3339(c *gin.Context, key string) (*time.Time, bool) { + raw := strings.TrimSpace(c.Query(key)) + if raw == "" { + return nil, true + } + parsed, err := time.Parse(time.RFC3339, raw) + if err != nil { + response.Error(c, http.StatusBadRequest, "invalid "+key) + return nil, false + } + return &parsed, true +} + +func parsePagination(c *gin.Context) (page int, pageSize int, ok bool) { + page = 1 + pageSize = 20 + if raw := strings.TrimSpace(c.Query("page")); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 { + response.Error(c, http.StatusBadRequest, "invalid page") + return 0, 0, false + } + page = parsed + } + if raw := strings.TrimSpace(c.Query("pageSize")); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 || parsed > 100 { + response.Error(c, http.StatusBadRequest, "invalid pageSize") + return 0, 0, false + } + pageSize = parsed + } + return page, pageSize, true +} + +// GetConfig godoc +// @Summary Get content moderation config +// @Tags admin-content-moderation +// @Produce json +// @Security BearerAuth +// @Success 200 {object} ContentModerationConfigResponseDoc +// @Router /admin/content-moderation/config [get] +func (h *Handler) GetConfig(c *gin.Context) { + cfg, err := h.service.GetConfig(c.Request.Context(), middleware.MustUserRole(c)) + if err != nil { + writeError(c, err) + return + } + categories := appcm.CategoryCatalog() + response.Success(c, ContentModerationConfigDataResponse{ + Config: toConfigResponse(cfg), + Categories: ContentModerationCategoryCatalogResponse{ + Text: categories["text"], + Image: categories["image"], + }, + }) +} + +// UpdateConfig godoc +// @Summary Update content moderation config +// @Tags admin-content-moderation +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param body body ContentModerationUpdateConfigRequest true "Content moderation configuration" +// @Success 200 {object} ContentModerationConfigUpdateResponseDoc +// @Router /admin/content-moderation/config [put] +func (h *Handler) UpdateConfig(c *gin.Context) { + var req ContentModerationUpdateConfigRequest + if err := c.ShouldBindJSON(&req); err != nil { + response.Error(c, http.StatusBadRequest, "invalid request body") + return + } + cfg, err := h.service.UpdateConfig(c.Request.Context(), middleware.MustUserRole(c), req.toApplicationInput()) + if err != nil { + writeError(c, err) + return + } + response.Success(c, ContentModerationConfigUpdateDataResponse{Config: toConfigResponse(cfg)}) +} + +// Probe godoc +// @Summary Probe content moderation service +// @Tags admin-content-moderation +// @Produce json +// @Security BearerAuth +// @Success 200 {object} ContentModerationProbeResponseDoc +// @Router /admin/content-moderation/probe [post] +func (h *Handler) Probe(c *gin.Context) { + result, err := h.service.Probe(c.Request.Context(), middleware.MustUserRole(c)) + if err != nil { + writeError(c, err) + return + } + response.Success(c, toProbeResponse(result)) +} + +// GetStats godoc +// @Summary Get content moderation daily stats +// @Tags admin-content-moderation +// @Produce json +// @Security BearerAuth +// @Param from query string false "Start time (RFC3339)" +// @Param to query string false "End time (RFC3339)" +// @Success 200 {object} ContentModerationStatsResponseDoc +// @Router /admin/content-moderation/stats [get] +func (h *Handler) GetStats(c *gin.Context) { + from, ok := parseOptionalRFC3339(c, "from") + if !ok { + return + } + to, ok := parseOptionalRFC3339(c, "to") + if !ok { + return + } + filter := appcm.StatsFilter{From: from, To: to} + items, err := h.service.GetStats(c.Request.Context(), middleware.MustUserRole(c), filter) + if err != nil { + writeError(c, err) + return + } + out := make([]ContentModerationDailyStatResponse, 0, len(items)) + for _, item := range items { + out = append(out, toDailyStatResponse(item)) + } + response.Success(c, ContentModerationStatsDataResponse{Items: out}) +} + +// parseOptionalUserID parses the optional userId query parameter. +// Empty means no filter (UserID 0). Invalid values yield 400 and ok=false. +func parseOptionalUserID(c *gin.Context) (uint, bool) { + raw := strings.TrimSpace(c.Query("userId")) + if raw == "" { + return 0, true + } + + // Limit bit size to the platform's uint width to avoid truncation on 32-bit. + parsed, err := strconv.ParseUint(raw, 10, strconv.IntSize) + if err != nil || parsed == 0 { + response.Error(c, http.StatusBadRequest, "invalid userId") + return 0, false + } + + return uint(parsed), true +} + +// ListEvents godoc +// @Summary List content moderation events +// @Tags admin-content-moderation +// @Produce json +// @Security BearerAuth +// @Param page query int false "Page number" +// @Param pageSize query int false "Page size" +// @Param result query string false "Result filter" +// @Param direction query string false "Direction filter" +// @Param modality query string false "Modality filter" +// @Param category query string false "Category filter" +// @Param userId query int false "User ID" +// @Param runId query string false "Run ID" +// @Param from query string false "Start time (RFC3339)" +// @Param to query string false "End time (RFC3339)" +// @Success 200 {object} ContentModerationEventListResponseDoc +// @Router /admin/content-moderation/events [get] +func (h *Handler) ListEvents(c *gin.Context) { + page, pageSize, ok := parsePagination(c) + if !ok { + return + } + userID, ok := parseOptionalUserID(c) + if !ok { + return + } + from, ok := parseOptionalRFC3339(c, "from") + if !ok { + return + } + to, ok := parseOptionalRFC3339(c, "to") + if !ok { + return + } + input := appcm.EventListInput{ + Direction: c.Query("direction"), + Modality: c.Query("modality"), + Result: c.Query("result"), + Category: c.Query("category"), + UserID: userID, + RunID: c.Query("runId"), + From: from, + To: to, + Page: page, + PageSize: pageSize, + } + items, total, err := h.service.ListEvents(c.Request.Context(), middleware.MustUserRole(c), input) + if err != nil { + writeError(c, err) + return + } + userIDs := make([]uint, 0, len(items)) + for _, item := range items { + userIDs = append(userIDs, item.UserID) + } + userLabels := h.resolveUserLabels(c.Request.Context(), userIDs) + out := make([]ContentModerationEventResponse, 0, len(items)) + for _, item := range items { + label := userLabels[item.UserID] + out = append(out, toEventResponse(item, label.Label, label.Username)) + } + response.Success(c, ContentModerationEventListDataResponse{ + Items: out, + Total: total, + Page: page, + PageSize: pageSize, + }) +} + +// GetEvent godoc +// @Summary Get content moderation event detail +// @Tags admin-content-moderation +// @Produce json +// @Security BearerAuth +// @Param eventID path string true "Moderation event ID" +// @Success 200 {object} ContentModerationEventDetailResponseDoc +// @Router /admin/content-moderation/events/{eventID} [get] +func (h *Handler) GetEvent(c *gin.Context) { + detail, err := h.service.GetEventDetail( + c.Request.Context(), + middleware.MustUserRole(c), + c.Param("eventID"), + ) + if err != nil { + writeError(c, err) + return + } + label := appadmin.UserLabel{} + if detail != nil { + label = h.resolveUserLabels(c.Request.Context(), []uint{detail.Event.UserID})[detail.Event.UserID] + h.service.RecordReviewAudit(c.Request.Context(), appcm.ReviewAuditInput{ + ActorUserID: middleware.MustUserID(c), + RequestID: middleware.MustRequestID(c), + Action: "content_moderation.event.view", + EventID: detail.Event.PublicID, + ClientIP: c.ClientIP(), + UserAgent: c.Request.UserAgent(), + Detail: map[string]bool{"retainedTextAvailable": detail.TextAvailable}, + }) + } + c.Header("Cache-Control", "no-store") + response.Success(c, toEventDetailResponse(detail, label.Label, label.Username)) +} + +// GetEventImage godoc +// @Summary Stream a isolated moderation image +// @Tags admin-content-moderation +// @Produce octet-stream +// @Security BearerAuth +// @Param eventID path string true "Moderation event ID" +// @Param index path int true "Image index" +// @Success 200 {file} binary +// @Router /admin/content-moderation/events/{eventID}/images/{index} [get] +func (h *Handler) GetEventImage(c *gin.Context) { + index, err := strconv.Atoi(c.Param("index")) + if err != nil || index < 0 { + response.Error(c, http.StatusBadRequest, "invalid image index") + return + } + data, mimeType, err := h.service.OpenEventImage( + c.Request.Context(), + middleware.MustUserRole(c), + c.Param("eventID"), + index, + ) + if err != nil { + writeError(c, err) + return + } + h.service.RecordReviewAudit(c.Request.Context(), appcm.ReviewAuditInput{ + ActorUserID: middleware.MustUserID(c), + RequestID: middleware.MustRequestID(c), + Action: "content_moderation.event_image.view", + EventID: c.Param("eventID"), + ClientIP: c.ClientIP(), + UserAgent: c.Request.UserAgent(), + Detail: map[string]int{"imageIndex": index}, + }) + c.Header("Cache-Control", "no-store") + c.Data(http.StatusOK, mimeType, data) +} + +func writeError(c *gin.Context, err error) { + switch { + case errors.Is(err, appcm.ErrSuperAdminRequired): + response.Error(c, http.StatusForbidden, "superadmin permission required") + case errors.Is(err, appcm.ErrAdminRequired): + response.Error(c, http.StatusForbidden, "admin permission required") + case errors.Is(err, appcm.ErrEventNotFound): + response.Error(c, http.StatusNotFound, "content moderation event not found") + case errors.Is(err, appcm.ErrServiceConfigRequired): + response.ErrorWithCode(c, http.StatusBadRequest, "content_moderation.config_required", err.Error()) + case errors.Is(err, appcm.ErrInvalidBaseURL), + errors.Is(err, appcm.ErrInvalidModel), + errors.Is(err, appcm.ErrInvalidTimeout), + errors.Is(err, appcm.ErrInvalidConcurrency), + errors.Is(err, appcm.ErrInvalidQueueCapacity), + errors.Is(err, appcm.ErrInvalidCategories), + errors.Is(err, appcm.ErrImageTextOnlyCategory), + errors.Is(err, appcm.ErrInvalidConfig): + response.ErrorWithCode(c, http.StatusBadRequest, "content_moderation.invalid_config", err.Error()) + case errors.Is(err, appcm.ErrProbeFailed): + response.ErrorWithCode(c, http.StatusBadRequest, "content_moderation.probe_failed", err.Error()) + case errors.Is(err, appcm.ErrInvalidEventFilter): + response.ErrorWithCode(c, http.StatusBadRequest, response.CodeRequestInvalidQuery, err.Error()) + default: + response.Error(c, http.StatusInternalServerError, "internal server error") + } +} diff --git a/backend/internal/transport/http/contentmoderation/handler_test.go b/backend/internal/transport/http/contentmoderation/handler_test.go new file mode 100644 index 000000000..65a38a68a --- /dev/null +++ b/backend/internal/transport/http/contentmoderation/handler_test.go @@ -0,0 +1,120 @@ +package contentmoderation + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" +) + +func TestParseOptionalUserID(t *testing.T) { + gin.SetMode(gin.TestMode) + + t.Run("missing userId returns zero without error", func(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/events", nil) + + userID, ok := parseOptionalUserID(c) + if !ok { + t.Fatal("expected ok=true for missing userId") + } + if userID != 0 { + t.Fatalf("expected UserID 0, got %d", userID) + } + }) + + t.Run("blank userId returns zero without error", func(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/events?userId=%20", nil) + + userID, ok := parseOptionalUserID(c) + if !ok { + t.Fatal("expected ok=true for blank userId") + } + if userID != 0 { + t.Fatalf("expected UserID 0, got %d", userID) + } + }) + + t.Run("valid userId is parsed", func(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/events?userId=42", nil) + + userID, ok := parseOptionalUserID(c) + if !ok { + t.Fatal("expected ok=true for valid userId") + } + if userID != 42 { + t.Fatalf("expected UserID 42, got %d", userID) + } + }) + + invalidCases := []struct { + name string + query string + }{ + {name: "non-numeric", query: "userId=abc"}, + {name: "negative", query: "userId=-1"}, + {name: "zero", query: "userId=0"}, + // Larger than any platform uint (exceeds ParseUint bit-size limit). + {name: "overflow bit width", query: "userId=18446744073709551616"}, + } + for _, tc := range invalidCases { + t.Run(tc.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/events?"+tc.query, nil) + + userID, ok := parseOptionalUserID(c) + if ok { + t.Fatalf("expected ok=false for %q, got userID=%d", tc.query, userID) + } + if userID != 0 { + t.Fatalf("expected UserID 0 on error, got %d", userID) + } + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected status 400, got %d", recorder.Code) + } + }) + } +} + +func TestParsePaginationRejectsInvalidValues(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, query := range []string{"page=0", "page=abc", "pageSize=0", "pageSize=101", "pageSize=abc"} { + t.Run(query, func(t *testing.T) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/events?"+query, nil) + if _, _, ok := parsePagination(c); ok { + t.Fatalf("expected %q to be rejected", query) + } + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", recorder.Code) + } + }) + } +} + +func TestParseOptionalRFC3339(t *testing.T) { + gin.SetMode(gin.TestMode) + valid := "2026-08-10T01:02:03Z" + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/stats?from="+valid, nil) + parsed, ok := parseOptionalRFC3339(c, "from") + if !ok || parsed == nil || !parsed.Equal(time.Date(2026, 8, 10, 1, 2, 3, 0, time.UTC)) { + t.Fatalf("unexpected parsed time: %v, ok=%v", parsed, ok) + } + + recorder := httptest.NewRecorder() + c, _ = gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/stats?from=not-a-time", nil) + if parsed, ok = parseOptionalRFC3339(c, "from"); ok || parsed != nil { + t.Fatalf("expected invalid time to be rejected: %v, ok=%v", parsed, ok) + } + if recorder.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", recorder.Code) + } +} diff --git a/backend/internal/transport/http/contentmoderation/module.go b/backend/internal/transport/http/contentmoderation/module.go new file mode 100644 index 000000000..7e01030b2 --- /dev/null +++ b/backend/internal/transport/http/contentmoderation/module.go @@ -0,0 +1,28 @@ +package contentmoderation + +import "github.com/gin-gonic/gin" + +// Module registers content moderation admin routes. +type Module struct { + Handler *Handler +} + +// NewModule creates the module. +func NewModule(handler *Handler) *Module { + return &Module{Handler: handler} +} + +// RegisterRoutes registers routes under the admin group. +func (m *Module) RegisterRoutes(adminGroup *gin.RouterGroup) { + if m == nil || m.Handler == nil { + return + } + group := adminGroup.Group("/content-moderation") + group.GET("/config", m.Handler.GetConfig) + group.PUT("/config", m.Handler.UpdateConfig) + group.POST("/probe", m.Handler.Probe) + group.GET("/stats", m.Handler.GetStats) + group.GET("/events", m.Handler.ListEvents) + group.GET("/events/:eventID", m.Handler.GetEvent) + group.GET("/events/:eventID/images/:index", m.Handler.GetEventImage) +} diff --git a/backend/internal/transport/http/conversation/dto_response.go b/backend/internal/transport/http/conversation/dto_response.go index d9ae6960d..ff1fb76a1 100644 --- a/backend/internal/transport/http/conversation/dto_response.go +++ b/backend/internal/transport/http/conversation/dto_response.go @@ -177,6 +177,7 @@ func ToConversationExportResponse(item *appconversation.ConversationExportResult } messages := make([]MessageResponse, 0, len(item.Messages)) for _, message := range item.Messages { + // User-owned archive: keep recoverable user content; assistant blocked body stays empty from DB. messages = append(messages, toMessageResponseWithRunAndFallback(message, runModels[strings.TrimSpace(message.RunID)], fallbackModel)) } @@ -197,6 +198,31 @@ func ToConversationExportResponse(item *appconversation.ConversationExportResult } } +// ToAdminConversationExportResponse redacts blocked originals for administrator bulk export. +func ToAdminConversationExportResponse(item *appconversation.ConversationExportResult) ConversationExportResponse { + resp := ToConversationExportResponse(item) + if item == nil { + return resp + } + runModels := make(map[string]model.Run, len(item.Runs)) + for _, run := range item.Runs { + if runID := strings.TrimSpace(run.RunID); runID != "" { + runModels[runID] = run + } + } + fallbackModel := "" + if item.Conversation != nil { + fallbackModel = item.Conversation.Model + } + messages := make([]MessageResponse, 0, len(item.Messages)) + for _, message := range item.Messages { + messages = append(messages, toMessageResponseWithRunAndFallbackAdmin(message, runModels[strings.TrimSpace(message.RunID)], fallbackModel)) + } + resp.Messages = messages + resp.Compatibility.Notes = "Admin export redacts blocked content. Originals are only available via content moderation event APIs." + return resp +} + // ConversationProjectResponse 对外会话项目响应 DTO。 type ConversationProjectResponse struct { PublicID string `json:"publicID"` @@ -785,11 +811,20 @@ type MessageResponse struct { ThumbsDownCount int64 `json:"thumbsDownCount"` BillingCost *MessageBillingCostResponse `json:"billingCost,omitempty"` ProcessTrace *MessageProcessTraceResponse `json:"processTrace,omitempty"` + Moderation *MessageModerationResponse `json:"moderation,omitempty"` EditedAt *time.Time `json:"editedAt" extensions:"x-nullable,!x-omitempty"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` } +// MessageModerationResponse exposes soft-moderation state to clients. +type MessageModerationResponse struct { + State string `json:"state,omitempty"` + Direction string `json:"direction,omitempty"` + EventID string `json:"eventID,omitempty"` + Categories []string `json:"categories,omitempty"` +} + func toTraceBlockResponse(b *model.MessageTraceBlock) *MessageTraceBlockResponse { if b == nil { return nil @@ -977,6 +1012,15 @@ func toMessageResponseWithRunAndFallback(m model.Message, run model.Run, fallbac if platformModelName == "" { platformModelName = strings.TrimSpace(fallbackModel) } + // Server-side redaction for blocked assistant content (user-facing keeps blocked user text). + content := m.Content + attachments := m.Attachments + processTrace := m.ProcessTrace + if strings.EqualFold(strings.TrimSpace(m.Status), "blocked") && m.Role == "assistant" { + content = "" + attachments = "[]" + processTrace = nil + } return MessageResponse{ ID: m.ID, ConversationID: m.ConversationID, @@ -986,7 +1030,7 @@ func toMessageResponseWithRunAndFallback(m model.Message, run model.Run, fallbac RunID: m.RunID, Role: m.Role, ContentType: m.ContentType, - Content: m.Content, + Content: content, BranchReason: m.BranchReason, SourceMessageID: m.SourceMessageID, TokenUsage: m.TokenUsage, @@ -999,7 +1043,7 @@ func toMessageResponseWithRunAndFallback(m model.Message, run model.Run, fallbac Status: m.Status, ErrorCode: m.ErrorCode, ErrorMessage: m.ErrorMessage, - Attachments: m.Attachments, + Attachments: attachments, PlatformModelName: platformModelName, UpstreamModelName: strings.TrimSpace(run.UpstreamModelName), ModelVendor: strings.TrimSpace(run.ModelVendor), @@ -1010,13 +1054,97 @@ func toMessageResponseWithRunAndFallback(m model.Message, run model.Run, fallbac ThumbsUpCount: m.ThumbsUpCount, ThumbsDownCount: m.ThumbsDownCount, BillingCost: toMessageBillingCostResponse(m), - ProcessTrace: toMessageProcessTraceResponse(m.ProcessTrace), + ProcessTrace: toMessageProcessTraceResponse(processTrace), + Moderation: toMessageModerationResponse(m, run), EditedAt: m.EditedAt, CreatedAt: m.CreatedAt, UpdatedAt: m.UpdatedAt, } } +func toMessageModerationResponse(m model.Message, run model.Run) *MessageModerationResponse { + eventID := strings.TrimSpace(m.ModerationEventID) + if eventID == "" { + eventID = strings.TrimSpace(run.ModerationEventID) + } + state := strings.TrimSpace(run.ModerationState) + if strings.EqualFold(strings.TrimSpace(m.Status), "blocked") { + state = "blocked" + } + if eventID == "" && state == "" { + return nil + } + categories := parseStringJSONArray(firstNonEmptyModerationJSON(m.ModerationCategoriesJSON, run.ModerationCategoriesJSON)) + direction := "" + if strings.EqualFold(m.Status, "blocked") && m.Role == "user" { + direction = "input" + } else if strings.EqualFold(m.Status, "blocked") { + direction = "output" + } + return &MessageModerationResponse{ + State: state, + Direction: direction, + EventID: eventID, + Categories: categories, + } +} + +func firstNonEmptyModerationJSON(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" && strings.TrimSpace(v) != "[]" { + return v + } + } + return "[]" +} + +func parseStringJSONArray(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "[]" || raw == "null" { + return nil + } + var items []string + if err := json.Unmarshal([]byte(raw), &items); err != nil { + return nil + } + return items +} + +// toMessageResponseWithRunAndFallbackAdmin redacts blocked originals for admin logs/export. +func toMessageResponseWithRunAndFallbackAdmin(m model.Message, run model.Run, fallbackModel string) MessageResponse { + resp := toMessageResponseWithRunAndFallback(m, run, fallbackModel) + if !strings.EqualFold(strings.TrimSpace(m.Status), "blocked") { + return resp + } + eventID := strings.TrimSpace(firstNonEmptyString(m.ModerationEventID, run.ModerationEventID)) + placeholder := "[blocked by content moderation" + if eventID != "" { + placeholder += "; event " + eventID + } + placeholder += "]" + if m.Role == "user" { + resp.Content = placeholder + resp.Attachments = "[]" + } else if m.Role == "assistant" { + resp.Content = "" + resp.Attachments = "[]" + resp.ProcessTrace = nil + if strings.TrimSpace(resp.ErrorMessage) == "" { + resp.ErrorMessage = placeholder + } + } + return resp +} + +func firstNonEmptyString(values ...string) string { + for _, v := range values { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + // ---------- Send Message ---------- // SendMessageResponse 发送消息响应 DTO。 diff --git a/backend/internal/transport/http/conversation/handler.go b/backend/internal/transport/http/conversation/handler.go index 026bd53cc..42ed1fcca 100644 --- a/backend/internal/transport/http/conversation/handler.go +++ b/backend/internal/transport/http/conversation/handler.go @@ -1,6 +1,7 @@ package conversation import ( + "encoding/json" "errors" "mime" "net/http" @@ -241,6 +242,45 @@ func streamErrorPayloadWithCode(code string, message string) map[string]interfac } } +// moderationBlockedStreamPayload is retained for recovery/reconnect assembly only. +// Live streams receive moderation_blocked via OnEvent after ApplyRunBlock commits. +func moderationBlockedStreamPayload(result *appconversation.SendMessageResult) map[string]interface{} { + payload := map[string]interface{}{ + "type": "moderation_blocked", + } + if result == nil { + return payload + } + if result.Moderation != nil && result.Moderation.Blocked { + payload["eventID"] = result.Moderation.EventID + payload["direction"] = result.Moderation.Direction + if len(result.Moderation.Categories) > 0 { + payload["categories"] = result.Moderation.Categories + } + return payload + } + eventID := strings.TrimSpace(result.AssistantMessage.ModerationEventID) + if eventID == "" { + eventID = strings.TrimSpace(result.UserMessage.ModerationEventID) + } + direction := "output" + if strings.EqualFold(strings.TrimSpace(result.UserMessage.Status), "blocked") { + direction = "input" + } + categoriesJSON := result.AssistantMessage.ModerationCategoriesJSON + if strings.TrimSpace(categoriesJSON) == "" || categoriesJSON == "[]" { + categoriesJSON = result.UserMessage.ModerationCategoriesJSON + } + var categories []string + _ = json.Unmarshal([]byte(categoriesJSON), &categories) + payload["eventID"] = eventID + payload["direction"] = direction + if len(categories) > 0 { + payload["categories"] = categories + } + return payload +} + func mapClientErrorMessage(err error) string { if err == nil { return "" diff --git a/backend/internal/transport/http/conversation/handler_media.go b/backend/internal/transport/http/conversation/handler_media.go index 48d7fd05b..d8f006eea 100644 --- a/backend/internal/transport/http/conversation/handler_media.go +++ b/backend/internal/transport/http/conversation/handler_media.go @@ -178,6 +178,23 @@ func (h *Handler) streamMediaTask( _ = flushStreamEvent(normalizeStreamEventPayload(eventType, payload)) return nil }) + if err == nil && result != nil && result.IsModerationBlocked() { + if !result.ModerationTerminalEmitted() { + _ = flushStreamEvent(moderationBlockedStreamPayload(result)) + } + if result.Billable { + billingCtx, billingCancel := context.WithTimeout(context.Background(), 10*time.Second) + usageLedger, billingErr := h.service.RecordSendMessageBilling(billingCtx, billingInput(result), authorization) + billingCancel() + if billingErr == nil { + appconversation.ApplyUsageBilling(&result.AssistantMessage, usageLedger) + } + } else { + _ = h.releaseSendMessageUsageAuthorization(authorization) + } + h.service.FinishMessageGeneration(clientRunID) + return + } if err != nil { if result == nil || !result.Billable { if releaseErr := h.releaseSendMessageUsageAuthorization(authorization); releaseErr != nil { diff --git a/backend/internal/transport/http/conversation/handler_message_send.go b/backend/internal/transport/http/conversation/handler_message_send.go index 35f0408ed..6268e7193 100644 --- a/backend/internal/transport/http/conversation/handler_message_send.go +++ b/backend/internal/transport/http/conversation/handler_message_send.go @@ -523,7 +523,7 @@ func (h *Handler) StreamMessage(c *gin.Context) { }) } - // 将中间事件(rag_search 等)通过 NDJSON 推送给客户端。 + // 将中间事件(含 moderation_*)通过 NDJSON 推送给客户端。 input.OnEvent = func(eventType string, payload map[string]interface{}) error { _ = flushStreamEvent(normalizeStreamEventPayload(eventType, payload)) return nil @@ -536,6 +536,22 @@ func (h *Handler) StreamMessage(c *gin.Context) { }) return nil }) + if err == nil && result != nil && result.IsModerationBlocked() { + // Guarantee a terminal event even if live OnEvent path missed emit. + if !result.ModerationTerminalEmitted() { + _ = flushStreamEvent(moderationBlockedStreamPayload(result)) + } + if result.Billable { + billingCtx, billingCancel := context.WithTimeout(context.Background(), 10*time.Second) + _ = h.recordAndApplySendMessageBilling(billingCtx, middleware.MustUserID(c), conversation, req, result, authorization) + billingCancel() + } else { + _ = h.releaseSendMessageUsageAuthorization(authorization) + } + h.service.FinishMessageGeneration(input.ClientRunID) + h.recordStreamSendMessageAuditAsync(c, conversation, req, result, "stream_message") + return + } if err != nil { if result != nil { if !result.Billable { @@ -668,7 +684,7 @@ func (h *Handler) ResumeMessageGenerationStream(c *gin.Context) { isTerminal := func(payload map[string]interface{}) bool { eventType, _ := payload["type"].(string) - return eventType == "completed" || eventType == "error" + return eventType == "completed" || eventType == "error" || eventType == "moderation_blocked" } terminalWritten := false writeEvent := func(payload map[string]interface{}) bool { diff --git a/backend/internal/transport/http/server.go b/backend/internal/transport/http/server.go index e9ad79195..2bc4cbaba 100644 --- a/backend/internal/transport/http/server.go +++ b/backend/internal/transport/http/server.go @@ -19,6 +19,7 @@ import ( authhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/auth" billinghttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/billing" channelhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/channel" + contentmoderationhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/contentmoderation" conversationhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/conversation" mcphttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/mcp" memoryhttp "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/transport/http/memory" @@ -50,21 +51,22 @@ type HealthChecker interface { // Modules 聚合可注册的业务模块。 type Modules struct { - Auth *authhttp.Module - AuthService middleware.SessionValidator - Channel *channelhttp.Module - Conversation *conversationhttp.Module - MCP *mcphttp.Module - Memory *memoryhttp.Module - Billing *billinghttp.Module - Admin *adminhttp.Module - Announcement *announcementhttp.Module - PromptPreset *promptpresethttp.Module - Skill *skillhttp.Module - Settings *settingshttp.Module - User *userhttp.Module - UserSettings *usersettingshttp.Module - StartupLog func(*zap.Logger) + Auth *authhttp.Module + AuthService middleware.SessionValidator + Channel *channelhttp.Module + Conversation *conversationhttp.Module + MCP *mcphttp.Module + Memory *memoryhttp.Module + Billing *billinghttp.Module + Admin *adminhttp.Module + ContentModeration *contentmoderationhttp.Module + Announcement *announcementhttp.Module + PromptPreset *promptpresethttp.Module + Skill *skillhttp.Module + Settings *settingshttp.Module + User *userhttp.Module + UserSettings *usersettingshttp.Module + StartupLog func(*zap.Logger) } // NewEngine 创建并注册 API 路由。 @@ -172,7 +174,7 @@ func NewEngine(cfg *config.Runtime, log *zap.Logger, modules Modules, hc HealthC if modules.User != nil { modules.User.RegisterRoutes(authRequired) } - if modules.Admin != nil || modules.Auth != nil || modules.Billing != nil || modules.Channel != nil || modules.MCP != nil || modules.Settings != nil || modules.Announcement != nil || modules.PromptPreset != nil || modules.Skill != nil { + if modules.Admin != nil || modules.Auth != nil || modules.Billing != nil || modules.Channel != nil || modules.MCP != nil || modules.Settings != nil || modules.Announcement != nil || modules.PromptPreset != nil || modules.Skill != nil || modules.ContentModeration != nil { adminGroup := authRequired.Group("/admin") adminGroup.Use(middleware.AdminOnly()) if modules.Auth != nil { @@ -181,6 +183,9 @@ func NewEngine(cfg *config.Runtime, log *zap.Logger, modules Modules, hc HealthC if modules.Admin != nil { modules.Admin.RegisterRoutes(adminGroup) } + if modules.ContentModeration != nil { + modules.ContentModeration.RegisterRoutes(adminGroup) + } if modules.Billing != nil { modules.Billing.RegisterAdminRoutes(adminGroup) } diff --git a/frontend/app/(admin)/admin/content-moderation/page.tsx b/frontend/app/(admin)/admin/content-moderation/page.tsx new file mode 100644 index 000000000..4de6953bc --- /dev/null +++ b/frontend/app/(admin)/admin/content-moderation/page.tsx @@ -0,0 +1,7 @@ +"use client"; + +import { AdminContentModeration } from "@/features/admin/components/sections/content-moderation/admin-content-moderation"; + +export default function AdminContentModerationPage() { + return ; +} diff --git a/frontend/features/admin/api/content-moderation.ts b/frontend/features/admin/api/content-moderation.ts new file mode 100644 index 000000000..4862aa438 --- /dev/null +++ b/frontend/features/admin/api/content-moderation.ts @@ -0,0 +1,103 @@ +import type { + Admin, + ContentModerationConfigDataResponse, + ContentModerationConfigUpdateDataResponse, + ContentModerationDailyStatResponse, + ContentModerationEventDetailResponse, + ContentModerationEventListDataResponse, + ContentModerationEventResponse, + ContentModerationProbeResponse, + ContentModerationServiceConfigResponse, + ContentModerationStatsDataResponse, + ContentModerationUpdateConfigRequest, +} from "@deeix/api-contract"; + +import { authedFetch, authedRequest } from "@/shared/api/authed-client"; + +export type ContentModerationConfig = ContentModerationServiceConfigResponse; +export type DailyStat = ContentModerationDailyStatResponse; +export type ModerationEvent = ContentModerationEventResponse; +export type ContentModerationEventDetail = ContentModerationEventDetailResponse; + +type ContentModerationEventListQuery = Pick< + Admin.ContentModerationEventsList.RequestQuery, + "page" | "pageSize" | "result" | "direction" +>; + +export async function getContentModerationConfig(accessToken: string) { + return authedRequest( + "/api/v1/admin/content-moderation/config", + { method: "GET", accessToken }, + true, + ); +} + +export async function updateContentModerationConfig( + accessToken: string, + payload: ContentModerationUpdateConfigRequest, +) { + return authedRequest( + "/api/v1/admin/content-moderation/config", + { method: "PUT", accessToken, body: payload }, + true, + ); +} + +export async function probeContentModeration(accessToken: string) { + return authedRequest( + "/api/v1/admin/content-moderation/probe", + { method: "POST", accessToken }, + true, + ); +} + +export async function getContentModerationStats(accessToken: string) { + return authedRequest( + "/api/v1/admin/content-moderation/stats", + { method: "GET", accessToken }, + true, + ); +} + +export async function listContentModerationEvents( + accessToken: string, + params: ContentModerationEventListQuery = {}, +) { + const query = new URLSearchParams(); + if (params.page) query.set("page", String(params.page)); + if (params.pageSize) query.set("pageSize", String(params.pageSize)); + if (params.result) query.set("result", params.result); + if (params.direction) query.set("direction", params.direction); + const suffix = query.toString() ? `?${query.toString()}` : ""; + return authedRequest( + `/api/v1/admin/content-moderation/events${suffix}`, + { method: "GET", accessToken }, + true, + ); +} + +export async function getContentModerationEvent(accessToken: string, eventID: string) { + return authedRequest( + `/api/v1/admin/content-moderation/events/${encodeURIComponent(eventID)}`, + { method: "GET", accessToken }, + true, + ); +} + +export async function fetchContentModerationEventImage( + accessToken: string, + eventID: string, + index: number, +): Promise<{ blob: Blob; mimeType: string }> { + const response = await authedFetch( + `/api/v1/admin/content-moderation/events/${encodeURIComponent(eventID)}/images/${index}`, + { + method: "GET", + accessToken, + cache: "no-store", + }, + ); + const mimeType = response.headers.get("Content-Type") || "image/png"; + const blob = await response.blob(); + return { blob, mimeType }; +} diff --git a/frontend/features/admin/components/admin-sidebar.tsx b/frontend/features/admin/components/admin-sidebar.tsx index b8e7681fc..b32258111 100644 --- a/frontend/features/admin/components/admin-sidebar.tsx +++ b/frontend/features/admin/components/admin-sidebar.tsx @@ -1,15 +1,13 @@ "use client"; -import * as React from "react"; +import { CircleArrowUp } from "lucide-react"; import Link from "next/link"; import { usePathname } from "next/navigation"; import { useTranslations } from "next-intl"; -import { CircleArrowUp } from "lucide-react"; - -import packageMeta from "@/package.json"; +import * as React from "react"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { ADMIN_SECTIONS, type AdminSection } from "@/features/admin/model/admin-sections"; import { AdminUpdateTooltipContent } from "@/features/admin/components/admin-update-tooltip-content"; +import { ADMIN_SECTIONS, type AdminSection } from "@/features/admin/model/admin-sections"; import { getCachedLatestReleaseSnapshot, getServerLatestReleaseSnapshot, @@ -17,6 +15,7 @@ import { subscribeLatestReleaseChange, } from "@/features/admin/model/update-check"; import { cn } from "@/lib/utils"; +import packageMeta from "@/package.json"; const ADMIN_SECTION_LABEL_KEYS: Record = { statistics: "sections.statistics", @@ -28,6 +27,7 @@ const ADMIN_SECTION_LABEL_KEYS: Record = { billing: "sections.billing", announcements: "sections.announcements", logs: "sections.logs", + "content-moderation": "sections.contentModeration", "login-settings": "sections.loginSettings", "conversation-settings": "sections.conversationSettings", "chat-files": "sections.chatFiles", diff --git a/frontend/features/admin/components/sections/content-moderation/admin-content-moderation.tsx b/frontend/features/admin/components/sections/content-moderation/admin-content-moderation.tsx new file mode 100644 index 000000000..3499c1d9d --- /dev/null +++ b/frontend/features/admin/components/sections/content-moderation/admin-content-moderation.tsx @@ -0,0 +1,441 @@ +"use client"; + +import { Save } from "lucide-react"; +import { AnimatePresence, motion } from "motion/react"; +import { useTranslations } from "next-intl"; +import * as React from "react"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { SpinnerLabel } from "@/components/ui/spinner"; +import { + type ContentModerationConfig, + getContentModerationConfig, + probeContentModeration, + updateContentModerationConfig, +} from "@/features/admin/api/content-moderation"; +import { resolveAdminErrorMessage } from "@/features/admin/utils/admin-error"; +import { cn } from "@/lib/utils"; +import { useAuthSession } from "@/shared/auth/auth-session-context"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { + SettingsFieldInset, + SettingsFieldItem, + SettingsFieldList, + SettingsFieldRow, + SettingsPage, + SettingsSection, + SettingsSectionSeparator, +} from "@/shared/components/settings-layout"; + +import { + type ServiceRuntimeState, + SettingsFieldEditor, + type SettingsFieldServiceRuntime, +} from "../shared/settings-runtime-panel"; +import { ModerationCategorySelector } from "./moderation-category-selector"; + +type ServiceConfigField = "baseUrl" | "model" | "timeoutSeconds" | "maxConcurrency" | "queueCapacity"; +type ServiceDraftField = ServiceConfigField | "apiKey"; + +export function AdminContentModeration() { + const t = useTranslations("adminContentModeration"); + const { user } = useAuthSession(); + const isSuperAdmin = user?.role === "superadmin"; + const [loading, setLoading] = React.useState(true); + const [saving, setSaving] = React.useState(false); + const [probing, setProbing] = React.useState(false); + const [config, setConfig] = React.useState(null); + const [savedConfig, setSavedConfig] = React.useState(null); + const [textCategories, setTextCategories] = React.useState([]); + const [imageCategories, setImageCategories] = React.useState([]); + const [apiKeyDraft, setApiKeyDraft] = React.useState(""); + const [probeRuntime, setProbeRuntime] = React.useState(null); + + const load = React.useCallback(async () => { + setLoading(true); + try { + const token = await resolveAccessToken(); + if (!token) return; + if (!isSuperAdmin) return; + const cfgRes = await getContentModerationConfig(token); + setConfig(cfgRes.config); + setSavedConfig(cfgRes.config); + setTextCategories(cfgRes.categories.text); + setImageCategories(cfgRes.categories.image); + } catch (error) { + toast.error(t("loadFailed"), { description: resolveAdminErrorMessage(error) }); + } finally { + setLoading(false); + } + }, [isSuperAdmin, t]); + + React.useEffect(() => { + void load(); + }, [load]); + + const serviceDirty = React.useMemo( + () => Boolean( + config && + savedConfig && + (config.enabled !== savedConfig.enabled || + config.baseUrl !== savedConfig.baseUrl || + config.model !== savedConfig.model || + config.timeoutSeconds !== savedConfig.timeoutSeconds || + config.maxConcurrency !== savedConfig.maxConcurrency || + config.queueCapacity !== savedConfig.queueCapacity || + apiKeyDraft.trim()), + ), + [apiKeyDraft, config, savedConfig], + ); + const policyDirty = React.useMemo( + () => Boolean(config && savedConfig && JSON.stringify(config.policy) !== JSON.stringify(savedConfig.policy)), + [config, savedConfig], + ); + + function updateServiceField( + key: Key, + value: ContentModerationConfig[Key], + ) { + setConfig((current) => (current ? { ...current, [key]: value } : current)); + setProbeRuntime(null); + } + + function updateServiceDraft(key: ServiceDraftField, value: string) { + switch (key) { + case "apiKey": + setApiKeyDraft(value); + setProbeRuntime(null); + break; + case "baseUrl": + case "model": + updateServiceField(key, value); + break; + case "timeoutSeconds": + updateServiceField(key, Number(value) || 10); + break; + case "maxConcurrency": + updateServiceField(key, Number(value) || 4); + break; + case "queueCapacity": + updateServiceField(key, Number(value) || 256); + break; + } + } + + const save = async () => { + if (!config || !isSuperAdmin) return; + const hasSelectedPolicy = [ + config.policy.inputTextCategories, + config.policy.inputImageCategories, + config.policy.outputTextCategories, + config.policy.outputImageCategories, + ].some((categories) => categories.length > 0); + if (config.enabled && !hasSelectedPolicy) { + toast.error(t("saveFailed"), { description: t("validation.policyRequired") }); + return; + } + if (config.enabled && !config.hasAPIKey && !apiKeyDraft.trim()) { + toast.error(t("saveFailed"), { description: t("validation.apiKeyRequired") }); + return; + } + setSaving(true); + try { + const token = await resolveAccessToken(); + if (!token) return; + const res = await updateContentModerationConfig(token, { + enabled: config.enabled, + baseUrl: config.baseUrl, + model: config.model, + timeoutSeconds: config.timeoutSeconds, + maxConcurrency: config.maxConcurrency, + queueCapacity: config.queueCapacity, + apiKey: apiKeyDraft.trim() || undefined, + policy: { + inputTextCategories: config.policy.inputTextCategories, + inputImageCategories: config.policy.inputImageCategories, + outputTextCategories: config.policy.outputTextCategories, + outputImageCategories: config.policy.outputImageCategories, + }, + }); + setConfig(res.config); + setSavedConfig(res.config); + setApiKeyDraft(""); + setProbeRuntime(null); + toast.success(t("saved")); + } catch (error) { + toast.error(t("saveFailed"), { description: resolveAdminErrorMessage(error) }); + } finally { + setSaving(false); + } + }; + + const probe = async () => { + if (!isSuperAdmin) return; + setProbing(true); + try { + const token = await resolveAccessToken(); + if (!token) return; + const res = await probeContentModeration(token); + const valid = res.text.valid && res.image.valid; + setProbeRuntime({ + status: valid ? "available" : "unhealthy", + reachable: valid, + message: [res.text.error, res.image.error].filter(Boolean).join(" · ") || undefined, + details: [ + { + label: t("probeText"), + value: `${res.text.valid ? t("valid") : t("invalid")} · ${res.text.latencyMS}ms`, + }, + { + label: t("probeImage"), + value: `${res.image.valid ? t("valid") : t("invalid")} · ${res.image.latencyMS}ms`, + }, + ], + }); + } catch (error) { + const message = resolveAdminErrorMessage(error); + setProbeRuntime({ status: "failed", reachable: false, message }); + toast.error(t("probeFailed"), { description: message }); + } finally { + setProbing(false); + } + }; + + if (loading) { + return ( + + +

{t("loading")}

+
+
+ ); + } + + if (!isSuperAdmin) { + return ( + + +

{t("superAdminOnly")}

+
+
+ ); + } + + if (!config || !savedConfig) { + return ( + + +

{t("loadFailed")}

+
+
+ ); + } + + const renderSaveAction = (visible: boolean) => + visible ? ( + + ) : null; + const serviceSaveAction = renderSaveAction(serviceDirty); + const policySaveAction = renderSaveAction(config.enabled && policyDirty); + const serviceFields = [ + { + key: "baseUrl", + type: "string", + value: config.baseUrl, + savedValue: savedConfig.baseUrl, + placeholder: undefined, + }, + { + key: "apiKey", + type: "password", + value: apiKeyDraft, + savedValue: "", + placeholder: t("fields.apiKeyPlaceholder"), + }, + { + key: "model", + type: "string", + value: config.model, + savedValue: savedConfig.model, + placeholder: undefined, + }, + { + key: "timeoutSeconds", + type: "int", + value: String(config.timeoutSeconds), + savedValue: String(savedConfig.timeoutSeconds), + placeholder: undefined, + }, + { + key: "maxConcurrency", + type: "int", + value: String(config.maxConcurrency), + savedValue: String(savedConfig.maxConcurrency), + placeholder: undefined, + }, + { + key: "queueCapacity", + type: "int", + value: String(config.queueCapacity), + savedValue: String(savedConfig.queueCapacity), + placeholder: undefined, + }, + ] as const; + const serviceConfigured = savedConfig.enabled && savedConfig.hasAPIKey; + const serviceRuntimeState: ServiceRuntimeState | null = serviceConfigured + ? probeRuntime + : { status: "unconfigured", reachable: false }; + const serviceRuntimeField: SettingsFieldServiceRuntime = { + runtime: serviceRuntimeState, + loading: probing, + actionDisabled: saving || probing || serviceDirty || !serviceConfigured, + pendingAction: probing ? "test" : "", + actions: [ + { + key: "test", + label: t("probe"), + icon: "bugplay", + action: "test", + spinWhen: "test", + }, + ], + onAction: (action) => { + if (action === "test") void probe(); + }, + }; + + return ( + + + + + { + setConfig((current) => (current ? { ...current, enabled: value === "true" } : current)); + setProbeRuntime(null); + }} + /> + + + {config.enabled ? ( + +
+ + + + {serviceFields.map((field) => ( + updateServiceDraft(field.key, value)} + /> + ))} + + + +
+
+ ) : null} +
+
+
+ + + + + + {( + [ + ["inputText", "inputTextCategories", textCategories], + ["inputImage", "inputImageCategories", imageCategories], + ["outputText", "outputTextCategories", textCategories], + ["outputImage", "outputImageCategories", imageCategories], + ] as const + ).map(([labelKey, field, options], index) => { + const selectedValue = config.policy[field] ?? []; + const selectedCount = options.filter((category) => + selectedValue.includes(category), + ).length; + return ( + + + + setConfig((current) => + current + ? { + ...current, + policy: { ...current.policy, [field]: next }, + } + : current, + ) + } + /> + + + ); + })} + + +
+ ); +} diff --git a/frontend/features/admin/components/sections/content-moderation/moderation-category-selector.tsx b/frontend/features/admin/components/sections/content-moderation/moderation-category-selector.tsx new file mode 100644 index 000000000..99efd83e4 --- /dev/null +++ b/frontend/features/admin/components/sections/content-moderation/moderation-category-selector.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { ChevronDownIcon } from "lucide-react"; + +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/utils"; + +type ModerationCategorySelectorProps = { + options: string[]; + value: string[]; + selectAllLabel: string; + emptyLabel: string; + selectedLabel: string; + disabled?: boolean; + disabledHint?: string; + onChange: (next: string[]) => void; +}; + +export function ModerationCategorySelector({ + options, + value, + selectAllLabel, + emptyLabel, + selectedLabel, + disabled, + disabledHint, + onChange, +}: ModerationCategorySelectorProps) { + const selected = new Set(value); + const selectedOptionCount = options.filter((category) => selected.has(category)).length; + const allSelected = options.length > 0 && selectedOptionCount === options.length; + + function updateCategory(category: string, checked: boolean) { + const next = new Set(selected); + if (checked) { + next.add(category); + } else { + next.delete(category); + } + onChange(Array.from(next).sort()); + } + + const trigger = ( + + + + ); + + return ( + + {disabled && disabledHint ? ( + + + {trigger} + + + {disabledHint} + + + ) : ( + trigger + )} + + 0 ? "indeterminate" : false} + className="pr-8 pl-2 font-medium [&>span:first-child]:right-2 [&>span:first-child]:left-auto" + onSelect={(event) => event.preventDefault()} + onCheckedChange={(checked) => + onChange(checked === true ? Array.from(new Set(options)).sort() : []) + } + > + {selectAllLabel} + + + {options.map((category) => ( + event.preventDefault()} + onCheckedChange={(checked) => updateCategory(category, checked === true)} + > + {category} + + ))} + + + ); +} diff --git a/frontend/features/admin/components/sections/logs/admin-logs.tsx b/frontend/features/admin/components/sections/logs/admin-logs.tsx index ccf39e7f8..66b0ad700 100644 --- a/frontend/features/admin/components/sections/logs/admin-logs.tsx +++ b/frontend/features/admin/components/sections/logs/admin-logs.tsx @@ -99,6 +99,8 @@ import { } from "@/shared/lib/billing-display"; import { ModelSelect, type ModelSelectOption } from "@/shared/components/model-select"; import { formatBytes } from "@/shared/lib/file-display"; +import { ModerationEventTable } from "@/features/admin/components/sections/logs/admin-moderation-events"; +import { useAuthSession } from "@/shared/auth/auth-session-context"; type LogDetail = | { kind: "audit"; item: AdminAuditLogDTO } @@ -2036,6 +2038,8 @@ function LogCleanupDialog({ export function AdminLogsPage() { const t = useTranslations("adminLogs"); + const { user } = useAuthSession(); + const isSuperAdmin = user?.role === "superadmin"; const [detail, setDetail] = React.useState(null); const [conversationDetailLoading, setConversationDetailLoading] = React.useState(false); const detailRequestRef = React.useRef(0); @@ -2140,6 +2144,7 @@ export function AdminLogsPage() { {t("tabs.auth")} {t("tabs.orders")} {t("tabs.conversation")} + {isSuperAdmin ? {t("tabs.moderation")} : null} setDetail({ kind: "audit", item })} /> @@ -2160,6 +2165,11 @@ export function AdminLogsPage() { void openConversationDetail(item)} /> + {isSuperAdmin ? ( + + + + ) : null} 0 ? String(fallbackID) : "-"); +} + +function DetailRow({ label, value, mono = false }: { label: string; value: React.ReactNode; mono?: boolean }) { + return ( +
+

{label}

+
+ {value ?? "-"} +
+
+ ); +} + +function DetailBlock({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+

{title}

+
{children}
+
+ ); +} + +function ModerationEventDetailSheet({ + eventID, + open, + onClose, +}: { + eventID: string; + open: boolean; + onClose: () => void; +}) { + const t = useTranslations("adminLogs.moderation"); + const locale = useLocale(); + const [loading, setLoading] = React.useState(false); + const [detail, setDetail] = React.useState(null); + const [images, setImages] = React.useState>([]); + const requestRef = React.useRef(0); + const imagesRef = React.useRef>([]); + const snap = useDialogSnapshot(open ? { eventID } : null); + + const revokeImages = React.useCallback((items: Array<{ index: number; url: string }>) => { + for (const item of items) URL.revokeObjectURL(item.url); + }, []); + + const replaceImages = React.useCallback( + (items: Array<{ index: number; url: string }>) => { + revokeImages(imagesRef.current); + imagesRef.current = items; + setImages(items); + }, + [revokeImages], + ); + + React.useEffect(() => { + if (!open || !eventID) return; + const requestID = ++requestRef.current; + setLoading(true); + setDetail(null); + replaceImages([]); + void (async () => { + try { + const token = await resolveAccessToken(); + if (!token) return; + const loaded = await getContentModerationEvent(token, eventID); + if (requestRef.current !== requestID) return; + setDetail(loaded); + if (loaded.imagesAvailable && Array.isArray(loaded.images) && loaded.images.length > 0) { + const loadedImages: Array<{ index: number; url: string }> = []; + for (const image of loaded.images) { + try { + const { blob } = await fetchContentModerationEventImage(token, eventID, image.index); + const url = URL.createObjectURL(blob); + if (requestRef.current !== requestID) { + URL.revokeObjectURL(url); + revokeImages(loadedImages); + return; + } + loadedImages.push({ index: image.index, url }); + } catch { + if (requestRef.current !== requestID) { + revokeImages(loadedImages); + return; + } + toast.error(t("imageLoadFailed")); + } + } + if (requestRef.current === requestID) replaceImages(loadedImages); + else revokeImages(loadedImages); + } + } catch (error) { + if (requestRef.current === requestID) { + toast.error(t("detailFailed"), { description: resolveAdminErrorMessage(error) }); + onClose(); + } + } finally { + if (requestRef.current === requestID) setLoading(false); + } + })(); + return () => { + requestRef.current += 1; + }; + }, [eventID, onClose, open, replaceImages, revokeImages, t]); + + React.useEffect(() => { + return () => { + requestRef.current += 1; + revokeImages(imagesRef.current); + imagesRef.current = []; + }; + }, [revokeImages]); + + const event = detail?.event; + const description = snap + ? `${event?.result || eventID} · ${formatDateTime(event?.createdAt, locale)}` + : ""; + + return ( + !next && onClose()}> + + + {t("detailTitle")} + {description} + +
+ {loading ? ( +
+ +
+ ) : detail && event ? ( + <> + + + + + + + + + + + + + + {detail.textAvailable && detail.decryptedText ? ( +
+

{t("detailText")}

+
+                    {detail.decryptedText}
+                  
+
+ ) : null} + + {images.length > 0 ? ( +
+

{t("detailImages")}

+
+ {images.map((image) => ( + {t("detailImageAlt", + ))} +
+
+ ) : null} + +
+

JSON

+
+                  
+                    {JSON.stringify(
+                      {
+                        event: detail.event,
+                        categories: detail.event.categories,
+                        categoryScores: detail.categoryScores,
+                        textAvailable: detail.textAvailable,
+                        imagesAvailable: detail.imagesAvailable,
+                        images: detail.images,
+                      },
+                      null,
+                      2,
+                    )}
+                  
+                
+
+ + ) : null} +
+
+
+ ); +} + +export function ModerationEventTable() { + const locale = useLocale(); + const t = useTranslations("adminLogs.moderation"); + const [loading, setLoading] = React.useState(true); + const [items, setItems] = React.useState([]); + const [total, setTotal] = React.useState(0); + const [page, setPage] = React.useState(1); + const [pageSize, setPageSize] = React.useState(20); + const [query, setQuery] = React.useState(""); + const [resultFilter, setResultFilter] = React.useState(""); + const [directionFilter, setDirectionFilter] = React.useState(""); + const [selectedEventID, setSelectedEventID] = React.useState(""); + + const load = React.useCallback(async () => { + setLoading(true); + try { + const token = await resolveAccessToken(); + if (!token) return; + const res = await listContentModerationEvents(token, { + page, + pageSize, + result: resultFilter.trim() || undefined, + direction: directionFilter.trim() || undefined, + }); + setItems(res.items ?? []); + setTotal(res.total ?? 0); + } catch (error) { + toast.error(t("loadFailed"), { description: resolveAdminErrorMessage(error) }); + } finally { + setLoading(false); + } + }, [directionFilter, page, pageSize, resultFilter, t]); + + React.useEffect(() => { + void load(); + }, [load]); + + const filteredItems = React.useMemo(() => { + const q = query.trim().toLowerCase(); + if (!q) return items; + return items.filter((item) => { + const haystack = [ + item.publicID, + item.userLabel, + item.username, + String(item.userID || ""), + item.result, + item.direction, + item.modality, + item.model, + item.runID, + item.messagePublicID, + item.errorCode, + item.contentSummary, + ] + .join(" ") + .toLowerCase(); + return haystack.includes(q); + }); + }, [items, query]); + + const pageCount = Math.max(1, Math.ceil(total / Math.max(1, pageSize))); + + return ( +
+ { + setResultFilter(value); + setPage(1); + }, + options: [ + { label: t("filters.all"), value: "" }, + { label: t("results.passed"), value: "passed" }, + { label: t("results.hit"), value: "hit" }, + { label: t("results.failedOpen"), value: "failed_open" }, + ], + }, + { + key: "direction", + label: t("columns.direction"), + value: directionFilter, + onValueChange: (value) => { + setDirectionFilter(value); + setPage(1); + }, + options: [ + { label: t("filters.all"), value: "" }, + { label: t("directions.input"), value: "input" }, + { label: t("directions.output"), value: "output" }, + ], + }, + ]} + loading={loading} + onRefresh={() => void load()} + /> + + + + + {t("columns.eventId")} + {t("columns.user")} + {t("columns.result")} + {t("columns.direction")} + {t("columns.modality")} + {t("columns.latency")} + {t("columns.createdAt")} + + + + {loading && filteredItems.length === 0 ? : null} + {!loading && filteredItems.length === 0 ? {t("empty")} : null} + {filteredItems.map((item) => ( + setSelectedEventID(item.publicID)} + > + {item.publicID} + + {resolveUserDisplayName(item.userLabel, item.username, item.userID)} + + {item.result} + {item.direction} + {item.modality} + {item.latencyMS}ms + {formatDateTime(item.createdAt, locale)} + + ))} + +
+ + { + setPageSize(next); + setPage(1); + }} + /> + + setSelectedEventID("")} + /> +
+ ); +} diff --git a/frontend/features/admin/components/sections/shared/settings-runtime-panel.tsx b/frontend/features/admin/components/sections/shared/settings-runtime-panel.tsx index 10eb97bdc..520f4e5b2 100644 --- a/frontend/features/admin/components/sections/shared/settings-runtime-panel.tsx +++ b/frontend/features/admin/components/sections/shared/settings-runtime-panel.tsx @@ -1,9 +1,9 @@ "use client"; -import * as React from "react"; import { BugPlay, Pause, Play, RefreshCw } from "lucide-react"; import { motion } from "motion/react"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -145,10 +145,11 @@ const RUNTIME_ICON_MAP = { } as const; function resolveRuntimeStatusKey(status?: string): ServiceRuntimeStatusKey { - return status === "running" ? "running" : "stopped"; + return status === "running" || status === "available" ? "running" : "stopped"; } type RuntimeStatusLabels = { + available: string; running: string; restarting: string; created: string; @@ -163,6 +164,8 @@ type RuntimeStatusLabels = { function resolveRuntimeStatusLabel(status: string | undefined, labels: RuntimeStatusLabels): string { switch (status) { + case "available": + return labels.available; case "running": return labels.running; case "restarting": @@ -192,6 +195,7 @@ function resolveRuntimeStatusBadgeTone(runtime?: ServiceRuntimeState | null): Se return "neutral"; } switch (runtime.status) { + case "available": case "running": return runtime.reachable ? "success" : "warning"; case "restarting": @@ -222,6 +226,7 @@ function resolveRuntimeMessageToneClassName({ return "text-muted-foreground"; } switch (runtime?.status) { + case "available": case "running": return "text-emerald-700"; case "unhealthy": @@ -378,6 +383,7 @@ export function SettingsFieldEditor({ const t = useTranslations("common"); const runtimeStatusLabels = React.useMemo( () => ({ + available: t("runtime.status.available"), running: t("runtime.status.running"), restarting: t("runtime.status.restarting"), created: t("runtime.status.created"), @@ -589,6 +595,8 @@ export function SettingsFieldEditor({ type="button" variant="secondary" size="icon" + aria-label={action.label} + title={action.label} className="size-8 shrink-0 rounded-md shadow-none transition-transform active:scale-90" disabled={disabled || action.disabled} onClick={action.onClick} @@ -623,6 +631,7 @@ export function ServiceRuntimePanel({ const t = useTranslations("common"); const runtimeStatusLabels = React.useMemo( () => ({ + available: t("runtime.status.available"), running: t("runtime.status.running"), restarting: t("runtime.status.restarting"), created: t("runtime.status.created"), diff --git a/frontend/features/admin/components/sections/statistics/admin-moderation-statistics.tsx b/frontend/features/admin/components/sections/statistics/admin-moderation-statistics.tsx new file mode 100644 index 000000000..d02be3f4c --- /dev/null +++ b/frontend/features/admin/components/sections/statistics/admin-moderation-statistics.tsx @@ -0,0 +1,427 @@ +"use client"; + +import * as React from "react"; +import { Activity, AlertTriangle, CheckCircle2, RefreshCw, ShieldAlert, Timer } from "lucide-react"; +import { useLocale, useTranslations } from "next-intl"; +import { Area, CartesianGrid, ComposedChart, Line, XAxis, YAxis } from "recharts"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { + ChartContainer, + ChartInteractiveLegend, + ChartTooltip, + type ChartConfig, + type ChartInteractiveLegendItem, +} from "@/components/ui/chart"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { type DailyStat, getContentModerationStats } from "@/features/admin/api/content-moderation"; +import { cn } from "@/lib/utils"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; + +type TrendMetric = "checks" | "hits" | "failures"; + +type DayAggregate = { + date: string; + label: string; + fullLabel: string; + checks: number; + hits: number; + failures: number; + latencySumMS: number; + latencyCount: number; +}; + +function MetricCard({ + label, + value, + icon, + loading, +}: { + label: string; + value: string; + icon: React.ReactNode; + loading: boolean; +}) { + return ( +
+
+ {icon} + {label} +
+ {loading ? ( + + ) : ( +

{value}

+ )} +
+ ); +} + +function compactNumber(value: number, locale: string): string { + if (!Number.isFinite(value) || value === 0) return "0"; + return new Intl.NumberFormat(locale, { + notation: "compact", + maximumFractionDigits: 1, + }).format(value); +} + +function formatLatency(value: number, locale: string): string { + if (!Number.isFinite(value) || value <= 0) return "0"; + if (value < 1000) return `${Math.round(value).toLocaleString(locale)}ms`; + return `${(value / 1000).toLocaleString(locale, { maximumFractionDigits: 2 })}s`; +} + +function formatPercent(value: number, locale: string): string { + if (!Number.isFinite(value) || value <= 0) return "0%"; + return `${value.toLocaleString(locale, { maximumFractionDigits: 1 })}%`; +} + +function parseDateKey(value: string): string { + return String(value ?? "").slice(0, 10); +} + +function aggregateDailyStats(items: DailyStat[], locale: string): DayAggregate[] { + const byDate = new Map(); + for (const item of items) { + // Category rows are breakdown counters. Their hitCount is in addition to the + // category="" summary row, so including them would double-count hits and can + // produce a hit rate above 100%. + if (item.category?.trim()) continue; + const date = parseDateKey(item.statDate); + if (!date) continue; + const current = byDate.get(date) ?? { + date, + label: date, + fullLabel: date, + checks: 0, + hits: 0, + failures: 0, + latencySumMS: 0, + latencyCount: 0, + }; + current.checks += item.checkCount || 0; + current.hits += item.hitCount || 0; + current.failures += item.failureCount || 0; + current.latencySumMS += item.latencySumMS || 0; + current.latencyCount += item.latencyCount || 0; + byDate.set(date, current); + } + + const sorted = Array.from(byDate.values()).sort((a, b) => a.date.localeCompare(b.date)); + return sorted.map((item) => { + const date = new Date(`${item.date}T00:00:00`); + const valid = !Number.isNaN(date.getTime()); + return { + ...item, + label: valid + ? new Intl.DateTimeFormat(locale, { month: "2-digit", day: "2-digit" }).format(date) + : item.date, + fullLabel: valid + ? new Intl.DateTimeFormat(locale, { year: "numeric", month: "2-digit", day: "2-digit" }).format(date) + : item.date, + }; + }); +} + +function ModerationTooltipContent({ + active, + payload, +}: { + active?: boolean; + payload?: Array<{ payload?: DayAggregate & { avgLatencyMS?: number } }>; +}) { + const t = useTranslations("adminStatistics.moderation"); + const locale = useLocale(); + const item = payload?.[0]?.payload; + if (!active || !item) return null; + return ( +
+

{item.fullLabel}

+
+
+ {t("metrics.checks")} + + {new Intl.NumberFormat(locale).format(item.checks)} + +
+
+ {t("metrics.hits")} + + {new Intl.NumberFormat(locale).format(item.hits)} + +
+
+ {t("metrics.failures")} + + {new Intl.NumberFormat(locale).format(item.failures)} + +
+
+ {t("metrics.latency")} + + {formatLatency(item.avgLatencyMS ?? 0, locale)} + +
+
+
+ ); +} + +export function AdminModerationStatisticsSection() { + const t = useTranslations("adminStatistics.moderation"); + const tRoot = useTranslations("adminStatistics"); + const locale = useLocale(); + const [loading, setLoading] = React.useState(true); + const [items, setItems] = React.useState([]); + const [trendMetric, setTrendMetric] = React.useState("checks"); + const [hiddenSeries, setHiddenSeries] = React.useState>(() => new Set()); + + const load = React.useCallback(async () => { + setLoading(true); + try { + const token = await resolveAccessToken(); + if (!token) return; + const res = await getContentModerationStats(token); + setItems(res.items ?? []); + } catch { + toast.error(t("loadFailed")); + } finally { + setLoading(false); + } + }, [t]); + + React.useEffect(() => { + void load(); + }, [load]); + + const days = React.useMemo(() => aggregateDailyStats(items, locale), [items, locale]); + const totals = React.useMemo(() => { + return days.reduce( + (acc, day) => { + acc.checks += day.checks; + acc.hits += day.hits; + acc.failures += day.failures; + acc.latencySumMS += day.latencySumMS; + acc.latencyCount += day.latencyCount; + return acc; + }, + { checks: 0, hits: 0, failures: 0, latencySumMS: 0, latencyCount: 0 }, + ); + }, [days]); + + const hitRate = totals.checks > 0 ? (totals.hits / totals.checks) * 100 : 0; + const avgLatencyMS = totals.latencyCount > 0 ? totals.latencySumMS / totals.latencyCount : 0; + + const chartData = React.useMemo( + () => + days.map((day) => ({ + ...day, + metricValue: day[trendMetric], + avgLatencyMS: day.latencyCount > 0 ? day.latencySumMS / day.latencyCount : 0, + })), + [days, trendMetric], + ); + + const chartConfig = React.useMemo( + () => ({ + metricValue: { + label: t(`metrics.${trendMetric}`), + color: "var(--chart-1)", + }, + avgLatencyMS: { + label: t("metrics.latency"), + color: "var(--chart-2)", + }, + }), + [t, trendMetric], + ); + + const legendItems = React.useMemo( + () => [ + { id: "metricValue", label: t(`metrics.${trendMetric}`), color: "var(--chart-1)" }, + { id: "avgLatencyMS", label: t("metrics.latency"), color: "var(--chart-2)" }, + ], + [t, trendMetric], + ); + + const hasData = days.some((day) => day.checks > 0 || day.hits > 0 || day.failures > 0); + + const toggleSeries = React.useCallback((id: string) => { + setHiddenSeries((current) => { + const next = new Set(current); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }, []); + + return ( +
+
+

{t("title")}

+ +
+ +
+ } + loading={loading} + /> + } + loading={loading} + /> + } + loading={loading} + /> + } + loading={loading} + /> +
+ +
+
+

{t("trendTitle")}

+ setTrendMetric(value as TrendMetric)}> + + {(["checks", "hits", "failures"] as const).map((metric) => ( + + {t(`metrics.${metric}`)} + + ))} + + +
+
+ {loading ? ( +
+ {Array.from({ length: 10 }).map((_, index) => ( + + ))} +
+ ) : !hasData ? ( +
+ {t("empty")} +
+ ) : ( +
+ + event.preventDefault()} + > + + + + + + + + + compactNumber(value, locale)} + /> + formatLatency(value, locale)} + /> + } + /> + + + + + +
+ )} +
+ {!loading && hasData ? ( +
+ + + {t("avgLatencyLabel")}: {formatLatency(avgLatencyMS, locale)} + +
+ ) : null} +
+
+ ); +} diff --git a/frontend/features/admin/components/sections/statistics/admin-statistics.tsx b/frontend/features/admin/components/sections/statistics/admin-statistics.tsx index f427edac3..32ecd5bed 100644 --- a/frontend/features/admin/components/sections/statistics/admin-statistics.tsx +++ b/frontend/features/admin/components/sections/statistics/admin-statistics.tsx @@ -27,6 +27,7 @@ import { StatisticsTrendChart, StatisticsUserRankingChart, } from "./admin-statistics-charts"; +import { AdminModerationStatisticsSection } from "./admin-moderation-statistics"; const ALL_MODELS_VALUE = "__all_models__"; @@ -384,6 +385,10 @@ export function AdminStatisticsPage() { /> + + + + ); } diff --git a/frontend/features/admin/model/admin-sections.ts b/frontend/features/admin/model/admin-sections.ts index dd8a1457b..c321a7f4d 100644 --- a/frontend/features/admin/model/admin-sections.ts +++ b/frontend/features/admin/model/admin-sections.ts @@ -8,6 +8,7 @@ export const ADMIN_SECTIONS = [ { id: "billing", label: "Billing", href: "/billing" }, { id: "announcements", label: "Announcements", href: "/announcements" }, { id: "logs", label: "Logs", href: "/logs" }, + { id: "content-moderation", label: "Content moderation", href: "/content-moderation" }, { id: "login-settings", label: "Login & auth", href: "/login" }, { id: "conversation-settings", label: "Conversation", href: "/conversation" }, { id: "chat-files", label: "Files & retrieval", href: "/chat-files" }, diff --git a/frontend/features/chat/hooks/use-chat-branch-state.ts b/frontend/features/chat/hooks/use-chat-branch-state.ts index 26d451b13..965708ac2 100644 --- a/frontend/features/chat/hooks/use-chat-branch-state.ts +++ b/frontend/features/chat/hooks/use-chat-branch-state.ts @@ -222,6 +222,7 @@ export function useChatBranchState({ liveRunIDs?: ReadonlySet; }) { const t = useTranslations("chat.messages"); + const submitT = useTranslations("chat.submit"); const resolveErrorMessage = useLocalizedErrorMessage(); const [branchSelections, setBranchSelections] = React.useState>({}); @@ -238,13 +239,15 @@ export function useChatBranchState({ generationInterrupted: t("generationInterrupted"), streamInterrupted: t("streamInterrupted"), imageRunning: t("imageRunning"), + moderationBlocked: submitT("moderationBlocked"), + moderationBlockedDescription: submitT("moderationBlockedDescription"), resolveErrorMessage: (errorCode: string, fallback: string, details?: UpstreamDebugInfo) => resolveErrorMessage(new ApiError(fallback, 502, details, errorCode), fallback), }, { liveRunIDs }, ), ), - [liveRunIDs, messages, resolveErrorMessage, t], + [liveRunIDs, messages, resolveErrorMessage, submitT, t], ); const serverMessagePublicIDs = React.useMemo( () => new Set(serverTreeMessages.map((item) => item.publicID).filter(Boolean)), diff --git a/frontend/features/chat/hooks/use-chat-message-submit.ts b/frontend/features/chat/hooks/use-chat-message-submit.ts index b99d78d07..c63395ddf 100644 --- a/frontend/features/chat/hooks/use-chat-message-submit.ts +++ b/frontend/features/chat/hooks/use-chat-message-submit.ts @@ -1,19 +1,18 @@ "use client"; -import * as React from "react"; import { useTranslations } from "next-intl"; +import * as React from "react"; import { toast } from "sonner"; - -import type { ChatAreaMessage, ImageLoadingAspectRatio } from "@/features/chat/types/messages"; -import type { - ChatModelOption, - PendingAttachment, - PendingExchange, - PendingExchangeMap, -} from "@/features/chat/types/chat-runtime"; +import { useHiddenQueuedParentRuns } from "@/features/chat/hooks/use-hidden-queued-parent-runs"; import type { ChatSubmitBlockReason } from "@/features/chat/model/chat-task"; import { resolveChatSubmitDecision } from "@/features/chat/model/chat-task"; -import { useHiddenQueuedParentRuns } from "@/features/chat/hooks/use-hidden-queued-parent-runs"; +import { + buildChildrenIndex, + parseAttachments, + toBranchKey, +} from "@/features/chat/model/chat-thread"; +import { sanitizeConversationOptions } from "@/features/chat/model/conversation-options"; +import { buildMediaImagePreviewMarkdown } from "@/features/chat/model/media-image-preview"; import { resolveAssistantInputSideUsageValue, resolveDefaultSubmissionParentMessage, @@ -25,29 +24,27 @@ import { preserveRicherLiveUpstreamThinkTrace, readLiveUpstreamThinkTrace, } from "@/features/chat/model/upstream-think-store"; +import type { + ChatModelOption, + PendingAttachment, + PendingExchange, + PendingExchangeMap, +} from "@/features/chat/types/chat-runtime"; +import type { ChatAreaMessage, ImageLoadingAspectRatio } from "@/features/chat/types/messages"; import { resolveErrorDetails, resolveErrorMessage, resolveErrorSummary, } from "@/features/chat/utils/chat-runtime"; import { - buildChildrenIndex, - parseAttachments, - toBranchKey, -} from "@/features/chat/model/chat-thread"; -import { sanitizeConversationOptions } from "@/features/chat/model/conversation-options"; -import { buildMediaImagePreviewMarkdown } from "@/features/chat/model/media-image-preview"; -import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; -import { notifyResponseCompletion } from "@/shared/lib/browser-notifications"; -import { + type ConversationStreamOptions, cancelMessageGeneration, getConversation, + streamMessage as streamConversationMessage, streamImageEdit, streamImageGeneration, - streamMessage as streamConversationMessage, streamVideoGeneration, updateMessage, - type ConversationStreamOptions, } from "@/shared/api/conversation"; import type { ConversationDTO, @@ -61,6 +58,8 @@ import type { } from "@/shared/api/conversation.types"; import { ApiError } from "@/shared/api/http-client"; import type { SkillSummaryDTO } from "@/shared/api/skills.types"; +import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; +import { notifyResponseCompletion } from "@/shared/lib/browser-notifications"; const CONVERSATION_METADATA_REFRESH_MAX_WAIT_MS = 45_000; const CONVERSATION_METADATA_REFRESH_INITIAL_DELAY_MS = 800; @@ -1116,6 +1115,42 @@ export function useChatMessageSubmit({ event.reasoning_tokens > 0 ? event.reasoning_tokens : current.assistantReasoningTokens, })); }, + onModerationChecking: () => { + updatePendingExchange(exchangeKey, (current) => ({ + ...current, + assistantFileProc: true, + assistantActivityLabel: t("moderationChecking"), + })); + }, + onModerationBlocked: (event) => { + const categories = Array.isArray(event.categories) ? event.categories : []; + updatePendingExchange(exchangeKey, (current) => ({ + ...current, + assistantPending: false, + assistantStreaming: false, + assistantFileProc: false, + assistantActivityLabel: undefined, + assistantText: "", + assistantAttachments: [], + assistantProcessTrace: undefined, + assistantStatus: "blocked", + assistantErrorCode: "content_moderation.blocked", + assistantErrorMessage: t("moderationBlocked"), + assistantInlineAlert: { + title: t("moderationBlocked"), + message: [ + t("moderationBlockedDescription"), + event.eventID ? t("moderationEventId", { id: event.eventID }) : "", + categories.length > 0 ? t("moderationCategories", { categories: categories.join(", ") }) : "", + ] + .filter(Boolean) + .join("\n"), + }, + })); + toast.error(t("moderationBlocked"), { + description: t("moderationBlockedDescription"), + }); + }, }; modelRunSequence = (nextModelRunSequenceRef.current.get(targetConversationScopeKey) ?? 0) + 1; nextModelRunSequenceRef.current.set(targetConversationScopeKey, modelRunSequence); @@ -1156,6 +1191,9 @@ export function useChatMessageSubmit({ const assistantMessageSucceeded = assistantMessageStatus === "success"; updatePendingExchange(exchangeKey, (current) => { const streamedText = current.assistantText; + const assistantMessageBlocked = + assistantMessageStatus.trim().toLowerCase() === "blocked" || + completed.assistantMessage.errorCode === "content_moderation.blocked"; const terminalErrorMessage = terminalStreamError ? resolveErrorMessage(streamEventErrorToApiError(terminalStreamError, t("retryLater")), terminalStreamError.message || t("retryLater")) : ""; @@ -1219,7 +1257,12 @@ export function useChatMessageSubmit({ assistantErrorCode: completed.assistantMessage.errorCode, assistantErrorMessage: completed.assistantMessage.errorMessage, assistantInlineAlert: - completed.assistantMessage.status === "error" || completed.assistantMessage.status === "interrupted" + assistantMessageBlocked + ? current.assistantInlineAlert ?? { + title: t("moderationBlocked"), + message: t("moderationBlockedDescription"), + } + : completed.assistantMessage.status === "error" || completed.assistantMessage.status === "interrupted" ? { title: t("generationInterrupted"), message: terminalErrorMessage || completedErrorMessage || t("retryLater"), @@ -1227,7 +1270,9 @@ export function useChatMessageSubmit({ } : undefined, assistantText: - streamedText === completed.assistantMessage.content + assistantMessageBlocked + ? "" + : streamedText === completed.assistantMessage.content ? current.assistantText : completed.assistantMessage.content, }; @@ -1358,6 +1403,15 @@ export function useChatMessageSubmit({ })); return false; } + if (error instanceof ApiError && error.errorCode === "content_moderation.blocked") { + // UI already updated via onModerationBlocked; settle as a soft block with retry. + shouldKeepConversationLayout = true; + releaseAttachments(effectiveAttachments); + if (conversationScopeKeyRef.current === targetConversationScopeKey) { + reload(); + } + return false; + } const errorMessage = resolveErrorMessage(error, t("retryLater")); const errorDetails = resolveErrorDetails(error); const errorSummary = resolveErrorSummary(error, t("retryLater")); diff --git a/frontend/features/chat/model/chat-thread.ts b/frontend/features/chat/model/chat-thread.ts index 76f9766cf..910b2d230 100644 --- a/frontend/features/chat/model/chat-thread.ts +++ b/frontend/features/chat/model/chat-thread.ts @@ -168,6 +168,8 @@ type MessageLabels = { generationInterrupted: string; streamInterrupted?: string; imageRunning?: string; + moderationBlocked?: string; + moderationBlockedDescription?: string; resolveErrorMessage?: (errorCode: string, fallback: string, details?: UpstreamDebugInfo) => string; }; @@ -229,7 +231,17 @@ export function mapServerMessage( msg.latencyMS = item.latencyMS ?? 0; msg.billingCost = item.billingCost; msg.processTrace = parseProcessTrace(item); - if ((item.status === "error" || item.status === "interrupted") && item.errorMessage?.trim()) { + const status = item.status.trim().toLowerCase(); + const moderationBlocked = status === "blocked" || item.errorCode === "content_moderation.blocked"; + if (moderationBlocked) { + msg.inlineAlert = { + title: labels.moderationBlocked || "Content blocked", + message: + labels.moderationBlockedDescription || + item.errorMessage?.trim() || + "This response was withdrawn after a safety check.", + }; + } else if ((status === "error" || status === "interrupted") && item.errorMessage?.trim()) { const details = extractInlineAlertDetails(item); msg.inlineAlert = { title: labels.generationInterrupted, diff --git a/frontend/i18n/messages.ts b/frontend/i18n/messages.ts index 9493daf1e..080db45c1 100644 --- a/frontend/i18n/messages.ts +++ b/frontend/i18n/messages.ts @@ -1,5 +1,7 @@ +import type { AppLocale } from "@/i18n/config"; import enAdminAnnouncements from "@/i18n/messages/en-US/admin-announcements.json"; import enAdminBilling from "@/i18n/messages/en-US/admin-billing.json"; +import enAdminContentModeration from "@/i18n/messages/en-US/admin-content-moderation.json"; import enAdminConversation from "@/i18n/messages/en-US/admin-conversation.json"; import enAdminFiles from "@/i18n/messages/en-US/admin-files.json"; import enAdminGroups from "@/i18n/messages/en-US/admin-groups.json"; @@ -11,8 +13,8 @@ import enAdminStatistics from "@/i18n/messages/en-US/admin-statistics.json"; import enAdminTools from "@/i18n/messages/en-US/admin-tools.json"; import enAdminUpstreams from "@/i18n/messages/en-US/admin-upstreams.json"; import enAdminUsers from "@/i18n/messages/en-US/admin-users.json"; -import enChat from "@/i18n/messages/en-US/chat.json"; import enAnnouncements from "@/i18n/messages/en-US/announcements.json"; +import enChat from "@/i18n/messages/en-US/chat.json"; import enCommon from "@/i18n/messages/en-US/common.json"; import enConversation from "@/i18n/messages/en-US/conversation.json"; import enErrors from "@/i18n/messages/en-US/errors.json"; @@ -23,7 +25,6 @@ import enPrompts from "@/i18n/messages/en-US/prompts.json"; import enRecent from "@/i18n/messages/en-US/recent.json"; import enSettings from "@/i18n/messages/en-US/settings.json"; import enShare from "@/i18n/messages/en-US/share.json"; -import type { AppLocale } from "@/i18n/config"; import { replaceDefaultBrandTitle } from "@/shared/config/branding"; const ENGLISH_MESSAGES = { @@ -52,6 +53,7 @@ const ENGLISH_MESSAGES = { adminTools: enAdminTools, adminUpstreams: enAdminUpstreams, adminUsers: enAdminUsers, + adminContentModeration: enAdminContentModeration, }; export type AppMessages = typeof ENGLISH_MESSAGES; @@ -134,6 +136,7 @@ export async function loadLocaleMessages(locale: AppLocale): Promise) => void; onUsage?: (event: Extract) => void; onInterrupted?: (event: Extract) => void; + onModerationChecking?: (event: Extract) => void; + onModerationBlocked?: (event: Extract) => void; +}; + +type StreamReadResult = { + completed: SendMessageResult | null; + moderationBlocked: Extract | null; }; async function readConversationStream( response: Response, options: ConversationStreamOptions, -): Promise { +): Promise { if (!response.body) { - return null; + return { completed: null, moderationBlocked: null }; } const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; let completed: SendMessageResult | null = null; + let moderationBlocked: Extract | null = null; + + const consumeEvent = (event: StreamMessageEvent) => { + if (event.type === "moderation_blocked") { + moderationBlocked = event; + } + const nextCompleted = handleStreamEvent(event, options, response.status); + if (nextCompleted) { + completed = nextCompleted; + } + }; while (true) { let readResult: ReadableStreamReadResult; @@ -992,11 +1034,7 @@ async function readConversationStream( buffer = remainder; for (const document of documents) { - const event = normalizeStreamEvent(JSON.parse(document)); - const nextCompleted = handleStreamEvent(event, options, response.status); - if (nextCompleted) { - completed = nextCompleted; - } + consumeEvent(normalizeStreamEvent(JSON.parse(document))); } if (done) { @@ -1006,14 +1044,10 @@ async function readConversationStream( const tail = buffer.trim(); if (tail) { - const event = normalizeStreamEvent(JSON.parse(tail)); - const nextCompleted = handleStreamEvent(event, options, response.status); - if (nextCompleted) { - completed = nextCompleted; - } + consumeEvent(normalizeStreamEvent(JSON.parse(tail))); } - return completed; + return { completed, moderationBlocked }; } async function postConversationStream( @@ -1041,11 +1075,23 @@ async function postConversationStream( throw new ApiError("stream body is empty", response.status); } - const completed = await readConversationStream(response, options); - if (!completed) { - throw new ApiError("stream completed without final payload", response.status); + const { completed, moderationBlocked } = await readConversationStream(response, options); + if (moderationBlocked) { + throw new ApiError( + "content blocked by moderation", + response.status, + { + eventID: moderationBlocked.eventID, + direction: moderationBlocked.direction, + categories: moderationBlocked.categories, + }, + "content_moderation.blocked", + ); + } + if (completed) { + return completed; } - return completed; + throw new ApiError("stream completed without final payload", response.status); } export async function streamMessage( diff --git a/frontend/shared/api/conversation.types.ts b/frontend/shared/api/conversation.types.ts index 4ab787be1..1637ebf53 100644 --- a/frontend/shared/api/conversation.types.ts +++ b/frontend/shared/api/conversation.types.ts @@ -1,19 +1,30 @@ import type { - BatchSetConversationProjectRequest as ContractBatchSetConversationProjectRequest, BatchSetConversationProjectResponse, ContextArtifactResponse, + BatchSetConversationProjectRequest as ContractBatchSetConversationProjectRequest, + CreateConversationProjectRequest as ContractCreateConversationProjectRequest, + CreateConversationRequest as ContractCreateConversationRequest, + CreateConversationShareRequest as ContractCreateConversationShareRequest, + RenameConversationRequest as ContractRenameConversationRequest, + ReorderConversationProjectsRequest as ContractReorderConversationProjectsRequest, + RevokeConversationSharesRequest as ContractRevokeConversationSharesRequest, + SendMessageRequest as ContractSendMessageRequest, + SetConversationArchiveRequest as ContractSetConversationArchiveRequest, + SetConversationProjectRequest as ContractSetConversationProjectRequest, + SetConversationStarRequest as ContractSetConversationStarRequest, + SetMessageFeedbackRequest as ContractSetMessageFeedbackRequest, + UpdateConversationLabelsRequest as ContractUpdateConversationLabelsRequest, + UpdateConversationProjectRequest as ContractUpdateConversationProjectRequest, + UpdateMessageRequest as ContractUpdateMessageRequest, ConversationDefaultModelCandidateResponse, ConversationDeleteResponse, ConversationExportResponse, - ConversationProjectResponse, ConversationPreviewMessageResponse, + ConversationProjectResponse, ConversationResponse, ConversationSearchPageResponse, ConversationSearchResultResponse, ConversationShareResponse, - CreateConversationProjectRequest as ContractCreateConversationProjectRequest, - CreateConversationRequest as ContractCreateConversationRequest, - CreateConversationShareRequest as ContractCreateConversationShareRequest, MessageBillingCostResponse, MessageFeedbackResponse, MessageProcessTraceResponse, @@ -26,20 +37,9 @@ import type { ModelProbeDebugResponse, PublicSharedConversationResponse, PublicSharedMessageResponse, - RenameConversationRequest as ContractRenameConversationRequest, - ReorderConversationProjectsRequest as ContractReorderConversationProjectsRequest, - RevokeConversationSharesRequest as ContractRevokeConversationSharesRequest, RevokeConversationSharesResponse, RunResponse, - SendMessageRequest as ContractSendMessageRequest, SendMessageResponse, - SetConversationArchiveRequest as ContractSetConversationArchiveRequest, - SetConversationProjectRequest as ContractSetConversationProjectRequest, - SetConversationStarRequest as ContractSetConversationStarRequest, - SetMessageFeedbackRequest as ContractSetMessageFeedbackRequest, - UpdateConversationProjectRequest as ContractUpdateConversationProjectRequest, - UpdateConversationLabelsRequest as ContractUpdateConversationLabelsRequest, - UpdateMessageRequest as ContractUpdateMessageRequest, } from "@deeix/api-contract"; import type { UserStorageQuotaDTO } from "@/shared/api/file.types"; @@ -294,6 +294,17 @@ export type StreamMessageEvent = seq?: number; data: SendMessageResult; } + | { + type: "moderation_checking"; + seq?: number; + } + | { + type: "moderation_blocked"; + seq?: number; + eventID?: string; + direction?: "input" | "output" | string; + categories?: string[]; + } | { type: "compact_done"; seq?: number; diff --git a/packages/api-contract/src/types.generated.ts b/packages/api-contract/src/types.generated.ts index 2a2d316ba..210fbda8d 100644 --- a/packages/api-contract/src/types.generated.ts +++ b/packages/api-contract/src/types.generated.ts @@ -588,6 +588,164 @@ export interface CleanupLogsResponseDoc { errorMsg: string; } +export interface ContentModerationCategoryCatalogResponse { + image: string[]; + text: string[]; +} + +export interface ContentModerationConfigDataResponse { + categories: ContentModerationCategoryCatalogResponse; + config: ContentModerationServiceConfigResponse; +} + +export interface ContentModerationConfigResponseDoc { + data: ContentModerationConfigDataResponse; + errorMsg: string; +} + +export interface ContentModerationConfigUpdateDataResponse { + config: ContentModerationServiceConfigResponse; +} + +export interface ContentModerationConfigUpdateResponseDoc { + data: ContentModerationConfigUpdateDataResponse; + errorMsg: string; +} + +export interface ContentModerationDailyStatResponse { + category: string; + checkCount: number; + contentItems: number; + direction: string; + failureCount: number; + hitCount: number; + latencyCount: number; + latencySumMS: number; + modality: string; + result: string; + statDate: string; +} + +export interface ContentModerationEventDetailResponse { + categoryScores: Record; + decryptedText?: string; + event: ContentModerationEventResponse; + images: ContentModerationIsolatedImageResponse[]; + imagesAvailable: boolean; + textAvailable: boolean; +} + +export interface ContentModerationEventDetailResponseDoc { + data: ContentModerationEventDetailResponse; + errorMsg: string; +} + +export interface ContentModerationEventListDataResponse { + items: ContentModerationEventResponse[]; + page: number; + pageSize: number; + total: number; +} + +export interface ContentModerationEventListResponseDoc { + data: ContentModerationEventListDataResponse; + errorMsg: string; +} + +export interface ContentModerationEventResponse { + categories: string[]; + contentSummary: string; + conversationID: number; + createdAt: string; + direction: string; + errorCode: string; + errorMessage: string; + latencyMS: number; + messagePublicID: string; + modality: string; + model: string; + policyVersion: number; + publicID: string; + result: string; + runID: string; + userID: number; + userLabel?: string; + username?: string; +} + +export interface ContentModerationIsolatedImageResponse { + sha256: string; + index: number; + mimeType: string; + sizeBytes: number; + sourceFileID?: string; +} + +export interface ContentModerationPolicyRequest { + inputImageCategories: string[]; + inputTextCategories: string[]; + outputImageCategories: string[]; + outputTextCategories: string[]; +} + +export interface ContentModerationPolicyResponse { + inputImageCategories: string[]; + inputTextCategories: string[]; + outputImageCategories: string[]; + outputTextCategories: string[]; + version: number; +} + +export interface ContentModerationProbeResponse { + image: ContentModerationProbeResultResponse; + text: ContentModerationProbeResultResponse; +} + +export interface ContentModerationProbeResponseDoc { + data: ContentModerationProbeResponse; + errorMsg: string; +} + +export interface ContentModerationProbeResultResponse { + error?: string; + latencyMS: number; + model?: string; + valid: boolean; +} + +export interface ContentModerationServiceConfigResponse { + apiKeyMasked?: string; + baseUrl: string; + enabled: boolean; + hasAPIKey: boolean; + maxConcurrency: number; + model: string; + policy: ContentModerationPolicyResponse; + queueCapacity: number; + timeoutSeconds: number; +} + +export interface ContentModerationStatsDataResponse { + items: ContentModerationDailyStatResponse[]; +} + +export interface ContentModerationStatsResponseDoc { + data: ContentModerationStatsDataResponse; + errorMsg: string; +} + +export interface ContentModerationUpdateConfigRequest { + apiKey?: string; + baseUrl?: string; + clearAPIKey?: boolean; + enabled?: boolean; + maxConcurrency?: number; + model?: string; + policy?: ContentModerationPolicyRequest; + queueCapacity?: number; + timeoutSeconds?: number; +} + export interface ContextArtifactResponse { content: string; createdAt: string; @@ -1512,6 +1670,13 @@ export interface MessageListResponseDoc { errorMsg: string; } +export interface MessageModerationResponse { + categories?: string[]; + direction?: string; + eventID?: string; + state?: string; +} + export interface MessageProcessTraceResponse { enabled: boolean; events?: MessageTraceEventResponse[]; @@ -1570,6 +1735,7 @@ export interface MessageResponse { latencyMS: number; modelIcon: string; modelVendor: string; + moderation?: MessageModerationResponse; myFeedback: string; outputTokens: number; parentMessageID: number | null; @@ -4163,6 +4329,152 @@ export namespace Admin { export type ResponseBody = UsageLogListResponseDoc; } + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationConfigList + * @summary Get content moderation config + * @request GET:/admin/content-moderation/config + * @secure + */ + export namespace ContentModerationConfigList { + export type RequestParams = {}; + export type RequestQuery = {}; + export type RequestBody = never; + export type RequestHeaders = {}; + export type ResponseBody = ContentModerationConfigResponseDoc; + } + + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationConfigUpdate + * @summary Update content moderation config + * @request PUT:/admin/content-moderation/config + * @secure + */ + export namespace ContentModerationConfigUpdate { + export type RequestParams = {}; + export type RequestQuery = {}; + export type RequestBody = ContentModerationUpdateConfigRequest; + export type RequestHeaders = {}; + export type ResponseBody = ContentModerationConfigUpdateResponseDoc; + } + + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationEventsList + * @summary List content moderation events + * @request GET:/admin/content-moderation/events + * @secure + */ + export namespace ContentModerationEventsList { + export type RequestParams = {}; + export type RequestQuery = { + /** Category filter */ + category?: string; + /** Direction filter */ + direction?: string; + /** Start time (RFC3339) */ + from?: string; + /** Modality filter */ + modality?: string; + /** Page number */ + page?: number; + /** Page size */ + pageSize?: number; + /** Result filter */ + result?: string; + /** Run ID */ + runId?: string; + /** End time (RFC3339) */ + to?: string; + /** User ID */ + userId?: number; + }; + export type RequestBody = never; + export type RequestHeaders = {}; + export type ResponseBody = ContentModerationEventListResponseDoc; + } + + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationEventsDetail + * @summary Get content moderation event detail + * @request GET:/admin/content-moderation/events/{eventID} + * @secure + */ + export namespace ContentModerationEventsDetail { + export type RequestParams = { + /** Moderation event ID */ + eventId: string; + }; + export type RequestQuery = {}; + export type RequestBody = never; + export type RequestHeaders = {}; + export type ResponseBody = ContentModerationEventDetailResponseDoc; + } + + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationEventsImagesDetail + * @summary Stream a isolated moderation image + * @request GET:/admin/content-moderation/events/{eventID}/images/{index} + * @secure + */ + export namespace ContentModerationEventsImagesDetail { + export type RequestParams = { + /** Moderation event ID */ + eventId: string; + /** Image index */ + index: number; + }; + export type RequestQuery = {}; + export type RequestBody = never; + export type RequestHeaders = {}; + export type ResponseBody = Blob; + } + + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationProbeCreate + * @summary Probe content moderation service + * @request POST:/admin/content-moderation/probe + * @secure + */ + export namespace ContentModerationProbeCreate { + export type RequestParams = {}; + export type RequestQuery = {}; + export type RequestBody = never; + export type RequestHeaders = {}; + export type ResponseBody = ContentModerationProbeResponseDoc; + } + + /** + * No description + * @tags admin-content-moderation + * @name ContentModerationStatsList + * @summary Get content moderation daily stats + * @request GET:/admin/content-moderation/stats + * @secure + */ + export namespace ContentModerationStatsList { + export type RequestParams = {}; + export type RequestQuery = { + /** Start time (RFC3339) */ + from?: string; + /** End time (RFC3339) */ + to?: string; + }; + export type RequestBody = never; + export type RequestHeaders = {}; + export type ResponseBody = ContentModerationStatsResponseDoc; + } + /** * @description 管理员分页查看对话运行轨迹、工具、MCP 与处理事件 * @tags admin