diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..395749f --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,29 @@ +name: FastAPI Auto Deploy + +on: + push: + branches: [ "develop" ] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: SSH Remote Commands for FastAPI + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.GCP_IP }} + username: ${{ secrets.GCP_USER }} + key: ${{ secrets.GCP_SSH_KEY }} + command_timeout: 30m + script: | + # 1. 파이썬 공용 폴더로 이동해서 최신 코드 땡겨오기 + cd /opt/analysis-engine + git pull origin develop + + # 2. 도커 설계도가 있는 메인 서버 폴더로 이동! + cd /opt/main-server + + # 3. 파이썬(veriq-analysis) 컨테이너만 콕 집어서 다시 빌드하고 실행 + sudo docker-compose up --build -d veriq-analysis + + diff --git a/.gitignore b/.gitignore index b7faf40..aa75ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ venv/ ENV/ env.bak/ venv.bak/ +.vscode # Spyder project settings .spyderproject @@ -173,7 +174,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Abstra # Abstra is an AI-powered process automation framework. @@ -186,7 +187,7 @@ cython_debug/ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore # and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder -# .vscode/ +.vscode/ # Ruff stuff: .ruff_cache/ @@ -205,3 +206,12 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# 검사기 테스트 결과 및 테스트 도구 +test +test/ +test-data +test-data/ +eval_results +tools +test_results \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ca6fe06 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "git.ignoreLimitWarning": true +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f8fda52 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index ed693aa..f0fb1be 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,11 @@ # analysis-engine -be2 -python- FastAPI -api: google safe browsing -*** +## 다음 수정 : +이것도 금일(0503) 내로 진행합니다...... -## 현재 단계 : API, DB 조회 오류 수정, URL정규화+단축/https검사 구현 -#### 지난 단계: API, DB 조회 구현 +#### 1. server info조회 결과와 context 간 연결, 결과 json에 확실히 반영되도록 안정화 -조회 부분까지의 업로드를 다른 브랜치에 하려 했으나 실수로 main에 집어넣었습니다. -오류 수정을 포함한 이후 단계부터는 develop, 또는 develop/기능 이름 브랜치에서 진행합니다. +#### 2. api조회 비동기 반영 수정 -*** - -### API, DB 조회 상세 설명 -+ 테스트를 위한 프로토타입 목적입니다. (최종 형태 X) - + 작동 구조만 맞추는 것에 가깝습니다. - + 더미 데이터로 테스트하며, 응답을 저장하지 않습니다. -+ API는 google safe browsing만을 조회합니다. -+ DB 조회 형식이 확정되지 않았습니다. - + case1> 요청-응답 : 분석 서버에서 메인 서버로 조회 요청 전송, 조회 결과를 응답받습니다. <현재 - + case2> 직접 조회(pymysql) : 분석 서버에서 MySQL DB로 직접 질의를 통해 조회합니다. +#### 3. feature 추출기 결합 + 학습한 모델 업로드 +*제발할수있었으면좋겠네요진짜* diff --git a/analyzers/__init__.py b/analyzers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analyzers/ai_analyze.py b/analyzers/ai_analyze.py index bcf8923..1db0703 100644 --- a/analyzers/ai_analyze.py +++ b/analyzers/ai_analyze.py @@ -1,4 +1,270 @@ -# ai를 이용하는 모든 부분을 해당 클래스에 선언. -# userinfo, host, domain등.... 가장 많은 파츠를 보지 않나 -# 스코어링도 ai로 한다면 여기에 클래스 정의하게 됨 -# scoring engine을 별도 분리할 생각이 그다지 없긴 한데... 아무튼간에. +# pre-trained ML 모델을 호출해 분석을 진행합니다. +# 현재 연결 대상: CharCNN + FEB/XGB score-level fusion +from core.context import AnalysisContext +from ml.charcnn.predictor import predict_url as predict_charcnn_url +from ml.xgb.predictor import predict_context as predict_xgb_context + + +FUSION_METHOD = "weighted_average" +XGB_WEIGHT = 0.65 +CHARCNN_WEIGHT = 0.35 + +# 한 모델이 매우 강하게 위험으로 판단한 경우, +# 가중 평균 점수에 최소 보정점을 더해 ML 결과에 반영한다. +HIGH_CONFIDENCE_THRESHOLD = 85 +HIGH_CONFIDENCE_BONUS = 25 + + +# ML 입력 URL 결정 함수 +# 요구사항: redirect 추적 결과가 있으면 final_url을 우선 사용한다. +def resolve_ml_target_url(context: AnalysisContext) -> str | None: + final_url = context.redirect.get("final_url") + if final_url: + return final_url + + normalized_url = context.url.get("normalized_url") + if normalized_url: + return normalized_url + + return context.request.get("original_url") + + +def ai_analyze(context: AnalysisContext) -> dict: + target_url = resolve_ml_target_url(context) + + charcnn_result = run_charcnn_inference(target_url) + charcnn_result = apply_charcnn_whitelist_score_cap(context, charcnn_result) + + xgb_result = run_xgb_inference(context) + fusion_result = fuse_ml_scores(xgb_result, charcnn_result) + + return build_ml_result( + score=fusion_result.get("score", 0), + threats=collect_ml_threats(xgb_result, charcnn_result, fusion_result), + charcnn=charcnn_result, + xgb=xgb_result, + fusion=fusion_result, + ) + + +def run_charcnn_inference(target_url: str | None) -> dict: + return predict_charcnn_url(target_url) + + +def run_xgb_inference(context: AnalysisContext) -> dict: + return predict_xgb_context(context) + + +def is_available(result: dict | None) -> bool: + return isinstance(result, dict) and result.get("available") is True + + +def safe_score(result: dict | None) -> int | None: + if not isinstance(result, dict): + return None + + if result.get("available") is not True: + return None + + try: + return max(0, min(100, int(round(float(result.get("score", 0)))))) + except (TypeError, ValueError): + return None + + +def fuse_ml_scores(xgb: dict, charcnn: dict) -> dict: + xgb_score = safe_score(xgb) + charcnn_score = safe_score(charcnn) + + contributors = [] + + if xgb_score is not None: + contributors.append("xgb") + if charcnn_score is not None: + contributors.append("charcnn") + + if xgb_score is not None and charcnn_score is not None: + # 현재 CharCNN이 일부 정상 URL에도 매우 높은 점수를 내는 경향이 있어 + # 기본 fusion은 XGB를 더 강하게 반영하는 가중 평균으로 유지한다. + base_score = round(xgb_score * XGB_WEIGHT + charcnn_score * CHARCNN_WEIGHT) + method = FUSION_METHOD + weights = {"xgb": XGB_WEIGHT, "charcnn": CHARCNN_WEIGHT} + elif xgb_score is not None: + base_score = xgb_score + method = "xgb_only" + weights = {"xgb": 1.0, "charcnn": 0.0} + elif charcnn_score is not None: + base_score = charcnn_score + method = "charcnn_only" + weights = {"xgb": 0.0, "charcnn": 1.0} + else: + base_score = 0 + method = "unavailable" + weights = {"xgb": 0.0, "charcnn": 0.0} + + bonus = HIGH_CONFIDENCE_BONUS if has_high_confidence_score(xgb_score, charcnn_score) else 0 + score = max(0, min(100, int(base_score + bonus))) + + return { + "available": bool(contributors), + "method": method, + "weights": weights, + "contributors": contributors, + "score": score, + } + + +def has_high_confidence_score(*scores: int | None) -> bool: + for score in scores: + if score is not None and score >= HIGH_CONFIDENCE_THRESHOLD: + return True + + return False + + +def collect_ml_threats(xgb: dict, charcnn: dict, fusion: dict) -> list[str]: + # 위협 라벨은 fusion 점수와 별개로, 개별 모델이 반환한 threats를 그대로 반영한다. + # 단, CharCNN whitelist가 적용된 경우 apply_charcnn_whitelist_score_cap()에서 + # charcnn.threats를 비우므로 top-level ml.threats에도 올라오지 않는다. + threats = [] + for result in (xgb, charcnn): + if not isinstance(result, dict): + continue + + for threat in result.get("threats", []) or []: + if threat not in threats: + threats.append(threat) + + return threats + + +def build_ml_result( + score: int = 0, + threats: list | None = None, + charcnn: dict | None = None, + xgb: dict | None = None, + fusion: dict | None = None, +) -> dict: + if threats is None: + threats = [] + + return { + "score": score, + "threats": threats, + "charcnn": charcnn or { + "available": False, + "score": 0, + "label": "unknown", + "threats": [], + }, + "xgb": xgb or { + "available": False, + "score": 0, + "label": "unknown", + "threats": [], + }, + "fusion": fusion or { + "available": False, + "method": "unavailable", + "score": 0, + "contributors": [], + }, + } + +# ============================================================ +# CharCNN whitelist score-cap post-processing +# ============================================================ +# 목적: +# - 유명 정상 도메인에서 CharCNN 오탐이 높게 나오는 경우만 완화한다. +# - rule 기반 검사에서 flags가 있으면 whitelist를 적용하지 않는다. +# - 다른 검사 항목, XGB, external, cert, redirect 결과는 건드리지 않는다. +# - whitelist 적용 여부를 별도 라벨로 반환하지 않는다. + +WHITELIST_DOMAINS = { + "google.com", "google.co.kr", "naver.com", "daum.net", "yahoo.com", "bing.com", + "facebook.com", "instagram.com", "twitter.com", "linkedin.com", "youtube.com", "kakao.com", + "apple.com", "microsoft.com", "github.com", "stackoverflow.com", "cloudflare.com", + "amazon.com", "aws.com", +} + +CHARCNN_WHITELIST_SCORE_CAP = 10 + + +def apply_charcnn_whitelist_score_cap(context: AnalysisContext, charcnn_result: dict | None) -> dict: + if not isinstance(charcnn_result, dict): + return { + "available": False, + "score": 0, + "label": "unknown", + "threats": [], + } + + if not is_available(charcnn_result): + return charcnn_result + + if has_rule_flags(context): + return charcnn_result + + if not is_whitelisted_context_domain(context): + return charcnn_result + + result = dict(charcnn_result) + capped_score = min(to_int_score(result.get("score", 0)), CHARCNN_WHITELIST_SCORE_CAP) + + result["score"] = capped_score + result["threats"] = [] + + # CharCNN 세부 label을 그대로 malicious로 두면 하위 결과를 읽는 쪽에서 혼동될 수 있어 + # score cap 이후에는 안전 쪽 label로 맞춘다. whitelist 관련 라벨은 별도로 추가하지 않는다. + if capped_score < 50: + result["label"] = "normal" + + return result + + +def has_rule_flags(context: AnalysisContext) -> bool: + rule_based = {} + + if context is not None: + detection = getattr(context, "detection", {}) or {} + rule_based = detection.get("rule_based", {}) or {} + + flags = rule_based.get("flags", []) + return bool(flags) + + +def is_whitelisted_context_domain(context: AnalysisContext) -> bool: + hostname = get_context_hostname(context) + + if not hostname: + return False + + for allowed_domain in WHITELIST_DOMAINS: + allowed_domain = normalize_domain(allowed_domain) + + if hostname == allowed_domain or hostname.endswith("." + allowed_domain): + return True + + return False + + +def get_context_hostname(context: AnalysisContext) -> str: + if context is None: + return "" + + url_info = getattr(context, "url", {}) or {} + return normalize_domain(url_info.get("hostname")) + + +def normalize_domain(domain: str | None) -> str: + if not domain: + return "" + + return str(domain).strip().lower().strip(".") + + +def to_int_score(value, default: int = 0) -> int: + try: + return max(0, min(100, int(round(float(value))))) + except (TypeError, ValueError): + return default \ No newline at end of file diff --git a/analyzers/external/__init__.py b/analyzers/external/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analyzers/external/req_gsb.py b/analyzers/external/req_gsb.py new file mode 100644 index 0000000..87069f7 --- /dev/null +++ b/analyzers/external/req_gsb.py @@ -0,0 +1,126 @@ +import os +import time +import requests + +from core.config import statusData as stat + +GSB_KEY = os.getenv("GOOGLE_SAFE_BROWSING_API_KEY") +GSB_CODE = "GSB" + + +# GSB 조회 메인 함수 +def req_gsb(target_url: str | None) -> dict: + raw_gsb = GSB_req_api(target_url) + parsed_gsb = parse_GSB_result(raw_gsb) + + return parsed_gsb + + +# GSB request +def GSB_req_api(url: str | None) -> dict: + if not url: + return {"status": stat.INVALID_INPUT, "data": None} + + if not GSB_KEY: + return {"status": stat.MISSING_API_KEY, "data": None} + + endpoint = f"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={GSB_KEY}" + body = GSB_req_build(url) + + start = time.time() + + try: + response = requests.post(endpoint, json=body, timeout=30) + elapsed = time.time() - start + + try: + response_data = response.json() if response.status_code == 200 else None + except ValueError: + response_data = None + + return { + "status": stat.SUCCESS, + "status_code": response.status_code, + "data": response_data, + "elapsed": elapsed + } + + except requests.exceptions.Timeout: + return {"status": stat.TIMEOUT, "data": None} + + except requests.exceptions.RequestException as e: + return { + "status": stat.REQUEST_FAILED, + "data": None, + "error": str(e) + } + + +# GSB request body 생성 +def GSB_req_build(url: str) -> dict: + return { + "client": { + "clientId": "url-analysis-query", + "clientVersion": "1.0" + }, + "threatInfo": { + "threatTypes": [ + "MALWARE", + "SOCIAL_ENGINEERING", + "UNWANTED_SOFTWARE", + "POTENTIALLY_HARMFUL_APPLICATION" + ], + "platformTypes": ["ANY_PLATFORM"], + "threatEntryTypes": ["URL"], + "threatEntries": [{"url": url}] + } + } + + +# GSB 응답 파싱 +def parse_GSB_result(raw: dict) -> dict: + status = raw.get("status") + + if status != stat.SUCCESS: + return { + "is_threat": False, + "status_type": [None, status], + "result_type": None, + "threat_labels": [] + } + + status_code = raw.get("status_code") + + if status_code != 200: + return { + "is_threat": False, + "status_type": [status_code, stat.CLIENT_ERROR], + "result_type": None, + "threat_labels": [] + } + + data = raw.get("data") or {} + matches = data.get("matches", []) + + if not matches: + return { + "is_threat": False, + "status_type": [200, stat.SUCCESS], + "result_type": True, + "threat_labels": [] + } + + threat_types = { + match.get("threatType") + for match in matches + if match.get("threatType") + } + + labels = [f"{GSB_CODE}:{threat}" for threat in threat_types] + + return { + "is_threat": True, + "status_type": [200, stat.SUCCESS], + "result_type": True, + "threat_labels": labels + } \ No newline at end of file diff --git a/analyzers/external/req_otx.py b/analyzers/external/req_otx.py new file mode 100644 index 0000000..fbd5a57 --- /dev/null +++ b/analyzers/external/req_otx.py @@ -0,0 +1,169 @@ +import os +import requests +from urllib.parse import quote, urlsplit + +try: + import tldextract +except ImportError: + tldextract = None + +from core.config import statusData as stat + +OTX_KEY = os.getenv("OTX_API_KEY") +TARGET_THREAT_KEYWORDS = [ + "PHISHING", + "MALWARE", + "RANSOMWARE", + "BOTNET", + "SPAM", + "C2", +] + + +def req_otx(target_url: str | None) -> dict: + raw_otx = OTX_req_api(target_url) + + # 화이트리스트면 OTX 신고 이력이 있어도 일단 무시 + if is_whitelisted(target_url): + return { + "status": raw_otx.get("status"), + "is_threat": False, + "pulse_count": 0, + "threat_labels": [], + } + + pulse_count, threat_labels = parse_OTX_result(raw_otx) + + return { + "status": raw_otx.get("status"), + "is_threat": pulse_count > 0, + "pulse_count": pulse_count, + "threat_labels": threat_labels, + } + + +def OTX_req_api(url: str | None) -> dict: + if not url: + return {"status": stat.INVALID_INPUT, "data": None} + + if not OTX_KEY: + return {"status": stat.MISSING_API_KEY, "data": None} + + endpoint, headers = OTX_req_build(url) + + try: + response = requests.get(endpoint, headers=headers, timeout=30) + + try: + response_data = response.json() if response.status_code == 200 else None + except ValueError: + response_data = None + + if response.status_code == 200: + return { + "status": stat.SUCCESS, + "status_code": response.status_code, + "data": response_data, + } + + return { + "status": stat.CLIENT_ERROR, + "status_code": response.status_code, + "data": response_data, + } + + except requests.exceptions.Timeout: + return {"status": stat.TIMEOUT, "data": None} + + except requests.exceptions.RequestException as e: + return { + "status": stat.REQUEST_FAILED, + "data": None, + "error": str(e), + } + + +def OTX_req_build(url: str) -> tuple[str, dict]: + encoded_url = quote(url, safe="") + endpoint = f"https://otx.alienvault.com/api/v1/indicators/url/{encoded_url}/general" + headers = {"X-OTX-API-KEY": OTX_KEY} + return endpoint, headers + + +def parse_OTX_result(raw: dict) -> tuple[int, list[str]]: + if raw.get("status") != stat.SUCCESS: + return 0, [] + + data = raw.get("data") or {} + pulse_info = data.get("pulse_info", {}) + count = pulse_info.get("count", 0) or 0 + + otx_labels = set() + + if count > 0: + pulses = pulse_info.get("pulses", []) + + for pulse in pulses: + tags = pulse.get("tags", []) + + for tag in tags: + tag_upper = str(tag).upper() + + if tag_upper in TARGET_THREAT_KEYWORDS: + otx_labels.add(f"OTX:{tag_upper}") + + if not otx_labels: + otx_labels.add("OTX:SUSPICIOUS") + + return count, list(otx_labels) + + +def is_whitelisted(url: str | None) -> bool: + """URL의 도메인이 화이트리스트에 포함되어 있는지 확인. + + tldextract가 설치되지 않은 환경에서도 import 자체가 실패하지 않도록 + stdlib 기반 fallback을 둔다. + """ + if not url: + return False + + WHITELIST_DOMAINS = { + "google.com", "google.co.kr", "naver.com", "daum.net", "yahoo.com", "bing.com", + "facebook.com", "instagram.com", "twitter.com", "linkedin.com", "youtube.com", "kakao.com", + "apple.com", "microsoft.com", "github.com", "stackoverflow.com", "cloudflare.com", "amazon.com", "aws.com" + } + + SAFE_SUFFIXES = { + "go.kr", "ac.kr", "re.kr", "mil.kr", "es.kr", "ms.kr", "hs.kr", "sc.kr", + "gov", "edu", "mil", "int" + } + + try: + domain, suffix = extract_domain_and_suffix(url) + + if domain in WHITELIST_DOMAINS: + return True + if suffix in SAFE_SUFFIXES: + return True + + return False + except Exception: + return False + + +def extract_domain_and_suffix(url: str) -> tuple[str, str]: + if tldextract is not None: + ext = tldextract.extract(url) + return f"{ext.domain}.{ext.suffix}", ext.suffix + + candidate = url if "://" in url else f"https://{url}" + hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".") + parts = hostname.split(".") if hostname else [] + + if len(parts) >= 3 and ".".join(parts[-2:]) in {"co.kr", "go.kr", "ac.kr", "or.kr", "ne.kr"}: + return ".".join(parts[-3:]), ".".join(parts[-2:]) + + if len(parts) >= 2: + return ".".join(parts[-2:]), parts[-1] + + return hostname, parts[-1] if parts else "" diff --git a/analyzers/external/req_whois.py b/analyzers/external/req_whois.py new file mode 100644 index 0000000..82500b9 --- /dev/null +++ b/analyzers/external/req_whois.py @@ -0,0 +1,109 @@ +from datetime import datetime, date +from urllib.parse import urlsplit + +try: + import whois +except ImportError: + whois = None + +try: + import tldextract +except ImportError: + tldextract = None + +from core.config import statusData as stat + + +def req_whois(target_url: str | None) -> dict: + raw_whois = WHOIS_req_api(target_url) + domain_date = parse_WHOIS_result(raw_whois) + + return { + "status": raw_whois.get("status"), + "creation_date": domain_date, + "domain_age_days": calculate_domain_age(domain_date) + } + + +def WHOIS_req_api(url: str | None) -> dict: + if not url: + return {"status": stat.INVALID_INPUT, "data": None} + + if whois is None: + return {"status": stat.REQUEST_FAILED, "data": None, "error": "python-whois_not_installed"} + + try: + domain = extract_registered_domain(url) + + if not domain: + return {"status": stat.INVALID_INPUT, "data": None} + + domain_info = whois.whois(domain) + return {"status": stat.SUCCESS, "data": domain_info} + except Exception as e: + return {"status": stat.REQUEST_FAILED, "data": None, "error": str(e)} + + +def extract_registered_domain(url: str) -> str | None: + if tldextract is not None: + ext = tldextract.extract(url) + if not ext.domain or not ext.suffix: + return None + return f"{ext.domain}.{ext.suffix}" + + candidate = url if "://" in url else f"https://{url}" + hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".") + if not hostname: + return None + + parts = hostname.split(".") + if len(parts) < 2: + return None + + if len(parts) >= 3 and ".".join(parts[-2:]) in {"co.kr", "go.kr", "ac.kr", "or.kr", "ne.kr"}: + return ".".join(parts[-3:]) + + return ".".join(parts[-2:]) + + +def parse_WHOIS_result(raw: dict) -> str | None: + if raw.get("status") != stat.SUCCESS: + return None + + data = raw.get("data") + if not data: + return None + + creation_date = data.get("creation_date") if isinstance(data, dict) else getattr(data, "creation_date", None) + + if isinstance(creation_date, list): + creation_date = next((item for item in creation_date if item), None) + + if isinstance(creation_date, datetime): + return creation_date.strftime("%Y-%m-%d") + + if isinstance(creation_date, date): + return creation_date.isoformat() + + if isinstance(creation_date, str): + creation_date = creation_date.strip() + + for fmt in ("%Y-%m-%d", "%Y.%m.%d", "%Y/%m/%d"): + try: + return datetime.strptime(creation_date[:10], fmt).strftime("%Y-%m-%d") + except ValueError: + pass + + return None + + +def calculate_domain_age(creation_date: str | None) -> int | None: + if not creation_date: + return None + + try: + created = datetime.strptime(creation_date, "%Y-%m-%d").date() + today = datetime.now().date() + return max((today - created).days, 0) + except ValueError: + return None diff --git a/analyzers/parameter_rule.py b/analyzers/parameter_rule.py new file mode 100644 index 0000000..bfa31b4 --- /dev/null +++ b/analyzers/parameter_rule.py @@ -0,0 +1,158 @@ +# 룰 기반의 파라미터 분석 로직입니다. +# query 파라미터, path, 인코딩 등을 분석해 의심스러운 사항을 기록합니다. +import re +from urllib.parse import parse_qsl, unquote + +from core.context import AnalysisContext + +FLAG_PERCENT_ENCODING = "percent_encoding_detected" +FLAG_DOUBLE_ENCODING = "double_encoding_suspected" + +FLAG_SUSPICIOUS_QUERY_PARAM = "suspicious_query_param_detected" +FLAG_EMBEDDED_URL_IN_QUERY = "embedded_url" +FLAG_SUSPICIOUS_QUERY_KEYWORD = "suspicious_query_keyword_detected" +SUSPICIOUS_QUERY_PARAM_NAMES = { + "url", "redirect", "redir", "next", "continue", + "target", "dest", "destination", "return", + "returnurl", "return_url", "callback" +} +SUSPICIOUS_KEYWORDS = { + "login", "signin", "verify", "verification", + "secure", "account", "update", "bank", + "payment", "confirm", "wallet", "auth" +} +FLAG_SUSPICIOUS_PATH_KEYWORD = "suspicious_path_keyword_detected" +FLAG_SUSPICIOUS_FRAGMENT_KEYWORD = "suspicious_fragment_keyword_detected" + +PERCENT_ENCODING_PATTERN = re.compile(r"%[0-9A-Fa-f]{2}") + + + +# 룰 기반 검사 수행 함수 +# context.url에서 필요한 값을 읽어 rule_based 형식의 dict를 반환 +def parameter_rule(context: AnalysisContext) -> dict: + path = context.url.get("path", "") + query = context.url.get("query", "") + fragment = context.url.get("fragment", "") + hostname = context.url.get("hostname", "") + + result = build_rule_based_result() + + analyze_encoding(path, query, fragment, result) + analyze_query(query, result) + analyze_path(path, result) + analyze_fragment(fragment, result) + analyze_hostname(hostname, result) + + return result + + +# 인코딩 검사- 퍼센트 인코딩 여부, 이중 인코딩 의심 여부를 분석 +def analyze_encoding(path: str, query: str, fragment: str, result: dict) -> None: + raw_text = f"{path}{query}{fragment}" + + # 퍼센트 인코딩 여부 + if PERCENT_ENCODING_PATTERN.search(raw_text): + if FLAG_PERCENT_ENCODING not in result["flags"]: + result["flags"].append(FLAG_PERCENT_ENCODING) + + # 1회 decode 후에도 %xx 패턴이 남으면 이중 인코딩 의심 + decoded_once = unquote(raw_text) + + if PERCENT_ENCODING_PATTERN.search(raw_text) and PERCENT_ENCODING_PATTERN.search(decoded_once): + if FLAG_DOUBLE_ENCODING not in result["flags"]: + result["flags"].append(FLAG_DOUBLE_ENCODING) + + +# query 문자열 분석 +def analyze_query(query: str, result: dict) -> None: + query_pairs = parse_qsl(query, keep_blank_values=True) + + for raw_key, raw_value in query_pairs: + key = raw_key.lower() + decoded_value = unquote(raw_value) + lower_value = decoded_value.lower() + + # 의심되는 파라미터 값 포함 여부 확인 + if key in SUSPICIOUS_QUERY_PARAM_NAMES: + if raw_key not in result["suspicious_query_params"]: + result["suspicious_query_params"].append(raw_key) + + if FLAG_SUSPICIOUS_QUERY_PARAM not in result["flags"]: + result["flags"].append(FLAG_SUSPICIOUS_QUERY_PARAM) + + # query값에 내포된 URL 존재여부 검사 + if ( + "http://" in lower_value or + "https://" in lower_value or + "www." in lower_value + ): + if decoded_value not in result["embedded_urls"]: + result["embedded_urls"].append(decoded_value) + + if FLAG_EMBEDDED_URL_IN_QUERY not in result["flags"]: + result["flags"].append(FLAG_EMBEDDED_URL_IN_QUERY) + + # query 값 안의 의심 키워드 검사 + for keyword in SUSPICIOUS_KEYWORDS: + if keyword in lower_value: + if keyword not in result["suspicious_keywords"]["query"]: + result["suspicious_keywords"]["query"].append(keyword) + + if FLAG_SUSPICIOUS_QUERY_KEYWORD not in result["flags"]: + result["flags"].append(FLAG_SUSPICIOUS_QUERY_KEYWORD) + + +# path 안의 의심 키워드 검사 +def analyze_path(path: str, result: dict) -> None: + lower_path = unquote(path).lower() + + for keyword in SUSPICIOUS_KEYWORDS: + if keyword in lower_path: + if keyword not in result["suspicious_keywords"]["path"]: + result["suspicious_keywords"]["path"].append(keyword) + + if FLAG_SUSPICIOUS_PATH_KEYWORD not in result["flags"]: + result["flags"].append(FLAG_SUSPICIOUS_PATH_KEYWORD) + + +# fragment 안의 의심 키워드 검사 +def analyze_fragment(fragment: str, result: dict) -> None: + lower_fragment = unquote(fragment).lower() + + for keyword in SUSPICIOUS_KEYWORDS: + if keyword in lower_fragment: + if keyword not in result["suspicious_keywords"]["fragment"]: + result["suspicious_keywords"]["fragment"].append(keyword) + + if FLAG_SUSPICIOUS_FRAGMENT_KEYWORD not in result["flags"]: + result["flags"].append(FLAG_SUSPICIOUS_FRAGMENT_KEYWORD) + + +# hostname 안의 의심 키워드 검사 +def analyze_hostname(hostname: str, result: dict) -> None: + lower_hostname = unquote(hostname).lower() + + for keyword in SUSPICIOUS_KEYWORDS: + if keyword in lower_hostname: + if keyword not in result["suspicious_keywords"]["hostname"]: + result["suspicious_keywords"]["hostname"].append(keyword) + + # hostname 라벨까지 추가하기 어려운 관계로 임시 할당 + if FLAG_SUSPICIOUS_QUERY_KEYWORD not in result["flags"]: + result["flags"].append(FLAG_SUSPICIOUS_QUERY_KEYWORD) + + +# rule_based 형식의 결과 dict 조립 +def build_rule_based_result() -> dict: + return { + "flags": [], + "suspicious_query_params": [], + "embedded_urls": [], + "suspicious_keywords": { + "path": [], + "query": [], + "fragment": [], + "hostname": [] + }, + } \ No newline at end of file diff --git a/analyzers/redirect_tracker.py b/analyzers/redirect_tracker.py index e69de29..a61130a 100644 --- a/analyzers/redirect_tracker.py +++ b/analyzers/redirect_tracker.py @@ -0,0 +1,189 @@ +# 리다이렉트 추적 검사 +import requests +from urllib.parse import urljoin +from core.context import AnalysisContext + + +REQUEST_TIMEOUT = 30 +MAX_REDIRECT_HOPS = 30 + +STATUS_SUCCESS = "success" +STATUS_CLIENT_ERROR = "client_error" +STATUS_REDIRECT = "redirect" +STATUS_REQUEST_FAILED = "request_failed" +STATUS_LOOP_DETECTED = "loop_detected" +STATUS_TOO_MANY_REDIRECTS = "too_many_redirects" +STATUS_INVALID_LOCATION = "invalid_location" + + + +def redirect_tracker(context: AnalysisContext) -> dict: + # 단축 URL : 단축 URL을 시작 URL로 함 + # 일반 URL : 정규화를 거친 URL을 시작 URL로 함 + if context.detection["shortened_url"]["detected"] is True: + start_url = context.request.get("original_url") + else: + start_url = context.url.get("normalized_url") + + if start_url and not start_url.startswith(("http://", "https://")): + start_url = "https://" + start_url + + current_url = start_url + visited_urls = set() + redirect_count = 0 + redirect_chain = [] + + while True: + # 루프 검사 + if current_url in visited_urls: + redirect_chain.append({ + "url": current_url, + "status_code": None, + "status_label": STATUS_LOOP_DETECTED, + "location": None + }) + return build_redirect_result( + redirect_count=redirect_count, + final_url=current_url, + redirect_chain=redirect_chain, + loop_detected=True, + status_type=[None, STATUS_LOOP_DETECTED], + ) + + # 방문했던 URL 리스트에 현재 URL 추가 + visited_urls.add(current_url) + + # 리다이렉트 요청 + response_info = request_once(current_url) + + status_code = response_info["status_code"] + location = response_info["location"] + status_type = response_info["status_type"] + + redirect_chain.append({ + "url": current_url, + "status_code": status_code, + "status_label": status_type, + "location": location + }) + + #---------- 리다이렉트 추적 + if status_type == STATUS_REDIRECT: + next_url = build_next_url(current_url, location) + + if not next_url: + return build_redirect_result( + redirect_count=redirect_count, + final_url=current_url, + redirect_chain=redirect_chain, + status_type=[status_code, STATUS_INVALID_LOCATION], + ) + + redirect_count += 1 + + if redirect_count > MAX_REDIRECT_HOPS: + return build_redirect_result( + redirect_count=redirect_count, + final_url=current_url, + redirect_chain=redirect_chain, + status_type=[status_code, STATUS_TOO_MANY_REDIRECTS], + ) + + current_url = next_url + continue + + #---------- 추적 종료 + if status_type == STATUS_SUCCESS: + return build_redirect_result( + redirect_count=redirect_count, + final_url=current_url, + redirect_chain=redirect_chain, + status_type=[status_code, STATUS_SUCCESS], + ) + + # 클라이언트 에러 종료 + if status_type == STATUS_CLIENT_ERROR: + return build_redirect_result( + redirect_count=redirect_count, + final_url=current_url, + redirect_chain=redirect_chain, + status_type=[status_code, STATUS_CLIENT_ERROR], + ) + + # 요청 실패 / 기타 종료 + return build_redirect_result( + redirect_count=redirect_count, + final_url=current_url, + redirect_chain=redirect_chain, + status_type=[status_code, STATUS_REQUEST_FAILED], + ) + + +# 상태코드 분류 +# 3xx+Location을 리다이렉트로 분류 +def classify_response_status(status_code: int, location: str | None) -> str: + if 200 <= status_code < 300: + return STATUS_SUCCESS + + if 300 <= status_code < 400: + if location: + return STATUS_REDIRECT + return STATUS_REQUEST_FAILED + + if 400 <= status_code < 500: + return STATUS_CLIENT_ERROR + + return STATUS_REQUEST_FAILED + + +# 현 URL에 한 번 요청, 자동 리다이렉트 X +def request_once(current_url: str) -> dict: + try: + response = requests.get( + current_url, + allow_redirects=False, + timeout=REQUEST_TIMEOUT + ) + + location = response.headers.get("Location") + status_type = classify_response_status(response.status_code, location) + + return { + "status_code": response.status_code, + "location": location, + "status_type": status_type, + } + + except requests.exceptions.RequestException as e: + return { + "status_code": None, + "location": None, + "status_type": STATUS_REQUEST_FAILED, + } + + +# 현 URL+Location 헤더로 이동할 URL 생성 +# location 존재 X->None, 상대 경로는 현재 URL 기준 변환 +def build_next_url(current_url: str, location: str | None) -> str | None: + if not location: + return None + return urljoin(current_url, location) + + +# context.redirect 형식에 맞춘 결과 dict 조립 +def build_redirect_result(**overrides) -> dict: + result = { + "was_checked": True, + "redirect_count": 0, + "final_url": None, + "redirect_chain": [], + "loop_detected": False, + "status_type": [None, None], + } + + for key in overrides: + if key not in result: + raise KeyError(f"허용되지 않은 redirect 필드: {key}") + + result.update(overrides) + return result \ No newline at end of file diff --git a/analyzers/req_api.py b/analyzers/req_api.py index 2782ba5..81d8cd5 100644 --- a/analyzers/req_api.py +++ b/analyzers/req_api.py @@ -1,77 +1,29 @@ # 외부 api 조회 -from dotenv import load_dotenv -import os -import requests -import time +# 파이프라인에서는 해당 파일만 호출합니다. +from analyzers.external.req_gsb import req_gsb +from analyzers.external.req_otx import req_otx +from analyzers.external.req_whois import req_whois -load_dotenv() +from core.context import AnalysisContext -API_KEY = os.getenv("GOOGLE_SAFE_BROWSING_API_KEY") -#요청 형식 -def build_request_body(url: str): - return { - "client": { - "clientId": "url-analysis-query", - "clientVersion": "1.0" - }, - "threatInfo": { - "threatTypes": [ - "MALWARE", - "SOCIAL_ENGINEERING", - "UNWANTED_SOFTWARE", - "POTENTIALLY_HARMFUL_APPLICATION" - ], - "platformTypes": ["ANY_PLATFORM"], - "threatEntryTypes": ["URL"], - "threatEntries": [ - {"url": url} - ] - } - } -#요청 본문 -def check_url_safe_browsing(url: str): - endpoint = f"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={API_KEY}" +# 외부 api 조회 함수 +def req_api(context: AnalysisContext) -> dict: + target_url = context.url.get("normalized_url") - body = build_request_body(url) - - start = time.time() - - try: - response = requests.post(endpoint, json=body, timeout=5) - elapsed = time.time() - start - - if response.status_code != 200: #### 200이 아님 이 아니라 문서 규약에 따라 400/404/401 등 응답 반환으로 조정필요 - return { - "error": True, - "status_code": response.status_code, - "elapsed": elapsed - } - - data = response.json() - - matches = data.get("matches", []) - - # 전반적으로 if-return이 아니라 exept throwing방식이 더 나을 것 같음 - # 기본 케이스는 조회 실패. - # exept로 200을 포함한 상태코드로 설정 - return { - "matched": len(matches) > 0, - "threat_types": list(set([m["threatType"] for m in matches])) if matches else [], - "elapsed": elapsed, - "raw": data - } - - except requests.exceptions.Timeout: # 이건 상태코드가 설정이 되어있지 않음 - return {"error": "timeout"} - except Exception as e: # 그냥 에러 말고 어떤 에러인지도 좀 써야 할 것 같다. - return {"error": str(e)} + # [1] GSB 파이프라인 + gsb_result = req_gsb(target_url) - -#테스트용 코드 -if __name__ == "__main__": - test_url = "http://malware.testing.google.test/testing/malware/" + # [2] OTX 파이프라인 + otx_result = req_otx(target_url) - result = check_url_safe_browsing(test_url) - print(result) + # [3] WHOIS 파이프라인 + whois_result = req_whois(target_url) + + # [4] 최종 조립 + return{ + "safe_browsing": gsb_result, + "otx": otx_result, + "whois": whois_result + } diff --git a/analyzers/req_cert.py b/analyzers/req_cert.py new file mode 100644 index 0000000..4e1887b --- /dev/null +++ b/analyzers/req_cert.py @@ -0,0 +1,246 @@ +import socket +import ssl + +from core.context import AnalysisContext +from core.config import statusData as stat +from core.config import certErrorData as Err + +CERT_TIMEOUT = 5 + + +def req_cert(context: AnalysisContext) -> dict: + hostname = context.url.get("hostname") + + raw = cert_req_api(hostname) + parsed = parse_cert_result(raw) + + return build_cert_result(parsed) + + +def cert_req_api(hostname: str | None) -> dict: + if not hostname: + return build_raw_cert_failure( + stat.CERT_INVALID_HOST, + Err.CERT_INVALID_HOST + ) + + try: + ssl_context = ssl.create_default_context() + + with socket.create_connection((hostname, 443), timeout=CERT_TIMEOUT) as sock: + with ssl_context.wrap_socket(sock, server_hostname=hostname) as ssock: + cert = ssock.getpeercert() + + return { + "status": stat.SUCCESS, + "data": cert + } + + except (socket.timeout, TimeoutError) as e: + status, error_code = classify_cert_error(e) + return build_raw_cert_failure(status, error_code) + + except ssl.SSLCertVerificationError as e: + status, error_code = classify_cert_error(e) + return build_raw_cert_failure(status, error_code) + + except ssl.SSLError as e: + status, error_code = classify_cert_error(e) + return build_raw_cert_failure(status, error_code) + + except socket.gaierror as e: + status, error_code = classify_cert_error(e) + return build_raw_cert_failure(status, error_code) + + except OSError as e: + status, error_code = classify_cert_error(e) + return build_raw_cert_failure(status, error_code) + + except Exception as e: + status, error_code = classify_cert_error(e) + return build_raw_cert_failure(status, error_code) + + +# 예외 원문을 구체적인 라벨화 +# status_type, error_code +def classify_cert_error(error: Exception) -> tuple[str, str]: + message = str(error).lower() + + if isinstance(error, (socket.timeout, TimeoutError)): + return stat.CERT_TIMEOUT, Err.CERT_TIMEOUT + + if isinstance(error, ssl.SSLCertVerificationError): + if "self-signed certificate" in message: + return stat.CERT_UNTRUSTED, Err.CERT_SELF_SIGNED + + if "expired" in message or "certificate has expired" in message: + return stat.CERT_EXPIRED, Err.CERT_EXPIRED + + if "not yet valid" in message: + return stat.CERT_NOT_YET_VALID, Err.CERT_NOT_YET_VALID + + if ( + "hostname" in message + or "ip address mismatch" in message + or "doesn't match" in message + or "does not match" in message + ): + return stat.CERT_HOSTNAME_MISMATCH, Err.CERT_HOSTNAME_MISMATCH + + if "revoked" in message: + # statusData에는 revoked 전용 상태가 없으므로 큰 분류는 untrusted로 둡니다. + return stat.CERT_UNTRUSTED, Err.CERT_REVOKED + + if ( + "unable to get local issuer certificate" in message + or "unable to verify the first certificate" in message + or "certificate verify failed" in message + or "issuer certificate" in message + ): + return stat.CERT_UNTRUSTED, Err.CERT_UNTRUSTED + + return stat.CERT_UNTRUSTED, Err.CERT_UNTRUSTED + + if isinstance(error, ssl.SSLError): + return stat.CERT_SSL_ERROR, Err.CERT_SSL_ERROR + + if isinstance(error, socket.gaierror): + return stat.CERT_INVALID_HOST, Err.CERT_INVALID_HOST + + if isinstance(error, ConnectionRefusedError): + return stat.CERT_CONNECTION_FAILED, Err.CERT_CONNECTION_FAILED + + if isinstance(error, OSError): + if ( + "name or service not known" in message + or "nodename nor servname provided" in message + or "getaddrinfo failed" in message + ): + return stat.CERT_INVALID_HOST, Err.CERT_INVALID_HOST + + if "connection refused" in message: + return stat.CERT_CONNECTION_FAILED, Err.CERT_CONNECTION_FAILED + + if "network is unreachable" in message or "no route to host" in message: + return stat.CERT_CONNECTION_FAILED, Err.CERT_CONNECTION_FAILED + + return stat.CERT_LOOKUP_FAILED, Err.CERT_LOOKUP_FAILED + + return stat.CERT_UNKNOWN_ERROR, Err.CERT_UNKNOWN_ERROR + + +def build_raw_cert_failure(status: str, error_code: str) -> dict: + return { + "status": status, + "error": error_code + } + + +def parse_cert_result(raw: dict) -> dict: + if not isinstance(raw, dict): + return build_cert_failure( + stat.CERT_UNKNOWN_ERROR, + Err.CERT_UNKNOWN_ERROR + ) + + status = raw.get("status") + + if status != stat.SUCCESS: + return build_cert_failure( + status or stat.CERT_UNKNOWN_ERROR, + normalize_error_code(raw.get("error")) + ) + + cert = raw.get("data") or {} + + if not cert: + return build_cert_failure( + stat.NO_CERTIFICATE, + Err.CERT_NO_CERTIFICATE + ) + + return { + "valid": True, + "issuer": parse_issuer(cert.get("issuer")), + "validFrom": cert.get("notBefore"), + "validTo": cert.get("notAfter"), + "status_type": stat.SUCCESS, + "error": [] + } + + +def normalize_error_code(error: str | None) -> str: + """ + cert_req_api 내부에서 만든 표준 error code만 최종 error에 남깁니다. + raw exception 문자열이 들어오면 CERT_UNKNOWN_ERROR로 치환합니다. + """ + valid_error_codes = { + Err.CERT_SELF_SIGNED, + Err.CERT_UNTRUSTED, + Err.CERT_EXPIRED, + Err.CERT_HOSTNAME_MISMATCH, + Err.CERT_NOT_YET_VALID, + Err.CERT_REVOKED, + Err.CERT_SSL_ERROR, + Err.CERT_INVALID_HOST, + Err.CERT_CONNECTION_FAILED, + Err.CERT_LOOKUP_FAILED, + Err.CERT_TIMEOUT, + Err.CERT_NO_CERTIFICATE, + Err.CERT_UNKNOWN_ERROR + } + + if isinstance(error, str) and error in valid_error_codes: + return error + + return Err.CERT_UNKNOWN_ERROR + + +def build_cert_failure(status: str, error: str | None = None) -> dict: + return { + "valid": False, + "issuer": None, + "validFrom": None, + "validTo": None, + "status_type": status, + "error": [normalize_error_code(error)] + } + + +def build_cert_result(parsed: dict) -> dict: + base = { + "valid": False, + "issuer": None, + "validFrom": None, + "validTo": None, + "status_type": None, + "error": [] + } + + base.update(parsed) + + if not isinstance(base.get("error"), list): + base["error"] = [normalize_error_code(base.get("error"))] + else: + base["error"] = [normalize_error_code(error) for error in base.get("error", [])] + + return base + + +def parse_issuer(issuer_tuple) -> str | None: + if not issuer_tuple: + return None + + flat = {} + + for group in issuer_tuple: + for key, value in group: + flat[key] = value + + if flat.get("organizationName"): + return flat["organizationName"] + + if flat.get("commonName"): + return flat["commonName"] + + return None \ No newline at end of file diff --git a/analyzers/req_db.py b/analyzers/req_db.py deleted file mode 100644 index 0a32ff7..0000000 --- a/analyzers/req_db.py +++ /dev/null @@ -1,38 +0,0 @@ -# 내부 DB 조회 -import requests -import os -import time - -DB_KEY = os.getenv("DB_API_URL") - -def check_url_exists(url: str): - start = time.time() - - try: - response = requests.post( - DB_KEY, - json={"url": url}, - timeout=5 - ) - - elapsed = time.time() - start - - if response.status_code != 200: - return { - "error": True, - "status_code": response.status_code, - "elapsed": elapsed - } - - data = response.json() - - return { - "exists": data.get("exists"), - "id": data.get("id"), - "elapsed": elapsed - } - - except requests.exceptions.Timeout: - return {"error": "timeout"} - except Exception as e: - return {"error": str(e)} diff --git a/analyzers/req_server/__init__.py b/analyzers/req_server/__init__.py new file mode 100644 index 0000000..d8673b8 --- /dev/null +++ b/analyzers/req_server/__init__.py @@ -0,0 +1,5 @@ +from .req_server import req_server + +__all__ = [ + "req_server", +] diff --git a/analyzers/req_server/constants.py b/analyzers/req_server/constants.py new file mode 100644 index 0000000..1987add --- /dev/null +++ b/analyzers/req_server/constants.py @@ -0,0 +1,10 @@ +STATUS_SUCCESS = "success" +STATUS_TIMEOUT = "timeout" +STATUS_REQUEST_FAILED = "request_failed" +STATUS_INVALID_INPUT = "invalid_input" +STATUS_UNEXPECTED_ERROR = "unexpected_error" + +SERVER_INFO_TIMEOUT = 3 +UNKNOWN_SERVER_INFO = "unknown" +UNKNOWN_LOCATION_INFO = "위치 정보 없음" +USER_AGENT = "veri-q-analysis-engine/1.0" diff --git a/analyzers/req_server/location.py b/analyzers/req_server/location.py new file mode 100644 index 0000000..c8b383a --- /dev/null +++ b/analyzers/req_server/location.py @@ -0,0 +1,70 @@ +COUNTRY_NAMES_KO = { + "KR": "대한민국", + "US": "미국", + "JP": "일본", + "CN": "중국", + "GB": "영국", + "DE": "독일", + "FR": "프랑스", + "CA": "캐나다", + "AU": "호주", + "SG": "싱가포르", + "HK": "홍콩", + "TW": "대만", +} + +KR_REGION_NAMES_KO = { + "11": "서울", + "26": "부산", + "27": "대구", + "28": "인천", + "29": "광주", + "30": "대전", + "31": "울산", + "36": "세종", + "41": "경기도", + "42": "강원도", + "43": "충청북도", + "44": "충청남도", + "45": "전라북도", + "46": "전라남도", + "47": "경상북도", + "48": "경상남도", + "49": "제주도", +} + +US_REGION_NAMES_KO = { + "CA": "캘리포니아주", + "IA": "아이오와주", + "NY": "뉴욕주", + "TX": "텍사스주", + "VA": "버지니아주", + "WA": "워싱턴주", +} + + +def format_location_ko(data: dict) -> str: + country_code = data.get("countryCode") + region_code = data.get("region") + country = COUNTRY_NAMES_KO.get(country_code, data.get("country")) + region = format_region_ko(country_code, region_code, data.get("regionName")) + city = data.get("city") + + parts = [] + for part in (country, region, city): + if part and part not in parts: + parts.append(part) + + return " ".join(parts) + + +def format_region_ko( + country_code: str | None, + region_code: str | None, + region_name: str | None, +) -> str | None: + if country_code == "KR": + return KR_REGION_NAMES_KO.get(region_code, region_name) + if country_code == "US": + return US_REGION_NAMES_KO.get(region_code, region_name) + return region_name diff --git a/analyzers/req_server/req_server.py b/analyzers/req_server/req_server.py new file mode 100644 index 0000000..cff7227 --- /dev/null +++ b/analyzers/req_server/req_server.py @@ -0,0 +1,48 @@ +from core.context import AnalysisContext +from core.config import ctxData as ctx + +from .constants import UNKNOWN_SERVER_INFO, UNKNOWN_LOCATION_INFO +from .server_info import fetch_server_type, fetch_server_location +from .server_url_utils import parse_target_host, hostname_idna + + +# server 정보 조회 메인 함수 +def req_server(context: AnalysisContext) -> dict: + hostname = context.url.get(ctx.Url_hostname) + hostname = hostname_idna(hostname) + port = context.url.get(ctx.Url_port) + scheme = context.url.get(ctx.Url_scheme) + + norm_url = context.url.get(ctx.Url_normalized_url) + origin_url = context.request.get(ctx.Req_original_url) + urls = [norm_url, origin_url] + + if not hostname: + hostname, fallback_port = parse_target_host(urls) + port = port or fallback_port + + port = port or 443 + + server_type = fetch_server_type(urls, scheme, hostname, port) + server_location = fetch_server_location(hostname) + server_status = build_server_status(server_type, server_location) + + return { + "server_type": server_type, + "server_location": server_location, + "server_status": server_status, + } + + + +def build_server_status(server_type: str, server_location: str) -> str: + type_ok = server_type != UNKNOWN_SERVER_INFO + location_ok = server_location != UNKNOWN_LOCATION_INFO + + if type_ok and location_ok: + return "success" + + if type_ok or location_ok: + return "partial" + + return "failed" diff --git a/analyzers/req_server/server_info.py b/analyzers/req_server/server_info.py new file mode 100644 index 0000000..631a880 --- /dev/null +++ b/analyzers/req_server/server_info.py @@ -0,0 +1,87 @@ +import socket +import urllib3 +import requests + +from .constants import ( + SERVER_INFO_TIMEOUT, + UNKNOWN_LOCATION_INFO, + UNKNOWN_SERVER_INFO, + USER_AGENT, +) +from .location import format_location_ko +from .server_url_utils import build_probe_urls + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + + + +# 서버 타입을 조회합니다. context 수정x +# 반환: server_type 문자열 +def fetch_server_type(urls:list, scheme, hostname: str | None, port: int | str | None) -> str: + # context에 저장되어 있는 정규화된 hostname를 받아서 사용합니다. + + if not hostname: + return UNKNOWN_SERVER_INFO + + for url in build_probe_urls(urls, scheme, hostname, port): + try: + response = requests.head( + url, + allow_redirects=True, + headers={"User-Agent": USER_AGENT}, + timeout=SERVER_INFO_TIMEOUT, + verify=False, + ) + server_type = response.headers.get("Server") + response.close() + + if server_type: + return server_type.strip() + + except requests.exceptions.RequestException: + continue + + return UNKNOWN_SERVER_INFO + + +# 서버 위치를 조회합니다. +# 반환: server_location 문자열 +def fetch_server_location(hostname: str | None) -> str: + if not hostname: + return UNKNOWN_LOCATION_INFO + + ip_address = resolve_ip_address(hostname) + if not ip_address: + return UNKNOWN_LOCATION_INFO + + try: + response = requests.get( + f"http://ip-api.com/json/{ip_address}", + params={"fields": "status,country,countryCode,region,regionName,city,query"}, + timeout=SERVER_INFO_TIMEOUT, + ) + data = response.json() + response.close() + + if data.get("status") == "success": + return format_location_ko(data) or ip_address + + except (requests.exceptions.RequestException, ValueError): + pass + + return ip_address + + +def resolve_ip_address(hostname: str) -> str | None: + try: + address_info = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM) + except OSError: + return None + + for family in (socket.AF_INET, socket.AF_INET6): + for info in address_info: + if info[0] == family: + return info[4][0] + + return None diff --git a/analyzers/req_server/server_url_utils.py b/analyzers/req_server/server_url_utils.py new file mode 100644 index 0000000..e8e7036 --- /dev/null +++ b/analyzers/req_server/server_url_utils.py @@ -0,0 +1,110 @@ +from urllib.parse import urlsplit + + + +def parse_target_host(urls: list) -> tuple[str | None, int | None]: + for candidate_url in urls or []: + hostname, port = parse_host_from_url(candidate_url) + if hostname: + return hostname_idna(hostname), port + + return None, None + + +def parse_host_from_url(url: str | None) -> tuple[str | None, int | None]: + if not isinstance(url, str): + return None, None + + candidate = url.strip() + if not candidate: + return None, None + + split_result = urlsplit(candidate) + if not split_result.hostname: + split_result = urlsplit(f"https://{candidate}") + + try: + port = split_result.port + except ValueError: + port = None + + return split_result.hostname, port + + +# hostname기준 HEAD 요청을 던질 URL 리스트 제작 함수 +def build_probe_urls(urls: list, scheme: str | None, hostname: str, port: int | str | None) -> list[str]: + result = [] + + for candidate_url in urls or []: + url = normalize_probe_url(candidate_url) + candidate_hostname, _ = parse_host_from_url(url) + + if hostname_idna(candidate_hostname) == hostname: + result.append(url) + + netloc = build_netloc(hostname, port) + + if scheme in ("http", "https"): + result.append(f"{scheme}://{netloc}") + + result.append(f"https://{netloc}") + result.append(f"http://{netloc}") + + return unique_values(result) + + +def normalize_probe_url(url: str | None) -> str | None: + if not isinstance(url, str): + return None + + candidate = url.strip() + if not candidate: + return None + + if not urlsplit(candidate).hostname: + candidate = f"https://{candidate}" + + return candidate + + +def build_netloc(hostname: str, port: int | str | None) -> str: + normalized_port = normalize_port(port) + if normalized_port and normalized_port not in (80, 443): + return f"{hostname}:{normalized_port}" + return hostname + + +# 기능 혼동 방지 위한 명칭 변경 +def hostname_idna(hostname: str | None) -> str | None: + if not isinstance(hostname, str): + return None + + normalized = hostname.strip().strip("[]").rstrip(".").lower() + if not normalized: + return None + + try: + return normalized.encode("idna").decode("ascii") + except UnicodeError: + return None + + +def normalize_port(port: int | str | None) -> int | None: + try: + normalized = int(port or 443) + except (TypeError, ValueError): + return None + + if 1 <= normalized <= 65535: + return normalized + return None + + +def unique_values(values: list[str | None]) -> list[str]: + unique = [] + + for value in values: + if value and value not in unique: + unique.append(value) + + return unique diff --git a/analyzers/safe_browsing/__init__.py b/analyzers/safe_browsing/__init__.py new file mode 100644 index 0000000..350f4ba --- /dev/null +++ b/analyzers/safe_browsing/__init__.py @@ -0,0 +1,19 @@ +# analyzers/external_analysis/__init__.py + +from .google_api import GoogleSafeBrowsing +from .schemas import SafeBrowsingResult + +# 1. 인스턴스를 미리 생성하여 '싱글톤'처럼 활용합니다. +# 이렇게 하면 호출할 때마다 매번 객체를 생성할 필요가 없어 효율적입니다. +_analyzer = GoogleSafeBrowsing() + +# 2. 외부(main.py)에서 사용할 공식 함수를 정의합니다. +async def check_url_safety(url: str) -> dict: + """ + URL의 안전성을 검사하여 BE3가 설계한 DB 규격에 맞는 결과를 반환합니다. + 반환 필드: apiChecked, apiProvider, apiResult + """ + return await _analyzer.analyze(url) + +# 3. 'from external_analysis import *' 를 했을 때 노출될 항목을 제한합니다. (캡슐화) +__all__ = ["check_url_safety"] \ No newline at end of file diff --git a/analyzers/safe_browsing/exceptions.py b/analyzers/safe_browsing/exceptions.py new file mode 100644 index 0000000..834d2f7 --- /dev/null +++ b/analyzers/safe_browsing/exceptions.py @@ -0,0 +1,28 @@ +# analyzers/external_analysis/exceptions.py + +class SafeBrowsingError(Exception): + """구글 세이프 브라우징 모듈의 기본 예외 클래스입니다.""" + def __init__(self, message="분석 중 오류가 발생했습니다.", status_code=500): + self.message = message + self.status_code = status_code + super().__init__(self.message) + +class InvalidAPIKeyError(SafeBrowsingError): + """API 키가 잘못되었거나 인증에 실패했을 때 발생합니다.""" + def __init__(self, message="Google API 키 인증에 실패했습니다."): + super().__init__(message, status_code=401) + +class QuotaExceededError(SafeBrowsingError): + """API 호출 할당량을 초과했을 때 발생합니다.""" + def __init__(self, message="API 호출 할당량을 초과했습니다. 무료 티어 확인이 필요합니다."): + super().__init__(message, status_code=429) + +class AnalysisTimeoutError(SafeBrowsingError): + """외부 API 응답이 너무 늦어 타임아웃이 발생했을 때 사용합니다.""" + def __init__(self, message="Google API 응답 시간이 초과되었습니다."): + super().__init__(message, status_code=504) + +class ServiceUnavailableError(SafeBrowsingError): + """구글 서버 자체에 문제가 있어 응답을 받을 수 없을 때 발생합니다.""" + def __init__(self, message="현재 Google 분석 서비스를 사용할 수 없습니다."): + super().__init__(message, status_code=503) \ No newline at end of file diff --git a/analyzers/safe_browsing/google_api.py b/analyzers/safe_browsing/google_api.py new file mode 100644 index 0000000..8376af7 --- /dev/null +++ b/analyzers/safe_browsing/google_api.py @@ -0,0 +1,92 @@ +import httpx +import os +import logging +from dotenv import load_dotenv + +# .env 파일에서 환경 변수 로드 +load_dotenv() + +# 로그 설정 (운영 환경에서의 트래킹을 위함) +logger = logging.getLogger(__name__) + +class GoogleSafeBrowsing: + def __init__(self): + self.api_key = os.getenv("GOOGLE_SAFE_BROWSING_KEY") + self.api_url = f"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={self.api_key}" + self.provider_name = "Google Safe Browsing" + + async def analyze(self, target_url: str) -> dict: + """ + URL을 분석하여 DB 엔티티 구조에 맞는 결과를 반환합니다. + """ + # API 키가 없을 경우의 방어 로직 + if not self.api_key: + logger.error("Google API Key가 설정되지 않았습니다.") + return { + "apiChecked": False, + "apiProvider": self.provider_name, + "apiResult": "CONFIG_ERROR" + } + + # Google API 요청 페이로드 구성 + payload = { + "client": { + "clientId": "veri-q-backend", + "clientVersion": "1.0.0" + }, + "threatInfo": { + "threatTypes": ["MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION"], + "platformTypes": ["ANY_PLATFORM"], + "threatEntryTypes": ["URL"], + "threatEntries": [{"url": target_url}] + } + } + + try: + # 비동기 HTTP 클라이언트를 사용한 호출 + async with httpx.AsyncClient() as client: + response = await client.post( + self.api_url, + json=payload, + timeout=5.0 # 외부 API 지연으로 인한 서버 중단 방지 + ) + + if response.status_code == 200: + result_data = response.json() + # Google API는 위협 발견 시 'matches' 리스트를 반환함 + if "matches" in result_data: + threat_type = result_data["matches"][0]["threatType"] + return { + "apiChecked": True, + "apiProvider": self.provider_name, + "apiResult": f"DANGER: {threat_type}" + } + else: + return { + "apiChecked": True, + "apiProvider": self.provider_name, + "apiResult": "SAFE" + } + + # 200 OK가 아닌 경우 (할당량 초과, 잘못된 요청 등) + logger.warning(f"Google API Error: {response.status_code}") + return { + "apiChecked": False, + "apiProvider": self.provider_name, + "apiResult": f"API_ERROR_{response.status_code}" + } + + except httpx.TimeoutException: + logger.error("Google API 호출 타임아웃 발생") + return { + "apiChecked": False, + "apiProvider": self.provider_name, + "apiResult": "TIMEOUT" + } + except Exception as e: + logger.error(f"Google API 분석 중 예기치 못한 에러: {str(e)}") + return { + "apiChecked": False, + "apiProvider": self.provider_name, + "apiResult": "INTERNAL_EXCEPTION" + } diff --git a/analyzers/safe_browsing/schemas.py b/analyzers/safe_browsing/schemas.py new file mode 100644 index 0000000..b3e4c41 --- /dev/null +++ b/analyzers/safe_browsing/schemas.py @@ -0,0 +1,31 @@ +# analyzers/external_analysis/schemas.py + +from pydantic import BaseModel, Field + +class SafeBrowsingResult(BaseModel): + """ + Google Safe Browsing 분석 결과를 담는 스키마입니다. + BE3가 설계한 DB 엔티티(ExternalApiInfo) 구조와 1:1 매핑됩니다. + """ + apiChecked: bool = Field( + ..., + description="외부 API 조회 성공 여부" + ) + apiProvider: str = Field( + default="Google Safe Browsing", + description="API 제공자 이름" + ) + apiResult: str = Field( + ..., + description="분석 결과 메시지 (예: SAFE, DANGER: MALWARE 등)" + ) + + class Config: + # JSON 결과 예시를 명시하여 Swagger 문서화를 돕습니다. + schema_extra = { + "example": { + "apiChecked": True, + "apiProvider": "Google Safe Browsing", + "apiResult": "SAFE" + } + } \ No newline at end of file diff --git a/analyzers/url_https_checker.py b/analyzers/url_https_checker.py deleted file mode 100644 index accda83..0000000 --- a/analyzers/url_https_checker.py +++ /dev/null @@ -1,3 +0,0 @@ -# 프로토콜-스킴 파트 분석. -# https 여부 확인이 주요 기능 -# 이건 그래도 파이프라인에 집어넣을거 생각하면 클래스 분리되어 있는 편이 나을듯 diff --git a/analyzers/url_shorten_checker.py b/analyzers/url_shorten_checker.py index e5397d1..b933e61 100644 --- a/analyzers/url_shorten_checker.py +++ b/analyzers/url_shorten_checker.py @@ -1 +1,73 @@ -# 단축 URL 여부 검사. 단, 해당 부분은 가장 먼저 검사하므로 파이프라인 내지 메인에 해당 코드가 삽입될 가능성 존재함 +# 단축 URL 여부 검사 +# 정규화된 Hostname을 받아 단축 URL 서비스 도메인과 일치 시, 단축 URL로 판별합니다. +from core.context import AnalysisContext + + +# 메인 함수 +# hostname이 단축 URL 서비스 도메인인지 검사 +def url_shorten_checker(context: AnalysisContext) -> dict: + + hostname = context.url.get("hostname") + return build_shorten_result(hostname) + + +# 실 검사 로직 +def check_shorten_domain(hostname: str | None) -> dict: + if not hostname: + return { + "detected": False, + "matched": "" + } + + if hostname in SHORTENER_DOMAINS: + return { + "detected": True, + "matched": hostname + } + + return { + "detected": False, + "matched": "" + } + + +# 결과 조립 +def build_shorten_result(hostname: str | None) -> dict: + result = { + "detected": False, + "evidence": [], + "details": { + "matched_shortener": "" + } + } + + check = check_shorten_domain(hostname) + + if check["detected"]: + result["detected"] = True + result["evidence"].append(check["matched"]) + result["details"]["matched_shortener"] = check["matched"] + + return result + + +# 단축 URL 서비스 도메인 목록 +SHORTENER_DOMAINS = { + "bit.ly", + "tinyurl.com", + "t.co", + "goo.gl", + "ow.ly", + "is.gd", + "buff.ly", + "adf.ly", + "bit.do", + "cutt.ly", + "url.kr", + "c11.kr", + "han.gl", + "me2.do", + "vo.la", + "lc.chat", + "buly.kr" +} \ No newline at end of file diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/base.py b/core/base.py deleted file mode 100644 index 795ef73..0000000 --- a/core/base.py +++ /dev/null @@ -1,2 +0,0 @@ -# 전체 검사기의 기본 구조 및 호출 형식을 통일하는 기반입니다. -# 한 번 정의하면 수정 X diff --git a/core/config.py b/core/config.py new file mode 100644 index 0000000..84079c5 --- /dev/null +++ b/core/config.py @@ -0,0 +1,132 @@ +# 오타 방지를 위한 상수화 클래스들을 한 곳에서 관리하는 파일입니다. +# from core.config import (사용할 class명) + + +# json의 result 라벨 +class riskLevelData: + THREAT = "THREAT" + CAUTION = "CAUTION" + SAFE = "SAFE" + + +# 상태 데이터 목록 +class statusData: + SUCCESS = "success" + TIMEOUT = "timeout" + REQUEST_FAILED = "request_failed" + INVALID_INPUT = "invalid_input" + CLIENT_ERROR = "client_error" + MISSING_API_KEY = "missing_api_key" + + NO_CERTIFICATE = "no_certificate" + UNEXPECTED_ERROR = "unexpected_error" + VERIFY_FAILED = "verify_failed" + SSL_ERROR = "ssl_error" + + # 인증서 신뢰 불가한 경우 + CERT_UNTRUSTED = "cert_untrusted" + CERT_EXPIRED = "cert_expired" + CERT_HOSTNAME_MISMATCH = "cert_hostname_mismatch" + CERT_NOT_YET_VALID = "cert_not_yet_valid" + CERT_LOOKUP_FAILED = "cert_lookup_failed" + CERT_TIMEOUT = "cert_timeout" + CERT_INVALID_HOST = "cert_invalid_host" + CERT_UNKNOWN_ERROR = "cert_unknown_error" + CERT_SSL_ERROR = "cert_ssl_error" + CERT_CONNECTION_FAILED = "cert_connection_failed" + +# 인증서 error 라벨 +class certErrorData: + CERT_SELF_SIGNED = "CERT_SELF_SIGNED" + CERT_UNTRUSTED = "CERT_UNTRUSTED" + CERT_EXPIRED = "CERT_EXPIRED" + CERT_HOSTNAME_MISMATCH = "CERT_HOSTNAME_MISMATCH" + CERT_NOT_YET_VALID = "CERT_NOT_YET_VALID" + CERT_REVOKED = "CERT_REVOKED" + CERT_SSL_ERROR = "CERT_SSL_ERROR" + CERT_INVALID_HOST = "CERT_INVALID_HOST" + CERT_CONNECTION_FAILED = "CERT_CONNECTION_FAILED" + CERT_LOOKUP_FAILED = "CERT_LOOKUP_FAILED" + CERT_TIMEOUT = "CERT_TIMEOUT" + CERT_NO_CERTIFICATE = "CERT_NO_CERTIFICATE" + CERT_UNKNOWN_ERROR = "CERT_UNKNOWN_ERROR" + +# context 접근 시 필드명 목록 +# !!context 변경 시 반드시 함께 변경해야 합니다!! +class ctxData: + Req_Uuid = "guest_Uuid" + Req_original_url = "original_url" + + Url_normalized_url = "normalized_url" + Url_scheme = "scheme" + Url_hostname = "hostname" + Url_port = "port" + Url_path = "path" + Url_query = "query" + Url_fragment = "fragment" + Url_is_https = "is_https" + Url_def_scheme = "has_explicit_scheme" + Url_def_port = "is_default_port" + + Red_count = "redirect_count" + Red_final_url = "final_url" + Red_chain = "redirect_chain" + Red_loop_det = "loop_detected" + Red_status = "status_type" + Red_result = "result" + + # detection + Det_short = "shortened_url" + Short_det = "detected" + Short_evd = "evidence" + + Det_rule = "rule_based" + Rule_flags = "flags" + Rule_Squery = "suspicious_query_params" + Rule_emedded_url = "embedded_urls" + Rule_Skeywords = "suspicious_keywords" + + # ml + Ml_score = "score" + Ml_threats = "threats" + + # scoring + Sco_score = "score" + Sco_riskLv = "risk_level" + Sco_reason = "reason_summary" + Sco_matched = "matched_rules" + + # external + Ex_server_type = "server_type" + Ex_server_loc = "server_location" + Ex_server_status = "server_status" + + EX_GSB = "safe_browsing" + GSB_is_threat = "is_threat" + GSB_status = "status_type" + GSB_result = "result_type" + GSB_threats = "threat_labels" + + EX_OTX = "otx" + OTX_status = "status" + OTX_is_threat = "is_threat" + OTX_pulse_count = "pulse_count" + OTX_threats = "threat_labels" + + EX_WHOIS = "whois" + WHO_status = "status" + WHO_date = "creation_date" + WHO_age = "domain_age_days" + + # certification + Cert_valid = "valid" + Cert_issuer = "issuer" + Cert_validFrom = "validFrom" + Cert_validTo = "validTo" + Cert_status = "status_type" + Cert_error = "error" + + # meta + Meta_status = "status" + Meta_steps = "completed_steps" + Meta_warn = "warnings" \ No newline at end of file diff --git a/core/context.py b/core/context.py index 0eb699b..9329e86 100644 --- a/core/context.py +++ b/core/context.py @@ -1,2 +1,113 @@ # 검사의 내용을 실시간 기록하는 페이지입니다. # 공유 노트 -> 검사 시 필요한 데이터는 여기에 임시 저장, 호출하는 것을 규약으로 합니다. +# 검사 요청 수신 시, 반드시 create_initial_context(scan_id, original_url)를 호출해 context를 생성합니다. +from dataclasses import dataclass, field +from typing import Any, Dict +# flags 삭제, info에 단계 및 기타 정보 일괄 저장하는 걸로. +https: dict = field(default_factory=lambda: { + "isSecure": False + }) + +@dataclass +class AnalysisContext: # key : gusetUuid, originalURL (request에만 저장됨) + request: Dict[str, Any] = field(default_factory=dict) # 수신한 요청 + url: Dict[str, Any] = field(default_factory=dict) # 정규화 거친 url + redirect: Dict[str, Any] = field(default_factory=dict) # 리다이렉트 추적 결과 + detection: Dict[str, Any] = field(default_factory=dict) # 룰 기반 검사기+단축 url + ml: Dict[str, Any] = field(default_factory=dict) # ml검사결과+ml점수 + external: Dict[str, Any] = field(default_factory=dict) # safe browsing/whois api + certificate: Dict[str, Any] = field(default_factory=dict) # 인증서 조회 + scoring: Dict[str, Any] = field(default_factory=dict) # 스코어링 결과 저장 + meta: Dict[str, Any] = field(default_factory=dict) # 실행 단계, 내부 errors 또는 기타 info + + +def create_initial_context(guest_Uuid: str, original_url: str) -> AnalysisContext: + return AnalysisContext( + request={ + "guest_Uuid": guest_Uuid, + "original_url": original_url + }, + url={ + "normalized_url": None, + "scheme": None, + "hostname": None, + "port": None, + "path": None, + "query": None, + "fragment": None, + "is_https": False, + "has_explicit_scheme": False, #-> 좀 더 쉽게 변경예정 + "is_default_port": False, + }, + redirect={ + "redirect_count": 0, + "final_url": None, + "redirect_chain": [], + "loop_detected": False, + "status_type": [], #status code, 상태 상수 + "result": None # str로 success/fail 저장 + }, + detection={ + "shortened_url": { + "detected": False, + "evidence": [] + }, + "rule_based": { + "flags": [], + "suspicious_query_params": [], + "embedded_urls": [], + "suspicious_keywords": { + "path": [], + "query": [], + "fragment": [], + "hostname": [] + }, + } + }, + ml = { + "score": None, + "threats": [] + }, + scoring={ + "score": None, + "risk_level": None, + "reason_summary": [], + "matched_rules": [], + }, + external={ # otx, whois 필드 추가, serverinfo 필드 추가 + "server_type": None, + "server_location": None, + "server_status": None, + "safe_browsing": { + "is_threat": False, + "status_type": None, + "result_type": None, + "threat_labels": [] + }, + "otx": { + "status": None, + "is_threat": False, + "pulse_count": 0, + "threat_labels": [] + }, + "whois": { + "status": None, + "creation_date": None, + "domain_age_days": 0 + } + }, + certificate = { + "valid": False, + "issuer": None, + "validFrom": None, + "validTo": None, + "status_type": None, + "error": [] + }, + meta={ + "status": "initialized", + "completed_steps": [], + "warnings": [], + "errors": [], + } + ) \ No newline at end of file diff --git a/core/ex_pipeline.py b/core/ex_pipeline.py new file mode 100644 index 0000000..4c83046 --- /dev/null +++ b/core/ex_pipeline.py @@ -0,0 +1,368 @@ +"""시연용 강제 위험 파이프라인. + +특정 원본 URL 1~2개만 기존 네트워크 검사 흐름을 타지 않고, +정규화 이후 검사 결과 context를 직접 채워서 위험 결과를 반환한다. +운영용 로직이 아니라 시연 안정화용 임시 분기다. +""" +import time +from typing import Callable + +from core.context import AnalysisContext +from core.config import statusData as stat, riskLevelData as lev +from services import builder, scoring, url_utils as utils + + +DEMO_RISK_CASES = { + "paypal.auth-sharefile.com": { + "display_name": "paypal_auth_sharefile_demo", + "rule_flags": [ + "suspicious_query_keyword_detected", + ], + "keywords": { + "hostname": ["auth"], + "path": [], + "query": ["auth"], + "fragment": [], + }, + # 실제 일반 pipeline 결과와 맞춘 표시용 값 + "server_type": "unknown", + "server_location": "Malaysia Kuala Lumpur", + "certificate": { + "valid": False, + "issuer": None, + "validFrom": None, + "validTo": None, + "status_type": "cert_timeout", + "error": ["CERT_TIMEOUT"], + }, + "domain_age_days": 266, + "ml_score": 60, + "ml_threats": ["CHARCNN:SUSPICIOUS_URL_PATTERN"], + "charcnn": { + "available": True, + "target_url": "https://paypal.auth-sharefile.com", + "score": 100, + "probability": 0.9997966885566711, + "label": "malicious", + "threshold": 0.5, + "threats": ["CHARCNN:SUSPICIOUS_URL_PATTERN"], + "error": None, + }, + "xgb": { + "available": True, + "score": 39, + "probability": 0.38680291175842285, + "label": "normal", + "raw_label": 0, + "threshold": 0.5, + "threats": [], + "error": None, + }, + }, + "g00gle-account-verify.com": { + "display_name": "g00gle_account_verify_demo", + "rule_flags": [ + "suspicious_query_keyword_detected", + ], + "keywords": { + "hostname": ["account", "verify"], + "path": [], + "query": ["account", "verify"], + "fragment": [], + }, + + # 일반 pipeline 결과 반영값 + "server_type": "unknown", + "server_location": "위치 정보 없음", + "domain_age_days": None, + + "certificate": { + "valid": False, + "issuer": None, + "validFrom": None, + "validTo": None, + "status_type": stat.SUCCESS, + "error": ["CERT_INVALID_HOST"], + }, + "charcnn": { + "available": True, + "target_url": "https://g00gle-account-verify.com", + "score": 100, + "probability": 0.99993896484375, + "label": "malicious", + "threshold": 0.5, + "threats": ["CHARCNN:SUSPICIOUS_URL_PATTERN"], + "error": None + }, + "xgb": { + "available": True, + "score": 4, + "probability": 0.03806363791227341, + "label": "normal", + "raw_label": 0, + "threshold": 0.5, + "threats": [], + "error": None + }, + "fusion": { + "available": True, + "method": "weighted_average", + "weights": {"xgb": 0.35, "charcnn": 0.65}, + "contributors": ["xgb", "charcnn"], + "score": 65 + } + } +} + +DEMO_STEP_DELAY_SECONDS = 0.5 + +ProgressCallback = Callable[[AnalysisContext, str, str, str], None] + + +def try_run_demo_risk_pipeline( + context: AnalysisContext, + progress_callback: ProgressCallback | None = None, +) -> dict | None: + """시연용 강제 위험 URL이면 최종 JSON을 반환하고, 아니면 None을 반환한다.""" + case = _match_demo_risk_case(context) + if case is None: + return None + + context.meta["demo_forced_risk"] = True + context.meta["demo_case"] = case.get("display_name") + + _progress(progress_callback, context, "DEMO_RISK_PIPELINE", "IN_PROGRESS", "시연용 위험 URL 분기를 실행하고 있습니다.") + + _progress(progress_callback, context, "SHORT_URL_CHECK", "IN_PROGRESS", "단축 URL 여부를 확인하고 있습니다.") + _fill_short_url_result(context) + _progress(progress_callback, context, "SHORT_URL_CHECK", "COMPLETED", "단축 URL 여부 확인이 완료되었습니다.") + + _progress(progress_callback, context, "URL_NORMALIZE", "IN_PROGRESS", "URL을 정규화하고 있습니다.") + utils.normalize_url(context) + _progress(progress_callback, context, "URL_NORMALIZE", "COMPLETED", "URL 정규화가 완료되었습니다.") + + _progress(progress_callback, context, "API_REQ", "IN_PROGRESS", "시연용 외부/인증서 검사 결과를 구성하고 있습니다.") + _fill_external_result(context, case) + _fill_certificate_result(context, case) + _progress(progress_callback, context, "API_REQ", "COMPLETED", "외부/인증서 검사 결과 구성이 완료되었습니다.") + + _progress(progress_callback, context, "RULE_ANALYSIS", "IN_PROGRESS", "도메인 및 파라미터 규칙 분석을 진행하고 있습니다.") + _fill_rule_result(context, case) + _progress(progress_callback, context, "RULE_ANALYSIS", "COMPLETED", "도메인 및 파라미터 규칙 분석이 완료되었습니다.") + + _progress(progress_callback, context, "REDIRECT_TRACE", "IN_PROGRESS", "리다이렉트 경로를 추적하고 있습니다.") + _fill_redirect_result(context) + _progress(progress_callback, context, "REDIRECT_TRACE", "COMPLETED", "리다이렉트 추적이 완료되었습니다.") + + _progress(progress_callback, context, "ML_ANALYSIS", "IN_PROGRESS", "ML 기반 위험 분석을 진행하고 있습니다.") + _fill_ml_result(context, case) + _progress(progress_callback, context, "ML_ANALYSIS", "COMPLETED", "ML 기반 위험 분석이 완료되었습니다.") + + _progress(progress_callback, context, "SCORING", "IN_PROGRESS", "최종 위험도를 계산하고 있습니다.") + context.scoring = scoring.score_analysis(context) + _ensure_threat_scoring(context, case) + _progress(progress_callback, context, "SCORING", "COMPLETED", "최종 위험도 계산이 완료되었습니다.") + + _progress(progress_callback, context, "DEMO_RISK_PIPELINE", "COMPLETED", "시연용 위험 URL 결과 생성이 완료되었습니다.") + + result = builder.build_result(context) + return builder.build_JSON(result) + + +def _match_demo_risk_case(context: AnalysisContext) -> dict | None: + """context.url["hostname"] 기준으로 시연용 위험 케이스를 찾는다. + + 앱/BE 경유 시 원본 URL 문자열은 /, scheme, 인코딩 차이로 달라질 수 있으므로 + 원본 URL 완전 일치가 아니라 pipeline 초반에 확보한 hostname으로만 매칭한다. + """ + hostname = _normalize_hostname(context.url.get("hostname")) + if not hostname: + return None + + return DEMO_RISK_CASES.get(hostname) + + +def _normalize_hostname(hostname: object) -> str: + return str(hostname or "").strip().lower().strip(".") + + +def _fill_short_url_result(context: AnalysisContext) -> None: + context.detection["shortened_url"] = { + "detected": False, + "evidence": [], + "details": {}, + } + + +def _fill_external_result(context: AnalysisContext, case: dict) -> None: + context.external.update({ + "server_type": case.get("server_type", "demo-server"), + "server_location": case.get("server_location", "demo-location"), + "server_status": stat.SUCCESS, + "safe_browsing": { + "is_threat": False, + "status_type": None, + "result_type": None, + "threat_labels": [], + }, + "otx": { + "status": None, + "is_threat": False, + "pulse_count": 0, + "threat_labels": [], + }, + "whois": { + "status": stat.SUCCESS, + "creation_date": None, + "domain_age_days": case.get("domain_age_days", 0), + }, + }) + + +def _fill_certificate_result(context: AnalysisContext, case: dict) -> None: + # 케이스별 인증서 표시값이 있으면 그대로 사용하고, + # 없으면 악성 URL도 정상 인증서를 가질 수 있다는 전제로 성공값을 넣는다. + certificate = case.get("certificate") + if isinstance(certificate, dict): + context.certificate = dict(certificate) + return + + context.certificate = { + "valid": True, + "issuer": "Demo CA", + "validFrom": None, + "validTo": None, + "status_type": stat.SUCCESS, + "error": [], + } + + +def _fill_rule_result(context: AnalysisContext, case: dict) -> None: + context.detection["rule_based"] = { + "flags": list(case.get("rule_flags", [])), + "suspicious_query_params": ["redirect", "return_url"], + "embedded_urls": [], + "suspicious_keywords": case.get("keywords", { + "path": [], + "query": [], + "fragment": [], + "hostname": [] + }), + } + + +def _fill_redirect_result(context: AnalysisContext) -> None: + normalized_url = context.url.get("normalized_url") or context.request.get("original_url") + context.redirect = { + "was_checked": True, + "redirect_count": 0, + "final_url": normalized_url, + "redirect_chain": [normalized_url] if normalized_url else [], + "loop_detected": False, + "status_type": [200, stat.SUCCESS], + "result": stat.SUCCESS, + } + + +def _fill_ml_result(context: AnalysisContext, case: dict) -> None: + charcnn = case.get("charcnn") + if isinstance(charcnn, dict): + charcnn_result = dict(charcnn) + else: + charcnn_result = { + "available": True, + "score": _clamp_score(case.get("charcnn_score", case.get("ml_score", 96))), + "label": "malicious", + "threats": ["CHARCNN:SUSPICIOUS_URL_PATTERN"], + } + + xgb = case.get("xgb") + if isinstance(xgb, dict): + xgb_result = dict(xgb) + else: + # XGB 표시값이 없는 케이스도 오류 없이 동작하게 한다. + xgb_result = { + "available": False, + "score": 0, + "label": "unknown", + "threats": [], + } + + fusion = case.get("fusion") + if isinstance(fusion, dict): + fusion_result = dict(fusion) + else: + fallback_score = case.get("ml_score", charcnn_result.get("score", 96)) + fusion_result = { + "available": True, + "method": case.get("fusion_method", "demo_forced_risk"), + "weights": {"xgb": 0.0, "charcnn": 1.0} if not xgb_result.get("available") else {"xgb": 0.65, "charcnn": 0.35}, + "contributors": ["charcnn"] if not xgb_result.get("available") else ["xgb", "charcnn"], + "score": _clamp_score(fallback_score), + } + + ml_score = _clamp_score(case.get("ml_score", fusion_result.get("score", charcnn_result.get("score", 96)))) + + if "ml_threats" in case: + ml_threats = list(case.get("ml_threats") or []) + else: + ml_threats = _collect_demo_ml_threats(charcnn_result, xgb_result, ml_score) + + context.ml = { + "score": ml_score, + "threats": ml_threats, + "charcnn": charcnn_result, + "xgb": xgb_result, + "fusion": fusion_result, + } + + +def _collect_demo_ml_threats(charcnn_result: dict, xgb_result: dict, ml_score: int) -> list[str]: + # 일반 ai_analyze.py처럼 최상위 ml.threats는 fusion/최종 ML 점수 기준으로 둔다. + if ml_score < 50: + return [] + + threats: list[str] = [] + for result in (charcnn_result, xgb_result): + if not isinstance(result, dict): + continue + for threat in result.get("threats", []) or []: + if threat not in threats: + threats.append(threat) + + return threats + + +def _ensure_threat_scoring(context: AnalysisContext, case: dict) -> None: + # scoring.score_analysis()를 정상 호출한 뒤, 혹시 정책 변경으로 THREAT 미만이 되면 + # 시연 URL 목적에 맞게 최소 위험 점수만 보장한다. + score = _clamp_score(context.scoring.get("score", 0)) + if score >= 65 and context.scoring.get("risk_level") == lev.THREAT: + return + + context.scoring = scoring.build_scoring_result( + internal_score=_clamp_score(context.scoring.get("internal_score", 0)), + ml_score=_clamp_score(context.scoring.get("ml_score", case.get("ml_score", 96))), + external_score=_clamp_score(context.scoring.get("external_score", 0)), + score=max(score, 90), + risk_level=lev.THREAT, + reason_summary=list(context.scoring.get("reason_summary", [])) + ["demo_forced_risk"], + matched_rules=list(context.scoring.get("matched_rules", [])) + ["DEMO:FORCED_MALICIOUS_URL"], + ) + + +def _progress(progress_callback: ProgressCallback | None,context: AnalysisContext,step: str,status: str,message: str) -> None: + if progress_callback is not None: + progress_callback(context, step, status, message) + + if status == "IN_PROGRESS" and DEMO_STEP_DELAY_SECONDS > 0: + time.sleep(DEMO_STEP_DELAY_SECONDS) + + +def _clamp_score(value) -> int: + try: + score = int(round(float(value))) + except (TypeError, ValueError): + score = 0 + return max(0, min(100, score)) + diff --git a/core/pipeline.py b/core/pipeline.py index 7dbd54a..db17443 100644 --- a/core/pipeline.py +++ b/core/pipeline.py @@ -1,2 +1,96 @@ # 선행 검사 결과에 따라 변동되는 검사 과정 세트를 지정합니다. -# 예시: 단축 url 여부에 따라 갈리는 검사 진행 과정을 각각 정의 +from analyzers import ( + parameter_rule as rule, + redirect_tracker as track, + url_shorten_checker as short, + ai_analyze as ai, + req_api as api, + req_cert as cert, +) +from analyzers.req_server import req_server +from services import ( + builder, + scoring, + url_utils as utils, + response as callback +) +from core import ex_pipeline + + + +def run_pipeline(context): + _send_progress(context, "TEST", "IN_PROGRESS", "파이프라인 진입 성공") + + # 단축 URL 검사 전에 hostname만 먼저 확보 + raw_url = context.request.get("original_url") + cleaned_url = utils.sanitize_input_url(raw_url) + parsed_result = utils.parse_url_components(cleaned_url) + + if not parsed_result.get("hostname"): + parsed_result = utils.parse_url_components(f"https://{cleaned_url}") + + context.url["sanitized_url"] = cleaned_url + context.url["hostname"] = (parsed_result.get("hostname") or "").lower() + + demo_result = ex_pipeline.try_run_demo_risk_pipeline(context, progress_callback=_send_progress) + if demo_result is not None: + return demo_result #시연 케이스 + + _send_progress(context, "SHORT_URL_CHECK", "IN_PROGRESS", "단축 URL 여부를 확인하고 있습니다.") + context.detection["shortened_url"] = short.url_shorten_checker(context) + _send_progress(context, "SHORT_URL_CHECK", "COMPLETED", "단축 URL 여부 확인이 완료되었습니다.") + + if context.detection["shortened_url"].get("detected", False): + _send_progress(context, "REDIRECT_TRACE", "IN_PROGRESS", "리다이렉트 경로를 추적하고 있습니다.") + context.redirect = track.redirect_tracker(context) + _send_progress(context, "REDIRECT_TRACE", "COMPLETED", "리다이렉트 추적이 완료되었습니다.") + + _send_progress(context, "URL_NORMALIZE", "IN_PROGRESS", "URL을 정규화하고 있습니다.") + utils.normalize_url(context) + _send_progress(context, "URL_NORMALIZE", "COMPLETED", "URL 정규화가 완료되었습니다.") + + _send_progress(context, "API_REQ", "IN_PROGRESS", "외부 위협 정보를 조회하고 있습니다.") + server_result = req_server(context) + context.external.update(server_result) + + api_result = api.req_api(context) + context.external.update(api_result) + + context.certificate = cert.req_cert(context) + _send_progress(context, "API_REQ", "COMPLETED", "외부 위협 정보 조회가 완료되었습니다.") + + _send_progress(context, "RULE_ANALYSIS", "IN_PROGRESS", "도메인 및 파라미터 규칙 분석을 진행하고 있습니다.") + context.detection["rule_based"] = rule.parameter_rule(context) + _send_progress(context, "RULE_ANALYSIS", "COMPLETED", "도메인 및 파라미터 규칙 분석이 완료되었습니다.") + + # CharCNN은 redirect를 거친 최종 URL을 입력으로 써야 하므로, + # 일반 URL의 redirect 추적을 ML 직전으로 당긴다. + if not context.detection.get("shortened_url", {}).get("detected", False): + _send_progress(context, "REDIRECT_TRACE", "IN_PROGRESS", "리다이렉트 경로를 추적하고 있습니다.") + context.redirect = track.redirect_tracker(context) + _send_progress(context, "REDIRECT_TRACE", "COMPLETED", "리다이렉트 추적이 완료되었습니다.") + + _send_progress(context, "ML_ANALYSIS", "IN_PROGRESS", "ML 기반 위험 분석을 진행하고 있습니다.") + context.ml = ai.ai_analyze(context) + _send_progress(context, "ML_ANALYSIS", "COMPLETED", "ML 기반 위험 분석이 완료되었습니다.") + + _send_progress(context, "SCORING", "IN_PROGRESS", "최종 위험도를 계산하고 있습니다.") + context.scoring = scoring.score_analysis(context) + _send_progress(context, "SCORING", "COMPLETED", "최종 위험도 계산이 완료되었습니다.") + + return _build_final_json(context) + + +# 중간 보고 전송용 wrapper +def _send_progress(context, step: str, status: str, message: str): + if context.meta.get("disable_callback") is True: + return + + guest_Uuid = context.request.get("guest_Uuid") + callback.send_progress(guest_Uuid, step, status, message) + + +# 최종 JSON 생성 +def _build_final_json(context) -> dict: + result = builder.build_result(context) + return builder.build_JSON(result) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..147ee65 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,64 @@ +services: + # 1. MySQL 데이터베이스 + veriq-db: + image: mysql:8.0 + container_name: veriq-db + restart: always + ports: + - "127.0.0.1:3306:3306" # 외부 IP 접속 차단, 로컬 호스트만 허용 + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: veriq_db + volumes: + - ./mysql_data:/var/lib/mysql + + # 2. Redis 캐시 서버 + veriq-redis: + image: redis:latest + container_name: veriq-redis + restart: always + ports: + - "127.0.0.1:6379:6379" + + # 3. Spring BE1 (Gateway) + veriq-gateway: + build: + context: . + dockerfile: Dockerfile + container_name: veriq-be1 + restart: always + ports: + - "8081:8081" + environment: + SPRING_PROFILES_ACTIVE: be1 + BE3_API_URL: "http://veriq-platform:8083/api/v1/scan/upload" # 도커 내부망 주소로 덮어쓰기 + depends_on: + - veriq-db + - veriq-redis + + # 4. Spring BE3 (Platform/Data Hub) + veriq-platform: + build: + context: . + dockerfile: Dockerfile + container_name: veriq-be3 + restart: always + ports: + - "8083:8083" + environment: + SPRING_PROFILES_ACTIVE: be3 + depends_on: + - veriq-db + - veriq-redis + + # 5. FastAPI (분석 엔진) + veriq-analysis: + build: + context: ../analysis-engine + dockerfile: Dockerfile + container_name: veriq-analysis + restart: always + ports: + - "8000:8000" + depends_on: + - veriq-db diff --git a/main.py b/main.py index 315b45a..fd638e6 100644 --- a/main.py +++ b/main.py @@ -1,39 +1,33 @@ # main interface for analysis -from fastapi import FastAPI -from pydantic import BaseModel from datetime import datetime +from fastapi import FastAPI + +from core.context import create_initial_context +from core.pipeline import run_pipeline +from models.receive import AnalysisRequest +from services import response as callback -from analyzers.req_api import check_url_safe_browsing -from analyzers.req_db import check_url_exists app = FastAPI() -# JSON 통신 여부 확인용 더미 응답 (DB에서 보내는 파일 형식에 맞춰서 수정 예정) -class AnalyzeRequest(BaseModel): - qr_url: str -# 기본 실행여부 @app.get("/") def root(): + print("connected") return {"message": "server is running"} + @app.post("/analyze") -def analyze(data: AnalyzeRequest): - return{ - "qr_url": data.qr_url, - "respond_test": "OK", - "analyzed_at": datetime.now().isoformat() - #"final_url": data.final_url, - #"risk_score": data.risk_score, - #"malicious_patterns": data.patterns - } - -# Safe Browsing 테스트 -@app.get("/test/safe-browsing") -def test_safe_browsing(url: str): - return check_url_safe_browsing(url) - -# DB 조회 테스트 -@app.get("/test/db") -def test_db(url: str): - return check_url_exists(url) \ No newline at end of file +def analyze(req: AnalysisRequest): + context = create_initial_context(req.guest_Uuid, req.url) + print("context created") + context.meta["scanned_at"] = datetime.now().isoformat() + + print("start pipeline") + result = run_pipeline(context) + + print("posting final result...") + guest_Uuid = context.request.get("guest_Uuid") + callback.send_final_result(guest_Uuid, result) + + return {"message": "analyze done", "result": result} diff --git a/ml/__init__.py b/ml/__init__.py new file mode 100644 index 0000000..e166116 --- /dev/null +++ b/ml/__init__.py @@ -0,0 +1 @@ +"""ML inference modules for the analysis engine.""" diff --git a/ml/charcnn/__init__.py b/ml/charcnn/__init__.py new file mode 100644 index 0000000..f042a03 --- /dev/null +++ b/ml/charcnn/__init__.py @@ -0,0 +1 @@ +"""CharCNN inference package.""" diff --git a/ml/charcnn/config.py b/ml/charcnn/config.py new file mode 100644 index 0000000..207872e --- /dev/null +++ b/ml/charcnn/config.py @@ -0,0 +1,16 @@ +from pathlib import Path + +BASE_DIR = Path(__file__).resolve().parent +SAVED_DIR = BASE_DIR / "saved" +MODEL_PATH = SAVED_DIR / "charcnn.pt" +VOCAB_PATH = SAVED_DIR / "char_vocab.json" + +MAX_LEN = 200 +EMBED_DIM = 64 +NUM_FILTERS = 128 +KERNEL_SIZES = [3, 4, 5] +DROPOUT = 0.3 +THRESHOLD = 0.5 + +PAD_TOKEN = "" +UNK_TOKEN = "" diff --git a/ml/charcnn/model.py b/ml/charcnn/model.py new file mode 100644 index 0000000..b5319cb --- /dev/null +++ b/ml/charcnn/model.py @@ -0,0 +1,44 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .config import DROPOUT, EMBED_DIM, KERNEL_SIZES, NUM_FILTERS + + +class CharCNN(nn.Module): + """Character-level CNN for binary URL risk prediction. + + The saved artifact is expected to be a state_dict trained with the same + architecture values in ml/charcnn/config.py. + """ + + def __init__( + self, + vocab_size: int, + embed_dim: int = EMBED_DIM, + num_filters: int = NUM_FILTERS, + kernel_sizes: list[int] | tuple[int, ...] = KERNEL_SIZES, + dropout: float = DROPOUT, + ): + super().__init__() + self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0) + self.convs = nn.ModuleList( + [nn.Conv1d(embed_dim, num_filters, kernel_size=k) for k in kernel_sizes] + ) + self.dropout = nn.Dropout(dropout) + self.fc = nn.Linear(num_filters * len(kernel_sizes), 1) + + def forward(self, x): + # x: [batch, seq_len] + x = self.embedding(x) # [batch, seq_len, embed_dim] + x = x.permute(0, 2, 1) # [batch, embed_dim, seq_len] + + pooled = [] + for conv in self.convs: + h = F.relu(conv(x)) # [batch, num_filters, conv_len] + h = F.max_pool1d(h, h.size(2)).squeeze(2) + pooled.append(h) + + x = torch.cat(pooled, dim=1) + x = self.dropout(x) + return self.fc(x).squeeze(1) diff --git a/ml/charcnn/predictor.py b/ml/charcnn/predictor.py new file mode 100644 index 0000000..bc4dc2e --- /dev/null +++ b/ml/charcnn/predictor.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import json +from functools import lru_cache +from pathlib import Path +from typing import Any + +from .config import ( + MAX_LEN, + MODEL_PATH, + PAD_TOKEN, + THRESHOLD, + UNK_TOKEN, + VOCAB_PATH, +) + +LABEL_MALICIOUS = "malicious" +LABEL_NORMAL = "normal" +LABEL_UNKNOWN = "unknown" +THREAT_LABEL = "CHARCNN:SUSPICIOUS_URL_PATTERN" + + +def _unavailable_result(url: str | None, error: str) -> dict[str, Any]: + return { + "available": False, + "target_url": url, + "score": 0, + "probability": None, + "label": LABEL_UNKNOWN, + "threshold": THRESHOLD, + "threats": [], + "error": error, + } + + +def load_vocab(vocab_path: Path = VOCAB_PATH) -> dict[str, int]: + with open(vocab_path, "r", encoding="utf-8") as f: + vocab = json.load(f) + + if PAD_TOKEN not in vocab or UNK_TOKEN not in vocab: + raise ValueError(f"vocab must contain {PAD_TOKEN!r} and {UNK_TOKEN!r}") + + return vocab + + +def encode_url(url: str, vocab: dict[str, int], max_len: int = MAX_LEN) -> list[int]: + url = str(url) + unk_idx = vocab[UNK_TOKEN] + pad_idx = vocab[PAD_TOKEN] + + encoded = [vocab.get(ch, unk_idx) for ch in url] + + if len(encoded) < max_len: + encoded += [pad_idx] * (max_len - len(encoded)) + else: + encoded = encoded[:max_len] + + return encoded + + +class CharCNNPredictor: + def __init__( + self, + model_path: Path = MODEL_PATH, + vocab_path: Path = VOCAB_PATH, + threshold: float = THRESHOLD, + ): + self.model_path = Path(model_path) + self.vocab_path = Path(vocab_path) + self.threshold = threshold + + if not self.model_path.exists(): + raise FileNotFoundError(f"CharCNN model file not found: {self.model_path}") + if not self.vocab_path.exists(): + raise FileNotFoundError(f"CharCNN vocab file not found: {self.vocab_path}") + + try: + import torch + except ImportError as exc: + raise ImportError("torch is required for CharCNN inference") from exc + + from .model import CharCNN + + self.torch = torch + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.vocab = load_vocab(self.vocab_path) + self.model = CharCNN(vocab_size=len(self.vocab)).to(self.device) + state_dict = torch.load(self.model_path, map_location=self.device) + self.model.load_state_dict(state_dict) + self.model.eval() + + def predict(self, url: str | None) -> dict[str, Any]: + if not url: + return _unavailable_result(url, "target_url_missing") + + encoded = encode_url(str(url), self.vocab) + x = self.torch.tensor([encoded], dtype=self.torch.long).to(self.device) + + with self.torch.no_grad(): + logit = self.model(x) + prob = self.torch.sigmoid(logit).item() + + score = int(round(prob * 100)) + label = LABEL_MALICIOUS if prob >= self.threshold else LABEL_NORMAL + threats = [THREAT_LABEL] if label == LABEL_MALICIOUS else [] + + return { + "available": True, + "target_url": url, + "score": score, + "probability": prob, + "label": label, + "threshold": self.threshold, + "threats": threats, + "error": None, + } + + +@lru_cache(maxsize=1) +def get_predictor() -> CharCNNPredictor: + return CharCNNPredictor() + + +def predict_url(url: str | None) -> dict[str, Any]: + try: + return get_predictor().predict(url) + except Exception as exc: + return _unavailable_result(url, str(exc)) diff --git a/ml/charcnn/saved/README.md b/ml/charcnn/saved/README.md new file mode 100644 index 0000000..713bb8d --- /dev/null +++ b/ml/charcnn/saved/README.md @@ -0,0 +1,11 @@ +# CharCNN artifacts + +여기에 학습 완료된 CharCNN 산출물 두 개를 넣는다. + +```text +charcnn.pt +char_vocab.json +``` + +두 파일은 반드시 같은 학습에서 나온 한 쌍이어야 한다. +현재 ZIP에는 실제 모델 가중치가 포함되어 있지 않으므로, 이 파일을 넣기 전까지 `context.ml["charcnn"]["available"]`은 `False`로 나온다. diff --git a/ml/charcnn/saved/char_vocab.json b/ml/charcnn/saved/char_vocab.json new file mode 100644 index 0000000..a5160fc --- /dev/null +++ b/ml/charcnn/saved/char_vocab.json @@ -0,0 +1,63 @@ +{ + "": 0, + "": 1, + "h": 2, + "t": 3, + "p": 4, + "s": 5, + ":": 6, + "/": 7, + "b": 8, + "e": 9, + "n": 10, + "y": 11, + "7": 12, + "2": 13, + "6": 14, + ".": 15, + "w": 16, + "i": 17, + "x": 18, + "c": 19, + "o": 20, + "m": 21, + "-": 22, + "1": 23, + "r": 24, + "a": 25, + "l": 26, + "g": 27, + "d": 28, + "f": 29, + "u": 30, + "k": 31, + "j": 32, + "z": 33, + "q": 34, + "0": 35, + "8": 36, + "v": 37, + "3": 38, + "5": 39, + "9": 40, + "4": 41, + "?": 42, + "=": 43, + "&": 44, + "%": 45, + "(": 46, + ")": 47, + "_": 48, + "#": 49, + "@": 50, + ";": 51, + "+": 52, + "*": 53, + "!": 54, + "$": 55, + "[": 56, + "]": 57, + "~": 58, + ",": 59, + "'": 60 +} \ No newline at end of file diff --git a/ml/charcnn/saved/charcnn.pt b/ml/charcnn/saved/charcnn.pt new file mode 100644 index 0000000..9269d3d Binary files /dev/null and b/ml/charcnn/saved/charcnn.pt differ diff --git a/ml/xgb/__init__.py b/ml/xgb/__init__.py new file mode 100644 index 0000000..a4cc26f --- /dev/null +++ b/ml/xgb/__init__.py @@ -0,0 +1 @@ +"""FEB + XGB inference package for the analysis engine.""" diff --git a/ml/xgb/config.py b/ml/xgb/config.py new file mode 100644 index 0000000..9ac1634 --- /dev/null +++ b/ml/xgb/config.py @@ -0,0 +1,45 @@ +from pathlib import Path + +BASE_DIR = Path(__file__).resolve().parent +SAVED_DIR = BASE_DIR / "saved" +XGB_MODEL_PATH = SAVED_DIR / "xgb_model.joblib" + +XGB_THRESHOLD = 0.5 +HIGH_RISK_SCORE_THRESHOLD = 80 + +XGB_THREAT_LABEL = "XGB:SUSPICIOUS_URL_FEATURES" +XGB_HIGH_RISK_THREAT_LABEL = "XGB:HIGH_RISK_URL" + +SENSITIVE_KEYWORDS = [ + "login", + "verify", + "secure", + "account", + "update", + "confirm", + "admin", + "signin", + "password", + "bank", + "payment", +] + +FREE_HOSTING_KEYWORDS = [ + "000webhost", + "github.io", + "pages.dev", + "netlify.app", + "vercel.app", + "firebaseapp.com", + "web.app", +] + +SUSPICIOUS_EXTENSIONS = [ + ".exe", + ".zip", + ".apk", + ".scr", + ".bat", + ".cmd", + ".msi", +] diff --git a/ml/xgb/feat_extractor.py b/ml/xgb/feat_extractor.py new file mode 100644 index 0000000..66d161f --- /dev/null +++ b/ml/xgb/feat_extractor.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import re +from urllib.parse import urlsplit + +from .config import FREE_HOSTING_KEYWORDS, SENSITIVE_KEYWORDS, SUSPICIOUS_EXTENSIONS +from .feat_schema import FEATURE_COLUMNS + + +def _safe_get(mapping: dict, key: str, default=None): + if not isinstance(mapping, dict): + return default + return mapping.get(key, default) + + +def _count_keywords(text: str, keywords: list[str]) -> int: + lower_text = text.lower() + return sum(1 for keyword in keywords if keyword in lower_text) + + +def _has_any(text: str, targets: list[str]) -> int: + lower_text = text.lower() + return int(any(target in lower_text for target in targets)) + + +def _is_ip_address(hostname: str) -> int: + if not hostname: + return 0 + pattern = r"^\d{1,3}(\.\d{1,3}){3}$" + return int(bool(re.match(pattern, hostname))) + + +def _target_url_from_context(context) -> str: + redirect = getattr(context, "redirect", {}) or {} + url_data = getattr(context, "url", {}) or {} + request = getattr(context, "request", {}) or {} + + return ( + _safe_get(redirect, "final_url") + or _safe_get(url_data, "normalized_url") + or _safe_get(request, "original_url") + or "" + ) + + +def _parse_target_url(target_url: str, fallback_url_data: dict) -> dict: + """Return URL fields without mutating engine context. + + The FEB training pipeline used the final analysis URL before feature extraction. + The current engine keeps context.url mostly fixed, so inference selects + redirect.final_url when available and parses it locally for lexical features. + """ + url = str(target_url or "").strip() + if url and not urlsplit(url).hostname: + url = f"https://{url}" + + try: + parsed = urlsplit(url) + except ValueError: + parsed = urlsplit("") + + hostname = (parsed.hostname or _safe_get(fallback_url_data, "hostname", "") or "").lower() + scheme = (parsed.scheme or _safe_get(fallback_url_data, "scheme", "") or "").lower() + normalized_url = url or _safe_get(fallback_url_data, "normalized_url", "") or "" + + return { + "normalized_url": normalized_url, + "hostname": hostname, + "path": parsed.path or _safe_get(fallback_url_data, "path", "") or "", + "query": parsed.query or _safe_get(fallback_url_data, "query", "") or "", + "is_https": scheme == "https" or bool(_safe_get(fallback_url_data, "is_https", False)), + } + + +def extract_features(context) -> dict: + fallback_url_data = getattr(context, "url", {}) or {} + url_data = _parse_target_url(_target_url_from_context(context), fallback_url_data) + detection = getattr(context, "detection", {}) or {} + redirect = getattr(context, "redirect", {}) or {} + certificate = getattr(context, "certificate", {}) or {} + + normalized_url = _safe_get(url_data, "normalized_url", "") or "" + hostname = _safe_get(url_data, "hostname", "") or "" + path = _safe_get(url_data, "path", "") or "" + query = _safe_get(url_data, "query", "") or "" + full_text = normalized_url.lower() + + shortened_url = _safe_get(detection, "shortened_url", {}) or {} + rule_based = _safe_get(detection, "rule_based", {}) or {} + + features = {} + + # G1 : length feature + features["url_length"] = len(normalized_url) + features["hostname_length"] = len(hostname) + features["path_length"] = len(path) + features["query_length"] = len(query) + features["subdomain_count"] = max(hostname.count(".") - 1, 0) if hostname else 0 + + # G2 : character feature + features["dot_count"] = normalized_url.count(".") + features["hyphen_count"] = normalized_url.count("-") + features["underscore_count"] = normalized_url.count("_") + features["slash_count"] = normalized_url.count("/") + features["question_mark_count"] = normalized_url.count("?") + features["equal_count"] = normalized_url.count("=") + features["ampersand_count"] = normalized_url.count("&") + features["at_count"] = normalized_url.count("@") + features["percent_count"] = normalized_url.count("%") + features["special_char_count"] = len(re.findall(r"[^a-zA-Z0-9]", normalized_url)) + + # G3 : keyword feature + features["keyword_count"] = _count_keywords(full_text, SENSITIVE_KEYWORDS) + features["has_sensitive_keyword"] = int(features["keyword_count"] > 0) + features["has_free_hosting_keyword"] = _has_any(full_text, FREE_HOSTING_KEYWORDS) + features["has_suspicious_extension"] = _has_any(full_text, SUSPICIOUS_EXTENSIONS) + + # G4 : numeric feature + digit_count = sum(ch.isdigit() for ch in normalized_url) + features["digit_count"] = digit_count + features["digit_ratio"] = digit_count / len(normalized_url) if normalized_url else 0.0 + features["has_ip_address"] = _is_ip_address(hostname) + + # G5 : structure/behavioral feature + features["is_https"] = int(bool(_safe_get(url_data, "is_https", False))) + features["is_shortened_url"] = int(bool(_safe_get(shortened_url, "detected", False))) + features["redirect_count"] = int(_safe_get(redirect, "redirect_count", 0) or 0) + features["has_redirect_loop"] = int(bool(_safe_get(redirect, "loop_detected", False))) + features["rule_flag_count"] = len(_safe_get(rule_based, "flags", []) or []) + features["suspicious_param_count"] = len(_safe_get(rule_based, "suspicious_query_params", []) or []) + features["embedded_url_count"] = len(_safe_get(rule_based, "embedded_urls", []) or []) + + # G6 : external/verification feature + features["certificate_valid"] = int(bool(_safe_get(certificate, "valid", False))) + + return {column: features.get(column, 0) for column in FEATURE_COLUMNS} diff --git a/ml/xgb/feat_schema.py b/ml/xgb/feat_schema.py new file mode 100644 index 0000000..1ccea06 --- /dev/null +++ b/ml/xgb/feat_schema.py @@ -0,0 +1,45 @@ +FEATURE_SCHEMA = { + # G1 : length feature + "url_length": int, + "hostname_length": int, + "path_length": int, + "query_length": int, + "subdomain_count": int, + + # G2 : character feature + "dot_count": int, + "hyphen_count": int, + "underscore_count": int, + "slash_count": int, + "question_mark_count": int, + "equal_count": int, + "ampersand_count": int, + "at_count": int, + "percent_count": int, + "special_char_count": int, + + # G3 : keyword feature + "keyword_count": int, + "has_sensitive_keyword": int, + "has_free_hosting_keyword": int, + "has_suspicious_extension": int, + + # G4 : numeric feature + "digit_count": int, + "digit_ratio": float, + "has_ip_address": int, + + # G5 : structure/behavioral feature + "is_https": int, + "is_shortened_url": int, + "redirect_count": int, + "has_redirect_loop": int, + "rule_flag_count": int, + "suspicious_param_count": int, + "embedded_url_count": int, + + # G6 : external/verification feature + "certificate_valid": int, +} + +FEATURE_COLUMNS = list(FEATURE_SCHEMA.keys()) diff --git a/ml/xgb/predictor.py b/ml/xgb/predictor.py new file mode 100644 index 0000000..7e54c5b --- /dev/null +++ b/ml/xgb/predictor.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import Any + +from .config import ( + HIGH_RISK_SCORE_THRESHOLD, + XGB_HIGH_RISK_THREAT_LABEL, + XGB_MODEL_PATH, + XGB_THRESHOLD, + XGB_THREAT_LABEL, +) +from .feat_extractor import extract_features +from .feat_schema import FEATURE_COLUMNS + +LABEL_MALICIOUS = "malicious" +LABEL_NORMAL = "normal" +LABEL_UNKNOWN = "unknown" + + +def _unavailable_result(error: str, features: dict | None = None) -> dict[str, Any]: + return { + "available": False, + "score": 0, + "probability": None, + "label": LABEL_UNKNOWN, + "threshold": XGB_THRESHOLD, + "threats": [], + "features": features or {}, + "error": error, + } + + +class XGBPredictor: + def __init__(self, model_path: Path = XGB_MODEL_PATH, threshold: float = XGB_THRESHOLD): + self.model_path = Path(model_path) + self.threshold = threshold + + if not self.model_path.exists(): + raise FileNotFoundError(f"XGB model file not found: {self.model_path}") + + try: + import joblib + import pandas as pd + except ImportError as exc: + raise ImportError("joblib and pandas are required for XGB inference") from exc + + self.pd = pd + self.model = joblib.load(self.model_path) + + def predict_from_features(self, features: dict) -> dict[str, Any]: + row = {column: features.get(column, 0) for column in FEATURE_COLUMNS} + x = self.pd.DataFrame([row], columns=FEATURE_COLUMNS) + + probability = float(self.model.predict_proba(x)[0][1]) + raw_label = int(probability >= self.threshold) + score = int(round(probability * 100)) + label = LABEL_MALICIOUS if raw_label == 1 else LABEL_NORMAL + + return { + "available": True, + "score": score, + "probability": probability, + "label": label, + "raw_label": raw_label, + "threshold": self.threshold, + "threats": self._build_threats(raw_label, score), + "features": row, + "error": None, + } + + def predict(self, context) -> dict[str, Any]: + features = extract_features(context) + return self.predict_from_features(features) + + def _build_threats(self, raw_label: int, score: int) -> list[str]: + if raw_label == 0: + return [] + + threats = [XGB_THREAT_LABEL] + if score >= HIGH_RISK_SCORE_THRESHOLD: + threats.append(XGB_HIGH_RISK_THREAT_LABEL) + return threats + + +@lru_cache(maxsize=1) +def get_predictor() -> XGBPredictor: + return XGBPredictor() + + +def predict_context(context) -> dict[str, Any]: + features = None + try: + features = extract_features(context) + return get_predictor().predict_from_features(features) + except Exception as exc: + return _unavailable_result(str(exc), features) diff --git a/ml/xgb/saved/xgb_model.joblib b/ml/xgb/saved/xgb_model.joblib new file mode 100644 index 0000000..6033a46 Binary files /dev/null and b/ml/xgb/saved/xgb_model.joblib differ diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/base.py b/models/base.py new file mode 100644 index 0000000..daa1999 --- /dev/null +++ b/models/base.py @@ -0,0 +1,13 @@ +# 전체 검사기의 기본 구조 및 호출 형식을 통일하는 기반입니다. +# 입력: context, 결과: 각 검사 종료 후 context에 직접 기록(반환X) +from core.context import AnalysisContext + +def run_detector(detector, context: AnalysisContext): + try: + detector(context) + context.meta["completed_steps"].append(detector.__name__) + except Exception as e: + context.meta["errors"].append({ + "step": detector.__name__, + "error": str(e) + }) \ No newline at end of file diff --git a/models/receive.py b/models/receive.py index 51f24a9..3d70f7e 100644 --- a/models/receive.py +++ b/models/receive.py @@ -1,6 +1,11 @@ # url 분석 요청 시 수신하는 데이터의 형식을 정의합니다. -from pydantic import BaseModel +from pydantic import BaseModel, Field + class AnalysisRequest(BaseModel): - scan_id: str # 분석 요청 식별 ID - url: str # 분석 대상 URL 원본 \ No newline at end of file + # [수정] BE3에서 넘겨주는 식별자 기준으로 변경 + guest_Uuid: str = Field(alias="guestUuid") + url: str + + class Config: + populate_by_name = True \ No newline at end of file diff --git a/models/response.py b/models/response.py deleted file mode 100644 index 0dcd5de..0000000 --- a/models/response.py +++ /dev/null @@ -1,106 +0,0 @@ -# 최종 응답 데이터. 즉, JSON 작성을 위한 데이터 구조 -# DB와 FE에 보내는 데이터의 내용이 다른 경우, 파이프라인 쪽에서 조립하고 해당 파일을 지우는 편이 나을 수 있을지도... -# 기능 개발 후 검토 -from dataclasses import dataclass, field, asdict -from typing import List, Optional -import json - - -# ========================= -# 1. 기능별 하위 DTO -# ========================= - -@dataclass -class HttpsResult: - """HTTPS 사용 여부""" - isSecure: bool - - -@dataclass -class ShortUrlResult: - """단축 URL 여부""" - isShortened: bool - - -@dataclass -class MlResult: - """ML 검사 결과""" - threats: List[str] = field(default_factory=list) - score: int = 0 - - -@dataclass -class ExternalApiResult: - """외부 API 조회 결과""" - checked: bool - provider: str - result: Optional[str] - - -@dataclass -class InternalDbResult: - """ - 내부 DB 조회 결과 - - exists: DB 내 URL 존재 여부 - - reportCount: 최근 신고 건수 - - blockCount: 최근 차단 건수 - """ - exists: bool - reportCount: Optional[int] = None - blockCount: Optional[int] = None - - -@dataclass -class RedirectResult: - """리다이렉션 추적 결과""" - finalUrl: str - redirectCount: int - - -@dataclass -class CertificateInfo: - """인증서 정보""" - valid: Optional[bool] = None - issuer: Optional[str] = None - validFrom: Optional[str] = None - validTo: Optional[str] = None - - -@dataclass -class ServerInfo: - """서버 유형 및 위치, 인증서 정보""" - type: str - location: str - certificate: Optional[CertificateInfo] = None - - -# ========================= -# 2. 최종 출력 DTO -# ========================= - -@dataclass -class AnalysisResponse: - """ - 최종 출력 JSON 구조 - """ - analysisTime: str - targetUrl: str - - https: HttpsResult - shortUrl: ShortUrlResult - ml: MlResult - externalApi: ExternalApiResult - internalDb: InternalDbResult - redirect: RedirectResult - serverInfo: ServerInfo - - score: int - riskLevel: str # safe / caution / threat - - def to_dict(self) -> dict: - """dataclass -> dict 변환""" - return asdict(self) - - def to_json(self) -> str: - """dataclass -> JSON 문자열 변환""" - return json.dumps(self.to_dict(), ensure_ascii=False, indent=2) diff --git a/models/result.py b/models/result.py index 1fac512..4e7684d 100644 --- a/models/result.py +++ b/models/result.py @@ -1,46 +1,53 @@ -# 결과 json형식 -# 차후 개발 단계에 따라 실제 반환값 삽입 - -{ - "analysisTime": "yyyy-mm-ddT18:30:00+09:00", - "targetUrl": "http://example.com", - "https": { - "isSecure": true/false - }, - "shortUrl": { - "isShortened": true/false - }, - "ml": { - "threats": [ - "위협 요소", - "배열 형식" - ], - "score": 65 #0~100 - }, - "externalApi": { - "checked": true/false, - "provider": "Google Safe Browsing", - "result": "safe" #실패/결과 없을 시 null - }, - "internalDb": { - "exists": true/false, - "reportCount": 3, - "blockCount": 1 #count 둘 모두 없으면 null - }, - "redirect": { - "finalUrl": "https://example.com/login", - "redirectCount": 2 #정수 - }, - "serverInfo": { - "type": "nginx", #서버 유형 - "location": "US", #서버 위치 - "certificate": { #인증서 정보 - "valid": true/false, - "issuer": "인증서 발급자", - "validFrom": "2026-01-01T00:00:00Z", - "validTo": "2026-04-01T00:00:00Z" - } - }, - "score": 72, #최종 점수 - "riskLevel": "safe/caution/threat" -} \ No newline at end of file +# 최종 응답 데이터. 즉, JSON 작성을 위한 데이터 구조 +from datetime import datetime, timezone +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +@dataclass +class AnalysisResult: + guestUuid: str + analysisTime: str + originalUrl: str + + https: dict = field(default_factory=lambda: { + "isSecure": False + }) + + shortUrl: dict = field(default_factory=lambda: { + "isShortened": False + }) + + ml: dict = field(default_factory=lambda: { + "score": 0, + "threats": [] + }) + + externalApi: dict = field(default_factory=lambda: { + "checked": False, + "provider": None, + "result": None + }) + reportCount: int | None = None + domainAge: int | None = None + + redirect: dict = field(default_factory=lambda: { + "finalUrl": None, + "redirectCount": 0 + }) + + threats: list[str] = field(default_factory=list) + + serverInfo: dict = field(default_factory=lambda: { + "type": None, + "location": None, + "certificate": { + "valid": False, + "issuer": None, + "validFrom": None, + "validTo": None + } + }) + + score: int = 0 + riskLevel: str = "SAFE" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4d3a550 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +annotated-doc==0.0.4 +annotated-types==0.7.0 +anyio==4.13.0 +click==8.3.2 +colorama==0.4.6 +fastapi==0.135.3 +h11==0.16.0 +idna==3.11 +pydantic==2.12.5 +pydantic_core==2.41.5 +python-dotenv==1.2.2 +starlette==1.0.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +uvicorn==0.44.0 +python-whois==0.9.6 +requests==2.32.3 +tldextract==5.3.1 +torch==2.11.0 +joblib==1.5.3 +pandas==3.0.2 +numpy==2.4.4 +python-dateutil==2.9.0.post0 +scikit-learn==1.8.0 +scipy==1.17.1 +six==1.17.0 +threadpoolctl==3.6.0 +tzdata==2026.2 +xgboost==3.2.0 \ No newline at end of file diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/builder.py b/services/builder.py new file mode 100644 index 0000000..7e5b967 --- /dev/null +++ b/services/builder.py @@ -0,0 +1,381 @@ +# 검사 결과를 정리하고, 전송할 수 있는 형태로 build합니다. +from datetime import datetime + +from models.result import AnalysisResult +from core.config import ctxData as ctx, statusData as stat, riskLevelData as lev + +from core.context import AnalysisContext + + + +def safe_dict(value) -> dict: + return value if isinstance(value, dict) else {} + + +def safe_int(value, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def clamp_score(value) -> int: + try: + score = int(round(float(value))) + except (TypeError, ValueError): + score = 0 + return max(0, min(100, score)) + + +def normalize_list(value) -> list: + if value is None: + return [] + if isinstance(value, list): + return [item for item in value if item is not None] + if isinstance(value, (tuple, set)): + return [item for item in value if item is not None] + return [value] + + +def normalize_risk_level(value: str | None) -> str: + if value in (lev.THREAT, lev.CAUTION, lev.SAFE): + return value + return lev.SAFE + + + +def add_unique(target: list, label) -> None: + if label is None: + return + + label = str(label) + if not label: + return + + if label not in target: + target.append(label) + + +# threats 작성 함수 +def build_threats(context: AnalysisContext) -> list[str]: + threats = [] + + if context.detection.get(ctx.Det_short, {}).get(ctx.Short_det): + add_unique(threats, "SHORTENED_URL") + + for flag in context.detection.get(ctx.Det_rule, {}).get(ctx.Rule_flags, []): + add_unique(threats, flag) + + for label in context.external.get(ctx.EX_GSB, {}).get(ctx.GSB_threats, []): + add_unique(threats, label) + + for label in context.external.get(ctx.EX_OTX, {}).get(ctx.OTX_threats, []): + add_unique(threats, label) + + ml_threats = normalize_list(safe_dict(context.ml).get(ctx.Ml_threats, [])) + + for label in ml_threats: + add_unique(threats, label) + + cert_errors = normalize_list(context.certificate.get(ctx.Cert_error, [])) + + for error in cert_errors: + add_unique(threats, error) + + for failure_label in build_failure_labels(context): + add_unique(threats, failure_label) + + return threats + + + +def build_failure_labels(context: AnalysisContext) -> list[str]: + """검사기 실패/미완료 상태를 threats 필드에 함께 노출하기 위한 라벨 목록을 만든다. + + 위협 라벨과 구분되도록 검사기 단위 라벨은 *_FAILED 형태로 통일한다. + 세부 실패 원인 중 이미 표준 라벨이 있는 인증서(CERT_*)는 기존 라벨을 유지하고, + 추가로 CERTIFICATE_FAILED를 붙인다. + """ + labels = [] + + append_external_failure_labels(labels, context) + append_redirect_failure_labels(labels, context) + append_server_failure_labels(labels, context) + append_certificate_failure_labels(labels, context) + append_ml_failure_labels(labels, context) + append_scoring_failure_labels(labels, context) + + return labels + + +def append_external_failure_labels(labels: list[str], context: AnalysisContext) -> None: + external = safe_dict(context.external) + + gsb = safe_dict(external.get(ctx.EX_GSB)) + if gsb.get(ctx.GSB_status) is not None and _is_failed(gsb.get(ctx.GSB_status)): + add_unique(labels, "GSB_FAILED") + + otx = safe_dict(external.get(ctx.EX_OTX)) + if otx.get(ctx.OTX_status) is not None and _is_failed(otx.get(ctx.OTX_status)): + add_unique(labels, "OTX_FAILED") + + whois = safe_dict(external.get(ctx.EX_WHOIS)) + if whois.get(ctx.WHO_status) is not None and _is_failed(whois.get(ctx.WHO_status)): + add_unique(labels, "WHOIS_FAILED") + + +def append_redirect_failure_labels(labels: list[str], context: AnalysisContext) -> None: + redirect = safe_dict(context.redirect) + status = _status_value(redirect.get(ctx.Red_status)) + + if not status or status == stat.SUCCESS: + return + + add_unique(labels, "REDIRECT_FAILED") + + detail_labels = { + "request_failed": "REDIRECT_REQUEST_FAILED", + "client_error": "REDIRECT_CLIENT_ERROR", + "loop_detected": "REDIRECT_LOOP_DETECTED", + "too_many_redirects": "REDIRECT_TOO_MANY_REDIRECTS", + "invalid_location": "REDIRECT_INVALID_LOCATION", + } + add_unique(labels, detail_labels.get(status)) + + if redirect.get(ctx.Red_loop_det): + add_unique(labels, "REDIRECT_LOOP_DETECTED") + + +def append_server_failure_labels(labels: list[str], context: AnalysisContext) -> None: + external = safe_dict(context.external) + server_status = _status_value(external.get(ctx.Ex_server_status)) + + if server_status == "failed": + add_unique(labels, "SERVER_INFO_FAILED") + + +def append_certificate_failure_labels(labels: list[str], context: AnalysisContext) -> None: + certificate = safe_dict(context.certificate) + status = _status_value(certificate.get(ctx.Cert_status)) + errors = normalize_list(certificate.get(ctx.Cert_error)) + + if (status and status != stat.SUCCESS) or errors: + add_unique(labels, "CERTIFICATE_FAILED") + + +def append_ml_failure_labels(labels: list[str], context: AnalysisContext) -> None: + ml_result = safe_dict(context.ml) + charcnn = safe_dict(ml_result.get("charcnn")) + xgb = safe_dict(ml_result.get("xgb")) + fusion = safe_dict(ml_result.get("fusion")) + + charcnn_failed = charcnn.get("available") is False and bool(charcnn.get("error")) + xgb_failed = xgb.get("available") is False and bool(xgb.get("error")) + + if charcnn_failed: + add_unique(labels, "CHARCNN_FAILED") + + if xgb_failed: + add_unique(labels, "XGB_FAILED") + + if fusion.get("available") is False and (charcnn_failed or xgb_failed): + add_unique(labels, "ML_FAILED") + + +def append_scoring_failure_labels(labels: list[str], context: AnalysisContext) -> None: + scoring = safe_dict(context.scoring) + reasons = normalize_list(scoring.get(ctx.Sco_reason)) + + scoring_failure_reasons = { + "internal_score_failed", + "external_score_failed", + } + + if any(reason in scoring_failure_reasons for reason in reasons): + add_unique(labels, "SCORING_FAILED") + + +# externalApi-result 작성 함수 +def build_external_result(context: AnalysisContext) -> str: + external = safe_dict(context.external) + safe_browsing = safe_dict(external.get(ctx.EX_GSB, {})) + otx = safe_dict(external.get(ctx.EX_OTX, {})) + whois = context.external.get(ctx.EX_WHOIS, {}) + + # 1. 위협 탐지 우선 + if safe_browsing.get("is_threat") or otx.get("is_threat"): + return "THREAT" + + # 2. 상태값 확인 + gsb_status = safe_browsing.get("status_type") + otx_status = otx.get("status") + whois_status = whois.get("status") + + gsb_success = _is_success(gsb_status) + otx_success = _is_success(otx_status) + whois_success = _is_success(whois_status) + + failed = [] + + if safe_browsing and not gsb_success: + failed.append("GSB") + + if otx and not otx_success: + failed.append("OTX") + + if whois and not whois_success: + failed.append("WHOIS") + + # 3. 실패 API 명시 + if failed: + return "_".join(failed) + "_FAILED" + + # 4. 하나라도 성공했고 위협이 없으면 SAFE + if gsb_success or otx_success: + return "SAFE" + + # 5. 아무것도 판단 못 함 + return "UNKNOWN" + +# externalApi 성공/실패 판정용 함수 +def _status_value(status): + if status is None: + return None + + if isinstance(status, (list, tuple)): + if len(status) >= 2: + value = status[1] + return None if value is None else str(value).lower() + if len(status) == 1: + value = status[0] + return None if value is None else str(value).lower() + return None + + return None if status is None else str(status).lower() + + +def _is_success(status) -> bool: + value = _status_value(status) + return value in ("success", "ok", "200") + + +def _is_failed(status) -> bool: + value = _status_value(status) + + if value is None: + return False + + return value not in ("success", "ok", "200") + + +# externalApi-provider 작성 함수 +def build_external_provider(context) -> str | None: + providers = [] + + safe_browsing = context.external.get("safe_browsing", {}) + otx = context.external.get("otx", {}) + + if safe_browsing.get("status_type") is not None: + providers.append("Google Safe Browsing") + + if otx.get("status") is not None: + providers.append("AlienVault OTX") + + return ", ".join(providers) if providers else None + + +# report count, domainage 추가 +def build_report_count(context: AnalysisContext) -> int | None: + otx = context.external.get("otx", {}) + pulse_count = otx.get("pulse_count") + + if isinstance(pulse_count, int): + return pulse_count + + try: + return int(pulse_count) + except (TypeError, ValueError): + return None + + +def build_domain_age(context: AnalysisContext) -> int | None: + whois = context.external.get("whois", {}) + domain_age_days = whois.get("domain_age_days") + + if isinstance(domain_age_days, int): + return domain_age_days + + try: + return int(domain_age_days) + except (TypeError, ValueError): + return None + + +def build_result(context: AnalysisContext) -> AnalysisResult: + external = safe_dict(context.external) + safe_browsing = safe_dict(external.get(ctx.EX_GSB, {})) + otx = safe_dict(external.get(ctx.EX_OTX, {})) + + result = AnalysisResult( + guestUuid=context.request.get("guest_Uuid", ""), + analysisTime=datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + originalUrl=context.request.get("original_url") or "" + ) + + result.https["isSecure"] = bool(context.url.get("is_https", False)) + + result.shortUrl["isShortened"] = bool( + context.detection.get(ctx.Det_short, {}).get(ctx.Short_det, False) + ) + + result.reportCount = build_report_count(context) + result.domainAge = build_domain_age(context) + result.threats = build_threats(context) + result.ml = safe_dict(context.ml) + + result.externalApi["checked"] = ( + safe_browsing.get(ctx.GSB_status) is not None + or safe_browsing.get(ctx.GSB_result) is not None + or otx.get(ctx.OTX_status) is not None + ) + result.externalApi["provider"] = build_external_provider(context) + result.externalApi["result"] = build_external_result(context) + + result.redirect["finalUrl"] = ( + context.redirect.get("final_url") + or context.url.get("normalized_url") + or context.request.get("original_url") + ) + result.redirect["redirectCount"] = safe_int(context.redirect.get("redirect_count", 0), 0) + + result.serverInfo["type"] = context.external.get("server_type") + result.serverInfo["location"] = context.external.get("server_location") + result.serverInfo["certificate"]["valid"] = context.certificate.get("valid") + result.serverInfo["certificate"]["issuer"] = context.certificate.get("issuer") + result.serverInfo["certificate"]["validFrom"] = context.certificate.get("validFrom") + result.serverInfo["certificate"]["validTo"] = context.certificate.get("validTo") + + scoring_result = safe_dict(context.scoring) + result.score = clamp_score(scoring_result.get("score", 0)) + result.riskLevel = normalize_risk_level(scoring_result.get("risk_level")) + + return result + + +def build_JSON(result: AnalysisResult) -> dict: + return { + "guestUuid": result.guestUuid, + "analysisTime": result.analysisTime, + "originalUrl": result.originalUrl, + "https": result.https, + "shortUrl": result.shortUrl, + "threats": result.threats, + "ml": result.ml, + "externalApi": result.externalApi, + "reportCount": result.reportCount, + "domainAge": result.domainAge, + "redirect": result.redirect, + "serverInfo": result.serverInfo, + "score": result.score, + "riskLevel": result.riskLevel + } \ No newline at end of file diff --git a/services/json_builder.py b/services/json_builder.py deleted file mode 100644 index 16efd60..0000000 --- a/services/json_builder.py +++ /dev/null @@ -1,2 +0,0 @@ -# 전송할 JSON 파일 제작용 -# sender이 별도 필요할까? 지금은 상관없을거같은데... diff --git a/services/response.py b/services/response.py new file mode 100644 index 0000000..80160f2 --- /dev/null +++ b/services/response.py @@ -0,0 +1,89 @@ +# BE3으로 전송하는 부분 +import os +import requests + +# [추가] 환경변수 +ML_SECRET_KEY = os.getenv("ML_SECRET_KEY") +CALLBACK_PROGRESS_URL = os.getenv("BE3_CALLBACK_PROGRESS_URL") +CALLBACK_FINAL_URL = os.getenv("BE3_CALLBACK_FINAL_URL") + +TIMEOUT_SECONDS = 5 + + + +# 중간보고 +def send_progress(guest_Uuid: str, step: str, status: str, message: str): + payload = { + "guestUuid": guest_Uuid, + "step": step, + "status": status, + "message": message + } + return _post_json(CALLBACK_PROGRESS_URL, guest_Uuid, payload) + + +# 최종보고 +def send_final_result(guest_Uuid: str, result_json: dict): + return _post_json(CALLBACK_FINAL_URL, guest_Uuid, result_json.copy()) + + +def _build_headers(guest_Uuid: str) -> dict: + return { + "X-ML-Secret": ML_SECRET_KEY or "", + "Content-Type": "application/json", + "guest_uuid": guest_Uuid, + } + + +def _post_json(url: str | None, guest_Uuid: str, payload: dict): + # URL 없으면 → 실패 JSON 생성 후 종료 + if not url: + payload.update({ + "status": "ERROR", + "message": "callback_url_missing" + }) + print("post_json: callback url missing") + return False + + if not ML_SECRET_KEY: + payload.update({ + "status": "ERROR", + "message": "ml_secret_missing" + }) + print("post_json: ml key missing") + return False + + try: + response = requests.post( + url, + json=payload, + headers=_build_headers(guest_Uuid), + timeout=TIMEOUT_SECONDS + ) + print("post_json: progress response") + + # 실패 시에도 payload에 반영 + if not response.ok: + payload.update({ + "status": "ERROR", + "message": f"http_{response.status_code}" + }) + print("post_json: https error") + return False + return payload + + except requests.exceptions.Timeout: + payload.update({ + "status": "ERROR", + "message": "timeout" + }) + print("post_json: timeout") + return False + + except requests.exceptions.RequestException as e: + payload.update({ + "status": "ERROR", + "message": str(e) + }) + print("post_json: ERROR") + return False \ No newline at end of file diff --git a/services/scoring.py b/services/scoring.py new file mode 100644 index 0000000..c5aac38 --- /dev/null +++ b/services/scoring.py @@ -0,0 +1,506 @@ +# 검사 결과에 기반해 점수를 매기는 로직입니다. +from core.context import AnalysisContext +from core.config import ctxData as ctx, statusData as stat, riskLevelData as lev +from core.config import certErrorData as Err + + +MIN_CAUTION_SCORE = 30 +COMBINED_CAUTION_SIGNAL_THRESHOLD = 3 +CHARCNN_SUSPICIOUS_LABEL = "CHARCNN:SUSPICIOUS_URL_PATTERN" +SERVER_INFO_FAILED_LABEL = "SERVER_INFO_FAILED" +REDIRECT_RISK_STATUSES = { + "request_failed", + "client_error", + "loop_detected", + "too_many_redirects", + "invalid_location", +} +SERVER_INFO_FAILED_STATUSES = { + "failed", + "request_failed", + "client_error", + "server_error", + "timeout", + "connection_failed", + "server_info_failed", +} + + +# 메인 점수 계산 함수 +def score_analysis(context: AnalysisContext) -> dict: + internal_score, internal_reasons = safe_score_part( + lambda: score_internal(context), + fallback_reason="internal_score_failed" + ) + ml_score = safe_score_ml(context) + external_score, external_reasons = safe_score_part( + lambda: score_external(context), + fallback_reason="external_score_failed" + ) + + final_score = round( + internal_score * 0.45 + + ml_score * 0.35 + + external_score * 0.20 + ) + + safe_browsing = safe_dict(context.external).get(ctx.EX_GSB, {}) + otx = safe_dict(context.external).get(ctx.EX_OTX, {}) + + # ML이 강하게 위험을 낸 경우에도 최소 점수를 보장한다. + # 기존 가중합만 쓰면 ML 100점이어도 최종 35점에 그쳐 threat까지 못 올라간다. + if ml_score >= 85: + final_score = max(final_score, 65) + elif ml_score >= 65: + final_score = max(final_score, 45) + + # 외부 API가 명확히 위협이라고 한 경우에는 최소 점수를 보장한다. + if safe_dict(safe_browsing).get(ctx.GSB_is_threat): + final_score = max(final_score, 85) + + if safe_dict(otx).get(ctx.OTX_is_threat): + final_score = max(final_score, 65) + + combined_caution_signals = collect_combined_caution_signals(context) + if len(combined_caution_signals) >= COMBINED_CAUTION_SIGNAL_THRESHOLD: + final_score = max(final_score, MIN_CAUTION_SCORE) + + final_score = clamp_score(final_score) + risk_level = decide_risk_level(final_score, safe_browsing, otx) + + reason_summary = unique_values(internal_reasons + external_reasons) + if len(combined_caution_signals) >= COMBINED_CAUTION_SIGNAL_THRESHOLD: + reason_summary.append("combined_minimum_caution") + matched_rules = collect_matched_rules(context) + + return build_scoring_result( + internal_score=internal_score, + ml_score=ml_score, + external_score=external_score, + score=final_score, + risk_level=risk_level, + reason_summary=reason_summary, + matched_rules=matched_rules + ) + + +# 각 점수 파트가 터져도 최종 scoring은 계속 진행되도록 방어한다. +def safe_score_part(func, fallback_reason: str) -> tuple[int, list[str]]: + try: + score, reasons = func() + return clamp_score(score), normalize_list(reasons) + except Exception: + return 0, [fallback_reason] + + +# 내부 로직 결과 점수 +def score_internal(context: AnalysisContext) -> tuple[int, list[str]]: + score = 0 + reasons = [] + + detection = safe_dict(context.detection) + shortened = safe_dict(detection.get(ctx.Det_short)) + if shortened.get(ctx.Short_det): + score += 15 + reasons.append("shortened_url") + + redirect = safe_dict(context.redirect) + redirect_count = safe_int(redirect.get(ctx.Red_count), 0) + + if redirect_count == 1: + score += 5 + reasons.append("redirect_1") + elif redirect_count == 2: + score += 10 + reasons.append("redirect_2") + elif redirect_count >= 3: + score += 15 + reasons.append("redirect_3_or_more") + + if bool(redirect.get(ctx.Red_loop_det)): + score += 30 + reasons.append("redirect_loop_detected") + + redirect_status = status_value(redirect.get(ctx.Red_status)) + if redirect_status == "too_many_redirects": + score += 25 + reasons.append("too_many_redirects") + + rule_based = safe_dict(detection.get(ctx.Det_rule)) + flags = normalize_list(rule_based.get(ctx.Rule_flags)) + + flag_scores = { + "suspicious_query_param_detected": 12, + "embedded_url": 15, + "suspicious_query_keyword_detected": 8, + "suspicious_path_keyword_detected": 8, + "suspicious_fragment_keyword_detected": 8, + "percent_encoding_detected": 8, + "double_encoding_suspected": 15, + } + + for flag in flags: + score += flag_scores.get(flag, 0) + reasons.append(flag) + + return clamp_score(score), reasons + + +def safe_score_ml(context: AnalysisContext) -> int: + try: + return score_ml(context) + except Exception: + return 0 + + +def score_ml(context: AnalysisContext) -> int: + ml_result = safe_dict(context.ml) + score = ml_result.get(ctx.Ml_score) + + # CharCNN 또는 XGB 중 하나라도 사용 가능하면 fusion score를 사용한다. + # 한 모델이 unavailable이어도 다른 모델 결과를 버리면 안 된다. + charcnn = safe_dict(ml_result.get("charcnn")) + xgb = safe_dict(ml_result.get("xgb")) + fusion = safe_dict(ml_result.get("fusion")) + + if fusion.get("available") is True: + return clamp_score(fusion.get("score", score)) + + if charcnn.get("available") is True or xgb.get("available") is True: + return clamp_score(safe_float(score, 0.0)) + + return 0 + + +# 외부 조회 결과 점수 +def score_external(context: AnalysisContext) -> tuple[int, list[str]]: + score = 0 + reasons = [] + + external = safe_dict(context.external) + + safe_browsing = safe_dict(external.get(ctx.EX_GSB)) + if safe_browsing.get(ctx.GSB_is_threat): + score += 40 + reasons.append("gsb_threat_detected") + + otx = safe_dict(external.get(ctx.EX_OTX)) + pulse_count = safe_int(otx.get(ctx.OTX_pulse_count), 0) + + if otx.get(ctx.OTX_is_threat): + score += 25 + reasons.append("otx_threat_detected") + + if 1 <= pulse_count <= 2: + score += 5 + reasons.append("otx_count_low") + elif 3 <= pulse_count <= 5: + score += 10 + reasons.append("otx_count_mid") + elif pulse_count >= 6: + score += 15 + reasons.append("otx_count_high") + + cert_score, cert_reasons = score_certificate(context) + score += cert_score + reasons.extend(cert_reasons) + + whois = safe_dict(external.get(ctx.EX_WHOIS)) + domain_age_days = safe_int(whois.get(ctx.WHO_age), 0) + + if domain_age_days > 0: + if domain_age_days <= 30: + score += 15 + reasons.append("very_young_domain") + elif domain_age_days <= 180: + score += 8 + reasons.append("young_domain") + + server_type = external.get(ctx.Ex_server_type) + server_status = external.get(ctx.Ex_server_status) + + # 서버 타입이 실제로 조회됐는데 unknown/빈 값이면 약한 가중치만 준다. + # 아무 조회도 안 된 기본 None 상태는 점수로 보지 않는다. + if server_status is not None and server_type in (None, "", "unknown"): + score += 3 + reasons.append("unknown_server_type") + + return clamp_score(score), reasons + + +def score_certificate(context: AnalysisContext) -> tuple[int, list[str]]: + certificate = safe_dict(context.certificate) + status = certificate.get(ctx.Cert_status) + errors = normalize_list(certificate.get(ctx.Cert_error)) + + # 인증서 검사를 아직 수행하지 않은 기본 상태. + if status is None and not errors: + return 0, [] + + # 정상 인증서. + if certificate.get(ctx.Cert_valid) is True or status in (stat.SUCCESS, "cert_success"): + return 0, [] + + risky_status_scores = { + stat.CERT_UNTRUSTED: 15, + stat.CERT_EXPIRED: 15, + stat.CERT_HOSTNAME_MISMATCH: 12, + stat.CERT_NOT_YET_VALID: 10, + } + + weak_status_scores = { + stat.CERT_SSL_ERROR: 5, + stat.CERT_LOOKUP_FAILED: 3, + stat.CERT_CONNECTION_FAILED: 3, + stat.CERT_TIMEOUT: 2, + stat.CERT_INVALID_HOST: 0, + stat.CERT_UNKNOWN_ERROR: 3, + stat.NO_CERTIFICATE: 5, + } + + error_scores = { + Err.CERT_SELF_SIGNED: 15, + Err.CERT_UNTRUSTED: 15, + Err.CERT_EXPIRED: 15, + Err.CERT_HOSTNAME_MISMATCH: 12, + Err.CERT_NOT_YET_VALID: 10, + Err.CERT_REVOKED: 15, + Err.CERT_SSL_ERROR: 5, + Err.CERT_LOOKUP_FAILED: 3, + Err.CERT_CONNECTION_FAILED: 3, + Err.CERT_TIMEOUT: 2, + Err.CERT_INVALID_HOST: 0, + Err.CERT_NO_CERTIFICATE: 5, + Err.CERT_UNKNOWN_ERROR: 3, + } + + score_candidates = [] + + if status in risky_status_scores: + score_candidates.append(risky_status_scores[status]) + elif status in weak_status_scores: + score_candidates.append(weak_status_scores[status]) + elif certificate.get(ctx.Cert_valid) is False: + # status가 없거나 모르는 값인데 valid만 False인 경우. 과가중하지 않는다. + score_candidates.append(3) + + for error in errors: + score_candidates.append(error_scores.get(error, 3)) + + score = max(score_candidates) if score_candidates else 0 + + reasons = [] + if score > 0: + # reason에는 raw error가 아니라 표준 status/error 라벨만 남긴다. + if status: + reasons.append(str(status)) + reasons.extend(str(error) for error in errors) + + return clamp_score(score), unique_values(reasons) + + +# 의심 신호가 여러 건 탐지되었을 때 최소 risklevel을 보장 +def collect_combined_caution_signals(context: AnalysisContext) -> list[str]: + signals = [] + + certificate = safe_dict(context.certificate) + for error in normalize_list(certificate.get(ctx.Cert_error)): + if error: + signals.append(str(error)) + + redirect = safe_dict(context.redirect) + if bool(redirect.get(ctx.Red_loop_det)): + signals.append("loop_detected") + + for status in collect_status_values( + redirect.get(ctx.Red_status), + redirect.get("status_type"), + redirect.get("result"), + redirect.get("result_type") + ): + if status in REDIRECT_RISK_STATUSES: + signals.append(status) + + external = safe_dict(context.external) + server_statuses = collect_status_values( + external.get(ctx.Ex_server_status), + external.get("server_status") + ) + if any(status in SERVER_INFO_FAILED_STATUSES for status in server_statuses): + signals.append(SERVER_INFO_FAILED_LABEL) + + ml_result = safe_dict(context.ml) + ml_threats = normalize_list(ml_result.get(ctx.Ml_threats)) + charcnn = safe_dict(ml_result.get("charcnn")) + charcnn_threats = normalize_list(charcnn.get("threats")) + + if CHARCNN_SUSPICIOUS_LABEL in ml_threats or CHARCNN_SUSPICIOUS_LABEL in charcnn_threats: + signals.append(CHARCNN_SUSPICIOUS_LABEL) + + return unique_values(signals) + + +def collect_status_values(*values) -> list[str]: + statuses = [] + + for value in values: + if value is None: + continue + + if isinstance(value, (list, tuple, set)): + status = status_value(value) + if status: + statuses.append(status) + + for item in value: + item_status = status_value(item) + if item_status: + statuses.append(item_status) + continue + + status = status_value(value) + if status: + statuses.append(status) + + return unique_values(statuses) + + +# 위험도 반환 +def decide_risk_level(final_score: int, safe_browsing: dict, otx: dict) -> str: + final_score = clamp_score(final_score) + safe_browsing = safe_dict(safe_browsing) + otx = safe_dict(otx) + + if safe_browsing.get(ctx.GSB_is_threat): + return lev.THREAT + + if otx.get(ctx.OTX_is_threat) and final_score >= 60: + return lev.THREAT + + if final_score >= 60: + return lev.THREAT + if final_score >= 30: + return lev.CAUTION + return lev.SAFE + + +def collect_matched_rules(context: AnalysisContext) -> list[str]: + matched = [] + + detection = safe_dict(context.detection) + + shortened = safe_dict(detection.get(ctx.Det_short)) + if shortened.get(ctx.Short_det): + matched.append("shortened_url") + + rule_based = safe_dict(detection.get(ctx.Det_rule)) + matched.extend(normalize_list(rule_based.get(ctx.Rule_flags))) + + external = safe_dict(context.external) + + safe_browsing = safe_dict(external.get(ctx.EX_GSB)) + matched.extend(normalize_list(safe_browsing.get(ctx.GSB_threats))) + + otx = safe_dict(external.get(ctx.EX_OTX)) + matched.extend(normalize_list(otx.get(ctx.OTX_threats))) + + ml_result = safe_dict(context.ml) + matched.extend(normalize_list(ml_result.get(ctx.Ml_threats))) + + certificate = safe_dict(context.certificate) + matched.extend(normalize_list(certificate.get(ctx.Cert_error))) + + return unique_values(matched) + + +def build_scoring_result(**overrides) -> dict: + result = { + "internal_score": 0, + "ml_score": 0, + "external_score": 0, + "score": 0, + "risk_level": lev.SAFE, + "reason_summary": [], + "matched_rules": [] + } + + for key in overrides: + if key not in result: + raise KeyError(f"허용되지 않은 scoring 필드: {key}") + + result.update(overrides) + + result["internal_score"] = clamp_score(result.get("internal_score")) + result["ml_score"] = clamp_score(result.get("ml_score")) + result["external_score"] = clamp_score(result.get("external_score")) + result["score"] = clamp_score(result.get("score")) + + if result.get("risk_level") not in (lev.THREAT, lev.CAUTION, lev.SAFE): + result["risk_level"] = lev.SAFE + + result["reason_summary"] = unique_values(normalize_list(result.get("reason_summary"))) + result["matched_rules"] = unique_values(normalize_list(result.get("matched_rules"))) + + return result + + +def safe_dict(value) -> dict: + return value if isinstance(value, dict) else {} + + +def normalize_list(value) -> list: + if value is None: + return [] + + if isinstance(value, list): + return [item for item in value if item is not None] + + if isinstance(value, tuple) or isinstance(value, set): + return [item for item in value if item is not None] + + return [value] + + +def unique_values(values: list) -> list: + result = [] + + for value in values: + if value is None: + continue + + if value not in result: + result.append(value) + + return result + + +def safe_int(value, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def safe_float(value, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def clamp_score(value) -> int: + return max(0, min(100, int(round(safe_float(value, 0.0))))) + + +def status_value(status) -> str | None: + if status is None: + return None + + if isinstance(status, (list, tuple)): + if len(status) >= 2: + return str(status[1]).lower() + if len(status) == 1: + return str(status[0]).lower() + return None + + return str(status).lower() \ No newline at end of file diff --git a/services/url_utils.py b/services/url_utils.py index c31427a..750ffe7 100644 --- a/services/url_utils.py +++ b/services/url_utils.py @@ -1 +1,148 @@ -# URL을 분해하고, 정규화하고, 필요하다면 관련 다른 기능들도 제공합니다. \ No newline at end of file +# URL을 분해하고, 정규화하고, 필요하다면 관련 다른 기능들도 제공합니다. +from urllib.parse import urlsplit, urlunsplit +from core.context import AnalysisContext + + +#입력 URL의 앞뒤 공백/탭/개행 제거 +def sanitize_input_url(raw_url: str | None) -> str: + if raw_url is None: + raise ValueError("URL 입력값이 None입니다.") + + cleaned_url = raw_url.strip() + if not cleaned_url: + raise ValueError("URL 입력값이 비어 있습니다.") + + return cleaned_url + + +#URL을 구성 요소별로 분해 +def parse_url_components(url: str) -> dict: + + split_result = urlsplit(url) # urlsplit() 결과 객체 + + scheme = split_result.scheme # URL scheme + netloc = split_result.netloc # 도메인+포트 + hostname = split_result.hostname # 순수 도메인 + port = split_result.port # 포트 번호 + path = split_result.path # 경로 + query = split_result.query # 쿼리 문자열 + fragment = split_result.fragment # fragment + + return { + "scheme": scheme, + "netloc": netloc, + "hostname": hostname, + "port": port, + "path": path, + "query": query, + "fragment": fragment + } #parsed_result + + +#URL에 scheme(http,https 등)이 존재하는지 확인 +#없을 시 https 부착 **검사 진행 위한 구조 정상화 목적** +#소문자화 scheme, scheme 존재 여부, https 여부를 반환합니다. +def normalize_scheme(parsed_result:dict): + scheme = parsed_result["scheme"] # scheme + scheme = scheme.lower() # 소문자로 저장 + scheme_exists = bool(scheme) # 보정 전 scheme 존재 여부 + is_https = (scheme == "https") # https 여부 (보정 전 기준) + + #scheme 미존재시 검사용으로 https 부착 + if scheme_exists is False: + scheme = "https" + + return { + "normalized_scheme": scheme, + "had_scheme_originally": scheme_exists, # 원본 scheme 존재 여부 + "is_https": is_https # 원본 url이 https였는지 여부 + } + + +# hostname 소문자화, 기본 포트 여부 판단 +# 수정 : 기본 포트일 시 기본포트 제거 부분 삭제. +# 사유 : feature 추출, server/인증서 조회 사용에 수월함 +def normalize_url_components(parsed_result: dict, normalized_scheme: str): + hostname = parsed_result["hostname"] # hostname + port = parsed_result["port"] # port + is_default_port = False # 기본 포트 여부(http:80/https:443) + + if hostname != None: + hostname = hostname.lower() + else: + hostname = "" + + # 기본 포트 여부 확인(http/https의 경우만) + if normalized_scheme == "http" and port == 80: + is_default_port = True + + elif normalized_scheme == "https" and port == 443: + is_default_port = True + + return { + "normalized_hostname": hostname, + "port": port, + "is_default_port": is_default_port, # 기본 포트 여부 + } #normalized_comp + + +#URL 재구성 +def rebuild_normalized_url(parsed_result: dict, normalized_scheme: str, normalized_comp: dict): + # hostname+port + if normalized_comp["port"] is None: + netloc = normalized_comp["normalized_hostname"] + else: + netloc = f"{normalized_comp['normalized_hostname']}:{normalized_comp['port']}" + + # normalized url (output) + integrated_url = urlunsplit(( + normalized_scheme, + netloc, + parsed_result["path"], + parsed_result["query"], + parsed_result["fragment"])) + + return { + "netloc": netloc, + "normalized_url": integrated_url + } + + +# context.url에 정규화 결과를 기록하는 메인 함수 +def normalize_url(context: AnalysisContext) -> None: + # 단축 URL이면 리다이렉트 후 최종 URL을 사용, 아니면 원본 URL 사용 + if context.detection["shortened_url"]["detected"]: + target_url = ( + context.redirect.get("final_url") + or context.request.get("original_url") + ) + else: + target_url = context.request.get("original_url") + + cleaned_url = sanitize_input_url(target_url) + parsed_result = parse_url_components(cleaned_url) + scheme_result = normalize_scheme(parsed_result) + normalized_comp = normalize_url_components( + parsed_result, + scheme_result["normalized_scheme"] + ) + rebuilt_result = rebuild_normalized_url( + parsed_result, + scheme_result["normalized_scheme"], + normalized_comp + ) + + # context.url에 최종 결과 기록 + context.url["normalized_url"] = rebuilt_result["normalized_url"] + context.url["scheme"] = scheme_result["normalized_scheme"] + context.url["hostname"] = normalized_comp["normalized_hostname"] + context.url["port"] = normalized_comp["port"] + context.url["path"] = parsed_result["path"] + context.url["query"] = parsed_result["query"] + context.url["fragment"] = parsed_result["fragment"] + context.url["is_https"] = scheme_result["is_https"] + context.url["has_explicit_scheme"] = scheme_result["had_scheme_originally"] + context.url["is_default_port"] = normalized_comp["is_default_port"] + + # 정규화 수행 완료 표시 + context.meta["normalized"] = True diff --git a/test/api_test.py b/test/api_test.py new file mode 100644 index 0000000..2dde98a --- /dev/null +++ b/test/api_test.py @@ -0,0 +1,35 @@ +# req_api 테스트 코드 +from req_api import req_api # 제공해주신 코드가 담긴 파일명으로 변경하세요. + +# 1. AnalysisContext를 흉내 내는 Mock 클래스 생성 +class MockContext: + def __init__(self, url: str): + # 원본 코드에서 context.url.get("normalized_url") 형태로 접근하므로 딕셔너리로 만듭니다. + self.url = {"normalized_url": url} + +def run_test(): + # 테스트용 URL 리스트 + test_urls = [ + # 1. 정상 URL 테스트 + "https://www.google.com", + + # 2. 악성 URL 테스트 (Google이 제공하는 Safe Browsing 테스트용 안전한 악성 URL) + "https://ey43.com/4/10379548" + ] + + for target_url in test_urls: + print(f"\n[테스트 시작] URL: {target_url}") + + # Mock Context 객체 생성 + context = MockContext(target_url) + + # 핵심 함수 호출 + result = req_api(context) + + # 결과 출력 + print("결과:") + import json + print(json.dumps(result, indent=4, ensure_ascii=False)) + +if __name__ == "__main__": + run_test() \ No newline at end of file diff --git a/test/test_api.py b/test/test_api.py new file mode 100644 index 0000000..8afa28b --- /dev/null +++ b/test/test_api.py @@ -0,0 +1,18 @@ +import asyncio +from analyzers.safe_browsing import check_url_safety + +async def run_test(): + # 1. 안전한 URL 테스트 + print("--- Test 1: Safe URL ---") + safe_url = "https://www.google.com" + result_safe = await check_url_safety(safe_url) + print(f"Result: {result_safe}\n") + + # 2. 위험한 URL 테스트 (구글 제공 테스트용 악성 URL) + print("--- Test 2: Dangerous URL ---") + malware_url = "http://testsafebrowsing.appspot.com/s/malware.html" + result_danger = await check_url_safety(malware_url) + print(f"Result: {result_danger}") + +if __name__ == "__main__": + asyncio.run(run_test()) \ No newline at end of file