From 16c0751ef0885c7ba24a6c7312c4ba143df9e211 Mon Sep 17 00:00:00 2001 From: zhengyang Date: Tue, 3 Mar 2026 16:51:40 +0800 Subject: [PATCH 1/5] [TOOL] Add tools to sync code from triton-ascend --- tools/sync_ascend/clean.sh | 22 + tools/sync_ascend/path_mapper.py | 361 +++++++++++++ tools/sync_ascend/sync_from_triton_ascend.sh | 507 +++++++++++++++++++ 3 files changed, 890 insertions(+) create mode 100755 tools/sync_ascend/clean.sh create mode 100644 tools/sync_ascend/path_mapper.py create mode 100755 tools/sync_ascend/sync_from_triton_ascend.sh diff --git a/tools/sync_ascend/clean.sh b/tools/sync_ascend/clean.sh new file mode 100755 index 000000000..036e2112d --- /dev/null +++ b/tools/sync_ascend/clean.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# 清理同步代码，恢复到同步之前 +# +# 用法: +# bash tools/sync_triton_ascend.sh [BASE_COMMIT] + +set -e + +git stash +git clean -xdf bin/ +git clean -xdf include/ +git clean -xdf lib/ +git clean -xdf python/ +git clean -xdf test/ +git clean -xdf third_party/ascend/ + +pushd third_party/flir/ +git stash +git clean -xdf include/ +git clean -xdf lib/ +popd diff --git a/tools/sync_ascend/path_mapper.py b/tools/sync_ascend/path_mapper.py new file mode 100644 index 000000000..d7615265c --- /dev/null +++ b/tools/sync_ascend/path_mapper.py @@ -0,0 +1,361 @@ +""" +读取 triton-ascend 的文件路径（相对仓库根），输出 JSON 格式的目标列表。 +每条记录：{"repo": "flagtree"|"flir", "path": "<相对目标仓库根的路径>"} +""" +import sys +import os +import json + +# ============================================================== +# 忽略规则（静默跳过，记录到 skipped.txt 但不报警告） +# +# SKIP_PREFIXES : 路径以这些前缀开头时忽略 +# SKIP_EXACT : 路径完全匹配时忽略 +# 注：triton-ascend 根目录下的文件（路径中不含 /）单独检测 +# ============================================================== +SKIP_PREFIXES = [ + 'docker/', + 'docs/', + 'third_party/ascend/AscendNPU-IR/', +] + +SKIP_EXACT = { + 'third_party/ascend/.gitignore', +} + +# ============================================================== +# 新增文件需人工处理的路径前缀 +# 当一个文件是新增（created.txt）且路径匹配以下前缀时， +# 在 created.txt 中标注 (需人工处理) +# ============================================================== +MANUAL_REVIEW_PREFIXES = [ + 'bin/', + 'include/', + 'lib/', + 'python/', + 'test/', +] + +# ============================================================== +# 前缀映射规则（PATH_PREFIX_RULES） +# +# 每条规则字段： +# src : triton-ascend 侧路径前缀（必填） +# dst : 目标路径前缀，strip src 后拼接（必填） +# dst_repo : 目标仓库，'flagtree' 或 'flir'（必填） +# rename : basename 重命名表 {old: new}，不填则不做重命名 +# +# 规则按顺序匹配，第一条命中即返回结果。 +# 需要同时写入多个目标（如 LIB_BOTH）的情形使用下方文件集合配置。 +# ============================================================== +PATH_PREFIX_RULES = [ + # ---------------------------------------------------------------- + # third_party/ascend/include/ → flir/include/ + # ---------------------------------------------------------------- + { + 'src': 'third_party/ascend/include/Dialect/TritonAscend/IR/', 'dst': 'include/npu/Dialect/TritonAscend/IR/', + 'dst_repo': 'flir' + }, + { + 'src': 'third_party/ascend/include/Dialect/TritonStructured/IR/', 'dst': + 'include/incubated/Dialect/TritonStructuredIncubated/IR/', 'dst_repo': 'flir', 'rename': { + 'TritonStructuredDialect.h': 'TritonStructuredDialectIncubated.h', + 'TritonStructuredDialect.td': 'TritonStructuredDialectIncubated.td', + } + }, + { + 'src': 'third_party/ascend/include/DiscreteMaskAccessConversion/', 'dst': + 'include/incubated/Conversion/DiscreteMaskAccessConversion/', 'dst_repo': 'flir' + }, + { + 'src': 'third_party/ascend/include/TritonToAnnotation/', 'dst': + 'include/incubated/Conversion/TritonToAnnotation/', 'dst_repo': 'flir' + }, + { + 'src': 'third_party/ascend/include/TritonToLinalg/', 'dst': + 'include/incubated/Conversion/TritonToLinalgIncubated/', 'dst_repo': 'flir', 'rename': { + 'TritonToLinalgPass.h': 'TritonToLinalgIncubatedPass.h', + } + }, + { + 'src': 'third_party/ascend/include/TritonToStructured/', 'dst': + 'include/incubated/Conversion/TritonToStructuredIncubated/', 'dst_repo': 'flir', 'rename': { + 'TritonToStructuredPass.h': 'TritonToStructuredIncubatedPass.h', + } + }, + { + 'src': 'third_party/ascend/include/TritonToUnstructure/', 'dst': + 'include/incubated/Conversion/TritonToUnstructureIncubated/', 'dst_repo': 'flir' + }, + { + 'src': 'third_party/ascend/include/Utils/', 'dst': 'include/incubated/Conversion/UtilsIncubated/', 'dst_repo': + 'flir' + }, + + # ---------------------------------------------------------------- + # third_party/ascend/include/ → flagtree/third_party/ascend/include/ + # ---------------------------------------------------------------- + { + 'src': 'third_party/ascend/include/TritonToHFusion/', 'dst': 'third_party/ascend/include/TritonToHFusion/', + 'dst_repo': 'flagtree' + }, + { + 'src': 'third_party/ascend/include/TritonToHIVM/', 'dst': 'third_party/ascend/include/TritonToHIVM/', + 'dst_repo': 'flagtree' + }, + { + 'src': 'third_party/ascend/include/TritonToLLVM/', 'dst': 'third_party/ascend/include/TritonToLLVM/', + 'dst_repo': 'flagtree' + }, + + # ---------------------------------------------------------------- + # third_party/ascend/lib/ → flir/lib/ + # ---------------------------------------------------------------- + { + 'src': 'third_party/ascend/lib/Dialect/TritonAscend/IR/', 'dst': 'lib/Dialect/TritonAscend/IR/', 'dst_repo': + 'flir' + }, + { + 'src': 'third_party/ascend/lib/Dialect/TritonStructured/IR/', 'dst': + 'lib/Dialect/TritonStructuredIncubated/IR/', 'dst_repo': 'flir', 'rename': { + 'TritonStructuredDialect.cpp': 'TritonStructuredDialectIncubated.cpp', + 'TritonStructuredOps.cpp': 'TritonStructuredOpsIncubated.cpp', + } + }, + { + 'src': 'third_party/ascend/lib/DiscreteMaskAccessConversion/', 'dst': + 'lib/Conversion/DiscreteMaskAccessConversion/', 'dst_repo': 'flir' + }, + { + 'src': 'third_party/ascend/lib/TritonToAnnotation/', 'dst': 'lib/Conversion/TritonToAnnotation/', 'dst_repo': + 'flir' + }, + { + 'src': 'third_party/ascend/lib/TritonToLinalg/', 'dst': 'lib/Conversion/TritonToLinalgIncubated/', 'dst_repo': + 'flir', 'rename': { + 'TritonToLinalgPass.cpp': 'TritonToLinalgIncubatedPass.cpp', + } + }, + { + 'src': 'third_party/ascend/lib/TritonToStructured/', 'dst': 'lib/Conversion/TritonToStructuredIncubated/', + 'dst_repo': 'flir', 'rename': { + 'TritonToStructuredPass.cpp': 'TritonToStructuredIncubatedPass.cpp', + } + }, + { + 'src': 'third_party/ascend/lib/TritonToUnstructure/', 'dst': 'lib/Conversion/TritonToUnstructureIncubated/', + 'dst_repo': 'flir' + }, + {'src': 'third_party/ascend/lib/Utils/', 'dst': 'lib/UtilsIncubated/', 'dst_repo': 'flir'}, + + # ---------------------------------------------------------------- + # third_party/ascend/lib/ → flagtree/third_party/ascend/lib/Conversion/ + # ---------------------------------------------------------------- + { + 'src': 'third_party/ascend/lib/TritonToHFusion/', 'dst': 'third_party/ascend/lib/Conversion/TritonToHFusion/', + 'dst_repo': 'flagtree' + }, + { + 'src': 'third_party/ascend/lib/TritonToHIVM/', 'dst': 'third_party/ascend/lib/Conversion/TritonToHIVM/', + 'dst_repo': 'flagtree' + }, + { + 'src': 'third_party/ascend/lib/TritonToLLVM/', 'dst': 'third_party/ascend/lib/Conversion/TritonToLLVM/', + 'dst_repo': 'flagtree' + }, + + # ---------------------------------------------------------------- + # include/runtime/libentry/ → spec/include/ + # ---------------------------------------------------------------- + { + 'src': 'include/runtime/libentry/', 'dst': 'third_party/ascend/backend/spec/include/runtime/libentry/', + 'dst_repo': 'flagtree' + }, + + # ---------------------------------------------------------------- + # third_party/ascend/unittest/ → flagtree 同路径 + # ---------------------------------------------------------------- + {'src': 'third_party/ascend/unittest/', 'dst': 'third_party/ascend/unittest/', 'dst_repo': 'flagtree'}, +] + +# ============================================================== +# 文件集合配置（用于需要按文件名分流的目录） +# ============================================================== + +# include/triton/Dialect/Triton/IR/ 下： +# 仅走 spec 的文件（其余未列出的归主干） +INCLUDE_TRITON_IR_SPEC = { + 'OpInterfaces.h', + 'TritonAttrDefs.td', + 'TritonOpInterfaces.td', + 'TritonOps.td', + 'TritonTypes.td', +} + +# lib/ 下： +# 只走 spec（主干无对应） +LIB_SPEC_ONLY = { + 'runtime/libentry/libentry.cpp', +} +# 同时写入主干 + spec +LIB_BOTH = { + 'Dialect/Triton/IR/Dialect.cpp', + 'Dialect/Triton/IR/Ops.cpp', + 'Dialect/Triton/IR/Traits.cpp', +} + +# python/triton/ 下： +# 只走 spec（主干标"不使用"） +PYTHON_TRITON_SPEC = { + 'compiler/code_generator.py', + 'compiler/compiler.py', + 'compiler/errors.py', + 'language/_utils.py', + 'language/core.py', + 'language/math.py', + 'language/semantic.py', + 'language/standard.py', + 'runtime/autotuner.py', + 'runtime/code_cache.py', + 'runtime/interpreter.py', + 'runtime/jit.py', + 'runtime/libentry.py', +} +# 只走主干（spec 无对应） +PYTHON_TRITON_MAIN = { + 'compiler/__init__.py', + 'language/__init__.py', + 'runtime/__init__.py', +} + +# python/src/ 下： +# 走 spec（third_party/ascend/python/src/） +PYTHON_SRC_SPEC = {'ir.cc', 'ir.h', 'main.cc'} +# 走主干 +PYTHON_SRC_MAIN = {'interpreter.cc', 'llvm.cc', 'passes.cc', 'passes.h'} + +# ============================================================== +# 辅助函数 +# ============================================================== + + +def _apply_rename(rename_table, path): + """对路径的 basename 应用重命名表""" + dn = os.path.dirname(path) + bn = os.path.basename(path) + new_bn = rename_table.get(bn, bn) + return os.path.join(dn, new_bn) if dn else new_bn + + +def _apply_rule(rule, path): + """对单条 PATH_PREFIX_RULES 规则求目标路径列表，未命中返回 None""" + src = rule['src'] + if not path.startswith(src): + return None + rest = path[len(src):] + rename_table = rule.get('rename') + if rename_table: + rest = _apply_rename(rename_table, rest) + return [{'repo': rule['dst_repo'], 'path': rule['dst'] + rest}] + + +# ============================================================== +# 主映射函数 +# ============================================================== + + +def map_path(triton_path): + """ + 返回 [{"repo": ..., "path": ...}, ...] 列表。 + 返回 'skip' 表示静默跳过（记录到 skipped.txt，不报警告）。 + 返回 None 表示未知路径（报警告）。 + """ + p = triton_path + + # 根目录文件（路径中不含 /） + if '/' not in p: + return 'skip' + + # SKIP_EXACT + if p in SKIP_EXACT: + return [] + + # SKIP_PREFIXES + # 同时处理带尾部 '/' 的前缀（目录内文件）和不带 '/' 的整体路径（如子模块引用） + for pfx in SKIP_PREFIXES: + if p.startswith(pfx) or p == pfx.rstrip('/'): + return 'skip' + + # PATH_PREFIX_RULES + for rule in PATH_PREFIX_RULES: + result = _apply_rule(rule, p) + if result is not None: + return result + + # third_party/ascend/ 其余文件：直接同路径到 flagtree + # (include/ 和 lib/ 已在上方规则中处理，不会走到这里) + if p.startswith('third_party/ascend/'): + return [{'repo': 'flagtree', 'path': p}] + + # include/ 文件 + if p.startswith('include/'): + rest = p[len('include/'):] + + if rest.startswith('triton/Dialect/Triton/IR/'): + bn = os.path.basename(rest) + if bn in INCLUDE_TRITON_IR_SPEC: + return [{'repo': 'flagtree', 'path': f'third_party/ascend/backend/spec/include/{rest}'}] + return [{'repo': 'flagtree', 'path': p}] + + return [{'repo': 'flagtree', 'path': p}] + + # lib/ 文件 + if p.startswith('lib/'): + rest = p[len('lib/'):] + if rest in LIB_SPEC_ONLY: + return [{'repo': 'flagtree', 'path': f'third_party/ascend/backend/spec/lib/{rest}'}] + if rest in LIB_BOTH: + return [ + {'repo': 'flagtree', 'path': p}, + {'repo': 'flagtree', 'path': f'third_party/ascend/backend/spec/lib/{rest}'}, + ] + return [{'repo': 'flagtree', 'path': p}] + + # python/triton/ 文件 + if p.startswith('python/triton/'): + rest = p[len('python/triton/'):] + if rest in PYTHON_TRITON_SPEC: + return [{'repo': 'flagtree', 'path': f'third_party/ascend/backend/spec/triton/{rest}'}] + # compiler/、language/、runtime/ 下的所有文件（含未在 PYTHON_TRITON_SPEC + # 中显式列出的新增文件）统一走 spec + if (rest.startswith('compiler/') or rest.startswith('language/') or rest.startswith('runtime/')): + return [{'repo': 'flagtree', 'path': f'third_party/ascend/backend/spec/triton/{rest}'}] + return [{'repo': 'flagtree', 'path': p}] + + # python/src/ 文件 + if p.startswith('python/src/'): + rest = p[len('python/src/'):] + if rest in PYTHON_SRC_SPEC: + return [{'repo': 'flagtree', 'path': f'third_party/ascend/python/src/{rest}'}] + return [{'repo': 'flagtree', 'path': p}] + + # bin/ 文件：直接同路径到 flagtree + if p.startswith('bin/'): + return [{'repo': 'flagtree', 'path': p}] + + # 未匹配 + return None + + +if __name__ == '__main__': + triton_path = sys.argv[1] + result = map_path(triton_path) + manual = isinstance(result, list) and any(t['repo'] == 'flagtree' and t['path'].startswith(pfx) + for t in result + for pfx in MANUAL_REVIEW_PREFIXES) + if result is None: + print(json.dumps({'status': 'unknown', 'targets': [], 'manual_review': False})) + elif result == 'skip': + print(json.dumps({'status': 'skip', 'targets': [], 'manual_review': False})) + else: + print(json.dumps({'status': 'ok', 'targets': result, 'manual_review': manual})) diff --git a/tools/sync_ascend/sync_from_triton_ascend.sh b/tools/sync_ascend/sync_from_triton_ascend.sh new file mode 100755 index 000000000..877c60cab --- /dev/null +++ b/tools/sync_ascend/sync_from_triton_ascend.sh @@ -0,0 +1,507 @@ +#!/usr/bin/env bash +# +# 将 triton-ascend 仓库从 BASE_COMMIT 到 HEAD 的新 commits 同步到 FlagTree。 +# 使用 git merge-file 做三路合并，保留 FlagTree 的特化修改，冲突处留标记供人工处理。 +# +# 用法: +# 配置 TRITON_DIR, FLAGTREE_DIR, FLIR_DIR, TRITON_ASCEND_DIR +# bash tools/sync_from_triton_ascend.sh [BASE_COMMIT] +# +# BASE_COMMIT 默认为 29d243e（上次同步点）。 +# 脚本不会执行 git commit，所有变更保留为工作区修改。 + +set -euo pipefail + +# ============================================================ +# 配置 +# ============================================================ +TRITON_DIR="/home/zhengyang/git/triton-3.2.x" # 用于生成 flagtree 与 openai triton 的对比，可置空 +FLAGTREE_DIR="/home/zhengyang/FlagTree/flagtree-3.2.x" # 目标主仓库 flagtree +FLIR_DIR="${FLAGTREE_DIR}/third_party/flir" # 目标子仓库 flir：不要求必须置于 flagtree 目录中 +TRITON_ASCEND_DIR="/home/zhengyang/FlagTree/triton-ascend" # 源仓库 triton-ascend +BASE_COMMIT="${1:-29d243e}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MAPPER_PY="${SCRIPT_DIR}/path_mapper.py" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +WORK_DIR="/tmp/flagtree_sync_from_triton_ascend_${TIMESTAMP}" +REPORT_DIR="${FLAGTREE_DIR}/sync_report_${TIMESTAMP}" + +mkdir -p "${WORK_DIR}" "${REPORT_DIR}" + +MODIFIED_LIST="${REPORT_DIR}/modified.txt" +CREATED_LIST="${REPORT_DIR}/created.txt" +CONFLICT_LIST="${REPORT_DIR}/conflict.txt" +DELETED_LIST="${REPORT_DIR}/deleted.txt" +SKIPPED_LIST="${REPORT_DIR}/skipped.txt" + +> "${MODIFIED_LIST}" +> "${CREATED_LIST}" +> "${CONFLICT_LIST}" +> "${DELETED_LIST}" +> "${SKIPPED_LIST}" + +# 路径映射逻辑见 tools/path_mapper.py + + +# ============================================================ +# 工具函数 +# ============================================================ + +log_info() { echo -e "\033[0;34m[INFO]\033[0m $*"; } +log_ok() { echo -e "\033[0;32m[OK]\033[0m $*"; } +log_warn() { echo -e "\033[0;33m[WARN]\033[0m $*"; } +log_error() { echo -e "\033[0;31m[ERROR]\033[0m $*"; } +log_conflict(){ echo -e "\033[0;35m[CONFLICT]\033[0m $*"; } + +# 生成列表条目：路径相同时只展示一次，不同时展示同步方向 +# 用法：fmt_entry +fmt_entry() { + local repo="$1" triton_file="$2" mapped_path="$3" + if [[ "$repo" == "flagtree" && "$triton_file" == "$mapped_path" ]]; then + echo "[flagtree] ${mapped_path}" + else + echo "[${repo}] ${mapped_path} <- [triton-ascend] ${triton_file}" + fi +} + +# 根据 repo 类型返回仓库根目录 +repo_dir() { + local repo="$1" + if [[ "$repo" == "flir" ]]; then + echo "${FLIR_DIR}" + else + echo "${FLAGTREE_DIR}" + fi +} + +# 解析 Python 输出：返回 status +get_status() { + python3 -c "import sys,json; d=json.loads(sys.stdin.read()); print(d['status'])" +} + +get_targets() { + # 输出每行 "|" + python3 -c " +import sys, json +d = json.loads(sys.stdin.read()) +for t in d['targets']: + print(t['repo'] + '|' + t['path']) +" +} + +get_manual_review() { + python3 -c " +import sys, json +d = json.loads(sys.stdin.read()) +print('1' if d.get('manual_review') else '0') +" +} + +# 写入指定报告文件，若 MAP_JSON 标记了 manual_review 则追加 (需人工处理) +# 用法：_write_entry [extra_suffix] +_write_entry() { + local list_file="$1" repo="$2" triton_file="$3" mapped_path="$4" map_json="$5" + local extra_suffix="${6:-}" + local entry + entry=$(fmt_entry "${repo}" "${triton_file}" "${mapped_path}") + [[ -n "${extra_suffix}" ]] && entry="${entry} ${extra_suffix}" + if [[ "$(echo "${map_json}" | get_manual_review)" == "1" ]]; then + echo "${entry} (需人工处理)" >> "${list_file}" + else + echo "${entry}" >> "${list_file}" + fi +} + +# ============================================================ +# Step 1: 获取变更文件列表 +# ============================================================ +log_info "=== Triton-Ascend → FlagTree 同步脚本 ===" +log_info "基准 commit: ${BASE_COMMIT}" +log_info "triton-ascend: ${TRITON_ASCEND_DIR}" +log_info "FlagTree: ${FLAGTREE_DIR}" +log_info "FLIR: ${FLIR_DIR}" +echo "" + +cd "${TRITON_ASCEND_DIR}" + +CURRENT_COMMIT=$(git rev-parse HEAD) +log_info "triton-ascend 当前 HEAD: ${CURRENT_COMMIT}" + +# 确认 BASE_COMMIT 存在 +if ! git rev-parse --verify "${BASE_COMMIT}" > /dev/null 2>&1; then + log_error "BASE_COMMIT '${BASE_COMMIT}' 在 triton-ascend 仓库中不存在，请检查。" + exit 1 +fi + +if [[ "${BASE_COMMIT}" == "$(git rev-parse HEAD)" ]]; then + log_warn "BASE_COMMIT 等于当前 HEAD，没有新的 commits 需要同步。" + exit 0 +fi + +log_info "新 commits 列表：" +git log --oneline "${BASE_COMMIT}..${CURRENT_COMMIT}" | sed 's/^/ /' +echo "" + +# 获取所有变更文件（含状态：A/M/D/R 等） +# 格式：\t<文件路径>（R 类型为 \t<旧路径>\t<新路径>） +CHANGED_FILES_RAW="${WORK_DIR}/changed_files_raw.txt" +git diff --name-status "${BASE_COMMIT}" "${CURRENT_COMMIT}" > "${CHANGED_FILES_RAW}" + +TOTAL=$(wc -l < "${CHANGED_FILES_RAW}") +log_info "共 ${TOTAL} 个文件变更" +echo "" + +# 生成 triton-ascend 基线到 HEAD 的完整 diff，存入报告目录 +TRITON_ASCEND_DIFF="${REPORT_DIR}/triton_ascend_changes.diff" +git diff "${BASE_COMMIT}" "${CURRENT_COMMIT}" > "${TRITON_ASCEND_DIFF}" +log_info "triton-ascend diff 已生成: ${TRITON_ASCEND_DIFF}" +echo "" + +# ============================================================ +# Step 2: 同步前，生成 flagtree 与官方 Triton 的对比 diff +# ============================================================ +TRITON_DIFF_DIR="${REPORT_DIR}/diff_with_openai_triton" +TRITON_DIFF_COUNT=0 + +if [[ ! -d "${TRITON_DIR}" ]]; then + log_warn "TRITON_DIR '${TRITON_DIR}' 不存在，跳过 Triton 代码比较" +else + log_info "[2/4] 生成同步前 flagtree 与官方 Triton 的对比..." + mkdir -p "${TRITON_DIFF_DIR}" + + while IFS=$'\t' read -r _st _f1 _f2; do + case "${_st:0:1}" in + D) continue ;; # 删除文件在 flagtree 侧无意义，跳过 + R) _tf="${_f2}" ;; # 重命名取新路径 + *) _tf="${_f1}" ;; + esac + + _mj=$(python3 "${MAPPER_PY}" "${_tf}") + [[ "$(echo "${_mj}" | get_status)" != "ok" ]] && continue + + while IFS='|' read -r _repo _mpath; do + [[ "$_repo" != "flagtree" ]] && continue + [[ "$_mpath" == third_party/* ]] && continue + + _ft_file="${FLAGTREE_DIR}/${_mpath}" + _triton_file="${TRITON_DIR}/${_mpath}" + + [[ ! -f "${_triton_file}" ]] && continue + [[ ! -f "${_ft_file}" ]] && continue + + mkdir -p "${TRITON_DIFF_DIR}/$(dirname "${_mpath}")" + _diff_file="${TRITON_DIFF_DIR}/${_mpath}.diff" + + if ! diff -u \ + --label "triton/${_mpath}" \ + --label "flagtree/${_mpath}" \ + "${_triton_file}" "${_ft_file}" > "${_diff_file}" 2>/dev/null; then + log_info " diff 已生成: ${_mpath}" + (( TRITON_DIFF_COUNT++ )) || true + else + # 无差异：保留文件，写入仅含标题行的空 diff + printf -- "--- triton/%s\n+++ flagtree/%s\n" "${_mpath}" "${_mpath}" > "${_diff_file}" + fi + done < <(echo "${_mj}" | get_targets) + done < "${CHANGED_FILES_RAW}" + + if [[ ${TRITON_DIFF_COUNT} -gt 0 ]]; then + log_ok "已生成 ${TRITON_DIFF_COUNT} 个与 openai triton 的对比 diff，目录: ${TRITON_DIFF_DIR}" + else + log_info "所有涉及文件与 openai triton 完全一致（或无对应文件）" + fi +fi +echo "" + +# ============================================================ +# Step 3: 逐文件处理（应用同步） +# ============================================================ +log_info "开始处理文件..." +echo "" + +MODIFIED_COUNT=0 +CREATED_COUNT=0 +CONFLICT_COUNT=0 +DELETED_COUNT=0 +SKIPPED_COUNT=0 + +while IFS=$'\t' read -r status file1 file2; do + # 处理 Rename（R）的情况：file1=旧路径，file2=新路径 + # 我们把重命名视为：删除 file1，添加 file2 + if [[ "${status}" == R* ]]; then + old_file="${file1}" + new_file="${file2}" + + # 处理旧路径（标记为已删除） + MAP_JSON=$(python3 "${MAPPER_PY}" "${old_file}") + MAP_STATUS=$(echo "${MAP_JSON}" | get_status) + if [[ "${MAP_STATUS}" == "ok" ]]; then + while IFS='|' read -r repo mapped_path; do + [[ -z "${mapped_path}" ]] && continue + _write_entry "${DELETED_LIST}" "${repo}" "${old_file}" "${mapped_path}" "${MAP_JSON}" "(RENAME_DEL)" + done < <(echo "${MAP_JSON}" | get_targets) + fi + + # 处理新路径（视为新增） + status="A" + file1="${new_file}" + fi + + triton_file="${file1}" + + # 查询路径映射 + MAP_JSON=$(python3 "${MAPPER_PY}" "${triton_file}") + MAP_STATUS=$(echo "${MAP_JSON}" | get_status) + + if [[ "${MAP_STATUS}" == "unknown" ]]; then + log_warn "未知路径（跳过）: [triton-ascend] ${triton_file}" + echo "[triton-ascend] ${triton_file} (未配置忽略路径)" >> "${SKIPPED_LIST}" + (( SKIPPED_COUNT++ )) || true + continue + fi + + if [[ "${MAP_STATUS}" == "skip" ]]; then + # 不在同步范围（如 test/ 目录），静默记录到 skipped + echo "[triton-ascend] ${triton_file}" >> "${SKIPPED_LIST}" + (( SKIPPED_COUNT++ )) || true + continue + fi + + TARGETS=$(echo "${MAP_JSON}" | get_targets) + if [[ -z "${TARGETS}" ]]; then + # 明确跳过（如 .gitignore），不记录 + log_info " 跳过（不同步）: ${triton_file}" + continue + fi + + # 遍历每个映射目标 + while IFS='|' read -r repo mapped_path; do + [[ -z "${mapped_path}" ]] && continue + + TARGET_REPO_DIR=$(repo_dir "${repo}") + TARGET_ABS="${TARGET_REPO_DIR}/${mapped_path}" + + # ---------- 文件删除 ---------- + if [[ "${status}" == "D" ]]; then + log_warn " 已删除（需人工确认是否同步删除）: ${triton_file}" + _write_entry "${DELETED_LIST}" "${repo}" "${triton_file}" "${mapped_path}" "${MAP_JSON}" + (( DELETED_COUNT++ )) || true + continue + fi + + # ---------- 文件新增 ---------- + if [[ "${status}" == "A" ]]; then + # 确保目标目录存在 + mkdir -p "$(dirname "${TARGET_ABS}")" + + # 从 triton-ascend 复制文件内容 + TRITON_NEW_CONTENT="${WORK_DIR}/triton_new_$(echo "${triton_file}" | tr '/' '_')" + git show "${CURRENT_COMMIT}:${triton_file}" > "${TRITON_NEW_CONTENT}" 2>/dev/null || { + log_warn " 无法获取新文件内容: ${triton_file}" + echo "[${repo}] ${mapped_path} <- [triton-ascend] ${triton_file} (无法读取)" >> "${SKIPPED_LIST}" + (( SKIPPED_COUNT++ )) || true + continue + } + + if [[ -f "${TARGET_ABS}" ]]; then + # 目标已存在，做差异报告 + log_warn " 新增但目标已存在（做 merge）: [${repo}] ${mapped_path}" + else + cp "${TRITON_NEW_CONTENT}" "${TARGET_ABS}" + log_ok " 新增: [${repo}] ${mapped_path}" + _write_entry "${CREATED_LIST}" "${repo}" "${triton_file}" "${mapped_path}" "${MAP_JSON}" + (( CREATED_COUNT++ )) || true + continue + fi + fi + + # ---------- 文件修改（包含上面"已存在"的新增走到这里的情况）---------- + # 获取 triton-ascend 中的 base 版本（BASE_COMMIT 处的内容） + TRITON_BASE="${WORK_DIR}/base_$(echo "${mapped_path}" | tr '/' '_')" + TRITON_NEW="${WORK_DIR}/new_$(echo "${mapped_path}" | tr '/' '_')" + + if ! git show "${BASE_COMMIT}:${triton_file}" > "${TRITON_BASE}" 2>/dev/null; then + # BASE_COMMIT 时该文件不存在（说明是新增文件但 status 不是 A，异常情况） + # 退而用空文件作为 base + touch "${TRITON_BASE}" + fi + + if ! git show "${CURRENT_COMMIT}:${triton_file}" > "${TRITON_NEW}" 2>/dev/null; then + log_warn " 无法读取 triton-ascend 新版本: ${triton_file}" + echo "[${repo}] ${mapped_path} <- [triton-ascend] ${triton_file} (无法读取新版本)" >> "${SKIPPED_LIST}" + (( SKIPPED_COUNT++ )) || true + continue + fi + + # 目标文件不存在时，直接创建 + if [[ ! -f "${TARGET_ABS}" ]]; then + mkdir -p "$(dirname "${TARGET_ABS}")" + cp "${TRITON_NEW}" "${TARGET_ABS}" + log_ok " 创建（目标不存在）: [${repo}] ${mapped_path}" + _write_entry "${CREATED_LIST}" "${repo}" "${triton_file}" "${mapped_path}" "${MAP_JSON}" + (( CREATED_COUNT++ )) || true + continue + fi + + # 三路合并： + # current = FlagTree 当前版本（含 FlagTree 特化改动） + # base = triton-ascend @ BASE_COMMIT（上次同步点） + # other = triton-ascend @ HEAD（新版本） + # git merge-file 会将结果写回 current 文件 + # 退出码 0 → 无冲突 + # 退出码 >0 → 有冲突（文件中已写入 <<<<<<< 标记） + # 退出码 <0 → 错误 + + MERGE_EXIT=0 + git merge-file \ + -L "flagtree/${mapped_path}" \ + -L "triton-ascend@${BASE_COMMIT}/${triton_file}" \ + -L "triton-ascend@HEAD/${triton_file}" \ + "${TARGET_ABS}" \ + "${TRITON_BASE}" \ + "${TRITON_NEW}" || MERGE_EXIT=$? + + if [[ ${MERGE_EXIT} -eq 0 ]]; then + log_ok " 已合并: [${repo}] ${mapped_path}" + _write_entry "${MODIFIED_LIST}" "${repo}" "${triton_file}" "${mapped_path}" "${MAP_JSON}" + (( MODIFIED_COUNT++ )) || true + elif [[ ${MERGE_EXIT} -gt 0 ]]; then + log_conflict " 合并冲突（请人工处理）: [${repo}] ${mapped_path}" + _write_entry "${CONFLICT_LIST}" "${repo}" "${triton_file}" "${mapped_path}" "${MAP_JSON}" "(${MERGE_EXIT} 处冲突)" + (( CONFLICT_COUNT++ )) || true + else + log_error " merge-file 错误: [${repo}] ${mapped_path}" + echo "[${repo}] ${mapped_path} (merge-file error)" >> "${SKIPPED_LIST}" + (( SKIPPED_COUNT++ )) || true + fi + + done <<< "${TARGETS}" + +done < "${CHANGED_FILES_RAW}" + +# ============================================================ +# Step 4: 对无需人工处理的新增/删除文件执行 git 暂存操作 +# ============================================================ +log_info "" +log_info "[4/5] 自动 git add / git rm..." + +GIT_ADD_COUNT=0 +GIT_RM_COUNT=0 + +# 从列表行中提取 [repo] path，跳过含 (需人工处理) 的行 +# 返回 " " 或空 +_parse_entry_no_manual() { + local _ln="$1" + [[ "$_ln" == *"(需人工处理)"* ]] && return + if [[ "$_ln" =~ ^\[([a-zA-Z]+)\][[:space:]]+([^[:space:]]+) ]]; then + echo "${BASH_REMATCH[1]} ${BASH_REMATCH[2]}" + fi +} + +# git add：遍历 created.txt +while IFS= read -r _line || [[ -n "$_line" ]]; do + [[ -z "$_line" ]] && continue + _parsed=$(_parse_entry_no_manual "$_line") + [[ -z "$_parsed" ]] && continue + _repo=$(echo "$_parsed" | cut -d' ' -f1) + _path=$(echo "$_parsed" | cut -d' ' -f2-) + _repo_dir=$(repo_dir "${_repo}") + _abs="${_repo_dir}/${_path}" + if [[ -f "$_abs" ]]; then + (cd "${_repo_dir}" && git add "${_path}") \ + && { log_ok " git add: [${_repo}] ${_path}"; (( GIT_ADD_COUNT++ )) || true; } \ + || log_warn " git add 失败: [${_repo}] ${_path}" + else + log_warn " git add 跳过（文件不存在）: [${_repo}] ${_path}" + fi +done < "${CREATED_LIST}" + +# git rm：遍历 deleted.txt，跳过 RENAME_DEL 条目（已由新增侧处理） +while IFS= read -r _line || [[ -n "$_line" ]]; do + [[ -z "$_line" ]] && continue + [[ "$_line" == *"(RENAME_DEL)"* ]] && continue + _parsed=$(_parse_entry_no_manual "$_line") + [[ -z "$_parsed" ]] && continue + _repo=$(echo "$_parsed" | cut -d' ' -f1) + _path=$(echo "$_parsed" | cut -d' ' -f2-) + _repo_dir=$(repo_dir "${_repo}") + _abs="${_repo_dir}/${_path}" + if [[ -f "$_abs" ]]; then + (cd "${_repo_dir}" && git rm "${_path}") \ + && { log_ok " git rm: [${_repo}] ${_path}"; (( GIT_RM_COUNT++ )) || true; } \ + || log_warn " git rm 失败: [${_repo}] ${_path}" + else + log_warn " git rm 跳过（文件不存在）: [${_repo}] ${_path}" + fi +done < "${DELETED_LIST}" + +log_info "自动暂存完成：git add ${GIT_ADD_COUNT} 个，git rm ${GIT_RM_COUNT} 个" + +# ============================================================ +# Step 5: 生成汇总报告 +# ============================================================ +SUMMARY="${REPORT_DIR}/summary.txt" +{ + echo "=========================================" + echo " Triton-Ascend → FlagTree 同步报告" + echo "=========================================" + echo "同步时间: $(date '+%Y-%m-%d %H:%M:%S')" + echo "triton-ascend: ${TRITON_ASCEND_DIR}" + echo "BASE_COMMIT: ${BASE_COMMIT}" + echo "HEAD_COMMIT: ${CURRENT_COMMIT}" + echo "" + _rdir=$(basename "${REPORT_DIR}") + echo "统计:" + echo " 已成功合并: ${MODIFIED_COUNT} 个文件" + echo " 新增创建: ${CREATED_COUNT} 个文件" + echo " 存在合并冲突: ${CONFLICT_COUNT} 个文件" + echo " 已删除: ${DELETED_COUNT} 个文件 ← 需人工处理" + echo " 跳过/未知路径: ${SKIPPED_COUNT} 个文件" + echo "" + echo "详细清单:" + echo " ${_rdir}/modified.txt" + echo " ${_rdir}/created.txt" + echo " ${_rdir}/conflict.txt" + echo " ${_rdir}/deleted.txt" + echo " ${_rdir}/skipped.txt" + echo "" + echo "flagtree 与 openai triton 原有 diff（${TRITON_DIFF_COUNT} 个文件）:" + echo " ${_rdir}/diff_with_openai_triton/" + echo "" + echo "triton-ascend 变更 diff:" + echo " ${_rdir}/triton_ascend_changes.diff" + echo "=========================================" +} | tee "${SUMMARY}" + +echo "" +log_info "报告目录: ${REPORT_DIR}" +echo "" + +if [[ ${CONFLICT_COUNT} -gt 0 ]]; then + echo "" + log_warn "以下文件存在合并冲突，请人工检查并解决（文件中已含 <<<<<<< 标记）：" + cat "${CONFLICT_LIST}" + echo "" +fi + +if [[ ${DELETED_COUNT} -gt 0 ]]; then + echo "" + log_warn "以下 triton-ascend 文件已被删除，请人工确认 FlagTree 侧是否也需同步删除：" + cat "${DELETED_LIST}" + echo "" +fi + +if [[ ${SKIPPED_COUNT} -gt 0 ]]; then + echo "" + log_warn "以下文件已跳过（不在同步范围内或路径未匹配），如有需要请人工处理：" + cat "${SKIPPED_LIST}" + echo "" +fi + +# 临时工作目录保留供调试，可手动删除 +log_info "临时工作目录（可手动删除）: ${WORK_DIR}" +log_info "报告目录: ${REPORT_DIR}" +log_info "摘要：${SUMMARY}" +log_info "完成" From d2c7b545a749c2d4115159ac44ee2de0b8791241 Mon Sep 17 00:00:00 2001 From: zhengyang Date: Tue, 3 Mar 2026 17:11:35 +0800 Subject: [PATCH 2/5] [TOOL] Add tools to sync code from triton-ascend --- tools/sync_ascend/clean.sh | 23 ++++++++++- tools/sync_ascend/sync_from_triton_ascend.sh | 40 ++++++++++++++++---- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/tools/sync_ascend/clean.sh b/tools/sync_ascend/clean.sh index 036e2112d..8f7148b6c 100755 --- a/tools/sync_ascend/clean.sh +++ b/tools/sync_ascend/clean.sh @@ -1,12 +1,31 @@ #!/bin/bash -# 清理同步代码，恢复到同步之前 +# 清理同步代码 +# 警告：脚本中涉及路径的所有修改均会撤销！ # # 用法: -# bash tools/sync_triton_ascend.sh [BASE_COMMIT] +# bash tools/sync_ascend/clean.sh set -e +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +_find_git_root() { + local _dir="$1" + while [[ "${_dir}" != "/" ]]; do + [[ -e "${_dir}/.git" ]] && { echo "${_dir}"; return 0; } + _dir="$(dirname "${_dir}")" + done + return 1 +} + +FLAGTREE_DIR="$(_find_git_root "${SCRIPT_DIR}")" || { + echo "[ERROR] 无法从 '${SCRIPT_DIR}' 向上找到 .git 目录，请在 flagtree 项目内执行脚本。" >&2 + exit 1 +} + +cd "${FLAGTREE_DIR}" + git stash git clean -xdf bin/ git clean -xdf include/ diff --git a/tools/sync_ascend/sync_from_triton_ascend.sh b/tools/sync_ascend/sync_from_triton_ascend.sh index 877c60cab..b933f49c4 100755 --- a/tools/sync_ascend/sync_from_triton_ascend.sh +++ b/tools/sync_ascend/sync_from_triton_ascend.sh @@ -4,8 +4,10 @@ # 使用 git merge-file 做三路合并，保留 FlagTree 的特化修改，冲突处留标记供人工处理。 # # 用法: -# 配置 TRITON_DIR, FLAGTREE_DIR, FLIR_DIR, TRITON_ASCEND_DIR -# bash tools/sync_from_triton_ascend.sh [BASE_COMMIT] +# export TRITON_ASCEND_DIR= # 必填：源仓库 triton-ascend +# export TRITON_DIR= # 可选：官方 triton，用于生成与 openai triton 的对比 diff +# export FLIR_DIR= # 可选：flir 子仓库，默认 /third_party/flir +# bash tools/sync_ascend/sync_from_triton_ascend.sh [BASE_COMMIT] # # BASE_COMMIT 默认为 29d243e（上次同步点）。 # 脚本不会执行 git commit，所有变更保留为工作区修改。 @@ -13,15 +15,37 @@ set -euo pipefail # ============================================================ -# 配置 +# 定位 flagtree 项目根目录（从脚本所在目录向上查找 .git） # ============================================================ -TRITON_DIR="/home/zhengyang/git/triton-3.2.x" # 用于生成 flagtree 与 openai triton 的对比，可置空 -FLAGTREE_DIR="/home/zhengyang/FlagTree/flagtree-3.2.x" # 目标主仓库 flagtree -FLIR_DIR="${FLAGTREE_DIR}/third_party/flir" # 目标子仓库 flir：不要求必须置于 flagtree 目录中 -TRITON_ASCEND_DIR="/home/zhengyang/FlagTree/triton-ascend" # 源仓库 triton-ascend +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +_find_git_root() { + local _dir="$1" + while [[ "${_dir}" != "/" ]]; do + [[ -e "${_dir}/.git" ]] && { echo "${_dir}"; return 0; } + _dir="$(dirname "${_dir}")" + done + return 1 +} + +FLAGTREE_DIR="$(_find_git_root "${SCRIPT_DIR}")" || { + echo "[ERROR] 无法从 '${SCRIPT_DIR}' 向上找到 .git 目录，请在 flagtree 项目内执行脚本。" >&2 + exit 1 +} + +# ============================================================ +# 配置（优先读取环境变量） +# ============================================================ +if [[ -z "${TRITON_ASCEND_DIR:-}" ]]; then + echo "[ERROR] 请设置环境变量 TRITON_ASCEND_DIR，指向 triton-ascend 仓库路径。" >&2 + exit 1 +fi + +TRITON_DIR="${TRITON_DIR:-}" # 可选，置空则跳过 openai triton 对比 +FLIR_DIR="${FLIR_DIR:-${FLAGTREE_DIR}/third_party/flir}" + BASE_COMMIT="${1:-29d243e}" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MAPPER_PY="${SCRIPT_DIR}/path_mapper.py" TIMESTAMP=$(date +%Y%m%d_%H%M%S) From 779cc8255d5e81c983dc32e879c32b808d5b7a4e Mon Sep 17 00:00:00 2001 From: zhengyang Date: Tue, 3 Mar 2026 17:39:50 +0800 Subject: [PATCH 3/5] [TOOL] Add tools to sync code from triton-ascend --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..94f480de9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf \ No newline at end of file From b3662be2a0a00828f4ca3ecdf879de013446ff22 Mon Sep 17 00:00:00 2001 From: zhengyang Date: Tue, 3 Mar 2026 17:54:08 +0800 Subject: [PATCH 4/5] [TOOL] Add tools to sync code from triton-ascend --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 94f480de9..6313b56c5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -* text=auto eol=lf \ No newline at end of file +* text=auto eol=lf From 42cf09be1ddc5156ee08af76468458fd67988068 Mon Sep 17 00:00:00 2001 From: zhengyang Date: Thu, 5 Mar 2026 19:49:57 +0800 Subject: [PATCH 5/5] [CI] Update flagtree-nvidia ci workflow --- .github/workflows/nv-build-and-test.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nv-build-and-test.yml b/.github/workflows/nv-build-and-test.yml index d4049e177..fc46f8276 100644 --- a/.github/workflows/nv-build-and-test.yml +++ b/.github/workflows/nv-build-and-test.yml @@ -14,7 +14,7 @@ concurrency: jobs: nv-build-and-test: - runs-on: nv-jiuding + runs-on: flagtree-nvidia if: ${{ github.repository == 'FlagTree/flagtree' || github.repository == 'flagos-ai/flagtree' }} steps: - name: Setup environment @@ -84,7 +84,7 @@ jobs: pip uninstall -y triton source ~/env.sh cd python - MAX_JOBS=32 python3.11 -m pip install . --no-build-isolation + MAX_JOBS=32 python3 -m pip install . --no-build-isolation - name: FlagTree Build on NVidia (triton_v3.2.x branch) if: ${{ steps.check_files.outputs.only_docs_changed != 'true' && env.TARGET_BRANCH == 'triton_v3.2.x' }} @@ -94,7 +94,7 @@ jobs: pip uninstall -y triton source ~/env-3.2.sh cd python - MAX_JOBS=32 python3.11 -m pip install . --no-build-isolation + MAX_JOBS=32 python3 -m pip install . --no-build-isolation - name: FlagTree Build on NVidia (triton_v3.3.x branch) if: ${{ steps.check_files.outputs.only_docs_changed != 'true' && env.TARGET_BRANCH == 'triton_v3.3.x' }} @@ -104,17 +104,18 @@ jobs: pip uninstall -y triton source ~/env-3.3.sh cd python - MAX_JOBS=32 python3.11 -m pip install . --no-build-isolation + MAX_JOBS=32 python3 -m pip install . --no-build-isolation - name: FlagTree Test on NVidia if: steps.check_files.outputs.only_docs_changed != 'true' shell: bash run: | set -x - python3.11 -m pytest -s python/test/unit \ + python3 -m pytest -s python/test/unit \ + --ignore=python/test/unit/language/test_line_info.py \ --ignore=python/test/unit/test_debug.py \ --ignore=python/test/unit/test_debug_dump.py \ --ignore=python/test/unit/tools/test_disasm.py if [ -d "python/test/operators" ]; then - python3.11 -m pytest -s python/test/operators + python3 -m pytest -s python/test/operators fi