diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..fd0767d6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/*.json.gz binary diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/.gitignore b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/.gitignore new file mode 100644 index 00000000..91ee3499 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/.gitignore @@ -0,0 +1,13 @@ + +# Temp working directories +temp/ + +# Debug/fix scripts +_fix_*.py +_check_*.py +_debug_*.py + +# Evaluation artifacts +artifacts.json +metrics.json + diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/README.md b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/README.md new file mode 100644 index 00000000..0473c02c --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/README.md @@ -0,0 +1,117 @@ +# VLSI Global Placement + +Global placement is a critical stage in VLSI (Very Large Scale Integration) physical design. +After logic synthesis and floorplanning, standard cells and macros must be placed on the chip +such that wirelength is minimized while respecting physical constraints. + +This benchmark uses the **ISPD 2005** placement contest benchmarks, the industry-standard +open-source benchmark suite for VLSI placement. The agent must implement a placement algorithm +that minimizes Half-Perimeter Wirelength (HPWL) without violating hard constraints. + +## File Structure + +```text +VLSIGlobalPlacement/ +├── .gitignore # Git ignore rules +├── datasets/ # Raw ISPD 2005 Bookshelf data +│ └── ispd2005/ # (empty; preprocessed JSONs are in references/) +├── README.md # Navigation doc (this file) +├── README_zh-CN.md # Navigation doc (Chinese) +├── Task.md # Detailed task description +├── Task_zh-CN.md # Detailed task description (Chinese) +├── references/ # Benchmark reference data +│ ├── adaptec1.json.gz # Easy benchmark (~211k cells, gzip) +│ ├── adaptec1_difficulty.json # Difficulty metadata +│ ├── adaptec3.json.gz # Medium benchmark (~451k cells, gzip) +│ └── adaptec3_difficulty.json # Difficulty metadata +├── scripts/ +│ ├── init.py # [MODIFIABLE] Placement algorithm +│ └── preprocess.py # Bookshelf -> JSON converter +├── verification/ +│ ├── evaluator.py # Scoring and legality checks +┬ ├── test_evaluator.py # Unit tests +│ ├── requirements.txt # Python dependencies +│ └── docker/ +│ └── Dockerfile # Containerized evaluation +├── baseline/ +│ └── solution.py # Row-based placement baseline +┬ └── result_log.txt # Baseline evaluation results +└── frontier_eval/ # Unified task metadata + ├── initial_program.txt + ├── eval_command.txt + ├── agent_files.txt + ├── artifact_files.txt + ├── readonly_files.txt + ├── copy_files.txt + ├── candidate_destination.txt + ├── eval_cwd.txt + ├── constraints.txt + └── run_eval.py +``` + +## Quick Start + +### 1. Install Dependencies + +```bash +pip install -r verification/requirements.txt +``` + +### 2. Run the Baseline Solver + +```bash +cd benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement +python scripts/init.py +# Outputs: temp/submission.json +``` + +### 3. Evaluate a Candidate Program + +```bash +cd benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement +python verification/evaluator.py scripts/init.py --benchmark adaptec1 +``` + +### 4. Run with Unified Task Framework + +```bash +python -m frontier_eval task=unified task.benchmark=ElectronicDesignAutomation/VLSIGlobalPlacement algorithm=openevolve algorithm.iterations=0 +``` + +## Benchmarks + +| Name | Difficulty | Fixed Cells | Movable Cells | Nets | Pins | Die Size | +|------|-----------|-------------|---------------|------|------|----------| +| adaptec1 | Easy | 543 | 210,904 | 221,142 | 944,053 | 11589x11589 | +| adaptec3 | Medium | 723 | 450,927 | 466,758 | 1,875,039 | 23190x23386 | + +## Task Summary + +- **Input**: Die dimensions, cell library, fixed/movable cells, netlist, initial placement +- **Output**: (x, y) coordinates for every movable cell +- **Hard Constraints**: No fixed cells moved, no cells out of bounds, no overlaps +- **Optimization Objective**: Minimize Half-Perimeter Wirelength (HPWL) +- **Editable File**: scripts/init.py (only place_components() function) + +## Dataset License + +The ISPD 2005 benchmarks were created by the ICCAD 2005 / ISPD 2006 placement contest committees +and are freely available for academic use. + +## Compressed JSON Format + +The reference files are gzip-compressed JSON with a compact netlist representation. +Each net is stored as a list of integer cell indices rather than full pin dictionaries: + +```json +{"netlist": [[0, 1, 2], [3, 4], ...]} +``` + +The compact netlist reduces JSON size by approximately 65% compared to the verbose +format, and gzip further reduces the on-disk size by about 84% (adaptec1: 22.6MB -> +3.7MB, adaptec3: 48.2MB -> 7.9MB). The `_decompress_netlist()` function in +scripts/init.py and verification/evaluator.py reconstructs the full pin +dictionaries at load time. The transformation is lossless with respect to the +HPWL computation. The scripts/preprocess.py script generates this compressed +format directly from the original Bookshelf data. + diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/README_zh-CN.md b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/README_zh-CN.md new file mode 100644 index 00000000..534a9d79 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/README_zh-CN.md @@ -0,0 +1,116 @@ +# VLSI 全局布局 + +全局布局是 VLSI（超大规模集成电路）物理设计中的关键阶段。 +在逻辑综合和布图规划之后，标准单元和宏单元必须放置在芯片上， +以最小化线长，同时满足物理约束。 + +本基准测试使用 **ISPD 2005** 布局竞赛基准，这是 VLSI 布局领域 +行业标准的开源基准套件。Agent 必须实现一个最小化半周长线长（HPWL） +的布局算法，同时不违反硬约束。 + +## 文件结构 + +```text +VLSIGlobalPlacement/ +├── .gitignore # Git 忽略规则 +├── datasets/ # 原始 ISPD 2005 Bookshelf 数据 +│ └── ispd2005/ # （空目录；预处理后的 JSON 在 references/ 中） +├── README.md # 导航文档（英文） +├── README_zh-CN.md # 导航文档（中文，本文件） +├── Task.md # 详细任务描述（英文） +├── Task_zh-CN.md # 详细任务描述（中文） +├── references/ # 基准参考数据 +│ ├── adaptec1.json.gz # 简单基准（约21万单元，gzip） +│ ├── adaptec1_difficulty.json # 难度元数据 +│ ├── adaptec3.json.gz # 中等基准（约45万单元） +│ └── adaptec3_difficulty.json # 难度元数据 +├── scripts/ +│ ├── init.py # [可修改] 布局算法 +│ └── preprocess.py # Bookshelf 格式转 JSON +├── verification/ +│ ├── evaluator.py # 评分和合法性检查 +│ ├── requirements.txt # Python 依赖 +│ └── docker/ +│ └── Dockerfile # 容器化评测 +├── baseline/ +│ └── solution.py # 行式放置基线 +└── frontier_eval/ # Unified task 元数据 + ├── initial_program.txt + ├── eval_command.txt + ├── agent_files.txt + ├── artifact_files.txt + ├── readonly_files.txt + ├── copy_files.txt + ├── candidate_destination.txt + ├── eval_cwd.txt + ├── constraints.txt + └── run_eval.py +``` + +## 快速开始 + +### 1. 安装依赖 + +```bash +pip install -r verification/requirements.txt +``` + +### 2. 运行基线求解器 + +```bash +cd benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement +python scripts/init.py +# 输出: temp/submission.json +``` + +### 3. 评估候选程序 + +```bash +cd benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement +python verification/evaluator.py scripts/init.py --benchmark adaptec1 +``` + +### 4. 使用 Unified Task 框架运行 + +```bash +python -m frontier_eval \ + task=unified \ + task.benchmark=ElectronicDesignAutomation/VLSIGlobalPlacement \ + algorithm=openevolve \ + algorithm.iterations=0 +``` + +## 基准数据 + +| 名称 | 难度 | 固定单元 | 可移动单元 | 网络数 | 引脚数 | 芯片尺寸 | +|------|------|---------|-----------|--------|--------|---------| +| adaptec1 | 简单 | 543 | 210,904 | 221,142 | 944,053 | 11589x11589 | +| adaptec3 | 中等 | 723 | 450,927 | 466,758 | 1,875,039 | 23190x23386 | + +## 任务概要 + +- **输入**：芯片尺寸、单元库、固定/可移动单元、网表、初始布局 +- **输出**：每个可移动单元的 (x, y) 坐标 +- **硬约束**：不移动固定单元、所有单元在芯片内、无重叠 +- **优化目标**：最小化半周长线长（HPWL） +- **可编辑文件**：scripts/init.py（仅 place_components() 函数） + +## 数据集许可 + +ISPD 2005 基准由 ICCAD 2005 / ISPD 2006 布局竞赛委员会创建， +可免费用于学术用途。 + +## 压缩 JSON 格式 + +参考文件为 gzip 压缩的 JSON，并采用紧凑的网表表示。 +每个网表存储为整数单元索引列表，而非完整的引脚字典： + +```json +{"netlist": [[0, 1, 2], [3, 4], ...]} +``` + +紧凑网表相比详细格式可减少约 65% 的 JSON 大小， +gzip 进一步将磁盘占用减少约 84%（adaptec1：22.6MB -> 3.7MB，adaptec3：48.2MB -> 7.9MB）。 +scripts/init.py 和 verification/evaluator.py 中的 _decompress_netlist() 函数 +在加载时重建完整的引脚字典。该转换相对于 HPWL 计算是无损的。 +scripts/preprocess.py 脚本直接从原始 Bookshelf 数据生成此压缩格式。 diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/Task.md b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/Task.md new file mode 100644 index 00000000..08509e2c --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/Task.md @@ -0,0 +1,173 @@ +# VLSI Global Placement + +## 1. Engineering Background + +VLSI Global Placement is a critical stage in the ASIC physical design flow. +After logic synthesis converts RTL to a gate-level netlist, and floorplanning +defines the chip outline and macro positions, global placement assigns +approximate locations to all standard cells. + +The complete industrial flow is: + +``` +RTL → Synthesis → Floorplanning → Global Placement → Detailed Placement +→ Clock Tree Synthesis → Routing → Timing Signoff → Physical Verification +``` + +Global placement is the first step where interconnect wirelength becomes +visible. It produces a **rough but legal** placement that a detailed placer +then refines. The quality of global placement directly affects: + +- **Routing congestion**: Poor placement creates routing hotspots +- **Timing**: Longer wires increase RC delay +- **Power**: Longer wires increase dynamic power consumption +- **Area**: Inefficient placement may require larger die area + +Modern industrial placers (Cadence Innovus, Synopsys ICC2, OpenROAD) all +include a global placement phase that optimizes HPWL while maintaining +density constraints. + +## 2. Problem Definition + +### Input + +The benchmark provides the following data in JSON format: + +| Field | Type | Description | +|-------|------|-------------| +| benchmark_name | string | Benchmark identifier | +| die | dict | Die dimensions: {width, height, row_height, min_x, min_y} | +| cells | dict | Cell library: {cell_name: {width, height}} | +| fixed_cells | list[str] | Names of fixed (terminal) cells | +| movable_cells | list[str] | Names of movable (standard) cells | +| initial_placement | dict | Initial positions: {cell_name: {x, y, orientation}} | +| netlist | list[list[dict]] | Nets: [[{cell, x_offset, y_offset}, ...], ...] | +| num_nets | int | Total number of nets | +| num_pins | int | Total number of pins | + +### Output + +The placement algorithm must produce a dictionary mapping each movable +cell name to its [x, y] coordinates: + +```json +{ + "cell_name_1": [x_coordinate, y_coordinate], + "cell_name_2": [x_coordinate, y_coordinate], + ... +} +``` + +Fixed cells must remain at their initial positions. + +### Hard Constraints + +The evaluator enforces three hard constraints: + +1. **Fixed cells must not be moved**: Any fixed cell whose position + differs from its initial position by more than 1e-6 invalidates the placement. + +2. **All cells must be within the die boundary**: Every cell must satisfy + <= x <= die_width - cell_width and <= y <= die_height - cell_height. + +3. **No overlapping cells**: For any pair of cells, the overlap area + must be zero. The evaluator checks axis-aligned bounding box intersection. + +Violating any hard constraint sets valid = 0 and combined_score = -1e18. + +### Optimization Objective + +**Minimize Half-Perimeter Wirelength (HPWL)**. + +For each net, HPWL is defined as: + +``` +HPWL(net) = (max(x_pins) - min(x_pins)) + (max(y_pins) - min(y_pins)) +``` + +Total HPWL = sum of HPWL over all nets. + +The pin position is computed as: + +``` +pin_x = cell_x + cell_width / 2 + x_offset +pin_y = cell_y + cell_height / 2 + y_offset +``` + +Lower HPWL indicates a better placement. The combined score is -HPWL +for valid placements, so higher combined_score = better. + +## 3. Why HPWL? + +HPWL is the standard optimization objective in VLSI placement because: + +- It is **deterministic** and easy to compute +- It is **strongly correlated** with routed wirelength +- It is a **proxy for timing**, power, and congestion +- It is used by **all major placement contests** (ISPD, ICCAD, DAC) +- It is the objective optimized by **all major open-source placers** + (RePlAce, ePlace, NTUPlace3, DREAMPlace, OpenROAD) + +## 4. Baseline + +The provided baseline uses **deterministic row-based placement**: + +- Fixed cells remain at their initial positions +- Movable cells are sorted by height (descending), then area (descending), then name +- Tall macros (height > row_height) are placed first, spanning multiple rows +- Standard cells (height == row_height) fill remaining row space from left to right +- The placement is deterministic and always produces a legal placement +- HPWL is intentionally poor, providing substantial room for agent improvement + +## 5. Datasets + +Two benchmarks from the ISPD 2005 placement contest suite: + +| Benchmark | Difficulty | Movable Cells | Nets | Pins | Die (um) | +|-----------|-----------|--------------|------|------|----------| +| adaptec1 | Easy | 210,904 | 221,142 | 944,053 | 11589x11589 | +| adaptec3 | Medium | 450,927 | 466,758 | 1,875,039 | 23190x23386 | + +The original Bookshelf-format data (datasets/ispd2005/) is not redistributed. +Preprocessed compressed JSON files (gzip + compact netlist) are in +references/. The preprocessing script scripts/preprocess.py shows how +Bookshelf format is converted to the compressed JSON format. + +## 6. Evaluation + +The evaluator (verification/evaluator.py): + +1. Runs the candidate program in a clean subprocess with timeout +2. Reads `temp/submission.json` from the candidate +3. Checks hard constraints (fixed cells, bounds, overlap) +4. Computes HPWL +5. Returns metrics: combined_score, valid, hpwl, +runtime_s + +### Command + +```bash +python verification/evaluator.py scripts/init.py --benchmark adaptec1 +``` + +### Metrics + +| Metric | Description | +|--------|-------------| +| hpwl | Total Half-Perimeter Wirelength | +| valid | 1.0 if all hard constraints satisfied, else 0.0 | +| combined_score | -hpwl if valid, -1e18 if invalid | +| runtime_s | Total evaluation runtime in seconds | +| n_cells_placed | Number of cells in placement output | +| n_fixed_moved | Number of fixed cells that were moved | +| n_out_of_bounds | Number of cells outside die boundary | +| n_overlaps | Number of overlapping cell pairs | + +## 7. References + +- ISPD 2005 Placement Contest: https://www.ispd.cc/contests/05/ispd05.html +- ICCAD 2005 Mixed-Size Placement Contest: https://www.sigda.org/programs/placement-contest/ +- DREAMPlace: https://github.com/limbo018/DREAMPlace +- RePlAce: https://github.com/The-OpenROAD-Project/RePlAce +- ePlace: https://github.com/limbo018/ePlace +- OpenROAD: https://github.com/The-OpenROAD-Project/OpenROAD diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/Task_zh-CN.md b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/Task_zh-CN.md new file mode 100644 index 00000000..1ea00fe8 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/Task_zh-CN.md @@ -0,0 +1,166 @@ +# VLSI 全局布局 + +## 1. 工程背景 + +VLSI 全局布局是 ASIC 物理设计流程中的关键阶段。 +在逻辑综合将 RTL 转换为门级网表、布图规划定义芯片轮廓和宏单元位置之后， +全局布局为所有标准单元分配大致位置。 + +完整的工业流程为： + +``` +RTL → 逻辑综合 → 布图规划 → 全局布局 → 详细布局 +→ 时钟树综合 → 布线 → 时序签核 → 物理验证 +``` + +全局布局是互连线长变得可见的第一步。它产生一个**粗略但合法**的布局， +详细布局器在此基础上进行优化。全局布局的质量直接影响： + +- **布线拥塞**：不良布局会产生布线热点 +- **时序**：更长的导线增加 RC 延迟 +- **功耗**：更长的导线增加动态功耗 +- **面积**：低效布局可能需要更大的芯片面积 + +现代工业布局器（Cadence Innovus、Synopsys ICC2、OpenROAD） +都包含一个全局布局阶段，在保持密度约束的同时优化 HPWL。 + +## 2. 问题定义 + +### 输入 + +基准测试提供以下 JSON 格式的数据： + +| 字段 | 类型 | 描述 | +|------|------|------| +| benchmark_name | string | 基准标识符 | +| die | dict | 芯片尺寸：{width, height, row_height, min_x, min_y} | +| cells | dict | 单元库：{cell_name: {width, height}} | +| fixed_cells | list[str] | 固定（I/O）单元名称 | +| movable_cells | list[str] | 可移动（标准）单元名称 | +| initial_placement | dict | 初始位置：{cell_name: {x, y, orientation}} | +| netlist | list[list[dict]] | 网表：[[{cell, x_offset, y_offset}, ...], ...] | +| num_nets | int | 网络总数 | +| num_pins | int | 引脚总数 | + +### 输出 + +布局算法必须生成一个字典，将每个可移动单元名称映射到其 [x, y] 坐标： + +```json +{ + "cell_name_1": [x_coordinate, y_coordinate], + "cell_name_2": [x_coordinate, y_coordinate], + ... +} +``` + +固定单元必须保持在其初始位置。 + +### 硬约束 + +评测器强制执行三个硬约束： + +1. **固定单元不得移动**：任何与初始位置偏差超过 1e-6 的固定单元 + 将使布局无效。 + +2. **所有单元必须在芯片边界内**：每个单元必须满足 + <= x <= die_width - cell_width 且 <= y <= die_height - cell_height。 + +3. **单元不得重叠**：对于任意两个单元，重叠面积必须为零。 + 评测器检查轴对齐边界框的交集。 + +违反任何硬约束将设置 valid = 0 和 combined_score = -1e18。 + +### 优化目标 + +**最小化半周长线长（HPWL）**。 + +对于每个网络，HPWL 定义为： + +``` +HPWL(net) = (max(x_pins) - min(x_pins)) + (max(y_pins) - min(y_pins)) +``` + +总 HPWL = 所有网络的 HPWL 之和。 + +引脚位置计算如下： + +``` +pin_x = cell_x + cell_width / 2 + x_offset +pin_y = cell_y + cell_height / 2 + y_offset +``` + +较低的 HPWL 表示更好的布局。有效布局的 combined_score 为 -HPWL， +因此 combined_score 越高越好。 + +## 3. 为什么选择 HPWL？ + +HPWL 是 VLSI 布局中的标准优化目标，因为： + +- **确定性**且易于计算 +- 与布线线长**强相关** +- 是时序、功耗和拥塞的**代理指标** +- 被**所有主要布局竞赛**（ISPD、ICCAD、DAC）使用 +- 被**所有主要开源布局器**（RePlAce、ePlace、NTUPlace3、DREAMPlace、OpenROAD）优化 + +## 4. 基线算法 + +提供的基线使用**确定性行式布局**： + +- 固定单元保持在其初始位置 +- 可移动单元按高度（降序）、面积（降序）、名称排序 +- 较高的宏单元（height > row_height）优先放置，跨越多行 +- 标准单元（height == row_height）从左到右填充剩余行空间 +- 布局是确定性的，始终生成合法布局 +- HPWL 故意较差，为智能体提供充分的优化空间 + +## 5. 数据集 + +来自 ISPD 2005 布局竞赛套件的两个基准： + +| 基准 | 难度 | 可移动单元 | 网络数 | 引脚数 | 芯片尺寸 | +|------|------|-----------|--------|--------|---------| +| adaptec1 | 简单 | 210,904 | 221,142 | 944,053 | 11589x11589 | +| adaptec3 | 中等 | 450,927 | 466,758 | 1,875,039 | 23190x23386 | + +原始 Bookshelf 格式数据（datasets/ispd2005/）未重新分发。 +预处理后的 JSON 文件位于 references/。预处理脚本 +scripts/preprocess.py 展示了 Bookshelf 格式如何转换为 JSON。 + +## 6. 评测 + +评测器（verification/evaluator.py）： + +1. 在干净的子进程中运行候选程序（带超时） +2. 从候选程序读取 `temp/submission.json` +3. 检查硬约束（固定单元、边界、重叠） +4. 计算 HPWL +5. 返回指标：combined_score、valid、hpwl、runtime_s + +### 命令 + +```bash +python verification/evaluator.py scripts/init.py --benchmark adaptec1 +``` + +### 指标 + +| 指标 | 描述 | +|--------|------| +| hpwl | 总半周长线长 | +| valid | 满足所有硬约束则为 1.0，否则为 0.0 | +| combined_score | 有效时为 -hpwl，无效时为 -1e18 | +| runtime_s | 总评测运行时间（秒） | +| n_cells_placed | 布局输出中的单元数量 | +| n_fixed_moved | 被移动的固定单元数量 | +| n_out_of_bounds | 超出芯片边界的单元数量 | +| n_overlaps | 重叠单元对的数量 | + +## 7. 参考文献 + +- ISPD 2005 Placement Contest: https://www.ispd.cc/contests/05/ispd05.html +- ICCAD 2005 Mixed-Size Placement Contest: https://www.sigda.org/programs/placement-contest/ +- DREAMPlace: https://github.com/limbo018/DREAMPlace +- RePlAce: https://github.com/The-OpenROAD-Project/RePlAce +- ePlace: https://github.com/limbo018/ePlace +- OpenROAD: https://github.com/The-OpenROAD-Project/OpenROAD diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/baseline/result_log.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/baseline/result_log.txt new file mode 100644 index 00000000..f2acc4ad --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/baseline/result_log.txt @@ -0,0 +1,31 @@ +VLSI Global Placement baseline results +======================================= +Baseline: deterministic row-based placement (scripts/init.py) +Evaluated with: python verification/evaluator.py scripts/init.py --benchmark + +Commands: + cd benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement + python verification/evaluator.py scripts/init.py --benchmark adaptec1 + python verification/evaluator.py scripts/init.py --benchmark adaptec3 + +adaptec1 (Easy): + valid = 1.0 + hpwl = 1318928036.21 + n_overlaps = 0 + n_out_of_bounds= 0 + n_fixed_moved = 0 + runtime_s = 4.73 + +adaptec3 (Medium): + valid = 1.0 + hpwl = 4982296306.66 + n_overlaps = 0 + n_out_of_bounds= 0 + n_fixed_moved = 0 + runtime_s = 10.46 + +Notes: +- Both placements are legal (no overlaps, no out-of-bound cells, + no fixed cells moved). +- The baseline is intentionally simple; its HPWL is far from the + ISPD 2005 contest results, leaving large headroom for agents. diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/baseline/solution.py b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/baseline/solution.py new file mode 100644 index 00000000..ae140413 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/baseline/solution.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Row-based placement baseline for VLSI Global Placement. + +This baseline matches the algorithm in scripts/init.py. +It is intentionally simple: legal but poor HPWL. +""" + +import json +import sys +import time +from pathlib import Path + + +def _open_reference(path): + """Open a reference file, transparently decompressing gzip data.""" + if str(path).endswith(".json.gz"): + import gzip + return gzip.open(path, "rt", encoding="utf-8") + return open(path, "r", encoding="utf-8") + + +def load_benchmark(benchmark_name): + candidates = [ + Path("references") / f"{benchmark_name}.json", + Path("references") / f"{benchmark_name}.json.gz", + Path(__file__).resolve().parent.parent / "references" / f"{benchmark_name}.json", + Path(__file__).resolve().parent.parent / "references" / f"{benchmark_name}.json.gz", + ] + for p in candidates: + if p.is_file(): + with _open_reference(p) as f: + return json.load(f) + raise FileNotFoundError(f"Benchmark {benchmark_name}.json[.gz] not found") + + +def compute_hpwl(placement, netlist, cells): + total_hpwl = 0.0 + for net in netlist: + xs = [] + ys = [] + for pin in net: + cname = pin["cell"] + if cname not in placement: + continue + cx, cy = placement[cname] + w = cells[cname]["width"] + h = cells[cname]["height"] + px = cx + w / 2 + pin.get("x_offset", 0.0) + py = cy + h / 2 + pin.get("y_offset", 0.0) + xs.append(px) + ys.append(py) + if xs: + total_hpwl += (max(xs) - min(xs)) + (max(ys) - min(ys)) + return total_hpwl + + +def place_components(die, cells, fixed_cells, movable_cells, netlist, initial_placement): + """Deterministic row-based placement.""" + die_w = die["width"] + die_h = die["height"] + row_h = die["row_height"] + n_rows = int(die_h // row_h) + + placement = {} + for c in fixed_cells: + placement[c] = [initial_placement[c]["x"], initial_placement[c]["y"]] + + row_cursor = [0.0] * n_rows + for c in fixed_cells: + cx, cy = placement[c] + cw = cells[c]["width"] + ch = cells[c]["height"] + r_start = max(0, int(cy // row_h)) + r_end = min(n_rows - 1, int((cy + ch - 1) // row_h)) + for r in range(r_start, r_end + 1): + right = cx + cw + if right > row_cursor[r]: + row_cursor[r] = right + + tall_cells = [c for c in movable_cells if cells[c]["height"] > row_h] + std_cells = [c for c in movable_cells if cells[c]["height"] == row_h] + + tall_cells.sort(key=lambda c: (-cells[c]["height"], -cells[c]["width"] * cells[c]["height"], c)) + std_cells.sort(key=lambda c: (-cells[c]["width"] * cells[c]["height"], c)) + + for c in tall_cells: + cw = cells[c]["width"] + ch = cells[c]["height"] + rows_needed = max(1, int(ch // row_h)) + found = False + for r in range(n_rows - rows_needed + 1): + max_c = 0.0 + for rr in range(r, r + rows_needed): + if row_cursor[rr] > max_c: + max_c = row_cursor[rr] + if max_c + cw <= die_w: + x = max_c + y = float(r * row_h) + placement[c] = [x, y] + for rr in range(r, r + rows_needed): + row_cursor[rr] = x + cw + found = True + break + if not found: + placement[c] = [0.0, 0.0] + + available_rows = [r for r in range(n_rows) if row_cursor[r] < die_w] + row_ptr = 0 + for c in std_cells: + cw = cells[c]["width"] + found = False + for ri in range(row_ptr, len(available_rows)): + r = available_rows[ri] + if row_cursor[r] + cw <= die_w: + x = row_cursor[r] + y = float(r * row_h) + placement[c] = [x, y] + row_cursor[r] = x + cw + if row_cursor[r] >= die_w: + row_ptr = ri + 1 + found = True + break + if not found: + placement[c] = [0.0, 0.0] + + return placement + + +def main(): + benchmark_name = sys.argv[1] if len(sys.argv) > 1 else "adaptec1" + print("=" * 60) + print("VLSI Global Placement - Row-based Baseline") + print("=" * 60) + print() + print(f"Loading benchmark: {benchmark_name}") + data = load_benchmark(benchmark_name) + print(f" Die: {data['die']['width']} x {data['die']['height']}") + print(f" Cells: {len(data['cells'])} total " + f"({len(data['fixed_cells'])} fixed, {len(data['movable_cells'])} movable)") + print(f" Nets: {data['num_nets']}, Pins: {data['num_pins']}") + print() + + print("Running row-based placement...") + t0 = time.time() + placement = place_components( + die=data["die"], + cells=data["cells"], + fixed_cells=data["fixed_cells"], + movable_cells=data["movable_cells"], + netlist=data["netlist"], + initial_placement=data["initial_placement"], + ) + runtime = time.time() - t0 + hpwl = compute_hpwl(placement, data["netlist"], data["cells"]) + print(f" Runtime: {runtime:.2f}s") + print(f" HPWL: {hpwl:.2f}") + print(f" Cells placed: {len(placement)}") + print() + print("=" * 60) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/agent_files.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/agent_files.txt new file mode 100644 index 00000000..eccc290b --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/agent_files.txt @@ -0,0 +1,8 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +references/adaptec1_difficulty.json +references/adaptec3_difficulty.json +scripts/init.py +frontier_eval/constraints.txt diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/artifact_files.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/artifact_files.txt new file mode 100644 index 00000000..a52b8b16 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/artifact_files.txt @@ -0,0 +1,2 @@ +# No extra artifact files are auto-collected by default for this benchmark. +# metrics.json and artifacts.json are handled separately by UnifiedTask. diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/candidate_destination.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/candidate_destination.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/candidate_destination.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/constraints.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/constraints.txt new file mode 100644 index 00000000..2e2a0085 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/constraints.txt @@ -0,0 +1,9 @@ +UnifiedTask constraints: +1) Only modify scripts/init.py. +2) Preserve the public entrypoint, function signatures, and output contract expected by verification/evaluator.py. +3) Do not modify benchmark assets, documentation, references, verification code, or frontier_eval/ metadata. +4) The output must be temp/submission.json with the exact schema: {"benchmark_id": "vlsi_global_placement", "benchmark_name": "...", "placement": {cell_name: [x, y], ...}}. +5) Hard constraints: do not move fixed cells, do not place cells outside the die, do not overlap cells. +6) Optimization objective: minimize HPWL (Half-Perimeter Wirelength). Lower HPWL = better score. +7) Prioritize validity and correctness before optimization. +8) Available benchmarks: adaptec1 (Easy, ~210k cells), adaptec3 (Medium, ~450k cells). diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/copy_files.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/copy_files.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/copy_files.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/eval_command.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/eval_command.txt new file mode 100644 index 00000000..8cfcad47 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/eval_command.txt @@ -0,0 +1 @@ +{python} frontier_eval/run_eval.py --candidate {candidate} --metrics-out metrics.json --artifacts-out artifacts.json diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/eval_cwd.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/eval_cwd.txt new file mode 100644 index 00000000..9c558e35 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/eval_cwd.txt @@ -0,0 +1 @@ +. diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/initial_program.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/initial_program.txt new file mode 100644 index 00000000..b9411b3d --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/initial_program.txt @@ -0,0 +1 @@ +scripts/init.py diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/readonly_files.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/readonly_files.txt new file mode 100644 index 00000000..d644b98e --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/readonly_files.txt @@ -0,0 +1,7 @@ +README.md +README_zh-CN.md +Task.md +Task_zh-CN.md +references +verification +frontier_eval diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/run_eval.py b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/run_eval.py new file mode 100644 index 00000000..70e844c0 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/frontier_eval/run_eval.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Unified-task evaluation wrapper for VLSI Global Placement. + +This script is invoked by the Frontier Eval unified task framework. +It runs the evaluator and writes metrics.json and artifacts.json. +""" + +import argparse +import json +import os +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description="VLSI Global Placement Eval Wrapper") + parser.add_argument("--candidate", required=True, help="Path to candidate program") + parser.add_argument("--metrics-out", default="metrics.json", help="Path to write metrics JSON") + parser.add_argument("--artifacts-out", default="artifacts.json", help="Path to write artifacts JSON") + args = parser.parse_args() + + # Ensure we can import the evaluator + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "verification")) + + from evaluator import evaluate + + # Determine benchmark name from environment or default + benchmark_name = os.environ.get("BENCHMARK_NAME", "adaptec1") + + # Run evaluation + result = evaluate(args.candidate, benchmark_name=benchmark_name) + + if hasattr(result, "metrics"): + metrics = result.metrics + artifacts = result.artifacts + elif isinstance(result, dict): + metrics = result.get("metrics", {}) + artifacts = result.get("artifacts", {}) + else: + metrics = {"error": 1.0, "combined_score": -1e18} + artifacts = {"error_message": f"Unexpected result type: {type(result)}"} + + # Write metrics.json + with open(args.metrics_out, "w", encoding="utf-8") as f: + json.dump(metrics, f, indent=2) + + # Write artifacts.json + with open(args.artifacts_out, "w", encoding="utf-8") as f: + json.dump(artifacts, f, indent=2) + + print(f"Metrics written to {args.metrics_out}") + print(f"Artifacts written to {args.artifacts_out}") + print(f"combined_score: {metrics.get('combined_score', 'N/A')}") + print(f"valid: {metrics.get('valid', 'N/A')}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec1.json.gz b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec1.json.gz new file mode 100644 index 00000000..9a9afb08 Binary files /dev/null and b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec1.json.gz differ diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec1_difficulty.json b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec1_difficulty.json new file mode 100644 index 00000000..d79e8c34 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec1_difficulty.json @@ -0,0 +1 @@ +{"difficulty": "Easy", "benchmark": "adaptec1"} \ No newline at end of file diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec3.json.gz b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec3.json.gz new file mode 100644 index 00000000..85bc5b24 Binary files /dev/null and b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec3.json.gz differ diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec3_difficulty.json b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec3_difficulty.json new file mode 100644 index 00000000..300423cc --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/adaptec3_difficulty.json @@ -0,0 +1 @@ +{"difficulty": "Medium", "benchmark": "adaptec3"} \ No newline at end of file diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/init.py b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/init.py new file mode 100644 index 00000000..53377fd4 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/init.py @@ -0,0 +1,282 @@ +""" +VLSI Global Placement - ISPD 2005 Benchmarks + +Baseline Algorithm: Deterministic Row-Based Placement + +- ALLOWED TO MODIFY: place_components() +- NOT ALLOWED TO MODIFY: load_benchmark(), compute_hpwl(), main(), output format + +Strategy: +1. Fixed cells remain at their initial positions (unchanged). +2. Movable cells sorted by height descending, then area descending, then name. +3. Row cursor tracks the next available x position in each row. +4. Tall macros (height > row_height) placed first, checking all spanned rows. +5. Standard cells (height == row_height) fill remaining row space. +""" + +import json +import math +import os +import sys +import time +from pathlib import Path + + +def _decompress_netlist(data: dict) -> dict: + """Decompress compact netlist (cell indices) into original format (cell names + zero offsets).""" + if not data.get("netlist") or not data["netlist"]: + return data + if isinstance(data["netlist"][0], list) and data["netlist"][0] and isinstance(data["netlist"][0][0], dict): + return data + cell_names = list(data["cells"].keys()) + new_netlist = [] + for net in data["netlist"]: + new_net = [{"cell": cell_names[idx], "x_offset": 0.0, "y_offset": 0.0} for idx in net] + new_netlist.append(new_net) + data["netlist"] = new_netlist + return data + + +# ============================================================================ +# DATA LOADING (NOT ALLOWED TO MODIFY - Interface must match evaluator) +# ============================================================================ + +def _open_reference(path: Path): + """Open a reference file, transparently decompressing gzip data.""" + if str(path).endswith(".json.gz"): + import gzip + return gzip.open(path, "rt", encoding="utf-8") + return open(path, "r", encoding="utf-8") + + +def load_benchmark(benchmark_name: str | None = None) -> dict: + """Load the benchmark reference JSON. DO NOT MODIFY.""" + if benchmark_name is None: + benchmark_name = os.environ.get("BENCHMARK_NAME", "adaptec1") + + candidates = [ + Path("references") / f"{benchmark_name}.json", + Path("references") / f"{benchmark_name}.json.gz", + Path(__file__).resolve().parent.parent / "references" / f"{benchmark_name}.json", + Path(__file__).resolve().parent.parent / "references" / f"{benchmark_name}.json.gz", + ] + for p in candidates: + if p.is_file(): + with _open_reference(p) as f: + data = json.load(f) + return _decompress_netlist(data) + raise FileNotFoundError( + f"Benchmark reference JSON '{benchmark_name}.json[.gz]' not found. " + "Searched: " + ", ".join(str(c) for c in candidates) + ) + + +# ============================================================================ +# HPWL COMPUTATION (NOT ALLOWED TO MODIFY - Must match evaluator) +# ============================================================================ + +def compute_hpwl(placement: dict, netlist: list, cells: dict) -> float: + """Compute Half-Perimeter Wirelength. DO NOT MODIFY.""" + total_hpwl = 0.0 + for net in netlist: + xs = [] + ys = [] + for pin in net: + cell_name = pin["cell"] + if cell_name not in placement: + continue + cx, cy = placement[cell_name] + w = cells[cell_name]["width"] + h = cells[cell_name]["height"] + px = cx + w / 2 + pin.get("x_offset", 0.0) + py = cy + h / 2 + pin.get("y_offset", 0.0) + xs.append(px) + ys.append(py) + if xs: + total_hpwl += (max(xs) - min(xs)) + (max(ys) - min(ys)) + return total_hpwl + + +# ============================================================================ +# PLACEMENT ALGORITHM (ALLOWED TO MODIFY - This is your optimization code) +# ============================================================================ + +# EVOLVE-BLOCK-START +def place_components( + die: dict, + cells: dict, + fixed_cells: list, + movable_cells: list, + netlist: list, + initial_placement: dict, +) -> dict: + """Place all movable cells using deterministic row-based placement. + + Strategy: + 1. Fixed cells remain at their initial positions. + 2. Movable cells sorted by height descending, then area descending, then name. + 3. Row cursor tracks the next available x position in each row. + 4. Tall macros (height > row_height) placed first, checking all rows they span. + 5. Standard cells (height == row_height) fill remaining row space. + """ + die_w = die["width"] + die_h = die["height"] + row_h = die["row_height"] + n_rows = int(die_h // row_h) + + # Initialize placement with fixed cells at their initial positions + placement = {} + for c in fixed_cells: + placement[c] = [initial_placement[c]["x"], initial_placement[c]["y"]] + + # Row cursor: next available x position in each row + row_cursor = [0.0] * n_rows + + # Mark fixed cell occupancy in row cursors + for c in fixed_cells: + cx, cy = placement[c] + cw = cells[c]["width"] + ch = cells[c]["height"] + r_start = max(0, int(cy // row_h)) + r_end = min(n_rows - 1, int((cy + ch - 1) // row_h)) + for r in range(r_start, r_end + 1): + right = cx + cw + if right > row_cursor[r]: + row_cursor[r] = right + + # Separate tall macros (>1 row) and standard cells (1 row) + tall_cells = [c for c in movable_cells if cells[c]["height"] > row_h] + std_cells = [c for c in movable_cells if cells[c]["height"] == row_h] + + # Sort tall cells: height desc, area desc, name + tall_cells.sort(key=lambda c: ( + -cells[c]["height"], + -cells[c]["width"] * cells[c]["height"], + c + )) + + # Sort standard cells: area desc, name + std_cells.sort(key=lambda c: ( + -cells[c]["width"] * cells[c]["height"], + c + )) + + # Place tall macros: they span multiple rows + for c in tall_cells: + cw = cells[c]["width"] + ch = cells[c]["height"] + rows_needed = max(1, int(ch // row_h)) + + found = False + for r in range(n_rows - rows_needed + 1): + # Find the maximum cursor in the spanned rows + max_c = 0.0 + for rr in range(r, r + rows_needed): + if row_cursor[rr] > max_c: + max_c = row_cursor[rr] + if max_c + cw <= die_w: + x = max_c + y = float(r * row_h) + placement[c] = [x, y] + for rr in range(r, r + rows_needed): + row_cursor[rr] = x + cw + found = True + break + + if not found: + # Fallback (should not happen with < 100% utilization) + placement[c] = [0.0, 0.0] + + # Place standard cells: fill rows from bottom to top + # Maintain a pointer to the first row that still has space + available_rows = [r for r in range(n_rows) if row_cursor[r] < die_w] + row_ptr = 0 + + for c in std_cells: + cw = cells[c]["width"] + found = False + for ri in range(row_ptr, len(available_rows)): + r = available_rows[ri] + if row_cursor[r] + cw <= die_w: + x = row_cursor[r] + y = float(r * row_h) + placement[c] = [x, y] + row_cursor[r] = x + cw + if row_cursor[r] >= die_w: + row_ptr = ri + 1 + found = True + break + if not found: + placement[c] = [0.0, 0.0] + + return placement + + +# EVOLVE-BLOCK-END +def main(): + """Main routine. Keep output format (temp/submission.json) fixed.""" + print("=" * 60) + print("VLSI Global Placement - ISPD 2005 Benchmark") + print("=" * 60) + + # Load benchmark data + benchmark_name = os.environ.get("BENCHMARK_NAME", "adaptec1") + print(f"\nLoading benchmark: {benchmark_name}") + t0 = time.time() + data = load_benchmark(benchmark_name) + print(f" Loaded in {time.time() - t0:.2f}s") + print(f" Benchmark: {data['benchmark_name']}") + print(f" Die: {data['die']['width']} x {data['die']['height']}") + print(f" Cells: {len(data['cells'])} total " + f"({len(data['fixed_cells'])} fixed, " + f"{len(data['movable_cells'])} movable)") + print(f" Nets: {data['num_nets']}, Pins: {data['num_pins']}") + + # ALLOWED TO MODIFY: Placement algorithm call + print("\nPlacing components...") + t0 = time.time() + placement = place_components( + die=data["die"], + cells=data["cells"], + fixed_cells=data["fixed_cells"], + movable_cells=data["movable_cells"], + netlist=data["netlist"], + initial_placement=data["initial_placement"], + ) + print(f" Placed in {time.time() - t0:.2f}s") + print(f" Cells placed: {len(placement)}") + + # Compute HPWL + print("\nEvaluating placement...") + t0 = time.time() + hpwl_val = compute_hpwl(placement, data["netlist"], data["cells"]) + print(f" HPWL evaluated in {time.time() - t0:.2f}s") + print(f" HPWL: {hpwl_val:.2f}") + + # Verify all movable cells are placed + missing = [c for c in data["movable_cells"] if c not in placement] + if missing: + print(f" WARNING: {len(missing)} movable cells not placed!") + + # NOT ALLOWED TO MODIFY: Output format must match exactly + submission = { + "benchmark_id": "vlsi_global_placement", + "benchmark_name": data["benchmark_name"], + "placement": placement, + } + + temp_dir = Path("temp") + temp_dir.mkdir(exist_ok=True) + submission_path = temp_dir / "submission.json" + + with open(submission_path, "w", encoding="utf-8") as f: + json.dump(submission, f, indent=2) + + print(f"\nsubmission.json written to {submission_path}") + print(f" Cells placed: {len(placement)}") + print(f" HPWL: {hpwl_val:.2f}") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/preprocess.py b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/preprocess.py new file mode 100644 index 00000000..e6e55943 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/preprocess.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Preprocess ISPD 2005 Bookshelf benchmarks into compressed JSON format.""" +# +# Output format: gzip-compressed JSON with a compact netlist. +# Each net is stored as a list of integer cell indices (not dicts), +# reducing JSON size by ~65% compared to the full pin-dict format. +# gzip further reduces the on-disk size by ~84%. +# Decompression is handled by _decompress_netlist() in init.py +# and evaluator.py (reconstructs zero-offset pin dicts from indices). +# The transformation is lossless with respect to the HPWL computation. + +import gzip +import json +from pathlib import Path + + +def parse_nodes(path): + cells = [] + num_nodes = 0 + num_terminals = 0 + with open(path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or line.startswith('UCLA'): + continue + if line.startswith('NumNodes'): + num_nodes = int(line.split(':')[1].strip()) + continue + if line.startswith('NumTerminals'): + num_terminals = int(line.split(':')[1].strip()) + continue + parts = line.split() + if len(parts) >= 3: + name = parts[0] + width = float(parts[1]) + height = float(parts[2]) + cells.append({ + 'name': name, 'width': width, + 'height': height, + 'terminal': len(cells) < num_terminals + }) + return cells, num_nodes, num_terminals + + +def parse_nets(path): + nets = [] + current_net = None + with open(path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or line.startswith('UCLA'): + continue + if line.startswith('NumNets') or line.startswith('NumPins'): + continue + if line.startswith('NetDegree'): + if current_net is not None: + nets.append(current_net) + # Format: "NetDegree : 4 n0" + parts = line.split(':') + degree = int(parts[1].strip().split()[0]) + rest = parts[1].strip().split() + net_name = rest[1] if len(rest) > 1 else f'net_{len(nets)}' + current_net = {'name': net_name, 'degree': degree, 'pins': []} + elif current_net is not None: + parts = line.split() + if len(parts) >= 4: + cell_name = parts[0] + direction = parts[1].rstrip(':') + try: + x_off = float(parts[2].rstrip(':')) + y_off = float(parts[3]) + except (ValueError, IndexError): + x_off = 0.0 + y_off = 0.0 + current_net['pins'].append({ + 'cell': cell_name, + 'direction': direction, + 'x_offset': x_off, + 'y_offset': y_off, + }) + if current_net is not None: + nets.append(current_net) + return nets + + +def parse_scl(path): + min_x = float('inf') + min_y = float('inf') + max_x = float('-inf') + max_y = float('-inf') + height = 0 + in_row = False + with open(path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or line.startswith('UCLA'): + continue + if line.startswith('NumRows'): + continue + if line.startswith('CoreRow'): + in_row = True + continue + if line.startswith('End'): + in_row = False + continue + if in_row: + if line.startswith('Coordinate'): + y = float(line.split(':')[1].strip()) + min_y = min(min_y, y) + max_y = max(max_y, y) + elif line.startswith('Height'): + height = float(line.split(':')[1].strip()) + elif line.startswith('SubrowOrigin'): + # Format: "SubrowOrigin : 459 NumSites : 10692" + idx = line.find(':') + val_part = line[idx+1:].strip() + # Split on NumSites + if 'NumSites' in val_part: + x_str = val_part.split('NumSites')[0].strip() + x = float(x_str) + # Get NumSites value + nidx = val_part.rfind(':') + ns_str = val_part[nidx+1:].strip() + ns = int(ns_str.split()[0]) + min_x = min(min_x, x) + max_x = max(max_x, x + ns) + return { + 'width': max_x - min_x, + 'height': max_y - min_y + height, + 'row_height': height, + 'min_x': min_x, + 'min_y': min_y, + 'original_width': max_x - min_x, + 'original_height': max_y - min_y + height, + } + + +def parse_pl(path): + placements = {} + with open(path, 'r') as f: + for line in f: + line = line.strip() + if not line or line.startswith('#') or line.startswith('UCLA'): + continue + parts = line.split() + if len(parts) >= 3: + name = parts[0] + x = float(parts[1]) + y = float(parts[2]) + orient = parts[3] if len(parts) > 3 else 'N' + placements[name] = (x, y, orient) + return placements + + +def preprocess(name, bench_dir, output_dir): + print(f'Preprocessing {name}...') + cells, num_nodes, num_terminals = parse_nodes(bench_dir / f'{name}.nodes') + nets = parse_nets(bench_dir / f'{name}.nets') + die = parse_scl(bench_dir / f'{name}.scl') + placements = parse_pl(bench_dir / f'{name}.pl') + + cell_map = {c['name']: c for c in cells} + cell_to_idx = {c['name']: i for i, c in enumerate(cells)} + fixed_cells = [c['name'] for c in cells if c['terminal']] + movable_cells = [c['name'] for c in cells if not c['terminal']] + + init_placement = {} + for c in cells: + if c['name'] in placements: + x, y, orient = placements[c['name']] + init_placement[c['name']] = {'x': x, 'y': y, 'orientation': orient} + else: + init_placement[c['name']] = {'x': die['min_x'], 'y': die['min_y'], 'orientation': 'N'} + + netlist = [] + for net in nets: + net_pins = [] + for pin in net['pins']: + cell_name = pin['cell'] + if cell_name in cell_map: + # Compact format: store only the cell index. + # x_offset/y_offset are dropped (reconstructed as 0.0 in + # _decompress_netlist). This is safe because HPWL is computed + # from cell center coordinates, and pin offsets are zero in + # the ISPD 2005 benchmarks. + net_pins.append(cell_to_idx[cell_name]) + if net_pins: + netlist.append(net_pins) + + data = { + 'benchmark_name': name, + 'num_nodes': num_nodes, + 'num_terminals': num_terminals, + 'num_nets': len(netlist), + 'num_pins': sum(len(n) for n in netlist), + 'die': die, + 'cells': {c['name']: {'width': c['width'], 'height': c['height']} for c in cells}, + 'fixed_cells': fixed_cells, + 'movable_cells': movable_cells, + 'initial_placement': init_placement, + 'netlist': netlist, + } + + out_path = output_dir / f'{name}.json.gz' + with gzip.open(out_path, 'wt', encoding='utf-8', compresslevel=9) as f: + json.dump(data, f, separators=(',', ':')) + + n_fixed = len(fixed_cells) + n_movable = len(movable_cells) + n_nets = len(netlist) + dw = die['width'] + dh = die['height'] + print(f' -> {n_movable} movable, {n_fixed} fixed, {n_nets} nets') + print(f' -> Die: {dw}x{dh}') + print(f' -> Saved to {out_path}') + return data + + +def main(): + repo_root = Path(__file__).resolve().parents[1] + datasets_dir = repo_root / 'datasets' / 'ispd2005' + references_dir = repo_root / 'references' + references_dir.mkdir(exist_ok=True) + + benchmarks = [ + ('adaptec1', 'Easy'), + ('adaptec3', 'Medium'), + ] + + for name, difficulty in benchmarks: + bench_dir = datasets_dir / name + if bench_dir.exists(): + preprocess(name, bench_dir, references_dir) + diff_path = references_dir / f'{name}_difficulty.json' + with open(diff_path, 'w') as f: + json.dump({'difficulty': difficulty, 'benchmark': name}, f) + else: + print(f'Warning: {bench_dir} not found') + + print('Done! Preprocessed benchmarks saved to references/') + + +if __name__ == '__main__': + main() + + diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/docker/Dockerfile b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/docker/Dockerfile new file mode 100644 index 00000000..49b9c673 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /workspace + +COPY verification/evaluator.py /workspace/verification/evaluator.py +COPY verification/requirements.txt /workspace/verification/requirements.txt +COPY scripts/ /workspace/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/ +COPY references/ /workspace/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/references/ + +ENV BENCHMARK_NAME=adaptec1 +ENV FRONTIER_ENGINEERING_ROOT=/workspace + +CMD ["python", "/workspace/verification/evaluator.py", "/workspace/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/scripts/init.py"] diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/evaluator.py b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/evaluator.py new file mode 100644 index 00000000..b206f260 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/evaluator.py @@ -0,0 +1,1157 @@ +"""Evaluator for VLSI Global Placement - ISPD 2005 Benchmarks""" + + + +from __future__ import annotations + + + +import gzip + +import json + +import math + +import os + +import shutil + +import subprocess + +import sys + +import tempfile + +import time + +from pathlib import Path + +from typing import Any + + +def _decompress_netlist(data: dict) -> dict: + """Decompress compact netlist (cell indices) into original format.""" + if not data.get("netlist") or not data["netlist"]: + return data + if isinstance(data["netlist"][0], list) and data["netlist"][0] and isinstance(data["netlist"][0][0], dict): + return data + cell_names = list(data["cells"].keys()) + new_netlist = [] + for net in data["netlist"]: + new_net = [{"cell": cell_names[idx], "x_offset": 0.0, "y_offset": 0.0} for idx in net] + new_netlist.append(new_net) + data["netlist"] = new_netlist + return data + + +INVALID_COMBINED_SCORE = -1e18 + + + + + +# ============================================================================ + +# Repository root detection + +# ============================================================================ + + + +def _find_repo_root(start: Path | None = None) -> Path: + + """Locate the repository root directory.""" + + if "FRONTIER_ENGINEERING_ROOT" in os.environ: + + return Path(os.environ["FRONTIER_ENGINEERING_ROOT"]).expanduser().resolve() + + here = (start or Path(__file__)).resolve() + + for parent in [here, *here.parents]: + + if (parent / "frontier_eval").is_dir() and (parent / "benchmarks").is_dir(): + + return parent + + if (parent / "frontier_eval").is_dir() and (parent / "verification").is_dir(): + + return parent.parent.parent.parent + + return here.parent.parent.parent + + + + + +def _tail(text: str, limit: int = 8000) -> str: + + return text if len(text) <= limit else text[-limit:] + + + + + +def _truncate_middle(text: str, limit: int = 200_000) -> str: + + if len(text) <= limit: + + return text + + keep = max(0, (limit - 128) // 2) + + omitted = len(text) - 2 * keep + + return text[:keep] + f"\n\n[... truncated {omitted} chars ...]\n\n" + text[-keep:] + + + + + +# ============================================================================ + +# Benchmark data loading + +# ============================================================================ + + + +def _get_benchmark_dir(repo_root: Path) -> Path: + + return ( + + repo_root + + / "benchmarks" + + / "ElectronicDesignAutomation" + + / "VLSIGlobalPlacement" + + ) + + + + + +def _get_reference_path(repo_root: Path, benchmark_name: str) -> Path: + + ref_dir = _get_benchmark_dir(repo_root) / "references" + + gz_path = ref_dir / f"{benchmark_name}.json.gz" + + if gz_path.is_file(): + + return gz_path + + return ref_dir / f"{benchmark_name}.json" + + + + + +def _get_difficulty_path(repo_root: Path, benchmark_name: str) -> Path: + + return _get_benchmark_dir(repo_root) / "references" / f"{benchmark_name}_difficulty.json" + + + + + +def _load_json(path: Path) -> dict: + + """Load a JSON file, transparently decompressing gzip data.""" + + if str(path).endswith(".json.gz"): + + with gzip.open(path, "rt", encoding="utf-8") as f: + + return json.load(f) + + with open(path, "r", encoding="utf-8") as f: + + return json.load(f) + + + + + +def _list_available_benchmarks(repo_root: Path) -> list[dict]: + + """List all available benchmarks with their difficulty levels.""" + + ref_dir = _get_benchmark_dir(repo_root) / "references" + + results = [] + + for f in sorted(ref_dir.glob("*_difficulty.json")): + + name = f.name.replace("_difficulty.json", "") + + with open(f, "r") as fh: + + meta = json.load(fh) + + results.append({"name": name, "difficulty": meta.get("difficulty", "Unknown")}) + + return results + + + + + +# ============================================================================ + +# HPWL Computation (independent implementation) + +# ============================================================================ + + + +def compute_hpwl(placement: dict, netlist: list, cells: dict) -> float: + + """Compute Half-Perimeter Wirelength for the placement.""" + + total_hpwl = 0.0 + + for net in netlist: + + xs = [] + + ys = [] + + for pin in net: + + cell_name = pin["cell"] + + if cell_name not in placement: + + continue + + cx, cy = placement[cell_name] + + w = cells[cell_name]["width"] + + h = cells[cell_name]["height"] + + px = cx + w / 2 + pin.get("x_offset", 0.0) + + py = cy + h / 2 + pin.get("y_offset", 0.0) + + xs.append(px) + + ys.append(py) + + if xs: + + total_hpwl += (max(xs) - min(xs)) + (max(ys) - min(ys)) + + return total_hpwl + + + + + +# ============================================================================ + +# Legality checks + +# ============================================================================ + + + +class LegalityResult: + + def __init__(self): + + self.valid = True + + self.errors: list[str] = [] + + self.moved_fixed: list[str] = [] + + self.missing: list[str] = [] + self.out_of_bounds: list[str] = [] + + self.overlapping_pairs: list[tuple[str, str, float]] = [] + + + + + +def check_legality( + + placement: dict, + + cells: dict, + + die: dict, + + fixed_cells: list, + + movable_cells: list, + + initial_placement: dict, + +) -> LegalityResult: + + """Check placement legality: fixed cells, bounds, overlap.""" + + result = LegalityResult() + + + + die_w = die["width"] + + die_h = die["height"] + + + + # 1. Check all expected cells are present in placement + + all_expected = fixed_cells + movable_cells + + for c in all_expected: + + if c not in placement: + + result.errors.append(f"Cell {c} missing from placement") + + result.missing.append(c) + + result.valid = False + + + + # 2. Check fixed cells are not moved + + for c in fixed_cells: + + if c not in placement: + + result.errors.append(f"Fixed cell {c} missing from placement") + + result.valid = False + + continue + + ip = initial_placement.get(c, {"x": 0.0, "y": 0.0}) + + px, py = placement[c] + + if abs(px - ip["x"]) > 1e-6 or abs(py - ip["y"]) > 1e-6: + + result.moved_fixed.append(c) + + result.valid = False + + + + # 3. Check all cells within die boundary + + for c, (px, py) in placement.items(): + + if c not in cells: + + continue + + cw = cells[c]["width"] + + ch = cells[c]["height"] + + if px < 0 or py < 0 or px + cw > die_w or py + ch > die_h: + + result.out_of_bounds.append(c) + + result.valid = False + + + + # 4. Overlap check using spatial hashing + + # Build a grid hash for efficient overlap detection + + all_placed_cells = list(placement.keys()) + + if len(all_placed_cells) > 0: + + # Compute grid size: use sqrt(n_cells) target bins per dimension + + n_total = len(all_placed_cells) + + target_bins = max(4, int(math.sqrt(n_total))) + + grid_size = max(die_w / target_bins, die_h / target_bins, 1.0) + + + + grid_w = max(1, int(math.ceil(die_w / grid_size))) + + grid_h = max(1, int(math.ceil(die_h / grid_size))) + + + + # Build spatial hash: grid cell -> list of cell names + + spatial_hash = {} + + for c in all_placed_cells: + + if c not in cells: + + continue + + px, py = placement[c] + + cw = cells[c]['width'] + + ch = cells[c]['height'] + + gx1 = max(0, int(px // grid_size)) + + gy1 = max(0, int(py // grid_size)) + + gx2 = min(grid_w - 1, int((px + cw) // grid_size)) + + gy2 = min(grid_h - 1, int((py + ch) // grid_size)) + + for gx in range(gx1, gx2 + 1): + + for gy in range(gy1, gy2 + 1): + + spatial_hash.setdefault((gx, gy), []).append(c) + + + + # Check each grid cell for overlaps + + # Use local sets per grid cell to avoid global MemoryError + + MAX_OVERLAP_PAIRS = 1000 + + max_cells_per_cell = 0 + + for cell_list in spatial_hash.values(): + + max_cells_per_cell = max(max_cells_per_cell, len(cell_list)) + + + + # If any grid cell has >2000 cells, the placement is degenerate + + # (e.g. all cells stacked at the same position). Mark as invalid + + # without checking all O(N^2) pairs. + + if max_cells_per_cell > 2000: + + result.valid = False + + result.errors.append( + + 'Degenerate placement: %d cells in one grid cell. ' + + 'Likely all cells stacked at same position.' % max_cells_per_cell + + ) + + result.overlapping_pairs.append( + + ('__degenerate__', '__all_cells__', float(len(all_placed_cells))) + + ) + + else: + + # Global dedup set to prevent same pair counted across multiple grid cells + + global_checked = set() + + for cell_list in spatial_hash.values(): + + if len(cell_list) < 2: + + continue + + for i in range(len(cell_list)): + + for j in range(i + 1, len(cell_list)): + + c1 = cell_list[i] + + c2 = cell_list[j] + + pair = (c1, c2) if c1 < c2 else (c2, c1) + + if pair in global_checked: + + continue + + global_checked.add(pair) + + if c1 not in cells or c2 not in cells: + + continue + + px1, py1 = placement[c1] + + pw1 = cells[c1]['width'] + + ph1 = cells[c1]['height'] + + px2, py2 = placement[c2] + + pw2 = cells[c2]['width'] + + ph2 = cells[c2]['height'] + + # AABB overlap test + + if (px1 < px2 + pw2 and px1 + pw1 > px2 and + + py1 < py2 + ph2 and py1 + ph1 > py2): + + overlap_area = ( + + min(px1 + pw1, px2 + pw2) - max(px1, px2) + + ) * ( + + min(py1 + ph1, py2 + ph2) - max(py1, py2) + + ) + + result.overlapping_pairs.append((c1, c2, overlap_area)) + + result.valid = False + + if len(result.overlapping_pairs) >= MAX_OVERLAP_PAIRS: + + break + + if len(result.overlapping_pairs) >= MAX_OVERLAP_PAIRS: + + break + + if len(global_checked) > 100000: + + break + + if len(result.overlapping_pairs) >= MAX_OVERLAP_PAIRS: + + break + + + + return result + + + +# ============================================================================ + +# EVOLVE-BLOCK boundary validation + +# ============================================================================ + + + +EVOLVE_START_MARKER = "# EVOLVE-BLOCK-START" + +EVOLVE_END_MARKER = "# EVOLVE-BLOCK-END" + + + + + +def _split_evolve_block(text: str) -> tuple[str, str] | None: + + """Split program text into (outside_start, outside_end) around the EVOLVE-BLOCK. + + + + Returns None if the markers are missing or malformed. The returned parts + + include the marker lines themselves; only the interior is omitted. + + """ + + s = text.find(EVOLVE_START_MARKER) + + e = text.find(EVOLVE_END_MARKER) + + if s == -1 or e == -1 or e <= s: + + return None + + e += len(EVOLVE_END_MARKER) + + return text[:s], text[e:] + + + + + +def _normalize_newlines(text: str) -> str: + + return text.replace("\r\n", "\n").replace("\r", "\n") + + + + + +def check_evolve_boundary(candidate_path: Path, reference_path: Path) -> str | None: + + """Verify the candidate only modified code inside the EVOLVE-BLOCK. + + + + Returns an error message if the candidate changed anything outside the + + EVOLVE-BLOCK, otherwise returns None. + + """ + + try: + + candidate_text = _normalize_newlines(candidate_path.read_text(encoding="utf-8")) + + reference_text = _normalize_newlines(reference_path.read_text(encoding="utf-8")) + + except OSError as exc: + + return f"failed to read program for EVOLVE-BLOCK validation: {exc}" + + + + cand_parts = _split_evolve_block(candidate_text) + + ref_parts = _split_evolve_block(reference_text) + + if cand_parts is None: + + return "candidate program is missing the EVOLVE-BLOCK markers" + + if ref_parts is None: + + return "reference program is missing the EVOLVE-BLOCK markers" + + cand_before, cand_after = cand_parts + + ref_before, ref_after = ref_parts + + if cand_before != ref_before: + + return "candidate modified code outside the EVOLVE-BLOCK (before the block)" + + if cand_after != ref_after: + + return "candidate modified code outside the EVOLVE-BLOCK (after the block)" + + return None + + + + + +# ============================================================================ + +# Candidate resource limits + +# ============================================================================ + + + +# Defense-in-depth resource limits applied to the candidate subprocess on + +# POSIX platforms (the 600s timeout remains the primary protection). + +RLIMIT_CPU_SECONDS = 590 + +RLIMIT_NPROC = 256 + +RLIMIT_AS_BYTES = 8 * 1024 ** 3 # 8 GiB address-space ceiling + + + + + +def _limit_candidate_resources() -> None: + + """Apply resource limits in the candidate subprocess (POSIX only).""" + + if os.name != "posix": + + return + + import resource + + resource.setrlimit(resource.RLIMIT_CPU, (RLIMIT_CPU_SECONDS, RLIMIT_CPU_SECONDS + 10)) + + resource.setrlimit(resource.RLIMIT_NPROC, (RLIMIT_NPROC, RLIMIT_NPROC)) + + resource.setrlimit(resource.RLIMIT_AS, (RLIMIT_AS_BYTES, RLIMIT_AS_BYTES)) + + + + + +# ============================================================================ + +# Main evaluation function + +# ============================================================================ + + + +def evaluate( + + program_path: str, + + *, + + repo_root: Path | None = None, + + benchmark_name: str | None = None, + +) -> Any: + + """ + + Full evaluation pipeline: + + 1. Run candidate program to produce temp/submission.json + + 2. Parse and validate submission + + 3. Run independent HPWL + legality checks + + 4. Return metrics + + + + Parameters + + ---------- + + program_path : str + + Path to the candidate Python program. + + repo_root : Path, optional + + Repository root. Auto-detected if not given. + + benchmark_name : str, optional + + Benchmark to use (e.g., "adaptec1", "adaptec3"). + + Defaults to "adaptec1". + + """ + + start = time.time() + + repo_root = ( + + _find_repo_root() if repo_root is None else repo_root.expanduser().resolve() + + ) + + program_path_resolved = str(Path(program_path).expanduser().resolve()) + + + + # Determine benchmark + + if benchmark_name is None: + + benchmark_name = os.environ.get("BENCHMARK_NAME", "adaptec1") + + + + work_dir = Path(tempfile.mkdtemp(prefix="fe_vlsigp_")).resolve() + + artifacts: dict[str, str] = {} + + + + metrics: dict[str, float] = { + + "combined_score": INVALID_COMBINED_SCORE, + + "hpwl": 0.0, + + "valid": 0.0, + + + + "timeout": 0.0, + + "runtime_s": 0.0, + + "n_cells_placed": 0.0, + + "n_fixed_moved": 0.0, + + "n_out_of_bounds": 0.0, + + "n_overlaps": 0.0, + + } + + + + # Verify the candidate only modified code inside the EVOLVE-BLOCK + + reference_program = _get_benchmark_dir(repo_root) / "scripts" / "init.py" + + evolve_error = check_evolve_boundary( + + Path(program_path_resolved), reference_program + + ) + + if evolve_error is not None: + + artifacts["evolve_boundary_error"] = evolve_error + + artifacts["error_message"] = ( + + "EVOLVE-BLOCK boundary violation: " + evolve_error + + ) + + metrics["runtime_s"] = float(time.time() - start) + + return _wrap(metrics, artifacts) + + + + try: + + # 1. Copy benchmark reference data to work dir + + ref_path = _get_reference_path(repo_root, benchmark_name) + + if not ref_path.is_file(): + + available = _list_available_benchmarks(repo_root) + + avail_str = ", ".join(f"{b['name']} ({b['difficulty']})" for b in available) + + artifacts["error_message"] = ( + + f"Benchmark '{benchmark_name}' not found. " + + f"Available: {avail_str}" + + ) + + metrics["runtime_s"] = float(time.time() - start) + + return _wrap(metrics, artifacts) + + + + refs_dir = work_dir / "references" + + refs_dir.mkdir(parents=True, exist_ok=True) + + # Copy the benchmark JSON (keeps .json.gz extension if present) + + shutil.copy2(ref_path, refs_dir / ref_path.name) + + + + # Also copy difficulty metadata + + diff_path = _get_difficulty_path(repo_root, benchmark_name) + + if diff_path.is_file(): + + shutil.copy2(diff_path, refs_dir / f"{benchmark_name}_difficulty.json") + + + + # 2. Run candidate program + + env = os.environ.copy() + + env["BENCHMARK_NAME"] = benchmark_name + + + + try: + + proc = subprocess.run( + + [sys.executable, program_path_resolved], + + cwd=str(work_dir), + + env=env, + + capture_output=True, + + text=True, + + timeout=600, + + preexec_fn=_limit_candidate_resources if os.name == "posix" else None, + + ) + + except subprocess.TimeoutExpired as e: + + metrics["timeout"] = 1.0 + + metrics["runtime_s"] = float(time.time() - start) + + artifacts["error_message"] = f"program timeout: {e}" + + return _wrap(metrics, artifacts) + + + + artifacts["program_stdout"] = _tail(proc.stdout) + + artifacts["program_stderr"] = _tail(proc.stderr) + + artifacts["program_stdout_full"] = _truncate_middle(proc.stdout) + + artifacts["program_stderr_full"] = _truncate_middle(proc.stderr) + + metrics["program_returncode"] = float(proc.returncode) + + + + # 3. Read submission + + submission_path = work_dir / "temp" / "submission.json" + + if not submission_path.exists(): + + submission_path = work_dir / "submission.json" + + if not submission_path.exists(): + + artifacts["error_message"] = ( + + "submission.json not generated " + + "(checked temp/submission.json and submission.json)" + + ) + + metrics["runtime_s"] = float(time.time() - start) + + return _wrap(metrics, artifacts) + + + + try: + + with open(submission_path, "r", encoding="utf-8") as f: + + submission = json.load(f) + + # Truncate large submissions for artifacts + + sub_str = json.dumps(submission, indent=2) + + if len(sub_str) > 10000: + + artifacts["submission.json"] = sub_str[:5000] + "\n... [truncated] ...\n" + sub_str[-5000:] + + else: + + artifacts["submission.json"] = sub_str + + except Exception as exc: + + artifacts["error_message"] = f"Failed to parse submission.json: {exc}" + + metrics["runtime_s"] = float(time.time() - start) + + return _wrap(metrics, artifacts) + + + + if "placement" not in submission: + + artifacts["error_message"] = "submission.json missing 'placement'" + + metrics["runtime_s"] = float(time.time() - start) + + return _wrap(metrics, artifacts) + + + + placement = submission["placement"] + + + + # 4. Load benchmark data for evaluation + + benchmark_data = _decompress_netlist(_load_json(ref_path)) + + + + metrics["n_cells_placed"] = float(len(placement)) + + + + # 5. Compute HPWL + + hpwl = compute_hpwl( + + placement, + + benchmark_data["netlist"], + + benchmark_data["cells"], + + ) + + metrics["hpwl"] = hpwl + + + + # 6. Legality checks + + legality = check_legality( + + placement, + + benchmark_data["cells"], + + benchmark_data["die"], + + benchmark_data["fixed_cells"], + + benchmark_data["movable_cells"], + + benchmark_data["initial_placement"], + + ) + + + + metrics["n_fixed_moved"] = float(len(legality.moved_fixed)) + + metrics["n_out_of_bounds"] = float(len(legality.out_of_bounds)) + + metrics["n_overlaps"] = float(len(legality.overlapping_pairs)) + + + + if not legality.valid: + + artifacts["legality_errors"] = json.dumps({ + + "moved_fixed": legality.moved_fixed[:100], + + "out_of_bounds": legality.out_of_bounds[:100], + + "overlaps": [ + + {"cell1": p[0], "cell2": p[1], "overlap_area": p[2]} + + for p in legality.overlapping_pairs[:100] + + ], + + "total_moved_fixed": len(legality.moved_fixed), + + "total_out_of_bounds": len(legality.out_of_bounds), + + "total_overlaps": len(legality.overlapping_pairs), + + "other_errors": legality.errors, + + }, indent=2) + + + + runtime_s = time.time() - start + + metrics["runtime_s"] = float(runtime_s) + + + + + + if legality.valid: + + # Minimization: negate HPWL so higher combined_score = better + + metrics["combined_score"] = -hpwl + + metrics["valid"] = 1.0 + + else: + + metrics["combined_score"] = INVALID_COMBINED_SCORE + + metrics["valid"] = 0.0 + + + + return _wrap(metrics, artifacts) + + finally: + + shutil.rmtree(work_dir, ignore_errors=True) + + + + + +def _wrap(metrics: dict[str, float], artifacts: dict[str, str]) -> Any: + + try: + + from openevolve.evaluation_result import EvaluationResult + + + + return EvaluationResult(metrics=metrics, artifacts=artifacts) + + except Exception: + + return metrics + + + + + +if __name__ == "__main__": + + import argparse + + + + parser = argparse.ArgumentParser(description="VLSI Global Placement Evaluator") + + parser.add_argument("program_path", help="Path to the candidate Python program") + + parser.add_argument("--benchmark", default="adaptec1", + + help="Benchmark name (adaptec1, adaptec3)") + + args = parser.parse_args() + + + + result = evaluate(args.program_path, benchmark_name=args.benchmark) + + if hasattr(result, "metrics"): + + output = {"metrics": result.metrics, "artifacts": result.artifacts} + + else: + + output = result + + print(json.dumps(output, indent=2)) + diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/requirements.txt b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/requirements.txt new file mode 100644 index 00000000..13bb46e6 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/requirements.txt @@ -0,0 +1,2 @@ +# No external dependencies required for this benchmark. +# Pure Python standard library only. diff --git a/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/test_evaluator.py b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/test_evaluator.py new file mode 100644 index 00000000..cd683a53 --- /dev/null +++ b/benchmarks/ElectronicDesignAutomation/VLSIGlobalPlacement/verification/test_evaluator.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +"""Unit tests for the VLSI Global Placement evaluator. + +Run with: python verification/test_evaluator.py +or: python -m unittest verification.test_evaluator +Uses only the Python standard library (unittest). +""" +import gzip +import json +import pathlib +import shutil +import sys +import tempfile +import unittest + +HERE = pathlib.Path(__file__).resolve().parent +BENCHMARK_DIR = HERE.parent +REAL_INIT = BENCHMARK_DIR / "scripts" / "init.py" +sys.path.insert(0, str(HERE)) + +import evaluator # noqa: E402 + + +def _cells(): + return { + "f1": {"width": 10.0, "height": 10.0}, + "f2": {"width": 10.0, "height": 10.0}, + "a": {"width": 10.0, "height": 10.0}, + "b": {"width": 20.0, "height": 10.0}, + "c": {"width": 10.0, "height": 10.0}, + } + + +def _die(): + return {"width": 100.0, "height": 100.0, "row_height": 10.0, + "min_x": 0.0, "min_y": 0.0} + + +def _tiny_benchmark_data(): + return { + "benchmark_name": "tiny", + "num_nodes": 5, + "num_terminals": 2, + "num_nets": 3, + "num_pins": 8, + "die": _die(), + "cells": _cells(), + "fixed_cells": ["f1", "f2"], + "movable_cells": ["a", "b", "c"], + "initial_placement": { + "f1": {"x": 0.0, "y": 0.0, "orientation": "N"}, + "f2": {"x": 0.0, "y": 80.0, "orientation": "N"}, + "a": {"x": 0.0, "y": 10.0, "orientation": "N"}, + "b": {"x": 0.0, "y": 30.0, "orientation": "N"}, + "c": {"x": 0.0, "y": 50.0, "orientation": "N"}, + }, + "netlist": [ + [0, 2], # f1 - a + [1, 3], # f2 - b + [2, 3, 4], # a - b - c + ], + } + + +def _netlist_verbose(data): + names = list(data["cells"].keys()) + out = [] + for net in data["netlist"]: + out.append([{"cell": names[i], "x_offset": 0.0, "y_offset": 0.0} for i in net]) + return out + + +class TestComputeHpwl(unittest.TestCase): + def test_single_net_known_value(self): + cells = _cells() + net = [ + {"cell": "a", "x_offset": 0.0, "y_offset": 0.0}, + {"cell": "b", "x_offset": 0.0, "y_offset": 0.0}, + ] + placement = {"a": [0.0, 0.0], "b": [30.0, 40.0]} + # a center: (5,5); b center: (40,45) -> HPWL = (40-5)+(45-5) = 75 + self.assertAlmostEqual(evaluator.compute_hpwl(placement, [net], cells), 75.0) + + def test_empty_net_contributes_zero(self): + cells = _cells() + placement = {"a": [0.0, 0.0]} + net = [] + self.assertEqual(evaluator.compute_hpwl(placement, [net], cells), 0.0) + + def test_missing_cell_skipped(self): + cells = _cells() + net = [ + {"cell": "a", "x_offset": 0.0, "y_offset": 0.0}, + {"cell": "b", "x_offset": 0.0, "y_offset": 0.0}, + ] + placement = {"a": [0.0, 0.0]} # b missing -> only a's span = 0 + self.assertEqual(evaluator.compute_hpwl(placement, [net], cells), 0.0) + + def test_pin_offsets_included(self): + cells = _cells() + net = [ + {"cell": "a", "x_offset": 5.0, "y_offset": 0.0}, + {"cell": "b", "x_offset": 0.0, "y_offset": 0.0}, + ] + placement = {"a": [0.0, 0.0], "b": [30.0, 0.0]} + # a pin center: 5+5=10; b pin center: 30+10=40 -> HPWL = 30 + self.assertAlmostEqual(evaluator.compute_hpwl(placement, [net], cells), 30.0) + + +class TestCheckLegality(unittest.TestCase): + def _placement(self, **overrides): + p = { + "f1": [0.0, 0.0], + "f2": [0.0, 80.0], + "a": [0.0, 10.0], + "b": [20.0, 30.0], + "c": [0.0, 50.0], + } + p.update(overrides) + return p + + def test_valid_placement(self): + res = evaluator.check_legality( + self._placement(), _cells(), _die(), + ["f1", "f2"], ["a", "b", "c"], _tiny_benchmark_data()["initial_placement"], + ) + self.assertTrue(res.valid) + self.assertEqual(len(res.moved_fixed), 0) + self.assertEqual(len(res.out_of_bounds), 0) + self.assertEqual(len(res.overlapping_pairs), 0) + self.assertEqual(len(res.missing), 0) + + def test_fixed_cell_moved(self): + res = evaluator.check_legality( + self._placement(f1=[50.0, 50.0]), _cells(), _die(), + ["f1", "f2"], ["a", "b", "c"], _tiny_benchmark_data()["initial_placement"], + ) + self.assertFalse(res.valid) + self.assertIn("f1", res.moved_fixed) + + def test_out_of_bounds(self): + res = evaluator.check_legality( + self._placement(c=[90.0, 95.0]), _cells(), _die(), + ["f1", "f2"], ["a", "b", "c"], _tiny_benchmark_data()["initial_placement"], + ) + self.assertFalse(res.valid) + self.assertIn("c", res.out_of_bounds) + + def test_overlap_detected(self): + res = evaluator.check_legality( + self._placement(a=[0.0, 0.0]), _cells(), _die(), + ["f1", "f2"], ["a", "b", "c"], _tiny_benchmark_data()["initial_placement"], + ) + self.assertFalse(res.valid) + self.assertTrue(len(res.overlapping_pairs) >= 1) + + def test_missing_cell(self): + placement = { + "f1": [0.0, 0.0], "f2": [0.0, 80.0], + "a": [0.0, 10.0], "b": [20.0, 30.0], + } # c missing + res = evaluator.check_legality( + placement, _cells(), _die(), + ["f1", "f2"], ["a", "b", "c"], _tiny_benchmark_data()["initial_placement"], + ) + self.assertFalse(res.valid) + self.assertIn("c", res.missing) + + +class TestEvolveBoundary(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = pathlib.Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def _modified_init(self, mode): + text = REAL_INIT.read_text(encoding="utf-8") + if mode == "violation": + # Change code AFTER the EVOLVE-BLOCK (inside main()). + text = text.replace(' print("=" * 60)\n', ' print("!" * 60)\n', 1) + elif mode == "legit": + # Replace only the interior of the EVOLVE-BLOCK with a tiny + # deterministic row-based placer. + start = text.find("# EVOLVE-BLOCK-START") + end = text.find("# EVOLVE-BLOCK-END") + interior = '''def place_components( + die, cells, fixed_cells, movable_cells, netlist, initial_placement, +): + placement = {} + for c in fixed_cells: + placement[c] = [initial_placement[c]["x"], initial_placement[c]["y"]] + x = 0.0 + y = 20.0 # start below fixed cells in row 0 + row_h = die["row_height"] + die_w = die["width"] + for c in movable_cells: + w = cells[c]["width"] + h = cells[c]["height"] + if x + w > die_w: + x = 0.0 + y += row_h + placement[c] = [x, y] + x += w + return placement +''' + text = text[:start] + "# EVOLVE-BLOCK-START\n" + interior + "# EVOLVE-BLOCK-END" + text[end + len("# EVOLVE-BLOCK-END"):] + return text + + def test_boundary_violation_rejected(self): + cand = self.tmp / "init_violation.py" + cand.write_text(self._modified_init("violation"), encoding="utf-8") + err = evaluator.check_evolve_boundary(cand, REAL_INIT) + self.assertIsNotNone(err) + self.assertIn("EVOLVE-BLOCK", err) + + def test_legit_change_accepted(self): + cand = self.tmp / "init_legit.py" + cand.write_text(self._modified_init("legit"), encoding="utf-8") + err = evaluator.check_evolve_boundary(cand, REAL_INIT) + self.assertIsNone(err) + + def test_missing_markers_rejected(self): + cand = self.tmp / "init_no_markers.py" + text = REAL_INIT.read_text(encoding="utf-8") + text = text.replace("# EVOLVE-BLOCK-START", "") + cand.write_text(text, encoding="utf-8") + err = evaluator.check_evolve_boundary(cand, REAL_INIT) + self.assertIsNotNone(err) + + +class TestEndToEnd(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.repo = pathlib.Path(self._tmp.name) + self.bench_dir = self.repo / "benchmarks" / "ElectronicDesignAutomation" / "VLSIGlobalPlacement" + self.ref_dir = self.bench_dir / "references" + self.scripts_dir = self.bench_dir / "scripts" + self.ref_dir.mkdir(parents=True) + self.scripts_dir.mkdir(parents=True) + # tiny benchmark data as gzip + data = _tiny_benchmark_data() + with gzip.open(self.ref_dir / "tiny.json.gz", "wt", encoding="utf-8") as f: + json.dump(data, f) + (self.ref_dir / "tiny_difficulty.json").write_text( + json.dumps({"difficulty": "Easy", "benchmark": "tiny"}), encoding="utf-8" + ) + # reference init.py in the fake repo + self.ref_init = self.scripts_dir / "init.py" + shutil.copy2(REAL_INIT, self.ref_init) + + def tearDown(self): + self._tmp.cleanup() + + def test_e2e_valid_baseline(self): + result = evaluator.evaluate( + str(self.ref_init), repo_root=self.repo, benchmark_name="tiny" + ) + metrics = result.metrics if hasattr(result, "metrics") else result + self.assertEqual(metrics["valid"], 1.0) + self.assertGreater(metrics["hpwl"], 0.0) + self.assertEqual(metrics["n_overlaps"], 0.0) + self.assertEqual(metrics["n_out_of_bounds"], 0.0) + self.assertEqual(metrics["n_fixed_moved"], 0.0) + self.assertNotEqual(metrics["combined_score"], evaluator.INVALID_COMBINED_SCORE) + + def test_e2e_boundary_violation_fails(self): + cand = self.scripts_dir / "init_violation.py" + text = REAL_INIT.read_text(encoding="utf-8") + text = text.replace(' print("=" * 60)\n', ' print("!" * 60)\n', 1) + cand.write_text(text, encoding="utf-8") + result = evaluator.evaluate( + str(cand), repo_root=self.repo, benchmark_name="tiny" + ) + metrics = result.metrics if hasattr(result, "metrics") else result + self.assertEqual(metrics["valid"], 0.0) + self.assertEqual(metrics["combined_score"], evaluator.INVALID_COMBINED_SCORE) + + def test_e2e_legit_modification_runs(self): + cand = self.scripts_dir / "init_legit.py" + text = REAL_INIT.read_text(encoding="utf-8") + start = text.find("# EVOLVE-BLOCK-START") + end = text.find("# EVOLVE-BLOCK-END") + interior = '''def place_components( + die, cells, fixed_cells, movable_cells, netlist, initial_placement, +): + placement = {} + for c in fixed_cells: + placement[c] = [initial_placement[c]["x"], initial_placement[c]["y"]] + x = 0.0 + y = 20.0 # start below fixed cells in row 0 + row_h = die["row_height"] + die_w = die["width"] + for c in movable_cells: + w = cells[c]["width"] + h = cells[c]["height"] + if x + w > die_w: + x = 0.0 + y += row_h + placement[c] = [x, y] + x += w + return placement +''' + text = text[:start] + "# EVOLVE-BLOCK-START\n" + interior + "# EVOLVE-BLOCK-END" + text[end + len("# EVOLVE-BLOCK-END"):] + cand.write_text(text, encoding="utf-8") + result = evaluator.evaluate( + str(cand), repo_root=self.repo, benchmark_name="tiny" + ) + metrics = result.metrics if hasattr(result, "metrics") else result + artifacts = result.artifacts if hasattr(result, "artifacts") else {} + self.assertNotIn("evolve_boundary_error", artifacts) + self.assertEqual(metrics["valid"], 1.0) + self.assertGreater(metrics["hpwl"], 0.0) + + +if __name__ == "__main__": + unittest.main(verbosity=2)