{ "name": "CodeGenie HarmonyOS Zero-to-One Bootstrap Eval", "generated_at": "2026-04-21T07:48:27.856Z", "summary": { "total": 5, "passed": 5, "failed": 0, "pass_rate": 1, "avg_wall_clock_ms": 685883, "avg_total_tokens": 849770, "avg_tool_calls_total": 45.8, "avg_skill_loads_total": 2 }, "by_kind": { "jscrash_fix": { "total": 0, "passed": 0, "failed": 0, "pass_rate": 0, "avg_wall_clock_ms": 0, "avg_total_tokens": 0, "avg_tool_calls_total": 0, "avg_skill_loads_total": 0 }, "project_bootstrap": { "total": 5, "passed": 5, "failed": 0, "pass_rate": 1, "avg_wall_clock_ms": 685883, "avg_total_tokens": 849770, "avg_tool_calls_total": 45.8, "avg_skill_loads_total": 2 }, "ui_increment": { "total": 0, "passed": 0, "failed": 0, "pass_rate": 0, "avg_wall_clock_ms": 0, "avg_total_tokens": 0, "avg_tool_calls_total": 0, "avg_skill_loads_total": 0 } }, "tasks": [ { "id": "bootstrap-hello-world", "kind": "project_bootstrap", "title": "Generate a Hello World HarmonyOS project and run it", "pass": true, "score": 100, "final_stage": "build", "rounds": 1, "wall_clock_ms": 203561, "input_tokens": 309913, "output_tokens": 2549, "reasoning_tokens": 524, "total_tokens": 312986, "tool_calls_total": 23, "tool_calls_by_name": { "todowrite": 5, "read": 4, "skill": 1, "bash": 2, "glob": 1, "edit": 4, "codegenie-mcp_init_project_path": 1, "codegenie-mcp_build_project": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 1, "skill_loads_by_name": { "deveco-create-project": 1 }, "steps": [ { "id": "create", "prompt": "请在当前工作目录从 0 到 1 生成一个名为 HelloWorld 的鸿蒙应用，提供一个简洁的 Hello World 页面，最后完成编译并尝试运行，如受环境限制请明确说明原因。", "goal": "从 0 到 1 生成 Hello World 鸿蒙工程，并完成编译与运行尝试", "stage": "build", "pass": true, "wall_clock_ms": 202711, "input_tokens": 309913, "output_tokens": 2549, "reasoning_tokens": 524, "total_tokens": 312986, "tool_calls_total": 23, "tool_calls_by_name": { "todowrite": 5, "read": 4, "skill": 1, "bash": 2, "glob": 1, "edit": 4, "codegenie-mcp_init_project_path": 1, "codegenie-mcp_build_project": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 1, "skill_loads_by_name": { "deveco-create-project": 1 } } ], "dir": "C:\\Users\\xi\\Desktop\\bench_mark\\codegenie-cli-benchmark\\artifacts_opencode\\artifact_bootstrap-0to1_20260421145118394\\runs\\bootstrap-hello-world", "session_id": "ses_251313cebffevHwBKm1pwJsq32", "assertions": [ { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: HelloWorld/AppScope/app.json5" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: HelloWorld/entry/src/main/ets/pages/Index.ets" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: HelloWorld/entry/build/default/outputs/default/app/entry-default.hap" } ] }, { "id": "bootstrap-luckin-coffee", "kind": "project_bootstrap", "title": "Generate a Luckin Coffee HarmonyOS app and run it", "pass": true, "score": 100, "final_stage": "build", "rounds": 1, "wall_clock_ms": 624800, "input_tokens": 1105498, "output_tokens": 12984, "reasoning_tokens": 966, "total_tokens": 1119448, "tool_calls_total": 57, "tool_calls_by_name": { "todowrite": 6, "read": 24, "skill": 1, "bash": 1, "codegenie-mcp_init_project_path": 2, "write": 7, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 4, "edit": 8, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 1, "skill_loads_by_name": { "deveco-create-project": 1 }, "steps": [ { "id": "create", "prompt": "我想要一个库幸咖啡的应用，里面有 4 个页面，分别是首页、菜单、会员卡、我的。首页有一个热销饮品的图片，然后依次是“到店取”“幸运送”“电商购”3 个功能入口，然后是“充值”“礼品卡”“团餐”“拼单”4 个功能入口，然后有一个优惠购的图片引导链接，标题写个大大的“库幸咖啡，每天9.9”，最下面是 3 个热销咖啡的图片介绍。菜单页面有一个当前店铺的信息，可供选择自提和外卖，店铺名写清楚是库幸咖啡，下面是一个点餐区域，点餐模块里面一定要有库幸咖啡的招牌“杨梅陨石拿铁”。我的页面最上面是一个个人信息，中间有 5 个快捷功能入口，分别为“我的订单”“咖啡钱包”“优惠券”“招商加盟”“账户余额”，然后是一个邀请好友的图片链接，最下面是 2 个赠送的礼品卡的图片。会员卡页面就放一个会员的图文介绍就行，标题写个“尊贵的VIP黄金会员”。请直接在当前工作目录完成这个鸿蒙应用，最后编译并尝试安装运行；如果受环境限制没法运行，请明确说明原因。", "goal": "从 0 到 1 生成库幸咖啡鸿蒙工程，并完成编译与运行尝试", "stage": "build", "pass": true, "wall_clock_ms": 623958, "input_tokens": 1105498, "output_tokens": 12984, "reasoning_tokens": 966, "total_tokens": 1119448, "tool_calls_total": 57, "tool_calls_by_name": { "todowrite": 6, "read": 24, "skill": 1, "bash": 1, "codegenie-mcp_init_project_path": 2, "write": 7, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 4, "edit": 8, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 1, "skill_loads_by_name": { "deveco-create-project": 1 } } ], "dir": "C:\\Users\\xi\\Desktop\\bench_mark\\codegenie-cli-benchmark\\artifacts_opencode\\artifact_bootstrap-0to1_20260421145118394\\runs\\bootstrap-luckin-coffee", "session_id": "ses_2512e219dffej3o4a2EHs11AOB", "assertions": [ { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: LuckinCoffee/AppScope/app.json5" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: LuckinCoffee/entry/src/main/ets/pages/Index.ets" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: LuckinCoffee/entry/build/default/outputs/default/app/entry-default.hap" } ] }, { "id": "bootstrap-fresh-delivery", "kind": "project_bootstrap", "title": "Generate a fresh delivery HarmonyOS app and run it", "pass": true, "score": 100, "final_stage": "build", "rounds": 1, "wall_clock_ms": 905103, "input_tokens": 1165111, "output_tokens": 21415, "reasoning_tokens": 998, "total_tokens": 1187524, "tool_calls_total": 52, "tool_calls_by_name": { "todowrite": 7, "skill": 2, "bash": 1, "read": 13, "codegenie-mcp_init_project_path": 2, "glob": 3, "write": 13, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 4, "edit": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 2, "skill_loads_by_name": { "deveco-create-project": 1, "arkui-knowledge": 1 }, "steps": [ { "id": "create", "prompt": "请在当前工作目录从 0 到 1 生成一个名为 FreshDelivery 的鸿蒙应用，包含“首页”“菜市场”“购物车”“我的”四个页面，其中首页包含动态天气提示栏、“今日特价”轮播区并展示 5 款限时折扣生鲜、“智能推荐”模块和“时令蔬菜”“新鲜肉禽”“海鲜水产”3 个快捷入口，菜市场页采用网格布局并提供带语音输入的搜索栏、六大分类“有机蔬菜”“精品肉类”“活鲜水产”“蛋奶制品”“速冻食品”“厨房调料”以及带产地溯源图标和新鲜度指数的商品卡片，购物车页支持修改数量、删除商品、结算和“凑单推荐”入口，我的页展示用户信息、会员等级进度条、“我的订单”“我的农场”“我的菜谱”“客服中心”“设置”等入口，最后完成编译并尝试运行，如受环境限制请明确说明原因。", "goal": "从 0 到 1 生成生鲜配送鸿蒙工程，并完成编译与运行尝试", "stage": "build", "pass": true, "wall_clock_ms": 904255, "input_tokens": 1165111, "output_tokens": 21415, "reasoning_tokens": 998, "total_tokens": 1187524, "tool_calls_total": 52, "tool_calls_by_name": { "todowrite": 7, "skill": 2, "bash": 1, "read": 13, "codegenie-mcp_init_project_path": 2, "glob": 3, "write": 13, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 4, "edit": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 2, "skill_loads_by_name": { "deveco-create-project": 1, "arkui-knowledge": 1 } } ], "dir": "C:\\Users\\xi\\Desktop\\bench_mark\\codegenie-cli-benchmark\\artifacts_opencode\\artifact_bootstrap-0to1_20260421145118394\\runs\\bootstrap-fresh-delivery", "session_id": "ses_2512498f5ffeM4lWdetR5U8nPV", "assertions": [ { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: FreshDelivery/AppScope/app.json5" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: FreshDelivery/entry/src/main/ets/pages/Index.ets" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: FreshDelivery/entry/build/default/outputs/default/app/entry-default.hap" } ] }, { "id": "bootstrap-travel-platform", "kind": "project_bootstrap", "title": "Generate a travel info HarmonyOS app and run it", "pass": true, "score": 100, "final_stage": "build", "rounds": 1, "wall_clock_ms": 775043, "input_tokens": 923491, "output_tokens": 17631, "reasoning_tokens": 1011, "total_tokens": 942133, "tool_calls_total": 51, "tool_calls_by_name": { "todowrite": 4, "skill": 3, "bash": 1, "codegenie-mcp_init_project_path": 2, "read": 14, "glob": 3, "write": 12, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 5, "edit": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 3, "skill_loads_by_name": { "deveco-create-project": 1, "arkui-knowledge": 1, "arkts-error-fixes": 1 }, "steps": [ { "id": "create", "prompt": "请在当前工作目录从 0 到 1 生成一个名为 TravelCompass 的鸿蒙应用，包含“首页”“目的地”“行程助手”“个人”四个页面，其中首页展示 5 个热门目的地“巴黎”“东京”“巴厘岛”“纽约”“马尔代夫”的幻灯片、5 个功能入口“旅游攻略”“优惠票”“当地美食”“酒店推荐”“旅行社报名”以及 3 条旅游资讯“夏季旅行优惠大放送，机票低至5折！”“巴厘岛新开网红酒店，限时特惠中！”“东京樱花季攻略，带你玩转最美春天”，目的地页展示上述 5 个目的地的特色和详细信息，行程助手页支持创建、编辑、删除行程并提供天气提醒、景点开放时间提醒、交通提醒等功能入口，个人页展示用户信息和收藏的旅游路线，最后完成编译并尝试运行，如受环境限制请明确说明原因。", "goal": "从 0 到 1 生成旅游资讯鸿蒙工程，并完成编译与运行尝试", "stage": "build", "pass": true, "wall_clock_ms": 774214, "input_tokens": 923491, "output_tokens": 17631, "reasoning_tokens": 1011, "total_tokens": 942133, "tool_calls_total": 51, "tool_calls_by_name": { "todowrite": 4, "skill": 3, "bash": 1, "codegenie-mcp_init_project_path": 2, "read": 14, "glob": 3, "write": 12, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 5, "edit": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 3, "skill_loads_by_name": { "deveco-create-project": 1, "arkui-knowledge": 1, "arkts-error-fixes": 1 } } ], "dir": "C:\\Users\\xi\\Desktop\\bench_mark\\codegenie-cli-benchmark\\artifacts_opencode\\artifact_bootstrap-0to1_20260421145118394\\runs\\bootstrap-travel-platform", "session_id": "ses_25116c93cffeF43A4ZKMSmb10l", "assertions": [ { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: TravelCompass/AppScope/app.json5" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: TravelCompass/entry/src/main/ets/pages/Index.ets" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: TravelCompass/entry/build/default/outputs/default/app/entry-default.hap" } ] }, { "id": "bootstrap-tech-news", "kind": "project_bootstrap", "title": "Generate a tech news HarmonyOS app and run it", "pass": true, "score": 100, "final_stage": "build", "rounds": 1, "wall_clock_ms": 920908, "input_tokens": 666621, "output_tokens": 19217, "reasoning_tokens": 921, "total_tokens": 686759, "tool_calls_total": 46, "tool_calls_by_name": { "todowrite": 4, "skill": 3, "bash": 1, "codegenie-mcp_init_project_path": 2, "read": 12, "write": 12, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 5, "edit": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 3, "skill_loads_by_name": { "deveco-create-project": 1, "arkui-knowledge": 1, "arkts-error-fixes": 1 }, "steps": [ { "id": "create", "prompt": "请在当前工作目录从 0 到 1 生成一个名为 TechNews 的鸿蒙应用，包含“头条”“快讯”“产品”“我的”四个页面，其中头条页采用三栏布局并分别展示当日重大科技新闻、各大科技博客实时标题流和主要科技公司股价动态，快讯页按时间倒序展示简短精炼的突发科技资讯，产品页以网格布局展示新发布硬件或软件评测并支持按“智能手机”“AI软件”等分类筛选且每张卡片包含图片、名称和聚合评测分数，我的页支持关注特定公司或技术话题并展示发布会信息列表，最后完成编译并尝试运行，如受环境限制请明确说明原因。", "goal": "从 0 到 1 生成科技资讯鸿蒙工程，并完成编译与运行尝试", "stage": "build", "pass": true, "wall_clock_ms": 920080, "input_tokens": 666621, "output_tokens": 19217, "reasoning_tokens": 921, "total_tokens": 686759, "tool_calls_total": 46, "tool_calls_by_name": { "todowrite": 4, "skill": 3, "bash": 1, "codegenie-mcp_init_project_path": 2, "read": 12, "write": 12, "codegenie-mcp_check_ets_files": 2, "codegenie-mcp_build_project": 5, "edit": 3, "codegenie-mcp_start_app": 2 }, "skill_loads_total": 3, "skill_loads_by_name": { "deveco-create-project": 1, "arkui-knowledge": 1, "arkts-error-fixes": 1 } } ], "dir": "C:\\Users\\xi\\Desktop\\bench_mark\\codegenie-cli-benchmark\\artifacts_opencode\\artifact_bootstrap-0to1_20260421145118394\\runs\\bootstrap-tech-news", "session_id": "ses_2510af5ebffeBKwN3RFV5Og70H", "assertions": [ { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: TechNews/AppScope/app.json5" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: TechNews/entry/src/main/ets/pages/Index.ets" }, { "type": "file_exists", "pass": true, "stage": "build", "detail": "file exists: TechNews/entry/build/default/outputs/default/app/entry-default.hap" } ] } ] }