1# 已有 Playwright 專案的話
2npm install --save-dev @midscene/web
3
4# 環境變數
5export MIDSCENE_MODEL_BASE_URL="https://api.openai.com/v1"
6export MIDSCENE_MODEL_NAME="gpt-4o"
7export OPENAI_API_KEY="sk-..."

 1import { test, expect } from '@playwright/test';
 2import { PlaywrightAgent } from '@midscene/web/playwright';
 3
 4test('search midscene on GitHub', async ({ page }) => {
 5  await page.goto('https://github.com');
 6  const agent = new PlaywrightAgent(page);
 7
 8  await agent.aiAction('input "midscene" in search box, press Enter');
 9  await agent.aiAssert('the first result is web-infra-dev/midscene');
10});

1# Android（需要 adb 在 PATH）
2npm install -g @midscene/cli @midscene/android
3
4# iOS（macOS only，需要 Xcode + WebDriverAgent）
5brew install ios-webkit-debug-proxy
6npm install -g @midscene/cli @midscene/ios
7
8# Desktop computer-use
9npm install -g @midscene/cli @midscene/computer

 1# vLLM serve UI-TARS-1.5-7B
 2pip install vllm
 3python -m vllm.entrypoints.openai.api_server \
 4  --model ByteDance-Seed/UI-TARS-1.5-7B \
 5  --port 8080 \
 6  --gpu-memory-utilization 0.9
 7
 8# Midscene 端
 9export MIDSCENE_MODEL_BASE_URL="http://localhost:8080/v1"
10export MIDSCENE_MODEL_NAME="ByteDance-Seed/UI-TARS-1.5-7B"
11export OPENAI_API_KEY="sk-empty"   # vLLM 不檢查

1git clone https://github.com/web-infra-dev/midscene
2cd midscene
3pnpm install              # 必須 pnpm
4pnpm run build:packages   # build 所有 packages
5pnpm run dev              # 同時 watch 多個 package

 1midscene/
 2├── packages/                       # 27 個 npm 套件
 3│   ├── core/                      # AI plan + agent loop 核心
 4│   ├── shared/                    # 共用 types / utils
 5│   ├── cli/                       # CLI（YAML 自動化）
 6│   ├── mcp/                       # MCP server（atomic tools for upper agents）
 7│   ├── visualizer/                # 視覺化回放
 8│   ├── recorder/                  # 互動錄製
 9│   ├── evaluation/                # 模型評測
10│   │
11│   ├── web-integration/           # Playwright + Puppeteer + Bridge mode
12│   ├── web-bridge-mcp/            # Bridge mode 的 MCP server
13│   │
14│   ├── android/                   # adb + scrcpy（Android 控制）
15│   ├── android-mcp/               # Android MCP server
16│   ├── android-playground/        # Android local playground
17│   │
18│   ├── ios/                       # WebDriverAgent（iOS）
19│   ├── ios-mcp/
20│   ├── ios-playground/
21│   │
22│   ├── computer/                  # 桌面通用（xvfb / RDP）
23│   ├── computer-{linux,mac,win}/  # 平台特化
24│   ├── computer-mcp/
25│   ├── computer-playground/
26│   │
27│   ├── harmony/                   # Huawei Harmony OS（hdc）
28│   ├── harmony-mcp/
29│   ├── harmony-playground/
30│   │
31│   ├── webdriver/                 # WebDriver protocol
32│   ├── playground/                # 共用 playground 基底
33│   └── playground-app/
34│
35├── apps/                          # 8 個應用
36│   ├── chrome-extension/          # 零代碼 web 體驗
37│   ├── studio/                    # Electron 桌面 GUI
38│   ├── report/                    # HTML 回放報告
39│   ├── site/                      # midscenejs.com 主站
40│   ├── playground/                # 主 playground
41│   ├── android-playground/
42│   ├── computer-playground/
43│   └── recorder-form/             # 互動錄製 UI
44│
45├── package.json                   # pnpm workspace + nx
46├── pnpm-workspace.yaml
47├── nx.json                        # Nx monorepo 設定
48├── biome.json                     # 取代 ESLint + Prettier
49└── AGENTS.md                      # 內部設計原則（agent rules）

1await agent.aiAction('click the search button, type "midscene", press Enter');
2await agent.aiTap('the login button');
3await agent.aiInput('username', 'foo@bar.com');
4await agent.aiScroll('down', 500);
5await agent.aiHover('the help icon');

1const data = await agent.aiQuery({
2  title: 'string',         // 從畫面 / DOM 抽出 title
3  price: 'number',
4  in_stock: 'boolean',
5});
6
7const list = await agent.aiQuery(
8  'array of product objects with name, price, image_url'
9);

1await agent.aiAssert('the cart shows 3 items');     // 失敗會 throw
2await agent.aiWaitFor('the loading spinner is gone', { timeoutMs: 30000 });
3const loc = await agent.aiLocate('the save button');// 不點，只回 bbox

 1# midscene.yaml
 2target:
 3  url: https://github.com
 4
 5tasks:
 6  - name: Search midscene
 7    flow:
 8      - ai: input "midscene" in search box, press Enter
 9      - aiAssert: the first result is web-infra-dev/midscene
10      - aiQuery:
11          structure:
12            star_count: number

1midscene midscene.yaml

 1import { test, expect } from '@playwright/test';
 2import { PlaywrightAgent } from '@midscene/web/playwright';
 3
 4test('login flow', async ({ page }) => {
 5  await page.goto('https://example.com/login');
 6  const agent = new PlaywrightAgent(page);
 7
 8  await agent.aiAction(`
 9    1. type "user@example.com" in email field
10    2. type "secret" in password field
11    3. click the Login button
12  `);
13
14  await agent.aiAssert('the page shows "Welcome, user"');
15});

 1import puppeteer from 'puppeteer';
 2import { PuppeteerAgent } from '@midscene/web/puppeteer';
 3
 4const browser = await puppeteer.launch();
 5const page = await browser.newPage();
 6const agent = new PuppeteerAgent(page);
 7
 8await page.goto('https://amazon.com');
 9await agent.aiAction('search for "midscene book"');
10const result = await agent.aiQuery({
11  first_title: 'string',
12  first_price: 'number',
13});
14console.log(result);

1import { BridgeAgent } from '@midscene/web/bridge';
2
3const agent = new BridgeAgent();
4await agent.connect();  // 連到 Chrome Extension 上的 bridge
5await agent.aiAction('open Gmail, search for invoices from this month');

1import { AndroidAgent } from '@midscene/android';
2
3const agent = new AndroidAgent({ deviceId: 'auto' }); // 第一個 adb device
4await agent.aiAction('open the Calculator app, calculate 7 × 8');
5const result = await agent.aiQuery({ display: 'string' });
6console.log(result.display); // "56"

1# 先啟動 WDA
2xcodebuild -project WebDriverAgent.xcodeproj \
3  -scheme WebDriverAgentRunner -destination "id=$UDID" test

1import { IOSAgent } from '@midscene/ios';
2
3const agent = new IOSAgent({ wdaUrl: 'http://localhost:8100' });
4await agent.aiAction('open Safari, navigate to midscenejs.com');

 1# YAML 跑單一檔
 2midscene run midscene.yaml
 3
 4# 整 directory
 5midscene run ./tests/
 6
 7# 啟 web UI playground
 8midscene playground
 9
10# Android playground
11midscene-android-playground
12
13# iOS playground
14midscene-ios-playground

1# 啟動 MCP server
2midscene-mcp --port 3000

1{
2  "mcpServers": {
3    "midscene": {
4      "command": "midscene-mcp",
5      "args": ["--target", "web"]
6    }
7  }
8}

1midscene run midscene.yaml
2# → midscene-report/<timestamp>/index.html
3
4open midscene-report/<timestamp>/index.html

1const agent = new PlaywrightAgent(page, {
2  cache: { enable: true, key: 'github-search-test' },
3});

1const agent = new PlaywrightAgent(page, {
2  systemPrompt: `You are testing an internal CRM. The user names are pseudonymized.`,
3});

1const plan = await agent.aiPlan('複雜任務：在 Gmail 找這個月的發票並下載');
2// plan = { steps: [...] }，可預覽再 execute
3for (const step of plan.steps) {
4  await agent.executeStep(step);
5  console.log(`✅ ${step.description}`);
6}

1python -m vllm.entrypoints.openai.api_server \
2  --model ByteDance-Seed/UI-TARS-1.5-7B \
3  --tensor-parallel-size 2 \
4  --max-num-seqs 16 \
5  --gpu-memory-utilization 0.9

1# 每次 PR run
2pnpm test:ai
3# → 自動產 midscene-report/<timestamp>/
4# → upload 到 S3 / Vercel preview
5# → PR 留言貼連結

1midscene-recorder
2# 開瀏覽器 → 手動操作 → 結束錄製
3# → 自動生成 midscene.yaml（每步 ai 動作 + 自動推斷 assertion）

1cd packages/evaluation
2pnpm run eval -- --model qwen-vl-max --tasks all
3# → 跑內建的 benchmark task suite，輸出 success rate / cost / latency

1// midscene.config.json
2{
3  "mcp": {
4    "tools": ["aiTap", "aiInput", "aiQuery"],
5    "exclude": ["aiAction"]  // 完整 aiAction 太強，限定 atomic tools
6  }
7}

1import { test } from '@playwright/test';
2import { PlaywrightAgent } from '@midscene/web/playwright';
3// 其他 fixture / hooks 與原 Playwright 完全相容

1// .mcp.json
2{
3  "mcpServers": {
4    "midscene-web": { "command": "midscene-mcp", "args": ["--target", "web"] },
5    "midscene-android": { "command": "midscene-android-mcp" }
6  }
7}

 1- name: Setup
 2  uses: pnpm/action-setup@v3
 3  with:
 4    version: 9
 5- run: pnpm install
 6- name: Midscene tests
 7  env:
 8    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 9    MIDSCENE_MODEL_NAME: gpt-4o
10    MIDSCENE_MODEL_BASE_URL: https://api.openai.com/v1
11  run: pnpm run e2e
12- uses: actions/upload-artifact@v4
13  with:
14    name: midscene-report
15    path: midscene-report/