1FORTE/
 2├── MIIT/                          # 訓練框架（基於 Otter）
 3│   ├── flamingo/                  # OpenFlamingo 模型定義
 4│   │   ├── modeling_flamingo.py   #   Flamingo 條件生成模型
 5│   │   └── mpt/                   #   MPT backbone
 6│   ├── otter/                     # Otter 模型（BrainGPT 基底）
 7│   │   └── modeling_otter.py      #   OtterForConditionalGeneration
 8│   └── pipeline/
 9│       ├── train/
10│       │   ├── instruction_following.py  # 主訓練腳本
11│       │   ├── data.py                   # 資料載入器
12│       │   └── train_utils.py            # 訓練工具
13│       ├── eval/                  # COCO 風格評估
14│       ├── serve/                 # Gradio demo 伺服器
15│       └── mimicit_utils/         # MIIT 資料集工具
16├── evaluation/                    # 推論框架（結構同 MIIT，用於 eval mode）
17│   └── pipeline/train/eval.py     # 推論主腳本
18├── data/
19│   ├── CQ500p_instruction.json    # CQ500 外部驗證指令
20│   ├── FORTE_brain.json           # 腦 CT 關鍵字詞典（4 類 × N 同義詞組）
21│   ├── FORTE_chestCT.json         # 胸 CT 關鍵字詞典
22│   ├── FORTE_abdomen.json         # 腹 CT 關鍵字詞典
23│   └── FORTE_CXR.json             # 胸 X 光關鍵字詞典
24├── Automatic_evaluation.py        # Step 1: NLG 自動評估
25├── Sentence_pairing.py            # Step 2: 語義句子配對
26├── FORTE.py                       # Step 3: FORTE 關鍵字評估
27├── Negation_removal.py            # Step 4: 否定移除
28├── train.sh                       # 訓練腳本入口
29├── eval.sh                        # 推論腳本入口
30├── environment.yml                # Conda 環境
31└── requirements.txt               # pip 依賴

 1# Step 1: Clone 專案
 2git clone https://github.com/charlierabea/FORTE.git
 3cd FORTE
 4
 5# Step 2: 建立 Conda 環境
 6conda env create -f environment.yml
 7conda activate forte
 8
 9# Step 3: 安裝 PyTorch（須匹配 CUDA 版本）
10# 範例：CUDA 11.8
11pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
12
13# Step 4: 安裝其餘依賴
14pip install -r requirements.txt
15
16# 注意：Horovod 安裝可能因 cmake 版本過舊而失敗
17# 推論不需要 Horovod，可略過此錯誤

 1# 建立目錄
 2mkdir -p checkpoints data
 3
 4# 1. 下載 Base Model（Otter-MPT-7B-Init）
 5#    來源：https://huggingface.co/luodian/OTTER-MPT7B-Init
 6#    放置於 ./checkpoints/OTTER-MPT7B-Init/
 7
 8# 2. 下載 BrainGPT Fine-tuned Model
 9#    來源：https://huggingface.co/Charliebear/BrainGPT
10#    放置於 ./checkpoints/OTTER_CLIP_BRAINGPT_hf/
11
12# 3. 下載 CQ500 外部驗證影像
13#    來源：Google Drive (見 README)
14#    放置於 ./data/CQ500p.json (影像索引)
15
16# 4. 下載 FORTE Keyword File
17#    來源：Google Drive (見 README)
18#    已內建：data/FORTE_brain.json 等 4 個 JSON
19
20# 目錄結構驗證
21ls checkpoints/
22# OTTER-MPT7B-Init/  OTTER_CLIP_BRAINGPT_hf/
23
24ls data/
25# CQ500p_instruction.json  CQ500p.json  FORTE_brain.json  ...

1# 執行推論腳本
2# 需要 GPU，使用 FSDP（Fully Sharded Data Parallel）加速
3bash ./eval.sh

 1# 設定 Python 路徑以正確引入 evaluation/ 模組
 2export PYHTONPATH="./evaluation/"
 3
 4# 使用 Accelerate + FSDP 啟動推論
 5accelerate launch \
 6    --config_file=./evaluation/pipeline/accelerate_configs/accelerate_config_fsdp.yaml \
 7    ./evaluation/pipeline/train/eval.py \
 8    --pretrained_model_name_or_path="./checkpoints/OTTER_CLIP_BRAINGPT_hf/" \
 9    --mimicit_path="./data/CQ500p_instruction.json" \
10    --images_path="./data/CQ500p.json" \
11    --batch_size=1 \
12    --warmup_steps_ratio=0.01 \
13    --workers=1

 1{
 2    "meta": {"version": "0.0.2", "time": "2024-02", "author": "big_data_center"},
 3    "data": {
 4        "CQ500p_INS_00001": {
 5            "instruction": "You are an AI assistant specialized in radiology topics. You are provided with brain CT slices from a single study. The number of slices is 24. Please generate medical descriptions based on the images in a consistent style. Use the following guidelines: - Degree: Indicate the intensity or state (e.g., normal, mild, chronic, old, etc). - Landmark: Specify the area of interest (e.g., intracerebral, midline, parenchyma, sulci, etc). - Feature: Describe any observed abnormalities (e.g., hemorrhage, atrophy, infarcts, etc). - Impression: Conclude with a clinical impression...",
 6            "answer": "",
 7            "image_ids": [
 8                "CQ500p_IMG_460_3_9",
 9                "CQ500p_IMG_460_3_10",
10                "...共 24 張 slice..."
11            ],
12            "rel_ins_ids": []
13        }
14    }
15}

 1# ===== Step 1: Automatic Evaluation (NLG 自動指標) =====
 2# 輸入：excel_files/evaluation_examples.xlsx (含 gt 與 parsed_output 欄位)
 3# 輸出：excel_files/automated_evaluation.xlsx (附加 BLEU/METEOR/ROUGE/CIDEr 分數)
 4
 5# 注意：須先移除 pycocoevalcap/eval.py 中的 Spice scorer 以避免錯誤
 6# 將以下行註解掉：
 7#     (Spice(), "SPICE")
 8
 9python3 Automatic_evaluation.py
10
11
12# ===== Step 2: Sentence Pairing (語義句子配對) =====
13# 使用 all-mpnet-base-v2 計算 GT 與 Output 各句子的 cosine similarity
14# 將最相似的句子配對，方便後續逐句比較
15# 輸入：excel_files/evaluation_examples.xlsx
16# 輸出：excel_files/sentencepaired_reports.xlsx
17
18python3 Sentence_pairing.py
19
20
21# ===== Step 3: FORTE Keyword Evaluation (領域關鍵字評估) =====
22# 從配對句子中擷取 4 類 FORTE 關鍵字，計算 Precision/Recall
23# 輸入：excel_files/sentencepaired_reports.xlsx + data/FORTE_brain.json
24# 輸出：excel_files/FORTE_evaluated.xlsx
25
26python3 FORTE.py
27
28
29# ===== Step 4: Negation Removal (否定移除) =====
30# 過濾掉 GT 或 Output 中 degree 欄含 "no" 的列
31# 避免 "no hemorrhage" 被誤計為 "hemorrhage" 的假陽性
32# 輸入：excel_files/FORTE_evaluated.xlsx
33# 輸出：excel_files/FORTE_negationremoval.xlsx
34
35python3 Negation_removal.py

 1# 準備訓練資料：
 2# 1. MED_instruction.json — 指令格式同 CQ500p_instruction.json
 3# 2. MED.json — 影像路徑對應檔
 4
 5# 修改 train.sh 中的路徑
 6export PYTHONPATH=.
 7
 8CUDA_VISIBLE_DEVICES=0,1 accelerate launch \
 9    --config_file=./MIIT/pipeline/accelerate_configs/accelerate_config_fsdp.yaml \
10    ./MIIT/pipeline/train/instruction_following.py \
11    --pretrained_model_name_or_path="./checkpoints/OTTER-MPT7B-Init" \
12    --mimicit_path="./data/MED_instruction.json" \
13    --images_path="./data/MED.json" \
14    --batch_size=1 \
15    --gradient_accumulation_steps=4 \
16    --num_epochs=3 \
17    --report_to_wandb \
18    --wandb_entity="your_institution" \
19    --run_name=BrainGPT_custom \
20    --wandb_project=BrainGPT_custom \
21    --workers=1 \
22    --lr_scheduler=cosine \
23    --learning_rate=1e-5 \
24    --warmup_steps_ratio=0.01

1# 每筆資料須包含：
2{
3    "instruction": "...",       # 提示模型生成報告的指令
4    "answer": "GT 報告文本",    # Ground truth（訓練時使用）
5    "image_ids": ["img_001", "img_002", ...],  # 對應影像 ID 清單
6    "rel_ins_ids": []           # 相關指令 ID（in-context learning 用）
7}

1真實 CT 影像 ─→ BrainGPT ─→ 合成放射科報告 ─→ 下游應用
2                                    │
3                    ┌───────────────┼───────────────┐
4                    ▼               ▼               ▼
5            NLP 模型訓練      CDSS 標註輔助    品質控管 benchmark
6            (RadBERT 等)    (半自動報告)     (FORTE metrics)

 1# 概念範例：同時使用 FORTE keyword metrics + RadFact factual score
 2
 3# Step 1: FORTE keyword evaluation
 4import json
 5import pandas as pd
 6
 7with open('data/FORTE_brain.json', 'r') as f:
 8    forte_keywords = json.load(f)
 9
10# 擷取 GT 與 Output 的關鍵字
11def extract_forte_keywords(text, keyword_dict):
12    """從報告文本中擷取 FORTE 四類關鍵字"""
13    text = f" {text.lower()} "
14    found = {}
15    for category in ['degree', 'landmark', 'feature', 'impression']:
16        found[category] = [k.strip() for k in keyword_dict[category] if k in text]
17    return found
18
19gt_keywords = extract_forte_keywords(gt_report, forte_keywords)
20pred_keywords = extract_forte_keywords(pred_report, forte_keywords)
21
22# Step 2: 計算 FORTE Precision / Recall per category
23# (詳見 FORTE.py 的 calculate_precision_recall 函式)
24
25# Step 3: 同時收集 RadFact factual consistency score
26# radfact_score = radfact_evaluate(gt_report, pred_report)
27
28# Step 4: 綜合報告
29# combined_score = weighted_average(forte_f1, radfact_score)

 1# 任何模型的輸出，只要存為 Excel (id / gt / parsed_output) 即可
 2# 套用 FORTE 四步驟評估
 3
 4# 1. 存為 Excel
 5import pandas as pd
 6results = pd.DataFrame({
 7    'id': case_ids,
 8    'gt': ground_truth_reports,
 9    'parsed_output': any_model_generated_reports
10})
11results.to_excel('excel_files/evaluation_examples.xlsx', index=False)
12
13# 2. 依序執行四步驟
14# python3 Automatic_evaluation.py
15# python3 Sentence_pairing.py
16# python3 FORTE.py
17# python3 Negation_removal.py

1distilabel Pipeline
2    └── Step 1: 收集 CT 影像 metadata
3    └── Step 2: 構建 MIIT instruction JSON
4    └── Step 3: 呼叫 BrainGPT 生成報告
5    └── Step 4: FORTE 評估品質過濾
6    └── Step 5: 匯出高品質合成報告語料