1Synthetic-Data-Generator/
 2├── SDG.py                  # 主程式：669 行 Python，包含所有生成邏輯
 3├── table_structure.sql     # MySQL Schema 定義 + 預載資料（化療方案、CUI 描述）
 4├── mapping.ttl             # RML (RDF Mapping Language) 對應規則
 5├── requirements.txt        # Python 依賴：rdfizer, mysql-connector, numpy, pandas
 6├── Dockerfile              # 基於 mysql:8.1.0 的容器定義
 7├── docker-compose.yml      # Docker Compose 編排設定
 8├── generate.sh             # 一鍵生成 Shell Script
 9├── CITATION.cff            # 引用資訊
10├── er-diagram-generated-data.jpg  # ER Diagram 圖片
11└── data/                   # 生成的輸出目錄（執行後產生）
12    ├── csv/                # 每張表一個 CSV 檔案
13    │   ├── patient.csv
14    │   ├── tumor_tnm.csv
15    │   └── ...
16    ├── synth_data.nt       # RDF N-Triples 格式
17    └── synth_data.sql.gz   # MySQL dump 壓縮檔

 1# 1. 複製專案
 2git clone https://github.com/SDM-TIB/Synthetic-Data-Generator.git
 3cd Synthetic-Data-Generator
 4
 5# 2. 確認 Docker 可用
 6docker --version
 7# Docker version 24.x.x 或更高
 8
 9# 3. (選用) 確認 docker-compose 可用
10docker compose version
11# Docker Compose version v2.x.x

1# 方法一：透過 docker-compose 建置
2docker-compose up -d --build
3
4# 方法二：手動建置
5docker build . -t sdmtib/sdg:latest

 1# Step 1: 啟動容器 (含建置)
 2docker-compose up -d --build
 3
 4# Step 2: 生成 100 位患者的乾淨資料 (mutation_prob = 0.0)
 5docker exec -it SDG bash -c "SDG -n 100 -p 0.0"
 6
 7# 預期輸出:
 8# Setting up the database: 0.234...
 9# Generating data: 2.156...
10# Dumping database: 0.512...
11# Dumping CSV: 0.078...
12# Finished generating the synthetic data. Total time: 3.021...
13
14# Step 3: 檢視生成的檔案
15ls -la ./data/
16# csv/
17# synth_data.nt
18# synth_data.sql.gz
19
20ls -la ./data/csv/
21# chemoterapy_cycle.csv
22# chemoterapy_schema.csv
23# comorbidity.csv
24# cui_description.csv
25# drug.csv
26# drug_chemoterapy_schema.csv
27# family_history.csv
28# oral_drug.csv
29# oral_drug_type.csv
30# patient.csv
31# radiotherapy.csv
32# surgery.csv
33# tumor_grade.csv
34# tumor_tnm.csv
35# tumor_type.csv
36
37# Step 4: 備份資料 (重要! 下次生成會覆蓋)
38cp -r ./data ./data_clean_100
39
40# Step 5: 生成 500 位患者的有雜訊資料
41docker exec -it SDG bash -c "SDG -n 500 -p 0.1"
42cp -r ./data ./data_noisy_500
43
44# Step 6: 完成後停止容器
45docker-compose down -v

1ehr,birth_date,diagnosis_date,age_at_diagnosis,first_treatment_date,surgery_date,death_date,age_at_death,er_positive,pr_positive,her2_overall_positive,ki67_percent_max_simp,neoadjuvant,menarche_age,menopause_pre,menopause_age,pregnancy,abort,birth,caesarean
21,1962-03-15,2019-07-22,57,2019-08-14,2019-09-05,,,,1,0,0,14,no,13,1,48,2,0,2,0
32,1958-11-02,2018-04-10,59,2018-05-03,2018-06-15,2023-09-28,64,1,1,0,18,no,12,1,50,3,1,2,0

 1# 生成 1000 位患者，突變機率 5%
 2./generate.sh 1000 0.05
 3
 4# generate.sh 內部會自動：
 5# 1. docker build . -t sdmtib/sdg:latest
 6# 2. docker run --name SDG -v ./data:/data -d sdmtib/sdg:latest
 7# 3. sleep 5s (等待 MySQL 啟動)
 8# 4. docker exec -it SDG bash -c "SDG -n 1000 -p 0.05"
 9# 5. docker rm -fv SDG (自動清理容器)
10
11# 檢視 RDF 輸出
12head -20 ./data/synth_data.nt
13# <http://research.tib.eu/paladin/entity/BC_HUPHM_1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://research.tib.eu/paladin/vocab/BCPatient> .
14# <http://research.tib.eu/paladin/entity/BC_HUPHM_1> <http://research.tib.eu/paladin/vocab/hasEHR> "1"^^<http://www.w3.org/2001/XMLSchema#integer> .
15# ...
16
17# 檢視 SQL dump
18zcat ./data/synth_data.sql.gz | head -30

 1#!/bin/bash
 2# batch_generate.sh -- 批次生成不同突變機率的資料集供比較
 3
 4# 啟動容器
 5docker-compose up -d --build
 6sleep 10  # 等待 MySQL 完全啟動
 7
 8PATIENTS=200
 9
10for PROB in 0.0 0.05 0.1 0.2 0.3; do
11    echo "=== 生成 mutation_prob=${PROB} ==="
12    docker exec -it SDG bash -c "SDG -n ${PATIENTS} -p ${PROB}"
13
14    # 備份到獨立資料夾
15    OUTDIR="./data_p${PROB}"
16    rm -rf "${OUTDIR}"
17    cp -r ./data "${OUTDIR}"
18
19    # 快速統計
20    echo "Patient CSV rows: $(wc -l < ${OUTDIR}/csv/patient.csv)"
21    echo "RDF triples: $(wc -l < ${OUTDIR}/synth_data.nt)"
22    echo "SQL dump size: $(du -h ${OUTDIR}/synth_data.sql.gz | cut -f1)"
23    echo ""
24done
25
26# 清理
27docker-compose down -v
28
29# 用 Python 進行簡單比較分析
30python3 << 'PYEOF'
31import pandas as pd
32import os
33
34probs = [0.0, 0.05, 0.1, 0.2, 0.3]
35stats = []
36
37for p in probs:
38    dir_path = f"./data_p{p}/csv"
39    if not os.path.exists(dir_path):
40        continue
41    pt = pd.read_csv(f"{dir_path}/patient.csv")
42    stats.append({
43        "mutation_prob": p,
44        "n_patients": len(pt),
45        "mean_age_dx": pt["age_at_diagnosis"].mean(),
46        "std_age_dx": pt["age_at_diagnosis"].std(),
47        "er_positive_rate": pt["er_positive"].mean(),
48        "death_rate": pt["death_date"].notna().mean() if "death_date" in pt.columns else None,
49        "mean_ki67": pt["ki67_percent_max_simp"].mean(),
50    })
51
52df = pd.DataFrame(stats)
53print("\n=== 各突變機率下的資料統計比較 ===")
54print(df.to_string(index=False))
55PYEOF

1=== 各突變機率下的資料統計比較 ===
2 mutation_prob  n_patients  mean_age_dx  std_age_dx  er_positive_rate  death_rate  mean_ki67
3           0.0         200        56.82       10.12              0.86        0.12      19.45
4          0.05         200        56.95       10.34              0.84        0.13      20.12
5           0.1         200        57.23       11.56              0.80        0.15      22.78
6           0.2         200        57.89       14.23              0.72        0.18      26.34
7           0.3         200        58.12       17.45              0.65        0.21      30.56

 1# 概念範例：SDG 輸出 → synthcity 隱私增強
 2import pandas as pd
 3from synthcity.plugins import Plugins
 4from synthcity.plugins.core.dataloader import GenericDataLoader
 5
 6# Step 1: 讀取 SDG 生成的 CSV
 7sdg_patients = pd.read_csv("./data/csv/patient.csv")
 8
 9# Step 2: 用 synthcity 的 DPGAN 在 SDG 資料上訓練
10loader = GenericDataLoader(sdg_patients, target_column="er_positive")
11syn_model = Plugins().get("dpgan", epsilon=1.0)  # epsilon 控制隱私預算
12syn_model.fit(loader)
13
14# Step 3: 生成帶 DP 保證的資料
15dp_data = syn_model.generate(count=1000)
16print(dp_data.dataframe().describe())

 1# 使用 Apache Jena Fuseki 載入 SDG 的 RDF 輸出
 2docker run -d --name fuseki \
 3  -p 3030:3030 \
 4  -v $(pwd)/data:/data \
 5  stain/jena-fuseki \
 6  /jena-fuseki/fuseki-server --mem /sdg
 7
 8# 上傳 RDF 資料
 9curl -X POST http://localhost:3030/sdg/data \
10  -H "Content-Type: application/n-triples" \
11  --data-binary @./data/synth_data.nt
12
13# SPARQL 查詢範例：查找所有 Stage IIIA 以上且 HER2+ 的患者
14curl -X POST http://localhost:3030/sdg/query \
15  --data-urlencode "query=
16    PREFIX paladin: <http://research.tib.eu/paladin/vocab/>
17    SELECT ?patient ?stage ?her2
18    WHERE {
19      ?patient a paladin:BCPatient ;
20               paladin:hasHER2OverallPositive ?her2 ;
21               paladin:hasStageDiagnosis ?stage .
22      FILTER(?her2 = 'Yes')
23      FILTER(?stage IN ('IIIA', 'IIIB', 'IIIC', 'IV'))
24    }
25    LIMIT 20
26  "

 1# 概念範例：SDG CSV → SDTM DM (Demographics) Domain
 2import pandas as pd
 3
 4patient = pd.read_csv("./data/csv/patient.csv")
 5
 6# 轉換為 SDTM DM domain
 7dm = pd.DataFrame({
 8    "STUDYID": "SDG-SYNTH-001",
 9    "DOMAIN": "DM",
10    "USUBJID": patient["ehr"].apply(lambda x: f"SDG-SYNTH-001-{x:04d}"),
11    "SUBJID": patient["ehr"].apply(lambda x: f"{x:04d}"),
12    "RFSTDTC": patient["first_treatment_date"],
13    "BRTHDTC": patient["birth_date"],
14    "AGE": patient["age_at_diagnosis"],
15    "AGEU": "YEARS",
16    "SEX": "F",  # 乳癌患者以女性為主
17    "RACE": "",  # SDG 目前不生成種族資料
18    "DTHFL": patient["death_date"].apply(lambda x: "Y" if pd.notna(x) else ""),
19    "DTHDTC": patient["death_date"],
20})
21
22dm.to_csv("sdtm_dm.csv", index=False)
23print(f"SDTM DM domain: {len(dm)} subjects")

 1# 概念範例：SDG 化療時間軸 → TemporAI 時間序列模型
 2import pandas as pd
 3
 4cycles = pd.read_csv("./data/csv/chemoterapy_cycle.csv")
 5patient = pd.read_csv("./data/csv/patient.csv")
 6
 7# 整合為時間序列格式
 8# 每位患者的化療週期序列可作為 TemporAI 的輸入
 9# 用於預測治療反應、生存分析等
10timeline = cycles.merge(patient[["ehr", "er_positive", "her2_overall_positive", "stage_diagnosis"]],
11                        on="ehr", how="left")
12timeline = timeline.sort_values(["ehr", "date"])
13print(f"時間序列記錄數: {len(timeline)}")
14print(f"涵蓋患者數: {timeline['ehr'].nunique()}")