1# 基本安裝（僅核心功能）
2pip install distilabel --upgrade
3
4# 建議使用 uv 建立隔離環境
5uv venv .venv-distilabel --python 3.11
6source .venv-distilabel/bin/activate
7uv pip install distilabel

 1# 使用 OpenAI（最常見）
 2pip install "distilabel[openai]"
 3
 4# 使用 Anthropic Claude
 5pip install "distilabel[anthropic]"
 6
 7# 使用本地 vLLM 推論
 8pip install "distilabel[vllm]"
 9
10# 使用 HuggingFace Transformers 本地推論
11pip install "distilabel[hf-transformers]"
12
13# 使用 Ollama 本地推論
14pip install "distilabel[ollama]"
15
16# 結構化輸出
17pip install "distilabel[outlines]"
18pip install "distilabel[instructor]"
19
20# 使用 Ray 分散式擴展
21pip install "distilabel[ray]"
22
23# 使用 Argilla 標注平台
24pip install "distilabel[argilla]"
25
26# 組合安裝（醫療 SDG 推薦組合）
27pip install "distilabel[openai,anthropic,hf-inference-endpoints,instructor,argilla]"

 1# OpenAI
 2export OPENAI_API_KEY="sk-..."
 3
 4# Anthropic
 5export ANTHROPIC_API_KEY="sk-ant-..."
 6
 7# HuggingFace
 8export HF_TOKEN="hf_..."
 9
10# Argilla（如需標注平台）
11export ARGILLA_API_URL="https://your-argilla-instance.com"
12export ARGILLA_API_KEY="your-argilla-key"

1# 確認安裝版本
2python -c "import distilabel; print(distilabel.__version__)"
3
4# 確認 CLI 可用
5python -m distilabel --help

 1"""
 2醫療 Q&A 合成資料生成 Pipeline
 3使用 TextGeneration + UltraFeedback 產出帶品質分數的訓練資料
 4"""
 5from datasets import Dataset
 6
 7from distilabel.models import OpenAILLM
 8from distilabel.pipeline import Pipeline
 9from distilabel.steps import KeepColumns, StepInput, step
10from distilabel.steps.tasks import TextGeneration, UltraFeedback
11
12# === 1. 定義 Seed Data (種子資料) ===
13seed_questions = [
14    {"instruction": "Explain the mechanism of action of PD-1 checkpoint inhibitors in treating renal cell carcinoma."},
15    {"instruction": "What are the key differences between ADC (antibody-drug conjugate) and traditional chemotherapy in terms of targeted delivery?"},
16    {"instruction": "Describe the standard pre-IND pharmacokinetics studies required by FDA for a biologic drug candidate."},
17    {"instruction": "How does CRISPR-Cas9 gene editing differ from traditional gene therapy approaches in treating genetic diseases?"},
18    {"instruction": "Explain the role of tumor microenvironment in immune evasion and its implications for immunotherapy."},
19]
20
21seed_dataset = Dataset.from_list(seed_questions)
22
23# === 2. 定義 LLM 模型 ===
24# 生成模型：負責生成回答
25generator_llm = OpenAILLM(
26    model="gpt-4o",
27    generation_kwargs={
28        "temperature": 0.7,
29        "max_new_tokens": 1024,
30    },
31)
32
33# 評判模型：負責品質評分
34judge_llm = OpenAILLM(
35    model="gpt-4o",
36    generation_kwargs={
37        "temperature": 0.0,  # 評判用低溫度確保一致性
38        "max_new_tokens": 512,
39    },
40)
41
42# === 3. 建構 Pipeline ===
43with Pipeline(name="medical-qa-sdg") as pipeline:
44
45    # Step 1: 文本生成 — 生成醫療問題的回答
46    generate = TextGeneration(
47        name="medical_answer_generation",
48        llm=generator_llm,
49        num_generations=3,  # 每個問題生成 3 個不同回答
50        system_prompt=(
51            "You are a senior biomedical researcher with expertise in "
52            "pharmacology, immunology, and regulatory science. "
53            "Provide detailed, accurate, and well-structured answers "
54            "suitable for training medical AI systems. "
55            "Include relevant citations format when applicable."
56        ),
57    )
58
59    # Step 2: UltraFeedback — 多維度 AI 品質評分
60    evaluate = UltraFeedback(
61        name="quality_evaluation",
62        llm=judge_llm,
63        aspect="overall-rating",  # 可選: helpfulness, honesty, truthfulness, instruction-following
64    )
65
66    # Step 3: 保留需要的欄位
67    keep = KeepColumns(
68        name="select_columns",
69        columns=[
70            "instruction",
71            "generation",
72            "ratings",
73            "rationales",
74        ],
75    )
76
77    # 連接步驟（>> 運算子定義依賴關係）
78    generate >> evaluate >> keep
79
80# === 4. 執行 Pipeline ===
81if __name__ == "__main__":
82    distiset = pipeline.run(dataset=seed_dataset)
83
84    # 查看結果
85    ds = distiset["default"]["train"]
86    print(f"生成 {len(ds)} 筆帶評分的醫療 QA 資料")
87    print(ds[0])
88
89    # 推送至 HuggingFace Hub
90    # distiset.push_to_hub("your-org/medical-qa-synthetic")

 1"""
 2醫療指令漸進進化 Pipeline
 3使用 EvolInstruct 將簡單指令自動升級為複雜多步驟指令
 4"""
 5from datasets import Dataset
 6
 7from distilabel.models import InferenceEndpointsLLM
 8from distilabel.pipeline import Pipeline
 9from distilabel.steps import KeepColumns
10from distilabel.steps.tasks import EvolInstruct, EvolComplexity
11
12# === 1. 簡單的醫療 Seed Instructions (種子指令) ===
13simple_instructions = Dataset.from_list([
14    {"instruction": "What is an ADC drug?"},
15    {"instruction": "How does immunotherapy work?"},
16    {"instruction": "What is a pre-IND meeting?"},
17    {"instruction": "Explain pharmacokinetics."},
18    {"instruction": "What is a Phase 1 clinical trial?"},
19    {"instruction": "Describe CRISPR technology."},
20    {"instruction": "What are biomarkers?"},
21    {"instruction": "How do monoclonal antibodies work?"},
22])
23
24# === 2. 使用 HuggingFace Inference Endpoints ===
25llm = InferenceEndpointsLLM(
26    model_id="meta-llama/Meta-Llama-3.1-70B-Instruct",
27    generation_kwargs={
28        "temperature": 0.8,
29        "max_new_tokens": 512,
30    },
31)
32
33# === 3. 建構 EvolInstruct Pipeline ===
34with Pipeline(name="medical-evol-instruct") as pipeline:
35
36    # EvolInstruct: 每個指令進化 3 輪
37    # 進化策略包含：增加約束條件、增加推理步驟、深化複雜度、
38    # 加入具體化描述、混合多個子問題
39    evol = EvolInstruct(
40        name="evolve_medical_instructions",
41        llm=llm,
42        num_evolutions=3,          # 進化輪數
43        store_evolutions=True,     # 保留每一輪的進化結果
44        generate_answers=True,     # 同時生成進化後指令的回答
45    )
46
47    # 評估進化後指令的複雜度
48    complexity = EvolComplexity(
49        name="score_complexity",
50        llm=llm,
51    )
52
53    keep = KeepColumns(
54        name="select_output",
55        columns=[
56            "instruction",       # 原始指令
57            "evolved_instruction",  # 最終進化指令
58            "answer",            # 進化指令的回答
59            "model_name",
60        ],
61    )
62
63    evol >> complexity >> keep
64
65# === 4. 執行並檢視進化結果 ===
66if __name__ == "__main__":
67    distiset = pipeline.run(dataset=simple_instructions)
68    ds = distiset["default"]["train"]
69
70    # 展示進化前後對比
71    for row in ds.select(range(3)):
72        print("=" * 60)
73        print(f"[原始] {row['instruction']}")
74        print(f"[進化] {row['evolved_instruction']}")
75        print(f"[回答] {row['answer'][:200]}...")
76        print()
77
78    # 典型進化結果範例：
79    # [原始] What is an ADC drug?
80    # [進化] Compare the payload delivery mechanisms of three FDA-approved
81    #        ADC drugs (T-DXd, enfortumab vedotin, sacituzumab govitecan),
82    #        analyzing how linker chemistry and DAR (drug-to-antibody ratio)
83    #        affect their therapeutic indices across different tumor types.
84    #        Include a discussion of the bystander effect and its clinical
85    #        implications for heterogeneous tumors.

 1"""
 2結構化醫療 NER 標注資料生成
 3使用 Instructor 確保 LLM 輸出符合 Pydantic schema
 4"""
 5from typing import List
 6
 7from datasets import Dataset
 8from pydantic import BaseModel, Field
 9
10from distilabel.models import OpenAILLM
11from distilabel.pipeline import Pipeline
12from distilabel.steps import KeepColumns
13from distilabel.steps.tasks import TextGeneration
14
15# === 1. 定義輸出 Schema (結構) ===
16class MedicalEntity(BaseModel):
17    """醫療命名實體"""
18    text: str = Field(description="Entity text span")
19    entity_type: str = Field(
20        description="Entity type",
21        enum=["DRUG", "DISEASE", "GENE", "PROTEIN", "PATHWAY", "DOSAGE", "BIOMARKER"]
22    )
23    start: int = Field(description="Start character offset")
24    end: int = Field(description="End character offset")
25
26class MedicalNERAnnotation(BaseModel):
27    """醫療文本 NER 標注"""
28    text: str = Field(description="Original clinical text")
29    entities: List[MedicalEntity] = Field(description="Extracted entities")
30    complexity: str = Field(
31        description="Text complexity level",
32        enum=["simple", "moderate", "complex"]
33    )
34
35# === 2. Seed Contexts (種子上下文) — 提供主題方向 ===
36seed_contexts = Dataset.from_list([
37    {"instruction": "Generate a synthetic clinical note about a patient with advanced NSCLC receiving pembrolizumab combination therapy. Include drug names, dosages, biomarker results (PD-L1, TMB), and disease progression notes."},
38    {"instruction": "Generate a synthetic pharmacovigilance report documenting adverse events of trastuzumab deruxtecan (T-DXd) in HER2-low breast cancer, including ILD grading and management."},
39    {"instruction": "Generate a synthetic clinical trial eligibility criteria section for a Phase 2 study of a novel bispecific antibody targeting CD3/CD20 in relapsed/refractory DLBCL."},
40])
41
42# === 3. 建構 Structured Generation Pipeline ===
43
44llm = OpenAILLM(
45    model="gpt-4o",
46    generation_kwargs={"temperature": 0.6, "max_new_tokens": 2048},
47    structured_output={
48        "format": "json",
49        "schema": MedicalNERAnnotation.model_json_schema(),
50    },
51)
52
53with Pipeline(name="medical-ner-sdg") as pipeline:
54
55    generate = TextGeneration(
56        name="generate_annotated_text",
57        llm=llm,
58        system_prompt=(
59            "You are a clinical NLP annotation expert. "
60            "Generate realistic synthetic clinical text and annotate all "
61            "medical entities with precise character offsets. "
62            "Ensure entity spans are accurate and non-overlapping. "
63            "The generated text should be realistic enough to train "
64            "a medical NER model."
65        ),
66    )
67
68    keep = KeepColumns(
69        name="select_output",
70        columns=["instruction", "generation", "model_name"],
71    )
72
73    generate >> keep
74
75# === 4. 執行 ===
76if __name__ == "__main__":
77    distiset = pipeline.run(dataset=seed_contexts)
78    ds = distiset["default"]["train"]
79
80    import json
81    for row in ds.select(range(2)):
82        parsed = json.loads(row["generation"])
83        print(f"Text: {parsed['text'][:100]}...")
84        print(f"Entities: {len(parsed['entities'])} found")
85        for ent in parsed["entities"][:5]:
86            print(f"  - [{ent['entity_type']}] \"{ent['text']}\"")
87        print()

 1# 從 HuggingFace 載入 seed 資料
 2from datasets import load_dataset
 3seed = load_dataset("bigbio/med_qa", split="train[:100]")
 4
 5# 執行 distilabel pipeline
 6distiset = pipeline.run(dataset=seed)
 7
 8# 推送到 HuggingFace Hub
 9distiset.push_to_hub(
10    repo_id="your-org/medical-sdg-output",
11    private=True,
12    token="hf_..."
13)

 1from distilabel.steps.argilla import TextGenerationToArgilla
 2
 3# 在 pipeline 中加入 Argilla 匯出步驟
 4export_to_argilla = TextGenerationToArgilla(
 5    name="export_to_argilla",
 6    dataset_name="medical-sdg-review",
 7    dataset_workspace="biomedical",
 8    api_url="https://your-argilla.com",
 9    api_key="your-key",
10)

 1from distilabel.models import vLLM, OllamaLLM
 2
 3# 選項 A: vLLM（高吞吐量）
 4local_llm = vLLM(
 5    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
 6    extra_kwargs={"tensor_parallel_size": 2},
 7)
 8
 9# 選項 B: Ollama（輕量部署）
10local_llm = OllamaLLM(
11    model="llama3.1:8b",
12    host="http://localhost:11434",
13)

1from distilabel.pipeline import Pipeline
2
3# 只需變更 Pipeline 的 with_ray 參數即可擴展
4with Pipeline(name="medical-sdg-at-scale").ray() as pipeline:
5    # pipeline 定義完全相同
6    # Ray 自動處理分散式排程與容錯
7    ...