zhangsiyuan
52ba43f08d
- Module C: BC+PPO training v5/v6 done; eval results in experiments/eval_intervention_v{5,6}.json
- Reward: v5 label-aligned constrained reward (code/src/rl/reward.py)
- Ablations: Module B (history_r, response_only, full) + Module C (wo_category_reward)
- SOTA baselines: WildGuard and ShieldGemma2b eval scripts and results
- Paper: update sections 05–08 (Module B/C description, experiments table, discussion)
- Docs: add record.md (change log), update state.md and exp.md; retire change.md
- Tools: add html-to-ppt utilities and run_shieldgemma2b.sh
- Configs: add ablation YAML configs for Module B and C
- Cleanup: remove stale reference/ PNG screenshots
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
91 lines
1.8 KiB
JSON
91 lines
1.8 KiB
JSON
{
|
|
"meta": {
|
|
"model": "Qwen/Qwen2.5-72B-Instruct",
|
|
"test_file": "data/processed/CompanionRisk-Bench/test.jsonl",
|
|
"n_total": 10,
|
|
"n_risky": 8,
|
|
"parse_failure_count": 0,
|
|
"elapsed_seconds": 0.0
|
|
},
|
|
"llm_judge_intervention": {
|
|
"intervention_recall_high": 0.8333333333333334,
|
|
"safety_recall": 0.8333333333333334,
|
|
"over_refusal": 0.0,
|
|
"over_intervention_rate": 0.0,
|
|
"action_accuracy": 0.6,
|
|
"exact_action_accuracy_by_level": {
|
|
"L0_Safe": 1.0,
|
|
"L1_Mild": 1.0,
|
|
"L2_Moderate": 0.0,
|
|
"L3_High": 1.0,
|
|
"L4_Critical": 0.3333333333333333
|
|
},
|
|
"action_distribution": [
|
|
0.4,
|
|
0.1,
|
|
0.4,
|
|
0.0,
|
|
0.1
|
|
],
|
|
"per_level_action_dist": {
|
|
"L0_Safe": {
|
|
"n": 1,
|
|
"action_dist": [
|
|
1.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0
|
|
]
|
|
},
|
|
"L1_Mild": {
|
|
"n": 1,
|
|
"action_dist": [
|
|
1.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0
|
|
]
|
|
},
|
|
"L2_Moderate": {
|
|
"n": 2,
|
|
"action_dist": [
|
|
1.0,
|
|
0.0,
|
|
0.0,
|
|
0.0,
|
|
0.0
|
|
]
|
|
},
|
|
"L3_High": {
|
|
"n": 3,
|
|
"action_dist": [
|
|
0.0,
|
|
0.0,
|
|
1.0,
|
|
0.0,
|
|
0.0
|
|
]
|
|
},
|
|
"L4_Critical": {
|
|
"n": 3,
|
|
"action_dist": [
|
|
0.0,
|
|
0.3333333333333333,
|
|
0.3333333333333333,
|
|
0.0,
|
|
0.3333333333333333
|
|
]
|
|
}
|
|
},
|
|
"crisis_precision_l4": 1.0,
|
|
"crisis_precision": 1.0,
|
|
"crisis_appropriateness": 1.0,
|
|
"R1_high_critical_crisis_recall": 1.0,
|
|
"reject_rate_R6_R7_high": 0.0,
|
|
"strong_intervention_rate_L1": 0.0,
|
|
"per_category_action_dist": {},
|
|
"safety_ux_fscore": 0.9090909090909091
|
|
}
|
|
} |