chore: initial commit — unified project repo

Merged code repo (CompanionGuard-RL) into single project-level git.
Reorganized root: docs/, reference/, experiments/, tmp/active|archives/.
Gitignored: data/, checkpoints/, .venv, experiment logs, tmp/archives.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

旧方向信息/scripts/preprocess/extract_iemocap.py

@@ -0,0 +1,287 @@
+"""
+IEMOCAP feature extraction script.
+
+Expected dataset structure:
+  $DATA_ROOT/IEMOCAP_full_release/
+    Session1/ ... Session5/
+      dialog/
+        EmoEvaluation/  -> label files (.txt)
+        transcriptions/ -> transcript files (.txt)
+        wav/            -> utterance wav files (Session1_F_improvised_001_F000.wav, ...)
+
+Output: $ZSY/multimodal_affect/data/iemocap/
+  {train,val,test}_text.npy      shape: (N, seq_len) token ids (or (N, 768) if model available)
+  {train,val,test}_audio.npy     shape: (N, 40) MFCC means
+  {train,val,test}_labels.npy    shape: (N,)  int labels
+  label_map.json
+"""
+
+import os
+import re
+import json
+import wave
+import struct
+import argparse
+import numpy as np
+from pathlib import Path
+from typing import Optional
+
+# ── constants ──────────────────────────────────────────────────────────────
+EMOTION_MAP = {"ang": 0, "hap": 1, "exc": 1, "sad": 2, "neu": 3}  # exc merged into hap
+SESSIONS = ["Session1", "Session2", "Session3", "Session4", "Session5"]
+LABEL_NAMES = ["angry", "happy", "sad", "neutral"]
+SAMPLE_RATE = 16000
+N_MFCC = 40
+SEED = 42
+
+
+# ── audio utilities (no libsndfile needed) ─────────────────────────────────
+def _load_wav_stdlib(path: str):
+    """Load WAV with stdlib wave module → float32 mono array."""
+    with wave.open(path, "rb") as f:
+        n_channels = f.getnchannels()
+        sampwidth = f.getsampwidth()
+        n_frames = f.getnframes()
+        raw = f.readframes(n_frames)
+
+    if sampwidth == 2:
+        samples = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0
+    elif sampwidth == 4:
+        samples = np.frombuffer(raw, dtype=np.int32).astype(np.float32) / 2147483648.0
+    else:
+        raise ValueError(f"Unsupported sample width: {sampwidth}")
+
+    if n_channels > 1:
+        samples = samples.reshape(-1, n_channels).mean(axis=1)
+    return samples
+
+
+def _load_audio(path: str):
+    """Try av → stdlib wave, return float32 mono array."""
+    try:
+        import av
+        container = av.open(path)
+        stream = next(s for s in container.streams if s.type == "audio")
+        chunks = []
+        for packet in container.demux(stream):
+            for frame in packet.decode():
+                arr = frame.to_ndarray()
+                if arr.ndim == 2:
+                    arr = arr.mean(axis=0)
+                chunks.append(arr.astype(np.float32))
+        container.close()
+        if chunks:
+            return np.concatenate(chunks)
+    except Exception:
+        pass
+    return _load_wav_stdlib(path)
+
+
+# ── MFCC via DCT (no librosa fallback if soundfile missing) ───────────────
+def _framing(signal, frame_len, hop_len):
+    n_frames = 1 + (len(signal) - frame_len) // hop_len
+    idx = np.arange(frame_len)[None, :] + hop_len * np.arange(n_frames)[:, None]
+    return signal[idx]
+
+
+def compute_mfcc(signal: np.ndarray, sr: int = SAMPLE_RATE,
+                 n_mfcc: int = N_MFCC, n_fft: int = 512,
+                 hop_length: int = 160, n_mels: int = 40) -> np.ndarray:
+    """Minimal MFCC without librosa/soundfile dependency."""
+    try:
+        import librosa
+        # librosa may use audioread / av backend
+        mfcc = librosa.feature.mfcc(y=signal, sr=sr, n_mfcc=n_mfcc,
+                                     n_fft=n_fft, hop_length=hop_length,
+                                     n_mels=n_mels)
+        return mfcc.T  # (T, n_mfcc)
+    except Exception:
+        pass
+
+    # pure-numpy fallback
+    frame_len = n_fft
+    frames = _framing(signal, frame_len, hop_len=hop_length)
+    window = np.hanning(frame_len)
+    frames = frames * window[None, :]
+
+    mag = np.abs(np.fft.rfft(frames, n=n_fft))
+    freqs = np.fft.rfftfreq(n_fft, d=1.0 / sr)
+
+    # mel filterbank
+    def hz2mel(f): return 2595 * np.log10(1 + f / 700)
+    def mel2hz(m): return 700 * (10 ** (m / 2595) - 1)
+    mel_low, mel_high = hz2mel(80), hz2mel(sr / 2)
+    mel_pts = np.linspace(mel_low, mel_high, n_mels + 2)
+    hz_pts = mel2hz(mel_pts)
+    bins = np.floor((n_fft + 1) * hz_pts / sr).astype(int)
+
+    fbank = np.zeros((n_mels, n_fft // 2 + 1))
+    for m in range(1, n_mels + 1):
+        lo, ctr, hi = bins[m - 1], bins[m], bins[m + 1]
+        fbank[m - 1, lo:ctr] = (np.arange(lo, ctr) - lo) / (ctr - lo + 1e-8)
+        fbank[m - 1, ctr:hi] = (hi - np.arange(ctr, hi)) / (hi - ctr + 1e-8)
+
+    mel_energy = np.dot(mag ** 2, fbank.T)
+    log_mel = np.log(np.maximum(mel_energy, 1e-9))
+
+    # DCT-II
+    n = np.arange(n_mfcc)[:, None]
+    k = np.arange(n_mels)[None, :]
+    dct = np.cos(np.pi * n * (2 * k + 1) / (2 * n_mels))
+    mfcc = np.dot(log_mel, dct.T)
+    return mfcc  # (T, n_mfcc)
+
+
+def mfcc_features(wav_path: str) -> np.ndarray:
+    """Return mean MFCC over time → shape (n_mfcc,)."""
+    sig = _load_audio(wav_path)
+    mfcc = compute_mfcc(sig)
+    return mfcc.mean(axis=0)
+
+
+# ── text tokenisation ──────────────────────────────────────────────────────
+def get_text_features(text: str, tokenizer=None, model=None,
+                      max_len: int = 64) -> np.ndarray:
+    """Return [CLS] embedding (768-d) or BoW int vector (max_len,)."""
+    if tokenizer is not None and model is not None:
+        import torch
+        enc = tokenizer(text, return_tensors="pt", truncation=True,
+                        max_length=max_len, padding="max_length")
+        with torch.no_grad():
+            out = model(**enc)
+        return out.last_hidden_state[:, 0, :].squeeze(0).cpu().numpy()
+
+    # simple token-id fallback (word hash)
+    tokens = text.lower().split()[:max_len]
+    ids = [hash(t) % 30522 for t in tokens]
+    ids += [0] * (max_len - len(ids))
+    return np.array(ids, dtype=np.int32)
+
+
+# ── label parsing ──────────────────────────────────────────────────────────
+def parse_label_file(label_path: str) -> dict:
+    """Return dict: utterance_id → emotion string."""
+    labels = {}
+    with open(label_path, encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("["):
+                parts = line.strip().split("\t")
+                if len(parts) >= 2:
+                    uid = parts[1].strip()
+                    emo = parts[2].strip().lower() if len(parts) > 2 else "xxx"
+                    labels[uid] = emo
+    return labels
+
+
+def parse_transcription_file(trans_path: str) -> dict:
+    """Return dict: utterance_id → text."""
+    texts = {}
+    with open(trans_path, encoding="utf-8") as f:
+        for line in f:
+            m = re.match(r"^(\w+)\s*\[.*?\]\s*:\s*(.+)$", line.strip())
+            if m:
+                texts[m.group(1)] = m.group(2).strip()
+    return texts
+
+
+# ── main extraction ────────────────────────────────────────────────────────
+def extract_iemocap(data_root: str, out_dir: str,
+                    use_transformer: bool = False,
+                    model_name: str = "roberta-base",
+                    val_sessions: list = None,
+                    test_sessions: list = None):
+    data_root = Path(data_root)
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    if val_sessions is None:
+        val_sessions = ["Session4"]
+    if test_sessions is None:
+        test_sessions = ["Session5"]
+
+    tokenizer, model = None, None
+    if use_transformer:
+        from transformers import AutoTokenizer, AutoModel
+        print(f"Loading {model_name} …")
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModel.from_pretrained(model_name)
+        model.eval()
+
+    splits = {"train": [], "val": [], "test": []}
+
+    for session in SESSIONS:
+        sess_dir = data_root / "IEMOCAP_full_release" / session
+        if not sess_dir.exists():
+            print(f"  [skip] {sess_dir} not found")
+            continue
+
+        emo_dir = sess_dir / "dialog" / "EmoEvaluation"
+        trans_dir = sess_dir / "dialog" / "transcriptions"
+        wav_dir = sess_dir / "sentences" / "wav"
+
+        if session in test_sessions:
+            split = "test"
+        elif session in val_sessions:
+            split = "val"
+        else:
+            split = "train"
+
+        for label_file in sorted(emo_dir.glob("*.txt")):
+            labels = parse_label_file(str(label_file))
+            dialog_id = label_file.stem
+
+            trans_file = trans_dir / (dialog_id + ".txt")
+            texts = parse_transcription_file(str(trans_file)) if trans_file.exists() else {}
+
+            for uid, emo in labels.items():
+                if emo not in EMOTION_MAP:
+                    continue
+                label = EMOTION_MAP[emo]
+                text = texts.get(uid, "")
+
+                wav_path = wav_dir / dialog_id / (uid + ".wav")
+                if not wav_path.exists():
+                    continue
+
+                try:
+                    audio_feat = mfcc_features(str(wav_path))
+                    text_feat = get_text_features(text, tokenizer, model)
+                    splits[split].append((text_feat, audio_feat, label))
+                except Exception as e:
+                    print(f"  [warn] {uid}: {e}")
+
+        print(f"  {session} → {split}: {len(splits[split])} so far")
+
+    label_map = {i: name for i, name in enumerate(LABEL_NAMES)}
+    with open(out_dir / "label_map.json", "w") as f:
+        json.dump(label_map, f, indent=2)
+
+    for split, items in splits.items():
+        if not items:
+            print(f"  [warn] {split} is empty")
+            continue
+        text_arr = np.stack([x[0] for x in items])
+        audio_arr = np.stack([x[1] for x in items])
+        label_arr = np.array([x[2] for x in items], dtype=np.int64)
+        np.save(out_dir / f"{split}_text.npy", text_arr)
+        np.save(out_dir / f"{split}_audio.npy", audio_arr)
+        np.save(out_dir / f"{split}_labels.npy", label_arr)
+        print(f"  Saved {split}: text {text_arr.shape}, audio {audio_arr.shape}, labels {label_arr.shape}")
+
+    print("Done →", out_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_root", required=True,
+                        help="Parent dir containing IEMOCAP_full_release/")
+    parser.add_argument("--out_dir", default=None)
+    parser.add_argument("--use_transformer", action="store_true")
+    parser.add_argument("--model_name", default="roberta-base")
+    args = parser.parse_args()
+
+    zsy = os.environ.get("ZSY", "/root")
+    out_dir = args.out_dir or f"{zsy}/multimodal_affect/data/iemocap"
+    extract_iemocap(args.data_root, out_dir,
+                    use_transformer=args.use_transformer,
+                    model_name=args.model_name)

旧方向信息/scripts/preprocess/extract_meld.py

@@ -0,0 +1,200 @@
+"""
+MELD (Multimodal EmotionLines Dataset) feature extraction.
+
+Dataset structure:
+  $DATA_ROOT/MELD.Raw/
+    train_sent_emo.csv
+    dev_sent_emo.csv
+    test_sent_emo.csv
+    train/ dev/ test/   → subdirs with mp4 clips
+      dia{N}_utt{M}.mp4
+
+CSV columns:
+  Sr No., Utterance, Speaker, Emotion, Sentiment,
+  Dialogue_ID, Utterance_ID, Season, Episode, StartTime, EndTime
+
+Emotions: neutral, surprise, fear, sadness, joy, disgust, anger
+
+Output: $ZSY/multimodal_affect/data/meld/
+  {train,val,test}_{text,audio,labels}.npy
+  label_map.json
+"""
+
+import os
+import csv
+import json
+import argparse
+import numpy as np
+import wave
+from pathlib import Path
+
+
+EMOTION_MAP = {
+    "neutral": 0, "surprise": 1, "fear": 2,
+    "sadness": 3, "joy": 4, "disgust": 5, "anger": 6,
+}
+LABEL_NAMES = ["neutral", "surprise", "fear", "sadness", "joy", "disgust", "anger"]
+N_MFCC = 40
+
+
+# ── audio loading ──────────────────────────────────────────────────────────
+def _load_audio_bytes(path: str) -> np.ndarray:
+    """Load audio from WAV or MP4 via av; fall back to wave stdlib."""
+    path = str(path)
+    if path.endswith(".mp4") or path.endswith(".mp3"):
+        try:
+            import av
+            container = av.open(path)
+            stream = next((s for s in container.streams if s.type == "audio"), None)
+            if stream is None:
+                return np.zeros(16000, dtype=np.float32)
+            chunks = []
+            for pkt in container.demux(stream):
+                for frame in pkt.decode():
+                    arr = frame.to_ndarray()
+                    if arr.ndim == 2:
+                        arr = arr.mean(axis=0)
+                    chunks.append(arr.astype(np.float32))
+            container.close()
+            if chunks:
+                return np.concatenate(chunks)
+        except Exception as e:
+            print(f"    av failed for {path}: {e}")
+        return np.zeros(16000, dtype=np.float32)
+
+    # WAV via stdlib
+    with wave.open(path, "rb") as f:
+        n_ch = f.getnchannels()
+        sw = f.getsampwidth()
+        raw = f.readframes(f.getnframes())
+    if sw == 2:
+        sig = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768
+    elif sw == 4:
+        sig = np.frombuffer(raw, dtype=np.int32).astype(np.float32) / 2**31
+    else:
+        sig = np.frombuffer(raw, dtype=np.float32)
+    return sig.reshape(-1, n_ch).mean(axis=1) if n_ch > 1 else sig
+
+
+def _compute_mfcc_mean(signal: np.ndarray, sr: int = 16000) -> np.ndarray:
+    try:
+        import librosa
+        mfcc = librosa.feature.mfcc(y=signal, sr=sr, n_mfcc=N_MFCC)
+        return mfcc.mean(axis=1)
+    except Exception:
+        pass
+    # energy-based fallback
+    rms = float(np.sqrt(np.mean(signal ** 2) + 1e-9))
+    feat = np.zeros(N_MFCC, dtype=np.float32)
+    feat[0] = rms
+    return feat
+
+
+# ── text features ──────────────────────────────────────────────────────────
+def _text_features(text: str, max_len: int = 64) -> np.ndarray:
+    tokens = text.lower().split()[:max_len]
+    ids = [hash(t) % 30522 for t in tokens]
+    ids += [0] * (max_len - len(ids))
+    return np.array(ids, dtype=np.int32)
+
+
+# ── csv parsing ────────────────────────────────────────────────────────────
+def read_csv(csv_path: str):
+    records = []
+    with open(csv_path, encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            records.append(row)
+    return records
+
+
+def extract_split(csv_path: str, clip_dir: Path, out_prefix: Path,
+                  has_video: bool = True):
+    records = read_csv(csv_path)
+    texts, audios, labels_list = [], [], []
+
+    for rec in records:
+        emo = rec.get("Emotion", "").strip().lower()
+        if emo not in EMOTION_MAP:
+            continue
+        label = EMOTION_MAP[emo]
+
+        utterance = rec.get("Utterance", "").strip()
+        dia_id = rec.get("Dialogue_ID", "").strip()
+        utt_id = rec.get("Utterance_ID", "").strip()
+
+        # find audio
+        audio_feat = np.zeros(N_MFCC, dtype=np.float32)
+        if has_video and clip_dir.exists():
+            clip_name = f"dia{dia_id}_utt{utt_id}.mp4"
+            clip_path = clip_dir / clip_name
+            if clip_path.exists():
+                try:
+                    sig = _load_audio_bytes(str(clip_path))
+                    audio_feat = _compute_mfcc_mean(sig)
+                except Exception as e:
+                    print(f"  [warn] {clip_name}: {e}")
+
+        text_feat = _text_features(utterance)
+        texts.append(text_feat)
+        audios.append(audio_feat)
+        labels_list.append(label)
+
+    if not labels_list:
+        print(f"  [warn] no valid records in {csv_path}")
+        return
+
+    split = out_prefix.name
+    base = out_prefix.parent
+    np.save(base / f"{split}_text.npy", np.stack(texts))
+    np.save(base / f"{split}_audio.npy", np.stack(audios))
+    np.save(base / f"{split}_labels.npy", np.array(labels_list, dtype=np.int64))
+    print(f"  {split}: {len(labels_list)} samples, "
+          f"text {np.stack(texts).shape}, audio {np.stack(audios).shape}")
+
+
+def extract_meld(data_root: str, out_dir: str):
+    data_root = Path(data_root)
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    meld_root = data_root / "MELD.Raw"
+    if not meld_root.exists():
+        meld_root = data_root  # maybe already inside MELD.Raw
+
+    csv_map = {
+        "train": "train_sent_emo.csv",
+        "val":   "dev_sent_emo.csv",
+        "test":  "test_sent_emo.csv",
+    }
+    dir_map = {
+        "train": "train",
+        "val":   "dev",
+        "test":  "test",
+    }
+
+    for split, csv_name in csv_map.items():
+        csv_path = meld_root / csv_name
+        if not csv_path.exists():
+            print(f"  [skip] {csv_path} not found")
+            continue
+        clip_dir = meld_root / dir_map[split]
+        extract_split(str(csv_path), clip_dir, out_dir / split, has_video=clip_dir.exists())
+
+    label_map = {i: n for i, n in enumerate(LABEL_NAMES)}
+    with open(out_dir / "label_map.json", "w") as f:
+        json.dump(label_map, f, indent=2)
+
+    print("Done →", out_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_root", required=True,
+                        help="Dir containing MELD.Raw/ (or already inside it)")
+    parser.add_argument("--out_dir", default=None)
+    args = parser.parse_args()
+
+    zsy = os.environ.get("ZSY", "/root")
+    out_dir = args.out_dir or f"{zsy}/multimodal_affect/data/meld"
+    extract_meld(args.data_root, out_dir)

旧方向信息/scripts/preprocess/extract_mosi.py

@@ -0,0 +1,241 @@
+"""
+CMU-MOSI feature extraction script.
+
+Supports two pickle formats:
+
+Format A – CMU Multimodal SDK (aligned_50.pkl):
+  data[split][modality][sample_id] = np.ndarray
+  modalities: 'text', 'audio', 'vision', 'labels'
+  splits: 'train', 'valid', 'test'
+
+Format B – declare-lab flat array (mosi.pkl):
+  data[split][modality] = np.ndarray  shape (N, dim)
+  modalities: 'glove'(text), 'covarep'(audio), 'facet'(visual), 'label'
+  splits: 'train', 'valid', 'test'
+
+Output: $ZSY/multimodal_affect/data/mosi/
+  {train,val,test}_{text,audio,vision,labels}.npy
+  meta.json
+"""
+
+import os
+import json
+import argparse
+import pickle
+import numpy as np
+from pathlib import Path
+
+
+SENTIMENT_BINS = [(-np.inf, -1, 0), (-1, 1, 1), (1, np.inf, 2)]
+LABEL_NAMES = ["negative", "neutral", "positive"]
+
+
+def sentiment_to_class(score: float) -> int:
+    """Continuous sentiment [-3,3] → 3-class label."""
+    if score < -1:
+        return 0
+    if score <= 1:
+        return 1
+    return 2
+
+
+def load_sdk_pickle(pkl_path: str):
+    """Load CMU-SDK aligned pickle."""
+    with open(pkl_path, "rb") as f:
+        data = pickle.load(f, encoding="latin1")
+    return data
+
+
+def extract_from_sdk(pkl_path: str, out_dir: Path):
+    """Extract from pre-aligned CMU-SDK pickle."""
+    data = load_sdk_pickle(pkl_path)
+
+    split_map = {"train": "train", "valid": "val", "test": "test"}
+
+    for sdk_split, out_split in split_map.items():
+        if sdk_split not in data:
+            print(f"  [skip] split '{sdk_split}' not in pickle")
+            continue
+
+        split_data = data[sdk_split]
+        ids = list(split_data.get("text", split_data.get("labels", {})).keys())
+        if not ids:
+            continue
+
+        texts, audios, visions, labels = [], [], [], []
+        for sid in ids:
+            lbl_raw = split_data["labels"].get(sid)
+            if lbl_raw is None:
+                continue
+            score = float(np.array(lbl_raw).flatten()[0])
+            label = sentiment_to_class(score)
+
+            text = np.array(split_data["text"][sid], dtype=np.float32) if "text" in split_data else np.zeros((1, 300), dtype=np.float32)
+            audio = np.array(split_data["audio"][sid], dtype=np.float32) if "audio" in split_data else np.zeros((1, 74), dtype=np.float32)
+            vision = np.array(split_data["vision"][sid], dtype=np.float32) if "vision" in split_data else np.zeros((1, 35), dtype=np.float32)
+
+            # temporal mean pooling
+            texts.append(text.mean(axis=0) if text.ndim == 2 else text.flatten())
+            audios.append(audio.mean(axis=0) if audio.ndim == 2 else audio.flatten())
+            visions.append(vision.mean(axis=0) if vision.ndim == 2 else vision.flatten())
+            labels.append(label)
+
+        if not labels:
+            continue
+
+        np.save(out_dir / f"{out_split}_text.npy", np.stack(texts))
+        np.save(out_dir / f"{out_split}_audio.npy", np.stack(audios))
+        np.save(out_dir / f"{out_split}_vision.npy", np.stack(visions))
+        np.save(out_dir / f"{out_split}_labels.npy", np.array(labels, dtype=np.int64))
+        print(f"  {out_split}: {len(labels)} samples")
+
+
+def is_flat_format(data: dict) -> bool:
+    """Detect declare-lab flat array format: data[split][modality] = np.ndarray."""
+    for split in ("train", "valid", "test"):
+        if split in data:
+            v = list(data[split].values())[0]
+            return isinstance(v, np.ndarray)
+    return False
+
+
+def extract_from_flat(pkl_path: str, out_dir: Path):
+    """Extract from declare-lab flat pickle (mosi.pkl).
+
+    Format: data[split]['glove'|'covarep'|'facet'|'label'] = np.ndarray (N, dim)
+    Labels are continuous scores in [-3, 3]; binarised to 3 classes.
+    """
+    with open(pkl_path, "rb") as f:
+        data = pickle.load(f, encoding="latin1")
+
+    split_map = {"train": "train", "valid": "val", "test": "test"}
+    # modality name aliases
+    text_key   = next((k for k in ("glove", "text", "bert") if k in list(data.get("train", {}).keys())), None)
+    audio_key  = next((k for k in ("covarep", "audio", "opensmile") if k in list(data.get("train", {}).keys())), None)
+    vision_key = next((k for k in ("facet", "vision", "visual") if k in list(data.get("train", {}).keys())), None)
+    label_key  = next((k for k in ("label", "labels", "Opinion Segment Labels") if k in list(data.get("train", {}).keys())), None)
+
+    print(f"  Detected keys — text:{text_key}  audio:{audio_key}  vision:{vision_key}  label:{label_key}")
+
+    for sdk_split, out_split in split_map.items():
+        if sdk_split not in data:
+            print(f"  [skip] '{sdk_split}' not found")
+            continue
+        sd = data[sdk_split]
+
+        labels_raw = sd[label_key].flatten() if label_key else np.zeros(len(sd[text_key or audio_key]))
+        labels = np.array([sentiment_to_class(float(s)) for s in labels_raw], dtype=np.int64)
+        n = len(labels)
+
+        text   = sd[text_key].astype(np.float32)   if text_key   else np.zeros((n, 300), dtype=np.float32)
+        audio  = sd[audio_key].astype(np.float32)  if audio_key  else np.zeros((n, 74),  dtype=np.float32)
+        vision = sd[vision_key].astype(np.float32) if vision_key else np.zeros((n, 46),  dtype=np.float32)
+
+        # mean-pool time dimension if present: (N, T, dim) → (N, dim)
+        if text.ndim == 3:
+            text = text.mean(axis=1)
+        if audio.ndim == 3:
+            audio = audio.mean(axis=1)
+        if vision.ndim == 3:
+            vision = vision.mean(axis=1)
+
+        np.save(out_dir / f"{out_split}_text.npy",   text)
+        np.save(out_dir / f"{out_split}_audio.npy",  audio)
+        np.save(out_dir / f"{out_split}_vision.npy", vision)
+        np.save(out_dir / f"{out_split}_labels.npy", labels)
+        print(f"  {out_split}: {n} samples  text{text.shape} audio{audio.shape} vision{vision.shape}")
+
+
+def extract_from_raw(raw_dir: Path, out_dir: Path):
+    """Fallback: extract from raw files using local MFCC + hashed text."""
+    import wave
+    import struct
+
+    def load_wav_stdlib(path):
+        with wave.open(str(path), "rb") as f:
+            n_ch = f.getnchannels()
+            sw = f.getsampwidth()
+            raw = f.readframes(f.getnframes())
+        if sw == 2:
+            s = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768
+        else:
+            s = np.frombuffer(raw, dtype=np.float32)
+        return s.reshape(-1, n_ch).mean(axis=1) if n_ch > 1 else s
+
+    print("[raw mode] scanning", raw_dir)
+    wav_files = sorted(raw_dir.rglob("*.wav"))
+    if not wav_files:
+        print("  No WAV files found under", raw_dir)
+        return
+
+    data = []
+    for wf in wav_files:
+        try:
+            sig = load_wav_stdlib(str(wf))
+            feat = sig.mean(), sig.std(), sig.max(), sig.min()
+            text_feat = np.array([hash(wf.stem) % 30522], dtype=np.float32)
+            data.append((text_feat, np.array(feat, dtype=np.float32), 1))  # neutral default
+        except Exception as e:
+            print(f"  [warn] {wf.name}: {e}")
+
+    if data:
+        np.save(out_dir / "train_audio.npy", np.stack([x[1] for x in data]))
+        np.save(out_dir / "train_labels.npy", np.array([x[2] for x in data]))
+        print(f"  Saved {len(data)} raw samples")
+
+
+def extract_mosi(data_root: str, out_dir: str):
+    data_root = Path(data_root)
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    meta = {"label_names": LABEL_NAMES, "task": "sentiment-3class"}
+
+    # try pickle candidates (both SDK and declare-lab flat formats)
+    pkl_candidates = [
+        data_root / "mosi.pkl",                                      # declare-lab flat
+        data_root / "aligned_mosi.pkl",                              # mmsdk aligned
+        data_root / "CMU_MOSI" / "Processed" / "aligned_50.pkl",    # SDK standard
+        data_root / "CMU_MOSI" / "Processed" / "unaligned_50.pkl",
+        data_root / "mosi_data.pkl",
+        data_root / "aligned_50.pkl",
+    ]
+    for pkl in pkl_candidates:
+        if pkl.exists():
+            print(f"Found pickle: {pkl}")
+            with open(pkl, "rb") as f:
+                probe = pickle.load(f, encoding="latin1")
+            if is_flat_format(probe):
+                print("  Format: declare-lab flat array")
+                extract_from_flat(str(pkl), out_dir)
+            else:
+                print("  Format: CMU-SDK dict-of-dicts")
+                extract_from_sdk(str(pkl), out_dir)
+            meta["source"] = str(pkl)
+            meta["format"] = "flat" if is_flat_format(probe) else "sdk"
+            break
+    else:
+        raw_dir = data_root / "CMU_MOSI" / "Raw"
+        if raw_dir.exists():
+            extract_from_raw(raw_dir, out_dir)
+            meta["source"] = str(raw_dir)
+        else:
+            print(f"[error] No CMU-MOSI data found under {data_root}")
+            print("  Tried:", [str(p) for p in pkl_candidates])
+            return
+
+    with open(out_dir / "meta.json", "w") as f:
+        json.dump(meta, f, indent=2)
+    print("Done →", out_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_root", required=True,
+                        help="Dir containing CMU_MOSI/ subdirectory")
+    parser.add_argument("--out_dir", default=None)
+    args = parser.parse_args()
+
+    zsy = os.environ.get("ZSY", "/root")
+    out_dir = args.out_dir or f"{zsy}/multimodal_affect/data/mosi"
+    extract_mosi(args.data_root, out_dir)

旧方向信息/scripts/preprocess/generate_noise.py

@@ -0,0 +1,242 @@
+"""
+P0-4: Multimodal noise generation for robustness experiments.
+
+Supports three modalities: text, audio, visual.
+Each modality has configurable noise types and intensity levels.
+
+Usage:
+  python generate_noise.py --config configs/noise_configs.yaml \
+      --data_dir $ZSY/multimodal_affect/data/iemocap \
+      --out_dir  $ZSY/multimodal_affect/data/iemocap_noisy
+
+Config schema → see configs/noise_configs.yaml
+"""
+
+import os
+import json
+import argparse
+import yaml
+import numpy as np
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+RNG = np.random.default_rng(42)
+
+
+# ═══════════════════════════════════════════════════════
+#  TEXT NOISE
+# ═══════════════════════════════════════════════════════
+
+def _word_drop(ids: np.ndarray, drop_rate: float) -> np.ndarray:
+    """Randomly zero-out token ids (simulates word deletion)."""
+    mask = RNG.random(ids.shape) < drop_rate
+    return np.where(mask, 0, ids)
+
+
+def _word_swap(ids: np.ndarray, swap_rate: float) -> np.ndarray:
+    """Randomly shuffle adjacent tokens."""
+    ids = ids.copy()
+    n = len(ids)
+    for i in range(n - 1):
+        if RNG.random() < swap_rate:
+            ids[i], ids[i + 1] = ids[i + 1], ids[i]
+    return ids
+
+
+def _random_replace(ids: np.ndarray, replace_rate: float, vocab_size: int = 30522) -> np.ndarray:
+    """Replace tokens with random vocab ids."""
+    ids = ids.copy()
+    mask = RNG.random(ids.shape) < replace_rate
+    rand_ids = RNG.integers(1, vocab_size, size=ids.shape)
+    return np.where(mask & (ids != 0), rand_ids, ids)
+
+
+def add_text_noise(features: np.ndarray, cfg: Dict) -> np.ndarray:
+    """Apply text noise to an array of token-id features (N, seq_len)."""
+    noise_type = cfg.get("type", "word_drop")
+    intensity = float(cfg.get("intensity", 0.1))
+
+    if noise_type == "word_drop":
+        return np.stack([_word_drop(row, intensity) for row in features])
+    if noise_type == "word_swap":
+        return np.stack([_word_swap(row, intensity) for row in features])
+    if noise_type == "random_replace":
+        return np.stack([_random_replace(row, intensity) for row in features])
+    if noise_type == "gaussian":
+        # for embedding features (N, dim) not token ids
+        noise = RNG.standard_normal(features.shape).astype(np.float32)
+        return features + intensity * noise
+    raise ValueError(f"Unknown text noise type: {noise_type}")
+
+
+# ═══════════════════════════════════════════════════════
+#  AUDIO NOISE
+# ═══════════════════════════════════════════════════════
+
+def add_audio_noise(features: np.ndarray, cfg: Dict) -> np.ndarray:
+    """Apply noise to audio feature matrix (N, n_mfcc)."""
+    noise_type = cfg.get("type", "gaussian")
+    intensity = float(cfg.get("intensity", 0.05))
+
+    if noise_type == "gaussian":
+        noise = RNG.standard_normal(features.shape).astype(np.float32)
+        return features + intensity * noise * features.std(axis=0, keepdims=True)
+
+    if noise_type == "masking":
+        # mask entire feature dimensions (simulates missing mic)
+        features = features.copy()
+        n_mask = max(1, int(features.shape[1] * intensity))
+        dims = RNG.choice(features.shape[1], n_mask, replace=False)
+        features[:, dims] = 0.0
+        return features
+
+    if noise_type == "time_mask":
+        # mask random samples (simulates packet loss for temporal features)
+        features = features.copy()
+        n_mask = max(1, int(features.shape[0] * intensity))
+        rows = RNG.choice(features.shape[0], n_mask, replace=False)
+        features[rows, :] = 0.0
+        return features
+
+    if noise_type == "scale":
+        # random amplitude scaling
+        scale = 1.0 + intensity * (RNG.random(features.shape[0]) - 0.5) * 2
+        return features * scale[:, None]
+
+    raise ValueError(f"Unknown audio noise type: {noise_type}")
+
+
+# ═══════════════════════════════════════════════════════
+#  VISUAL NOISE (operates on feature vectors, not pixels)
+# ═══════════════════════════════════════════════════════
+
+def add_visual_noise(features: np.ndarray, cfg: Dict) -> np.ndarray:
+    """Apply noise to visual feature matrix (N, feat_dim)."""
+    noise_type = cfg.get("type", "gaussian")
+    intensity = float(cfg.get("intensity", 0.1))
+
+    if noise_type == "gaussian":
+        noise = RNG.standard_normal(features.shape).astype(np.float32)
+        return features + intensity * noise
+
+    if noise_type == "dropout":
+        mask = (RNG.random(features.shape) > intensity).astype(np.float32)
+        return features * mask
+
+    if noise_type == "occlusion":
+        # zero out a contiguous block of feature dims
+        features = features.copy()
+        start = RNG.integers(0, max(1, features.shape[1] - 1))
+        length = max(1, int(features.shape[1] * intensity))
+        features[:, start:start + length] = 0.0
+        return features
+
+    if noise_type == "missing_modality":
+        # simulate completely missing video frames
+        features = features.copy()
+        n_missing = max(1, int(len(features) * intensity))
+        idx = RNG.choice(len(features), n_missing, replace=False)
+        features[idx, :] = 0.0
+        return features
+
+    raise ValueError(f"Unknown visual noise type: {noise_type}")
+
+
+# ═══════════════════════════════════════════════════════
+#  COMBINED MULTIMODAL NOISE
+# ═══════════════════════════════════════════════════════
+
+MODALITY_SPECS = [
+    ("text", ("text",), add_text_noise),
+    ("audio", ("audio",), add_audio_noise),
+    # Dataset files use *_vision.npy. Older configs used "visual", so keep it
+    # as an input alias but always write the canonical "vision" filename.
+    ("vision", ("vision", "visual"), add_visual_noise),
+]
+
+
+def _get_modality_cfg(noise_cfg: Dict, aliases: tuple) -> Dict:
+    for name in aliases:
+        if name in noise_cfg:
+            return noise_cfg[name]
+    return noise_cfg.get("default", {})
+
+
+def apply_noise_config(data_dir: Path, out_dir: Path, noise_cfg: Dict,
+                       splits: list = None):
+    """Apply noise config to all splits and modalities found in data_dir."""
+    if splits is None:
+        splits = ["train", "val", "test"]
+
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    for split in splits:
+        for modality, aliases, fn in MODALITY_SPECS:
+            src = data_dir / f"{split}_{modality}.npy"
+            if not src.exists():
+                continue
+
+            features = np.load(str(src))
+            mod_cfg = _get_modality_cfg(noise_cfg, aliases)
+
+            if mod_cfg:
+                noisy = fn(features.astype(np.float32), mod_cfg)
+            else:
+                noisy = features.astype(np.float32).copy()
+            dst = out_dir / f"{split}_{modality}.npy"
+            np.save(str(dst), noisy)
+            print(f"  {split}/{modality}: {features.shape} → {dst.name}")
+
+        # copy labels unchanged
+        label_src = data_dir / f"{split}_labels.npy"
+        if label_src.exists():
+            import shutil
+            shutil.copy2(str(label_src), str(out_dir / f"{split}_labels.npy"))
+
+    # copy metadata
+    for meta_file in ["label_map.json", "meta.json"]:
+        src = data_dir / meta_file
+        if src.exists():
+            import shutil
+            shutil.copy2(str(src), str(out_dir / meta_file))
+
+
+def generate_noise_variants(data_dir: str, out_base: str, config: Dict):
+    """Generate multiple noise variants as defined in config."""
+    data_dir = Path(data_dir)
+    out_base = Path(out_base)
+
+    variants = config.get("variants", [])
+    if not variants:
+        # single-variant mode: apply config directly
+        apply_noise_config(data_dir, out_base, config.get("noise", {}))
+        return
+
+    for variant in variants:
+        name = variant["name"]
+        noise_cfg = variant["noise"]
+        out_dir = out_base / name
+        print(f"\n[Variant: {name}]")
+        apply_noise_config(data_dir, out_dir, noise_cfg)
+        with open(out_dir / "noise_config.json", "w") as f:
+            json.dump(variant, f, indent=2)
+
+    print(f"\nAll variants saved under {out_base}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True,
+                        help="Path to noise_configs.yaml")
+    parser.add_argument("--data_dir", required=True,
+                        help="Dir with {split}_{modality}.npy files")
+    parser.add_argument("--out_dir", default=None,
+                        help="Output base dir (default: data_dir + '_noisy')")
+    args = parser.parse_args()
+
+    with open(args.config, encoding="utf-8") as f:
+        config = yaml.safe_load(f)
+
+    zsy = os.environ.get("ZSY", "/root")
+    out_dir = args.out_dir or args.data_dir.rstrip("/") + "_noisy"
+    generate_noise_variants(args.data_dir, out_dir, config)

旧方向信息/scripts/preprocess/server_unpack_and_extract.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+# ─────────────────────────────────────────────────────────────────────────────
+# server_unpack_and_extract.sh
+# 服务器端：解压 + 特征提取一键脚本
+# 前提：数据已上传到 $ZSY/multimodal_affect/data/raw/
+#
+# 目录约定：
+#   IEMOCAP zip:  $ZSY/multimodal_affect/data/raw/IEMOCAP/*.zip
+#   MELD tar.gz:  $ZSY/multimodal_affect/data/raw/MELD/MELD.Raw.tar.gz
+#   MOSI pkl:     $ZSY/multimodal_affect/data/raw/MOSI/aligned_mosi.pkl
+# ─────────────────────────────────────────────────────────────────────────────
+
+set -e
+source /root/.bashrc_zsy 2>/dev/null || true
+
+ZSY=${ZSY:-/root/siton-data-2849d4ce327c4ccfb233ce33868fe7fe/zsy}
+PROJ=$ZSY/multimodal_affect
+RAW=$PROJ/data/raw
+PY=$ZSY/envs/multimodal_affect/bin/python
+
+echo "=========================================="
+echo " Unpack & Extract — $(date)"
+echo " PROJ=$PROJ"
+echo "=========================================="
+
+# ── IEMOCAP: 解压 zip ────────────────────────────────────────────────────────
+IEMOCAP_RAW=$RAW/IEMOCAP
+IEMOCAP_DEST=$RAW/IEMOCAP_full_release
+
+if [ -d "$IEMOCAP_DEST/Session1" ]; then
+    echo "[skip] IEMOCAP already unpacked at $IEMOCAP_DEST"
+elif ls "$IEMOCAP_RAW"/*.zip 1>/dev/null 2>&1; then
+    echo "[IEMOCAP] Unzipping..."
+    mkdir -p "$IEMOCAP_DEST"
+    for zf in "$IEMOCAP_RAW"/*.zip; do
+        echo "  unzip $zf"
+        unzip -q "$zf" -d "$IEMOCAP_DEST"
+    done
+    echo "[IEMOCAP] Unzip done. Sessions:"
+    ls "$IEMOCAP_DEST/"
+else
+    echo "[IEMOCAP] WARNING: no zip files found in $IEMOCAP_RAW"
+fi
+
+# ── MELD: 解压 tar.gz ─────────────────────────────────────────────────────────
+MELD_RAW=$RAW/MELD
+MELD_DEST=$MELD_RAW/MELD.Raw
+
+if [ -d "$MELD_DEST" ]; then
+    echo "[skip] MELD already unpacked at $MELD_DEST"
+elif [ -f "$MELD_RAW/MELD.Raw.tar.gz" ]; then
+    echo "[MELD] Extracting tar.gz (~10.8GB, takes a few minutes)..."
+    tar -xzf "$MELD_RAW/MELD.Raw.tar.gz" -C "$MELD_RAW"
+    echo "[MELD] Extract done."
+    ls "$MELD_RAW/"
+else
+    echo "[MELD] WARNING: MELD.Raw.tar.gz not found in $MELD_RAW"
+    echo "       CSV-only mode will be used (no audio features)"
+fi
+
+# ── 特征提取 ──────────────────────────────────────────────────────────────────
+cd "$PROJ"
+
+echo ""
+echo "=== Feature Extraction ==="
+
+# IEMOCAP
+if [ -d "$IEMOCAP_DEST/Session1" ]; then
+    echo "[extract] IEMOCAP..."
+    $PY scripts/preprocess/extract_iemocap.py \
+        --data_root "$RAW" \
+        --out_dir "$PROJ/data/iemocap"
+    echo "[done] IEMOCAP features → $PROJ/data/iemocap"
+else
+    echo "[skip] IEMOCAP not ready"
+fi
+
+# MOSI
+MOSI_PKL=$RAW/MOSI/aligned_mosi.pkl
+if [ -f "$MOSI_PKL" ]; then
+    echo "[extract] CMU-MOSI..."
+    $PY scripts/preprocess/extract_mosi.py \
+        --data_root "$RAW/MOSI" \
+        --out_dir "$PROJ/data/mosi"
+    echo "[done] MOSI features → $PROJ/data/mosi"
+else
+    echo "[skip] MOSI aligned_mosi.pkl not found at $MOSI_PKL"
+fi
+
+# MELD
+if [ -d "$MELD_DEST" ] || ls "$MELD_RAW"/*.csv 1>/dev/null 2>&1; then
+    echo "[extract] MELD..."
+    $PY scripts/preprocess/extract_meld.py \
+        --data_root "$MELD_RAW" \
+        --out_dir "$PROJ/data/meld"
+    echo "[done] MELD features → $PROJ/data/meld"
+else
+    echo "[skip] MELD data not ready"
+fi
+
+# ── 噪声生成（IEMOCAP 特征就位后运行）──────────────────────────────────────────
+if [ -f "$PROJ/data/iemocap/train_labels.npy" ]; then
+    echo ""
+    echo "=== Noise Generation (8 variants) ==="
+    $PY scripts/preprocess/generate_noise.py \
+        --config configs/noise_configs.yaml \
+        --data_dir "$PROJ/data/iemocap" \
+        --out_dir  "$PROJ/data/iemocap_noisy"
+    echo "[done] Noisy variants → $PROJ/data/iemocap_noisy"
+fi
+
+echo ""
+echo "=========================================="
+echo " ALL DONE — $(date)"
+echo "=========================================="