Refactor JspaceAI codebase: remove main_multimodal.py and main_v2.py, add main_chat.py for interactive dialogue mode with online learning capabilities.

main.py

@@ -1,245 +1,242 @@
 #!/usr/bin/env python3
 """
-JspaceAI —— 主程序入口
+JspaceAI —— 全部接入版（具身 Agent + 完整神经系统 + 自主心智）
 
-对比实验：
-    1. JSpaceModel（工作空间 + J-space 广播 + ODE 动力学）
-    2. FlatBaseline（同样参数量的扁平 MLP）
+接入全部 5 个感官输入 + 4 个输出执行器 + 神经系统（小脑/海马体/基底神经节/中枢神经）
++ 自主心智（好奇心驱动 + 状态持久化 + 自我模型 + 元学习）。
 
-任务：多模态连续时间序列预测（4 种模式切换）
-
-输出：
-    - 训练/评估 loss 曲线对比
-    - 注意力热力图（哪个专家在哪个时段被激活）
-    - 工作空间 ||w|| 演化曲线（输出门控信号）
-    - 预测 vs 真实序列对比
+模式：
+    --mode test:    子系统自检（权限 + I/O + 模型 + 海马体回忆）
+    --mode live:    实时具身循环 + 自主心智（默认安全：不控制鼠标键盘）
+    --mode safe:    同 live 但更保守（更高动作门控阈值）
 
 运行：
-    python main.py
-    python main.py --steps 1000 --device cuda
+    python main.py --mode test
+    python main.py --mode live --steps 100
+    python main.py --mode safe --steps 50
+    python main.py --mode live --steps 200 --unsafe   # 允许鼠标键盘输出
+
+再次运行会从上次状态继续（海洋不蒸发）。
 """
 from __future__ import annotations
-
 import argparse
 import torch
 import numpy as np
 import matplotlib.pyplot as plt
 from pathlib import Path
+import time
 
 from jspaceai import (
-    JSpaceConfig, JSpaceModel, FlatBaseline,
-    ContinuousSequenceTask, Trainer,
+    MultimodalConfig, MultimodalJSpaceModel, EmbodiedAgent,
+    AutonomousMind, PLATFORM,
+    get_screen_size, print_permission_guide,
+    check_camera_permission, check_microphone_permission,
+    check_input_monitoring_permission,
 )
 
 
-def count_params(model: torch.nn.Module) -> int:
-    return sum(p.numel() for p in model.parameters() if p.requires_grad)
-
-
-def run_experiment(n_steps: int, device: str, outdir: Path):
-    outdir.mkdir(parents=True, exist_ok=True)
-    torch.manual_seed(42)
-    np.random.seed(42)
-
-    # 任务
-    task = ContinuousSequenceTask(input_dim=8, seq_len=64, seed=42)
-
-    # 模型 1: JSpaceModel
-    config = JSpaceConfig(
-        input_dim=8,
-        workspace_dim=32,
-        expert_dim=16,
-        num_experts=5,
-        num_wells=4,
-        ode_steps=4,
-        dt=0.1,
-        tau_w=0.3,
-        output_threshold=0.5,
-        jacobian_sparsity=8,
-        noise_std=0.01,
+def get_config() -> MultimodalConfig:
+    return MultimodalConfig(
+        vocab_size=50, embed_dim=16, input_dim=8, workspace_dim=64,
+        expert_dim=24, num_experts=12, num_wells=4, ode_steps=3,
+        dt=0.1, tau_w=0.3, jacobian_sparsity=16, noise_std=0.01,
+        img_size=32, audio_frame_size=1024, keyboard_vocab=128,
     )
-    jspace_model = JSpaceModel(config)
 
-    # 模型 2: FlatBaseline（参数量对齐）
-    # JSpaceModel 大约的参数量：
-    #   专家: 5 × (4×16 + 4 + 16×8 + 16×32 + 16×32) ≈ 5 × 1108 = 5540
-    #   工作空间: (8+32)×32 + 32 + 32×32 + 32 ≈ 2144
-    #   预测头: 32×32 + 32 + 32×8 + 8 ≈ 1384
-    #   总计 ≈ 9000
-    flat_model = FlatBaseline(input_dim=8, hidden_dim=90, num_layers=2)
 
+def test_subsystems():
+    """子系统自检"""
     print("=" * 60)
-    print("JspaceAI 对比实验")
+    print(f"具身 Agent 子系统测试 | 平台: {PLATFORM}")
     print("=" * 60)
-    print(f"任务: 多模态连续序列预测 (input_dim=8, seq_len=64)")
-    print(f"训练步数: {n_steps}")
-    print(f"设备: {device}")
-    print()
-    print(f"JSpaceModel 参数量: {count_params(jspace_model):,}")
-    print(f"FlatBaseline 参数量: {count_params(flat_model):,}")
-    print()
 
-    # 训练
-    print("-" * 60)
-    print("训练 JSpaceModel...")
-    trainer_js = Trainer(jspace_model, task, lr=1e-3, device=device)
-    history_js = trainer_js.train(n_steps=n_steps, batch_size=32, eval_interval=n_steps // 10)
+    print("\n1. 权限检查:")
+    checks = {
+        '摄像头': check_camera_permission(),
+        '麦克风': check_microphone_permission(),
+        '键盘/鼠标监听': check_input_monitoring_permission(),
+    }
+    for name, ok in checks.items():
+        print(f"   {name}: {'OK' if ok else '需要权限'}")
+    if not all(checks.values()):
+        print_permission_guide()
 
-    print()
-    print("-" * 60)
-    print("训练 FlatBaseline...")
-    trainer_flat = Trainer(flat_model, task, lr=1e-3, device=device)
-    history_flat = trainer_flat.train(n_steps=n_steps, batch_size=32, eval_interval=n_steps // 10)
+    sw, sh = get_screen_size()
+    print(f"   屏幕尺寸: {sw}x{sh}")
 
-    # 评估
-    print()
+    print("\n2. 模型 + Agent 初始化:")
+    config = get_config()
+    model = MultimodalJSpaceModel(config)
+    print(f"   模型参数: {sum(p.numel() for p in model.parameters()):,}")
+    print(f"   专家分工: {model.expert_modality}")
+
+    agent = EmbodiedAgent(
+        model, device='cpu',
+        enable_mouse_output=False, enable_keyboard_output=False,
+        enable_audio_output=True, enable_screen_output=False,
+    )
+    print(f"   小脑参数: {sum(p.numel() for p in agent.cerebellum.parameters()):,}")
+    print(f"   海马体容量: {agent.hippocampus.capacity}")
+    print(f"   反射弧数: {len(agent.cns.reflexes)}")
+
+    print("\n3. 单步循环测试（2秒采集）:")
+    agent.senses.start()
+    time.sleep(2)
+    for _ in range(5):
+        info = agent.step_once()
+        print(f"   step {info['step']:2d} | mod {info['modality']:8s} | "
+              f"||w|| {info['w_norm']:.3f} | action {info['action']['action_idx']} | "
+              f"executed {info['action']['executed']} | mem {info['memories_count']}")
+        time.sleep(0.5)
+    agent.senses.stop()
+    agent.audio_actuator.stop()
+
+    print("\n4. 海马体回忆测试:")
+    if agent.hippocampus and agent.hippocampus.size() > 0:
+        w = agent.state['w'][0].cpu().numpy()
+        for i, m in enumerate(agent.hippocampus.recall(w, top_k=3)):
+            print(f"   记忆 {i}: sim={m['similarity']:.3f} ctx={m['context']}")
+
+    print("\n所有子系统测试完成")
+
+
+def live(n_steps: int, device: str, safe_mode: bool = False, unsafe: bool = False):
+    """实时具身循环 + 自主心智"""
+    print(f"平台: {PLATFORM}")
+    sw, sh = get_screen_size()
+    print(f"屏幕: {sw}x{sh}")
+
+    allow_output = unsafe and not safe_mode
+    if safe_mode:
+        print("\n安全模式：不执行鼠标/键盘动作")
+        print_permission_guide()
+    elif not allow_output:
+        print("\n默认模式：仅感知 + 音频/屏幕输出（不控制鼠标键盘）")
+        print("  如需鼠标键盘输出，加 --unsafe")
+
+    config = get_config()
+    model = MultimodalJSpaceModel(config).to(device)
+    mp = Path('outputs/multimodal_model.pt')
+    if mp.exists():
+        try:
+            ckpt = torch.load(mp, map_location=device, weights_only=False)
+            model.load_state_dict(ckpt['model'], strict=False)
+            print(f"已加载模型: {mp}")
+        except Exception:
+            print("模型加载失败，随机初始化")
+    else:
+        print("未找到训练模型，随机初始化")
+    model.eval()
+
+    agent = EmbodiedAgent(
+        model, device=device,
+        enable_mouse_output=allow_output,
+        enable_keyboard_output=allow_output,
+        enable_audio_output=True,
+        enable_screen_output=not safe_mode,
+        risk_threshold=0.5 if safe_mode else 0.3,
+    )
+    mind = AutonomousMind(agent, save_dir='outputs/mind', device=device)
+
+    print("\n" + "=" * 60)
+    print("自主心智 - 全感官具身循环")
     print("=" * 60)
-    print("最终评估")
-    print("=" * 60)
-    eval_xs = task.generate_batch(128)
-    eval_loss_js = trainer_js.evaluate(eval_xs)
-    eval_loss_flat = trainer_flat.evaluate(eval_xs)
-    print(f"JSpaceModel  eval MSE: {eval_loss_js:.6f}")
-    print(f"FlatBaseline eval MSE: {eval_loss_flat:.6f}")
-    winner = "JSpaceModel" if eval_loss_js < eval_loss_flat else "FlatBaseline"
-    improvement = abs(eval_loss_js - eval_loss_flat) / max(eval_loss_js, eval_loss_flat) * 100
-    print(f"胜者: {winner} (相对优势 {improvement:.1f}%)")
+    print("好奇心驱动 + 状态持久化 + 自我模型 + 元学习")
+    print(f"运行 {n_steps} 步（Ctrl+C 中断，状态自动保存）\n")
+
+    log = []
+
+    def on_step(info):
+        log.append(info)
+        if info['step'] % 10 == 0:
+            print(f"  step {info['step']:4d} | mod {info['modality']:8s} | "
+                  f"||w|| {info['w_norm']:.3f} | curio {info['curiosity']:.3f} | "
+                  f"success {info['success']:.2f} | weak={info['weakness']} | "
+                  f"mem {info['memory_count']}")
+
+    mind.run(n_steps=n_steps, interval=0.2, save_every=30, on_step=on_step)
+
+    print("\n" + mind.introspect())
 
     # 可视化
-    print()
-    print("生成可视化...")
+    if log:
+        fig, axes = plt.subplots(2, 3, figsize=(18, 10))
+        steps = [s['step'] for s in log]
 
-    # 1. Loss 曲线对比
-    fig, axes = plt.subplots(2, 2, figsize=(14, 10))
+        # ||w|| 按模态着色
+        ax = axes[0, 0]
+        wn = [s['w_norm'] for s in log]
+        mods = [s['modality'] for s in log]
+        cm = {'image': 'green', 'screen': 'purple', 'audio': 'blue',
+              'keyboard': 'orange', 'mouse': 'red', 'idle': 'gray'}
+        colors = [cm.get(m, 'gray') for m in mods]
+        ax.scatter(steps, wn, c=colors, alpha=0.7, s=30)
+        ax.set_xlabel('Step'); ax.set_ylabel('||w||')
+        ax.set_title('Workspace Norm (by modality)'); ax.grid(True, alpha=0.3)
+        from matplotlib.patches import Patch
+        ax.legend(handles=[Patch(facecolor=c, label=m) for m, c in cm.items()],
+                  fontsize=7)
 
-    ax = axes[0, 0]
-    ax.plot(history_js, label='JSpaceModel', alpha=0.7, linewidth=0.8)
-    ax.plot(history_flat, label='FlatBaseline', alpha=0.7, linewidth=0.8)
-    # 平滑曲线
-    if len(history_js) > 20:
-        smooth_js = np.convolve(history_js, np.ones(20)/20, mode='valid')
-        smooth_flat = np.convolve(history_flat, np.ones(20)/20, mode='valid')
-        ax.plot(smooth_js, label='JSpace (smoothed)', linewidth=2)
-        ax.plot(smooth_flat, label='Flat (smoothed)', linewidth=2)
-    ax.set_xlabel('Step')
-    ax.set_ylabel('MSE Loss')
-    ax.set_title('Training Loss')
-    ax.legend()
-    ax.set_yscale('log')
-    ax.grid(True, alpha=0.3)
-
-    # 2. 注意力热力图
-    test_xs = task.generate_batch(1)  # 单条序列
-    alpha = trainer_js.get_attention(test_xs)  # (1, T, num_experts)
-    if alpha is not None:
+        # 好奇心 + 成功度
         ax = axes[0, 1]
-        alpha_np = alpha[0].cpu().numpy()  # (T, num_experts)
-        im = ax.imshow(alpha_np.T, aspect='auto', cmap='hot', interpolation='nearest')
-        ax.set_xlabel('Time step')
-        ax.set_ylabel('Expert index')
-        ax.set_title('J-space Attention (which expert is active)')
-        plt.colorbar(im, ax=ax)
-        # 标注时段边界
-        for i in range(1, 64 // 16):
-            ax.axvline(x=i * 16, color='cyan', linestyle='--', alpha=0.7)
-        ax.set_xticks(range(0, 64, 16))
+        ax.plot(steps, [s['curiosity'] for s in log], 'r-', label='curiosity')
+        ax.plot(steps, [s['success'] for s in log], 'g-', label='success')
+        ax.legend(); ax.set_title('Curiosity & Success')
+        ax.grid(True, alpha=0.3)
 
-    # 3. 工作空间 ||w|| 演化
-    jspace_model.eval()
-    with torch.no_grad():
-        xs = test_xs.to(device)
-        _, info = jspace_model(xs)
-    w_norm = info['w_norm'][0].cpu().numpy()
+        # 模态分布
+        ax = axes[0, 2]
+        mc = {}
+        for m in mods:
+            mc[m] = mc.get(m, 0) + 1
+        ax.bar(mc.keys(), mc.values(),
+               color=[cm.get(m, 'gray') for m in mc.keys()])
+        ax.set_title('Modality Distribution')
 
-    ax = axes[1, 0]
-    ax.plot(w_norm, label='||w||', linewidth=2)
-    ax.axhline(y=config.output_threshold, color='r', linestyle='--',
-               label=f'threshold={config.output_threshold}', alpha=0.7)
-    ax.set_xlabel('Time step')
-    ax.set_ylabel('||w||')
-    ax.set_title('Workspace norm (output gating signal)')
-    ax.legend()
-    ax.grid(True, alpha=0.3)
-    for i in range(1, 64 // 16):
-        ax.axvline(x=i * 16, color='gray', linestyle='--', alpha=0.5)
+        # 自我模型
+        ax = axes[1, 0]
+        fc = log[-1]['self_confidence']
+        ax.barh(list(fc.keys()), list(fc.values()),
+                color=plt.cm.RdYlGn(list(fc.values())))
+        ax.set_xlim(0, 1); ax.set_title('Self Model (confidence)')
 
-    # 4. 预测 vs 真实（取第 0 维）
-    ax = axes[1, 1]
-    with torch.no_grad():
-        xs = test_xs.to(device)
-        preds_js, _ = jspace_model(xs)
-        preds_flat, _ = flat_model(xs)
-    true_seq = test_xs[0, 1:, 0].numpy()
-    pred_js = preds_js[0, :-1, 0].cpu().numpy()
-    pred_flat = preds_flat[0, :-1, 0].cpu().numpy()
+        # 世界模型 loss
+        ax = axes[1, 1]
+        ax.plot(steps, [s['world_loss'] for s in log], 'orange')
+        ax.set_title('World Model Loss'); ax.grid(True, alpha=0.3)
 
-    ax.plot(true_seq, label='True', linewidth=2, color='black')
-    ax.plot(pred_js, label='JSpace', alpha=0.8)
-    ax.plot(pred_flat, label='Flat', alpha=0.8)
-    ax.set_xlabel('Time step')
-    ax.set_ylabel('Value (dim 0)')
-    ax.set_title('Prediction vs True (dim 0)')
-    ax.legend()
-    ax.grid(True, alpha=0.3)
-    for i in range(1, 64 // 16):
-        ax.axvline(x=i * 16, color='gray', linestyle='--', alpha=0.3)
+        # 记忆数
+        ax = axes[1, 2]
+        ax.plot(steps, [s['memory_count'] for s in log], 'teal')
+        ax.set_title('Hippocampus Memory Count'); ax.grid(True, alpha=0.3)
 
-    plt.tight_layout()
-    fig_path = outdir / 'experiment.png'
-    plt.savefig(fig_path, dpi=150, bbox_inches='tight')
-    print(f"可视化已保存: {fig_path}")
-    plt.close()
-
-    # 保存模型
-    torch.save({
-        'jspace_model': jspace_model.state_dict(),
-        'flat_model': flat_model.state_dict(),
-        'config': config,
-        'eval_loss_js': eval_loss_js,
-        'eval_loss_flat': eval_loss_flat,
-    }, outdir / 'models.pt')
-
-    print()
-    print("=" * 60)
-    print("实验完成")
-    print("=" * 60)
-    print(f"JSpaceModel MSE:  {eval_loss_js:.6f}")
-    print(f"FlatBaseline MSE: {eval_loss_flat:.6f}")
-    print(f"胜者: {winner}")
-    print()
-    print("关键观察点（看 experiment.png）:")
-    print("  1. Loss 曲线: JSpace 是否收敛更快/更低？")
-    print("  2. 注意力热力图: 不同时段是否激活不同专家？")
-    print("  3. ||w|| 曲线: 模式切换时是否有明显尖峰？")
-    print("  4. 预测对比: 模式切换处哪个模型更鲁棒？")
+        plt.tight_layout()
+        Path('outputs').mkdir(exist_ok=True)
+        plt.savefig('outputs/embodied_live.png', dpi=150, bbox_inches='tight')
+        print(f"\n可视化: outputs/embodied_live.png")
 
 
 def main():
-    parser = argparse.ArgumentParser(description='JspaceAI 对比实验')
-    parser.add_argument('--steps', type=int, default=500,
-                        help='训练步数 (default: 500)')
-    parser.add_argument('--device', type=str, default='cpu',
-                        help='设备 (cpu / cuda / mps)')
-    parser.add_argument('--outdir', type=str, default='outputs',
-                        help='输出目录')
-    args = parser.parse_args()
+    p = argparse.ArgumentParser(description='JspaceAI 全部接入版（具身 + 自主心智）')
+    p.add_argument('--mode', default='test', choices=['test', 'live', 'safe'],
+                   help='运行模式: test=自检, live=实时循环, safe=安全模式')
+    p.add_argument('--steps', type=int, default=100, help='live/safe 模式步数')
+    p.add_argument('--device', default='cpu', help='设备 (cpu/cuda/mps/auto)')
+    p.add_argument('--unsafe', action='store_true',
+                   help='允许鼠标键盘输出（默认禁用）')
+    args = p.parse_args()
 
-    # 自动检测设备
-    if args.device == 'auto':
-        if torch.cuda.is_available():
-            device = 'cuda'
-        elif torch.backends.mps.is_available():
-            device = 'mps'
-        else:
-            device = 'cpu'
-    else:
-        device = args.device
+    dev = args.device
+    if dev == 'auto':
+        dev = 'cuda' if torch.cuda.is_available() else (
+            'mps' if torch.backends.mps.is_available() else 'cpu')
 
-    run_experiment(
-        n_steps=args.steps,
-        device=device,
-        outdir=Path(args.outdir),
-    )
+    if args.mode == 'test':
+        test_subsystems()
+    elif args.mode == 'live':
+        live(args.steps, dev, safe_mode=False, unsafe=args.unsafe)
+    elif args.mode == 'safe':
+        live(args.steps, dev, safe_mode=True, unsafe=False)
 
 
 if __name__ == '__main__':