使用 NVIDIA Alpamayo 2 Super 生成轨迹、推理路径和自动标注

from alpamayo2_super import helper
from alpamayo2_super.load_physical_aiavdataset import load_physical_aiavdataset
from alpamayo2_super.models.alpamayo2_super import Alpamayo2Super
from alpamayo2_super.visualization import plot_inference_result

data = load_physical_aiavdataset(
  "030c760c-ae38-49aa-9ad8-f5650a545d26",
   t0_us=2000000,
)

model = Alpamayo2Super.from_pretrained("nvidia/Alpamayo2-Super", dtype=torch.bfloat16, device_map="cuda:0")
model_inputs = helper.prepare_model_inputs(data, model.config, model.tokenizer)
model_inputs = helper.to_device(model_inputs, "cuda")

torch.cuda.manual_seed_all(42)
with torch.autocast("cuda", dtype=torch.bfloat16):
   pred_xyz, pred_rot, logprob, extra = model.sample_trajectories_from_data(
       data=model_inputs,
       top_p=0.98,
       temperature=0.6,
       num_traj_samples=1,
       diffusion_kwargs={"inference_step": 10},
       return_extra=True,
   )

fig, metadata = plot_inference_result(
   data=data,
   pred_xyz=pred_xyz,
   extra=extra,
)

uv run alpasim_wizard deploy=local topology=2gpu driver=alpamayo2 
wizard.log_dir=$PWD/tutorial eval.video.video_layouts=[REASONING_OVERLAY]

# Should be used in defaults list, e.g.
# - /driver: alpamayo2
# Type validation happens at driver runtime via OmegaConf.structured merge

defaults:
  - alpamayo_configs    # Camera and simulation configs for 4-cam 10Hz
  - _self_                      # YAML values override schema defaults

# Alpamayo 2 Super Driver Configuration for Alpasim

# Logging level (uses wizard's global setting)
log_level: ${wizard.log_level}

# Model configuration
model:
  model_type: alpamayo2  # Entry-point name in alpasim.models registry
  # HuggingFace model ID (requires cached download or hf authentication in the driver container)
  checkpoint_path: "nvidia/Alpamayo2-Super"
  # # Alternative local path to a pre-downloaded model
  # checkpoint_path: "/mnt/drivers/alpamayo2/Alpamayo2-Super"
  device: "cuda"
  # Enable classifier-free guidance navigation sampling (NOTE: this requires 2 GPUs with at least 70 GB VRAM).
  # Set to true only when sufficient GPU memory is available.
  use_classifier_free_guidance_nav: false

# Server configuration
host: "0.0.0.0"
port: ???

# Inference configuration
inference:
  use_cameras:
     - camera_cross_left_120fov
     - camera_front_wide_120fov
     - camera_cross_right_120fov
     - camera_front_tele_30fov
  max_batch_size: 1  # A2Super is memory intensive, start with batch size 1
  subsample_factor: 1
  context_length: 4  # A2Super uses 4 temporal frames per camera

# Route configuration — A2Super uses language-only navigation, not waypoint commands
route:
  use_waypoint_commands: false

# Output configuration
output_dir: "/mnt/output/driver"

# Trajectory optimization (disabled by default for A2Super)
trajectory_optimizer:
  enabled: false

plot_debug_images: false

from alpamayo2_super import helper
from alpamayo2_super.load_physical_aiavdataset import load_physical_aiavdataset
from alpamayo2_super.models.alpamayo2_super import Alpamayo2Super
from alpamayo2_super.text_tasks import generate_text, prepare_text_generation_inputs

data = load_physical_aiavdataset(
  "030c760c-ae38-49aa-9ad8-f5650a545d26",
   t0_us=2000000,
)

model = Alpamayo2Super.from_pretrained("nvidia/Alpamayo2-Super", dtype=torch.bfloat16, device_map="cuda:0")
task_inputs = prepare_text_generation_inputs(
   data=data,
   model_config=model.config,
   tokenizer=model.tokenizer,
   task="meta_action",
)
task_inputs = helper.to_device(task_inputs, "cuda")

torch.cuda.manual_seed_all(42)
with torch.autocast("cuda", dtype=torch.bfloat16):
   result = generate_text(
       model,
       task_inputs,
       top_p=0.98,
       temperature=0.6,
       max_new_tokens=512,
   )

cot = result["cot"][0]
meta_action = result["meta_action"][0]
print("Chain-of-Causation:\n", cot)
print("\nMeta-action:\n", meta_action)

from alpamayo2_super import helper
from alpamayo2_super.load_physical_aiavdataset import load_physical_aiavdataset
from alpamayo2_super.models.alpamayo2_super import Alpamayo2Super
from alpamayo2_super.text_tasks import generate_text, prepare_text_generation_inputs

data = load_physical_aiavdataset(
  "ea4a6729-bf33-4997-905b-cd58774a3580",
   t0_us=7500000,
)

model = Alpamayo2Super.from_pretrained("nvidia/Alpamayo2-Super", dtype=torch.bfloat16, device_map="cuda:0")
task_inputs = prepare_vqa_inputs(
    data=data,
    model_config=model.config,
    tokenizer=model.tokenizer,
    question="Describe the driving scene and identify the key traffic elements that should influence ego behavior.",
)
task_inputs = helper.to_device(task_inputs, "cuda")
with torch.autocast("cuda", dtype=torch.bfloat16):
    result = generate_text(
        model,
        task_inputs,
        top_p=1.0,
        temperature=0.1,
        max_new_tokens=1024,
    )
answer = result["answer"][0]
print(answer)

from alpamayo2_super import helper
from alpamayo2_super.load_physical_aiavdataset import load_physical_aiavdataset
from alpamayo2_super.models.alpamayo2_super import Alpamayo2Super
from alpamayo2_super.text_tasks import generate_text, prepare_text_generation_inputs

data = load_physical_aiavdataset(
  "b5f3756c-4f0e-4298-a1ff-cc92ed392ae0",
   t0_us=11000000,
)

model = Alpamayo2Super.from_pretrained("nvidia/Alpamayo2-Super", dtype=torch.bfloat16, device_map="cuda:0")

task_inputs = prepare_text_generation_inputs(
   data=data,
   model_config=model.config,
   tokenizer=model.tokenizer,
   task="auto_labeling",
)
task_inputs = helper.to_device(task_inputs, "cuda")

torch.cuda.manual_seed_all(42)
with torch.autocast("cuda", dtype=torch.bfloat16):
   result = generate_text(
       model,
       task_inputs,
       top_p=0.98,
       temperature=0.6,
       max_new_tokens=1024,
   )

auto_labeling_text = result["cot_auto_labeling"][0]
auto_labeling_json = result["cot_auto_labeling_json"][0]
print(json.dumps(auto_labeling_json, indent=2))

# ... same imports and model loading as above ...

# Get the model to predict a future trajectory of its own, and use that in auto-labeling as the "observed" future motion.
trajectory_inputs = helper.prepare_model_inputs(data, model.config, model.tokenizer)
trajectory_inputs = helper.to_device(trajectory_inputs, "cuda")

torch.cuda.manual_seed_all(42)
with torch.autocast("cuda", dtype=torch.bfloat16):
   pred_xyz, pred_rot, _, extra = model.sample_trajectories_from_data(
        data=trajectory_inputs,
        top_p=0.98,
        temperature=0.6,
        num_traj_samples=1,
        diffusion_kwargs={"inference_step": 10},
        return_extra=True,
   )
   future_xyz = pred_xyz[:, 0, 0].detach().cpu()
   future_rot = pred_rot[:, 0, 0].detach().cpu()
   # In case you want to see the model's reasoning, uncomment these:
   # trajectory_cot = str(extra["cot"].reshape(-1)[0])
   # print("trajectory_cot:\n", trajectory_cot)

task_inputs = prepare_text_generation_inputs(
   data=data,
   model_config=model.config,
   tokenizer=model.tokenizer,
   task="auto_labeling",
   # This is where the predicted trajectories are passed in: 
   future_xyz=future_xyz,
   future_rot=future_rot,
)
task_inputs = helper.to_device(task_inputs, "cuda")

torch.cuda.manual_seed_all(42)
with torch.autocast("cuda", dtype=torch.bfloat16):
   result = generate_text(
       model,
       task_inputs,
       top_p=0.98,
       temperature=0.6,
       max_new_tokens=1024,
   )

auto_labeling_text = result["cot_auto_labeling"][0]
auto_labeling_json = result["cot_auto_labeling_json"][0]
print(json.dumps(auto_labeling_json, indent=2))