借助 NVIDIA Omniverse NuRec 在车辆平台上扩展智能汽车感知

import os
from huggingface_hub import login, snapshot_download

login(token=os.getenv("HF_TOKEN"))

# Download one reconstructed scene (USDZ) from the 26.04 release
snapshot_download(
    repo_id="nvidia/PhysicalAI-Autonomous-Vehicles-NuRec",
	repo_type="dataset",
    allow_patterns="sample_set/26.04_release/<scene-uuid>/*",
)

git clone https://github.com/NVIDIA/nurec-skills.git

export SCENE_DIR=/absolute/path/to/scene
export SCENE_FILE=<SCENE_UUID>.usdz
export RIG_DIR=/absolute/path/to/rig
export RIG_FILE=minimal-synthetic-target-rig.json
export OUTPUT_DIR=/absolute/path/to/output
mkdir -p "$OUTPUT_DIR"

export NUREC_IMAGE='nvcr.io/nvidia/nre/nre-ga:26.04.01@sha256:97f43e7130c5636ce3e80ea3184d97f56a87fdd989b05cce42230881dbdea284'
docker pull "$NUREC_IMAGE"
docker image inspect "$NUREC_IMAGE" --format '{{range .RepoDigests}}{{println .}}{{end}}'

docker run --rm --gpus all --shm-size=64g \
  -v "$SCENE_DIR":/inputs/scene:ro \
  -v "$RIG_DIR":/inputs/rig:ro \
  -v "$OUTPUT_DIR":/outputs \
  "$NUREC_IMAGE" export-custom-rig-trajectory \
    --artifact-path="/inputs/scene/$SCENE_FILE" \
    --rig-json="/inputs/rig/$RIG_FILE" \
    --output=/outputs/custom_rig_trajectories.json

docker run --rm --gpus all --shm-size=64g \
  -v "$SCENE_DIR":/inputs/scene:ro \
  -v "$OUTPUT_DIR":/outputs \
  "$NUREC_IMAGE" render \
    --artifact-path="/inputs/scene/$SCENE_FILE" \
    --output-dir=/outputs/smoke \
    --custom-rig-trajectory=/outputs/custom_rig_trajectories.json \
    --no-replicate-training-views \
    --renderer=default \
    --image-format=png \
    --frame-step=30 \
    --image-scale=1 \
    --frame-naming=frame-end-timestamp \
    --camera-id=camera_front_synthetic_120fov

git clone https://github.com/NVIDIA/nurec-skills.git
python nurec-skills/skills/nurec-fixer/scripts/validate_setup.py

git clone https://github.com/NVIDIA/harmonizer.git
cd harmonizer
docker build -t harmonizer-cosmos-env -f Dockerfile.cosmos .
hf auth login
./download_checkpoints.sh
test -f models/diffusion_harmonizer.pkl
test -d src/checkpoints/nvidia/Cosmos-Predict2-0.6B-Text2Image

export HARMONIZER_DIR=/absolute/path/to/harmonizer
export RENDER_ROOT=/absolute/path/to/output/render
export CAMERA_ID=camera_front_synthetic_120fov
docker run --rm --gpus all --ipc=host \
 --entrypoint python \
 -v "$HARMONIZER_DIR":/work \
 -v "$RENDER_ROOT":/frames \
 -w /work/src \
 harmonizer-cosmos-env \
 inference_pix2pix_turbo_harmonizer.py \
 --input_image="/frames/$CAMERA_ID" \
 --model_path=/work/models/diffusion_harmonizer.pkl \
 --model_identifier=harmonized \
 --timestep=250 \
 --resolution=1024 \
 --use_sched

git clone https://github.com/NVIDIA/nurec-skills.git

clip/
└── raw/
   ├── generated/
   │   ├── front_wide.mp4
   │   ├── front_tele.mp4
   │   ├── cross_left.mp4
   │   ├── cross_right.mp4
   │   ├── rear_left.mp4
   │   ├── rear_right.mp4
   │   └── rear.mp4
   └── clipgt/parquets/
       ├── calibration_estimate.parquet
       ├── egomotion_estimate.parquet
       └── object_fused.parquet

export CONVERTER_DIR="$WORK_DIR/cosmos-clipgt-converter"
cp -R "$NUREC_SKILLS_REPO/skills/ncore/ncore_template" "$CONVERTER_DIR"

frame_timestamp_us = anchor_timestamp_us + i * 1,000,000 / fps

export COSMOS_CONVERTER="$TUTORIAL_REPO/tools/cosmos_ncore/build_cosmos_ncore_v4.py"
test -f "$COSMOS_CONVERTER"
env -u PYTHONPATH "$NCORE_PYTHON" "$COSMOS_CONVERTER" \
 --root "$INPUT_ROOT" \
 --anchor-us "$ANCHOR_TIMESTAMP_US" \
 --sequence-id "$SEQUENCE_ID" \
 --fps "$FPS" \
 --store-type itar

export NCORE_DIR="$INPUT_ROOT/ncore/$SEQUENCE_ID"
export NCORE_MANIFEST="$NCORE_DIR/$SEQUENCE_ID.json"
export NCORE_VALIDATOR="$TUTORIAL_REPO/tools/cosmos_ncore/validate_cosmos_ncore_v4.py"
test -s "$NCORE_MANIFEST"
python3 -m json.tool "$NCORE_MANIFEST" > /dev/null

env -u PYTHONPATH "$NCORE_PYTHON" "$NCORE_VALIDATOR" \
 --root "$INPUT_ROOT" \
 --sequence-id "$SEQUENCE_ID"

export NCORE_REPO=/absolute/path/to/ncore
export NCORE_MANIFEST=/absolute/path/to/sequence/<sequence-id>.json

  (
    cd "$NCORE_REPO"
    bazel run //tools/ncore_vis -- \
      v4 \
      --component-group="$NCORE_MANIFEST"
  )

<sequence-id>/
├── <sequence-id>.json
├── build_manifest.json
├── <sequence-id>.ncore4.zarr.itar
├── <sequence-id>.ncore4-<camera-id>.zarr.itar
└── ... one camera archive per converted camera

export NUREC_AUX_IMAGE=nvcr.io/nvidia/nre/nre-tools-ga:26.04.00
python3 "$NUREC_SKILLS_REPO/skills/nre/scripts/validate_setup.py" --strict
docker pull "$NUREC_AUX_IMAGE"
docker run --rm --gpus all --shm-size=2g \
 --env NGC_API_KEY \
--volume "$NCORE_DIR:/workdir/dataset" \
 "$NUREC_AUX_IMAGE" ncore-aux-data \
 --dataset-path="/workdir/dataset/$SEQUENCE_ID.json" \
 --output-dir=/workdir/dataset \
 --segmentation-backend=mask2former \
 --depth-backend=depthanythingv2 \
 --max-depth-m=80 \
 --ego-mask \
 --no-lidar-seg-camvis \
 --no-seg-logits \
 --zarr-store-type=itar \
 --store-meta \
 --num-threads=auto

<sequence-id>/
├── <sequence-id>.json
├── build_manifest.json
├── <sequence-id>.ncore4.zarr.itar
├── <sequence-id>.ncore4-<camera-id>.zarr.itar
└── ... one camera archive per converted camera
├── <sequence-id>.aux.sseg.zarr.itar
├── <sequence-id>.aux.depth.zarr.itar
├── <sequence-id>.aux.egomask.zarr.itar
└── <sequence-id>.aux-meta.json