{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# CLI Demo\n", "This notebook illustrates the basic usage of Vis4D. We run the provided Faster R-CNN toy config to run training and inference on COCO images." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Inference\n", "Run inference on the validation set." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[08/07 03:11:01 Vis4D]: \u001b[0mEnvironment info: PyTorch version: 2.7.1\n", "Is debug build: False\n", "CUDA used to build PyTorch: None\n", "ROCM used to build PyTorch: N/A\n", "\n", "OS: macOS 15.6 (arm64)\n", "GCC version: Could not collect\n", "Clang version: 17.0.0 (clang-1700.0.13.5)\n", "CMake version: Could not collect\n", "Libc version: N/A\n", "\n", "Python version: 3.12.9 | packaged by Anaconda, Inc. | (main, Feb 6 2025, 12:55:12) [Clang 14.0.6 ] (64-bit runtime)\n", "Python platform: macOS-15.6-arm64-arm-64bit\n", "Is CUDA available: False\n", "CUDA runtime version: No CUDA\n", "CUDA_MODULE_LOADING set to: N/A\n", "GPU models and configuration: No CUDA\n", "Nvidia driver version: No CUDA\n", "cuDNN version: No CUDA\n", "HIP runtime version: N/A\n", "MIOpen runtime version: N/A\n", "Is XNNPACK available: True\n", "\n", "CPU:\n", "Apple M4\n", "\n", "Versions of relevant libraries:\n", "[pip3] mypy_extensions==1.1.0\n", "[pip3] numpy==1.26.4\n", "[pip3] pytorch-lightning==2.5.2\n", "[pip3] torch==2.7.1\n", "[pip3] torchmetrics==1.7.4\n", "[pip3] torchvision==0.22.1\n", "[conda] numpy 1.26.4 pypi_0 pypi\n", "[conda] pytorch-lightning 2.5.2 pypi_0 pypi\n", "[conda] torch 2.7.1 pypi_0 pypi\n", "[conda] torchmetrics 1.7.4 pypi_0 pypi\n", "[conda] torchvision 0.22.1 pypi_0 pypi\n", "GPU available: True (mps), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/lightning/pytorch/trainer/setup.py:177: GPU available but not used. You can set it by doing `Trainer(accelerator='gpu')`.\n", "\u001b[32m[08/07 03:11:01 Vis4D]: \u001b[0mLoad checkpoint from http path: https://download.pytorch.org/models/resnet50-0676ba61.pth\n", "Downloading: \"https://download.pytorch.org/models/resnet50-0676ba61.pth\" to /var/folders/xd/tqr32hx17vz95rkzbx76ngsh0000gn/T/tmpok5awk2d/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth\n", "100%|███████████████████████████████████████| 97.8M/97.8M [00:00<00:00, 166MB/s]\n", "/Users/royyang/Workspace/vis4d/vis4d/common/ckpt.py:377: UserWarning: The model and loaded state dict do not match exactly\n", "\n", "unexpected key in source state_dict: fc.weight, fc.bias\n", "\n", " rank_zero_warn(err_msg)\n", "\u001b[32m[08/07 03:11:02 Vis4D]: \u001b[0mLoad checkpoint from http path: https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n", "Downloading: \"https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\" to /var/folders/xd/tqr32hx17vz95rkzbx76ngsh0000gn/T/tmpok5awk2d/.cache/torch/hub/checkpoints/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n", "100%|████████████████████████████████████████| 160M/160M [00:04<00:00, 34.1MB/s]\n", "\u001b[32m[08/07 03:11:07 Vis4D]: \u001b[0mGenerating COCODataset(root=data/coco_test, split=train, use_pascal_voc_cats=False) data mapping...\n", "\u001b[32m[08/07 03:11:07 Vis4D]: \u001b[0mLoading COCODataset(root=data/coco_test, split=train, use_pascal_voc_cats=False) takes 0.00 seconds.\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'test_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.\n", "\u001b[32m[08/07 03:11:08 Vis4D]: \u001b[0mTesting: 1/2, ETA: 0:00:01, 1.14s/it\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mTesting: 2/2, ETA: 0:00:00, 1.01s/it\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mRunning evaluator CocoEvaluator(annotation_path=data/coco_test/annotations/instances_train.json) with Det metric... \n", "Loading and preparing results...\n", "DONE (t=0.00s)\n", "creating index...\n", "index created!\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mDet/AP: 0.3974\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mDet/AP50: 0.5497\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mDet/AP75: 0.4414\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mDet/APs: 0.2868\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mDet/APm: 0.6932\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mDet/APl: 0.6000\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0mShowing results for metric: Det\n", "\u001b[32m[08/07 03:11:09 Vis4D]: \u001b[0m\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.397\n", " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.550\n", " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.441\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.287\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.693\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.600\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.278\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.429\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.454\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.348\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.693\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.633\n", "\n" ] } ], "source": [ "!vis4d test --config faster_rcnn_example.py --config.params.num_epochs 1" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Training\n", "Run the training for 1 epoch and inference on the validation set." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m[08/07 03:11:20 Vis4D]: \u001b[0mEnvironment info: PyTorch version: 2.7.1\n", "Is debug build: False\n", "CUDA used to build PyTorch: None\n", "ROCM used to build PyTorch: N/A\n", "\n", "OS: macOS 15.6 (arm64)\n", "GCC version: Could not collect\n", "Clang version: 17.0.0 (clang-1700.0.13.5)\n", "CMake version: Could not collect\n", "Libc version: N/A\n", "\n", "Python version: 3.12.9 | packaged by Anaconda, Inc. | (main, Feb 6 2025, 12:55:12) [Clang 14.0.6 ] (64-bit runtime)\n", "Python platform: macOS-15.6-arm64-arm-64bit\n", "Is CUDA available: False\n", "CUDA runtime version: No CUDA\n", "CUDA_MODULE_LOADING set to: N/A\n", "GPU models and configuration: No CUDA\n", "Nvidia driver version: No CUDA\n", "cuDNN version: No CUDA\n", "HIP runtime version: N/A\n", "MIOpen runtime version: N/A\n", "Is XNNPACK available: True\n", "\n", "CPU:\n", "Apple M4\n", "\n", "Versions of relevant libraries:\n", "[pip3] mypy_extensions==1.1.0\n", "[pip3] numpy==1.26.4\n", "[pip3] pytorch-lightning==2.5.2\n", "[pip3] torch==2.7.1\n", "[pip3] torchmetrics==1.7.4\n", "[pip3] torchvision==0.22.1\n", "[conda] numpy 1.26.4 pypi_0 pypi\n", "[conda] pytorch-lightning 2.5.2 pypi_0 pypi\n", "[conda] torch 2.7.1 pypi_0 pypi\n", "[conda] torchmetrics 1.7.4 pypi_0 pypi\n", "[conda] torchvision 0.22.1 pypi_0 pypi\n", "GPU available: True (mps), used: False\n", "TPU available: False, using: 0 TPU cores\n", "HPU available: False, using: 0 HPUs\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/lightning/pytorch/trainer/setup.py:177: GPU available but not used. You can set it by doing `Trainer(accelerator='gpu')`.\n", "Seed set to 798931529\n", "\u001b[32m[08/07 03:11:20 Vis4D]: \u001b[0mGlobal seed set to 798931529\n", "\u001b[32m[08/07 03:11:20 Vis4D]: \u001b[0mLoad checkpoint from http path: https://download.pytorch.org/models/resnet50-0676ba61.pth\n", "Downloading: \"https://download.pytorch.org/models/resnet50-0676ba61.pth\" to /var/folders/xd/tqr32hx17vz95rkzbx76ngsh0000gn/T/tmpx4ximh4c/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth\n", "100%|███████████████████████████████████████| 97.8M/97.8M [00:00<00:00, 164MB/s]\n", "/Users/royyang/Workspace/vis4d/vis4d/common/ckpt.py:377: UserWarning: The model and loaded state dict do not match exactly\n", "\n", "unexpected key in source state_dict: fc.weight, fc.bias\n", "\n", " rank_zero_warn(err_msg)\n", "\u001b[32m[08/07 03:11:21 Vis4D]: \u001b[0mLoad checkpoint from http path: https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n", "Downloading: \"https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\" to /var/folders/xd/tqr32hx17vz95rkzbx76ngsh0000gn/T/tmpx4ximh4c/.cache/torch/hub/checkpoints/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n", "100%|████████████████████████████████████████| 160M/160M [00:04<00:00, 33.8MB/s]\n", "\u001b[32m[08/07 03:11:26 Vis4D]: \u001b[0mGenerating COCODataset(root=data/coco_test, split=train, use_pascal_voc_cats=False) data mapping...\n", "\u001b[32m[08/07 03:11:26 Vis4D]: \u001b[0mLoading COCODataset(root=data/coco_test, split=train, use_pascal_voc_cats=False) takes 0.00 seconds.\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/torch/utils/data/dataloader.py:683: UserWarning: 'pin_memory' argument is set as true but not supported on MPS now, then device pinned memory won't be used.\n", " warnings.warn(warn_msg)\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/lightning/pytorch/loops/fit_loop.py:310: The number of training batches (2) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n", "\n", " | Name | Type | Params | Mode \n", "---------------------------------------------------\n", "0 | loss_module | LossModule | 0 | train\n", "1 | model | FasterRCNN | 41.8 M | train\n", "---------------------------------------------------\n", "41.5 M Trainable params\n", "225 K Non-trainable params\n", "41.8 M Total params\n", "167.012 Total estimated model params size (MB)\n", "161 Modules in train mode\n", "29 Modules in eval mode\n", "\u001b[32m[08/07 03:11:26 Vis4D]: \u001b[0mGenerating COCODataset(root=data/coco_test, split=train, use_pascal_voc_cats=False) data mapping...\n", "\u001b[32m[08/07 03:11:26 Vis4D]: \u001b[0mLoading COCODataset(root=data/coco_test, split=train, use_pascal_voc_cats=False) takes 0.00 seconds.\n", "/Users/royyang/anaconda3/envs/vis4d/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=9` in the `DataLoader` to improve performance.\n", "\u001b[32m[08/07 03:11:28 Vis4D]: \u001b[0mEpoch 1: 1/2, ETA: 0:00:02, 2.72s/it, loss: 1.4686, RPNLoss.loss_cls: 0.4833, RPNLoss.loss_bbox: 0.2011, RCNNLoss.rcnn_loss_cls: 0.5931, RCNNLoss.rcnn_loss_bbox: 0.1911\n", "\u001b[32m[08/07 03:11:31 Vis4D]: \u001b[0mEpoch 1: 2/2, ETA: 0:00:00, 2.65s/it, loss: 1.5583, RPNLoss.loss_cls: 0.1662, RPNLoss.loss_bbox: 0.0978, RCNNLoss.rcnn_loss_cls: 0.8646, RCNNLoss.rcnn_loss_bbox: 0.4296\n", "\u001b[32m[08/07 03:11:32 Vis4D]: \u001b[0mValidation: 1/2, ETA: 0:00:00, 1.26it/s\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mValidation: 2/2, ETA: 0:00:00, 1.26it/s\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mRunning evaluator CocoEvaluator(annotation_path=data/coco_test/annotations/instances_train.json) with Det metric... \n", "Loading and preparing results...\n", "DONE (t=0.00s)\n", "creating index...\n", "index created!\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mDet/AP: 0.3194\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mDet/AP50: 0.4682\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mDet/AP75: 0.3778\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mDet/APs: 0.2240\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mDet/APm: 0.5861\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mDet/APl: 0.3133\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0mShowing results for metric: Det\n", "\u001b[32m[08/07 03:11:33 Vis4D]: \u001b[0m\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.319\n", " Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.468\n", " Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.378\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.224\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.586\n", " Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.313\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.162\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.396\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.396\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.315\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.621\n", " Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.367\n", "\n", "`Trainer.fit` stopped: `max_epochs=1` reached.\n" ] } ], "source": [ "!vis4d fit --config faster_rcnn_example.py --config.params.num_epochs 1" ] } ], "metadata": { "kernelspec": { "display_name": "vis4d", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 4 }