Skip to content

Commit 4800d3c

Browse files
cjaverliatcjaverliat
committed
Merge branch 'master' of https://github.com/liris-xr/kineo 
2 parents ab1dcf0 + 993509c commit 4800d3c

27 files changed

Lines changed: 3411 additions & 438 deletions

‎.claude/settings.local.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"permissions": {
3+
"allow": [
4+
"Bash(find:*)",
5+
"Bash(python:*)"
6+
]
7+
}
8+
}

‎configs/demo/offline/dwpose_single_person_pairwise_calib_export.yaml‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
output_root_dir: "${oc.env:OUTPUT_ROOT_DIR,./outputs}/infer_dwpose_single_person_pairwise_calib"
2+
cache_root_dir: "${oc.env:CACHE_ROOT_DIR,./cache}/infer_dwpose_single_person_pairwise_calib"
3+
use_cache: False
4+
batch_size: 32
5+
use_half_precision: True
6+
shared_intrinsics: True
7+
# Don't use faces and hands in the pipeline (body only), except when triangulating the final result.
8+
# This is because hands and faces are often noisy, yet their score is anormaly high giving false positives.
9+
keypoints_indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
10+
11+
rtmlib_bbox_keypoints_detection_frame_step: 1
12+
13+
pipeline:
14+
seed: 19
15+
16+
stages:
17+
rtmlib_bbox_keypoints_detection:
18+
_target_: kineo.pipeline.stages.rtmlib.rtmlib_bbox_keypoints_detection.RtmlibBboxKeypointsDetectionStage
19+
name: "Rtmlib Bbox Keypoints Detection"
20+
order: 5
21+
keypoints_model: "https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/rtmpose-l_simcc-ucoco_dw-ucoco_270e-384x288-2438fd99_20230728.zip"
22+
keypoints_model_input_shape_hw: [288, 384]
23+
runtime_cfg:
24+
_target_: kineo.pipeline.stages.rtmlib.rtmlib_bbox_keypoints_detection.RtmlibBboxKeypointsDetectionRuntimeConfig
25+
use_cache: ${use_cache}
26+
cache_output_path_template: "${cache_root_dir}/{sequence_name}/{annotation_key}.pkl"
27+
show: False
28+
frame_step: ${rtmlib_bbox_keypoints_detection_frame_step}
29+
30+
# Infer intrinsics before NLF to provide it to the model
31+
moge_intrinsics_estimation:
32+
_target_: kineo.pipeline.stages.moge_intrinsics_estimation.MoGeIntrinsicsEstimationStage
33+
name: "MoGe Intrinsics Estimation"
34+
model_name_or_path: "Ruicheng/moge-2-vitl"
35+
order: 20
36+
runtime_cfg:
37+
_target_: kineo.pipeline.stages.moge_intrinsics_estimation.MoGeIntrinsicsEstimationRuntimeConfig
38+
use_cache: ${use_cache}
39+
use_half_precision: ${use_half_precision}
40+
cache_output_path_template: "${cache_root_dir}/{sequence_name}/{annotation_key}.pkl"
41+
42+
global_time_resampling:
43+
_target_: kineo.pipeline.stages.global_time_resampling.GlobalTimeResamplingStage
44+
name: "Global Time Resampling"
45+
order: 30
46+
runtime_cfg:
47+
_target_: kineo.pipeline.stages.global_time_resampling.GlobalTimeResamplingRuntimeConfig
48+
target_fps: 50
49+
interpolation_dt_thr_s: 0.3
50+
51+
keypoints_pairs_sampling:
52+
_target_: kineo.pipeline.stages.keypoints_pairs_sampling.KeypointsPairsSamplingStage
53+
name: "Keypoints Pairs Sampling"
54+
order: 40
55+
runtime_cfg:
56+
_target_: kineo.pipeline.stages.keypoints_pairs_sampling.KeypointsPairsSamplingRuntimeConfig
57+
max_points_pairs: 4000
58+
pair_avg_conf_score_thr: 0.5
59+
keypoints_indices: ${keypoints_indices}
60+
61+
# calibration_points_video_export:
62+
# _target_: kineo.pipeline.stages.calibration_points_video_export.CalibrationPointsVideoExportStage
63+
# name: "Export calibration points video"
64+
# order: 45
65+
# runtime_cfg:
66+
# _target_: kineo.pipeline.stages.calibration_points_video_export.CalibrationPointsVideoExportRuntimeConfig
67+
# view1_id: "view_0"
68+
# view2_id: "view_1"
69+
# point_radius: 5
70+
# line_thickness: 1
71+
# show: False
72+
# keypoints_indices: ${keypoints_indices}
73+
74+
sfm_camera_extrinsics_initialization:
75+
_target_: kineo.pipeline.stages.sfm_camera_extrinsics_initialization.SfMCameraExtrinsicsInitializationStage
76+
name: "SfM Camera Extrinsics Initialization"
77+
order: 50
78+
runtime_cfg:
79+
_target_: kineo.pipeline.stages.sfm_camera_extrinsics_initialization.SfMCameraExtrinsicsInitializationRuntimeConfig
80+
ransac_confidence: 0.99
81+
ransac_reproj_threshold: 3.0
82+
n_refinement_iters: 200
83+
tolerance_grad: 1e-05
84+
tolerance_change: 1e-09
85+
86+
bundle_adjustment_sampling:
87+
_target_: kineo.pipeline.stages.bundle_adjustment_sampling.BundleAdjustmentSamplingStage
88+
name: "Bundle Adjustment Sampling"
89+
order: 55
90+
runtime_cfg:
91+
_target_: kineo.pipeline.stages.bundle_adjustment_sampling.BundleAdjustmentSamplingRuntimeConfig
92+
n_kp_samples: 400
93+
min_kp_score: 0.5
94+
95+
bundle_adjustment_1: # Only optimize camera extrinsics
96+
_target_: kineo.pipeline.stages.bundle_adjustment.BundleAdjustmentStage
97+
name: "Bundle Adjustment (First Pass)"
98+
order: 60
99+
runtime_cfg:
100+
_target_: kineo.pipeline.stages.bundle_adjustment.BundleAdjustmentRuntimeConfig
101+
n_iters: 300
102+
use_lbfgs: True
103+
optimize_distortion_coefficients: False
104+
optimize_focal_length: False
105+
optimize_principal_point: False
106+
optimize_rotation: True
107+
optimize_translation: True
108+
shared_intrinsics: ${shared_intrinsics}
109+
huber_delta: 1.0
110+
tolerance_grad: 1e-05
111+
tolerance_change: 1e-09
112+
dist_coeffs_regularization_weight: 0.0
113+
114+
bundle_adjustment_2:
115+
_target_: kineo.pipeline.stages.bundle_adjustment.BundleAdjustmentStage
116+
name: "Bundle Adjustment (Second Pass)"
117+
order: 70
118+
runtime_cfg:
119+
_target_: kineo.pipeline.stages.bundle_adjustment.BundleAdjustmentRuntimeConfig
120+
n_iters: 300
121+
use_lbfgs: True
122+
optimize_distortion_coefficients: False
123+
optimize_focal_length: True
124+
optimize_principal_point: False
125+
optimize_rotation: True
126+
optimize_translation: True
127+
shared_intrinsics: ${shared_intrinsics}
128+
huber_delta: 1.0
129+
tolerance_grad: 1e-05
130+
tolerance_change: 1e-09
131+
dist_coeffs_regularization_weight: 0.0
132+
133+
bundle_adjustment_3:
134+
_target_: kineo.pipeline.stages.bundle_adjustment.BundleAdjustmentStage
135+
name: "Bundle Adjustment (Third Pass)"
136+
order: 80
137+
runtime_cfg:
138+
_target_: kineo.pipeline.stages.bundle_adjustment.BundleAdjustmentRuntimeConfig
139+
n_iters: 300
140+
optimize_distortion_coefficients: True
141+
optimize_focal_length: True
142+
optimize_principal_point: False
143+
optimize_rotation: True
144+
optimize_translation: True
145+
shared_intrinsics: ${shared_intrinsics}
146+
huber_delta: 1.0
147+
dist_coeffs_regularization_weight: 1.0
148+
use_lbfgs: True
149+
150+
mvs_triangulation:
151+
_target_: kineo.pipeline.stages.mvs_triangulation.MVSTriangulationStage
152+
name: "MVS Triangulation"
153+
order: 90
154+
runtime_cfg:
155+
_target_: kineo.pipeline.stages.mvs_triangulation.MVSTriangulationRuntimeConfig
156+
use_eigendecomposition: True
157+
158+
moge_global_scale_estimation:
159+
_target_: kineo.pipeline.stages.moge_global_scale_estimation.MoGeGlobalScaleEstimationStage
160+
name: "MoGe Global Scale Estimation"
161+
model_name_or_path: "Ruicheng/moge-2-vitl"
162+
order: 100
163+
runtime_cfg:
164+
_target_: kineo.pipeline.stages.moge_global_scale_estimation.MoGeGlobalScaleEstimationRuntimeConfig
165+
use_half_precision: ${use_half_precision}
166+
keypoints_indices: ${keypoints_indices}
167+
168+
global_scale_application:
169+
_target_: kineo.pipeline.stages.global_scale_application.GlobalScaleApplicationStage
170+
name: "Global Scale Application"
171+
order: 110
172+
173+
scene_reorientation:
174+
_target_: kineo.pipeline.stages.scene_reorientation.SceneReorientationStage
175+
name: "Scene Reorientation"
176+
order: 120
177+
runtime_cfg:
178+
_target_: kineo.pipeline.stages.scene_reorientation.SceneReorientationRuntimeConfig
179+
min_kps_quantile: 0.01
180+
181+
# bundle_adjustment_history_rerun_export:
182+
# _target_: kineo.pipeline.stages.bundle_adjustment_history_rerun_export.BundleAdjustmentHistoryRerunExportStage
183+
# name: "Bundle Adjustment History Rerun Export"
184+
# order: 121
185+
# runtime_cfg:
186+
# _target_: kineo.pipeline.stages.bundle_adjustment_history_rerun_export.BundleAdjustmentHistoryRerunExportRuntimeConfig
187+
# output_path_template: "./outputs/rerun/{sequence_name}_ba_history.rrd"
188+
189+
background_subtraction:
190+
_target_: kineo.pipeline.stages.background_subtraction.BackgroundSubtractionStage
191+
name: "Background Subtraction"
192+
order: 125
193+
runtime_cfg:
194+
_target_: kineo.pipeline.stages.background_subtraction.BackgroundSubtractionRuntimeConfig
195+
run_skyseg: False
196+
197+
vggt_scene_reconstruction:
198+
_target_: kineo.pipeline.stages.vggt_scene_reconstruction.VGGTSceneReconstructionStage
199+
name: "Scene Reconstruction"
200+
order: 130
201+
model_name_or_path: "./checkpoints/vggt-1b.pt"
202+
runtime_cfg:
203+
_target_: kineo.pipeline.stages.vggt_scene_reconstruction.VGGTSceneReconstructionRuntimeConfig
204+
205+
rerun_export:
206+
_target_: kineo.pipeline.stages.rerun_export.RerunExportStage
207+
name: "Rerun Export"
208+
order: 160
209+
runtime_cfg:
210+
_target_: kineo.pipeline.stages.rerun_export.RerunExportRuntimeConfig
211+
output_path_template: "${output_root_dir}/offline_demo/{sequence_name}.rrd"
212+
max_world_points_to_show: 1000000
213+
world_points_confidence_threshold: 0.5
214+
world_z_clipping_threshold: 1.7
215+
remove_world_points_outside_scene_radius: True
216+
217+
keypoints_radius_2d: 1
218+
keypoints_radius_3d: 0.02
219+
220+
skeleton_bones_thickness_3d: 0.010
221+
skeleton_joint_radius_3d: 0.020
222+
223+
skeleton_bones_thickness_2d: 1.0
224+
skeleton_joint_radius_2d: 2.5
225+
226+
min_kps_score_3d: 0.2
227+
min_kps_score_2d: 0.2
228+
229+
video_quality: 25
230+
image_plane_distance: 0.2
231+
232+
log_disconnected_joints: False
233+
234+
log_videos: True
235+
236+
log_world_reconstruction: True
237+
log_pred_cameras: True
238+
log_pred_keypoints_2d: True
239+
log_pred_keypoints_3d: True
240+
log_pred_skeletons_2d: True
241+
log_pred_skeletons_3d: True
242+
log_pred_smpl: True
243+
log_pred_smpl_skeleton_2d: True
244+
245+
log_gt_keypoints_2d: False
246+
log_gt_keypoints_3d: False
247+
log_gt_skeletons_2d: False
248+
log_gt_skeletons_3d: False
249+
log_gt_cameras: False
250+
251+
annotations_export:
252+
_target_: kineo.pipeline.stages.annotations_export.AnnotationsExportStage
253+
name: "Annotations Export"
254+
order: 170
255+
runtime_cfg:
256+
_target_: kineo.pipeline.stages.annotations_export.AnnotationsExportRuntimeConfig
257+
annotations_keys: "*"
258+
excluded_annotations_keys: []
259+
not_found_error: False
260+
output_path_template: "${output_root_dir}/offline_demo/annotations/{sequence_name}/{annotation_key}.pkl"

‎configs/demo/offline/dwpose_single_person_sam2.yaml‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ pipeline:
5252
_target_: kineo.pipeline.stages.rtmlib.rtmlib_keypoints_detection.RtmlibKeypointsDetectionRuntimeConfig
5353
use_cache: ${use_cache}
5454
cache_output_path_template: "${cache_root_dir}/{sequence_name}/{annotation_key}.pkl"
55-
show: True
55+
show: False
5656
frame_step: ${rtmlib_keypoints_detection_frame_step}
5757

5858
global_time_resampling:
@@ -181,6 +181,14 @@ pipeline:
181181
_target_: kineo.pipeline.stages.scene_reorientation.SceneReorientationRuntimeConfig
182182
min_kps_quantile: 0.01
183183

184+
bundle_adjustment_history_rerun_export:
185+
_target_: kineo.pipeline.stages.bundle_adjustment_history_rerun_export.BundleAdjustmentHistoryRerunExportStage
186+
name: "Bundle Adjustment History Rerun Export"
187+
order: 121
188+
runtime_cfg:
189+
_target_: kineo.pipeline.stages.bundle_adjustment_history_rerun_export.BundleAdjustmentHistoryRerunExportRuntimeConfig
190+
output_path_template: "./outputs/rerun/{sequence_name}_ba_history.rrd"
191+
184192
background_subtraction:
185193
_target_: kineo.pipeline.stages.background_subtraction.BackgroundSubtractionStage
186194
name: "Background Subtraction"
@@ -209,35 +217,36 @@ pipeline:
209217
output_path_template: "${output_root_dir}/offline_demo/{sequence_name}.rrd"
210218
max_world_points_to_show: 1000000
211219
world_points_confidence_threshold: 0.5
212-
world_z_clipping_threshold: 1.5
220+
world_z_clipping_threshold: 2.0
213221
remove_world_points_outside_scene_radius: True
214-
world_translation: [0.0, 0.0, -0.1]
215222

216-
start_frame_idx: 1164
223+
keypoints_radius_2d: 1
224+
keypoints_radius_3d: 0.02
225+
226+
skeleton_bones_thickness_3d: 0.010
227+
skeleton_joint_radius_3d: 0.020
217228

218-
skeleton_joint_radius_3d: 0.010
219-
skeleton_joint_radius_2d: 6.0
220-
skeleton_bones_thickness_3d: 0.005
221-
skeleton_bones_thickness_2d: 3.0
229+
skeleton_bones_thickness_2d: 1.0
230+
skeleton_joint_radius_2d: 2.5
222231

223-
min_kps_score_3d: 0.15
224-
min_kps_score_2d: 0.15
232+
min_kps_score_3d: 0.2
233+
min_kps_score_2d: 0.2
225234

226235
video_quality: 25
227236
image_plane_distance: 0.2
228237

229-
log_disconnected_joints: True
238+
log_disconnected_joints: False
230239

231240
log_videos: True
232-
video_view_ids: "view_1"
233241

234242
log_world_reconstruction: True
235243
log_pred_cameras: True
236-
log_pred_keypoints_2d: False
237-
log_pred_keypoints_3d: False
244+
log_pred_keypoints_2d: True
245+
log_pred_keypoints_3d: True
238246
log_pred_skeletons_2d: True
239247
log_pred_skeletons_3d: True
240-
log_pred_smpl: False
248+
log_pred_smpl: True
249+
log_pred_smpl_skeleton_2d: True
241250

242251
log_gt_keypoints_2d: False
243252
log_gt_keypoints_3d: False

0 commit comments

Comments
 (0)