From 2b7243d1be5add574fa20fb67ccd37a342b7c3ee Mon Sep 17 00:00:00 2001 From: hofee <64160135+GitHofee@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:17:54 +0800 Subject: [PATCH 1/6] upd infernce --- configs/local/inference_config.yaml | 20 +++---- core/seq_dataset.py | 34 +++++++++--- core/seq_dataset_preprocessed.py | 85 +++++++++++++++++++++++++++++ runners/inferencer.py | 58 +++++++++++++++----- utils/reconstruction.py | 8 ++- utils/render.py | 28 +++++++--- 6 files changed, 191 insertions(+), 42 deletions(-) create mode 100644 core/seq_dataset_preprocessed.py diff --git a/configs/local/inference_config.yaml b/configs/local/inference_config.yaml index f8a33a4..8d78c32 100644 --- a/configs/local/inference_config.yaml +++ b/configs/local/inference_config.yaml @@ -14,17 +14,17 @@ runner: dataset_list: - OmniObject3d_test - blender_script_path: "/data/hofee/project/nbv_rec/blender/data_renderer.py" - output_dir: "/data/hofee/data/inference_global_full_on_testset" + blender_script_path: "C:\\Document\\Local Project\\nbv_rec\\blender\\data_renderer.py" + output_dir: "C:\\Document\\Datasets\\inference_scan_pts_overlap_global_full_on_testset" pipeline: nbv_reconstruction_pipeline voxel_size: 0.003 dataset: OmniObject3d_train: - root_dir: "/data/hofee/data/new_full_data" - model_dir: "/data/hofee/data/scaled_object_meshes" - source: seq_reconstruction_dataset - split_file: "/data/hofee/data/sample.txt" + root_dir: "C:\\Document\\Datasets\\inference_test" + model_dir: "C:\\Document\\Datasets\\scaled_object_meshes" + source: seq_reconstruction_dataset_preprocessed + split_file: "C:\\Document\\Datasets\\data_list\\sample.txt" type: test filter_degree: 75 ratio: 1 @@ -34,10 +34,10 @@ dataset: load_from_preprocess: True OmniObject3d_test: - root_dir: "/data/hofee/data/new_full_data" - model_dir: "/data/hofee/data/scaled_object_meshes" - source: seq_reconstruction_dataset - split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_test.txt" + root_dir: "C:\\Document\\Datasets\\inference_test" + model_dir: "C:\\Document\\Datasets\\scaled_object_meshes" + source: seq_reconstruction_dataset_preprocessed + split_file: "C:\\Document\\Datasets\\data_list\\OmniObject3d_test.txt" type: test filter_degree: 75 eval_list: diff --git a/core/seq_dataset.py b/core/seq_dataset.py index 1c129bc..68774d5 100644 --- a/core/seq_dataset.py +++ b/core/seq_dataset.py @@ -8,7 +8,7 @@ import torch import os import sys -sys.path.append(r"/data/hofee/project/nbv_rec/nbv_reconstruction") +sys.path.append(r"C:\Document\Local Project\nbv_rec\nbv_reconstruction") from utils.data_load import DataLoadUtil from utils.pose import PoseUtil @@ -55,7 +55,9 @@ class SeqReconstructionDataset(BaseDataset): def get_datalist(self): datalist = [] - for scene_name in self.scene_name_list: + total = len(self.scene_name_list) + for idx, scene_name in enumerate(self.scene_name_list): + print(f"processing {scene_name} ({idx}/{total})") seq_num = DataLoadUtil.get_label_num(self.root_dir, scene_name) scene_max_coverage_rate = 0 max_coverage_rate_list = [] @@ -182,17 +184,33 @@ if __name__ == "__main__": seed = 0 torch.manual_seed(seed) np.random.seed(seed) + ''' + OmniObject3d_test: + root_dir: "H:\\AI\\Datasets\\packed_test_data" + model_dir: "H:\\AI\\Datasets\\scaled_object_meshes" + source: seq_reconstruction_dataset + split_file: "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt" + type: test + filter_degree: 75 + eval_list: + - pose_diff + - coverage_rate_increase + ratio: 0.1 + batch_size: 1 + num_workers: 12 + pts_num: 8192 + load_from_preprocess: True + ''' config = { - "root_dir": "/data/hofee/data/new_full_data", + "root_dir": "H:\\AI\\Datasets\\packed_test_data", "source": "seq_reconstruction_dataset", - "split_file": "/data/hofee/data/sample.txt", + "split_file": "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt", "load_from_preprocess": True, - "ratio": 0.5, - "batch_size": 2, + "ratio": 1, "filter_degree": 75, "num_workers": 0, - "pts_num": 4096, - "type": namespace.Mode.TRAIN, + "pts_num": 8192, + "type": "test", } ds = SeqReconstructionDataset(config) print(len(ds)) diff --git a/core/seq_dataset_preprocessed.py b/core/seq_dataset_preprocessed.py new file mode 100644 index 0000000..9f84bf4 --- /dev/null +++ b/core/seq_dataset_preprocessed.py @@ -0,0 +1,85 @@ +import numpy as np +from PytorchBoot.dataset import BaseDataset +import PytorchBoot.namespace as namespace +import PytorchBoot.stereotype as stereotype +from PytorchBoot.config import ConfigManager +from PytorchBoot.utils.log_util import Log +import pickle +import torch +import os +import sys + +sys.path.append(r"C:\Document\Local Project\nbv_rec\nbv_reconstruction") + +from utils.data_load import DataLoadUtil +from utils.pose import PoseUtil +from utils.pts import PtsUtil + + +@stereotype.dataset("seq_reconstruction_dataset_preprocessed") +class SeqReconstructionDatasetPreprocessed(BaseDataset): + def __init__(self, config): + super(SeqReconstructionDatasetPreprocessed, self).__init__(config) + self.config = config + self.root_dir = config["root_dir"] + self.real_root_dir = r"H:\AI\Datasets\packed_test_data" + self.item_list = os.listdir(self.root_dir) + + def __getitem__(self, index): + data = pickle.load(open(os.path.join(self.root_dir, self.item_list[index]), "rb")) + data_item = { + "first_scanned_pts": np.asarray(data["first_scanned_pts"], dtype=np.float32), # Ndarray(S x Nv x 3) + "first_scanned_coverage_rate": data["first_scanned_coverage_rate"], # List(S): Float, range(0, 1) + "first_scanned_n_to_world_pose_9d": np.asarray(data["first_scanned_n_to_world_pose_9d"], dtype=np.float32), # Ndarray(S x 9) + "seq_max_coverage_rate": data["seq_max_coverage_rate"], # Float, range(0, 1) + "best_seq_len": data["best_seq_len"], # Int + "scene_name": data["scene_name"], # String + "gt_pts": np.asarray(data["gt_pts"], dtype=np.float32), # Ndarray(N x 3) + "scene_path": os.path.join(self.real_root_dir, data["scene_name"]), # String + "O_to_L_pose": np.asarray(data["O_to_L_pose"], dtype=np.float32), + } + return data_item + + def __len__(self): + return len(self.item_list) + + +# -------------- Debug ---------------- # +if __name__ == "__main__": + import torch + + seed = 0 + torch.manual_seed(seed) + np.random.seed(seed) + ''' + OmniObject3d_test: + root_dir: "H:\\AI\\Datasets\\packed_test_data" + model_dir: "H:\\AI\\Datasets\\scaled_object_meshes" + source: seq_reconstruction_dataset + split_file: "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt" + type: test + filter_degree: 75 + eval_list: + - pose_diff + - coverage_rate_increase + ratio: 0.1 + batch_size: 1 + num_workers: 12 + pts_num: 8192 + load_from_preprocess: True + ''' + config = { + "root_dir": "H:\\AI\\Datasets\\packed_test_data", + "source": "seq_reconstruction_dataset", + "split_file": "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt", + "load_from_preprocess": True, + "ratio": 1, + "filter_degree": 75, + "num_workers": 0, + "pts_num": 8192, + "type": "test", + } + ds = SeqReconstructionDataset(config) + print(len(ds)) + print(ds.__getitem__(10)) + diff --git a/runners/inferencer.py b/runners/inferencer.py index 50f838c..0b990b5 100644 --- a/runners/inferencer.py +++ b/runners/inferencer.py @@ -19,7 +19,7 @@ from PytorchBoot.dataset import BaseDataset from PytorchBoot.runners.runner import Runner from PytorchBoot.utils import Log from PytorchBoot.status import status_manager - +from utils.data_load import DataLoadUtil @stereotype.runner("inferencer") class Inferencer(Runner): def __init__(self, config_path): @@ -35,7 +35,12 @@ class Inferencer(Runner): ''' Experiment ''' self.load_experiment("nbv_evaluator") - self.stat_result = {} + self.stat_result_path = os.path.join(self.output_dir, "stat.json") + if os.path.exists(self.stat_result_path): + with open(self.stat_result_path, "r") as f: + self.stat_result = json.load(f) + else: + self.stat_result = {} ''' Test ''' self.test_config = ConfigManager.get(namespace.Stereotype.RUNNER, namespace.Mode.TEST) @@ -68,22 +73,21 @@ class Inferencer(Runner): test_set_name = test_set.get_name() total=int(len(test_set)) - scene_name_list = test_set.get_scene_name_list() - for i in range(total): - scene_name = scene_name_list[i] + for i in tqdm(range(total), desc=f"Processing {test_set_name}", ncols=100): + data = test_set.__getitem__(i) + scene_name = data["scene_name"] inference_result_path = os.path.join(self.output_dir, test_set_name, f"{scene_name}.pkl") if os.path.exists(inference_result_path): Log.info(f"Inference result already exists for scene: {scene_name}") continue - data = test_set.__getitem__(i) + status_manager.set_progress("inference", "inferencer", f"Batch[{test_set_name}]", i+1, total) - scene_name = data["scene_name"] output = self.predict_sequence(data) self.save_inference_result(test_set_name, data["scene_name"], output) status_manager.set_progress("inference", "inferencer", f"dataset", len(self.test_set_list), len(self.test_set_list)) - def predict_sequence(self, data, cr_increase_threshold=0, max_iter=50, max_retry=5): + def predict_sequence(self, data, cr_increase_threshold=0.001, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 7): scene_name = data["scene_name"] Log.info(f"Processing scene: {scene_name}") status_manager.set_status("inference", "inferencer", "scene", scene_name) @@ -102,16 +106,23 @@ class Inferencer(Runner): ''' data for inference ''' input_data = {} + scanned_pts = [] input_data["combined_scanned_pts"] = torch.tensor(data["first_scanned_pts"][0], dtype=torch.float32).to(self.device).unsqueeze(0) input_data["scanned_n_to_world_pose_9d"] = [torch.tensor(data["first_scanned_n_to_world_pose_9d"], dtype=torch.float32).to(self.device)] input_data["mode"] = namespace.Mode.TEST input_pts_N = input_data["combined_scanned_pts"].shape[1] - first_frame_target_pts, first_frame_target_normals = RenderUtil.render_pts(first_frame_to_world, scene_path, self.script_path, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose) + root = os.path.dirname(scene_path) + display_table_info = DataLoadUtil.get_display_table_info(root, scene_name) + radius = display_table_info["radius"] + scan_points = np.asarray(ReconstructionUtil.generate_scan_points(display_table_top=0,display_table_radius=radius)) + first_frame_target_pts, first_frame_target_normals, first_frame_scan_points_indices = RenderUtil.render_pts(first_frame_to_world, scene_path, self.script_path, scan_points, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose) scanned_view_pts = [first_frame_target_pts] + history_indices = [first_frame_scan_points_indices] last_pred_cr, added_pts_num = self.compute_coverage_rate(scanned_view_pts, None, down_sampled_model_pts, threshold=voxel_threshold) - + scanned_pts.append(first_frame_target_pts) retry_duplication_pose = [] retry_no_pts_pose = [] + retry_overlap_pose = [] retry = 0 pred_cr_seq = [last_pred_cr] success = 0 @@ -129,7 +140,22 @@ class Inferencer(Runner): try: start_time = time.time() - new_target_pts, new_target_normals = RenderUtil.render_pts(pred_pose, scene_path, self.script_path, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose) + new_target_pts, new_target_normals, new_scan_points_indices = RenderUtil.render_pts(pred_pose, scene_path, self.script_path, scan_points, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose) + #import ipdb; ipdb.set_trace() + if not ReconstructionUtil.check_scan_points_overlap(history_indices, new_scan_points_indices, scan_points_threshold): + curr_overlap_area_threshold = overlap_area_threshold + else: + curr_overlap_area_threshold = overlap_area_threshold * 0.5 + + downsampled_new_target_pts = PtsUtil.voxel_downsample_point_cloud(new_target_pts, voxel_threshold) + overlap, new_added_pts_num = ReconstructionUtil.check_overlap(downsampled_new_target_pts, down_sampled_model_pts, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=voxel_threshold, require_new_added_pts_num = True) + if not overlap: + retry += 1 + retry_overlap_pose.append(pred_pose.cpu().numpy().tolist()) + continue + + scanned_pts.append(new_target_pts) + history_indices.append(new_scan_points_indices) end_time = time.time() print(f"Time taken for rendering: {end_time - start_time} seconds") except Exception as e: @@ -147,14 +173,16 @@ class Inferencer(Runner): continue start_time = time.time() - pred_cr, new_added_pts_num = self.compute_coverage_rate(scanned_view_pts, new_target_pts, down_sampled_model_pts, threshold=voxel_threshold) + pred_cr, covered_pts_num = self.compute_coverage_rate(scanned_view_pts, new_target_pts, down_sampled_model_pts, threshold=voxel_threshold) end_time = time.time() print(f"Time taken for coverage rate computation: {end_time - start_time} seconds") print(pred_cr, last_pred_cr, " max: ", data["seq_max_coverage_rate"]) + print("new added pts num: ", new_added_pts_num) if pred_cr >= data["seq_max_coverage_rate"] - 1e-3: print("max coverage rate reached!: ", pred_cr) success += 1 - elif new_added_pts_num < 10: + elif new_added_pts_num < 5: + success += 1 print("min added pts num reached!: ", new_added_pts_num) if pred_cr <= last_pred_cr + cr_increase_threshold: retry += 1 @@ -180,6 +208,7 @@ class Inferencer(Runner): input_data["scanned_n_to_world_pose_9d"] = input_data["scanned_n_to_world_pose_9d"][0].cpu().numpy().tolist() result = { + "scanned_pts": scanned_pts, "pred_pose_9d_seq": input_data["scanned_n_to_world_pose_9d"], "combined_scanned_pts": input_data["combined_scanned_pts"], "target_pts_seq": scanned_view_pts, @@ -189,6 +218,7 @@ class Inferencer(Runner): "scene_name": scene_name, "retry_no_pts_pose": retry_no_pts_pose, "retry_duplication_pose": retry_duplication_pose, + "retry_overlap_pose": retry_overlap_pose, "best_seq_len": data["best_seq_len"], } self.stat_result[scene_name] = { @@ -216,7 +246,7 @@ class Inferencer(Runner): os.makedirs(dataset_dir) output_path = os.path.join(dataset_dir, f"{scene_name}.pkl") pickle.dump(output, open(output_path, "wb")) - with open(os.path.join(dataset_dir, "stat.json"), "w") as f: + with open(self.stat_result_path, "w") as f: json.dump(self.stat_result, f) diff --git a/utils/reconstruction.py b/utils/reconstruction.py index c478ad4..6645ee9 100644 --- a/utils/reconstruction.py +++ b/utils/reconstruction.py @@ -32,13 +32,15 @@ class ReconstructionUtil: @staticmethod - def check_overlap(new_point_cloud, combined_point_cloud, overlap_area_threshold=25, voxel_size=0.01): + def check_overlap(new_point_cloud, combined_point_cloud, overlap_area_threshold=25, voxel_size=0.01, require_new_added_pts_num=False): kdtree = cKDTree(combined_point_cloud) distances, _ = kdtree.query(new_point_cloud) - overlapping_points = np.sum(distances < voxel_size*2) + overlapping_points_num = np.sum(distances < voxel_size*2) cm = 0.01 voxel_size_cm = voxel_size / cm - overlap_area = overlapping_points * voxel_size_cm * voxel_size_cm + overlap_area = overlapping_points_num * voxel_size_cm * voxel_size_cm + if require_new_added_pts_num: + return overlap_area > overlap_area_threshold, len(new_point_cloud)-np.sum(distances < voxel_size*1.2) return overlap_area > overlap_area_threshold diff --git a/utils/render.py b/utils/render.py index 5c1f5a2..504a10d 100644 --- a/utils/render.py +++ b/utils/render.py @@ -54,7 +54,22 @@ class RenderUtil: return points_camera_world @staticmethod - def render_pts(cam_pose, scene_path, script_path, voxel_threshold=0.005, filter_degree=75, nO_to_nL_pose=None, require_full_scene=False): + def get_scan_points_indices(scan_points, mask, display_table_mask_label, cam_intrinsic, cam_extrinsic): + scan_points_homogeneous = np.hstack((scan_points, np.ones((scan_points.shape[0], 1)))) + points_camera = np.dot(np.linalg.inv(cam_extrinsic), scan_points_homogeneous.T).T[:, :3] + points_image_homogeneous = np.dot(cam_intrinsic, points_camera.T).T + points_image_homogeneous /= points_image_homogeneous[:, 2:] + pixel_x = points_image_homogeneous[:, 0].astype(int) + pixel_y = points_image_homogeneous[:, 1].astype(int) + h, w = mask.shape[:2] + valid_indices = (pixel_x >= 0) & (pixel_x < w) & (pixel_y >= 0) & (pixel_y < h) + mask_colors = mask[pixel_y[valid_indices], pixel_x[valid_indices]] + selected_points_indices = np.where((mask_colors == display_table_mask_label).all(axis=-1))[0] + selected_points_indices = np.where(valid_indices)[0][selected_points_indices] + return selected_points_indices + + @staticmethod + def render_pts(cam_pose, scene_path, script_path, scan_points, voxel_threshold=0.005, filter_degree=75, nO_to_nL_pose=None, require_full_scene=False): nO_to_world_pose = DataLoadUtil.get_real_cam_O_from_cam_L(cam_pose, nO_to_nL_pose, scene_path=scene_path) @@ -74,12 +89,7 @@ class RenderUtil: 'blender', '-b', '-P', script_path, '--', temp_dir ], capture_output=True, text=True) end_time = time.time() - print(result) print(f"-- Time taken for blender: {end_time - start_time} seconds") - if result.returncode != 0: - print("Blender script failed:") - print(result.stderr) - return None path = os.path.join(temp_dir, "tmp") cam_info = DataLoadUtil.load_cam_info(path, binocular=True) depth_L, depth_R = DataLoadUtil.load_depth( @@ -116,11 +126,15 @@ class RenderUtil: target_points, target_normals = PtsUtil.filter_points( target_points, sampled_target_normal_L, cam_info["cam_to_world"], theta_limit = filter_degree, z_range=(RenderUtil.min_z, RenderUtil.max_z) ) + + scan_points_indices_L = RenderUtil.get_scan_points_indices(scan_points, mask_img_L, RenderUtil.display_table_mask_label, cam_info["cam_intrinsic"], cam_info["cam_to_world"]) + scan_points_indices_R = RenderUtil.get_scan_points_indices(scan_points, mask_img_R, RenderUtil.display_table_mask_label, cam_info["cam_intrinsic"], cam_info["cam_to_world_R"]) + scan_points_indices = np.intersect1d(scan_points_indices_L, scan_points_indices_R) if not has_points: target_points = np.zeros((0, 3)) target_normals = np.zeros((0, 3)) end_time = time.time() print(f"-- Time taken for processing: {end_time - start_time} seconds") #import ipdb; ipdb.set_trace() - return target_points, target_normals \ No newline at end of file + return target_points, target_normals, scan_points_indices \ No newline at end of file From 5bcd0fc6e313da92f175d79889eacb9e37d92ee3 Mon Sep 17 00:00:00 2001 From: hofee <64160135+GitHofee@users.noreply.github.com> Date: Mon, 4 Nov 2024 23:49:12 +0800 Subject: [PATCH 2/6] upd --- configs/local/inference_config.yaml | 24 ++++++++++++------------ runners/inferencer.py | 8 ++------ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/configs/local/inference_config.yaml b/configs/local/inference_config.yaml index 8d78c32..13a7acc 100644 --- a/configs/local/inference_config.yaml +++ b/configs/local/inference_config.yaml @@ -20,18 +20,18 @@ runner: voxel_size: 0.003 dataset: - OmniObject3d_train: - root_dir: "C:\\Document\\Datasets\\inference_test" - model_dir: "C:\\Document\\Datasets\\scaled_object_meshes" - source: seq_reconstruction_dataset_preprocessed - split_file: "C:\\Document\\Datasets\\data_list\\sample.txt" - type: test - filter_degree: 75 - ratio: 1 - batch_size: 1 - num_workers: 12 - pts_num: 8192 - load_from_preprocess: True + # OmniObject3d_train: + # root_dir: "C:\\Document\\Datasets\\inference_test1" + # model_dir: "C:\\Document\\Datasets\\scaled_object_meshes" + # source: seq_reconstruction_dataset_preprocessed + # split_file: "C:\\Document\\Datasets\\data_list\\sample.txt" + # type: test + # filter_degree: 75 + # ratio: 1 + # batch_size: 1 + # num_workers: 12 + # pts_num: 8192 + # load_from_preprocess: True OmniObject3d_test: root_dir: "C:\\Document\\Datasets\\inference_test" diff --git a/runners/inferencer.py b/runners/inferencer.py index 0b990b5..a38b9e8 100644 --- a/runners/inferencer.py +++ b/runners/inferencer.py @@ -87,7 +87,7 @@ class Inferencer(Runner): status_manager.set_progress("inference", "inferencer", f"dataset", len(self.test_set_list), len(self.test_set_list)) - def predict_sequence(self, data, cr_increase_threshold=0.001, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 7): + def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 7): scene_name = data["scene_name"] Log.info(f"Processing scene: {scene_name}") status_manager.set_status("inference", "inferencer", "scene", scene_name) @@ -106,7 +106,6 @@ class Inferencer(Runner): ''' data for inference ''' input_data = {} - scanned_pts = [] input_data["combined_scanned_pts"] = torch.tensor(data["first_scanned_pts"][0], dtype=torch.float32).to(self.device).unsqueeze(0) input_data["scanned_n_to_world_pose_9d"] = [torch.tensor(data["first_scanned_n_to_world_pose_9d"], dtype=torch.float32).to(self.device)] input_data["mode"] = namespace.Mode.TEST @@ -119,7 +118,6 @@ class Inferencer(Runner): scanned_view_pts = [first_frame_target_pts] history_indices = [first_frame_scan_points_indices] last_pred_cr, added_pts_num = self.compute_coverage_rate(scanned_view_pts, None, down_sampled_model_pts, threshold=voxel_threshold) - scanned_pts.append(first_frame_target_pts) retry_duplication_pose = [] retry_no_pts_pose = [] retry_overlap_pose = [] @@ -154,7 +152,6 @@ class Inferencer(Runner): retry_overlap_pose.append(pred_pose.cpu().numpy().tolist()) continue - scanned_pts.append(new_target_pts) history_indices.append(new_scan_points_indices) end_time = time.time() print(f"Time taken for rendering: {end_time - start_time} seconds") @@ -182,7 +179,7 @@ class Inferencer(Runner): print("max coverage rate reached!: ", pred_cr) success += 1 elif new_added_pts_num < 5: - success += 1 + #success += 1 print("min added pts num reached!: ", new_added_pts_num) if pred_cr <= last_pred_cr + cr_increase_threshold: retry += 1 @@ -208,7 +205,6 @@ class Inferencer(Runner): input_data["scanned_n_to_world_pose_9d"] = input_data["scanned_n_to_world_pose_9d"][0].cpu().numpy().tolist() result = { - "scanned_pts": scanned_pts, "pred_pose_9d_seq": input_data["scanned_n_to_world_pose_9d"], "combined_scanned_pts": input_data["combined_scanned_pts"], "target_pts_seq": scanned_view_pts, From 6f427785b3395f681951bead746e439da35d9fa7 Mon Sep 17 00:00:00 2001 From: hofee <64160135+GitHofee@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:17:20 -0600 Subject: [PATCH 3/6] upd inference --- configs/local/inference_config.yaml | 2 +- configs/local/view_generate_config.yaml | 8 ++++---- runners/inferencer.py | 17 ++++++++++++----- runners/view_generator.py | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/configs/local/inference_config.yaml b/configs/local/inference_config.yaml index 13a7acc..1e6d89b 100644 --- a/configs/local/inference_config.yaml +++ b/configs/local/inference_config.yaml @@ -15,7 +15,7 @@ runner: - OmniObject3d_test blender_script_path: "C:\\Document\\Local Project\\nbv_rec\\blender\\data_renderer.py" - output_dir: "C:\\Document\\Datasets\\inference_scan_pts_overlap_global_full_on_testset" + output_dir: "C:\\Document\\Datasets\\debug_output" pipeline: nbv_reconstruction_pipeline voxel_size: 0.003 diff --git a/configs/local/view_generate_config.yaml b/configs/local/view_generate_config.yaml index b58d672..803bae8 100644 --- a/configs/local/view_generate_config.yaml +++ b/configs/local/view_generate_config.yaml @@ -8,11 +8,11 @@ runner: root_dir: experiments generate: port: 5002 - from: 600 + from: 0 to: -1 # -1 means all - object_dir: /media/hofee/data/data/object_meshes_part1 - table_model_path: "/media/hofee/data/data/others/table.obj" - output_dir: /media/hofee/repository/data_part_1 + object_dir: C:\\Document\\Datasets\\ball_meshes + table_model_path: C:\\Document\\Datasets\\table.obj + output_dir: C:\\Document\\Datasets\\debug_ball_generate_view binocular_vision: true plane_size: 10 max_views: 512 diff --git a/runners/inferencer.py b/runners/inferencer.py index a38b9e8..0ba8a4f 100644 --- a/runners/inferencer.py +++ b/runners/inferencer.py @@ -76,6 +76,8 @@ class Inferencer(Runner): for i in tqdm(range(total), desc=f"Processing {test_set_name}", ncols=100): data = test_set.__getitem__(i) scene_name = data["scene_name"] + if scene_name != "omniobject3d-suitcase_001": + continue inference_result_path = os.path.join(self.output_dir, test_set_name, f"{scene_name}.pkl") if os.path.exists(inference_result_path): Log.info(f"Inference result already exists for scene: {scene_name}") @@ -87,7 +89,7 @@ class Inferencer(Runner): status_manager.set_progress("inference", "inferencer", f"dataset", len(self.test_set_list), len(self.test_set_list)) - def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 7): + def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 5): scene_name = data["scene_name"] Log.info(f"Processing scene: {scene_name}") status_manager.set_status("inference", "inferencer", "scene", scene_name) @@ -110,10 +112,12 @@ class Inferencer(Runner): input_data["scanned_n_to_world_pose_9d"] = [torch.tensor(data["first_scanned_n_to_world_pose_9d"], dtype=torch.float32).to(self.device)] input_data["mode"] = namespace.Mode.TEST input_pts_N = input_data["combined_scanned_pts"].shape[1] + root = os.path.dirname(scene_path) display_table_info = DataLoadUtil.get_display_table_info(root, scene_name) radius = display_table_info["radius"] scan_points = np.asarray(ReconstructionUtil.generate_scan_points(display_table_top=0,display_table_radius=radius)) + first_frame_target_pts, first_frame_target_normals, first_frame_scan_points_indices = RenderUtil.render_pts(first_frame_to_world, scene_path, self.script_path, scan_points, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose) scanned_view_pts = [first_frame_target_pts] history_indices = [first_frame_scan_points_indices] @@ -124,6 +128,7 @@ class Inferencer(Runner): retry = 0 pred_cr_seq = [last_pred_cr] success = 0 + last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], 0.002) import time while len(pred_cr_seq) < max_iter and retry < max_retry: start_time = time.time() @@ -174,13 +179,9 @@ class Inferencer(Runner): end_time = time.time() print(f"Time taken for coverage rate computation: {end_time - start_time} seconds") print(pred_cr, last_pred_cr, " max: ", data["seq_max_coverage_rate"]) - print("new added pts num: ", new_added_pts_num) if pred_cr >= data["seq_max_coverage_rate"] - 1e-3: print("max coverage rate reached!: ", pred_cr) success += 1 - elif new_added_pts_num < 5: - #success += 1 - print("min added pts num reached!: ", new_added_pts_num) if pred_cr <= last_pred_cr + cr_increase_threshold: retry += 1 retry_duplication_pose.append(pred_pose.cpu().numpy().tolist()) @@ -202,6 +203,12 @@ class Inferencer(Runner): if success > 3: break last_pred_cr = pred_cr + pts_num = voxel_downsampled_combined_scanned_pts_np.shape[0] + if pts_num - last_pts_num < 10: + retry += 1 + print("delta pts num < 10:", pts_num, last_pts_num) + last_pts_num = pts_num + input_data["scanned_n_to_world_pose_9d"] = input_data["scanned_n_to_world_pose_9d"][0].cpu().numpy().tolist() result = { diff --git a/runners/view_generator.py b/runners/view_generator.py index 634ccbf..2e28c44 100644 --- a/runners/view_generator.py +++ b/runners/view_generator.py @@ -9,7 +9,7 @@ class ViewGenerator(Runner): self.config_path = config_path def run(self): - result = subprocess.run(['/home/hofee/blender-4.0.2-linux-x64/blender', '-b', '-P', '../blender/run_blender.py', '--', self.config_path]) + result = subprocess.run(['blender', '-b', '-P', '../blender/run_blender.py', '--', self.config_path]) print() def create_experiment(self, backup_name=None): From 6a608ea74bc1b177dea2b3f7f7cea97ac821cbad Mon Sep 17 00:00:00 2001 From: hofee Date: Wed, 6 Nov 2024 20:07:33 +0800 Subject: [PATCH 4/6] upd inference_server --- runners/inference_server.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/runners/inference_server.py b/runners/inference_server.py index e71b28b..ac62fc9 100644 --- a/runners/inference_server.py +++ b/runners/inference_server.py @@ -25,6 +25,7 @@ class InferencerServer(Runner): self.pipeline:torch.nn.Module = ComponentFactory.create(namespace.Stereotype.PIPELINE, self.pipeline_name) self.pipeline = self.pipeline.to(self.device) self.pts_num = 8192 + self.voxel_size = 0.002 ''' Experiment ''' self.load_experiment("inferencer_server") @@ -34,20 +35,14 @@ class InferencerServer(Runner): scanned_pts = data["scanned_pts"] scanned_n_to_world_pose_9d = data["scanned_n_to_world_pose_9d"] combined_scanned_views_pts = np.concatenate(scanned_pts, axis=0) - fps_downsampled_combined_scanned_pts, fps_idx = PtsUtil.fps_downsample_point_cloud( - combined_scanned_views_pts, self.pts_num, require_idx=True + voxel_downsampled_combined_scanned_pts = PtsUtil.voxel_downsample_point_cloud( + combined_scanned_views_pts, self.voxel_size + ) + fps_downsampled_combined_scanned_pts, fps_idx = PtsUtil.fps_downsample_point_cloud( + voxel_downsampled_combined_scanned_pts, self.pts_num, require_idx=True ) - # combined_scanned_views_pts_mask = np.zeros(len(scanned_pts), dtype=np.uint8) - # start_idx = 0 - # for i in range(len(scanned_pts)): - # end_idx = start_idx + len(scanned_pts[i]) - # combined_scanned_views_pts_mask[start_idx:end_idx] = i - # start_idx = end_idx - - # fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx] input_data["scanned_pts"] = scanned_pts - # input_data["scanned_pts_mask"] = np.asarray(fps_downsampled_combined_scanned_pts_mask, dtype=np.uint8) input_data["scanned_n_to_world_pose_9d"] = np.asarray(scanned_n_to_world_pose_9d, dtype=np.float32) input_data["combined_scanned_pts"] = np.asarray(fps_downsampled_combined_scanned_pts, dtype=np.float32) return input_data From 493639287e0ba297d56f99cba1c4b9af2d99d5d2 Mon Sep 17 00:00:00 2001 From: hofee <64160135+GitHofee@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:42:44 +0800 Subject: [PATCH 5/6] update calculating pts_num in inference.py --- configs/local/view_generate_config.yaml | 8 ++++---- runners/inferencer.py | 18 +++++++----------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/configs/local/view_generate_config.yaml b/configs/local/view_generate_config.yaml index 803bae8..4396de7 100644 --- a/configs/local/view_generate_config.yaml +++ b/configs/local/view_generate_config.yaml @@ -8,11 +8,11 @@ runner: root_dir: experiments generate: port: 5002 - from: 0 - to: -1 # -1 means all - object_dir: C:\\Document\\Datasets\\ball_meshes + from: 1 + to: 50 # -1 means all + object_dir: C:\\Document\\Datasets\\scaled_object_meshes table_model_path: C:\\Document\\Datasets\\table.obj - output_dir: C:\\Document\\Datasets\\debug_ball_generate_view + output_dir: C:\\Document\\Datasets\\debug_generate_view binocular_vision: true plane_size: 10 max_views: 512 diff --git a/runners/inferencer.py b/runners/inferencer.py index 0ba8a4f..238da0b 100644 --- a/runners/inferencer.py +++ b/runners/inferencer.py @@ -128,7 +128,7 @@ class Inferencer(Runner): retry = 0 pred_cr_seq = [last_pred_cr] success = 0 - last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], 0.002) + last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], 0.002).shape[0] import time while len(pred_cr_seq) < max_iter and retry < max_retry: start_time = time.time() @@ -151,7 +151,7 @@ class Inferencer(Runner): curr_overlap_area_threshold = overlap_area_threshold * 0.5 downsampled_new_target_pts = PtsUtil.voxel_downsample_point_cloud(new_target_pts, voxel_threshold) - overlap, new_added_pts_num = ReconstructionUtil.check_overlap(downsampled_new_target_pts, down_sampled_model_pts, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=voxel_threshold, require_new_added_pts_num = True) + overlap, _ = ReconstructionUtil.check_overlap(downsampled_new_target_pts, down_sampled_model_pts, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=voxel_threshold, require_new_added_pts_num = True) if not overlap: retry += 1 retry_overlap_pose.append(pred_pose.cpu().numpy().tolist()) @@ -175,27 +175,22 @@ class Inferencer(Runner): continue start_time = time.time() - pred_cr, covered_pts_num = self.compute_coverage_rate(scanned_view_pts, new_target_pts, down_sampled_model_pts, threshold=voxel_threshold) + pred_cr, _ = self.compute_coverage_rate(scanned_view_pts, new_target_pts, down_sampled_model_pts, threshold=voxel_threshold) end_time = time.time() print(f"Time taken for coverage rate computation: {end_time - start_time} seconds") print(pred_cr, last_pred_cr, " max: ", data["seq_max_coverage_rate"]) if pred_cr >= data["seq_max_coverage_rate"] - 1e-3: print("max coverage rate reached!: ", pred_cr) success += 1 - if pred_cr <= last_pred_cr + cr_increase_threshold: - retry += 1 - retry_duplication_pose.append(pred_pose.cpu().numpy().tolist()) - continue + retry = 0 pred_cr_seq.append(pred_cr) scanned_view_pts.append(new_target_pts) - down_sampled_new_pts_world = PtsUtil.random_downsample_point_cloud(new_target_pts, input_pts_N) - new_pts = down_sampled_new_pts_world input_data["scanned_n_to_world_pose_9d"] = [torch.cat([input_data["scanned_n_to_world_pose_9d"][0], pred_pose_9d], dim=0)] - combined_scanned_pts = np.concatenate([input_data["combined_scanned_pts"][0].cpu().numpy(), new_pts], axis=0) + combined_scanned_pts = np.vstack(scanned_view_pts) voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_pts, 0.002) random_downsampled_combined_scanned_pts_np = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, input_pts_N) input_data["combined_scanned_pts"] = torch.tensor(random_downsampled_combined_scanned_pts_np, dtype=torch.float32).unsqueeze(0).to(self.device) @@ -204,8 +199,9 @@ class Inferencer(Runner): break last_pred_cr = pred_cr pts_num = voxel_downsampled_combined_scanned_pts_np.shape[0] - if pts_num - last_pts_num < 10: + if pts_num - last_pts_num < 10 and pred_cr < data["seq_max_coverage_rate"] - 1e-3: retry += 1 + retry_duplication_pose.append(pred_pose.cpu().numpy().tolist()) print("delta pts num < 10:", pts_num, last_pts_num) last_pts_num = pts_num From 2c8ef203218cf6cec284b27104994499d32b5a09 Mon Sep 17 00:00:00 2001 From: hofee Date: Wed, 20 Nov 2024 15:24:45 +0800 Subject: [PATCH 6/6] upd ab_global_only --- configs/local/inference_config.yaml | 12 ++--- core/seq_dataset.py | 84 ++++++++++++----------------- core/seq_dataset_preprocessed.py | 5 +- runners/inferencer.py | 69 ++++++++++++------------ utils/render.py | 9 +--- 5 files changed, 80 insertions(+), 99 deletions(-) diff --git a/configs/local/inference_config.yaml b/configs/local/inference_config.yaml index 1e6d89b..64bccbf 100644 --- a/configs/local/inference_config.yaml +++ b/configs/local/inference_config.yaml @@ -14,11 +14,11 @@ runner: dataset_list: - OmniObject3d_test - blender_script_path: "C:\\Document\\Local Project\\nbv_rec\\blender\\data_renderer.py" - output_dir: "C:\\Document\\Datasets\\debug_output" + blender_script_path: "/media/hofee/data/project/python/nbv_reconstruction/blender/data_renderer.py" + output_dir: "/media/hofee/data/data/new_inference_test_output" pipeline: nbv_reconstruction_pipeline voxel_size: 0.003 - + min_new_area: 1.0 dataset: # OmniObject3d_train: # root_dir: "C:\\Document\\Datasets\\inference_test1" @@ -34,10 +34,10 @@ dataset: # load_from_preprocess: True OmniObject3d_test: - root_dir: "C:\\Document\\Datasets\\inference_test" - model_dir: "C:\\Document\\Datasets\\scaled_object_meshes" + root_dir: "/media/hofee/data/data/new_testset_output" + model_dir: "/media/hofee/data/data/scaled_object_meshes" source: seq_reconstruction_dataset_preprocessed - split_file: "C:\\Document\\Datasets\\data_list\\OmniObject3d_test.txt" + # split_file: "C:\\Document\\Datasets\\data_list\\OmniObject3d_test.txt" type: test filter_degree: 75 eval_list: diff --git a/core/seq_dataset.py b/core/seq_dataset.py index 68774d5..e27563b 100644 --- a/core/seq_dataset.py +++ b/core/seq_dataset.py @@ -8,7 +8,7 @@ import torch import os import sys -sys.path.append(r"C:\Document\Local Project\nbv_rec\nbv_reconstruction") +sys.path.append(r"/media/hofee/data/project/python/nbv_reconstruction/nbv_reconstruction") from utils.data_load import DataLoadUtil from utils.pose import PoseUtil @@ -47,6 +47,8 @@ class SeqReconstructionDataset(BaseDataset): with open(self.split_file_path, "r") as f: for line in f: scene_name = line.strip() + if not os.path.exists(os.path.join(self.root_dir, scene_name)): + continue scene_name_list.append(scene_name) return scene_name_list @@ -58,29 +60,19 @@ class SeqReconstructionDataset(BaseDataset): total = len(self.scene_name_list) for idx, scene_name in enumerate(self.scene_name_list): print(f"processing {scene_name} ({idx}/{total})") - seq_num = DataLoadUtil.get_label_num(self.root_dir, scene_name) - scene_max_coverage_rate = 0 - max_coverage_rate_list = [] scene_max_cr_idx = 0 - for seq_idx in range(seq_num): - label_path = DataLoadUtil.get_label_path( - self.root_dir, scene_name, seq_idx - ) - label_data = DataLoadUtil.load_label(label_path) - max_coverage_rate = label_data["max_coverage_rate"] - if max_coverage_rate > scene_max_coverage_rate: - scene_max_coverage_rate = max_coverage_rate - scene_max_cr_idx = seq_idx - max_coverage_rate_list.append(max_coverage_rate) - best_label_path = DataLoadUtil.get_label_path(self.root_dir, scene_name, scene_max_cr_idx) - best_label_data = DataLoadUtil.load_label(best_label_path) - first_frame = best_label_data["best_sequence"][0] - best_seq_len = len(best_label_data["best_sequence"]) + frame_len = DataLoadUtil.get_scene_seq_length(self.root_dir, scene_name) + + for i in range(frame_len): + path = DataLoadUtil.get_path(self.root_dir, scene_name, i) + pts = DataLoadUtil.load_from_preprocessed_pts(path, "npy") + if pts.shape[0] == 0: + continue datalist.append({ "scene_name": scene_name, - "first_frame": first_frame, - "best_seq_len": best_seq_len, - "max_coverage_rate": scene_max_coverage_rate, + "first_frame": i, + "best_seq_len": -1, + "max_coverage_rate": 1.0, "label_idx": scene_max_cr_idx, }) return datalist @@ -131,8 +123,7 @@ class SeqReconstructionDataset(BaseDataset): scanned_n_to_world_pose, ) = ([], [], []) view = data_item_info["first_frame"] - frame_idx = view[0] - coverage_rate = view[1] + frame_idx = view view_path = DataLoadUtil.get_path(self.root_dir, scene_name, frame_idx) cam_info = DataLoadUtil.load_cam_info(view_path, binocular=True) @@ -144,7 +135,7 @@ class SeqReconstructionDataset(BaseDataset): target_point_cloud, self.pts_num ) scanned_views_pts.append(downsampled_target_point_cloud) - scanned_coverages_rate.append(coverage_rate) + n_to_world_6d = PoseUtil.matrix_to_rotation_6d_numpy( np.asarray(n_to_world_pose[:3, :3]) ) @@ -161,7 +152,6 @@ class SeqReconstructionDataset(BaseDataset): gt_pts = self.seq_combined_pts(scene_name, frame_list) data_item = { "first_scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3) - "first_scanned_coverage_rate": scanned_coverages_rate, # List(S): Float, range(0, 1) "first_scanned_n_to_world_pose_9d": np.asarray(scanned_n_to_world_pose, dtype=np.float32), # Ndarray(S x 9) "seq_max_coverage_rate": max_coverage_rate, # Float, range(0, 1) "best_seq_len": best_seq_len, # Int @@ -180,39 +170,35 @@ class SeqReconstructionDataset(BaseDataset): # -------------- Debug ---------------- # if __name__ == "__main__": import torch + from tqdm import tqdm + import pickle + import os seed = 0 torch.manual_seed(seed) np.random.seed(seed) - ''' - OmniObject3d_test: - root_dir: "H:\\AI\\Datasets\\packed_test_data" - model_dir: "H:\\AI\\Datasets\\scaled_object_meshes" - source: seq_reconstruction_dataset - split_file: "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt" - type: test - filter_degree: 75 - eval_list: - - pose_diff - - coverage_rate_increase - ratio: 0.1 - batch_size: 1 - num_workers: 12 - pts_num: 8192 - load_from_preprocess: True - ''' + config = { - "root_dir": "H:\\AI\\Datasets\\packed_test_data", + "root_dir": "/media/hofee/data/data/new_testset", "source": "seq_reconstruction_dataset", - "split_file": "H:\\AI\\Datasets\\data_list\\OmniObject3d_test.txt", + "split_file": "/media/hofee/data/data/OmniObject3d_test.txt", "load_from_preprocess": True, - "ratio": 1, "filter_degree": 75, "num_workers": 0, "pts_num": 8192, - "type": "test", + "type": namespace.Mode.TEST, } - ds = SeqReconstructionDataset(config) - print(len(ds)) - print(ds.__getitem__(10)) + output_dir = "/media/hofee/data/data/new_testset_output" + os.makedirs(output_dir, exist_ok=True) + + ds = SeqReconstructionDataset(config) + for i in tqdm(range(len(ds)), desc="processing dataset"): + output_path = os.path.join(output_dir, f"item_{i}.pkl") + item = ds.__getitem__(i) + for key, value in item.items(): + if isinstance(value, np.ndarray): + item[key] = value.tolist() + #import ipdb; ipdb.set_trace() + with open(output_path, "wb") as f: + pickle.dump(item, f) diff --git a/core/seq_dataset_preprocessed.py b/core/seq_dataset_preprocessed.py index 9f84bf4..c92b5db 100644 --- a/core/seq_dataset_preprocessed.py +++ b/core/seq_dataset_preprocessed.py @@ -15,21 +15,19 @@ from utils.data_load import DataLoadUtil from utils.pose import PoseUtil from utils.pts import PtsUtil - @stereotype.dataset("seq_reconstruction_dataset_preprocessed") class SeqReconstructionDatasetPreprocessed(BaseDataset): def __init__(self, config): super(SeqReconstructionDatasetPreprocessed, self).__init__(config) self.config = config self.root_dir = config["root_dir"] - self.real_root_dir = r"H:\AI\Datasets\packed_test_data" + self.real_root_dir = r"/media/hofee/data/data/new_testset" self.item_list = os.listdir(self.root_dir) def __getitem__(self, index): data = pickle.load(open(os.path.join(self.root_dir, self.item_list[index]), "rb")) data_item = { "first_scanned_pts": np.asarray(data["first_scanned_pts"], dtype=np.float32), # Ndarray(S x Nv x 3) - "first_scanned_coverage_rate": data["first_scanned_coverage_rate"], # List(S): Float, range(0, 1) "first_scanned_n_to_world_pose_9d": np.asarray(data["first_scanned_n_to_world_pose_9d"], dtype=np.float32), # Ndarray(S x 9) "seq_max_coverage_rate": data["seq_max_coverage_rate"], # Float, range(0, 1) "best_seq_len": data["best_seq_len"], # Int @@ -43,7 +41,6 @@ class SeqReconstructionDatasetPreprocessed(BaseDataset): def __len__(self): return len(self.item_list) - # -------------- Debug ---------------- # if __name__ == "__main__": import torch diff --git a/runners/inferencer.py b/runners/inferencer.py index 238da0b..5d0cf16 100644 --- a/runners/inferencer.py +++ b/runners/inferencer.py @@ -23,11 +23,15 @@ from utils.data_load import DataLoadUtil @stereotype.runner("inferencer") class Inferencer(Runner): def __init__(self, config_path): + super().__init__(config_path) self.script_path = ConfigManager.get(namespace.Stereotype.RUNNER, "blender_script_path") self.output_dir = ConfigManager.get(namespace.Stereotype.RUNNER, "output_dir") self.voxel_size = ConfigManager.get(namespace.Stereotype.RUNNER, "voxel_size") + self.min_new_area = ConfigManager.get(namespace.Stereotype.RUNNER, "min_new_area") + CM = 0.01 + self.min_new_pts_num = self.min_new_area * (CM / self.voxel_size) **2 ''' Pipeline ''' self.pipeline_name = self.config[namespace.Stereotype.PIPELINE] self.pipeline:torch.nn.Module = ComponentFactory.create(namespace.Stereotype.PIPELINE, self.pipeline_name) @@ -74,22 +78,24 @@ class Inferencer(Runner): total=int(len(test_set)) for i in tqdm(range(total), desc=f"Processing {test_set_name}", ncols=100): - data = test_set.__getitem__(i) - scene_name = data["scene_name"] - if scene_name != "omniobject3d-suitcase_001": + try: + data = test_set.__getitem__(i) + scene_name = data["scene_name"] + inference_result_path = os.path.join(self.output_dir, test_set_name, f"{scene_name}.pkl") + if os.path.exists(inference_result_path): + Log.info(f"Inference result already exists for scene: {scene_name}") + continue + + status_manager.set_progress("inference", "inferencer", f"Batch[{test_set_name}]", i+1, total) + output = self.predict_sequence(data) + self.save_inference_result(test_set_name, data["scene_name"], output) + except Exception as e: + Log.error(f"Error in scene {scene_name}, {e}") continue - inference_result_path = os.path.join(self.output_dir, test_set_name, f"{scene_name}.pkl") - if os.path.exists(inference_result_path): - Log.info(f"Inference result already exists for scene: {scene_name}") - continue - - status_manager.set_progress("inference", "inferencer", f"Batch[{test_set_name}]", i+1, total) - output = self.predict_sequence(data) - self.save_inference_result(test_set_name, data["scene_name"], output) status_manager.set_progress("inference", "inferencer", f"dataset", len(self.test_set_list), len(self.test_set_list)) - def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 5): + def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 10, max_success=3): scene_name = data["scene_name"] Log.info(f"Processing scene: {scene_name}") status_manager.set_status("inference", "inferencer", "scene", scene_name) @@ -128,13 +134,11 @@ class Inferencer(Runner): retry = 0 pred_cr_seq = [last_pred_cr] success = 0 - last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], 0.002).shape[0] + last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], voxel_threshold).shape[0] import time - while len(pred_cr_seq) < max_iter and retry < max_retry: - start_time = time.time() + while len(pred_cr_seq) < max_iter and retry < max_retry and success < max_success: + Log.green(f"iter: {len(pred_cr_seq)}, retry: {retry}/{max_retry}, success: {success}/{max_success}") output = self.pipeline(input_data) - end_time = time.time() - print(f"Time taken for inference: {end_time - start_time} seconds") pred_pose_9d = output["pred_pose_9d"] pred_pose = torch.eye(4, device=pred_pose_9d.device) @@ -142,7 +146,6 @@ class Inferencer(Runner): pred_pose[:3,3] = pred_pose_9d[0,6:] try: - start_time = time.time() new_target_pts, new_target_normals, new_scan_points_indices = RenderUtil.render_pts(pred_pose, scene_path, self.script_path, scan_points, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose) #import ipdb; ipdb.set_trace() if not ReconstructionUtil.check_scan_points_overlap(history_indices, new_scan_points_indices, scan_points_threshold): @@ -153,15 +156,14 @@ class Inferencer(Runner): downsampled_new_target_pts = PtsUtil.voxel_downsample_point_cloud(new_target_pts, voxel_threshold) overlap, _ = ReconstructionUtil.check_overlap(downsampled_new_target_pts, down_sampled_model_pts, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=voxel_threshold, require_new_added_pts_num = True) if not overlap: + Log.yellow("no overlap!") retry += 1 retry_overlap_pose.append(pred_pose.cpu().numpy().tolist()) continue history_indices.append(new_scan_points_indices) - end_time = time.time() - print(f"Time taken for rendering: {end_time - start_time} seconds") except Exception as e: - Log.warning(f"Error in scene {scene_path}, {e}") + Log.error(f"Error in scene {scene_path}, {e}") print("current pose: ", pred_pose) print("curr_pred_cr: ", last_pred_cr) retry_no_pts_pose.append(pred_pose.cpu().numpy().tolist()) @@ -169,40 +171,41 @@ class Inferencer(Runner): continue if new_target_pts.shape[0] == 0: - print("no pts in new target") + Log.red("no pts in new target") retry_no_pts_pose.append(pred_pose.cpu().numpy().tolist()) retry += 1 continue - start_time = time.time() pred_cr, _ = self.compute_coverage_rate(scanned_view_pts, new_target_pts, down_sampled_model_pts, threshold=voxel_threshold) - end_time = time.time() - print(f"Time taken for coverage rate computation: {end_time - start_time} seconds") - print(pred_cr, last_pred_cr, " max: ", data["seq_max_coverage_rate"]) + Log.yellow(f"{pred_cr}, {last_pred_cr}, max: , {data['seq_max_coverage_rate']}") if pred_cr >= data["seq_max_coverage_rate"] - 1e-3: print("max coverage rate reached!: ", pred_cr) - success += 1 + - retry = 0 pred_cr_seq.append(pred_cr) scanned_view_pts.append(new_target_pts) input_data["scanned_n_to_world_pose_9d"] = [torch.cat([input_data["scanned_n_to_world_pose_9d"][0], pred_pose_9d], dim=0)] combined_scanned_pts = np.vstack(scanned_view_pts) - voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_pts, 0.002) + voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_pts, voxel_threshold) random_downsampled_combined_scanned_pts_np = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, input_pts_N) input_data["combined_scanned_pts"] = torch.tensor(random_downsampled_combined_scanned_pts_np, dtype=torch.float32).unsqueeze(0).to(self.device) - if success > 3: - break + last_pred_cr = pred_cr pts_num = voxel_downsampled_combined_scanned_pts_np.shape[0] - if pts_num - last_pts_num < 10 and pred_cr < data["seq_max_coverage_rate"] - 1e-3: + Log.info(f"delta pts num:,{pts_num - last_pts_num },{pts_num}, {last_pts_num}") + + if pts_num - last_pts_num < self.min_new_pts_num and pred_cr <= data["seq_max_coverage_rate"] - 1e-2: retry += 1 retry_duplication_pose.append(pred_pose.cpu().numpy().tolist()) - print("delta pts num < 10:", pts_num, last_pts_num) + Log.red(f"delta pts num < {self.min_new_pts_num}:, {pts_num}, {last_pts_num}") + elif pts_num - last_pts_num < self.min_new_pts_num and pred_cr > data["seq_max_coverage_rate"] - 1e-2: + success += 1 + Log.success(f"delta pts num < {self.min_new_pts_num}:, {pts_num}, {last_pts_num}") + last_pts_num = pts_num diff --git a/utils/render.py b/utils/render.py index 504a10d..aa2f4e2 100644 --- a/utils/render.py +++ b/utils/render.py @@ -84,12 +84,10 @@ class RenderUtil: params_data_path = os.path.join(temp_dir, "params.json") with open(params_data_path, 'w') as f: json.dump(params, f) - start_time = time.time() result = subprocess.run([ - 'blender', '-b', '-P', script_path, '--', temp_dir + '/home/hofee/blender-4.0.2-linux-x64/blender', '-b', '-P', script_path, '--', temp_dir ], capture_output=True, text=True) - end_time = time.time() - print(f"-- Time taken for blender: {end_time - start_time} seconds") + # print(result) path = os.path.join(temp_dir, "tmp") cam_info = DataLoadUtil.load_cam_info(path, binocular=True) depth_L, depth_R = DataLoadUtil.load_depth( @@ -97,7 +95,6 @@ class RenderUtil: cam_info["far_plane"], binocular=True ) - start_time = time.time() mask_L, mask_R = DataLoadUtil.load_seg(path, binocular=True) normal_L = DataLoadUtil.load_normal(path, binocular=True, left_only=True) ''' target points ''' @@ -134,7 +131,5 @@ class RenderUtil: if not has_points: target_points = np.zeros((0, 3)) target_normals = np.zeros((0, 3)) - end_time = time.time() - print(f"-- Time taken for processing: {end_time - start_time} seconds") #import ipdb; ipdb.set_trace() return target_points, target_normals, scan_points_indices \ No newline at end of file