upd

configs/local/inference_config.yaml

@@ -18,7 +18,7 @@ runner:
   output_dir: "/media/hofee/data/data/new_inference_test_output"
   pipeline: nbv_reconstruction_pipeline
   voxel_size: 0.003
-  min_new_area: 1.0
+  
 dataset:
   # OmniObject3d_train:
   #   root_dir: "C:\\Document\\Datasets\\inference_test1"

configs/local/view_generate_config.yaml

@@ -7,17 +7,19 @@ runner:
     name: debug
     root_dir: experiments
   generate:
-    port: 5002
-    from: 1
-    to: 50 # -1 means all
-    object_dir: C:\\Document\\Datasets\\scaled_object_meshes
-    table_model_path: C:\\Document\\Datasets\\table.obj
-    output_dir: C:\\Document\\Datasets\\debug_generate_view
+    port: 5000
+    from: 0
+    to: -1 # -1 means all
+    object_dir: /media/hofee/data/data/scaled_object_meshes
+    table_model_path: "/media/hofee/data/data/others/table.obj"
+    output_dir: /media/hofee/data/data/new_testset
+    object_list_path: /media/hofee/data/data/OmniObject3d_test.txt
+    use_list: True
     binocular_vision: true
     plane_size: 10
     max_views: 512
     min_views: 128
-    random_view_ratio: 0.02
+    random_view_ratio: 0.01
     min_cam_table_included_degree: 20
     max_diag: 0.7
     min_diag: 0.01

core/seq_dataset.py

@@ -47,9 +47,8 @@ class SeqReconstructionDataset(BaseDataset):
         with open(self.split_file_path, "r") as f:
             for line in f:
                 scene_name = line.strip()
-                if not os.path.exists(os.path.join(self.root_dir, scene_name)):
-                    continue
-                scene_name_list.append(scene_name)
+                if os.path.exists(os.path.join(self.root_dir, scene_name)):
+                    scene_name_list.append(scene_name)
         return scene_name_list
     
     def get_scene_name_list(self):
@@ -169,6 +168,7 @@ class SeqReconstructionDataset(BaseDataset):
 
 # -------------- Debug ---------------- #
 if __name__ == "__main__":
+    #import ipdb; ipdb.set_trace()
     import torch
     from tqdm import tqdm
     import pickle
@@ -199,6 +199,6 @@ if __name__ == "__main__":
         for key, value in item.items():
             if isinstance(value, np.ndarray):
                 item[key] = value.tolist()
-        #import ipdb; ipdb.set_trace()
+        import ipdb; ipdb.set_trace()
         with open(output_path, "wb") as f:
-            pickle.dump(item, f)
+            pickle.dump(item, f)

core/seq_dataset_preprocessed.py

@@ -15,6 +15,7 @@ from utils.data_load import DataLoadUtil
 from utils.pose import PoseUtil
 from utils.pts import PtsUtil
 
+
 @stereotype.dataset("seq_reconstruction_dataset_preprocessed")
 class SeqReconstructionDatasetPreprocessed(BaseDataset):
     def __init__(self, config):
@@ -41,6 +42,7 @@ class SeqReconstructionDatasetPreprocessed(BaseDataset):
     def __len__(self):
         return len(self.item_list)
 
+
 # -------------- Debug ---------------- #
 if __name__ == "__main__":
     import torch

preprocess/pack_upload_data.py

@@ -29,8 +29,8 @@ def pack_all_scenes(root, scene_list, output_dir):
         pack_scene_data(root, scene, output_dir)
 
 if __name__ == "__main__":
-    root = r"H:\AI\Datasets\nbv_rec_part2"
-    output_dir = r"H:\AI\Datasets\upload_part2"
+    root = r"/media/hofee/repository/data_part_1"
+    output_dir = r"/media/hofee/repository/upload_part1"
     scene_list = os.listdir(root)
     from_idx = 0
     to_idx = len(scene_list)

preprocess/preprocessor.py

@@ -164,10 +164,10 @@ def save_scene_data(root, scene, scene_idx=0, scene_total=1,file_type="txt"):
 
 if __name__ == "__main__":
     #root = "/media/hofee/repository/new_data_with_normal"
-    root = r"H:\AI\Datasets\nbv_rec_part2"
+    root = "/media/hofee/data/data/new_testset"
     scene_list = os.listdir(root)
     from_idx = 0 # 1000
-    to_idx = 600 # 1500
+    to_idx = len(scene_list) # 1500
 
 
     cnt = 0
@@ -179,7 +179,11 @@ if __name__ == "__main__":
             print(f"Scene {scene} has been processed")
             cnt+=1
             continue
-        save_scene_data(root, scene, cnt, total, file_type="npy")
+        try:
+            save_scene_data(root, scene, cnt, total, file_type="npy")
+        except Exception as e:
+            print(f"Error occurred when processing scene {scene}")
+            print(e)
         cnt+=1
         end = time.time()
         print(f"Time cost: {end-start}")

runners/inference_server.py

@@ -25,7 +25,6 @@ class InferencerServer(Runner):
         self.pipeline:torch.nn.Module = ComponentFactory.create(namespace.Stereotype.PIPELINE, self.pipeline_name)
         self.pipeline = self.pipeline.to(self.device)
         self.pts_num = 8192
-        self.voxel_size = 0.002
             
         ''' Experiment '''
         self.load_experiment("inferencer_server")
@@ -35,14 +34,20 @@ class InferencerServer(Runner):
         scanned_pts = data["scanned_pts"]
         scanned_n_to_world_pose_9d = data["scanned_n_to_world_pose_9d"]
         combined_scanned_views_pts = np.concatenate(scanned_pts, axis=0)
-        voxel_downsampled_combined_scanned_pts = PtsUtil.voxel_downsample_point_cloud(
-            combined_scanned_views_pts, self.voxel_size
-        )
         fps_downsampled_combined_scanned_pts, fps_idx = PtsUtil.fps_downsample_point_cloud(
-            voxel_downsampled_combined_scanned_pts, self.pts_num, require_idx=True
+            combined_scanned_views_pts, self.pts_num, require_idx=True
         )
+        # combined_scanned_views_pts_mask = np.zeros(len(scanned_pts), dtype=np.uint8)
+        # start_idx = 0
+        # for i in range(len(scanned_pts)):
+        #     end_idx = start_idx + len(scanned_pts[i])
+        #     combined_scanned_views_pts_mask[start_idx:end_idx] = i
+        #     start_idx = end_idx
+
+        # fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx]
 
         input_data["scanned_pts"] = scanned_pts
+        # input_data["scanned_pts_mask"] = np.asarray(fps_downsampled_combined_scanned_pts_mask, dtype=np.uint8)
         input_data["scanned_n_to_world_pose_9d"] = np.asarray(scanned_n_to_world_pose_9d, dtype=np.float32)
         input_data["combined_scanned_pts"] = np.asarray(fps_downsampled_combined_scanned_pts, dtype=np.float32)
         return input_data

runners/inferencer.py

@@ -23,15 +23,11 @@ from utils.data_load import DataLoadUtil
 @stereotype.runner("inferencer")
 class Inferencer(Runner):
     def __init__(self, config_path):
-        
         super().__init__(config_path)
     
         self.script_path = ConfigManager.get(namespace.Stereotype.RUNNER, "blender_script_path")
         self.output_dir = ConfigManager.get(namespace.Stereotype.RUNNER, "output_dir")
         self.voxel_size = ConfigManager.get(namespace.Stereotype.RUNNER, "voxel_size")
-        self.min_new_area = ConfigManager.get(namespace.Stereotype.RUNNER, "min_new_area")
-        CM = 0.01
-        self.min_new_pts_num = self.min_new_area * (CM / self.voxel_size) **2
         ''' Pipeline '''
         self.pipeline_name = self.config[namespace.Stereotype.PIPELINE]
         self.pipeline:torch.nn.Module = ComponentFactory.create(namespace.Stereotype.PIPELINE, self.pipeline_name)
@@ -78,24 +74,22 @@ class Inferencer(Runner):
 
                 total=int(len(test_set))
                 for i in tqdm(range(total), desc=f"Processing {test_set_name}", ncols=100):
-                    try:
-                        data = test_set.__getitem__(i)
-                        scene_name = data["scene_name"]
-                        inference_result_path = os.path.join(self.output_dir, test_set_name, f"{scene_name}.pkl")
-                        if os.path.exists(inference_result_path):
-                            Log.info(f"Inference result already exists for scene: {scene_name}")
-                            continue
-                        
-                        status_manager.set_progress("inference", "inferencer", f"Batch[{test_set_name}]", i+1, total)
-                        output = self.predict_sequence(data)
-                        self.save_inference_result(test_set_name, data["scene_name"], output)
-                    except Exception as e:
-                        Log.error(f"Error in scene {scene_name}, {e}")
+                    data = test_set.__getitem__(i)
+                    scene_name = data["scene_name"]
+                    if scene_name != "omniobject3d-book_004":
                         continue
+                    inference_result_path = os.path.join(self.output_dir, test_set_name, f"{scene_name}.pkl")
+                    if os.path.exists(inference_result_path):
+                        Log.info(f"Inference result already exists for scene: {scene_name}")
+                        continue
+                    
+                    status_manager.set_progress("inference", "inferencer", f"Batch[{test_set_name}]", i+1, total)
+                    output = self.predict_sequence(data)
+                    self.save_inference_result(test_set_name, data["scene_name"], output)
                     
             status_manager.set_progress("inference", "inferencer", f"dataset", len(self.test_set_list), len(self.test_set_list))
         
-    def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 10, max_success=3):
+    def predict_sequence(self, data, cr_increase_threshold=0, overlap_area_threshold=25, scan_points_threshold=10, max_iter=50, max_retry = 5):
         scene_name = data["scene_name"]
         Log.info(f"Processing scene: {scene_name}")
         status_manager.set_status("inference", "inferencer", "scene", scene_name)
@@ -120,6 +114,7 @@ class Inferencer(Runner):
         input_pts_N = input_data["combined_scanned_pts"].shape[1]
         
         root = os.path.dirname(scene_path)
+        
         display_table_info = DataLoadUtil.get_display_table_info(root, scene_name)
         radius = display_table_info["radius"]
         scan_points = np.asarray(ReconstructionUtil.generate_scan_points(display_table_top=0,display_table_radius=radius))
@@ -134,11 +129,13 @@ class Inferencer(Runner):
         retry = 0
         pred_cr_seq = [last_pred_cr]
         success = 0
-        last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], voxel_threshold).shape[0]
+        last_pts_num = PtsUtil.voxel_downsample_point_cloud(data["first_scanned_pts"][0], 0.002).shape[0]
         import time
-        while len(pred_cr_seq) < max_iter and retry < max_retry and success < max_success:
-            Log.green(f"iter: {len(pred_cr_seq)}, retry: {retry}/{max_retry}, success: {success}/{max_success}")
+        while len(pred_cr_seq) < max_iter and retry < max_retry:
+            start_time = time.time()
             output = self.pipeline(input_data)
+            end_time = time.time()
+            print(f"Time taken for inference: {end_time - start_time} seconds")
             pred_pose_9d = output["pred_pose_9d"]
             pred_pose = torch.eye(4, device=pred_pose_9d.device)
 
@@ -146,6 +143,7 @@ class Inferencer(Runner):
             pred_pose[:3,3] = pred_pose_9d[0,6:]
             
             try:
+                start_time = time.time()
                 new_target_pts, new_target_normals, new_scan_points_indices = RenderUtil.render_pts(pred_pose, scene_path, self.script_path, scan_points, voxel_threshold=voxel_threshold, filter_degree=filter_degree, nO_to_nL_pose=O_to_L_pose)
                 #import ipdb; ipdb.set_trace()
                 if not ReconstructionUtil.check_scan_points_overlap(history_indices, new_scan_points_indices, scan_points_threshold):
@@ -156,14 +154,15 @@ class Inferencer(Runner):
                 downsampled_new_target_pts = PtsUtil.voxel_downsample_point_cloud(new_target_pts, voxel_threshold)
                 overlap, _ = ReconstructionUtil.check_overlap(downsampled_new_target_pts, down_sampled_model_pts, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=voxel_threshold, require_new_added_pts_num = True)
                 if not overlap:
-                    Log.yellow("no overlap!")
                     retry += 1
                     retry_overlap_pose.append(pred_pose.cpu().numpy().tolist())
                     continue
                 
                 history_indices.append(new_scan_points_indices)
+                end_time = time.time()
+                print(f"Time taken for rendering: {end_time - start_time} seconds")
             except Exception as e:
-                Log.error(f"Error in scene {scene_path}, {e}")
+                Log.warning(f"Error in scene {scene_path}, {e}")
                 print("current pose: ", pred_pose)
                 print("curr_pred_cr: ", last_pred_cr)
                 retry_no_pts_pose.append(pred_pose.cpu().numpy().tolist())
@@ -171,66 +170,41 @@ class Inferencer(Runner):
                 continue
             
             if new_target_pts.shape[0] == 0:
-                Log.red("no pts in new target")
+                print("no pts in new target")
                 retry_no_pts_pose.append(pred_pose.cpu().numpy().tolist())
                 retry += 1
                 continue
             
+            start_time = time.time()
             pred_cr, _ = self.compute_coverage_rate(scanned_view_pts, new_target_pts, down_sampled_model_pts, threshold=voxel_threshold)
-            Log.yellow(f"{pred_cr}, {last_pred_cr}, max: , {data['seq_max_coverage_rate']}")
+            end_time = time.time()
+            print(f"Time taken for coverage rate computation: {end_time - start_time} seconds")
+            print(pred_cr, last_pred_cr, " max: ", data["seq_max_coverage_rate"])
             if pred_cr >= data["seq_max_coverage_rate"] - 1e-3:
                 print("max coverage rate reached!: ", pred_cr)
-                
+                success += 1
         
 
+            retry = 0
             pred_cr_seq.append(pred_cr)
             scanned_view_pts.append(new_target_pts)
             
             input_data["scanned_n_to_world_pose_9d"] = [torch.cat([input_data["scanned_n_to_world_pose_9d"][0], pred_pose_9d], dim=0)]
             
             combined_scanned_pts = np.vstack(scanned_view_pts)
-            voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_pts, voxel_threshold)
+            voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_pts, 0.002)
             random_downsampled_combined_scanned_pts_np = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, input_pts_N)
             input_data["combined_scanned_pts"] = torch.tensor(random_downsampled_combined_scanned_pts_np, dtype=torch.float32).unsqueeze(0).to(self.device)
 
-            
+            if success > 3:
+                break
             last_pred_cr = pred_cr
             pts_num = voxel_downsampled_combined_scanned_pts_np.shape[0]
-            Log.info(f"delta pts num:,{pts_num - last_pts_num },{pts_num}, {last_pts_num}")
-
-            if pts_num - last_pts_num < self.min_new_pts_num and pred_cr <= data["seq_max_coverage_rate"] - 1e-2:
+            if pts_num - last_pts_num < 10 and pred_cr < data["seq_max_coverage_rate"] - 1e-3:
                 retry += 1
                 retry_duplication_pose.append(pred_pose.cpu().numpy().tolist())
-                Log.red(f"delta pts num < {self.min_new_pts_num}:, {pts_num}, {last_pts_num}")
-            elif pts_num - last_pts_num < self.min_new_pts_num and pred_cr > data["seq_max_coverage_rate"] - 1e-2:
-                success += 1
-                Log.success(f"delta pts num < {self.min_new_pts_num}:, {pts_num}, {last_pts_num}")
-
+                print("delta pts num < 10:", pts_num, last_pts_num)
             last_pts_num = pts_num
-            
-
-        input_data["scanned_n_to_world_pose_9d"] = input_data["scanned_n_to_world_pose_9d"][0].cpu().numpy().tolist()
-        result = {
-            "pred_pose_9d_seq": input_data["scanned_n_to_world_pose_9d"],
-            "combined_scanned_pts": input_data["combined_scanned_pts"],
-            "target_pts_seq": scanned_view_pts,
-            "coverage_rate_seq": pred_cr_seq,
-            "max_coverage_rate": data["seq_max_coverage_rate"],
-            "pred_max_coverage_rate": max(pred_cr_seq),
-            "scene_name": scene_name,
-            "retry_no_pts_pose": retry_no_pts_pose,
-            "retry_duplication_pose": retry_duplication_pose,
-            "retry_overlap_pose": retry_overlap_pose,
-            "best_seq_len": data["best_seq_len"],
-        }
-        self.stat_result[scene_name] = {
-            "coverage_rate_seq": pred_cr_seq,
-            "pred_max_coverage_rate": max(pred_cr_seq),
-            "pred_seq_len": len(pred_cr_seq),
-        }
-        print('success rate: ', max(pred_cr_seq))
-
-        return result
 
     def compute_coverage_rate(self, scanned_view_pts, new_pts, model_pts, threshold=0.005):
         if new_pts is not None:

runners/view_generator.py

@@ -9,7 +9,7 @@ class ViewGenerator(Runner):
         self.config_path = config_path
         
     def run(self):
-        result = subprocess.run(['blender', '-b', '-P', '../blender/run_blender.py', '--', self.config_path])
+        result = subprocess.run(['/home/hofee/blender-4.0.2-linux-x64/blender', '-b', '-P', '../blender/run_blender.py', '--', self.config_path])
         print()
 
     def create_experiment(self, backup_name=None):

utils/render.py

@@ -84,10 +84,13 @@ class RenderUtil:
             params_data_path = os.path.join(temp_dir, "params.json")
             with open(params_data_path, 'w') as f:
                 json.dump(params, f)
+            start_time = time.time()
             result = subprocess.run([
                 '/home/hofee/blender-4.0.2-linux-x64/blender', '-b', '-P', script_path, '--', temp_dir
             ], capture_output=True, text=True)
-            # print(result)
+            end_time = time.time()
+
+            print(f"-- Time taken for blender: {end_time - start_time} seconds")
             path = os.path.join(temp_dir, "tmp")
             cam_info = DataLoadUtil.load_cam_info(path, binocular=True)
             depth_L, depth_R = DataLoadUtil.load_depth(
@@ -95,6 +98,7 @@ class RenderUtil:
                     cam_info["far_plane"], 
                     binocular=True
                 )
+            start_time = time.time()
             mask_L, mask_R = DataLoadUtil.load_seg(path, binocular=True)
             normal_L = DataLoadUtil.load_normal(path, binocular=True, left_only=True)
             ''' target points '''
@@ -131,5 +135,7 @@ class RenderUtil:
             if not has_points:
                 target_points = np.zeros((0, 3))
                 target_normals = np.zeros((0, 3))
+            end_time = time.time()
+            print(f"-- Time taken for processing: {end_time - start_time} seconds")
             #import ipdb; ipdb.set_trace()
             return target_points, target_normals, scan_points_indices