fix nan

app_generate_strategy.py

@@ -5,5 +5,5 @@ from runners.strategy_generator import StrategyGenerator
 class DataGenerateApp:
     @staticmethod
     def start():
-        StrategyGenerator("configs/server/server_strategy_generate_config.yaml").run()
+        StrategyGenerator("configs/local/strategy_generate_config.yaml").run()
         

app_split.py

@@ -5,5 +5,5 @@ from runners.data_spliter import DataSpliter
 class DataSplitApp:
     @staticmethod
     def start():
-        DataSpliter("configs/server/split_dataset_config.yaml").run()
+        DataSpliter("configs/server/server_split_dataset_config.yaml").run()
         

configs/local/strategy_generate_config.yaml

@@ -12,17 +12,16 @@ runner:
       
   generate:
     voxel_threshold: 0.003
-    soft_overlap_threshold: 0.3
-    hard_overlap_threshold: 0.6
+    overlap_area_threshold: 30
+    compute_with_normal: False
+    scan_points_threshold: 10
     overwrite: False
-    seq_num: 15
+    seq_num: 10
     dataset_list:
       - OmniObject3d
   
 datasets:
     OmniObject3d:
-      root_dir: /media/hofee/repository/full_data_output 
-      from: 0
-      to: -1 # -1 means end
-
-    
+      root_dir: /data/hofee/nbv_rec_part2_preprocessed
+      from: 155
+      to: 165 # ..-1 means end

configs/local/train_config.yaml

@@ -84,7 +84,7 @@ module:
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 2048
+    main_feat_dim: 3072
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False

configs/local/view_generate_config.yaml

@@ -7,12 +7,12 @@ runner:
     name: debug
     root_dir: experiments
   generate:
-    port: 5000
-    from: 0
+    port: 5002
+    from: 600
     to: -1 # -1 means all
-    object_dir: H:\\AI\\Datasets\\object_meshes_part2
-    table_model_path: "H:\\AI\\Datasets\\table.obj"
-    output_dir: C:\\Document\\Datasets\\nbv_rec_part2
+    object_dir: /media/hofee/data/data/object_meshes_part1
+    table_model_path: "/media/hofee/data/data/others/table.obj"
+    output_dir: /media/hofee/repository/data_part_1
     binocular_vision: true
     plane_size: 10
     max_views: 512

configs/server/server_split_dataset_config.yaml

@@ -0,0 +1,22 @@
+
+runner:
+  general:
+    seed: 0
+    device: cpu
+    cuda_visible_devices: "0,1,2,3,4,5,6,7"
+    
+  experiment:
+    name: server_split_dataset
+    root_dir: "experiments"
+  
+  split: # 
+    root_dir: "/data/hofee/data/new_full_data"
+    type: "unseen_instance" # "unseen_category"
+    datasets:
+      OmniObject3d_train: 
+        path: "/data/hofee/data/new_full_data_list/OmniObject3d_train.txt"
+        ratio: 0.9
+
+      OmniObject3d_test: 
+        path: "/data/hofee/data/new_full_data_list/OmniObject3d_test.txt"
+        ratio: 0.1

configs/server/server_train_config.yaml

@@ -7,13 +7,13 @@ runner:
     parallel: False
     
   experiment:
-    name: full_w_global_feat_wo_local_pts_feat
+    name: train_ab_global_and_partial_global
     root_dir: "experiments"
     use_checkpoint: False
     epoch: -1 # -1 stands for last epoch
     max_epochs: 5000
     save_checkpoint_interval: 1
-    test_first: True  
+    test_first: False  
 
   train:
     optimizer:
@@ -25,60 +25,60 @@ runner:
   test:
     frequency: 3 # test frequency
     dataset_list:
-      - OmniObject3d_test
+      #- OmniObject3d_test
       - OmniObject3d_val
 
-  pipeline: nbv_reconstruction_global_pts_pipeline
+  pipeline: nbv_reconstruction_pipeline
   
 dataset:
   OmniObject3d_train:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/data/new_full_data"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_train.txt"
     type: train
     cache: True
     ratio: 1
-    batch_size: 160
-    num_workers: 16
-    pts_num: 4096
+    batch_size: 80
+    num_workers: 128
+    pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_test:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/data/new_full_data"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt"
+    split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_test.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.05
-    batch_size: 160
+    ratio: 1
+    batch_size: 80
     num_workers: 12
-    pts_num: 4096
+    pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_val:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/data/new_full_data"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_train.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.005
-    batch_size: 160
+    ratio: 0.1
+    batch_size: 80
     num_workers: 12
-    pts_num: 4096
+    pts_num: 8192
     load_from_preprocess: True
 
 
 pipeline:
-  nbv_reconstruction_local_pts_pipeline:
+  nbv_reconstruction_pipeline:
     modules:
       pts_encoder: pointnet_encoder
       seq_encoder: transformer_seq_encoder
@@ -87,16 +87,6 @@ pipeline:
     eps: 1e-5
     global_scanned_feat: True
 
-  nbv_reconstruction_global_pts_pipeline:
-    modules:
-      pts_encoder: pointnet_encoder
-      pose_seq_encoder: transformer_seq_encoder
-      pose_encoder: pose_encoder
-      view_finder: gf_view_finder
-    eps: 1e-5
-    global_scanned_feat: True
-
-
     
 module:
 
@@ -107,11 +97,11 @@ module:
     feature_transform: False
 
   transformer_seq_encoder:
-    embed_dim: 1344
+    embed_dim: 320
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
-    output_dim: 2048
+    output_dim: 1024
 
   gf_view_finder:
     t_feat_dim: 128
@@ -128,6 +118,9 @@ module:
     pose_dim: 9
     out_dim: 256
 
+  pts_num_encoder:
+    out_dim: 64
+
 loss_function:
   gf_loss:
 

core/nbv_dataset.py

@@ -7,8 +7,9 @@ from PytorchBoot.utils.log_util import Log
 import torch
 import os
 import sys
+import time
 
-sys.path.append(r"/home/data/hofee/project/nbv_rec/nbv_reconstruction")
+sys.path.append(r"/data/hofee/project/nbv_rec/nbv_reconstruction")
 
 from utils.data_load import DataLoadUtil
 from utils.pose import PoseUtil
@@ -31,7 +32,7 @@ class NBVReconstructionDataset(BaseDataset):
         self.load_from_preprocess = config.get("load_from_preprocess", False)
 
         if self.type == namespace.Mode.TEST:
-            self.model_dir = config["model_dir"]
+            #self.model_dir = config["model_dir"]
             self.filter_degree = config["filter_degree"]
         if self.type == namespace.Mode.TRAIN:
             scale_ratio = 1
@@ -66,7 +67,9 @@ class NBVReconstructionDataset(BaseDataset):
                 if max_coverage_rate > scene_max_coverage_rate:
                     scene_max_coverage_rate = max_coverage_rate
                 max_coverage_rate_list.append(max_coverage_rate)
-            mean_coverage_rate = np.mean(max_coverage_rate_list)
+                
+            if max_coverage_rate_list:
+                mean_coverage_rate = np.mean(max_coverage_rate_list)
 
             for seq_idx in range(seq_num):
                 label_path = DataLoadUtil.get_label_path(
@@ -112,6 +115,15 @@ class NBVReconstructionDataset(BaseDataset):
         except Exception as e:
             Log.error(f"Save cache failed: {e}")
 
+    def voxel_downsample_with_mapping(self, point_cloud, voxel_size=0.003):
+        voxel_indices = np.floor(point_cloud / voxel_size).astype(np.int32)
+        unique_voxels, inverse, counts = np.unique(voxel_indices, axis=0, return_inverse=True, return_counts=True)
+        idx_sort = np.argsort(inverse)
+        idx_unique = idx_sort[np.cumsum(counts)-counts]
+        downsampled_points = point_cloud[idx_unique]
+        return downsampled_points, inverse
+
+
     def __getitem__(self, index):
         data_item_info = self.datalist[index]
         scanned_views = data_item_info["scanned_views"]
@@ -122,7 +134,10 @@ class NBVReconstructionDataset(BaseDataset):
             scanned_views_pts,
             scanned_coverages_rate,
             scanned_n_to_world_pose,
-        ) = ([], [], [], [])
+        ) = ([], [], [])
+        start_time = time.time()
+        start_indices = [0]
+        total_points = 0
         for view in scanned_views:
             frame_idx = view[0]
             coverage_rate = view[1]
@@ -144,8 +159,12 @@ class NBVReconstructionDataset(BaseDataset):
             n_to_world_trans = n_to_world_pose[:3, 3]
             n_to_world_9d = np.concatenate([n_to_world_6d, n_to_world_trans], axis=0)
             scanned_n_to_world_pose.append(n_to_world_9d)
+            total_points += len(downsampled_target_point_cloud)
+            start_indices.append(total_points)
 
 
+        end_time = time.time()
+        #Log.info(f"load data time: {end_time - start_time}")
         nbv_idx, nbv_coverage_rate = nbv[0], nbv[1]
         nbv_path = DataLoadUtil.get_path(self.root_dir, scene_name, nbv_idx)
         cam_info = DataLoadUtil.load_cam_info(nbv_path)
@@ -158,29 +177,27 @@ class NBVReconstructionDataset(BaseDataset):
         best_to_world_9d = np.concatenate(
             [best_to_world_6d, best_to_world_trans], axis=0
         )
-
+        
         combined_scanned_views_pts = np.concatenate(scanned_views_pts, axis=0)
-        fps_downsampled_combined_scanned_pts, fps_idx = PtsUtil.fps_downsample_point_cloud(
-            combined_scanned_views_pts, self.pts_num, require_idx=True
-        )
-        
-        combined_scanned_views_pts_mask = np.zeros(len(scanned_views_pts), dtype=np.uint8)
-        
-        start_idx = 0
-        for i in range(len(scanned_views_pts)):
-            end_idx = start_idx + len(scanned_views_pts[i])
-            combined_scanned_views_pts_mask[start_idx:end_idx] = i
-            start_idx = end_idx
-            
-        fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx]
-        
-        
-        
-        
+        voxel_downsampled_combined_scanned_pts_np, inverse = self.voxel_downsample_with_mapping(combined_scanned_views_pts, 0.003)
+        random_downsampled_combined_scanned_pts_np, random_downsample_idx = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, self.pts_num, require_idx=True)
+
+        all_idx_unique = np.arange(len(voxel_downsampled_combined_scanned_pts_np))
+        all_random_downsample_idx = all_idx_unique[random_downsample_idx]
+        scanned_pts_mask = []
+        for idx, start_idx in enumerate(start_indices):
+            if idx == len(start_indices) - 1:
+                break
+            end_idx = start_indices[idx+1]
+            view_inverse = inverse[start_idx:end_idx]
+            view_unique_downsampled_idx = np.unique(view_inverse)
+            view_unique_downsampled_idx_set = set(view_unique_downsampled_idx)
+            mask = np.array([idx in view_unique_downsampled_idx_set for idx in all_random_downsample_idx])
+            scanned_pts_mask.append(mask)
         data_item = {
             "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3)
-            "scanned_pts_mask": np.asarray(fps_downsampled_combined_scanned_pts_mask,dtype=np.uint8), # Ndarray(N), range(0, S)
-            "combined_scanned_pts": np.asarray(fps_downsampled_combined_scanned_pts, dtype=np.float32), # Ndarray(N x 3)
+            "combined_scanned_pts": np.asarray(random_downsampled_combined_scanned_pts_np, dtype=np.float32), # Ndarray(N x 3)
+            "scanned_pts_mask": np.asarray(scanned_pts_mask, dtype=np.bool), # Ndarray(N)
             "scanned_coverage_rate": scanned_coverages_rate, # List(S): Float, range(0, 1)
             "scanned_n_to_world_pose_9d": np.asarray(scanned_n_to_world_pose, dtype=np.float32), # Ndarray(S x 9)
             "best_coverage_rate": nbv_coverage_rate, # Float, range(0, 1)
@@ -206,7 +223,9 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["scanned_n_to_world_pose_9d"] = [
                 torch.tensor(item["scanned_n_to_world_pose_9d"]) for item in batch
             ]
-            
+            collate_data["scanned_pts_mask"] = [
+                torch.tensor(item["scanned_pts_mask"]) for item in batch
+            ]
             ''' ------ Fixed Length ------ '''
             
             collate_data["best_to_world_pose_9d"] = torch.stack(
@@ -215,17 +234,14 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["combined_scanned_pts"] = torch.stack(
                 [torch.tensor(item["combined_scanned_pts"]) for item in batch]
             )
-            collate_data["scanned_pts_mask"] = torch.stack(
-                [torch.tensor(item["scanned_pts_mask"]) for item in batch]
-            )
-                
+            
             for key in batch[0].keys():
                 if key not in [
                     "scanned_pts",
-                    "scanned_pts_mask",
                     "scanned_n_to_world_pose_9d",
                     "best_to_world_pose_9d",
                     "combined_scanned_pts",
+                    "scanned_pts_mask",
                 ]:
                     collate_data[key] = [item[key] for item in batch]
             return collate_data
@@ -241,10 +257,9 @@ if __name__ == "__main__":
     torch.manual_seed(seed)
     np.random.seed(seed)
     config = {
-        "root_dir": "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy",
-        "model_dir": "/home/data/hofee/project/nbv_rec/data/scaled_object_meshes",
+        "root_dir": "/data/hofee/nbv_rec_part2_preprocessed",
         "source": "nbv_reconstruction_dataset",
-        "split_file": "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt",
+        "split_file": "/data/hofee/data/sample.txt",
         "load_from_preprocess": True,
         "ratio": 0.5,
         "batch_size": 2,

core/pipeline.py

@@ -1,4 +1,5 @@
 import torch
+import time
 from torch import nn
 import PytorchBoot.namespace as namespace
 import PytorchBoot.stereotype as stereotype
@@ -6,10 +7,10 @@ from PytorchBoot.factory.component_factory import ComponentFactory
 from PytorchBoot.utils import Log
 
 
-@stereotype.pipeline("nbv_reconstruction_global_pts_n_num_pipeline")
-class NBVReconstructionGlobalPointsPipeline(nn.Module):
+@stereotype.pipeline("nbv_reconstruction_pipeline")
+class NBVReconstructionPipeline(nn.Module):
     def __init__(self, config):
-        super(NBVReconstructionGlobalPointsPipeline, self).__init__()
+        super(NBVReconstructionPipeline, self).__init__()
         self.config = config
         self.module_config = config["modules"]
         
@@ -19,12 +20,8 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         self.pose_encoder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["pose_encoder"]
         )
-        self.pts_num_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["pts_num_encoder"]
-        )
-        
-        self.transformer_seq_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["transformer_seq_encoder"]
+        self.seq_encoder = ComponentFactory.create(
+            namespace.Stereotype.MODULE, self.module_config["seq_encoder"]
         )
         self.view_finder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["view_finder"]
@@ -32,7 +29,6 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         
 
         self.eps = float(self.config["eps"])
-        self.enable_global_scanned_feat = self.config["global_scanned_feat"]
 
     def forward(self, data):
         mode = data["mode"]
@@ -92,50 +88,50 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         scanned_n_to_world_pose_9d_batch = data[
             "scanned_n_to_world_pose_9d"
         ]  # List(B): Tensor(S x 9)
-        scanned_pts_mask_batch = data[
-            "scanned_pts_mask"
-        ]  # Tensor(B x N)
+
+        scanned_pts_mask_batch = data["scanned_pts_mask"] # List(B): Tensor(N)
 
         device = next(self.parameters()).device
 
         embedding_list_batch = []
 
         combined_scanned_pts_batch = data["combined_scanned_pts"]  # Tensor(B x N x 3)
-        global_scanned_feat, perpoint_scanned_feat_batch = self.pts_encoder.encode_points(
+        global_scanned_feat, per_point_feat_batch = self.pts_encoder.encode_points(
             combined_scanned_pts_batch, require_per_point_feat=True
-        )  # global_scanned_feat: Tensor(B x Dg), perpoint_scanned_feat: Tensor(B x N x Dl)
-
-        for scanned_n_to_world_pose_9d, scanned_mask, perpoint_scanned_feat in zip(
-            scanned_n_to_world_pose_9d_batch,
-            scanned_pts_mask_batch,
-            perpoint_scanned_feat_batch,
-        ):
-            scanned_target_pts_num = [] # List(S): Int
-            partial_feat_seq = []
+        )  # global_scanned_feat: Tensor(B x Dg)
+        batch_size = len(scanned_n_to_world_pose_9d_batch)
+        for i in range(batch_size):
+            seq_len = len(scanned_n_to_world_pose_9d_batch[i])
+            scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d_batch[i].to(device)  # Tensor(S x 9)
+            scanned_pts_mask = scanned_pts_mask_batch[i] # Tensor(S x N)
+            per_point_feat = per_point_feat_batch[i] # Tensor(N x Dp)
+            partial_point_feat_seq = []
+            for j in range(seq_len):
+                partial_per_point_feat = per_point_feat[scanned_pts_mask[j]]
+                if partial_per_point_feat.shape[0] == 0:
+                    partial_point_feat = torch.zeros(per_point_feat.shape[1], device=device)
+                else:
+                    partial_point_feat = torch.mean(partial_per_point_feat, dim=0) # Tensor(Dp)
+                partial_point_feat_seq.append(partial_point_feat)
+            partial_point_feat_seq = torch.stack(partial_point_feat_seq, dim=0) # Tensor(S x Dp)
             
-            seq_len = len(scanned_n_to_world_pose_9d)
-            for seq_idx in range(seq_len): 
-                partial_idx_in_combined_pts = scanned_mask == seq_idx # Ndarray(V), N->V idx mask
-                partial_perpoint_feat = perpoint_scanned_feat[partial_idx_in_combined_pts] # Ndarray(V x Dl)
-                partial_feat = torch.mean(partial_perpoint_feat, dim=0)[0] # Tensor(Dl)
-                partial_feat_seq.append(partial_feat)
-                scanned_target_pts_num.append(partial_perpoint_feat.shape[0])
-                
-            scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.int32).to(device) # Tensor(S)
-            scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
-            
-            pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp)
-            pts_num_feat_seq = self.pts_num_encoder.encode_pts_num(scanned_target_pts_num)  # Tensor(S x Dn)
-            partial_feat_seq = torch.stack(partial_feat_seq) # Tensor(S x Dl)
-            
-            seq_embedding = torch.cat([pose_feat_seq, pts_num_feat_seq, partial_feat_seq], dim=-1) # Tensor(S x (Dp+Dn+Dl))
-            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dn+Dl))
-
-        seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
+            pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp) 
 
+            seq_embedding = torch.cat([partial_point_feat_seq, pose_feat_seq], dim=-1)
+            
+            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp))
+        
+        seq_feat = self.seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
         main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))
 
         if torch.isnan(main_feat).any():
+            for i in range(len(main_feat)):
+                if torch.isnan(main_feat[i]).any():
+                    scanned_pts_mask = scanned_pts_mask_batch[i]
+                    Log.info(f"scanned_pts_mask shape: {scanned_pts_mask.shape}")
+                    Log.info(f"scanned_pts_mask sum: {scanned_pts_mask.sum()}")
+                    import ipdb
+                    ipdb.set_trace()
             Log.error("nan in main_feat", True)
 
         return main_feat

preprocess/clean_preprocessed_data.py

@@ -0,0 +1,43 @@
+import os
+import shutil
+
+def clean_scene_data(root, scene):
+    # 清理目标点云数据
+    pts_dir = os.path.join(root, scene, "pts")
+    if os.path.exists(pts_dir):
+        shutil.rmtree(pts_dir)
+        print(f"已删除 {pts_dir}")
+
+    # 清理法线数据
+    nrm_dir = os.path.join(root, scene, "nrm")
+    if os.path.exists(nrm_dir):
+        shutil.rmtree(nrm_dir)
+        print(f"已删除 {nrm_dir}")
+
+    # 清理扫描点索引数据
+    scan_points_indices_dir = os.path.join(root, scene, "scan_points_indices")
+    if os.path.exists(scan_points_indices_dir):
+        shutil.rmtree(scan_points_indices_dir)
+        print(f"已删除 {scan_points_indices_dir}")
+
+    # 删除扫描点数据文件
+    scan_points_file = os.path.join(root, scene, "scan_points.txt")
+    if os.path.exists(scan_points_file):
+        os.remove(scan_points_file)
+        print(f"已删除 {scan_points_file}")
+
+def clean_all_scenes(root, scene_list):
+    for idx, scene in enumerate(scene_list):
+        print(f"正在清理场景 {scene} ({idx+1}/{len(scene_list)})")
+        clean_scene_data(root, scene)
+
+if __name__ == "__main__":
+    root = r"c:\Document\Local Project\nbv_rec\nbv_reconstruction\temp"
+    scene_list = os.listdir(root)
+    from_idx = 0
+    to_idx = len(scene_list)
+    print(f"正在清理场景 {scene_list[from_idx:to_idx]}")
+
+    clean_all_scenes(root, scene_list[from_idx:to_idx])
+    print("清理完成")
+

preprocess/pack_preprocessed_data.py

@@ -0,0 +1,48 @@
+import os
+import shutil
+
+def pack_scene_data(root, scene, output_dir):
+    scene_dir = os.path.join(output_dir, scene)
+    if not os.path.exists(scene_dir):
+        os.makedirs(scene_dir)
+
+    pts_dir = os.path.join(root, scene, "pts")
+    if os.path.exists(pts_dir):
+        shutil.move(pts_dir, os.path.join(scene_dir, "pts"))
+        
+    scan_points_indices_dir = os.path.join(root, scene, "scan_points_indices")
+    if os.path.exists(scan_points_indices_dir):
+        shutil.move(scan_points_indices_dir, os.path.join(scene_dir, "scan_points_indices")) 
+
+    scan_points_file = os.path.join(root, scene, "scan_points.txt")
+    if os.path.exists(scan_points_file):
+        shutil.move(scan_points_file, os.path.join(scene_dir, "scan_points.txt"))
+
+    model_pts_nrm_file = os.path.join(root, scene, "points_and_normals.txt")
+    if os.path.exists(model_pts_nrm_file):
+        shutil.move(model_pts_nrm_file, os.path.join(scene_dir, "points_and_normals.txt"))
+
+    camera_dir = os.path.join(root, scene, "camera_params")
+    if os.path.exists(camera_dir):
+        shutil.move(camera_dir, os.path.join(scene_dir, "camera_params"))
+
+    scene_info_file = os.path.join(root, scene, "scene_info.json")
+    if os.path.exists(scene_info_file):
+        shutil.move(scene_info_file, os.path.join(scene_dir, "scene_info.json"))
+
+def pack_all_scenes(root, scene_list, output_dir):
+    for idx, scene in enumerate(scene_list):
+        print(f"正在打包场景 {scene} ({idx+1}/{len(scene_list)})")
+        pack_scene_data(root, scene, output_dir)
+
+if __name__ == "__main__":
+    root = r"H:\AI\Datasets\nbv_rec_part2"
+    output_dir = r"H:\AI\Datasets\scene_info_part2"
+    scene_list = os.listdir(root)
+    from_idx = 0
+    to_idx = len(scene_list)
+    print(f"正在打包场景 {scene_list[from_idx:to_idx]}")
+
+    pack_all_scenes(root, scene_list[from_idx:to_idx], output_dir)
+    print("打包完成")
+

preprocess/pack_upload_data.py

@@ -0,0 +1,41 @@
+import os
+import shutil
+
+def pack_scene_data(root, scene, output_dir):
+    scene_dir = os.path.join(output_dir, scene)
+    if not os.path.exists(scene_dir):
+        os.makedirs(scene_dir)
+
+    pts_dir = os.path.join(root, scene, "pts")
+    if os.path.exists(pts_dir):
+        shutil.move(pts_dir, os.path.join(scene_dir, "pts"))
+
+    camera_dir = os.path.join(root, scene, "camera_params")
+    if os.path.exists(camera_dir):
+        shutil.move(camera_dir, os.path.join(scene_dir, "camera_params"))
+
+    scene_info_file = os.path.join(root, scene, "scene_info.json")
+    if os.path.exists(scene_info_file):
+        shutil.move(scene_info_file, os.path.join(scene_dir, "scene_info.json"))
+
+    label_dir = os.path.join(root, scene, "label")
+    if os.path.exists(label_dir):
+        shutil.move(label_dir, os.path.join(scene_dir, "label"))
+
+
+def pack_all_scenes(root, scene_list, output_dir):
+    for idx, scene in enumerate(scene_list):
+        print(f"packing {scene} ({idx+1}/{len(scene_list)})")
+        pack_scene_data(root, scene, output_dir)
+
+if __name__ == "__main__":
+    root = r"H:\AI\Datasets\nbv_rec_part2"
+    output_dir = r"H:\AI\Datasets\upload_part2"
+    scene_list = os.listdir(root)
+    from_idx = 0
+    to_idx = len(scene_list)
+    print(f"packing {scene_list[from_idx:to_idx]}")
+
+    pack_all_scenes(root, scene_list[from_idx:to_idx], output_dir)
+    print("packing done")
+

preprocess/preprocessor.py

@@ -91,8 +91,8 @@ def get_scan_points_indices(scan_points, mask, display_table_mask_label, cam_int
 def save_scene_data(root, scene, scene_idx=0, scene_total=1,file_type="txt"):
     
     ''' configuration '''
-    target_mask_label = (0, 255, 0, 255)
-    display_table_mask_label=(0, 0, 255, 255)
+    target_mask_label = (0, 255, 0)
+    display_table_mask_label=(0, 0, 255)
     random_downsample_N = 32768
     voxel_size=0.003
     filter_degree = 75
@@ -153,6 +153,7 @@ def save_scene_data(root, scene, scene_idx=0, scene_total=1,file_type="txt"):
         
         if not has_points:
             target_points = np.zeros((0, 3))
+            target_normals = np.zeros((0, 3))
             
         save_target_points(root, scene, frame_id, target_points, file_type=file_type)
         save_target_normals(root, scene, frame_id, target_normals, file_type=file_type)
@@ -163,17 +164,10 @@ def save_scene_data(root, scene, scene_idx=0, scene_total=1,file_type="txt"):
 
 if __name__ == "__main__":
     #root = "/media/hofee/repository/new_data_with_normal"
-    root = r"C:\\Document\\Local Project\\nbv_rec\\nbv_reconstruction\\temp"
-    # list_path = r"/media/hofee/repository/full_list.txt"
-    # scene_list = []
-    
-    # with open(list_path, "r") as f:
-    #     for line in f:
-    #         scene_list.append(line.strip())
+    root = r"H:\AI\Datasets\nbv_rec_part2"
     scene_list = os.listdir(root)
     from_idx = 0 # 1000
-    to_idx = 1 # 1500
-    print(scene_list)
+    to_idx = 600 # 1500
 
 
     cnt = 0
@@ -181,6 +175,10 @@ if __name__ == "__main__":
     total = to_idx - from_idx
     for scene in scene_list[from_idx:to_idx]:
         start = time.time()
+        if os.path.exists(os.path.join(root, scene, "scan_points.txt")):
+            print(f"Scene {scene} has been processed")
+            cnt+=1
+            continue
         save_scene_data(root, scene, cnt, total, file_type="npy")
         cnt+=1
         end = time.time()

runners/inferece_server.py

@@ -0,0 +1,109 @@
+import os
+import json
+import torch
+import numpy as np
+from flask import Flask, request, jsonify
+
+import PytorchBoot.namespace as namespace
+import PytorchBoot.stereotype as stereotype
+from PytorchBoot.factory import ComponentFactory
+
+from PytorchBoot.runners.runner import Runner
+from PytorchBoot.utils import Log
+
+from utils.pts import PtsUtil
+
+@stereotype.runner("inferencer")
+class InferencerServer(Runner):
+    def __init__(self, config_path):
+        super().__init__(config_path)
+    
+        ''' Web Server '''
+        self.app = Flask(__name__)
+        ''' Pipeline '''
+        self.pipeline_name = self.config[namespace.Stereotype.PIPELINE]
+        self.pipeline:torch.nn.Module = ComponentFactory.create(namespace.Stereotype.PIPELINE, self.pipeline_name)
+        self.pipeline = self.pipeline.to(self.device)
+            
+        ''' Experiment '''
+        self.load_experiment("nbv_evaluator")
+
+    def get_input_data(self, data):
+        input_data = {}
+        scanned_pts = data["scanned_pts"]
+        scanned_n_to_world_pose_9d = data["scanned_n_to_world_pose_9d"]
+        combined_scanned_views_pts = np.concatenate(scanned_pts, axis=0)
+        fps_downsampled_combined_scanned_pts, fps_idx = PtsUtil.fps_downsample_point_cloud(
+            combined_scanned_views_pts, self.pts_num, require_idx=True
+        )
+        combined_scanned_views_pts_mask = np.zeros(len(scanned_pts), dtype=np.uint8)
+        start_idx = 0
+        for i in range(len(scanned_pts)):
+            end_idx = start_idx + len(scanned_pts[i])
+            combined_scanned_views_pts_mask[start_idx:end_idx] = i
+            start_idx = end_idx
+
+        fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx]
+
+        input_data["scanned_pts_mask"] = np.asarray(fps_downsampled_combined_scanned_pts_mask, dtype=np.uint8)
+        input_data["scanned_n_to_world_pose_9d"] = np.asarray(scanned_n_to_world_pose_9d, dtype=np.float32)
+        input_data["combined_scanned_pts"] = np.asarray(fps_downsampled_combined_scanned_pts, dtype=np.float32)
+        return input_data
+    
+    def get_result(self, output_data):
+
+        estimated_delta_rot_9d = output_data["pred_pose_9d"]
+        result = {
+            "estimated_delta_rot_9d": estimated_delta_rot_9d.tolist()
+        }
+        return result
+    
+    def run(self):
+        Log.info("Loading from epoch {}.".format(self.current_epoch))
+    
+        @self.app.route("/inference", methods=["POST"])
+        def inference():
+            data = request.json
+            input_data = self.get_input_data(data)
+            output_data = self.pipeline.forward_test(input_data)
+            result = self.get_result(output_data)
+            return jsonify(result)
+        
+
+        self.app.run(host="0.0.0.0", port=5000)
+
+    def get_checkpoint_path(self, is_last=False):
+        return os.path.join(self.experiment_path, namespace.Direcotry.CHECKPOINT_DIR_NAME,
+                            "Epoch_{}.pth".format(
+                                self.current_epoch if self.current_epoch != -1 and not is_last else "last"))
+
+    def load_checkpoint(self, is_last=False):
+        self.load(self.get_checkpoint_path(is_last))
+        Log.success(f"Loaded checkpoint from {self.get_checkpoint_path(is_last)}")
+        if is_last:
+            checkpoint_root = os.path.join(self.experiment_path, namespace.Direcotry.CHECKPOINT_DIR_NAME)
+            meta_path = os.path.join(checkpoint_root, "meta.json")
+            if not os.path.exists(meta_path):
+                raise FileNotFoundError(
+                    "No checkpoint meta.json file in the experiment {}".format(self.experiments_config["name"]))
+            file_path = os.path.join(checkpoint_root, "meta.json")
+            with open(file_path, "r") as f:
+                meta = json.load(f)
+            self.current_epoch = meta["last_epoch"]
+            self.current_iter = meta["last_iter"]
+
+    def load_experiment(self, backup_name=None):
+        super().load_experiment(backup_name)
+        self.current_epoch = self.experiments_config["epoch"]
+        self.load_checkpoint(is_last=(self.current_epoch == -1))
+
+    def create_experiment(self, backup_name=None):
+        super().create_experiment(backup_name)
+       
+        
+    def load(self, path):
+        state_dict = torch.load(path)
+        self.pipeline.load_state_dict(state_dict)
+
+
+

runners/strategy_generator.py

@@ -24,12 +24,15 @@ class StrategyGenerator(Runner):
         }
         self.overwrite = ConfigManager.get("runner", "generate", "overwrite")
         self.seq_num = ConfigManager.get("runner","generate","seq_num")
+        self.overlap_area_threshold = ConfigManager.get("runner","generate","overlap_area_threshold")
+        self.compute_with_normal = ConfigManager.get("runner","generate","compute_with_normal")
+        self.scan_points_threshold = ConfigManager.get("runner","generate","scan_points_threshold")
         
 
         
     def run(self):
         dataset_name_list =  ConfigManager.get("runner", "generate", "dataset_list")
-        voxel_threshold, soft_overlap_threshold, hard_overlap_threshold = ConfigManager.get("runner","generate","voxel_threshold"), ConfigManager.get("runner","generate","soft_overlap_threshold"), ConfigManager.get("runner","generate","hard_overlap_threshold")
+        voxel_threshold = ConfigManager.get("runner","generate","voxel_threshold")
         for dataset_idx in range(len(dataset_name_list)):
             dataset_name = dataset_name_list[dataset_idx]
             status_manager.set_progress("generate_strategy", "strategy_generator", "dataset", dataset_idx, len(dataset_name_list))
@@ -51,7 +54,7 @@ class StrategyGenerator(Runner):
                     cnt += 1
                     continue
                 
-                self.generate_sequence(root_dir, scene_name,voxel_threshold, soft_overlap_threshold, hard_overlap_threshold)
+                self.generate_sequence(root_dir, scene_name,voxel_threshold)
                 cnt += 1
             status_manager.set_progress("generate_strategy", "strategy_generator", "scene", total, total)
         status_manager.set_progress("generate_strategy", "strategy_generator", "dataset", len(dataset_name_list), len(dataset_name_list))
@@ -64,28 +67,36 @@ class StrategyGenerator(Runner):
     def load_experiment(self, backup_name=None):
         super().load_experiment(backup_name)
     
-    def generate_sequence(self, root, scene_name, voxel_threshold, soft_overlap_threshold, hard_overlap_threshold):
+    def generate_sequence(self, root, scene_name, voxel_threshold):
         status_manager.set_status("generate_strategy", "strategy_generator", "scene", scene_name)
         frame_num = DataLoadUtil.get_scene_seq_length(root, scene_name)
         
         model_points_normals = DataLoadUtil.load_points_normals(root, scene_name)
         model_pts = model_points_normals[:,:3]
-        down_sampled_model_pts = PtsUtil.voxel_downsample_point_cloud(model_pts, voxel_threshold)
+        down_sampled_model_pts, idx = PtsUtil.voxel_downsample_point_cloud(model_pts, voxel_threshold, require_idx=True)
+        down_sampled_model_nrm = model_points_normals[idx, 3:]
         pts_list = []
+        nrm_list = []
         scan_points_indices_list = []
         non_zero_cnt = 0
 
         for frame_idx in range(frame_num):
             status_manager.set_progress("generate_strategy", "strategy_generator", "loading frame", frame_idx, frame_num)
             pts_path = os.path.join(root,scene_name, "pts", f"{frame_idx}.npy")
+            nrm_path = os.path.join(root,scene_name, "nrm", f"{frame_idx}.npy")
             idx_path = os.path.join(root,scene_name, "scan_points_indices", f"{frame_idx}.npy")
-            point_cloud = np.load(pts_path)
-            sampled_point_cloud = PtsUtil.voxel_downsample_point_cloud(point_cloud, voxel_threshold)
-            indices = np.load(idx_path) 
-            pts_list.append(sampled_point_cloud)
             
+            pts = np.load(pts_path)
+            if self.compute_with_normal:
+                if pts.shape[0] == 0:
+                    nrm = np.zeros((0,3))
+                else:
+                    nrm = np.load(nrm_path)
+                nrm_list.append(nrm)
+            pts_list.append(pts)
+            indices = np.load(idx_path) 
             scan_points_indices_list.append(indices)
-            if sampled_point_cloud.shape[0] > 0:
+            if pts.shape[0] > 0:
                 non_zero_cnt += 1
         status_manager.set_progress("generate_strategy", "strategy_generator", "loading frame", frame_num, frame_num)
         
@@ -93,7 +104,7 @@ class StrategyGenerator(Runner):
         init_view_list = []
         idx = 0
         while len(init_view_list) < seq_num and idx < len(pts_list):
-            if pts_list[idx].shape[0] > 100:
+            if pts_list[idx].shape[0] > 50:
                 init_view_list.append(idx)
             idx += 1
 
@@ -102,8 +113,13 @@ class StrategyGenerator(Runner):
         for init_view in init_view_list:
             status_manager.set_progress("generate_strategy", "strategy_generator", "computing sequence", seq_idx, len(init_view_list))
             start = time.time()
-            limited_useful_view, _, _ = ReconstructionUtil.compute_next_best_view_sequence_with_overlap(down_sampled_model_pts, pts_list, scan_points_indices_list = scan_points_indices_list,init_view=init_view, 
-                                                                                                        threshold=voxel_threshold, soft_overlap_threshold=soft_overlap_threshold, hard_overlap_threshold= hard_overlap_threshold, scan_points_threshold=10, status_info=self.status_info)
+
+            if not self.compute_with_normal:    
+                limited_useful_view, _, _ = ReconstructionUtil.compute_next_best_view_sequence(down_sampled_model_pts, pts_list, scan_points_indices_list = scan_points_indices_list,init_view=init_view, 
+                                                                                                            threshold=voxel_threshold, scan_points_threshold=self.scan_points_threshold, overlap_area_threshold=self.overlap_area_threshold, status_info=self.status_info)
+            else:
+                limited_useful_view, _, _ = ReconstructionUtil.compute_next_best_view_sequence_with_normal(down_sampled_model_pts, down_sampled_model_nrm, pts_list, nrm_list, scan_points_indices_list = scan_points_indices_list,init_view=init_view, 
+                                                                                                            threshold=voxel_threshold, scan_points_threshold=self.scan_points_threshold, overlap_area_threshold=self.overlap_area_threshold, status_info=self.status_info)
             end = time.time()
             print(f"Time: {end-start}")
             data_pairs = self.generate_data_pairs(limited_useful_view)

runners/view_generator.py

@@ -9,7 +9,7 @@ class ViewGenerator(Runner):
         self.config_path = config_path
         
     def run(self):
-        result = subprocess.run(['blender', '-b', '-P', '../blender/run_blender.py', '--', self.config_path])
+        result = subprocess.run(['/home/hofee/blender-4.0.2-linux-x64/blender', '-b', '-P', '../blender/run_blender.py', '--', self.config_path])
         print()
 
     def create_experiment(self, backup_name=None):

utils/data_load.py

@@ -22,8 +22,10 @@ class DataLoadUtil:
         float_channels = ['R', 'G', 'B']
         img_data = []
         for channel in float_channels:
-            channel_data = exr_file.channel(channel, Imath.PixelType(Imath.PixelType.FLOAT))
-            img_data.append(np.frombuffer(channel_data, dtype=np.float32).reshape((height, width)))
+            channel_data = exr_file.channel(channel)
+            img_data.append(np.frombuffer(channel_data, dtype=np.float16).reshape((height, width)))
+
+        # 将各通道组合成一个 (height, width, 3) 的 RGB 图像
         img = np.stack(img_data, axis=-1)
         return img
 
@@ -51,6 +53,8 @@ class DataLoadUtil:
     @staticmethod
     def get_label_num(root, scene_name):
         label_dir = os.path.join(root, scene_name, "label")
+        if not os.path.exists(label_dir):
+            return 0
         return len(os.listdir(label_dir))
 
     @staticmethod
@@ -132,8 +136,8 @@ class DataLoadUtil:
         if binocular and not left_only:
 
             def clean_mask(mask_image):
-                green = [0, 255, 0, 255]
-                red = [255, 0, 0, 255]
+                green = [0, 255, 0]
+                red = [255, 0, 0]
                 threshold = 2
                 mask_image = np.where(
                     np.abs(mask_image - green) <= threshold, green, mask_image
@@ -208,6 +212,17 @@ class DataLoadUtil:
         else:
             pts = np.load(npy_path)
         return pts
+    
+    @staticmethod
+    def load_from_preprocessed_nrm(path, file_type="npy"):
+        npy_path = os.path.join(
+            os.path.dirname(path), "nrm", os.path.basename(path) + "." + file_type
+        )
+        if file_type == "txt":
+            nrm = np.loadtxt(npy_path)
+        else:
+            nrm = np.load(npy_path)
+        return nrm
 
     @staticmethod
     def cam_pose_transformation(cam_pose_before):

utils/pts.py

@@ -5,18 +5,47 @@ import torch
 class PtsUtil:
 
     @staticmethod
-    def voxel_downsample_point_cloud(point_cloud, voxel_size=0.005):
+    def voxel_downsample_point_cloud(point_cloud, voxel_size=0.005, require_idx=False):
         voxel_indices = np.floor(point_cloud / voxel_size).astype(np.int32)
-        unique_voxels = np.unique(voxel_indices, axis=0, return_inverse=True)
-        return unique_voxels[0]*voxel_size
+        if require_idx:
+            _, inverse, counts = np.unique(voxel_indices, axis=0, return_inverse=True, return_counts=True)
+            idx_sort = np.argsort(inverse)
+            idx_unique = idx_sort[np.cumsum(counts)-counts]
+            downsampled_points = point_cloud[idx_unique]
+            return downsampled_points, idx_unique
+        else:
+            import ipdb; ipdb.set_trace()
+            unique_voxels = np.unique(voxel_indices, axis=0, return_inverse=False)
+            return unique_voxels*voxel_size
+
+    @staticmethod
+    def voxel_downsample_point_cloud_o3d(point_cloud, voxel_size=0.005):
+        pcd = o3d.geometry.PointCloud()
+        pcd.points = o3d.utility.Vector3dVector(point_cloud)
+        pcd = pcd.voxel_down_sample(voxel_size)
+        return np.asarray(pcd.points)
     
     @staticmethod
-    def random_downsample_point_cloud(point_cloud, num_points, require_idx=False):
+    def voxel_downsample_point_cloud_and_trace_o3d(point_cloud, voxel_size=0.005):
+        pcd = o3d.geometry.PointCloud()
+        pcd.points = o3d.utility.Vector3dVector(point_cloud)
+        max_bound = pcd.get_max_bound()
+        min_bound = pcd.get_min_bound()
+        pcd = pcd.voxel_down_sample_and_trace(voxel_size, max_bound, min_bound, True)
+        
+        return np.asarray(pcd.points)
+       
+    @staticmethod
+    def random_downsample_point_cloud(point_cloud, num_points, require_idx=False, replace=True):
         if point_cloud.shape[0] == 0:
             if require_idx:
                 return point_cloud, np.array([])
             return point_cloud
-        idx = np.random.choice(len(point_cloud), num_points, replace=True)
+        if not replace and num_points > len(point_cloud):
+            if require_idx:
+                return point_cloud, np.arange(len(point_cloud))
+            return point_cloud
+        idx = np.random.choice(len(point_cloud), num_points, replace=replace)
         if require_idx:
             return point_cloud[idx], idx
         return point_cloud[idx]

utils/reconstruction.py

@@ -8,16 +8,23 @@ class ReconstructionUtil:
     def compute_coverage_rate(target_point_cloud, combined_point_cloud, threshold=0.01):
         kdtree = cKDTree(combined_point_cloud)
         distances, _ = kdtree.query(target_point_cloud)
-        covered_points_num = np.sum(distances < threshold)
+        covered_points_num = np.sum(distances < threshold*2)
         coverage_rate = covered_points_num / target_point_cloud.shape[0]
         return coverage_rate, covered_points_num
     
+    @staticmethod
     def compute_coverage_rate_with_normal(target_point_cloud, combined_point_cloud, target_normal, combined_normal, threshold=0.01, normal_threshold=0.1):
         kdtree = cKDTree(combined_point_cloud)
         distances, indices = kdtree.query(target_point_cloud)
-        is_covered_by_distance = distances < threshold
+        is_covered_by_distance = distances < threshold*2
         normal_dots = np.einsum('ij,ij->i', target_normal, combined_normal[indices])
         is_covered_by_normal = normal_dots > normal_threshold
+
+        pts_nrm_target = np.hstack([target_point_cloud, target_normal])
+        np.savetxt("pts_nrm_target.txt",  pts_nrm_target)
+        pts_nrm_combined = np.hstack([combined_point_cloud, combined_normal])
+        np.savetxt("pts_nrm_combined.txt", pts_nrm_combined)
+        import ipdb; ipdb.set_trace()
         covered_points_num = np.sum(is_covered_by_distance & is_covered_by_normal)
         coverage_rate = covered_points_num / target_point_cloud.shape[0]
 
@@ -25,15 +32,14 @@ class ReconstructionUtil:
         
     
     @staticmethod
-    def compute_overlap_rate(new_point_cloud, combined_point_cloud, threshold=0.01):
+    def check_overlap(new_point_cloud, combined_point_cloud, overlap_area_threshold=25, voxel_size=0.01):
         kdtree = cKDTree(combined_point_cloud)
         distances, _ = kdtree.query(new_point_cloud)
-        overlapping_points = np.sum(distances < threshold)
-        if new_point_cloud.shape[0] == 0:
-            overlap_rate = 0
-        else:
-            overlap_rate = overlapping_points / new_point_cloud.shape[0]
-        return overlap_rate
+        overlapping_points = np.sum(distances < voxel_size*2)
+        cm = 0.01
+        voxel_size_cm = voxel_size / cm
+        overlap_area = overlapping_points * voxel_size_cm * voxel_size_cm
+        return overlap_area > overlap_area_threshold
 
 
     @staticmethod
@@ -49,14 +55,14 @@ class ReconstructionUtil:
         return new_added_points
     
     @staticmethod
-    def compute_next_best_view_sequence_with_overlap(target_point_cloud, point_cloud_list, scan_points_indices_list, threshold=0.01, soft_overlap_threshold=0.5, hard_overlap_threshold=0.7, init_view = 0, scan_points_threshold=5, status_info=None):
+    def compute_next_best_view_sequence(target_point_cloud, point_cloud_list, scan_points_indices_list, threshold=0.01, overlap_area_threshold=25, init_view = 0, scan_points_threshold=5, status_info=None):
         selected_views = [init_view]
         combined_point_cloud = point_cloud_list[init_view]
         history_indices = [scan_points_indices_list[init_view]]
         
         max_rec_pts = np.vstack(point_cloud_list)
         downsampled_max_rec_pts = PtsUtil.voxel_downsample_point_cloud(max_rec_pts, threshold)
-        
+        combined_point_cloud = PtsUtil.voxel_downsample_point_cloud(combined_point_cloud, threshold)
         max_rec_pts_num = downsampled_max_rec_pts.shape[0]
         max_real_rec_pts_coverage, _ = ReconstructionUtil.compute_coverage_rate(target_point_cloud, downsampled_max_rec_pts, threshold)
         
@@ -69,6 +75,7 @@ class ReconstructionUtil:
         cnt_processed_view = 0
         remaining_views.remove(init_view)
         curr_rec_pts_num = combined_point_cloud.shape[0]
+        drop_output_ratio = 0.4
         
         import time
         while remaining_views:
@@ -78,27 +85,23 @@ class ReconstructionUtil:
             best_covered_num = 0
         
             for view_index in remaining_views:
+                if np.random.rand() < drop_output_ratio:
+                    continue
                 if point_cloud_list[view_index].shape[0] == 0:
                     continue
                 if selected_views:
                     new_scan_points_indices = scan_points_indices_list[view_index]
                     if not ReconstructionUtil.check_scan_points_overlap(history_indices, new_scan_points_indices, scan_points_threshold):
-                        overlap_threshold = hard_overlap_threshold
+                        curr_overlap_area_threshold = overlap_area_threshold
                     else:
-                        overlap_threshold = soft_overlap_threshold
-                    start = time.time()
-                    overlap_rate = ReconstructionUtil.compute_overlap_rate(point_cloud_list[view_index],combined_point_cloud, threshold)
-                    end = time.time()
-                    # print(f"overlap_rate Time: {end-start}")
-                    if overlap_rate < overlap_threshold:
+                        curr_overlap_area_threshold = overlap_area_threshold * 0.5
+                        
+                    if not ReconstructionUtil.check_overlap(point_cloud_list[view_index], combined_point_cloud, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=threshold):
                         continue
                 
-                start = time.time()
                 new_combined_point_cloud = np.vstack([combined_point_cloud, point_cloud_list[view_index]])
                 new_downsampled_combined_point_cloud = PtsUtil.voxel_downsample_point_cloud(new_combined_point_cloud,threshold)
                 new_coverage, new_covered_num = ReconstructionUtil.compute_coverage_rate(downsampled_max_rec_pts, new_downsampled_combined_point_cloud, threshold)
-                end = time.time()   
-                #print(f"compute_coverage_rate Time: {end-start}")
                 coverage_increase = new_coverage - current_coverage
                 if coverage_increase > best_coverage_increase:
                     best_coverage_increase = coverage_increase
@@ -107,6 +110,100 @@ class ReconstructionUtil:
                     best_combined_point_cloud = new_downsampled_combined_point_cloud
                     
             
+            if best_view is not None:
+                if best_coverage_increase <=1e-3 or best_covered_num - current_covered_num <= 5:
+                    break
+                
+                selected_views.append(best_view)
+                best_rec_pts_num = best_combined_point_cloud.shape[0]
+                print(f"Current rec pts num: {curr_rec_pts_num}, Best rec pts num: {best_rec_pts_num}, Best cover pts: {best_covered_num}, Max rec pts num: {max_rec_pts_num}")
+                print(f"Current coverage: {current_coverage+best_coverage_increase}, Best coverage increase: {best_coverage_increase}, Max Real coverage: {max_real_rec_pts_coverage}")
+                current_covered_num = best_covered_num
+                curr_rec_pts_num = best_rec_pts_num
+                combined_point_cloud = best_combined_point_cloud
+                remaining_views.remove(best_view)
+                history_indices.append(scan_points_indices_list[best_view])
+                current_coverage += best_coverage_increase
+                cnt_processed_view += 1
+                if status_info is not None:
+                    sm = status_info["status_manager"]
+                    app_name = status_info["app_name"]
+                    runner_name = status_info["runner_name"]
+                    sm.set_status(app_name, runner_name, "current coverage", current_coverage)
+                    sm.set_progress(app_name, runner_name, "processed view", cnt_processed_view, len(point_cloud_list))
+            
+                view_sequence.append((best_view, current_coverage))
+                
+            else:
+                break
+        if status_info is not None:
+            sm = status_info["status_manager"]
+            app_name = status_info["app_name"]
+            runner_name = status_info["runner_name"]
+            sm.set_progress(app_name, runner_name, "processed view", len(point_cloud_list), len(point_cloud_list))
+        return view_sequence, remaining_views, combined_point_cloud
+    
+    @staticmethod
+    def compute_next_best_view_sequence_with_normal(target_point_cloud, target_normal, point_cloud_list, normal_list, scan_points_indices_list, threshold=0.01, overlap_area_threshold=25, init_view = 0, scan_points_threshold=5, status_info=None):
+        selected_views = [init_view]
+        combined_point_cloud = point_cloud_list[init_view]
+        combined_normal = normal_list[init_view]
+        history_indices = [scan_points_indices_list[init_view]]
+        
+        max_rec_pts = np.vstack(point_cloud_list)
+        max_rec_nrm = np.vstack(normal_list)
+        downsampled_max_rec_pts, idx = PtsUtil.voxel_downsample_point_cloud(max_rec_pts, threshold, require_idx=True)
+        downsampled_max_rec_nrm = max_rec_nrm[idx]  
+        max_rec_pts_num = downsampled_max_rec_pts.shape[0]
+        try:
+            max_real_rec_pts_coverage, _ = ReconstructionUtil.compute_coverage_rate_with_normal(target_point_cloud, downsampled_max_rec_pts, target_normal, downsampled_max_rec_nrm, threshold)
+        except:
+            import ipdb; ipdb.set_trace()
+        
+        new_coverage, new_covered_num = ReconstructionUtil.compute_coverage_rate_with_normal(downsampled_max_rec_pts, combined_point_cloud, downsampled_max_rec_nrm, combined_normal, threshold)
+        current_coverage = new_coverage
+        current_covered_num = new_covered_num
+        
+        remaining_views = list(range(len(point_cloud_list)))
+        view_sequence = [(init_view, current_coverage)]
+        cnt_processed_view = 0
+        remaining_views.remove(init_view)
+        curr_rec_pts_num = combined_point_cloud.shape[0]
+        
+        while remaining_views:
+            best_view = None
+            best_coverage_increase = -1
+            best_combined_point_cloud = None
+            best_combined_normal = None
+            best_covered_num = 0
+        
+            for view_index in remaining_views:
+                if point_cloud_list[view_index].shape[0] == 0:
+                    continue
+                if selected_views:
+                    new_scan_points_indices = scan_points_indices_list[view_index]
+                    if not ReconstructionUtil.check_scan_points_overlap(history_indices, new_scan_points_indices, scan_points_threshold):
+                        curr_overlap_area_threshold = overlap_area_threshold
+                    else:
+                        curr_overlap_area_threshold = overlap_area_threshold * 0.5
+                        
+                    if not ReconstructionUtil.check_overlap(point_cloud_list[view_index], combined_point_cloud, overlap_area_threshold = curr_overlap_area_threshold, voxel_size=threshold):
+                        continue
+                
+                new_combined_point_cloud = np.vstack([combined_point_cloud, point_cloud_list[view_index]])
+                new_combined_normal = np.vstack([combined_normal, normal_list[view_index]])
+                new_downsampled_combined_point_cloud, idx = PtsUtil.voxel_downsample_point_cloud(new_combined_point_cloud,threshold, require_idx=True)
+                new_downsampled_combined_normal = new_combined_normal[idx]
+                new_coverage, new_covered_num = ReconstructionUtil.compute_coverage_rate_with_normal(downsampled_max_rec_pts, new_downsampled_combined_point_cloud, downsampled_max_rec_nrm, new_downsampled_combined_normal, threshold)
+                coverage_increase = new_coverage - current_coverage
+                if coverage_increase > best_coverage_increase:
+                    best_coverage_increase = coverage_increase
+                    best_view = view_index
+                    best_covered_num = new_covered_num
+                    best_combined_point_cloud = new_downsampled_combined_point_cloud
+                    best_combined_normal = new_downsampled_combined_normal
+                    
+            
             if best_view is not None:
                 if best_coverage_increase <=1e-3 or best_covered_num - current_covered_num <= 5:
                     break
@@ -118,6 +215,7 @@ class ReconstructionUtil:
                 current_covered_num = best_covered_num
                 curr_rec_pts_num = best_rec_pts_num
                 combined_point_cloud = best_combined_point_cloud
+                combined_normal = best_combined_normal
                 remaining_views.remove(best_view)
                 history_indices.append(scan_points_indices_list[best_view])
                 current_coverage += best_coverage_increase

utils/vis.py

@@ -47,6 +47,42 @@ class visualizeUtil:
         all_combined_pts = np.vstack(all_combined_pts)
         downsampled_all_pts = PtsUtil.voxel_downsample_point_cloud(all_combined_pts, 0.001)
         np.savetxt(os.path.join(output_dir, "all_combined_pts.txt"), downsampled_all_pts)
+
+    @staticmethod
+    def save_seq_cam_pos_and_cam_axis(root, scene, frame_idx_list, output_dir):
+        all_cam_pos = []
+        all_cam_axis = []
+        for i in frame_idx_list:
+            path = DataLoadUtil.get_path(root, scene, i)
+            cam_info = DataLoadUtil.load_cam_info(path, binocular=True)
+            cam_pose = cam_info["cam_to_world"]
+            cam_pos = cam_pose[:3, 3]
+            cam_axis = cam_pose[:3, 2] 
+            
+            num_samples = 10
+            sample_points = [cam_pos + 0.02*t * cam_axis for t in range(num_samples)]
+            sample_points = np.array(sample_points)
+        
+            all_cam_pos.append(cam_pos)
+            all_cam_axis.append(sample_points)
+        
+        all_cam_pos = np.array(all_cam_pos)
+        all_cam_axis = np.array(all_cam_axis).reshape(-1, 3)
+        np.savetxt(os.path.join(output_dir, "seq_cam_pos.txt"), all_cam_pos)
+        np.savetxt(os.path.join(output_dir, "seq_cam_axis.txt"), all_cam_axis)
+        
+    @staticmethod
+    def save_seq_combined_pts(root, scene, frame_idx_list, output_dir):
+        all_combined_pts = []   
+        for i in frame_idx_list:
+            path = DataLoadUtil.get_path(root, scene, i)
+            pts = DataLoadUtil.load_from_preprocessed_pts(path,"npy")
+            if pts.shape[0] == 0:
+                continue
+            all_combined_pts.append(pts)
+        all_combined_pts = np.vstack(all_combined_pts)
+        downsampled_all_pts = PtsUtil.voxel_downsample_point_cloud(all_combined_pts, 0.001)
+        np.savetxt(os.path.join(output_dir, "seq_combined_pts.txt"), downsampled_all_pts)
         
     @staticmethod
     def save_target_mesh_at_world_space(
@@ -120,18 +156,37 @@ class visualizeUtil:
         sampled_visualized_normal = np.array(sampled_visualized_normal).reshape(-1, 3)
         np.savetxt(os.path.join(output_dir, "target_pts.txt"), sampled_target_points)
         np.savetxt(os.path.join(output_dir, "target_normal.txt"), sampled_visualized_normal)
+
+    @staticmethod
+    def save_pts_nrm(root, scene, frame_idx, output_dir, binocular=False):
+        path = DataLoadUtil.get_path(root, scene, frame_idx)
+        pts_world = DataLoadUtil.load_from_preprocessed_pts(path, "npy")
+        nrm_camera = DataLoadUtil.load_from_preprocessed_nrm(path, "npy")
+        cam_info = DataLoadUtil.load_cam_info(path, binocular=binocular)
+        cam_to_world = cam_info["cam_to_world"]
+        nrm_world = nrm_camera @ cam_to_world[:3, :3].T
+        visualized_nrm = []
+        num_samples = 10
+        for i in range(len(pts_world)):
+            for t in range(num_samples):
+                visualized_nrm.append(pts_world[i] - 0.02 * t * nrm_world[i])
+        
+        visualized_nrm = np.array(visualized_nrm)
+        np.savetxt(os.path.join(output_dir, "nrm.txt"), visualized_nrm)
+        np.savetxt(os.path.join(output_dir, "pts.txt"), pts_world)
         
-
-
 # ------ Debug ------
 
 if __name__ == "__main__":
-    root = r"/home/yan20/nbv_rec/project/franka_control/temp"
+    root = r"C:\Document\Local Project\nbv_rec\nbv_reconstruction\temp"
     model_dir = r"H:\\AI\\Datasets\\scaled_object_box_meshes"
     scene = "box"
     output_dir = r"C:\Document\Local Project\nbv_rec\nbv_reconstruction\test"
     
     #visualizeUtil.save_all_cam_pos_and_cam_axis(root, scene, output_dir)
-    visualizeUtil.save_all_combined_pts(root, scene, output_dir)
-    visualizeUtil.save_target_mesh_at_world_space(root, model_dir, scene)
-    #visualizeUtil.save_points_and_normals(root, scene,"10", output_dir, binocular=True)
+    # visualizeUtil.save_all_combined_pts(root, scene, output_dir)
+    # visualizeUtil.save_seq_combined_pts(root, scene, [0, 121, 286, 175, 111,366,45,230,232,225,255,17,199,78,60], output_dir)
+    # visualizeUtil.save_seq_cam_pos_and_cam_axis(root, scene, [0, 121, 286, 175, 111,366,45,230,232,225,255,17,199,78,60], output_dir)
+    # visualizeUtil.save_target_mesh_at_world_space(root, model_dir, scene)
+    #visualizeUtil.save_points_and_normals(root, scene,"10", output_dir, binocular=True)
+    visualizeUtil.save_pts_nrm(root, scene, "116", output_dir, binocular=True)