upd

app_split.py

@@ -5,5 +5,5 @@ from runners.data_spliter import DataSpliter
 class DataSplitApp:
     @staticmethod
     def start():
-        DataSpliter("configs/server/split_dataset_config.yaml").run()
+        DataSpliter("configs/server/server_split_dataset_config.yaml").run()
         

configs/local/strategy_generate_config.yaml

@@ -12,7 +12,7 @@ runner:
       
   generate:
     voxel_threshold: 0.003
-    overlap_area_threshold: 25
+    overlap_area_threshold: 30
     compute_with_normal: False
     scan_points_threshold: 10
     overwrite: False
@@ -22,8 +22,6 @@ runner:
   
 datasets:
     OmniObject3d:
-      root_dir: H:\\AI\\Datasets\\nbv_rec_part2
+      root_dir: /data/hofee/nbv_rec_part2_preprocessed
-      from: 0
+      from: 155
-      to: 300 # -1 means end
+      to: 165 # ..-1 means end
-
-    

configs/local/train_config.yaml

@@ -84,7 +84,7 @@ module:
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 2048
+    main_feat_dim: 3072
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False

configs/server/server_split_dataset_config.yaml

@@ -0,0 +1,22 @@
+
+runner:
+  general:
+    seed: 0
+    device: cpu
+    cuda_visible_devices: "0,1,2,3,4,5,6,7"
+    
+  experiment:
+    name: debug
+    root_dir: "experiments"
+  
+  split: # 
+    root_dir: "/data/hofee/data/packed_preprocessed_data"
+    type: "unseen_instance" # "unseen_category"
+    datasets:
+      OmniObject3d_train: 
+        path: "/data/hofee/data/OmniObject3d_train.txt"
+        ratio: 0.9
+
+      OmniObject3d_test: 
+        path: "/data/hofee/data/OmniObject3d_test.txt"
+        ratio: 0.1

configs/server/server_train_config.yaml

@@ -7,13 +7,13 @@ runner:
     parallel: False
     
   experiment:
-    name: full_w_global_feat_wo_local_pts_feat
+    name: overfit_ab_local_only
     root_dir: "experiments"
     use_checkpoint: False
     epoch: -1 # -1 stands for last epoch
     max_epochs: 5000
     save_checkpoint_interval: 1
-    test_first: True  
+    test_first: False  
 
   train:
     optimizer:
@@ -25,60 +25,60 @@ runner:
   test:
     frequency: 3 # test frequency
     dataset_list:
-      - OmniObject3d_test
+      #- OmniObject3d_test
       - OmniObject3d_val
 
-  pipeline: nbv_reconstruction_global_pts_pipeline
+  pipeline: nbv_reconstruction_pipeline
   
 dataset:
   OmniObject3d_train:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: train
     cache: True
     ratio: 1
-    batch_size: 160
+    batch_size: 32
     num_workers: 16
-    pts_num: 4096
+    pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_test:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.05
+    ratio: 1
-    batch_size: 160
+    batch_size: 32
     num_workers: 12
-    pts_num: 4096
+    pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_val:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.005
+    ratio: 1
-    batch_size: 160
+    batch_size: 32
     num_workers: 12
-    pts_num: 4096
+    pts_num: 8192
     load_from_preprocess: True
 
 
 pipeline:
-  nbv_reconstruction_local_pts_pipeline:
+  nbv_reconstruction_pipeline:
     modules:
       pts_encoder: pointnet_encoder
       seq_encoder: transformer_seq_encoder
@@ -87,27 +87,17 @@ pipeline:
     eps: 1e-5
     global_scanned_feat: True
 
-  nbv_reconstruction_global_pts_pipeline:
-    modules:
-      pts_encoder: pointnet_encoder
-      pose_seq_encoder: transformer_seq_encoder
-      pose_encoder: pose_encoder
-      view_finder: gf_view_finder
-    eps: 1e-5
-    global_scanned_feat: True
-
-
     
 module:
 
   pointnet_encoder:
     in_dim: 3
-    out_dim: 1024
+    out_dim: 512
     global_feat: True
     feature_transform: False
 
   transformer_seq_encoder:
-    embed_dim: 1344
+    embed_dim: 768
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
@@ -128,6 +118,9 @@ module:
     pose_dim: 9
     out_dim: 256
 
+  pts_num_encoder:
+    out_dim: 64
+
 loss_function:
   gf_loss:
 

core/nbv_dataset.py

@@ -8,7 +8,7 @@ import torch
 import os
 import sys
 
-sys.path.append(r"/home/data/hofee/project/nbv_rec/nbv_reconstruction")
+sys.path.append(r"/data/hofee/project/nbv_rec/nbv_reconstruction")
 
 from utils.data_load import DataLoadUtil
 from utils.pose import PoseUtil
@@ -31,10 +31,10 @@ class NBVReconstructionDataset(BaseDataset):
         self.load_from_preprocess = config.get("load_from_preprocess", False)
 
         if self.type == namespace.Mode.TEST:
-            self.model_dir = config["model_dir"]
+            #self.model_dir = config["model_dir"]
             self.filter_degree = config["filter_degree"]
         if self.type == namespace.Mode.TRAIN:
-            scale_ratio = 1
+            scale_ratio = 50
             self.datalist = self.datalist*scale_ratio
         if self.cache:
             expr_root = ConfigManager.get("runner", "experiment", "root_dir")
@@ -66,7 +66,9 @@ class NBVReconstructionDataset(BaseDataset):
                 if max_coverage_rate > scene_max_coverage_rate:
                     scene_max_coverage_rate = max_coverage_rate
                 max_coverage_rate_list.append(max_coverage_rate)
-            mean_coverage_rate = np.mean(max_coverage_rate_list)
+                
+            if max_coverage_rate_list:
+                mean_coverage_rate = np.mean(max_coverage_rate_list)
 
             for seq_idx in range(seq_num):
                 label_path = DataLoadUtil.get_label_path(
@@ -112,6 +114,10 @@ class NBVReconstructionDataset(BaseDataset):
         except Exception as e:
             Log.error(f"Save cache failed: {e}")
 
+    def voxel_downsample_with_mask(self, pts, voxel_size):
+        pass
+
+
     def __getitem__(self, index):
         data_item_info = self.datalist[index]
         scanned_views = data_item_info["scanned_views"]
@@ -122,7 +128,7 @@ class NBVReconstructionDataset(BaseDataset):
             scanned_views_pts,
             scanned_coverages_rate,
             scanned_n_to_world_pose,
-        ) = ([], [], [], [])
+        ) = ([], [], [])
         for view in scanned_views:
             frame_idx = view[0]
             coverage_rate = view[1]
@@ -159,28 +165,8 @@ class NBVReconstructionDataset(BaseDataset):
             [best_to_world_6d, best_to_world_trans], axis=0
         )
 
-        combined_scanned_views_pts = np.concatenate(scanned_views_pts, axis=0)
-        fps_downsampled_combined_scanned_pts, fps_idx = PtsUtil.fps_downsample_point_cloud(
-            combined_scanned_views_pts, self.pts_num, require_idx=True
-        )
-        
-        combined_scanned_views_pts_mask = np.zeros(len(scanned_views_pts), dtype=np.uint8)
-        
-        start_idx = 0
-        for i in range(len(scanned_views_pts)):
-            end_idx = start_idx + len(scanned_views_pts[i])
-            combined_scanned_views_pts_mask[start_idx:end_idx] = i
-            start_idx = end_idx
-            
-        fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx]
-        
-        
-        
-        
         data_item = {
             "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3)
-            "scanned_pts_mask": np.asarray(fps_downsampled_combined_scanned_pts_mask,dtype=np.uint8), # Ndarray(N), range(0, S)
-            "combined_scanned_pts": np.asarray(fps_downsampled_combined_scanned_pts, dtype=np.float32), # Ndarray(N x 3)
             "scanned_coverage_rate": scanned_coverages_rate, # List(S): Float, range(0, 1)
             "scanned_n_to_world_pose_9d": np.asarray(scanned_n_to_world_pose, dtype=np.float32), # Ndarray(S x 9)
             "best_coverage_rate": nbv_coverage_rate, # Float, range(0, 1)
@@ -212,12 +198,6 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["best_to_world_pose_9d"] = torch.stack(
                 [torch.tensor(item["best_to_world_pose_9d"]) for item in batch]
             )
-            collate_data["combined_scanned_pts"] = torch.stack(
-                [torch.tensor(item["combined_scanned_pts"]) for item in batch]
-            )
-            collate_data["scanned_pts_mask"] = torch.stack(
-                [torch.tensor(item["scanned_pts_mask"]) for item in batch]
-            )
                 
             for key in batch[0].keys():
                 if key not in [
@@ -225,7 +205,6 @@ class NBVReconstructionDataset(BaseDataset):
                     "scanned_pts_mask",
                     "scanned_n_to_world_pose_9d",
                     "best_to_world_pose_9d",
-                    "combined_scanned_pts",
                 ]:
                     collate_data[key] = [item[key] for item in batch]
             return collate_data
@@ -241,10 +220,9 @@ if __name__ == "__main__":
     torch.manual_seed(seed)
     np.random.seed(seed)
     config = {
-        "root_dir": "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy",
+        "root_dir": "/data/hofee/data/packed_preprocessed_data",
-        "model_dir": "/home/data/hofee/project/nbv_rec/data/scaled_object_meshes",
         "source": "nbv_reconstruction_dataset",
-        "split_file": "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt",
+        "split_file": "/data/hofee/data/OmniObject3d_train.txt",
         "load_from_preprocess": True,
         "ratio": 0.5,
         "batch_size": 2,

core/pipeline.py

@@ -1,4 +1,5 @@
 import torch
+import time
 from torch import nn
 import PytorchBoot.namespace as namespace
 import PytorchBoot.stereotype as stereotype
@@ -6,10 +7,10 @@ from PytorchBoot.factory.component_factory import ComponentFactory
 from PytorchBoot.utils import Log
 
 
-@stereotype.pipeline("nbv_reconstruction_global_pts_n_num_pipeline")
+@stereotype.pipeline("nbv_reconstruction_pipeline")
-class NBVReconstructionGlobalPointsPipeline(nn.Module):
+class NBVReconstructionPipeline(nn.Module):
     def __init__(self, config):
-        super(NBVReconstructionGlobalPointsPipeline, self).__init__()
+        super(NBVReconstructionPipeline, self).__init__()
         self.config = config
         self.module_config = config["modules"]
         
@@ -19,12 +20,8 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         self.pose_encoder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["pose_encoder"]
         )
-        self.pts_num_encoder = ComponentFactory.create(
+        self.seq_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["pts_num_encoder"]
+            namespace.Stereotype.MODULE, self.module_config["seq_encoder"]
-        )
-        
-        self.transformer_seq_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["transformer_seq_encoder"]
         )
         self.view_finder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["view_finder"]
@@ -58,7 +55,10 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         return perturbed_x, random_t, target_score, std
 
     def forward_train(self, data):
+        start_time = time.time()
         main_feat = self.get_main_feat(data)
+        end_time = time.time()
+        print("get_main_feat time: ", end_time - start_time)
         """ get std """
         best_to_world_pose_9d_batch = data["best_to_world_pose_9d"]
         perturbed_x, random_t, target_score, std = self.pertube_data(
@@ -92,48 +92,23 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         scanned_n_to_world_pose_9d_batch = data[
             "scanned_n_to_world_pose_9d"
         ]  # List(B): Tensor(S x 9)
-        scanned_pts_mask_batch = data[
+        scanned_pts_batch = data[
-            "scanned_pts_mask"
+            "scanned_pts"
-        ]  # Tensor(B x N)
+        ]
-
         device = next(self.parameters()).device
 
         embedding_list_batch = []
 
-        combined_scanned_pts_batch = data["combined_scanned_pts"]  # Tensor(B x N x 3)
+        for scanned_n_to_world_pose_9d, scanned_pts in zip(scanned_n_to_world_pose_9d_batch, scanned_pts_batch):
-        global_scanned_feat, perpoint_scanned_feat_batch = self.pts_encoder.encode_points(
-            combined_scanned_pts_batch, require_per_point_feat=True
-        )  # global_scanned_feat: Tensor(B x Dg), perpoint_scanned_feat: Tensor(B x N x Dl)
-
-        for scanned_n_to_world_pose_9d, scanned_mask, perpoint_scanned_feat in zip(
-            scanned_n_to_world_pose_9d_batch,
-            scanned_pts_mask_batch,
-            perpoint_scanned_feat_batch,
-        ):
-            scanned_target_pts_num = [] # List(S): Int
-            partial_feat_seq = []
-            
-            seq_len = len(scanned_n_to_world_pose_9d)
-            for seq_idx in range(seq_len): 
-                partial_idx_in_combined_pts = scanned_mask == seq_idx # Ndarray(V), N->V idx mask
-                partial_perpoint_feat = perpoint_scanned_feat[partial_idx_in_combined_pts] # Ndarray(V x Dl)
-                partial_feat = torch.mean(partial_perpoint_feat, dim=0)[0] # Tensor(Dl)
-                partial_feat_seq.append(partial_feat)
-                scanned_target_pts_num.append(partial_perpoint_feat.shape[0])
-                
-            scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.int32).to(device) # Tensor(S)
             scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
-            
+            scanned_pts = scanned_pts.to(device)  # Tensor(S x N x 3)
-            pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp)
+            pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp) 
-            pts_num_feat_seq = self.pts_num_encoder.encode_pts_num(scanned_target_pts_num)  # Tensor(S x Dn)
+            pts_feat_seq = self.pts_encoder.encode_points(scanned_pts, require_per_point_feat=False)  # Tensor(S x Dl)
-            partial_feat_seq = torch.stack(partial_feat_seq) # Tensor(S x Dl)
+            seq_embedding = torch.cat([pose_feat_seq, pts_feat_seq], dim=-1) # Tensor(S x (Dp+Dl))
-            
+            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dl))
-            seq_embedding = torch.cat([pose_feat_seq, pts_num_feat_seq, partial_feat_seq], dim=-1) # Tensor(S x (Dp+Dn+Dl))
+        
-            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dn+Dl))
+        seq_feat = self.seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
-
+        main_feat = seq_feat # Tensor(B x Ds)
-        seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
-
-        main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))
 
         if torch.isnan(main_feat).any():
             Log.error("nan in main_feat", True)

runners/strategy_generator.py

@@ -85,14 +85,16 @@ class StrategyGenerator(Runner):
             pts_path = os.path.join(root,scene_name, "pts", f"{frame_idx}.npy")
             nrm_path = os.path.join(root,scene_name, "nrm", f"{frame_idx}.npy")
             idx_path = os.path.join(root,scene_name, "scan_points_indices", f"{frame_idx}.npy")
+            
             pts = np.load(pts_path)
-            if pts.shape[0] == 0:
+            if self.compute_with_normal:
-                nrm = np.zeros((0,3))
+                if pts.shape[0] == 0:
-            else:
+                    nrm = np.zeros((0,3))
-                nrm = np.load(nrm_path)
+                else:
-            indices = np.load(idx_path) 
+                    nrm = np.load(nrm_path)
+                nrm_list.append(nrm)
             pts_list.append(pts)
-            nrm_list.append(nrm)
+            indices = np.load(idx_path) 
             scan_points_indices_list.append(indices)
             if pts.shape[0] > 0:
                 non_zero_cnt += 1

utils/data_load.py

@@ -53,6 +53,8 @@ class DataLoadUtil:
     @staticmethod
     def get_label_num(root, scene_name):
         label_dir = os.path.join(root, scene_name, "label")
+        if not os.path.exists(label_dir):
+            return 0
         return len(os.listdir(label_dir))
 
     @staticmethod

utils/reconstruction.py

@@ -62,7 +62,7 @@ class ReconstructionUtil:
         
         max_rec_pts = np.vstack(point_cloud_list)
         downsampled_max_rec_pts = PtsUtil.voxel_downsample_point_cloud(max_rec_pts, threshold)
-        
+        combined_point_cloud = PtsUtil.voxel_downsample_point_cloud(combined_point_cloud, threshold)
         max_rec_pts_num = downsampled_max_rec_pts.shape[0]
         max_real_rec_pts_coverage, _ = ReconstructionUtil.compute_coverage_rate(target_point_cloud, downsampled_max_rec_pts, threshold)
         
@@ -75,6 +75,7 @@ class ReconstructionUtil:
         cnt_processed_view = 0
         remaining_views.remove(init_view)
         curr_rec_pts_num = combined_point_cloud.shape[0]
+        drop_output_ratio = 0.4
         
         import time
         while remaining_views:
@@ -84,6 +85,8 @@ class ReconstructionUtil:
             best_covered_num = 0
         
             for view_index in remaining_views:
+                if np.random.rand() < drop_output_ratio:
+                    continue
                 if point_cloud_list[view_index].shape[0] == 0:
                     continue
                 if selected_views: