global_only: debug

configs/server/server_train_config.yaml

@@ -3,11 +3,11 @@ runner:
   general:
     seed: 0
     device: cuda
-    cuda_visible_devices: "1"
+    cuda_visible_devices: "0"
     parallel: False
     
   experiment:
-    name: debug
+    name: overfit_ab_global_only
     root_dir: "experiments"
     use_checkpoint: False
     epoch: -1 # -1 stands for last epoch
@@ -28,57 +28,57 @@ runner:
       #- OmniObject3d_test
       - OmniObject3d_val
 
-  pipeline: nbv_reconstruction_global_pts_n_num_pipeline
+  pipeline: nbv_reconstruction_pipeline
   
 dataset:
   OmniObject3d_train:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: train
     cache: True
     ratio: 1
-    batch_size: 160
+    batch_size: 80
     num_workers: 16
     pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_test:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.05
+    ratio: 1
-    batch_size: 160
+    batch_size: 80
     num_workers: 12
     pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_val:
-    root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.005
+    ratio: 1
-    batch_size: 160
+    batch_size: 80
     num_workers: 12
     pts_num: 8192
     load_from_preprocess: True
 
 
 pipeline:
-  nbv_reconstruction_local_pts_pipeline:
+  nbv_reconstruction_pipeline:
     modules:
       pts_encoder: pointnet_encoder
       seq_encoder: transformer_seq_encoder
@@ -87,25 +87,6 @@ pipeline:
     eps: 1e-5
     global_scanned_feat: True
 
-  nbv_reconstruction_global_pts_pipeline:
-    modules:
-      pts_encoder: pointnet_encoder
-      pose_seq_encoder: transformer_seq_encoder
-      pose_encoder: pose_encoder
-      view_finder: gf_view_finder
-    eps: 1e-5
-    global_scanned_feat: True
-
-  nbv_reconstruction_global_pts_n_num_pipeline:
-    modules:
-      pts_encoder: pointnet_encoder
-      transformer_seq_encoder: transformer_seq_encoder
-      pose_encoder: pose_encoder
-      view_finder: gf_view_finder
-      pts_num_encoder: pts_num_encoder
-    eps: 1e-5
-    global_scanned_feat: True
-
     
 module:
 
@@ -116,16 +97,16 @@ module:
     feature_transform: False
 
   transformer_seq_encoder:
-    embed_dim: 384
+    embed_dim: 256
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
-    output_dim: 2048
+    output_dim: 1024
 
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 3072
+    main_feat_dim: 2048
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False

core/nbv_dataset.py

@@ -34,7 +34,7 @@ class NBVReconstructionDataset(BaseDataset):
             #self.model_dir = config["model_dir"]
             self.filter_degree = config["filter_degree"]
         if self.type == namespace.Mode.TRAIN:
-            scale_ratio = 100
+            scale_ratio = 50
             self.datalist = self.datalist*scale_ratio
         if self.cache:
             expr_root = ConfigManager.get("runner", "experiment", "root_dir")
@@ -169,19 +169,9 @@ class NBVReconstructionDataset(BaseDataset):
         voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_views_pts, 0.002)
         random_downsampled_combined_scanned_pts_np = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, self.pts_num)
         
-        combined_scanned_views_pts_mask = np.zeros(len(combined_scanned_views_pts), dtype=np.uint8)
-        
-        start_idx = 0
-        for i in range(len(scanned_views_pts)):
-            end_idx = start_idx + len(scanned_views_pts[i])
-            combined_scanned_views_pts_mask[start_idx:end_idx] = i
-            start_idx = end_idx
-        fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx]
-        
         data_item = {
             "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3)
-            "scanned_pts_mask": np.asarray(fps_downsampled_combined_scanned_pts_mask,dtype=np.uint8), # Ndarray(N), range(0, S)
+            "combined_scanned_pts": np.asarray(random_downsampled_combined_scanned_pts_np, dtype=np.float32), # Ndarray(N x 3)
-            "combined_scanned_pts": np.asarray(fps_downsampled_combined_scanned_pts, dtype=np.float32), # Ndarray(N x 3)
             "scanned_coverage_rate": scanned_coverages_rate, # List(S): Float, range(0, 1)
             "scanned_n_to_world_pose_9d": np.asarray(scanned_n_to_world_pose, dtype=np.float32), # Ndarray(S x 9)
             "best_coverage_rate": nbv_coverage_rate, # Float, range(0, 1)
@@ -216,14 +206,9 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["combined_scanned_pts"] = torch.stack(
                 [torch.tensor(item["combined_scanned_pts"]) for item in batch]
             )
-            collate_data["scanned_pts_mask"] = torch.stack(
-                [torch.tensor(item["scanned_pts_mask"]) for item in batch]
-            )
-                
             for key in batch[0].keys():
                 if key not in [
                     "scanned_pts",
-                    "scanned_pts_mask",
                     "scanned_n_to_world_pose_9d",
                     "best_to_world_pose_9d",
                     "combined_scanned_pts",

core/pipeline.py

@@ -7,10 +7,10 @@ from PytorchBoot.factory.component_factory import ComponentFactory
 from PytorchBoot.utils import Log
 
 
-@stereotype.pipeline("nbv_reconstruction_global_pts_n_num_pipeline")
+@stereotype.pipeline("nbv_reconstruction_pipeline")
-class NBVReconstructionGlobalPointsPipeline(nn.Module):
+class NBVReconstructionPipeline(nn.Module):
     def __init__(self, config):
-        super(NBVReconstructionGlobalPointsPipeline, self).__init__()
+        super(NBVReconstructionPipeline, self).__init__()
         self.config = config
         self.module_config = config["modules"]
         
@@ -20,12 +20,8 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         self.pose_encoder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["pose_encoder"]
         )
-        self.pts_num_encoder = ComponentFactory.create(
+        self.seq_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["pts_num_encoder"]
+            namespace.Stereotype.MODULE, self.module_config["seq_encoder"]
-        )
-        
-        self.transformer_seq_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["transformer_seq_encoder"]
         )
         self.view_finder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["view_finder"]
@@ -33,7 +29,6 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         
 
         self.eps = float(self.config["eps"])
-        self.enable_global_scanned_feat = self.config["global_scanned_feat"]
 
     def forward(self, data):
         mode = data["mode"]
@@ -96,46 +91,23 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module):
         scanned_n_to_world_pose_9d_batch = data[
             "scanned_n_to_world_pose_9d"
         ]  # List(B): Tensor(S x 9)
-        scanned_pts_mask_batch = data[
-            "scanned_pts_mask"
-        ]  # Tensor(B x N)
 
         device = next(self.parameters()).device
 
         embedding_list_batch = []
 
         combined_scanned_pts_batch = data["combined_scanned_pts"]  # Tensor(B x N x 3)
-        global_scanned_feat, perpoint_scanned_feat_batch = self.pts_encoder.encode_points(
+        global_scanned_feat = self.pts_encoder.encode_points(
-            combined_scanned_pts_batch, require_per_point_feat=True
+            combined_scanned_pts_batch, require_per_point_feat=False
-        )  # global_scanned_feat: Tensor(B x Dg), perpoint_scanned_feat: Tensor(B x N x Dl)
+        )  # global_scanned_feat: Tensor(B x Dg)
 
-        for scanned_n_to_world_pose_9d, scanned_mask, perpoint_scanned_feat in zip(
+        for scanned_n_to_world_pose_9d in scanned_n_to_world_pose_9d_batch:
-            scanned_n_to_world_pose_9d_batch,
-            scanned_pts_mask_batch,
-            perpoint_scanned_feat_batch,
-        ):
-            scanned_target_pts_num = [] # List(S): Int
-            partial_feat_seq = []
-            
-            seq_len = len(scanned_n_to_world_pose_9d)
-            for seq_idx in range(seq_len): 
-                partial_idx_in_combined_pts = scanned_mask == seq_idx # Ndarray(V), N->V idx mask
-                partial_perpoint_feat = perpoint_scanned_feat[partial_idx_in_combined_pts] # Ndarray(V x Dl)
-                partial_feat = torch.mean(partial_perpoint_feat, dim=0) # Tensor(Dl)
-                partial_feat_seq.append(partial_feat)
-                scanned_target_pts_num.append(partial_perpoint_feat.shape[0])
-            
-                
-            scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.float32).unsqueeze(-1).to(device) # Tensor(S x 1)
             scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
-            pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp)
+            pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp) 
-            pts_num_feat_seq = self.pts_num_encoder.encode_pts_num(scanned_target_pts_num)  # Tensor(S x Dn)
+            seq_embedding = pose_feat_seq
-            partial_feat_seq = torch.stack(partial_feat_seq) # Tensor(S x Dl)
+            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp))
-            seq_embedding = torch.cat([pose_feat_seq, pts_num_feat_seq, partial_feat_seq], dim=-1) # Tensor(S x (Dp+Dn+Dl))
-            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dn+Dl))
-
         
-        seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
+        seq_feat = self.seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
         main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))
 
         if torch.isnan(main_feat).any():