global_and_local: debug

configs/server/server_train_config.yaml

@@ -7,7 +7,7 @@ runner:
     parallel: False
     
   experiment:
-    name: train_ab_global_only
+    name: overfit_ab_global_and_local
     root_dir: "experiments"
     use_checkpoint: False
     epoch: -1 # -1 stands for last epoch
@@ -25,53 +25,53 @@ runner:
   test:
     frequency: 3 # test frequency
     dataset_list:
-      - OmniObject3d_test
+      #- OmniObject3d_test
       - OmniObject3d_val
 
   pipeline: nbv_reconstruction_pipeline
   
 dataset:
   OmniObject3d_train:
-    root_dir: "/data/hofee/data/new_full_data"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: train
     cache: True
     ratio: 1
-    batch_size: 80
-    num_workers: 128
+    batch_size: 32
+    num_workers: 16
     pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_test:
-    root_dir: "/data/hofee/data/new_full_data"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_test.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.1
-    batch_size: 80
+    ratio: 1
+    batch_size: 32
     num_workers: 12
     pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_val:
-    root_dir: "/data/hofee/data/new_full_data"
+    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/data/hofee/data/new_full_data_list/OmniObject3d_train.txt"
+    split_file: "/data/hofee/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 0.01
-    batch_size: 80
+    ratio: 1
+    batch_size: 32
     num_workers: 12
     pts_num: 8192
     load_from_preprocess: True
@@ -92,21 +92,21 @@ module:
 
   pointnet_encoder:
     in_dim: 3
-    out_dim: 1024
+    out_dim: 512
     global_feat: True
     feature_transform: False
 
   transformer_seq_encoder:
-    embed_dim: 256
+    embed_dim: 768
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
-    output_dim: 1024
+    output_dim: 2048
 
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 2048
+    main_feat_dim: 2560
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False

core/nbv_dataset.py

@@ -206,9 +206,11 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["combined_scanned_pts"] = torch.stack(
                 [torch.tensor(item["combined_scanned_pts"]) for item in batch]
             )
+                
             for key in batch[0].keys():
                 if key not in [
                     "scanned_pts",
+                    "scanned_pts_mask",
                     "scanned_n_to_world_pose_9d",
                     "best_to_world_pose_9d",
                     "combined_scanned_pts",

core/pipeline.py

@@ -29,6 +29,7 @@ class NBVReconstructionPipeline(nn.Module):
         
 
         self.eps = float(self.config["eps"])
+        self.enable_global_scanned_feat = self.config["global_scanned_feat"]
 
     def forward(self, data):
         mode = data["mode"]
@@ -54,7 +55,10 @@ class NBVReconstructionPipeline(nn.Module):
         return perturbed_x, random_t, target_score, std
 
     def forward_train(self, data):
+        start_time = time.time()
         main_feat = self.get_main_feat(data)
+        end_time = time.time()
+        print("get_main_feat time: ", end_time - start_time)
         """ get std """
         best_to_world_pose_9d_batch = data["best_to_world_pose_9d"]
         perturbed_x, random_t, target_score, std = self.pertube_data(
@@ -88,7 +92,9 @@ class NBVReconstructionPipeline(nn.Module):
         scanned_n_to_world_pose_9d_batch = data[
             "scanned_n_to_world_pose_9d"
         ]  # List(B): Tensor(S x 9)
-
+        scanned_pts_batch = data[
+            "scanned_pts"
+        ]
         device = next(self.parameters()).device
 
         embedding_list_batch = []
@@ -98,11 +104,13 @@ class NBVReconstructionPipeline(nn.Module):
             combined_scanned_pts_batch, require_per_point_feat=False
         )  # global_scanned_feat: Tensor(B x Dg)
 
-        for scanned_n_to_world_pose_9d in scanned_n_to_world_pose_9d_batch:
+        for scanned_n_to_world_pose_9d, scanned_pts in zip(scanned_n_to_world_pose_9d_batch, scanned_pts_batch):
             scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
+            scanned_pts = scanned_pts.to(device)  # Tensor(S x N x 3)
             pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp) 
-            seq_embedding = pose_feat_seq
-            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp))
+            pts_feat_seq = self.pts_encoder.encode_points(scanned_pts, require_per_point_feat=False)  # Tensor(S x Dl)
+            seq_embedding = torch.cat([pose_feat_seq, pts_feat_seq], dim=-1) # Tensor(S x (Dp+Dl))
+            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dl))
         
         seq_feat = self.seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
         main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))