global_and_local: config

configs/server/server_train_config.yaml

@@ -28,7 +28,7 @@ runner:
       #- OmniObject3d_test
       - OmniObject3d_val
 
-  pipeline: nbv_reconstruction_global_pts_n_num_pipeline
+  pipeline: nbv_reconstruction_pipeline
   
 dataset:
   OmniObject3d_train:
@@ -78,7 +78,7 @@ dataset:
 
 
 pipeline:
-  nbv_reconstruction_local_pts_pipeline:
+  nbv_reconstruction_pipeline:
     modules:
       pts_encoder: pointnet_encoder
       seq_encoder: transformer_seq_encoder
@@ -87,25 +87,6 @@ pipeline:
     eps: 1e-5
     global_scanned_feat: True
 
-  nbv_reconstruction_global_pts_pipeline:
-    modules:
-      pts_encoder: pointnet_encoder
-      pose_seq_encoder: transformer_seq_encoder
-      pose_encoder: pose_encoder
-      view_finder: gf_view_finder
-    eps: 1e-5
-    global_scanned_feat: True
-
-  nbv_reconstruction_global_pts_n_num_pipeline:
-    modules:
-      pts_encoder: pointnet_encoder
-      transformer_seq_encoder: transformer_seq_encoder
-      pose_encoder: pose_encoder
-      view_finder: gf_view_finder
-      pts_num_encoder: pts_num_encoder
-    eps: 1e-5
-    global_scanned_feat: True
-
     
 module:
 
@@ -120,12 +101,12 @@ module:
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
-    output_dim: 1024
+    output_dim: 2048
 
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 2048
+    main_feat_dim: 3072
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False

core/pipeline.py

@@ -92,7 +92,9 @@ class NBVReconstructionPipeline(nn.Module):
         scanned_n_to_world_pose_9d_batch = data[
             "scanned_n_to_world_pose_9d"
         ]  # List(B): Tensor(S x 9)
-
+        scanned_pts_batch = data[
+            "scanned_pts"
+        ]
         device = next(self.parameters()).device
 
         embedding_list_batch = []
@@ -102,11 +104,13 @@ class NBVReconstructionPipeline(nn.Module):
             combined_scanned_pts_batch, require_per_point_feat=False
         )  # global_scanned_feat: Tensor(B x Dg)
 
-        for scanned_n_to_world_pose_9d in scanned_n_to_world_pose_9d_batch:
+        for scanned_n_to_world_pose_9d, scanned_pts in zip(scanned_n_to_world_pose_9d_batch, scanned_pts_batch):
             scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
+            scanned_pts = scanned_pts.to(device)  # Tensor(S x N x 3)
             pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d)  # Tensor(S x Dp) 
-            seq_embedding = pose_feat_seq
-            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp))
+            pts_feat_seq = self.pts_encoder.encode_points(scanned_pts, require_per_point_feat=False)  # Tensor(S x Dl)
+            seq_embedding = torch.cat([pose_feat_seq, pts_feat_seq], dim=-1) # Tensor(S x (Dp+Dl))
+            embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dl))
         
         seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
         main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))