configs/server/server_train_config.yaml

@@ -7,7 +7,7 @@ runner:
     parallel: False
     
   experiment:
-    name: overfit_ab_local_only
+    name: debug
     root_dir: "experiments"
     use_checkpoint: False
     epoch: -1 # -1 stands for last epoch
@@ -32,46 +32,46 @@ runner:
   
 dataset:
   OmniObject3d_train:
-    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
+    root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/data/hofee/data/sample.txt"
+    split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt"
     type: train
     cache: True
     ratio: 1
-    batch_size: 32
+    batch_size: 160
     num_workers: 16
     pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_test:
-    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
+    root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/data/hofee/data/sample.txt"
+    split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 1
+    ratio: 0.05
-    batch_size: 32
+    batch_size: 160
     num_workers: 12
     pts_num: 8192
     load_from_preprocess: True
 
   OmniObject3d_val:
-    root_dir: "/data/hofee/nbv_rec_part2_preprocessed"
+    root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new"
     model_dir: "../data/scaled_object_meshes"
     source: nbv_reconstruction_dataset
-    split_file: "/data/hofee/data/sample.txt"
+    split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt"
     type: test
     cache: True
     filter_degree: 75
     eval_list:
       - pose_diff
-    ratio: 1
+    ratio: 0.005
-    batch_size: 32
+    batch_size: 160
     num_workers: 12
     pts_num: 8192
     load_from_preprocess: True
@@ -92,12 +92,12 @@ module:
 
   pointnet_encoder:
     in_dim: 3
-    out_dim: 512
+    out_dim: 1024
     global_feat: True
     feature_transform: False
 
   transformer_seq_encoder:
-    embed_dim: 768
+    embed_dim: 256
     num_heads: 4
     ffn_dim: 256
     num_layers: 3
@@ -106,7 +106,7 @@ module:
   gf_view_finder:
     t_feat_dim: 128
     pose_feat_dim: 256
-    main_feat_dim: 2048
+    main_feat_dim: 3072
     regression_head: Rx_Ry_and_T
     pose_mode: rot_matrix
     per_point_feature: False

core/nbv_dataset.py

@@ -34,7 +34,7 @@ class NBVReconstructionDataset(BaseDataset):
             #self.model_dir = config["model_dir"]
             self.filter_degree = config["filter_degree"]
         if self.type == namespace.Mode.TRAIN:
-            scale_ratio = 50
+            scale_ratio = 100
             self.datalist = self.datalist*scale_ratio
         if self.cache:
             expr_root = ConfigManager.get("runner", "experiment", "root_dir")
@@ -165,8 +165,13 @@ class NBVReconstructionDataset(BaseDataset):
             [best_to_world_6d, best_to_world_trans], axis=0
         )
 
+        combined_scanned_views_pts = np.concatenate(scanned_views_pts, axis=0)
+        voxel_downsampled_combined_scanned_pts_np = PtsUtil.voxel_downsample_point_cloud(combined_scanned_views_pts, 0.002)
+        random_downsampled_combined_scanned_pts_np = PtsUtil.random_downsample_point_cloud(voxel_downsampled_combined_scanned_pts_np, self.pts_num)
+        
         data_item = {
             "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3)
+            "combined_scanned_pts": np.asarray(random_downsampled_combined_scanned_pts_np, dtype=np.float32), # Ndarray(N x 3)
             "scanned_coverage_rate": scanned_coverages_rate, # List(S): Float, range(0, 1)
             "scanned_n_to_world_pose_9d": np.asarray(scanned_n_to_world_pose, dtype=np.float32), # Ndarray(S x 9)
             "best_coverage_rate": nbv_coverage_rate, # Float, range(0, 1)
@@ -198,6 +203,12 @@ class NBVReconstructionDataset(BaseDataset):
             collate_data["best_to_world_pose_9d"] = torch.stack(
                 [torch.tensor(item["best_to_world_pose_9d"]) for item in batch]
             )
+            collate_data["combined_scanned_pts"] = torch.stack(
+                [torch.tensor(item["combined_scanned_pts"]) for item in batch]
+            )
+            collate_data["scanned_pts_mask"] = torch.stack(
+                [torch.tensor(item["scanned_pts_mask"]) for item in batch]
+            )
                 
             for key in batch[0].keys():
                 if key not in [
@@ -205,6 +216,7 @@ class NBVReconstructionDataset(BaseDataset):
                     "scanned_pts_mask",
                     "scanned_n_to_world_pose_9d",
                     "best_to_world_pose_9d",
+                    "combined_scanned_pts",
                 ]:
                     collate_data[key] = [item[key] for item in batch]
             return collate_data

core/pipeline.py

@@ -20,8 +20,8 @@ class NBVReconstructionPipeline(nn.Module):
         self.pose_encoder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["pose_encoder"]
         )
-        self.seq_encoder = ComponentFactory.create(
+        self.transformer_seq_encoder = ComponentFactory.create(
-            namespace.Stereotype.MODULE, self.module_config["seq_encoder"]
+            namespace.Stereotype.MODULE, self.module_config["transformer_seq_encoder"]
         )
         self.view_finder = ComponentFactory.create(
             namespace.Stereotype.MODULE, self.module_config["view_finder"]
@@ -99,6 +99,11 @@ class NBVReconstructionPipeline(nn.Module):
 
         embedding_list_batch = []
 
+        combined_scanned_pts_batch = data["combined_scanned_pts"]  # Tensor(B x N x 3)
+        global_scanned_feat = self.pts_encoder.encode_points(
+            combined_scanned_pts_batch, require_per_point_feat=False
+        )  # global_scanned_feat: Tensor(B x Dg)
+
         for scanned_n_to_world_pose_9d, scanned_pts in zip(scanned_n_to_world_pose_9d_batch, scanned_pts_batch):
             scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device)  # Tensor(S x 9)
             scanned_pts = scanned_pts.to(device)  # Tensor(S x N x 3)
@@ -107,8 +112,8 @@ class NBVReconstructionPipeline(nn.Module):
             seq_embedding = torch.cat([pose_feat_seq, pts_feat_seq], dim=-1) # Tensor(S x (Dp+Dl))
             embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dl))
         
-        seq_feat = self.seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
+        seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds)
-        main_feat = seq_feat # Tensor(B x Ds)
+        main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg))
 
         if torch.isnan(main_feat).any():
             Log.error("nan in main_feat", True)