From 63a246c0c87d42f04076a459adcfdc88c954b09c Mon Sep 17 00:00:00 2001 From: hofee Date: Mon, 28 Oct 2024 19:15:48 +0000 Subject: [PATCH] debug new training --- configs/local/train_config.yaml | 2 +- configs/server/server_train_config.yaml | 32 +++++++++++++++++-------- core/global_pts_n_num_pipeline.py | 11 ++++----- core/nbv_dataset.py | 9 +++---- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/configs/local/train_config.yaml b/configs/local/train_config.yaml index 226c302..77e97a9 100644 --- a/configs/local/train_config.yaml +++ b/configs/local/train_config.yaml @@ -84,7 +84,7 @@ module: gf_view_finder: t_feat_dim: 128 pose_feat_dim: 256 - main_feat_dim: 2048 + main_feat_dim: 3072 regression_head: Rx_Ry_and_T pose_mode: rot_matrix per_point_feature: False diff --git a/configs/server/server_train_config.yaml b/configs/server/server_train_config.yaml index bf12e1b..bcc767d 100644 --- a/configs/server/server_train_config.yaml +++ b/configs/server/server_train_config.yaml @@ -7,7 +7,7 @@ runner: parallel: False experiment: - name: full_w_global_feat_wo_local_pts_feat + name: debug root_dir: "experiments" use_checkpoint: False epoch: -1 # -1 stands for last epoch @@ -28,14 +28,14 @@ runner: - OmniObject3d_test - OmniObject3d_val - pipeline: nbv_reconstruction_global_pts_pipeline + pipeline: nbv_reconstruction_global_pts_n_num_pipeline dataset: OmniObject3d_train: - root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy" + root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new" model_dir: "../data/scaled_object_meshes" source: nbv_reconstruction_dataset - split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt" + split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt" type: train cache: True ratio: 1 @@ -45,10 +45,10 @@ dataset: load_from_preprocess: True OmniObject3d_test: - root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy" + root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new" model_dir: "../data/scaled_object_meshes" source: nbv_reconstruction_dataset - split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_test.txt" + split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt" type: test cache: True filter_degree: 75 @@ -61,10 +61,10 @@ dataset: load_from_preprocess: True OmniObject3d_val: - root_dir: "/home/data/hofee/project/nbv_rec/data/nbv_rec_data_512_preproc_npy" + root_dir: "/home/data/hofee/project/nbv_rec/data/sample_for_training_new" model_dir: "../data/scaled_object_meshes" source: nbv_reconstruction_dataset - split_file: "/home/data/hofee/project/nbv_rec/data/OmniObject3d_train.txt" + split_file: "/home/data/hofee/project/nbv_rec/data/sample.txt" type: test cache: True filter_degree: 75 @@ -96,6 +96,15 @@ pipeline: eps: 1e-5 global_scanned_feat: True + nbv_reconstruction_global_pts_n_num_pipeline: + modules: + pts_encoder: pointnet_encoder + transformer_seq_encoder: transformer_seq_encoder + pose_encoder: pose_encoder + view_finder: gf_view_finder + pts_num_encoder: pts_num_encoder + eps: 1e-5 + global_scanned_feat: True module: @@ -107,7 +116,7 @@ module: feature_transform: False transformer_seq_encoder: - embed_dim: 1344 + embed_dim: 384 num_heads: 4 ffn_dim: 256 num_layers: 3 @@ -116,7 +125,7 @@ module: gf_view_finder: t_feat_dim: 128 pose_feat_dim: 256 - main_feat_dim: 2048 + main_feat_dim: 3072 regression_head: Rx_Ry_and_T pose_mode: rot_matrix per_point_feature: False @@ -128,6 +137,9 @@ module: pose_dim: 9 out_dim: 256 + pts_num_encoder: + out_dim: 64 + loss_function: gf_loss: diff --git a/core/global_pts_n_num_pipeline.py b/core/global_pts_n_num_pipeline.py index 04a360b..e8b5e95 100644 --- a/core/global_pts_n_num_pipeline.py +++ b/core/global_pts_n_num_pipeline.py @@ -117,22 +117,21 @@ class NBVReconstructionGlobalPointsPipeline(nn.Module): for seq_idx in range(seq_len): partial_idx_in_combined_pts = scanned_mask == seq_idx # Ndarray(V), N->V idx mask partial_perpoint_feat = perpoint_scanned_feat[partial_idx_in_combined_pts] # Ndarray(V x Dl) - partial_feat = torch.mean(partial_perpoint_feat, dim=0)[0] # Tensor(Dl) + partial_feat = torch.mean(partial_perpoint_feat, dim=0) # Tensor(Dl) partial_feat_seq.append(partial_feat) scanned_target_pts_num.append(partial_perpoint_feat.shape[0]) - - scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.int32).to(device) # Tensor(S) - scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device) # Tensor(S x 9) + + scanned_target_pts_num = torch.tensor(scanned_target_pts_num, dtype=torch.float32).unsqueeze(-1).to(device) # Tensor(S x 1) + scanned_n_to_world_pose_9d = scanned_n_to_world_pose_9d.to(device) # Tensor(S x 9) pose_feat_seq = self.pose_encoder.encode_pose(scanned_n_to_world_pose_9d) # Tensor(S x Dp) pts_num_feat_seq = self.pts_num_encoder.encode_pts_num(scanned_target_pts_num) # Tensor(S x Dn) partial_feat_seq = torch.stack(partial_feat_seq) # Tensor(S x Dl) - seq_embedding = torch.cat([pose_feat_seq, pts_num_feat_seq, partial_feat_seq], dim=-1) # Tensor(S x (Dp+Dn+Dl)) embedding_list_batch.append(seq_embedding) # List(B): Tensor(S x (Dp+Dn+Dl)) + seq_feat = self.transformer_seq_encoder.encode_sequence(embedding_list_batch) # Tensor(B x Ds) - main_feat = torch.cat([seq_feat, global_scanned_feat], dim=-1) # Tensor(B x (Ds+Dg)) if torch.isnan(main_feat).any(): diff --git a/core/nbv_dataset.py b/core/nbv_dataset.py index 4013d98..05fa4bf 100644 --- a/core/nbv_dataset.py +++ b/core/nbv_dataset.py @@ -34,7 +34,7 @@ class NBVReconstructionDataset(BaseDataset): self.model_dir = config["model_dir"] self.filter_degree = config["filter_degree"] if self.type == namespace.Mode.TRAIN: - scale_ratio = 1 + scale_ratio = 100 self.datalist = self.datalist*scale_ratio if self.cache: expr_root = ConfigManager.get("runner", "experiment", "root_dir") @@ -122,7 +122,7 @@ class NBVReconstructionDataset(BaseDataset): scanned_views_pts, scanned_coverages_rate, scanned_n_to_world_pose, - ) = ([], [], [], []) + ) = ([], [], []) for view in scanned_views: frame_idx = view[0] coverage_rate = view[1] @@ -164,7 +164,7 @@ class NBVReconstructionDataset(BaseDataset): combined_scanned_views_pts, self.pts_num, require_idx=True ) - combined_scanned_views_pts_mask = np.zeros(len(scanned_views_pts), dtype=np.uint8) + combined_scanned_views_pts_mask = np.zeros(len(combined_scanned_views_pts), dtype=np.uint8) start_idx = 0 for i in range(len(scanned_views_pts)): @@ -174,9 +174,6 @@ class NBVReconstructionDataset(BaseDataset): fps_downsampled_combined_scanned_pts_mask = combined_scanned_views_pts_mask[fps_idx] - - - data_item = { "scanned_pts": np.asarray(scanned_views_pts, dtype=np.float32), # Ndarray(S x Nv x 3) "scanned_pts_mask": np.asarray(fps_downsampled_combined_scanned_pts_mask,dtype=np.uint8), # Ndarray(N), range(0, S)