更新

AlchemistZoro · AlchemistZoro · commit 47ec36ff6d3c · 2023-02-25T11:09:57.000+08:00
diff --git a/model/modeling_roberta.py b/model/modeling_roberta.py
@@ -73,8 +73,12 @@ def forward(
         if self.position_embedding_type == "absolute":
             position_embeddings = self.position_embeddings(position_ids)
             embeddings += position_embeddings
+
+        
         embeddings = self.LayerNorm(embeddings)
+
         embeddings = self.dropout(embeddings)
+
         return embeddings
     
     def pool_input_embeds(self,inputs_embeds,pool_mask):
@@ -167,4 +171,9 @@ def pool_input_embeds(inputs_embeds,pool_mask):
     pool_mask[0][3:5] = 1
     pool_mask[-1][2:5] = 1
     
-    pool_input_embeds(inputs_embeds,pool_mask)
+    word_embeddings = nn.Embedding(3,2)
+    idx = torch.tensor([0,0,1])
+    # token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
+    print(word_embeddings(idx).shape)
+    
+    # pool_input_embeds(inputs_embeds,pool_mask)
