diff --git a/TTS/tts/models/base_tts.py b/TTS/tts/models/base_tts.py
index 14bc9180..f1fdbd33 100644
--- a/TTS/tts/models/base_tts.py
+++ b/TTS/tts/models/base_tts.py
@@ -102,7 +102,7 @@ class BaseTTS(BaseModel):
                 config.d_vector_dim if "d_vector_dim" in config and config.d_vector_dim is not None else 512
             )
         # init speaker embedding layer
-        if config.use_speaker_embedding:
+        if config.use_speaker_embedding and not config.use_d_vector_file::
             print(" > Init speaker_embedding layer.")
             self.speaker_embedding = nn.Embedding(self.num_speakers, self.embedded_speaker_dim)
             self.speaker_embedding.weight.data.normal_(0, 0.3)