diff --git a/python/ctranslate2/converters/transformers.py b/python/ctranslate2/converters/transformers.py
index d98c65860..4e10aabd9 100644
--- a/python/ctranslate2/converters/transformers.py
+++ b/python/ctranslate2/converters/transformers.py
@@ -356,7 +356,7 @@ def set_attention(self, spec, attention, self_attention=False):
         self.set_linear(spec.linear[-1], attention.out_proj)
 
     def set_common_layers(self, spec, module):
-        spec.scale_embeddings = module.embed_scale
+        spec.scale_embeddings = getattr(module, 'embed_scale', False)
         self.set_position_encodings(spec.position_encodings, module.embed_positions)
         self.set_embeddings(
             (