diff --git a/python/ctranslate2/converters/transformers.py b/python/ctranslate2/converters/transformers.py
index c5a674967..1b2d0c98f 100644
--- a/python/ctranslate2/converters/transformers.py
+++ b/python/ctranslate2/converters/transformers.py
@@ -915,7 +915,8 @@ def set_config(self, config, model, tokenizer):
         if gen_config is not None:
             config.suppress_ids = gen_config.suppress_tokens
             config.suppress_ids_begin = gen_config.begin_suppress_tokens
-            config.alignment_heads = gen_config.alignment_heads
+            if hasattr(gen_config, "alignment_heads"):
+                config.alignment_heads = gen_config.alignment_heads
             if hasattr(gen_config, "lang_to_id"):
                 config.lang_ids = sorted(gen_config.lang_to_id.values())
         else: