reorg structure of model implementations

Author: zigzagcai

internlm/checkpoint/checkpoint_manager.py

@@ -16,8 +16,8 @@
     auto_resume_sanity_check,
     ckpt_info_sanity_check,
 )
-from internlm.model.base_model import BaseModel
-from internlm.model.registry import model_initializer
+from internlm.model_implementations.registry import model_initializer
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.monitor import send_alert_message
 from internlm.solver.optimizer import HybridZeroOptimizer, HybridZeroOptimizer_v2
 from internlm.utils.common import get_current_device

internlm/checkpoint/load_funcs.py

@@ -1,8 +1,8 @@
 # Copyright (c) InternLM. All rights reserved.
 
-from internlm.model.modeling_internlm import InternLM1
-from internlm.model.modeling_internlm2 import InternLM2
-from internlm.model.modeling_llama import Llama2
+from internlm.model_implementations.transformers.modeling_internlm import InternLM1
+from internlm.model_implementations.transformers.modeling_internlm2 import InternLM2
+from internlm.model_implementations.transformers.modeling_llama import Llama2
 from internlm.utils.logger import get_logger
 
 logger = get_logger(__file__)

internlm/core/trainer_builder.py

@@ -19,12 +19,12 @@
 from internlm.initialize.initialize_trainer import initialize_trainer
 from internlm.model.losses.ce_loss import InternLoss
 from internlm.model.metrics import AccPerplex
+from internlm.model_inject.inject import inject_model
 from internlm.monitor.monitor import send_alert_message
 from internlm.train.pipeline import (
     get_scheduler_hooks,
     initialize_llm_profile,
     initialize_optimizer,
-    inject_model,
     load_new_batch,
     record_current_batch_training_metrics,
 )

internlm/model_implementations/__init__.py

@@ -6,12 +6,12 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context import global_context as gpc
 from internlm.core.parallel.shard import pipeline_parallel_sharding_wrapper
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.linear import (
     ParallelLinearWithCommExt,
     ScaleColumnParallelLinear,
 )
-from internlm.model.registry import model_initializer
+from internlm.model_implementations.registry import model_initializer
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.utils.common import get_current_device
 from internlm.utils.lazy import LazyObject
 from internlm.utils.logger import get_logger

internlm/model/builder.py internlm/model_implementations/builder.py

@@ -4,16 +4,16 @@
 
 from typing import Callable
 
-from internlm.model.modeling_baichuan2 import Baichuan2
-from internlm.model.modeling_gemma import Gemma
-from internlm.model.modeling_internlm import InternLM1
-from internlm.model.modeling_internlm2 import InternLM2
-from internlm.model.modeling_llama import Llama2
-from internlm.model.modeling_llava import Llava
-from internlm.model.modeling_mixtral import MixtralMoE
-from internlm.model.modeling_moe import Internlm1MoE
-from internlm.model.modeling_qwen2 import Qwen2
-from internlm.model.modeling_qwen2_moe import Qwen2Moe
+from internlm.model_implementations.transformers.modeling_baichuan2 import Baichuan2
+from internlm.model_implementations.transformers.modeling_gemma import Gemma
+from internlm.model_implementations.transformers.modeling_internlm import InternLM1
+from internlm.model_implementations.transformers.modeling_internlm2 import InternLM2
+from internlm.model_implementations.transformers.modeling_llama import Llama2
+from internlm.model_implementations.transformers.modeling_llava import Llava
+from internlm.model_implementations.transformers.modeling_mixtral import MixtralMoE
+from internlm.model_implementations.transformers.modeling_moe import Internlm1MoE
+from internlm.model_implementations.transformers.modeling_qwen2 import Qwen2
+from internlm.model_implementations.transformers.modeling_qwen2_moe import Qwen2Moe
 from internlm.utils.common import SingletonMeta
 from internlm.utils.utils import ModelType
 

internlm/model/registry.py internlm/model_implementations/registry.py

@@ -17,7 +17,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA
@@ -27,6 +26,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model_implementations/transformers/__init__.py

@@ -16,7 +16,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA
@@ -26,6 +25,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model/base_model.py internlm/model_implementations/transformers/base_model.py

@@ -15,7 +15,6 @@
 from internlm.core.naive_amp import set_output_attr_to_module
 from internlm.core.parallel.shard import partition_uniform
 from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA
@@ -27,6 +26,7 @@
     internlm1_mha_pre_load_convert,
     internlm1_mha_save_convert,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model/modeling_baichuan2.py internlm/model_implementations/transformers/modeling_baichuan2.py

@@ -18,7 +18,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA
@@ -29,6 +28,7 @@
     convert_attn_kwargs_to_args,
     get_parallel_size_from_file,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model/modeling_gemma.py internlm/model_implementations/transformers/modeling_gemma.py

@@ -16,7 +16,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA
@@ -26,6 +25,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model/modeling_internlm.py internlm/model_implementations/transformers/modeling_internlm.py

@@ -7,13 +7,13 @@
 from internlm.core.context.parallel_context import global_context as gpc
 from internlm.core.naive_amp import set_output_attr_to_module
 from internlm.initialize.initialize_tensor import normal_, uniform_
-from internlm.model.base_model import BaseModel
 from internlm.model.llava.clip_builder import build_vision_tower
 from internlm.model.llava.projector_builder import build_vision_projector
-from internlm.model.modeling_llama import Llama2Decoder
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.norm import new_layer_norm
+from internlm.model_implementations.transformers.base_model import BaseModel
+from internlm.model_implementations.transformers.modeling_llama import Llama2Decoder
 from internlm.utils.logger import get_logger
 
 logger = get_logger(__file__)

internlm/model/modeling_internlm2.py internlm/model_implementations/transformers/modeling_internlm2.py

@@ -10,7 +10,6 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context.parallel_context import global_context as gpc
 from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import SWA
@@ -21,6 +20,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 

internlm/model/modeling_llama.py internlm/model_implementations/transformers/modeling_llama.py

@@ -10,7 +10,6 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context.parallel_context import global_context as gpc
 from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA
@@ -23,6 +22,7 @@
     internlm1_mha_pre_load_convert,
     internlm1_mha_save_convert,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 

internlm/model/modeling_llava.py internlm/model_implementations/transformers/modeling_llava.py

@@ -16,7 +16,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import SWA
@@ -26,6 +25,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger
 from internlm.utils.storage_manager import get_fns, llm_load, llm_save

internlm/model/modeling_mixtral.py internlm/model_implementations/transformers/modeling_mixtral.py

@@ -14,7 +14,6 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import SWA
@@ -25,6 +24,7 @@
     convert_attn_args_to_kwargs,
     convert_attn_kwargs_to_args,
 )
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.solver.activation_checkpoint import activation_checkpoint
 from internlm.utils.logger import get_logger