reorg structure of model implementations

Author: zigzagcai

internlm/checkpoint/checkpoint_manager.py

@@ -16,8 +16,8 @@
     auto_resume_sanity_check,
     ckpt_info_sanity_check,
 )
-from internlm.model.base_model import BaseModel
-from internlm.model.registry import model_initializer
+from internlm.model_implementations.registry import model_initializer
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.monitor import send_alert_message
 from internlm.solver.optimizer import HybridZeroOptimizer, HybridZeroOptimizer_v2
 from internlm.utils.common import get_current_device

internlm/checkpoint/load_funcs.py

@@ -1,8 +1,8 @@
 # Copyright (c) InternLM. All rights reserved.
 
-from internlm.model.modeling_internlm import InternLM1
-from internlm.model.modeling_internlm2 import InternLM2
-from internlm.model.modeling_llama import Llama2
+from internlm.model_implementations.transformers.modeling_internlm import InternLM1
+from internlm.model_implementations.transformers.modeling_internlm2 import InternLM2
+from internlm.model_implementations.transformers.modeling_llama import Llama2
 from internlm.utils.logger import get_logger
 
 logger = get_logger(__file__)

internlm/core/trainer_builder.py

@@ -17,14 +17,14 @@
 from internlm.data.train_state import get_train_state
 from internlm.eval.evaluation import evaluate_on_val_dls
 from internlm.initialize.initialize_trainer import initialize_trainer
+from internlm.model_inject.inject import inject_model
 from internlm.model.losses.ce_loss import InternLoss
 from internlm.model.metrics import AccPerplex
 from internlm.monitor.monitor import send_alert_message
 from internlm.train.pipeline import (
     get_scheduler_hooks,
     initialize_llm_profile,
     initialize_optimizer,
-    inject_model,
     load_new_batch,
     record_current_batch_training_metrics,
 )

internlm/model_implementations/__init__.py

@@ -6,12 +6,12 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context import global_context as gpc
 from internlm.core.parallel.shard import pipeline_parallel_sharding_wrapper
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.linear import (
     ParallelLinearWithCommExt,
     ScaleColumnParallelLinear,
 )
-from internlm.model.registry import model_initializer
+from internlm.model_implementations.registry import model_initializer
 from internlm.utils.common import get_current_device
 from internlm.utils.lazy import LazyObject
 from internlm.utils.logger import get_logger

internlm/model/builder.py internlm/model_implementations/builder.py

@@ -4,16 +4,16 @@
 
 from typing import Callable
 
-from internlm.model.modeling_baichuan2 import Baichuan2
-from internlm.model.modeling_gemma import Gemma
-from internlm.model.modeling_internlm import InternLM1
-from internlm.model.modeling_internlm2 import InternLM2
-from internlm.model.modeling_llama import Llama2
-from internlm.model.modeling_llava import Llava
-from internlm.model.modeling_mixtral import MixtralMoE
-from internlm.model.modeling_moe import Internlm1MoE
-from internlm.model.modeling_qwen2 import Qwen2
-from internlm.model.modeling_qwen2_moe import Qwen2Moe
+from internlm.model_implementations.transformers.modeling_baichuan2 import Baichuan2
+from internlm.model_implementations.transformers.modeling_gemma import Gemma
+from internlm.model_implementations.transformers.modeling_internlm import InternLM1
+from internlm.model_implementations.transformers.modeling_internlm2 import InternLM2
+from internlm.model_implementations.transformers.modeling_llama import Llama2
+from internlm.model_implementations.transformers.modeling_llava import Llava
+from internlm.model_implementations.transformers.modeling_mixtral import MixtralMoE
+from internlm.model_implementations.transformers.modeling_moe import Internlm1MoE
+from internlm.model_implementations.transformers.modeling_qwen2 import Qwen2
+from internlm.model_implementations.transformers.modeling_qwen2_moe import Qwen2Moe
 from internlm.utils.common import SingletonMeta
 from internlm.utils.utils import ModelType
 

internlm/model/registry.py internlm/model_implementations/registry.py

@@ -17,7 +17,7 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA

internlm/model_implementations/transformers/__init__.py

@@ -16,7 +16,7 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA

internlm/model/base_model.py internlm/model_implementations/transformers/base_model.py

@@ -15,7 +15,7 @@
 from internlm.core.naive_amp import set_output_attr_to_module
 from internlm.core.parallel.shard import partition_uniform
 from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA

internlm/model/modeling_baichuan2.py internlm/model_implementations/transformers/modeling_baichuan2.py

@@ -18,7 +18,7 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA

internlm/model/modeling_gemma.py internlm/model_implementations/transformers/modeling_gemma.py

@@ -16,7 +16,7 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import GQA

internlm/model/modeling_internlm.py internlm/model_implementations/transformers/modeling_internlm.py

@@ -7,10 +7,10 @@
 from internlm.core.context.parallel_context import global_context as gpc
 from internlm.core.naive_amp import set_output_attr_to_module
 from internlm.initialize.initialize_tensor import normal_, uniform_
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.llava.clip_builder import build_vision_tower
 from internlm.model.llava.projector_builder import build_vision_projector
-from internlm.model.modeling_llama import Llama2Decoder
+from internlm.model_implementations.transformers.modeling_llama import Llama2Decoder
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.norm import new_layer_norm

internlm/model/modeling_internlm2.py internlm/model_implementations/transformers/modeling_internlm2.py

@@ -10,7 +10,7 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context.parallel_context import global_context as gpc
 from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import SWA

internlm/model/modeling_llama.py internlm/model_implementations/transformers/modeling_llama.py

@@ -10,7 +10,7 @@
 from internlm.core.context import ParallelMode
 from internlm.core.context.parallel_context import global_context as gpc
 from internlm.initialize.initialize_tensor import normal_, scaled_init_method_normal
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import MHA

internlm/model/modeling_llava.py internlm/model_implementations/transformers/modeling_llava.py

@@ -16,7 +16,7 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import SWA

internlm/model/modeling_mixtral.py internlm/model_implementations/transformers/modeling_mixtral.py

@@ -14,7 +14,7 @@
     scaled_init_method_uniform,
     uniform_,
 )
-from internlm.model.base_model import BaseModel
+from internlm.model_implementations.transformers.base_model import BaseModel
 from internlm.model.modules.embedding import Embedding1D
 from internlm.model.modules.linear import new_linear
 from internlm.model.modules.mha import SWA