diff --git a/LanguageModeling/BERT/config.py b/LanguageModeling/BERT/config.py
index f809f18..79ec270 100644
--- a/LanguageModeling/BERT/config.py
+++ b/LanguageModeling/BERT/config.py
@@ -58,7 +58,8 @@ def get_parser(parser=None):
                         help='use use fp16 or not')
     parser.add_argument('--use_xla', type=str2bool, nargs='?', const=True,
                         help='Whether to use use xla')
-
+    parser.add_argument("--num_accumulation_steps", type=int, default=1,
+                        help='Number of accumulation steps before gradient update, Global batch size = num_accumulation_steps * train_batch_size')
     parser.add_argument("--optimizer_type", type=str, default="adam",
                         help="Optimizer used for training - LAMB or ADAM")
 
diff --git a/LanguageModeling/BERT/oneflow_auto_bert.sh b/LanguageModeling/BERT/oneflow_auto_bert.sh
new file mode 100755
index 0000000..1623b67
--- /dev/null
+++ b/LanguageModeling/BERT/oneflow_auto_bert.sh
@@ -0,0 +1,200 @@
+#!/bin/bash
+
+BENCH_ROOT=$1
+PYTHON_WHL=$2
+CMP_OLD=$3
+
+# PYTHON_WHL=oneflow-0.3.5+cu112.git.325160b-cp38-cp38-linux_x86_64.whl
+# CMP_OLD=325160bcfb786b166b063e669aea345fadee2da7
+
+BERT_OSSDIR=oss://oneflow-staging/branch/master/bert/
+DOWN_FILE="wget https://oneflow-staging.oss-cn-beijing.aliyuncs.com/branch/master/bert/${CMP_OLD}/out.tar.gz"
+# DOWN_FILE="ossutil64 cp ${BERT_OSSDIR}$CMP_OLD/out.tar.gz .; "
+ENABLE_FP32=0
+GPU_NUM_PER_NODE=8
+BSZ=64
+
+PORT=57520
+
+PYTHON="python3.8"
+DOCKER_USER=root
+
+multi_machine() 
+{
+    # param 1 node
+    NUM_NODES=$1
+
+    # param 2 run cmd
+    RUN_CMD=$2
+
+    # param 3 output file
+    OUTPUT_FILE=$3
+
+    # param 4 python 
+    PYTHON=$4
+
+    # param 5  
+    IS_F32=$5
+
+    declare -a host_list=("10.11.0.2" "10.11.0.3" "10.11.0.4" "10.11.0.5")
+
+    if [ $NUM_NODES -gt ${#host_list[@]} ]
+    then
+        echo num_nodes should be less than or equal to length of host_list.
+        exit
+    fi
+
+    hosts=("${host_list[@]:0:${NUM_NODES}}")
+    echo "Working on hosts:${hosts[@]}"
+
+    ips=${hosts[0]}
+    for host in "${hosts[@]:1}" 
+    do
+        ips+=",${host}"
+    done
+
+    for host in "${hosts[@]:1}" 
+    do
+    echo "start training on ${host}"
+
+        echo -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+            nohup $RUN_CMD 0 $PYTHON  >/dev/null 2>&1 &" 
+
+        ssh -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+            nohup $RUN_CMD 0 $PYTHON  >/dev/null 2>&1 &"   
+
+    done
+
+    # copy files to master host and start work
+    host=${hosts[0]}
+    echo "start training on ${host}"
+
+    echo $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+        $RUN_CMD 1 $PYTHON "
+    ssh -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+        $RUN_CMD 1 $PYTHON "
+
+
+    for host in "${hosts[@]}" 
+    do
+        echo $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+            mkdir -p out/${OUTPUT_FILE}; mv -f log out/${OUTPUT_FILE}/log_1 "    
+        ssh -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+            mkdir -p out/${OUTPUT_FILE}; mv -f log out/${OUTPUT_FILE}/log_1 "
+    done
+
+    # Result analysis
+
+    host=${hosts[0]}
+    echo "start training on ${host}"
+
+    echo -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+        $PYTHON tools/result_analysis.py $IS_F32 \
+        --cmp1_file=./old/$OUTPUT_FILE/log_1/out.json \
+        --cmp2_file=./out/$OUTPUT_FILE/log_1/out.json \
+        --out=./pic/$OUTPUT_FILE.png "
+
+
+    ssh -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+        $PYTHON tools/result_analysis.py $IS_F32 \
+        --cmp1_file=./old/$OUTPUT_FILE/log_1/out.json \
+        --cmp2_file=./out/$OUTPUT_FILE/log_1/out.json \
+        --out=./pic/$OUTPUT_FILE.png "
+
+    echo "multi_machine done"
+
+}
+
+
+#######################################################################################
+#                     0 prepare the host list ips for training
+########################################################################################
+ALL_NODES=4
+
+declare -a host_list=("10.11.0.2" "10.11.0.3" "10.11.0.4" "10.11.0.5")
+
+if [ $ALL_NODES -gt ${#host_list[@]} ]
+then 
+    echo num_nodes should be less than or equal to length of host_list.
+    exit
+fi
+
+hosts=("${host_list[@]:0:${ALL_NODES}}")
+echo "Working on hosts:${hosts[@]}"
+
+ips=${hosts[0]}
+for host in "${hosts[@]:1}" 
+do
+   ips+=",${host}"
+done
+
+# #######################################################################################
+# #                     1 prepare oneflow_temp folder on each host
+# ########################################################################################
+
+for host in "${hosts[@]}" 
+do
+    ssh -p $PORT $DOCKER_USER@$host " rm -rf ~/oneflow_temp ; mkdir -p ~/oneflow_temp"
+    scp -P $PORT -r $BENCH_ROOT  $DOCKER_USER@$host:~/oneflow_temp/
+    echo "test--->"
+    # scp -P $PORT -r $PYTHON_WHL  $DOCKER_USER@$host:~/oneflow_temp/
+    # ssh -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/; \
+    #    $PYTHON -m pip install $PYTHON_WHL; "
+
+    ssh -p $PORT $DOCKER_USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+                                    mkdir -p pic; rm -rf pic/*; mkdir -p out; rm -rf out/* "
+    
+
+done
+
+#_______________________________________________________________________________________________
+host=${hosts[0]}
+ssh -p $PORT $DOCKER_USER@$host "cd ~; rm -rf ~/out; \
+                                ${DOWN_FILE}; \
+                                tar xvf out.tar.gz; \
+                                cp -rf ~/out ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT/old;"
+
+
+#######################################################################################
+#                                2   run   single
+########################################################################################
+
+if [ "$ENABLE_FP32" = 1 ];then
+    float_types=(0 1)
+else
+    float_types=(0 )
+fi
+num_nodes=(1 4)
+optimizers=(adam lamb)
+accumulations=(1 2)
+FLOAT_STR=(f16 f32)
+NUM_NODE_STR=(null single multi multi multi)
+
+
+
+for ftype in ${float_types[@]}
+do
+    for num_node in ${num_nodes[@]}
+    do
+        for optimizer in ${optimizers[@]}
+        do
+            for accumulation in ${accumulations[@]}
+            do
+                name=${NUM_NODE_STR[$num_node]}_bert_${FLOAT_STR[$ftype]}_pretraining_
+                multi_machine ${num_node} "sh train_perbert.sh 1 1 ${BSZ} 1 ${optimizer} ${GPU_NUM_PER_NODE} $num_node " \
+                "${name}${GPU_NUM_PER_NODE}gpu_${BSZ}bs_accumulation-${accumulation}_${optimizer}_debug" \
+                $PYTHON "--f32=${ftype}"
+            done #end accumulations
+        done  #end optimizer
+    done #end num_node
+done #float_types
+
+
+host=${hosts[0]}
+echo "start tar on ${host}"
+
+ssh $USER@$host "cd ~/oneflow_temp/OneFlow-Benchmark/LanguageModeling/BERT; \
+    tar  -zcvf out.tar.gz  out; \
+    $PYTHON  tools/stitching_pic.py --dir=pic --out_file=./pic/all.png "
+
+echo "multi_machine done"
diff --git a/LanguageModeling/BERT/prepare_auto_docker.sh b/LanguageModeling/BERT/prepare_auto_docker.sh
new file mode 100755
index 0000000..18aacbe
--- /dev/null
+++ b/LanguageModeling/BERT/prepare_auto_docker.sh
@@ -0,0 +1,50 @@
+NUM_NODES=${1-4}
+
+
+#######################################################################################
+#                     0 prepare the host list ips for training
+########################################################################################
+declare -a host_list=("10.11.0.2" "10.11.0.3" "10.11.0.4" "10.11.0.5")
+
+if [ $NUM_NODES -gt ${#host_list[@]} ]
+then 
+    echo num_nodes should be less than or equal to length of host_list.
+    exit
+fi
+
+hosts=("${host_list[@]:0:${NUM_NODES}}")
+echo "Working on hosts:${hosts[@]}"
+
+ips=${hosts[0]}
+for host in "${hosts[@]:1}" 
+do
+   ips+=",${host}"
+done
+
+#######################################################################################
+#                     1 prepare docker image
+########################################################################################
+WORK_PATH=`pwd`
+
+wget https://oneflow-staging.oss-cn-beijing.aliyuncs.com/branch/master/bert/docker_image/oneflow_autobert.tar
+
+for host in "${hosts[@]}" 
+do
+  ssh $USER@$host "mkdir -p ~/oneflow_docker_temp; rm -rf ~/oneflow_docker_temp/*"
+  scp -r oneflow_autobert.tar  $USER@$host:~/oneflow_docker_temp
+  ssh $USER@$host " docker load --input ~/oneflow_docker_temp/oneflow_autobert.tar; "
+
+  echo "tesst--->"
+  ssh $USER@$host "  \
+        docker run  --runtime=nvidia --rm -i -d --privileged --shm-size=16g \
+            --ulimit memlock=-1 --net=host  \
+            --name oneflow-auto-test   \
+            --cap-add=IPC_LOCK --device=/dev/infiniband  \
+            -v /data/bert/:/data/bert/ \
+            -v /datasets/bert/:/datasets/bert/  \
+            -v /datasets/ImageNet/OneFlow/:/datasets/ImageNet/OneFlow/ \
+            -v /data/imagenet/ofrecord:/data/imagenet/ofrecord \
+            -v ${WORK_PATH}:/workspace/oneflow-test \
+            -w /workspace/oneflow-test  \
+            oneflow:cu11.2-ubuntu18.04 bash   -c \"/usr/sbin/sshd -p 57520 && bash\" "
+done
diff --git a/LanguageModeling/BERT/run_pretraining.py b/LanguageModeling/BERT/run_pretraining.py
index 253e487..3d296e6 100755
--- a/LanguageModeling/BERT/run_pretraining.py
+++ b/LanguageModeling/BERT/run_pretraining.py
@@ -29,9 +29,24 @@
 parser.add_argument("--data_part_num", type=int, default=32, help="data part number in dataset")
 parser.add_argument("--iter_num", type=int, default=1144000, help="total iterations to run")
 parser.add_argument("--batch_size_per_device", type=int, default=64)
+parser.add_argument("--debug", type=int, default=0)
+parser.add_argument("--data_load_random", type=int, default=1)
+
 args = parser.parse_args()
 configs.print_args(args)
 
+
+if  args.debug == 1:
+    flow.config.enable_debug_mode(True)
+    print('Enable Debug !!!!!!!')
+
+if  args.data_load_random == 1:
+    random_tmp=True
+    print('Enable random loading of data !!!!!!!')
+else:
+    random_tmp=False
+    print('Disable random loading of data !!!!!!!')
+
 batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device
 
 
@@ -39,8 +54,8 @@ def BertDecoder(data_dir, batch_size, data_part_num, seq_length, max_predictions
     ofrecord = flow.data.ofrecord_reader(data_dir,
                                          batch_size=batch_size,
                                          data_part_num=data_part_num,
-                                         random_shuffle = False,
-                                         shuffle_after_epoch=False)
+                                         random_shuffle = random_tmp,
+                                         shuffle_after_epoch=random_tmp)
     blob_confs = {}
     def _blob_conf(name, shape, dtype=flow.int32):
         blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype)
@@ -101,12 +116,16 @@ def main():
     flow.env.log_dir(args.log_dir)
     flow.config.enable_debug_mode(True)
     
+    flow.config.enable_legacy_model_io()
+    flow.config.enable_model_io_v2(True)
+    
     InitNodes(args)
 
     snapshot = Snapshot(args.model_save_dir, args.model_load_dir)
 
+    print('num_accumulation_steps:', args.num_accumulation_steps)
     metric = Metric(desc='train', print_steps=args.loss_print_every_n_iter, 
-                    batch_size=batch_size, keys=['total_loss', 'mlm_loss', 'nsp_loss'])
+                    batch_size=batch_size * args.num_accumulation_steps, keys=['total_loss', 'mlm_loss', 'nsp_loss'])
     for step in range(args.iter_num):
         PretrainJob().async_get(metric.metric_cb(step))
         #PretrainJob().async_get(metric.metric_cb(step, epoch=3))
diff --git a/LanguageModeling/BERT/run_squad.sh b/LanguageModeling/BERT/run_squad.sh
new file mode 100644
index 0000000..4fd9e4b
--- /dev/null
+++ b/LanguageModeling/BERT/run_squad.sh
@@ -0,0 +1,74 @@
+BENCH_ROOT_DIR=/path/to/
+# pretrained model dir
+PRETRAINED_MODEL=/DATA/disk1/of_output/uncased_L-12_H-768_A-12_oneflow
+
+# squad ofrecord dataset dir
+DATA_ROOT=/DATA/disk1/of_output/bert/of_squad
+
+# `vocab.txt` dir
+REF_ROOT_DIR=/DATA/disk1/of_output/uncased_L-12_H-768_A-12
+
+# `evaluate-v*.py` and `dev-v*.json` dir
+SQUAD_TOOL_DIR=/DATA/disk1/of_output/bert/of_squad
+db_version=${1:-"v2.0"}
+if [ $db_version = "v1.1" ]; then
+  train_example_num=88614
+  eval_example_num=10833
+  version_2_with_negative="False"
+elif [ $db_version = "v2.0" ]; then
+  train_example_num=131944
+  eval_example_num=12232
+  version_2_with_negative="True"
+else
+  echo "db_version must be 'v1.1' or 'v2.0'"
+  exit
+fi
+
+train_data_dir=$DATA_ROOT/train-$db_version
+eval_data_dir=$DATA_ROOT/dev-$db_version
+LOGFILE=./bert_fp_training.log
+export PYTHONUNBUFFERED=1
+export ONEFLOW_DEBUG_MODE=True
+export CUDA_VISIBLE_DEVICES=7
+# finetune and eval SQuAD,
+# `predictions.json` will be saved to folder `./squad_output`
+python3 $BENCH_ROOT_DIR/run_squad.py \
+  --model=SQuAD \
+  --do_train=True \
+  --do_eval=True \
+  --gpu_num_per_node=1 \
+  --learning_rate=3e-5 \
+  --batch_size_per_device=16 \
+  --eval_batch_size_per_device=16 \
+  --num_epoch=3 \
+  --use_fp16 \
+  --version_2_with_negative=$version_2_with_negative \
+  --loss_print_every_n_iter=20 \
+  --do_lower_case=True \
+  --seq_length=384 \
+  --num_hidden_layers=12 \
+  --num_attention_heads=12 \
+  --max_position_embeddings=512 \
+  --type_vocab_size=2 \
+  --vocab_size=30522 \
+  --attention_probs_dropout_prob=0.1 \
+  --hidden_dropout_prob=0.1 \
+  --hidden_size_per_head=64 \
+  --train_data_dir=$train_data_dir \
+  --train_example_num=$train_example_num \
+  --eval_data_dir=$eval_data_dir \
+  --eval_example_num=$eval_example_num \
+  --log_dir=./log \
+  --model_load_dir=${PRETRAINED_MODEL} \
+  --save_last_snapshot=True \
+  --model_save_dir=./squad_snapshots \
+  --vocab_file=$REF_ROOT_DIR/vocab.txt \
+  --predict_file=$SQUAD_TOOL_DIR/dev-${db_version}.json \
+  --output_dir=./squad_output 2>&1 | tee ${LOGFILE}
+
+
+# evaluate predictions.json to get metrics
+python3 $SQUAD_TOOL_DIR/evaluate-${db_version}.py \
+  $SQUAD_TOOL_DIR/dev-${db_version}.json \
+  ./squad_output/predictions.json
+
diff --git a/LanguageModeling/BERT/stop_all_docker.sh b/LanguageModeling/BERT/stop_all_docker.sh
new file mode 100755
index 0000000..78c6078
--- /dev/null
+++ b/LanguageModeling/BERT/stop_all_docker.sh
@@ -0,0 +1,31 @@
+NUM_NODES=${1-4}
+
+#######################################################################################
+#                     0 prepare the host list ips for training
+########################################################################################
+declare -a host_list=("10.11.0.2" "10.11.0.3" "10.11.0.4" "10.11.0.5")
+
+if [ $NUM_NODES -gt ${#host_list[@]} ]
+then 
+    echo num_nodes should be less than or equal to length of host_list.
+    exit
+fi
+
+hosts=("${host_list[@]:0:${NUM_NODES}}")
+echo "Working on hosts:${hosts[@]}"
+
+ips=${hosts[0]}
+for host in "${hosts[@]:1}" 
+do
+   ips+=",${host}"
+done
+
+#######################################################################################
+#                     1 prepare docker image
+########################################################################################
+
+for host in "${hosts[@]}" 
+do
+  ssh $USER@$host "docker kill oneflow-auto-test"
+
+done
diff --git a/LanguageModeling/BERT/tools/README.md b/LanguageModeling/BERT/tools/README.md
new file mode 100644
index 0000000..3a9baa7
--- /dev/null
+++ b/LanguageModeling/BERT/tools/README.md
@@ -0,0 +1,12 @@
+## Oneflow BERT automatic test tools
+Automatically execute bert modle with different parameters and analyze the results
+### Dependent environment
+- matplotlib
+    ```
+    pip install  matplotlib
+    ```
+
+### Features
+- `analysis.py`,Analyze the log file to get the total loss mlm loss nsp loos through GPU memory
+- `result_analysis.py`,Analyze the running results of the two versions and output reports
+- `stitching_pic.py` Multiple pictures are spliced ​​together
diff --git a/LanguageModeling/BERT/tools/analysis.py b/LanguageModeling/BERT/tools/analysis.py
new file mode 100644
index 0000000..c25b342
--- /dev/null
+++ b/LanguageModeling/BERT/tools/analysis.py
@@ -0,0 +1,63 @@
+import argparse
+import re
+import json
+
+from ctypes import *
+
+
+
+def collect_loss(log_file, gpu_num):
+    print("loss : ",log_file)
+
+    f = open(log_file,"r")  
+    lines = f.readlines()
+    total_loss = []
+    mlm_loss = []
+    nsp_loss = []
+    throughput =[]
+    memory=[]
+
+    pattern = re.compile(r"step:\s*(\d+)\s*,\s*total_loss:\s*(\d+\.?\d+)\s*,\s*mlm_loss:\s*(\d+\.?\d+)\s*,\s*nsp_loss:\s*(\d+\.?\d+)\s*,\s*throughput:\s*(\d+\.?\d+)\s*")
+    for line in lines:
+        if(line.split(':')[0] == 'step'):
+            
+            match = pattern.match(line)
+            if match:
+                total_loss.append(match.group(2))
+                mlm_loss.append(match.group(3))
+                nsp_loss.append(match.group(4))
+                throughput.append(match.group(5))
+        if(line.split(' [MiB]\\n')[0] == 'b\'memory.used'):
+            str_tmp = line.split(' [MiB]\\n')[1]
+
+            for i in range(gpu_num):
+                memory.append(str_tmp.split(' MiB\\n')[i])
+
+    return total_loss, mlm_loss, nsp_loss,throughput, memory
+
+
+def main():
+    parser = argparse.ArgumentParser(description="collect GPU device memory usage")
+    parser.add_argument("--log_file", type=str, default=None)
+    parser.add_argument("--mem_file", type=str, default=None)
+    parser.add_argument("--out_file", type=str, default=None)
+    parser.add_argument("--gpu_num", type=int, default=1)
+
+    args = parser.parse_args()
+
+    total_loss, mlm_loss, nsp_loss,throughput, memory = collect_loss(args.log_file, args.gpu_num)
+
+    out={}
+    out['total_loss'] = total_loss
+    out['mlm_loss'] = mlm_loss
+    out['nsp_loss'] = nsp_loss
+    out['throughput'] = throughput
+    out['memory'] = memory
+
+    string = json.dumps(out)
+    with open(args.out_file,'w')as f:
+        f.write(string)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/LanguageModeling/BERT/tools/gpu_memory_usage.py b/LanguageModeling/BERT/tools/gpu_memory_usage.py
new file mode 100644
index 0000000..6f3daa8
--- /dev/null
+++ b/LanguageModeling/BERT/tools/gpu_memory_usage.py
@@ -0,0 +1,63 @@
+import time
+import argparse
+import pynvml
+
+
+class Device(object):
+    class Status:
+        INIT = "INIT"
+        DETECTING = "DETECTING"
+        STOP = "STOP"
+
+    start_detecting_mem_threshold = 32 * 1024 * 1024
+
+    def __init__(self, handle):
+        self.handle = handle
+        self.status = self.Status.INIT
+        self.max_mem_usage = 0
+
+    def update(self):
+        info = pynvml.nvmlDeviceGetMemoryInfo(self.handle)
+        if self.status == self.Status.INIT:
+            if info.used > self.start_detecting_mem_threshold:
+                self.status = self.Status.DETECTING
+        elif self.status == self.Status.DETECTING:
+            if info.used < self.start_detecting_mem_threshold:
+                self.status = self.Status.STOP
+                return False
+            else:
+                self.max_mem_usage = max(self.max_mem_usage, info.used)
+        elif self.status == self.Status.STOP:
+            raise ValueError("detecting is stop")
+        else:
+            raise ValueError("invalid status")
+
+        return True
+
+
+def main():
+    parser = argparse.ArgumentParser(description="collect GPU device memory usage")
+    parser.add_argument("-g", type=int, default=1, help="number of gpu devices")
+    parser.add_argument("-n", type=float, default=1, help="metrics rate")
+    args = parser.parse_args()
+
+    pynvml.nvmlInit()
+    n_gpus = args.g
+    devices = [Device(pynvml.nvmlDeviceGetHandleByIndex(i)) for i in range(n_gpus)]
+
+    running = True
+    while running:
+        time.sleep(args.n)
+        running = False
+        for device in devices:
+            running |= device.update()
+
+    pynvml.nvmlShutdown()
+    for i, device in enumerate(devices):
+        max_mem_usage_mbytes = device.max_mem_usage / 1024 / 1024
+        print(f"{max_mem_usage_mbytes:.2f}")
+
+
+if __name__ == "__main__":
+    main()
+    
diff --git a/LanguageModeling/BERT/tools/result_analysis.py b/LanguageModeling/BERT/tools/result_analysis.py
new file mode 100644
index 0000000..fe8b988
--- /dev/null
+++ b/LanguageModeling/BERT/tools/result_analysis.py
@@ -0,0 +1,134 @@
+import json
+import matplotlib.pyplot as plt
+import argparse
+import numpy as np
+
+def read_file(file1, file2):
+
+    with open(file1,'r') as load_f:
+        dict1 = json.load(load_f)
+
+    with open(file2,'r') as load_f:
+        dict2 = json.load(load_f)
+
+
+    return dict1, dict2
+
+def analysis_loss(y3, f32):
+
+    # Calculate the correlation coefficient
+
+    # if f32 == 1:
+    #     iter = len(y3)
+
+    # else:
+    # len = y3.shape[0]
+    # min = y3.min()
+    # max = y3.max()
+    # mean = np.mean(y3)
+    # var = np.var(y3) 
+
+    # random = np.random.randn(len)/10000.0
+    # print(random)
+    # print(y3)
+    # lst=[y3,random]
+    # res=np.corrcoef(lst)
+    # print('----->', res)
+
+    len = y3.shape[0]
+
+    if f32 == 1:
+        iter = np.count_nonzero(y3==0)
+        tmp = iter/len
+        print('count zeor = ', iter)
+        if iter/len > 0.6:
+            print('Test passed')
+            return 1
+        else:
+            print('Test failed')
+            return 0
+    else:
+
+        mean = np.mean(y3)
+        var = np.var(y3) 
+        print('F16---->', abs(mean), var)
+
+        if abs(mean) < 0.001 and var < 0.00001:
+            print('Test passed')
+            return 1
+        else:
+            print('Test failed')
+            return 0
+
+def drawing_loss(dict1, dict2, f32, image):
+
+
+    m1 = dict1["memory"]
+    m2 = dict2["memory"]
+    
+    print(m1)
+    print(m2)
+    table_len = len(m1)
+    row_labels = ['old','new']
+    table_vals = [m1,m2]
+
+
+    y1 = dict1["total_loss"]
+    y2 = dict2["total_loss"]
+    y1=list(map(float,y1))
+    y2=list(map(float,y2))
+    x = np.arange(1, len(y1)+1)
+    y1 = np.array(y1)
+    y2 = np.array(y2)
+    y3 = np.subtract(y1,y2)
+
+    result = analysis_loss(y3, f32)
+
+    print(x)
+    print(y1)
+    print(y2)
+    print(y3)
+    fig = plt.figure(figsize=(24,8), dpi=150)
+    plt.figure(1) 
+    ax = fig.add_subplot()
+
+
+    ax1 = plt.subplot(121)
+    plt.xlabel('iterations')
+    plt.plot(x,y1,color='red',label='Diachronic version')
+    plt.plot(x,y2,color='blue',label='Current version')
+    plt.title('Loss comparison')
+
+    plt.legend(loc='best')
+    
+    ax2 = plt.subplot(122)
+    plt.xlabel('iterations')
+    plt.plot(x,y3,color='red')  
+    plt.title('Loss difference')
+    plt.table(cellText=table_vals, rowLabels=row_labels, colWidths=[0.05]*table_len, loc='best')
+
+    plt.suptitle(image.split('/')[1].split('.')[0],fontsize=20,x=0.5,y=0.98)
+
+    if result == 1:
+        plt.text(0.9, 1,'PASS', fontsize=50, color='blue', transform=ax.transAxes)
+    else:
+        plt.text(0.9, 1,'FAILED',fontsize=50,color='red',transform=ax.transAxes)
+    plt.savefig(image)
+    
+
+def main():
+    print('test')
+    parser = argparse.ArgumentParser(description="Compare and analyze training results and output icons")
+    parser.add_argument("--cmp1_file", type=str, default=None)
+    parser.add_argument("--cmp2_file", type=str, default=None)
+    parser.add_argument("--out_file", type=str, default=None)
+    parser.add_argument("--f32", type=int, default=0)
+    args = parser.parse_args()
+
+    dict1, dict2 = read_file(args.cmp1_file, args.cmp2_file)
+
+    drawing_loss(dict1, dict2, args.f32, args.out_file)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/LanguageModeling/BERT/tools/stitching_pic.py b/LanguageModeling/BERT/tools/stitching_pic.py
new file mode 100644
index 0000000..dac5c33
--- /dev/null
+++ b/LanguageModeling/BERT/tools/stitching_pic.py
@@ -0,0 +1,38 @@
+import matplotlib.pyplot as plt
+import argparse
+import os
+from PIL import Image
+
+def stiiching_pic(dir, out_file):
+
+    hight = 1200 
+    width = 3600
+    file_lsit = os.listdir(dir)
+    target = Image.new('RGBA', (width, hight*len(file_lsit)))
+    left = 0
+    right = hight
+    for file in file_lsit:
+
+        tmp = dir+'/'+file
+        print(tmp)
+        image = Image.open(tmp)
+        # print(image)
+        # print(target)
+        target.paste(image, (0, left, width, right))
+        left += hight 
+        right += hight
+    target.save(out_file)
+
+
+def main():
+    print('test')
+    parser = argparse.ArgumentParser(description="Stitching pictures")
+    parser.add_argument("--dir", type=str, default=None)
+    parser.add_argument("--out_file", type=str, default=None)
+    args = parser.parse_args()
+
+    stiiching_pic(args.dir, args.out_file)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/LanguageModeling/BERT/train_perbert.sh b/LanguageModeling/BERT/train_perbert.sh
new file mode 100644
index 0000000..079dadf
--- /dev/null
+++ b/LanguageModeling/BERT/train_perbert.sh
@@ -0,0 +1,132 @@
+USE_FP16=${1:-0}
+#DEBUG
+DEBUG_MODE=${2:-0}
+#
+BATCH_SIZE=${3:-64}
+#accumulation
+ACCUMULATION_STEMPS=${4-1}
+#
+OPTIMIZER=${5-adam}
+#GPU
+GPUS_PER_NODE=${6:-8}
+#
+NNODES=${7:-1}
+#
+MASTER=${8:-0}
+
+PYTHON=${9:-python}
+
+LOG_FOLDER=./log/
+
+PRINT_ITER=1
+ITER_NUM=130
+
+NODE_IPS='10.11.0.2','10.11.0.3','10.11.0.4','10.11.0.5'
+
+INIT_MODEL=/data/bert/initial_model/
+#DATA_DIR=/data/bert_dataset
+DATA_DIR=/data/bert/wiki_seq_len_128/
+
+##########################################################################################################
+#                                           FP
+##########################################################################################################
+echo ${USE_FP16}
+
+
+if [ "$USE_FP16" = 1 ];then
+  FP_CMD=--use_fp16
+  FP_NAME=f16
+  echo "USE_FP16"
+else
+  FP_CMD=
+  FP_NAME=f32
+  echo "USE_FP32"
+fi
+##########################################################################################################
+#                                           DEBUG_NAME
+##########################################################################################################
+# if [ DEBUG_MODE==1 ];then
+#   DEBUG_NAME=debug
+# else
+#   DEBUG_NAME=
+# fi
+
+##########################################################################################################
+#                                           Create folder
+##########################################################################################################
+#bert_f32_pretraining_8gpu_64bs_100iter_lamb_debug
+mkdir -p $LOG_FOLDER
+
+LOGFILE=$LOG_FOLDER/bert_pretraining_${FP_NAME}_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter_${OPTIMIZER}\
+_${DEBUG_NAME}.log
+
+MODEL_DIR=./snapshots/
+
+MEM_FILE=$LOG_FOLDER/memory.log
+
+echo LOGFILE=$LOGFILE
+echo DATA_DIR=$DATA_DIR
+
+rm -rf ${MODEL_DIR}/*
+rm -rf ${LOG_FOLDER}/*
+
+NVPROF=baseline-report_${NODE_RANK}
+#-o ${NVPROF} 
+
+export NCCL_DEBUG=INFO
+export PYTHONUNBUFFERED=1
+
+
+#nsys profile --stats=true -o ${NVPROF}  \
+
+$PYTHON run_pretraining.py \
+  --gpu_num_per_node=${GPUS_PER_NODE} \
+  --num_nodes=${NNODES} \
+  --node_ips=$NODE_IPS \
+  --learning_rate=1e-4 \
+  --warmup_proportion=0.01 \
+  --weight_decay_rate=0.01 \
+  --batch_size_per_device=${BATCH_SIZE} \
+  --iter_num=${ITER_NUM} \
+  --loss_print_every_n_iter=${PRINT_ITER} \
+  --seq_length=128 \
+  --max_predictions_per_seq=20 \
+  --num_hidden_layers=12 \
+  --num_attention_heads=12 \
+  --num_accumulation_steps=${ACCUMULATION_STEMPS} \
+  --max_position_embeddings=512 \
+  --type_vocab_size=2 \
+  --vocab_size=30522 \
+  --attention_probs_dropout_prob=0 \
+  --hidden_dropout_prob=0 \
+  --hidden_size_per_head=64 \
+  --data_part_num=64 \
+  --data_dir=$DATA_DIR \
+  --log_dir=${LOG_FOLDER} \
+  --model_save_every_n_iter=10000 \
+  --save_last_snapshot=True \
+  --model_save_dir=./snapshots \
+  --debug=${DEBUG_MODE} \
+  --data_load_random=0 \
+  --model_load_dir=${INIT_MODEL} \
+  ${FP_CMD} \
+  --optimizer_type=${OPTIMIZER} \
+  2>&1 | tee ${LOGFILE} 
+
+echo "Writting log to ${LOGFILE}"
+
+SQLITE=$LOG_FOLDER/bert_pretraining_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter.sqlite
+QDREP=$LOG_FOLDER/bert_pretraining_${GPUS_PER_NODE}gpu_${BATCH_SIZE}bs_${ITER_NUM}iter.qdrep
+
+# mv $NVPROF.sqlite $SQLITE
+# mv $NVPROF.qdrep  $QDREP
+
+if [ "$MASTER" = 1 ]; then
+  json_file=${LOG_FOLDER}out.json
+  python tools/analysis.py \
+    --log_file=$LOGFILE \
+    --mem_file=$MEM_FILE \
+    --out_file=$json_file \
+    --gpu_num=$GPUS_PER_NODE 
+
+fi
diff --git a/LanguageModeling/BERT/util.py b/LanguageModeling/BERT/util.py
index d2637dd..a355e98 100755
--- a/LanguageModeling/BERT/util.py
+++ b/LanguageModeling/BERT/util.py
@@ -21,6 +21,7 @@
 import pandas as pd
 from datetime import datetime
 import oneflow as flow
+import subprocess
 
 
 def InitNodes(args):
@@ -116,6 +117,9 @@ def metric_cb(self, step=0, **kwargs):
         def callback(outputs):
             if step == 0: self._clear()
 
+            if step == 1:
+                print(subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv ", shell=True))
+
             for key in self.keys:
                 self.metric_dict[key] += outputs[key].sum()
                 self.metric_dict['n_' + key] += outputs[key].size
@@ -131,7 +135,7 @@ def callback(outputs):
                 for key in self.keys:
                     value = self.metric_dict[key] / self.metric_dict['n_' + key]
                     self.update_and_save(key, value, step, **kwargs)
-                print(', '.join(('{}: {}' if type(v) is int else '{}: {:.3f}').format(k, v) \
+                print(', '.join(('{}: {}' if type(v) is int else '{}: {}').format(k, v) \
                                 for k, v in self.metric_dict.items()), time.time())
                 self._clear()
 
@@ -160,6 +164,7 @@ def CreateOptimizer(args):
 def GetFunctionConfig(args):
     config = flow.function_config()
     config.enable_auto_mixed_precision(args.use_fp16)
+    config.train.num_gradient_accumulation_steps(args.num_accumulation_steps)
     if args.use_xla:
         config.use_xla_jit(True)
     config.enable_fuse_add_to_output(True)
diff --git a/requirements.txt b/requirements.txt
index 1eedb7a..1b55c50 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
 numpy>=1.17.2
 pandas>=1.0.4
-pillow>=7.2.0
\ No newline at end of file
+pillow>=7.2.0
+matplotlib>=3.4.2
\ No newline at end of file