Fix Hugginface Transformers version (#295)

* Fix Hugginface Transformers version

Signed-off-by: Abolfazl Shahbazi <[email protected]>

* Fix the patch file name too

Signed-off-by: Abolfazl Shahbazi <[email protected]>

* Fix the warnings in patch file

Signed-off-by: Abolfazl Shahbazi <[email protected]>

Author: ashahba

quickstart/language_modeling/pytorch/bert_large/inference/cpu/README.md

@@ -83,8 +83,8 @@ export MODEL_DIR=$(pwd)
 cd quickstart/language_modeling/pytorch/bert_large/inference/cpu
 git clone https://github.com/huggingface/transformers.git
 cd transformers
-git checkout v4.10.0
-git apply ../enable_ipex_for_bert-base.diff
+git checkout v3.0.2
+git apply ../enable_ipex_for_squad.diff
 pip install -e ./
 
 # Env vars

quickstart/language_modeling/pytorch/bert_large/inference/cpu/enable_ipex_for_squad.diff

@@ -8,8 +8,8 @@ index 2bd4e90..e18a296 100644
  import timeit
 +import time
 +import sys
-+import threading 
- 
++import threading
+
  import numpy as np
  import torch
 @@ -45,19 +48,22 @@ from transformers.data.metrics.squad_metrics import (
@@ -18,29 +18,29 @@ index 2bd4e90..e18a296 100644
  from transformers.data.processors.squad import SquadResult, SquadV1Processor, SquadV2Processor
 -
 +import intel_extension_for_pytorch as ipex
- 
+
  try:
      from torch.utils.tensorboard import SummaryWriter
  except ImportError:
      from tensorboardX import SummaryWriter
- 
+
 -
  logger = logging.getLogger(__name__)
- 
+
  MODEL_CONFIG_CLASSES = list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
  MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
- 
+
 +def trace_handler(prof):
 +    print(prof.key_averages().table(
 +        sort_by="self_cpu_time_total", row_limit=-1))
 +    prof.export_chrome_trace("./log/test_trace_" + str(prof.step_num) + ".json")
- 
+
  def set_seed(args):
      random.seed(args.seed)
 @@ -264,19 +270,95 @@ def train(args, train_dataset, model, tokenizer):
- 
+
      return global_step, tr_loss / global_step
- 
+
 +def wrap_model(model, args):
 +    model.eval()
 +    ipex.nn.utils._model_convert.replace_dropout_with_identity(model)
@@ -63,7 +63,7 @@ index 2bd4e90..e18a296 100644
 +        # enable fusion path work(need to run two interation).
 +        with torch.no_grad():
 +            y = model(dumpy_tensor, dumpy_tensor, dumpy_tensor)
-+            y = model(dumpy_tensor, dumpy_tensor, dumpy_tensor) 
++            y = model(dumpy_tensor, dumpy_tensor, dumpy_tensor)
 +            #dumpy_tensor = torch.ones((128, 384), dtype=torch.long)
 +            #y = model(dumpy_tensor, dumpy_tensor, dumpy_tensor)
 +            #dumpy_tensor = torch.ones((81, 384), dtype=torch.long)
@@ -79,9 +79,9 @@ index 2bd4e90..e18a296 100644
 +    elif args.use_jit:
 +        with torch.no_grad():
 +            model = torch.jit.trace(model, jit_inputs, strict=False)
-+            #model = torch.jit._recursive.wrap_cpp_module(torch._C._freeze_module(model._c, preserveParameters=True)) 
++            #model = torch.jit._recursive.wrap_cpp_module(torch._C._freeze_module(model._c, preserveParameters=True))
 +            model = torch.jit.freeze(model)
-+    return model 
++    return model
 +
 +def benchmark_evaluate(args, model, eval_dataloader):
 +    total_time = 0
@@ -107,32 +107,32 @@ index 2bd4e90..e18a296 100644
 +                "token_type_ids": batch[2],
 +            }
 +            time_start = time.time()
-+            outputs = model(**inputs)          
++            outputs = model(**inputs)
 +            #prof.step()
 +            time_end = time.time()
 +            i += 1
 +            if i > args.perf_begin_iter:
 +                total_time +=(time_end - time_start)
 +            if i >= args.perf_run_iters + args.perf_begin_iter:
 +                throughput = args.eval_batch_size * args.perf_run_iters / total_time
-+                print("Throughput: {:.3f} sentence/s".format(throughput))     
++                print("Throughput: {:.3f} sentence/s".format(throughput))
 +                break
- 
+
  def evaluate(args, model, tokenizer, prefix=""):
      dataset, examples, features = load_and_cache_examples(args, tokenizer, evaluate=True, output_examples=True)
 -
      if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
          os.makedirs(args.output_dir)
 -
-+ 
++
      args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
- 
+
 +    model = wrap_model(model, args)
      # Note that DistributedSampler samples randomly
      eval_sampler = SequentialSampler(dataset)
      eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
 -
-+      
++
      # multi-gpu evaluate
      if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
          model = torch.nn.DataParallel(model)
@@ -141,7 +141,7 @@ index 2bd4e90..e18a296 100644
      logger.info("  Num examples = %d", len(dataset))
      logger.info("  Batch size = %d", args.eval_batch_size)
 -
-+    
++
 +    if args.do_calibration:
 +        conf = ipex.quantization.QuantConf(qscheme=torch.per_tensor_affine)
 +        for step, batch in enumerate(eval_dataloader):
@@ -168,18 +168,18 @@ index 2bd4e90..e18a296 100644
 +            for t in threads:
 +                t.join()
 +        else:
-+            benchmark_evaluate(args, model, eval_dataloader)        
++            benchmark_evaluate(args, model, eval_dataloader)
 +        exit()
      all_results = []
      start_time = timeit.default_timer()
 -
-+    
++
      for batch in tqdm(eval_dataloader, desc="Evaluating"):
          model.eval()
          batch = tuple(t.to(args.device) for t in batch)
 @@ -658,6 +768,36 @@ def main():
      parser.add_argument("--server_port", type=str, default="", help="Can be used for distant debugging.")
- 
+
      parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
 +    parser.add_argument(
 +        "--bf16",
@@ -198,7 +198,7 @@ index 2bd4e90..e18a296 100644
 +                        help='use llga int8 in pytorch jit model')
 +    parser.add_argument('--int8_fp32', dest='int8_fp32', action='store_true',
 +                        help='use int8 fp32 mix precision')
-+    parser.add_argument("--int8_config", type=str, default="config.json", 
++    parser.add_argument("--int8_config", type=str, default="config.json",
 +                        help="quantization config file for int8 mode")
 +    parser.add_argument("--do_calibration", action='store_true',
 +                        help="Enable calibration process")
@@ -212,20 +212,20 @@ index 2bd4e90..e18a296 100644
 +                        help="Total cores used for this process, used for share_weight mode")
 +
      args = parser.parse_args()
- 
+
      if args.doc_stride >= args.max_seq_length - args.max_query_length:
 diff --git a/src/transformers/modeling_bert.py b/src/transformers/modeling_bert.py
 index 23d25cf..b281147 100644
 --- a/src/transformers/modeling_bert.py
 +++ b/src/transformers/modeling_bert.py
 @@ -139,7 +139,7 @@ def mish(x):
      return x * torch.tanh(nn.functional.softplus(x))
- 
- 
+
+
 -ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new, "mish": mish}
 +ACT2FN = {"gelu": torch.nn.functional.gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new, "mish": mish}
- 
- 
+
+
  BertLayerNorm = torch.nn.LayerNorm
 @@ -239,6 +239,8 @@ class BertSelfAttention(nn.Module):
          attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
@@ -235,4 +235,4 @@ index 23d25cf..b281147 100644
 +                attention_mask = attention_mask.to(attention_scores.dtype)
              # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
              attention_scores = attention_scores + attention_mask
- 
+