update nllb recipe and readme

recipes/nllb/README.md

@@ -2,12 +2,34 @@
 
 ## Conversion
 
+### 1. Sentencepiece with OpenNMT Tokenizer
+
 ```bash
 eole convert HF --model_dir facebook/nllb-200-1.3B --output ./nllb-1.3b --token $HF_TOKEN --tokenizer onmt
 ```
 
+### 2. HuggingFace Tokenizer
+
+```bash
+eole convert HF --model_dir facebook/nllb-200-1.3B --output ./nllb-1.3b --token $HF_TOKEN
+```
+
+
 ## Inference
 
 ```bash
-eole predict -c inference.yaml
+echo "What is the weather like in Tahiti?" > test.en
+```
+
+
+### 1. Sentencepiece with OpenNMT Tokenizer
+
+```bash
+eole predict -c inference-pyonmttok.yaml
+```
+
+### 2. HuggingFace Tokenizer
+
+```bash
+eole predict -c inference-hf.yaml
 ```

recipes/nllb/inference.yaml → recipes/nllb/inference-hf.yaml

@@ -1,13 +1,11 @@
 model_path: "nllb-1.3b"
-transforms: ["sentencepiece", "prefix"]
+transforms: ["prefix", "huggingface_tokenize"]
 transforms_configs:
   prefix:
     src_prefix: "</s> eng_Latn"
     tgt_prefix: "deu_Latn"
-  sentencepiece:
-    src_subword_model: ./flores200_sacrebleu_tokenizer_spm.model
-    tgt_subword_model: ./flores200_sacrebleu_tokenizer_spm.model
-
+  huggingface_tokenize:
+    huggingface_model: facebook/nllb-200-1.3B
 
 tgt_file_prefix: true
 

recipes/nllb/inference-pyonmttok.yaml

@@ -0,0 +1,21 @@
+model_path: "nllb-1.3b"
+# transforms: ["sentencepiece", "prefix"]
+transforms: ["prefix", "huggingface_tokenize"]
+transforms_configs:
+  prefix:
+    src_prefix: "</s> eng_Latn"
+    tgt_prefix: "deu_Latn"
+  # sentencepiece:
+  #   src_subword_model: ./flores200_sacrebleu_tokenizer_spm.model
+  #   tgt_subword_model: ./flores200_sacrebleu_tokenizer_spm.model
+  huggingface_tokenize:
+    huggingface_model: facebook/nllb-200-1.3B
+
+tgt_file_prefix: true
+
+gpu_ranks: [0]
+world_size: 1
+beam_size: 5
+
+src: test.en
+output: test.de