fix: update imports and dependencies to address vulnerabilities

langtest/datahandler/datasource.py

@@ -1393,7 +1393,7 @@ def __init__(self, dataset: dict, task: TaskManager):
             task (str): Task to be evaluated on.
         """
         self.dataset_name = dataset["data_source"]
-        self.sub_name = dataset.get("subset", "sst2")
+        self.sub_name = dataset.get("subset", "SetFit/sst2")
         self.task = task
 
     @staticmethod

langtest/modelhandler/transformers_modelhandler.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 import logging
 import numpy as np
 from functools import lru_cache
@@ -407,62 +407,138 @@ def __call__(
 
 
 class PretrainedModelForTranslation(ModelAPI):
-    """Transformers pretrained model for translation tasks
+    """Transformers pretrained model for translation tasks.
 
     Args:
-        model (transformers.pipeline.Pipeline): Pretrained HuggingFace translation pipeline for predictions.
+        tokenizer: Pretrained HuggingFace tokenizer.
+        model: Pretrained HuggingFace sequence-to-sequence or causal LM model.
+        source_lang (str): Default source language for translation.
+        target_lang (str): Default target language for translation.
+        is_encoder_decoder (bool): Whether the loaded model is an encoder-decoder type.
     """
 
-    def __init__(self, model):
-        """Constructor method
-
-        Args:
-            model (transformers.pipeline.Pipeline): Pretrained HuggingFace NER pipeline for predictions.
-        """
-        assert isinstance(model, Pipeline), ValueError(
-            Errors.E079(Pipeline=Pipeline, type_model=type(model))
-        )
+    def __init__(
+        self,
+        tokenizer: AutoTokenizer,
+        model: Any,
+        source_lang: str = "English",
+        target_lang: str = "German",
+        is_encoder_decoder: bool = True,
+    ):
+        self.tokenizer = tokenizer
         self.model = model
+        self.source_lang = source_lang
+        self.target_lang = target_lang
+        self.is_encoder_decoder = is_encoder_decoder
 
     @classmethod
-    def load_model(cls, path: str, *args, **kwargs) -> "Pipeline":
-        """Load the Translation model into the `model` attribute.
+    def load_model(cls, path: str, *args, **kwargs) -> "PretrainedModelForTranslation":
+        """Load the Translation model into the `model` attribute."""
+        from transformers import (
+            AutoTokenizer,
+            AutoModelForSeq2SeqLM,
+            AutoModelForCausalLM,
+            AutoConfig,
+        )
 
-        Args:
-            path (str):
-                path to model or model name
+        source_lang = kwargs.pop("source_language", None)
+        target_lang = kwargs.pop("target_language", None)
 
-        Returns:
-            'Pipeline':
-        """
-        from ..langtest import HARNESS_CONFIG as harness_config
+        # Fallback to langtest configurations if not explicitly provided
+        if not source_lang or not target_lang:
+            try:
+                from langtest import HARNESS_CONFIG
 
-        config = harness_config["model_parameters"]
-        tgt_lang = config.get("target_language") or kwargs.get("target_language")
+                config_harness = HARNESS_CONFIG.get("model_parameters", {})
+                source_lang = source_lang or config_harness.get(
+                    "source_language", "English"
+                )
+                target_lang = target_lang or config_harness.get(
+                    "target_language", "German"
+                )
+            except ImportError:
+                source_lang = source_lang or "English"
+                target_lang = target_lang or "German"
+
+        tokenizer_loaded = AutoTokenizer.from_pretrained(path)
+        model_config = AutoConfig.from_pretrained(path)
+        is_enc_dec = model_config.is_encoder_decoder
 
-        if "t5" in path:
-            return cls(pipeline(f"translation_en_to_{tgt_lang}", model=path))
+        # Pass remaining args/kwargs (like device_map) to the model loading
+        if is_enc_dec:
+            model_loaded = AutoModelForSeq2SeqLM.from_pretrained(path, *args, **kwargs)
         else:
-            return cls(pipeline(model=path, src_lang="en", tgt_lang=tgt_lang))
+            print(
+                f"Warning: Model '{path}' is not an encoder-decoder model. "
+                "Loading as AutoModelForCausalLM. Performance may vary."
+            )
+            model_loaded = AutoModelForCausalLM.from_pretrained(path, *args, **kwargs)
+
+        return cls(
+            tokenizer_loaded,
+            model_loaded,
+            source_lang,
+            target_lang,
+            is_encoder_decoder=is_enc_dec,
+        )
 
     @lru_cache(maxsize=102400)
-    def predict(self, text: str, **kwargs) -> TranslationOutput:
-        """Perform predictions on the input text.
+    def predict(
+        self,
+        text: str,
+        source_lang: Optional[str] = None,
+        target_lang: Optional[str] = None,
+        **kwargs,
+    ) -> TranslationOutput:
+        """
+        Perform predictions on the input text. Wraps kwargs to enable LRU caching.
 
         Args:
             text (str): Input text to perform translation on.
-            kwargs: Additional keyword arguments.
-
+            source_lang (Optional[str]): Source language for translation. Defaults to the instance's source_lang.
+            target_lang (Optional[str]): Target language for translation. Defaults to the instance's target_lang.
 
         Returns:
-            TranslationOutput: Output model for translation tasks
+            TranslationOutput: Translated text from the input text.
         """
-        prediction = self.model(text, **kwargs)[0]["translation_text"]
+        # Convert kwargs to a hashable frozenset to prevent TypeError in lru_cache
+        current_source = source_lang or self.source_lang
+        current_target = target_lang or self.target_lang
+
+        input_text = f"translate {current_source} to {current_target}: {text}"
+        inputs = self.tokenizer(input_text, return_tensors="pt").to(self.model.device)
+
+        # Safely extract and construct generation arguments
+        gen_kwargs = {
+            "max_length": kwargs.pop("max_length", 512),
+            "num_beams": kwargs.pop("num_beams", 1),
+            **kwargs,
+        }
+
+        # Hugging Face crashes if diversity_penalty is set without num_beam_groups > 1
+        num_beam_groups = kwargs.pop("num_beam_groups", 1)
+        if num_beam_groups > 1:
+            gen_kwargs["num_beam_groups"] = num_beam_groups
+            gen_kwargs["diversity_penalty"] = kwargs.pop("diversity_penalty", 0.0)
+
+        # Generate tokens
+        output_tokens = self.model.generate(**inputs, **gen_kwargs)
+
+        # Decode based on architecture
+        if self.is_encoder_decoder:
+            prediction = self.tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+        else:
+            # For Causal LMs, strip the input prompt from the output
+            input_length = inputs["input_ids"].shape[1]
+            prediction = self.tokenizer.decode(
+                output_tokens[0][input_length:], skip_special_tokens=True
+            )
+
         return TranslationOutput(translation_text=prediction)
 
     def __call__(self, text: str, *args, **kwargs) -> TranslationOutput:
         """Alias of the 'predict' method"""
-        return self.predict(text=text, **kwargs)
+        return self.predict(text=text, *args, **kwargs)
 
 
 class PretrainedModelForWinoBias(ModelAPI):

langtest/pipelines/transformers/ner_pipeline.py

@@ -126,7 +126,7 @@ def train(self):
             model=self.model,
             args=TrainingArguments(output_dir=self.output_dir, **self.training_args),
             train_dataset=self.train_dataset,
-            tokenizer=self.tokenizer,
+            processing_class=self.tokenizer,
         )
         trainer.train()
         self.model.save_pretrained(self.output_dir)
@@ -215,7 +215,7 @@ def retrain(self):
             ),
             train_dataset=self.augmented_train_dataset,
             eval_dataset=self.eval_dataset,
-            tokenizer=self.tokenizer,
+            processing_class=self.tokenizer,
         )
         trainer.train()
         self.model.save_pretrained(f"augmented_{self.output_dir}")

langtest/prompts.py

@@ -32,7 +32,8 @@ def get_template(self):
         order_less = []
 
         sorted_fields = sorted(
-            self.__dict__.keys(), key=lambda x: self.__field_order.index(x.lower())
+            (k for k in self.__dict__.keys() if k.lower() in self.__field_order),
+            key=lambda x: self.__field_order.index(x.lower()),
         )
 
         for field in sorted_fields:
@@ -126,7 +127,7 @@ def get_input_variables(self):
     def prompt_style(self):
         """Generate a prompt based on the prompt type."""
         if self.prompt_type in ["chat", "instruct"]:
-            from langchain.prompts import (
+            from langchain_core.prompts import (
                 ChatPromptTemplate,
                 FewShotChatMessagePromptTemplate,
             )
@@ -149,7 +150,7 @@ def prompt_style(self):
             return final_prompt
 
         elif self.prompt_type == "completion":
-            from langchain.prompts import FewShotPromptTemplate, PromptTemplate
+            from langchain_core.prompts import FewShotPromptTemplate, PromptTemplate
 
             template = "".join(v for _, v in self.get_template)
             template = f"{template.replace('Answer:', '')}"

langtest/transform/constants.py

@@ -146203,10 +146203,10 @@
 bad_word_list= ['2g1c', '2 girls 1 cup', 'acrotomophilia', 'alabama hot pocket', 'alaskan pipeline', 'anal', 'anilingus', 'anus', 'apeshit', 'arsehole', 'ass', 'asshole', 'assmunch', 'auto erotic', 'autoerotic', 'babeland', 'baby batter', 'baby juice', 'ball gag', 'ball gravy', 'ball kicking', 'ball licking', 'ball sack', 'ball sucking', 'bangbros', 'bangbus', 'bareback', 'barely legal', 'barenaked', 'bastard', 'bastardo', 'bastinado', 'bbw', 'bdsm', 'beaner', 'beaners', 'beaver cleaver', 'beaver lips', 'beastiality', 'bestiality', 'big black', 'big breasts', 'big knockers', 'big tits', 'bimbos', 'birdlock', 'bitch', 'bitches', 'black cock', 'blonde action', 'blonde on blonde action', 'blowjob', 'blow job', 'blow your load', 'blue waffle', 'blumpkin', 'bollocks', 'bondage', 'boner', 'boob', 'boobs', 'booty call', 'brown showers', 'brunette action', 'bukkake', 'bulldyke', 'bullet vibe', 'bullshit', 'bung hole', 'bunghole', 'busty', 'butt', 'buttcheeks', 'butthole', 'camel toe', 'camgirl', 'camslut', 'camwhore', 'carpet muncher', 'carpetmuncher', 'chocolate rosebuds', 'cialis', 'circlejerk', 'cleveland steamer', 'clit', 'clitoris', 'clover clamps', 'clusterfuck', 'cock', 'cocks', 'coprolagnia', 'coprophilia', 'cornhole', 'coon', 'coons', 'creampie', 'cum', 'cumming', 'cumshot', 'cumshots', 'cunnilingus', 'cunt', 'darkie', 'date rape', 'daterape', 'deep throat', 'deepthroat', 'dendrophilia', 'dick', 'dildo', 'dingleberry', 'dingleberries', 'dirty pillows', 'dirty sanchez', 'doggie style', 'doggiestyle', 'doggy style', 'doggystyle', 'dog style', 'dolcett', 'domination', 'dominatrix', 'dommes', 'donkey punch', 'double dong', 'double penetration', 'dp action', 'dry hump', 'dvda', 'eat my ass', 'ecchi', 'ejaculation', 'erotic', 'erotism', 'escort', 'eunuch', 'fag', 'faggot', 'fecal', 'felch', 'fellatio', 'feltch', 'female squirting', 'femdom', 'figging', 'fingerbang', 'fingering', 'fisting', 'foot fetish', 'footjob', 'frotting', 'fuck', 'fuck buttons', 'fuckin', 'fucking', 'fucktards', 'fudge packer', 'fudgepacker', 'futanari', 'gangbang', 'gang bang', 'gay sex', 'genitals', 'giant cock', 'girl on', 'girl on top', 'girls gone wild', 'goatcx', 'goatse', 'god damn', 'gokkun', 'golden shower', 'goodpoop', 'goo girl', 'goregasm', 'grope', 'group sex', 'g-spot', 'guro', 'hand job', 'handjob', 'hard core', 'hardcore', 'hentai', 'homoerotic', 'honkey', 'hooker', 'horny', 'hot carl', 'hot chick', 'how to kill', 'how to murder', 'huge fat', 'humping', 'incest', 'intercourse', 'jack off', 'jail bait', 'jailbait', 'jelly donut', 'jerk off', 'jigaboo', 'jiggaboo', 'jiggerboo', 'jizz', 'juggs', 'kike', 'kinbaku', 'kinkster', 'kinky', 'knobbing', 'leather restraint', 'leather straight jacket', 'lemon party', 'livesex', 'lolita', 'lovemaking', 'make me come', 'male squirting', 'masturbate', 'masturbating', 'masturbation', 'menage a trois', 'milf', 'missionary position', 'mong', 'motherfucker', 'mound of venus', 'mr hands', 'muff diver', 'muffdiving', 'nambla', 'nawashi', 'negro', 'neonazi', 'nigga', 'nigger', 'nig nog', 'nimphomania', 'nipple', 'nipples', 'nsfw', 'nsfw images', 'nude', 'nudity', 'nutten', 'nympho', 'nymphomania', 'octopussy', 'omorashi', 'one cup two girls', 'one guy one jar', 'orgasm', 'orgy', 'paedophile', 'paki', 'panties', 'panty', 'pedobear', 'pedophile', 'pegging', 'penis', 'phone sex', 'piece of shit', 'pikey', 'pissing', 'piss pig', 'pisspig', 'playboy', 'pleasure chest', 'pole smoker', 'ponyplay', 'poof', 'poon', 'poontang', 'punany', 'poop chute', 'poopchute', 'porn', 'porno', 'pornography', 'prince albert piercing', 'pthc', 'pubes', 'pussy', 'queaf', 'queef', 'quim', 'raghead', 'raging boner', 'rape', 'raping', 'rapist', 'rectum', 'reverse cowgirl', 'rimjob', 'rimming', 'rosy palm', 'rosy palm and her 5 sisters', 'rusty trombone', 'sadism', 'santorum', 'scat', 'schlong', 'scissoring', 'semen', 'sex', 'sexcam', 'sexo', 'sexy', 'sexual', 'sexually', 'sexuality', 'shaved beaver', 'shaved pussy', 'shemale', 'shibari', 'shit', 'shitblimp', 'shitty', 'shota', 'shrimping', 'skeet', 'slanteye', 'slut', 's&m', 'smut', 'snatch', 'snowballing', 'sodomize', 'sodomy', 'spastic', 'spic', 'splooge', 'splooge moose', 'spooge', 'spread legs', 'spunk', 'strap on', 'strapon', 'strappado', 'strip club', 'style doggy', 'suck', 'sucks', 'suicide girls', 'sultry women', 'swastika', 'swinger', 'tainted love', 'taste my', 'tea bagging', 'threesome', 'throating', 'thumbzilla', 'tied up', 'tight white', 'tit', 'tits', 'titties', 'titty', 'tongue in a', 'topless', 'tosser', 'towelhead', 'tranny', 'tribadism', 'tub girl', 'tubgirl', 'tushy', 'twat', 'twink', 'twinkie', 'two girls one cup', 'undressing', 'upskirt', 'urethra play', 'urophilia', 'vagina', 'venus mound', 'viagra', 'vibrator', 'violet wand', 'vorarephilia', 'voyeur', 'voyeurweb', 'voyuer', 'vulva', 'wank', 'wetback', 'wet dream', 'white power', 'whore', 'worldsex', 'wrapping men', 'wrinkled starfish', 'xx', 'xxx', 'yaoi', 'yellow showers', 'yiffy', 'zoophilia', '🖕']
 
 DATASETS = {
-    'sst2': {
+    'SetFit/sst2': {
         'subsets': [''],
         'labels': [['Negative Sentiment', 'Positive Sentiment']],
-        'fields': [['sentence']],
+        'fields': [['text']],
         'label_names': ['label'],
     },
     'rotten_tomatoes': {

langtest/transform/grammar.py

@@ -177,21 +177,18 @@ def transform(
         sample_list: List[Sample], prob: Optional[float] = 1.0, *args, **kwargs
     ):
         if try_import_lib("transformers"):
-            from transformers import pipeline
+            from langtest.utils.hf_utils import paraphrase
 
-            pipe = pipeline(
-                "text2text-generation", model="humarin/chatgpt_paraphraser_on_T5_base"
-            )
             for idx, sample in enumerate(sample_list):
                 if isinstance(sample, str):
-                    test_case = pipe(sample, max_length=11000, num_return_sequences=1)[0][
-                        "generated_text"
-                    ]
+                    test_case = paraphrase(
+                        sample, max_length=11000, num_return_sequences=1
+                    )[0]
                     sample_list[idx] = test_case
                 else:
-                    test_case = pipe(
+                    test_case = paraphrase(
                         sample.original, max_length=1000, num_return_sequences=1
-                    )[0]["generated_text"]
+                    )[0]
                     sample.test_case = test_case
                     sample.category = "grammar"
 

langtest/transform/utils.py

@@ -562,7 +562,10 @@ def get_criteria_scores(self, case_id, question_id, section_id=None):
         df = self.filter(self.criteria, case_id=case_id, question_id=question_id)
         if section_id:
             df = df[df["section_id"] == section_id]
-        return [float(score.replace(",", ".")) for score in df["criteria_score_possible"]]
+        return [
+            float(score.replace(",", ".")) if isinstance(score, str) else score
+            for score in df["criteria_score_possible"]
+        ]
 
     @lru_cache(maxsize=4)
     def load_csv(self, filename) -> pd.DataFrame:
@@ -572,7 +575,7 @@ def load_csv(self, filename) -> pd.DataFrame:
         )
         try:
             # save the csv file into `~/.langtest/` directory
-            location_path = os.path.expanduser(f"~/.langtest/amgea/{filename}")
+            location_path = os.path.expanduser(f"~/.langtest/amega/{filename}")
             os.makedirs(os.path.dirname(location_path), exist_ok=True)
             if not os.path.exists(location_path):
                 import requests
@@ -582,7 +585,7 @@ def load_csv(self, filename) -> pd.DataFrame:
                 with open(location_path, "wb") as f:
                     f.write(response.content)
 
-            return pd.read_csv(location_path, delimiter=";")
+            return pd.read_csv(location_path, delimiter=",")
         except (FileNotFoundError, pd.errors.ParserError) as e:
             print(f"Error loading {filename}: {e}")
             return pd.DataFrame()
@@ -603,7 +606,7 @@ def dataset_info(self) -> pd.DataFrame:
         for case_id in cases_ids:
 
             case_branch = self.cases[self.cases["case_id"] == case_id][
-                "case_brunch"
+                "case_branch"
             ].values[0]
             case_title = self.cases[self.cases["case_id"] == case_id][
                 "case_title"
@@ -937,15 +940,15 @@ def aggregate_results(self, benchmark_data, evaluation_results):
         """
         # Get case info
         case_data = benchmark_data.get_case_data(case_id=self.case_id)[0]
-        case_brunch = case_data["case_brunch"]
+        case_branch = case_data["case_branch"]
 
         # Prepare base aggregated data
         aggregated_data = {
             "generator_model_name_or_path": self.generator_model_name_or_path,
             "generator_type": self.generator_type,
             "evaluator_model_name_or_path": self.model,
             "case_id": self.case_id,
-            "case_brunch": case_brunch,
+            "case_branch": case_branch,
         }
 
         # Initialize per-question fields
@@ -976,7 +979,7 @@ def aggregate_results(self, benchmark_data, evaluation_results):
             results_list.append(
                 [
                     aggregated_data["case_id"],
-                    aggregated_data["case_brunch"],
+                    aggregated_data["case_branch"],
                     q_id,
                     question_possible,
                     question_initial,
@@ -1001,7 +1004,7 @@ def aggregate_results(self, benchmark_data, evaluation_results):
         results_list.append(
             [
                 self.case_id,
-                aggregated_data["case_brunch"],
+                aggregated_data["case_branch"],
                 "total",
                 aggregated_data["case_possible_score"],
                 aggregated_data["case_initial_score"],

langtest/utils/custom_types/helpers.py

@@ -425,8 +425,8 @@ def llm_prompt_eval(
         bool: True if the model's prediction passes the LLM evaluation, False otherwise.
 
     """
-    from langchain.evaluation.qa import QAEvalChain
-    from langchain.prompts import PromptTemplate
+    from langchain_classic.evaluation.qa import QAEvalChain
+    from langchain_core.prompts import PromptTemplate
 
     PROMPT = PromptTemplate(
         input_variables=["query", "answer", "result"],