Merge pull request #9 from thoraxe/extend-question-validator

enhances the question validator

Author: thoraxe

modules/question_validator.py

@@ -54,24 +54,32 @@ def validate_question(self, conversation, query, **kwargs):
 - You are a question classifying tool
 - You are an expert in kubernetes and openshift
 - Your job is to determine if a question is about kubernetes or openshift and to provide a one word response
-- If a question is not about kubernetes or openshift, answer with only the word "INVALID"
-- If a question is about kubernetes or openshift, answer with only the word "VALID"
-- Do not provide explanation, only respond with the single chosen word
+- If a question is not about kubernetes or openshift, answer with only the word INVALID
+- If a question is about kubernetes or openshift, answer with the word VALID
+- If a question is not about creating kubernetes or openshift yaml, answer with the word NOYAML
+- If a question is about creating kubernetes or openshift yaml, add the word YAML
+- Use a comma to separate the words
+- Do not provide explanation, only respond with the chosen words
 
 Example Question:
 Can you make me lunch with ham and cheese?
 Example Response:
-INVALID
+INVALID,NOYAML
 
 Example Question:
 Why is the sky blue?
 Example Response:
-INVALID
+INVALID,NOYAML
 
 Example Question:
 Can you help configure my cluster to automatically scale?
 Example Response:
-VALID
+VALID,NOYAML
+
+Example Question:
+please give me a vertical pod autoscaler configuration to manage my frontend deployment automatically.  Don't update the workload if there are less than 2 pods running.
+Example Response:
+VALID,YAML
 
 Question:
 {query}
@@ -82,19 +90,23 @@ def validate_question(self, conversation, query, **kwargs):
         self.logger.info(conversation + " Validating query")
         self.logger.info(conversation + " usng model: " + model)
 
-        bare_llm = get_watsonx_predictor(model=model, min_new_tokens=1, max_new_tokens=2)
+        bare_llm = get_watsonx_predictor(model=model, min_new_tokens=1, max_new_tokens=4)
         llm_chain = LLMChain(llm=bare_llm, prompt=prompt_instructions, verbose=verbose)
 
         task_query = prompt_instructions.format(query=query)
 
         self.logger.info(conversation + " task query: " + task_query)
 
         response = llm_chain(inputs={"query": query})
+        clean_response = str(response['text']).strip()
 
-        self.logger.info(conversation + " response: " + str(response))
+        self.logger.info(conversation + " response: " + clean_response)
 
-        # should only return "VALID" or "INVALID"
-        return response['text'].strip()
+        # will return an array:
+        # [INVALID,NOYAML]
+        # [VALID,NOYAML]
+        # [VALID,YAML]
+        return clean_response.split(",")
 
 if __name__ == "__main__":
     """to execute, from the repo root, use python -m modules.question_validator.py"""

ols.py

@@ -120,25 +120,50 @@ def ols2_request(llm_request: LLMRequest):
     question_validator = QuestionValidator()
 
     is_valid = question_validator.validate_question(conversation, llm_request.query)
-    if is_valid == "INVALID":
+    if is_valid[0] == "INVALID":
+        logging.info(conversation + " question was determined to not be k8s/ocp, so rejecting")
         llm_response.response = ("Sorry, I can only answer questions about "
                                  "OpenShift and Kubernetes. This does not look "
                                  "like something I know how to handle.")
         raise HTTPException(status_code=422, detail=llm_response.dict())
-    if is_valid == "VALID":
-        # the LLM thought the question was valid, so pass it to the YAML generator
-        yaml_generator = YamlGenerator()
-        generated_yaml = yaml_generator.generate_yaml(conversation, llm_request.query)
-
-        # TODO: raise an exception on a failure of the yaml generator
-
-        # filter/clean/lint the YAML response
-
-        # RAG for supporting documentation
-
-        # generate a user-friendly response to wrap the YAML and/or the supporting information
-        llm_response.response = generated_yaml
-        return llm_response
+    if is_valid[0] == "VALID":
+        logging.info(conversation + " question is about k8s/ocp")
+        # the LLM thought the question was valid, so decide if it's about YAML or not
+
+        if is_valid[1] == "NOYAML":
+            logging.info(conversation + " question is not about yaml, so send for generic info")
+            llm_response.response = "Documentation-based response here"
+            return llm_response
+        elif is_valid[1] == "YAML":
+            logging.info(conversation + " question is about yaml, so send to the YAML generator")
+            yaml_generator = YamlGenerator()
+            generated_yaml = yaml_generator.generate_yaml(conversation, llm_request.query)
+
+            if generated_yaml == "some failure":
+                # we didn't get any kind of yaml markdown block back from the model
+                llm_response.response = (
+                    "Sorry, something bad happened internally. Please try again."
+                )
+                raise HTTPException(status_code=500, detail=llm_response.dict())
+
+            # we got some kind of valid yaml back from the yaml generator, so proceed
+
+            # filter/clean/lint the YAML response
+
+            # RAG for supporting documentation
+
+            # generate a user-friendly response to wrap the YAML and/or the supporting information
+            llm_response.response = generated_yaml
+
+
+            return llm_response
+        else:
+            # something weird happened, so generate an internal error
+            # something bad happened with the validation
+            llm_response.response = (
+                "Sorry, something bad happened internally. Please try again."
+            )
+        raise HTTPException(status_code=500, detail=llm_response.dict())
     else:
         # something bad happened with the validation
         llm_response.response = (