Auto-Precise-Prompts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright &copy; 2024 Praneeth Vadlapati"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from common_functions import get_lm_response, extract_data, print_error, \\\n",
    "\t\t\t\t\t\t\t\tprint_progress, model_small, backticks\n",
    "\n",
    "processing_criteria = 'Provide name and age of people whose age is below 35.'\n",
    "initial_prompt_template = 'Here is the input data: {structured_input_data}.\\n' + processing_criteria\n",
    "data_format = 'csv'  # 'csv' or 'json'\n",
    "\n",
    "structured_input_data = ''' ```csv\n",
    "Name,Gender,Age,City\n",
    "John,Male,25,NYC\n",
    "Jane,Female,30,LA\n",
    "Doe,Male,38,Chicago\n",
    "Emily,Female,48,Houston\n",
    "Henry,Male,66,Philadelphia\n",
    "``` '''.strip()\n",
    "expected_response = ''' ```csv\n",
    "Name,Age\n",
    "John,25\n",
    "Jane,30\n",
    "``` '''.strip()\n",
    "example_response = ''' ```csv\n",
    "Name,Age\n",
    "Alice,22\n",
    "Bob,30\n",
    "``` '''.strip()\n",
    "\n",
    "structured_input_data_json = ''' ```json\n",
    "[\n",
    "\t{\"Name\": \"John\", \"Gender\": \"Male\", \"Age\": 25, \"City\": \"NYC\"},\n",
    "\t{\"Name\": \"Jane\", \"Gender\": \"Female\", \"Age\": 30, \"City\": \"LA\"},\n",
    "\t{\"Name\": \"Doe\", \"Gender\": \"Male\", \"Age\": 38, \"City\": \"Chicago\"},\n",
    "\t{\"Name\": \"Emily\", \"Gender\": \"Female\", \"Age\": 48, \"City\": \"Houston\"},\n",
    "\t{\"Name\": \"Henry\", \"Gender\": \"Male\", \"Age\": 66, \"City\": \"Philadelphia\"}\n",
    "]\n",
    "``` '''.strip()\n",
    "expected_response_json = ''' ```json\n",
    "[\n",
    "\t{\"Name\": \"John\", \"Age\": 25},\n",
    "\t{\"Name\": \"Jane\", \"Age\": 30}\n",
    "]\n",
    "``` '''.strip()\n",
    "example_response_json = ''' ```json\n",
    "[\n",
    "\t{\"Name\": \"Alice\", \"Age\": 22},\n",
    "\t{\"Name\": \"Bob\", \"Age\": 30}\n",
    "]\n",
    "``` '''.strip()\n",
    "\n",
    "if data_format == 'json':\n",
    "\tstructured_input_data = structured_input_data_json\n",
    "\texpected_response = expected_response_json\n",
    "\texample_response = example_response_json\n",
    "\n",
    "\n",
    "initial_prompt = initial_prompt_template.format(\n",
    "    structured_input_data=structured_input_data, example_response=example_response)\n",
    "\n",
    "expected_data = extract_data(expected_response, data_format)\n",
    "response_length_limit = int(len(expected_response) * 10)\n",
    "\n",
    "def shorten_response(current_response):\n",
    "\tif current_response and len(current_response) > response_length_limit:\n",
    "\t\tcurrent_response = current_response[:response_length_limit] + '...'\n",
    "\treturn current_response\n",
    "\n",
    "current_template_file = f'{data_format} prompt_template_optimized.txt'\n",
    "current_prompt_template = None\n",
    "if os.path.exists(current_template_file):\n",
    "\twith open(current_template_file) as file:\n",
    "\t\tcurrent_prompt_template = file.read().strip()\n",
    "\n",
    "prompt_loaded = True if current_prompt_template else False\n",
    "\n",
    "if not current_prompt_template:\n",
    "\tcurrent_prompt_template = str(initial_prompt_template)  # copy initial string\n",
    "current_prompt = current_prompt_template.format(\n",
    "    structured_input_data=structured_input_data, example_response=example_response)\n",
    "\n",
    "shortened_prompt_template = None\n",
    "shortened_template_file = f'{data_format} prompt_template_shortened.txt'\n",
    "if os.path.exists(shortened_template_file):\n",
    "\twith open(shortened_template_file) as f:\n",
    "\t\tshortened_prompt_template = f.read().strip()\n",
    "\n",
    "loaded_short_prompt = True if shortened_prompt_template else False\n",
    "\n",
    "shortened_prompt = shortened_prompt_template.format(\n",
    "\tstructured_input_data=structured_input_data, example_response=example_response) \\\n",
    "\tif shortened_prompt_template else None\n",
    "\n",
    "print(f'Model: {model_small}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompteng_log_file = 'test_prompt_eng.log'\n",
    "with open(prompteng_log_file, 'w') as f:\n",
    "\tf.write('')\n",
    "\n",
    "trial_log_file = 'trial.log'\n",
    "with open(trial_log_file, 'w') as f:\n",
    "\tf.write('')\n",
    "\n",
    "trial_log_file2 = trial_log_file + '2'\n",
    "with open(trial_log_file2, 'w') as f:\n",
    "\tf.write('')\n",
    "\n",
    "last_wrong_response = None\n",
    "\n",
    "def find_accuracy(current_prompt, model=None, no_logs=False, trial_mode=False, total_trials=7):\n",
    "\tglobal last_wrong_response, current_response\n",
    "\tcorrect_responses = 0\n",
    "\n",
    "\tif not no_logs:\n",
    "\t\tprint(' Finding accuracy', end='', flush=True)\n",
    "\t\twith open(trial_log_file, 'a') as f:\n",
    "\t\t\tf.write(f'\\n\\n' + '='*40 + f' Checking {total_trials} times ' + '='*40 + '\\n\\n')\n",
    "\n",
    "\tfor trial_attempt_num in range(total_trials):\n",
    "\t\tcurrent_response = get_lm_response(current_prompt, model=model)\n",
    "\t\ttry:\n",
    "\t\t\tif extract_data(current_response, data_format) == expected_data:\n",
    "\t\t\t\tcorrect_responses += 1\n",
    "\t\t\t\twith open(trial_log_file2, 'a') as f:\n",
    "\t\t\t\t\tf.write(f'Response: {current_response}\\n\\n')\n",
    "\t\t\t\t\tf.write('_'*120 + '\\n')\n",
    "\t\t\telse:\n",
    "\t\t\t\tlast_wrong_response = current_response\n",
    "\t\t\t\tif not no_logs:\n",
    "\t\t\t\t\twith open(trial_log_file, 'a') as f:\n",
    "\t\t\t\t\t\tf.write(f'Response: {current_response}\\n\\n')\n",
    "\t\t\t\t\t\tf.write('_'*120 + '\\n')\n",
    "\t\t\tif not no_logs:\n",
    "\t\t\t\tprint_progress()\n",
    "\t\texcept Exception as e:\n",
    "\t\t\tif not no_logs:\n",
    "\t\t\t\twith open(trial_log_file, 'a') as f:\n",
    "\t\t\t\t\tf.write(f'Error: {e}\\n\\n')\n",
    "\t\t\t\t\tf.write('_'*120 + '\\n')\n",
    "\t\t\tprint_error(e)\n",
    "\t\tif trial_mode and not correct_responses and trial_attempt_num >= (0.4*total_trials):\n",
    "\t\t\tbreak\n",
    "\n",
    "\tif not no_logs:\n",
    "\t\tprint()  # because end='' was used in the last print statement\n",
    "\treturn correct_responses / total_trials  # trial_accuracy\n",
    "\n",
    "print('Setup done.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimize the prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attempt_num = 0\n",
    "prompt_eng_success = False\n",
    "\n",
    "attempts_limit = 4\n",
    "trial_cutoff_score = 0.8\n",
    "\n",
    "trial_accuracy = None\n",
    "initial_accuracy = None\n",
    "\n",
    "prompt_eng_template = '''\n",
    "Value of structured_input_data variable (includes backticks):\n",
    "{structured_input_data}\n",
    "\\n --- \n",
    "Processing criteria: {processing_criteria}\n",
    "\\n --- \n",
    "Current prompt template: {current_prompt_template}\n",
    "\\n --- \n",
    "Accuracy of current prompt: {trial_accuracy}\n",
    "Current response: {current_response}\n",
    "\\n --- \n",
    "Expected response: {expected_response}\n",
    "\\n --- \n",
    "\n",
    "Act as a Prompt Engineer and an expert Linguist. Write prompt to process structured data using language model.\n",
    "Rewrite prompt template to generate expected response (including its special characters and backticks).\n",
    "Use example_response placeholder to indicate a sample response. Don't add your own sample in the template.\n",
    "\n",
    "Don't include answer or expected response.\n",
    "CSV responses must include expected column names without extra columns.\n",
    "Write only new prompt template without any other text.\n",
    "At last, emphasize on the processing criteria I mentioned.\n",
    "Backticks and format must be exactly same as the example response.\n",
    "\n",
    "Avoid backticks like ```.\n",
    "Mention \"[structured_input_data]\" placeholder to indicate input data.\n",
    "Make sure response includes backticks.\n",
    "Don't miss both placeholders mentioned in curly braces.\n",
    "Add \"**Write like**: [example_response]\" as placeholder to mention example response.\n",
    "'''.strip()\n",
    "\n",
    "def optimize_prompt():\n",
    "\tglobal current_prompt_template, current_prompt, trial_accuracy, attempt_num, \\\n",
    "\t\tcurrent_response, prompt_eng_success, initial_accuracy\n",
    "\tif prompt_loaded:\n",
    "\t\tprompt_eng_success = True\n",
    "\t\tprint(f'\\nCurrent prompt template was already loaded. New prompt is not required.')\n",
    "\t\ttrial_accuracy = trial_cutoff_score\n",
    "\t\treturn  # temporarily avoid running\n",
    "\n",
    "\twhile True:\n",
    "\t\t# -------------- Find accuracy of the current prompt template --------------\n",
    "\t\ttrial_accuracy = find_accuracy(current_prompt)\n",
    "\t\tif current_prompt_template == initial_prompt_template:\n",
    "\t\t\tinitial_accuracy = trial_accuracy\n",
    "\t\tprint(f'Trial accuracy: {trial_accuracy}')\n",
    "\t\tif trial_accuracy >= trial_cutoff_score:\n",
    "\t\t\tprint(f'Trial accuracy: {trial_accuracy} (Good).')\n",
    "\t\t\tprompt_eng_success = True\n",
    "\t\telse:\n",
    "\t\t\tprint(f'Trial accuracy: {trial_accuracy} (Failed).')\n",
    "\t\t\tcurrent_response = last_wrong_response\n",
    "\t\t\tif attempt_num > attempts_limit:\n",
    "\t\t\t\tprint(f'Prompt Optimization failed after {attempt_num} attempts. Exiting.')\n",
    "\t\t\t\tbreak\n",
    "\t\t\tif attempt_num:  # non-zero\n",
    "\t\t\t\tprint(f'Failed attempt {attempt_num}')\n",
    "\n",
    "\t\t# -------------- Save current prompt template --------------\n",
    "\t\tif prompt_eng_success:\n",
    "\t\t\twith open(current_template_file, 'w') as f:\n",
    "\t\t\t\tf.write(current_prompt_template)\n",
    "\t\t\tif attempt_num <= 2:  # first few attempts\n",
    "\t\t\t\tif trial_accuracy == 1:  # stop if accuracy 100%\n",
    "\t\t\t\t\tprint(f'Accuracy is {trial_accuracy} (BEST). Exiting.')\n",
    "\t\t\t\t\tbreak\n",
    "\t\t\t\t# if prompt success but <100% accuracy at first attempt, proceed to improve the prompt\n",
    "\t\t\telse:  # in next attempts, stop if accuracy is good\n",
    "\t\t\t\tbreak\n",
    "\t\telse:\n",
    "\t\t\tattempt_num += 1\n",
    "\t\t\twith open(prompteng_log_file, 'a') as f:\n",
    "\t\t\t\tf.write(f'Prompt: {current_prompt_template}\\n\\n')\n",
    "\t\t\t\tf.write('-'*80 + '\\n')\n",
    "\t\t\t\tf.write(f'Last Wrong Response: {shorten_response(current_response)}\\n\\n')\n",
    "\t\t\t\tf.write('_'*120 + '\\n')\n",
    "\n",
    "\t\t# -------------- Change prompt using LLM --------------\n",
    "\t\tprompteng_attempts_limit = 3\n",
    "\t\tprompt_eng_attempt = 0\n",
    "\t\twhile prompt_eng_attempt < prompteng_attempts_limit:\n",
    "\t\t\tprompt_eng_attempt += 1\n",
    "\t\t\tprompteng_prompt = prompt_eng_template.format(\n",
    "\t\t\t\tstructured_input_data=structured_input_data,\n",
    "\t\t\t\texample_response=example_response,\n",
    "\t\t\t\tprocessing_criteria=processing_criteria,\n",
    "\t\t\t\tcurrent_prompt_template=current_prompt_template,\n",
    "\t\t\t\tcurrent_response=current_response,\n",
    "\t\t\t\texpected_response=expected_response,\n",
    "\t\t\t\ttrial_accuracy=int(trial_accuracy * 100),\n",
    "\t\t\t).replace('[', '{').replace(']', '}')\n",
    "\t\t\tprint(f'Calling LLM for prompt engineering attempt {prompt_eng_attempt}...')\n",
    "\t\t\tcurrent_prompt_template = get_lm_response(prompteng_prompt, use_large_model=True)\n",
    "\t\t\t# if prompt has more than 250 words, try again\n",
    "\t\t\tif len(current_prompt_template.split()) > 500:\n",
    "\t\t\t\tprint('Prompt is too long. Trying again.')\n",
    "\t\t\t\tcurrent_prompt_template = None\n",
    "\t\t\t\tcontinue\n",
    "\t\t\ttry:\n",
    "\t\t\t\t# -------------- Test the new prompt --------------\n",
    "\t\t\t\tif 'structured_input_data' not in current_prompt_template or \\\n",
    "\t\t\t\t\t\t'example_response' not in current_prompt_template:\n",
    "\t\t\t\t\tprint('Prompt template has missing placeholders. Trying again.')\n",
    "\t\t\t\t\tprompt_eng_attempt -= 1\n",
    "\t\t\t\t\tcontinue\n",
    "\t\t\t\tif backticks in current_prompt_template:\n",
    "\t\t\t\t\tprint('Prompt template has backticks. Trying again.')\n",
    "\t\t\t\t\tprompt_eng_attempt -= 1\n",
    "\t\t\t\t\tcontinue\n",
    "\t\t\t\tcurrent_prompt = current_prompt_template.format(\n",
    "\t\t\t\t\tstructured_input_data=structured_input_data, example_response=example_response)\n",
    "\t\t\t\tbreak  # prompt creation successful\n",
    "\t\t\texcept:\n",
    "\t\t\t\tprint_error()\n",
    "\t\t\t\tcontinue\n",
    "\t\tif prompt_eng_attempt >= prompteng_attempts_limit:\n",
    "\t\t\tprint('Prompt engineering failed. Exiting.')\n",
    "\t\t\tbreak\n",
    "\n",
    "optimize_prompt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(current_prompt_template)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shorten the prompt, maintaining the accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shorten_prompt_using_function(shortener_function, max_attempts=5):\n",
    "\tglobal current_prompt_template, trial_accuracy, shortened_prompt_template, shortened_prompt\n",
    "\tif loaded_short_prompt:\n",
    "\t\tprint('Shortened prompt template is already loaded from file. Exiting.')\n",
    "\t\treturn\n",
    "\tif not prompt_eng_success:\n",
    "\t\tif not trial_accuracy:  # Trial conducted and failed\n",
    "\t\t\tprint('Prompt engineering failed. Could not send the prompt for shortening.')\n",
    "\t\treturn\n",
    "\n",
    "\tshorten_attempt_num = 0\n",
    "\twhile shorten_attempt_num < max_attempts:\n",
    "\t\tshorten_attempt_num += 1\n",
    "\t\tshortened_prompt_template = shortener_function(str(current_prompt_template))  # pass a copy\n",
    "\t\tif len(shortened_prompt_template) > len(current_prompt_template):\n",
    "\t\t\tprint('Prompt became longer. Retrying.')\n",
    "\t\t\tcontinue\n",
    "\t\ttry:\n",
    "\t\t\tif 'structured_input_data' not in shortened_prompt_template or \\\n",
    "\t\t\t\t\t'example_response' not in shortened_prompt_template:\n",
    "\t\t\t\tprint('Shortened prompt template has missing placeholders. Retrying.')\n",
    "\t\t\t\tcontinue\n",
    "\t\t\tshortened_prompt = shortened_prompt_template.format(\n",
    "       \t\t\tstructured_input_data=structured_input_data, example_response=example_response)\n",
    "\t\texcept:\n",
    "\t\t\tprint('Shortened prompt failed to format. Retrying.')\n",
    "\t\t\tcontinue\n",
    "\n",
    "\t\tshort_trial_accuracy = find_accuracy(shortened_prompt, trial_mode=True)\n",
    "\t\tif short_trial_accuracy < trial_accuracy:\n",
    "\t\t\tprint(f'Shortened prompt accuracy is {short_trial_accuracy}, not improved. Retrying...')\n",
    "\t\t\tcontinue\n",
    "\t\telse:\n",
    "\t\t\tprint(f'Shortened prompt accuracy: {short_trial_accuracy}')\n",
    "\n",
    "\t\told_length = len(current_prompt_template)\n",
    "\t\treduction_percent = ((old_length - len(shortened_prompt_template)) / old_length) * 100\n",
    "\t\tif reduction_percent <= 0:\n",
    "\t\t\tprint('Length of Shortened prompt is same as original prompt.')\n",
    "\t\t\tif short_trial_accuracy <= trial_accuracy:  # same length without improvement in accuracy\n",
    "\t\t\t\tcontinue\n",
    "\t\t\tprint('\\t But it is more accurate.')  # proceed with next steps\n",
    "\n",
    "\t\tcurrent_prompt_template = shortened_prompt_template\n",
    "\t\ttrial_accuracy = short_trial_accuracy\n",
    "\t\twith open(shortened_template_file, 'w') as f:\n",
    "\t\t\tf.write(shortened_prompt_template)\n",
    "\t\t\t\n",
    "\t\tprint(f'\\n SUCCESS: Shortened prompt is {reduction_percent:.2f}% shorter. Saved prompt.')\n",
    "\t\tif reduction_percent <= 0:\n",
    "\t\t\tprint('Accuracy improved, but length not reduced. Retrying to shorten the prompt further...')\n",
    "\t\t\tshorten_attempt_num -= 1\n",
    "\t\t\tcontinue\n",
    "\n",
    "\t\tbreak  # shortened by maintaining accuracy\n",
    "\n",
    "\tif shorten_attempt_num >= max_attempts:\n",
    "\t\tprint(f'Failed to shorten the prompt after {max_attempts} attempts. Exited.')\n",
    "\n",
    "\t\tshortened_prompt_template = current_prompt_template\n",
    "\t\twith open(shortened_template_file, 'w') as f:\n",
    "\t\t\tf.write(shortened_prompt_template)\n",
    "\n",
    "\t\tshortened_prompt = shortened_prompt_template.format(\n",
    "\t\t\tstructured_input_data=structured_input_data, example_response=example_response)\n",
    "\n",
    "\n",
    "shorten_instruction = '''\n",
    "Current prompt template:\n",
    "{current_prompt_template}\n",
    "\\n --- \n",
    "Be a Prompt Engineer. Shorten the above prompt template.\n",
    "Make sure the prompt is short and concise.\n",
    "Retain the placeholders values and key information.\n",
    "Do not remove placeholders 'structured_input_data' and 'example_response'.\n",
    "Return only the shortened prompt without any other text.\n",
    "When a language model uses the prompt to generate response, backticks and format must be exactly same as the example response.\n",
    "Make sure response from the model includes the formatted data inside backticks with the format like {example_response}\n",
    "'''.strip()\n",
    "\n",
    "def shorten_prompt_LLM(template):\n",
    "\treturn get_lm_response(\n",
    "\t\tshorten_instruction.format(current_prompt_template=template,\n",
    "\t\t\t\t\t\t\t\t\texample_response=example_response),\n",
    "\t\tuse_large_model=True\n",
    "\t)\n",
    "\n",
    "shorten_prompt_using_function(shorten_prompt_LLM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing final prompt using multiple models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_to_test = []\n",
    "if model_small not in models_to_test:\n",
    "\tmodels_to_test.append(model_small)\n",
    "\n",
    "if not prompt_eng_success:\n",
    "    print('Prompt engineering failed. Could not send the prompt for final testing.')\n",
    "else:\n",
    "\tfor model in models_to_test:\n",
    "\t\tprint(f'\\nUsing model {model} (type: {data_format})...')\n",
    "\t\tmodel_trial_accuracy = find_accuracy(current_prompt, model=model, total_trials=10)\n",
    "\t\tprint(model_trial_accuracy)\n",
    "\t\tif shortened_prompt:\n",
    "\t\t\tmodel_shortened_accuracy = find_accuracy(shortened_prompt, model=model, total_trials=10) \\\n",
    "\t\t\t\t\t\t\t\t\t\tif current_prompt != shortened_prompt else model_trial_accuracy\n",
    "\t\t\tshortened_prompt_text = f'& {model_shortened_accuracy} (shortened prompt) '\n",
    "\t\t\tprint(model_shortened_accuracy)\n",
    "\t\telse:\n",
    "\t\t\tshortened_prompt_text = ''\n",
    "\t\tmodel_initial_accuracy = find_accuracy(initial_prompt, model=model, total_trials=10) \\\n",
    "\t\t\t\t\t\t\t\t\tif initial_accuracy == None or model != model_small else initial_accuracy\n",
    "\t\tprint(model_initial_accuracy)\n",
    "\t\tval = f'{model}: ({data_format}) {model_trial_accuracy} (optimized prompt) {shortened_prompt_text}& {model_initial_accuracy} (initial prompt)\\n'\n",
    "\t\twith open('results.txt', 'a') as f:\n",
    "\t\t\tf.write(val)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}