2023/QualityandSafetyforLLMApplications/L5_Passive_and_active_monitoring.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7e9fb5ff-1179-4b4d-8b25-fed249361c6f",
   "metadata": {},
   "source": [
    "# Lesson 5: Passive and active monitoring"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "414d2d9d-cad1-4548-90c0-115ebb2a51ec",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c50b87ce-17a5-4cf3-8f90-4420f31df3fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13489fee-6538-4b50-8cc2-c873f7cc293b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import whylogs as why"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41bc1e60-3db6-4059-be9d-7ef5015c6e4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fd4400b-6240-4a1a-8d45-a62d4d062f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langkit import llm_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd0cce47-ea95-4d2a-a8d6-4fa057db73cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_metrics.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b60e9758-1b61-4121-84f6-67c4bc072de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from whylogs.experimental.core.udf_schema import register_dataset_udf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5885749-6f72-4200-9e20-0c3a8f303270",
   "metadata": {},
   "outputs": [],
   "source": [
    "from whylogs.experimental.core.udf_schema import udf_schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e3f81e-3ca1-4fad-9df3-6eae571543a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_schema = udf_schema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6275cf6-d3d7-4ba1-85b0-d0ba5da344f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_logger = why.logger(schema=udf_schema())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63fa2886-c738-49a8-9d1e-d604d41d1eeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_logger = why.logger(\n",
    "    model = \"rolling\",\n",
    "    interval = 1,\n",
    "    when = \"H\",\n",
    "    schema = udf_schema()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70ad41b3",
   "metadata": {},
   "source": [
    "**Note**: To accessthe WhyLabs platform that was built in the previous lessons: https://hub.whylabsapp.com/resources/model-1/profiles?profile=ref-EBT5yDFL0lyq0r93&sessionToken=session-pPdc5R9m"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4c366d6-dfe9-4ff0-8e8d-d4d5e167a5ce",
   "metadata": {},
   "source": [
    "## Building your own active monitoring guardrails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08e675d9-05af-47ed-b852-0e652ca03030",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d9fce34-626b-41c7-a875-971b6b045106",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = helpers.get_openai_key()\n",
    "openai.base_url = helpers.get_openai_base_url()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9fe03da-4065-40f3-9195-5321086ae7d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_llm_logger = why.logger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0290cec1-7961-45dc-8178-56541d1b1047",
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_request():\n",
    "    # Take request\n",
    "    request = input(\"\\nEnter your desired item to make a recipe\" \\\n",
    "                    \"(or 'quit'):\")\n",
    "    if request.lower() == \"quit\":\n",
    "        raise KeyboardInterrupt()\n",
    "        \n",
    "    # Log request\n",
    "    active_llm_logger.log({\"request\": request})\n",
    "\n",
    "    return request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33832180-d4e7-4167-a714-b9253c0747e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prompt_llm(request):\n",
    "    # Transform prompt\n",
    "    prompt = f\"\"\"Please give me a short recipe for creating\"\\\n",
    "    the following item in up to 6 steps. Each step of the recipe \"\\\n",
    "    should be summarized in no more than 200 characters.\"\\\n",
    "    Item: {request}\"\"\"\n",
    "\n",
    "    # Log prompt\n",
    "    active_llm_logger.log({\"prompt\": prompt})\n",
    "\n",
    "    # Collect response from LLM\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model = \"gpt-3.5-turbo\",\n",
    "        messages = [{\n",
    "            \"role\": \"system\",\n",
    "            \"content\": prompt\n",
    "        }]\n",
    "    )[\"choices\"][0][\"message\"][\"content\"]\n",
    "\n",
    "    # Log response\n",
    "    active_llm_logger.log({\"response\": response})\n",
    "\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42b4b09d-0b47-4706-abe9-72edb04e4d35",
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_reply_success(request,response):\n",
    "    # Create and print user reply\n",
    "    reply = f\"\\nSuccess! Here is the recipe for\"\\\n",
    "            f\"{request}:\\n{response}\"\n",
    "    print(reply)\n",
    "\n",
    "    #Log reply\n",
    "    active_llm_logger.log({\"reply\": reply})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee5755da-f487-4bb5-abc2-4f75f19b972e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_reply_failure(request = \"your request\"):\n",
    "    # Create and print user reply\n",
    "    reply = (\"\\nUnfortunately, we are not able to provide a recipe for \" \\\n",
    "            f\"{request} at this time. Please try Recipe Creator 900 \" \\\n",
    "            f\"in the future.\")\n",
    "    print(reply)\n",
    "\n",
    "    #Log reply\n",
    "    active_llm_logger.log({\"reply\": reply})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dc3b03d-c87f-4d70-8ba2-f747e321266b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LLMApplicationValidationError(ValueError):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "880760c1-bb08-4f99-81b8-6de02a80f119",
   "metadata": {},
   "outputs": [],
   "source": [
    "while True:\n",
    "    try:\n",
    "        request = user_request()\n",
    "        response = prompt_llm(request)\n",
    "        user_reply_success(request, response)\n",
    "    except KeyboardInterrupt:\n",
    "        break\n",
    "    except LLMApplicationValidationError:\n",
    "        user_reply_failure(request)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee702c5e-78e9-41f1-a06a-744e3a173c11",
   "metadata": {},
   "outputs": [],
   "source": [
    "from whylogs.core.relations import Predicate\n",
    "from whylogs.core.metrics.condition_count_metric import Condition\n",
    "from whylogs.core.validators import ConditionValidator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d8e2065-d802-4dca-81b6-3d7137d786db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def raise_error(validator_name, condition_name, value):\n",
    "    raise LLMApplicationValidationError(\n",
    "        f\"Failed {validator_name} with value {value}.\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af74ca18-002e-4afe-937c-520de465c569",
   "metadata": {},
   "outputs": [],
   "source": [
    "low_condition = {\"<0.3\": Condition(Predicate().less_than(0.3))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de5f4609-932f-41de-a11d-38b8ca556967",
   "metadata": {},
   "outputs": [],
   "source": [
    "toxicity_validator = ConditionValidator(\n",
    "    name = \"Toxic\",\n",
    "    conditions = low_condition,\n",
    "    actions = [raise_error]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82ef1e8c-064b-432c-bda4-c7f920b397dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "refusal_validator = ConditionValidator(\n",
    "    name = \"Refusal\",\n",
    "    conditions = low_condition,\n",
    "    actions = [raise_error]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b88ab523-88b2-4817-8a36-0a216e65583f",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_validators = {\n",
    "    \"prompt.toxicity\": [toxicity_validator],\n",
    "    \"response.refusal_similarity\": [refusal_validator]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "182e8950-ccb1-43e6-886f-fe02d23a1b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_llm_logger = why.logger(\n",
    "    model = \"rolling\",\n",
    "    interval = 5,\n",
    "    when = \"M\",\n",
    "    base_name = \"active_llm\",\n",
    "    schema = udf_schema(validators = llm_validators)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "809c648e-4d74-414c-b87e-eef694db46a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_llm_logger.log(\n",
    "    {\"response\":\"I'm sorry, but I can't answer that.\"}\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb01aa07",
   "metadata": {},
   "source": [
    " ⚠️ **Disclaimer**: Please be aware that the code may not capture all safety concerns and some undesired responses can still pass through. We encourage you to explore ways in which you can make the monitoring system more robust."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e78e61d-4ecc-4448-9592-ba157baef6a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "while True:\n",
    "    try:\n",
    "        request = user_request()\n",
    "        response = prompt_llm(request)\n",
    "        user_reply_success(request, response)\n",
    "    except KeyboardInterrupt:\n",
    "        break\n",
    "    except LLMApplicationValidationError:\n",
    "        user_reply_failure(request)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c966f1ec-73aa-4df2-ad45-648a27977880",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}