Sentiment feedback

ansible_ai_connect_chatbot/src/AnsibleChatbot/AnsibleChatbot.tsx

@@ -37,7 +37,6 @@ import lightspeedLogoDark from "../assets/lightspeed_dark.svg";
 
 import "./AnsibleChatbot.scss";
 import {
-  botMessage,
   inDebugMode,
   modelsSupported,
   useChatbot,
@@ -82,6 +81,7 @@ export const AnsibleChatbot: React.FunctionComponent = () => {
   const {
     messages,
     setMessages,
+    botMessage,
     isLoading,
     handleSend,
     alertMessage,

ansible_ai_connect_chatbot/src/App.test.tsx

@@ -89,6 +89,12 @@ describe("App tests", () => {
     ).toBeInTheDocument();
     expect(screen.getByText("Create variables")).toBeInTheDocument();
 
+    const thumbsUpIcon = screen.getByRole("button", { name: "Good response" });
+    await act(async () => fireEvent.click(thumbsUpIcon));
+
+    const thumbsDownIcon = screen.getByRole("button", { name: "Bad response" });
+    await act(async () => fireEvent.click(thumbsDownIcon));
+
     const clearContextButton = screen.getByLabelText("Clear context");
     await act(async () => fireEvent.click(clearContextButton));
     expect(
@@ -116,6 +122,27 @@ describe("App tests", () => {
     ).not.toBeVisible();
   });
 
+  it("ThumbsDown icon test", async () => {
+    mockAxios(200);
+    renderApp();
+    const textArea = screen.getByLabelText("Send a message...");
+    await act(async () => userEvent.type(textArea, "Hello"));
+    const sendButton = screen.getByLabelText("Send button");
+    await act(async () => fireEvent.click(sendButton));
+    expect(
+      screen.getByText(
+        "In Ansible, the precedence of variables is determined by the order...",
+      ),
+    ).toBeInTheDocument();
+    expect(screen.getByText("Create variables")).toBeInTheDocument();
+
+    const thumbsDownIcon = screen.getByRole("button", { name: "Bad response" });
+    await act(async () => fireEvent.click(thumbsDownIcon));
+
+    const sureButton = screen.getByText("Sure!");
+    await act(async () => fireEvent.click(sureButton));
+  });
+
   it("Chat service returns 500", async () => {
     mockAxios(500);
     const view = renderApp();
@@ -157,6 +184,56 @@ describe("App tests", () => {
     );
   });
 
+  it("Feedback API returns 500", async () => {
+    mockAxios(200);
+    const view = renderApp();
+    const textArea = screen.getByLabelText("Send a message...");
+    await act(async () => userEvent.type(textArea, "Hello"));
+    const sendButton = screen.getByLabelText("Send button");
+    await act(async () => fireEvent.click(sendButton));
+    expect(
+      screen.getByText(
+        "In Ansible, the precedence of variables is determined by the order...",
+      ),
+    ).toBeInTheDocument();
+    expect(screen.getByText("Create variables")).toBeInTheDocument();
+
+    vi.restoreAllMocks();
+    mockAxios(500);
+
+    const thumbsUpIcon = screen.getByRole("button", { name: "Good response" });
+    await act(async () => fireEvent.click(thumbsUpIcon));
+    const alert = view.container.querySelector(".pf-v6-c-alert__description");
+    const textContent = alert?.textContent;
+    expect(textContent).toEqual("Feedback API returned status_code 500");
+  });
+
+  it("Feedback API returns an unexpected error", async () => {
+    mockAxios(200);
+    const view = renderApp();
+    const textArea = screen.getByLabelText("Send a message...");
+    await act(async () => userEvent.type(textArea, "Hello"));
+    const sendButton = screen.getByLabelText("Send button");
+    await act(async () => fireEvent.click(sendButton));
+    expect(
+      screen.getByText(
+        "In Ansible, the precedence of variables is determined by the order...",
+      ),
+    ).toBeInTheDocument();
+    expect(screen.getByText("Create variables")).toBeInTheDocument();
+
+    vi.restoreAllMocks();
+    mockAxios(-1, true);
+
+    const thumbsUpIcon = screen.getByRole("button", { name: "Good response" });
+    await act(async () => fireEvent.click(thumbsUpIcon));
+    const alert = view.container.querySelector(".pf-v6-c-alert__description");
+    const textContent = alert?.textContent;
+    expect(textContent).toEqual(
+      "An unexpected error occured: Error: mocked error",
+    );
+  });
+
   it("Color theme switch", async () => {
     mockAxios(200);
     const view = renderApp();

ansible_ai_connect_chatbot/src/Constants.ts

@@ -15,3 +15,14 @@ export const FOOTNOTE_TITLE = "Verify accuracy";
 /* Footnote description */
 export const FOOTNOTE_DESCRIPTION =
   "While Lightspeed strives for accuracy, there's always a possibility of errors. It's a good practice to verify critical information from reliable sources, especially if it's crucial for decision-making or actions.";
+
+/* Sentiments */
+export enum Sentiment {
+  THUMBS_UP = 0,
+  THUMBS_DOWN = 1,
+}
+
+export const GITHUB_NEW_ISSUE_URL =
+  "https://github.com/ansible/ansible-lightspeed-va-feedback/issues/new" +
+  "?assignees=korenaren&labels=bug%2Ctriage&projects=&template=chatbot_feedback.yml" +
+  "&title=Chatbot+response+can+be+improved";

ansible_ai_connect_chatbot/src/index.css

@@ -11,6 +11,10 @@
   inset: 0;
   content: "";
   background-color: var(--pf-t--global--background--color--floating--default);
-  opacity: 0.8;
+  opacity: 0;
   box-shadow: var(--pf-t--global--box-shadow--sm);
 }
+
+.action-button-clicked {
+  background-color: #80808040 !important;
+}

ansible_ai_connect_chatbot/src/types/Message.ts

@@ -1,4 +1,5 @@
 import type { MessageProps } from "@patternfly/virtual-assistant/dist/dynamic/Message";
+import { Sentiment } from "../Constants";
 
 // Types for OLS (OpenShift lightspeed-service) POST /v1/query API
 type LLMRequest = {
@@ -34,3 +35,9 @@ export type ReferencedDocumentsProp = {
 export type ExtendedMessage = MessageProps & {
   referenced_documents: ReferencedDocument[];
 };
+
+export type ChatFeedback = {
+  query: string;
+  response: ChatResponse;
+  sentiment: Sentiment;
+};

ansible_ai_connect_chatbot/src/useChatbot/useChatbot.ts

@@ -5,11 +5,17 @@ import type {
   ExtendedMessage,
   ChatRequest,
   ChatResponse,
+  ChatFeedback,
 } from "../types/Message";
 import type { LLMModel } from "../types/Model";
 import logo from "../assets/lightspeed.svg";
 import userLogo from "../assets/user_logo.png";
-import { API_TIMEOUT, TIMEOUT_MSG } from "../Constants";
+import {
+  API_TIMEOUT,
+  GITHUB_NEW_ISSUE_URL,
+  Sentiment,
+  TIMEOUT_MSG,
+} from "../Constants";
 
 const userName = document.getElementById("user_name")?.innerText ?? "User";
 const botName =
@@ -45,18 +51,6 @@ export const inDebugMode = () => {
   return import.meta.env.PROD ? debug === "true" : debug !== "false";
 };
 
-export const botMessage = (content: string): MessageProps => ({
-  role: "bot",
-  content,
-  name: botName,
-  avatar: logo,
-  timestamp: getTimestamp(),
-  actions: {
-    positive: { onClick: () => console.log("Good response") },
-    negative: { onClick: () => console.log("Bad response") },
-  },
-});
-
 const isTimeoutError = (e: any) =>
   e?.name === "AxiosError" &&
   e?.message === `timeout of ${API_TIMEOUT}ms exceeded`;
@@ -69,6 +63,33 @@ export const timeoutMessage = (): MessageProps => ({
   timestamp: getTimestamp(),
 });
 
+export const feedbackMessage = (f: ChatFeedback): MessageProps => ({
+  role: "bot",
+  content:
+    f.sentiment === Sentiment.THUMBS_UP
+      ? "Thank you for your feedback!"
+      : "Thank you for your feedback. If you have more to share, please click the button below (_requires GitHub login_).",
+  name: botName,
+  avatar: logo,
+  timestamp: getTimestamp(),
+  quickResponses:
+    f.sentiment === Sentiment.THUMBS_UP
+      ? []
+      : [
+          {
+            content: "Sure!",
+            id: "response",
+            onClick: () =>
+              window
+                .open(
+                  `${GITHUB_NEW_ISSUE_URL}&conversation_id=${f.response.conversation_id}&prompt=${f.query}&response=${f.response.response}`,
+                  "_blank",
+                )
+                ?.focus(),
+          },
+        ],
+});
+
 type AlertMessage = {
   title: string;
   message: string;
@@ -99,6 +120,85 @@ export const useChatbot = () => {
     setMessages((msgs: ExtendedMessage[]) => [...msgs, newMessage]);
   };
 
+  const botMessage = (
+    response: ChatResponse | string,
+    query = "",
+  ): MessageProps => {
+    const sendFeedback = async (sentiment: Sentiment) => {
+      if (typeof response === "object") {
+        handleFeedback({ query, response, sentiment });
+      }
+    };
+    const message: MessageProps = {
+      role: "bot",
+      content: typeof response === "object" ? response.response : response,
+      name: botName,
+      avatar: logo,
+      timestamp: getTimestamp(),
+    };
+
+    message.actions = {
+      positive: {
+        onClick: () => {
+          sendFeedback(Sentiment.THUMBS_UP);
+          if (message.actions) {
+            message.actions.positive.isDisabled = true;
+            message.actions.negative.isDisabled = true;
+            message.actions.positive.className = "action-button-clicked";
+          }
+        },
+      },
+      negative: {
+        onClick: () => {
+          sendFeedback(Sentiment.THUMBS_DOWN);
+          if (message.actions) {
+            message.actions.positive.isDisabled = true;
+            message.actions.negative.isDisabled = true;
+            message.actions.negative.className = "action-button-clicked";
+          }
+        },
+      },
+    };
+    return message;
+  };
+
+  const handleFeedback = async (feedbackRequest: ChatFeedback) => {
+    try {
+      const csrfToken = readCookie("csrftoken");
+      const resp = await axios.post(
+        "/api/v0/ai/feedback/",
+        {
+          chatFeedback: feedbackRequest,
+        },
+        {
+          headers: {
+            "Content-Type": "application/json",
+            "X-CSRFToken": csrfToken,
+          },
+        },
+      );
+      if (resp.status === 200) {
+        const newBotMessage = {
+          referenced_documents: [],
+          ...feedbackMessage(feedbackRequest),
+        };
+        addMessage(newBotMessage);
+      } else {
+        setAlertMessage({
+          title: "Error",
+          message: `Feedback API returned status_code ${resp.status}`,
+          variant: "danger",
+        });
+      }
+    } catch (e) {
+      setAlertMessage({
+        title: "Error",
+        message: `An unexpected error occured: ${e}`,
+        variant: "danger",
+      });
+    }
+  };
+
   const handleSend = async (message: string) => {
     const userMessage: ExtendedMessage = {
       role: "user",
@@ -146,10 +246,8 @@ export const useChatbot = () => {
         if (!conversationId) {
           setConversationId(chatResponse.conversation_id);
         }
-        const newBotMessage = {
-          referenced_documents,
-          ...botMessage(chatResponse.response),
-        };
+        const newBotMessage: any = botMessage(chatResponse, message);
+        newBotMessage.referenced_documents = referenced_documents;
         addMessage(newBotMessage);
       } else {
         setAlertMessage({
@@ -180,6 +278,7 @@ export const useChatbot = () => {
   return {
     messages,
     setMessages,
+    botMessage,
     isLoading,
     handleSend,
     alertMessage,