From e3ae39e2baba55a1c18e2bc8ca985ea596cf3279 Mon Sep 17 00:00:00 2001
From: Dave Glover <dglover@microsoft.com>
Date: Tue, 10 Sep 2024 20:48:25 +1000
Subject: [PATCH 1/3] gpt-4-vision

---
 .../src/components/playground/ChatCard.tsx    | 431 +++++++++++-------
 .../src/components/playground/Message.tsx     |  46 +-
 src/playground/src/pages/playground/Chat.tsx  |   5 +-
 3 files changed, 313 insertions(+), 169 deletions(-)

diff --git a/src/playground/src/components/playground/ChatCard.tsx b/src/playground/src/components/playground/ChatCard.tsx
index d72c3d57..45fb3cff 100644
--- a/src/playground/src/components/playground/ChatCard.tsx
+++ b/src/playground/src/components/playground/ChatCard.tsx
@@ -1,15 +1,14 @@
 import {
-  makeStyles,
-  Body1,
-  Button,
-  CardFooter,
-  Field,
-  Textarea,
-  Spinner,
-  shorthands
+    makeStyles,
+    Body1,
+    Button,
+    CardFooter,
+    Textarea,
+    Spinner,
+    shorthands
 } from "@fluentui/react-components";
 import { Dispatch, useEffect, useRef, useState } from "react";
-import { Delete24Regular, SendRegular } from "@fluentui/react-icons";
+import { Delete24Regular, SendRegular, Attach24Regular } from "@fluentui/react-icons";
 import { Message } from "./Message";
 import { Response } from "./Response";
 import { useEventDataContext } from "../../providers/EventDataProvider";
@@ -17,175 +16,281 @@ import { Card } from "./Card";
 import { ChatResponseMessageExtended } from "../../pages/playground/Chat.state";
 
 interface CardProps {
-  onPromptEntered: Dispatch<string>;
-  messageList: ChatResponseMessageExtended[];
-  onClear: () => void;
-  isLoading: boolean;
-  canChat: boolean;
+    onPromptEntered: Dispatch<string>;
+    messageList: ChatResponseMessageExtended[];
+    onClear: () => void;
+    isLoading: boolean;
+    canChat: boolean;
 }
 
 const useStyles = makeStyles({
-  dialog: {
-    display: "block",
-  },
-  smallButton: {
-    width: "100%",
-    height: "40%",
-    maxWidth: "none",
-    textAlign: "left",
-    marginBottom: "12px"
-  },
-  startCard: {
-    display: "flex",
-    maxWidth: "80%",
-    ...shorthands.margin("35%", "20%"),
-    ...shorthands.padding("20px", "0px"),
-  },
-  chatCard: {
-    display: "flex",
-    height: "calc(100vh - 92px)",
-  },
+    dialog: {
+        display: "block",
+    },
+    buttonContainer: {
+        display: "flex",
+        justifyContent: "flex-end",  // Align buttons to the right
+    },
+    smallButton: {
+        marginBottom: "12px",
+        ...shorthands.margin("4px"),
+    },
+    startCard: {
+        display: "flex",
+        maxWidth: "80%",
+        ...shorthands.margin("35%", "20%"),
+        ...shorthands.padding("20px", "0px"),
+    },
+    chatCard: {
+        display: "flex",
+        height: "calc(100vh - 100px)",
+    },
+    wrapper: {
+        display: "flex",
+        flexDirection: "column",
+        justifyContent: "flex-end", // Ensure content sticks to the bottom
+        height: "120px",
+        maxHeight: "120px",
+    },
+    userQuery: {
+        flexGrow: 0,                // Do not grow to fill remaining space
+        marginBottom: "10px",
+        overflowY: "auto",          // Scroll if content overflows
+    }
 });
 
 export const ChatCard = ({
-  onPromptEntered,
-  messageList,
-  onClear,
-  isLoading,
-  canChat,
+    onPromptEntered,
+    messageList,
+    onClear,
+    isLoading,
+    canChat,
 }: CardProps) => {
-  const chat = useStyles();
-  const chatContainerRef = useRef<HTMLDivElement>(null);
-  const { isAuthorized } = useEventDataContext();
-
-  useEffect(() => {
-    if (chatContainerRef.current) {
-      chatContainerRef.current.scrollTop =
-        chatContainerRef.current.scrollHeight;
-    }
-  }, [messageList]);
-
-  return (
-    <Card header="Chat session" className={chat.chatCard} >
-
-      {isAuthorized && (
-        <>
-          <div
-            id={"chatContainer"}
-            style={{ overflowY: "auto" }}
-            ref={chatContainerRef}
-          >
-
-            {messageList.length > 1 ? (
-              messageList.map((message, index) => {
-                if (message.role === "system") {
-                  return null;
-                }
-                return message.role === "user" ? (
-                  <Message key={index} message={message} />
-                ) : (
-                  <Response key={index} message={message} />
-                );
-              })
-            ) : (
-              <Card className={chat.startCard}>
-                <Body1 style={{ textAlign: "center" }}>
-                  {!canChat && (<h2>Select a model</h2>)}
-                  {canChat && (
-                    <>
-                      <h2>Start chatting</h2>
-                      Test your assistant by sending queries below. Then adjust your assistant setup to improve the assistant's responses.
-                    </>
-                  )}
-                </Body1>
-              </Card>
+    const chat = useStyles();
+    const chatContainerRef = useRef<HTMLDivElement>(null);
+    const { isAuthorized } = useEventDataContext();
+
+    useEffect(() => {
+        if (chatContainerRef.current) {
+            chatContainerRef.current.scrollTop =
+                chatContainerRef.current.scrollHeight;
+        }
+    }, [messageList]);
+
+    return (
+        <Card header="Chat session" className={chat.chatCard} >
+
+            {isAuthorized && (
+                <>
+                    <div
+                        id={"chatContainer"}
+                        style={{ overflowY: "auto" }}
+                        ref={chatContainerRef}
+                    >
+
+                        {messageList.length > 1 ? (
+                            messageList.map((message, index) => {
+                                if (message.role === "system") {
+                                    return null;
+                                }
+                                return message.role === "user" ? (
+                                    <Message key={index} message={message} />
+                                ) : (
+                                    <Response key={index} message={message} />
+                                );
+                            })
+                        ) : (
+                            <Card className={chat.startCard}>
+                                <Body1 style={{ textAlign: "center" }}>
+                                    {!canChat && (<h2>Select a model</h2>)}
+                                    {canChat && (
+                                        <>
+                                            <h2>Start chatting</h2>
+                                            Test your assistant by sending queries below. Then adjust your assistant setup to improve the assistant's responses.
+                                        </>
+                                    )}
+                                </Body1>
+                            </Card>
+                        )}
+                    </div>
+                    {isLoading && <Spinner />}
+                    {isAuthorized && (
+                        <ChatInput
+                            promptSubmitted={onPromptEntered}
+                            onClear={onClear}
+                            canChat={canChat}
+                        />
+                    )}
+                    {!isAuthorized && (
+                        <>
+                            <CardFooter style={{ height: "10vh" }}>
+                                <p>Please enter your event code to start the chat session.</p>
+                            </CardFooter>
+                        </>
+                    )}
+                </>
             )}
-          </div>
-          {isLoading && <Spinner />}
-          {isAuthorized && (
-            <ChatInput
-              promptSubmitted={onPromptEntered}
-              onClear={onClear}
-              canChat={canChat}
-            />
-          )}
-          {!isAuthorized && (
-            <>
-              <CardFooter style={{ height: "10vh" }}>
-                <p>Please enter your event code to start the chat session.</p>
-              </CardFooter>
-            </>
-          )}
-        </>
-      )}
-    </Card>
-  );
+        </Card>
+    );
 };
 
 const hasPrompt = (prompt: string) => {
-  const regex = /^\s*$/;
-  return !regex.test(prompt);
+    const regex = /^\s*$/;
+    return !regex.test(prompt);
 };
 
 function ChatInput({
-  promptSubmitted,
-  onClear,
-  canChat,
+    promptSubmitted,
+    onClear,
+    canChat,
 }: {
-  promptSubmitted: Dispatch<string>;
-  onClear: () => void;
-  canChat: boolean;
+    promptSubmitted: Dispatch<string>;
+    onClear: () => void;
+    canChat: boolean;
 }) {
-  const [userPrompt, setPrompt] = useState("");
-
-  const chat = useStyles();
-  return (
-    <CardFooter style={{ height: "10vh" }}>
-      <Field className="user-query" style={{ width: "100%" }}>
-        <Textarea
-          value={userPrompt}
-          placeholder="Type user query here (Shift + Enter for new line)"
-          disabled={!canChat}
-          onChange={(event) => setPrompt(event.target.value)}
-          onKeyDown={(event) => {
-            if (
-              event.key === "Enter" &&
-              !event.shiftKey &&
-              hasPrompt(userPrompt)
-            ) {
-              promptSubmitted(userPrompt);
-              setPrompt("");
-              event.preventDefault();
+    const [userPrompt, setPrompt] = useState("");
+    const [files, setFiles] = useState<{ name: string, dataUrl: string }[]>([]);
+    const maxFiles = 10;
+
+    const chat = useStyles();
+
+    // Handle file selection and convert files to base64 data URLs
+    const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
+        const selectedFiles = Array.from(e.target.files || []);
+
+        if (selectedFiles.length + files.length > maxFiles) {
+            alert(`You can only upload up to ${maxFiles} files.`);
+            return;
+        }
+
+        try {
+            const newFiles = await Promise.all(
+                selectedFiles.map(file => new Promise<{ name: string, dataUrl: string }>((resolve, reject) => {
+                    const reader = new FileReader();
+
+                    reader.onload = () => {
+                        const dataUrl = reader.result?.toString();
+                        if (dataUrl) {
+                            resolve({ name: file.name, dataUrl });
+                        } else {
+                            reject(new Error('Failed to read file data'));
+                        }
+                    };
+
+                    reader.onerror = () => {
+                        console.error(`Error reading file: ${file.name}`);
+                        reject(new Error(`Error reading file: ${file.name}`));
+                    };
+
+                    reader.readAsDataURL(file);
+                }))
+            );
+
+            setFiles(prevFiles => [...prevFiles, ...newFiles]);
+        } catch (error) {
+            console.error('Error encoding files to base64:', error);
+            alert('Error uploading files, please try again.');
+        } finally {
+            // Clear the file input value to allow re-uploading the same file
+            if (fileInputRef.current) {
+                fileInputRef.current.value = '';
             }
-          }}
-        />
-      </Field>
-      <div>
-        <Button
-          className={chat.smallButton}
-          id={"send-button"}
-          icon={<SendRegular />}
-          iconPosition="before"
-          appearance="primary"
-          onClick={() => {
-            promptSubmitted(userPrompt);
-            setPrompt("");
-          }}
-          disabled={!canChat || !hasPrompt(userPrompt)}
-        >
-          Send
-        </Button>
-        <Button
-          className={chat.smallButton}
-          id="clear-button"
-          disabled={!canChat}
-          icon={<Delete24Regular />}
-          iconPosition="before"
-          onClick={onClear}
-        >
-          Clear
-        </Button>
-      </div>
-    </CardFooter>
-  );
+        }
+    };
+
+    const fileInputRef = useRef<HTMLInputElement | null>(null);
+
+    const triggerFileInput = () => {
+        if (fileInputRef.current) {
+            fileInputRef.current.click();
+        }
+    };
+
+    const handleSend = () => {
+        if (files.length > 0) {
+            const promptImageData = [
+                { type: "text", text: userPrompt },
+                ...files.map(file => ({
+                    type: "image_url",
+                    imageUrl: { url: file.dataUrl }
+                }))
+            ];
+            promptSubmitted(JSON.stringify(promptImageData));
+        }
+        else {
+            const promptTextData = userPrompt;
+            promptSubmitted(JSON.stringify(promptTextData));
+        }
+        setPrompt(""); // Clear the prompt
+    };
+
+    const clearAll = () => {
+        setFiles([]);  // Clear all uploaded files
+        onClear();
+        setPrompt("");
+    };
+
+    return (
+        <div className={chat.wrapper}>
+            <Textarea className={chat.wrapper}
+                value={userPrompt}
+                placeholder="Type user query here (Shift + Enter for new line)"
+                disabled={!canChat}
+                onChange={(event) => setPrompt(event.target.value)}
+                onKeyDown={(event) => {
+                    if (
+                        event.key === "Enter" &&
+                        !event.shiftKey &&
+                        hasPrompt(userPrompt)
+                    ) {
+                        handleSend();
+                        setPrompt("");
+                        event.preventDefault();
+                    }
+                }}
+            />
+            <div className={chat.buttonContainer}>
+                {/* Hidden file input */}
+                <input
+                    ref={fileInputRef}
+                    id="upload-button"
+                    type="file"
+                    accept="image/png, image/jpeg, image/jpg, image/gif, image/webp"
+                    onChange={handleFileChange}
+                    style={{ display: 'none' }} // This hides the file input
+                    multiple // Enable multiple file selection
+                />
+
+                {/* Button to trigger the file input */}
+                <Button
+                    className={chat.smallButton}
+                    id="upload-button"
+                    icon={<Attach24Regular />}
+                    iconPosition="before"
+                    disabled={!canChat}
+                    onClick={triggerFileInput}>
+                    {files?.length} of {maxFiles}
+                </Button>
+                <Button
+                    className={chat.smallButton}
+                    id="clear-button"
+                    disabled={!canChat}
+                    icon={<Delete24Regular />}
+                    iconPosition="before"
+                    onClick={clearAll}
+                >
+                </Button>
+                <Button
+                    className={chat.smallButton}
+                    id={"send-button"}
+                    icon={<SendRegular />}
+                    iconPosition="before"
+                    appearance="primary"
+                    onClick={handleSend} // Use handleSend function to check for file or prompt
+                    disabled={!canChat || !hasPrompt(userPrompt)} // Disable if no input or file
+                >
+                </Button>
+            </div>
+        </div>
+    );
 }
diff --git a/src/playground/src/components/playground/Message.tsx b/src/playground/src/components/playground/Message.tsx
index b2f617d1..903bac0d 100644
--- a/src/playground/src/components/playground/Message.tsx
+++ b/src/playground/src/components/playground/Message.tsx
@@ -13,6 +13,12 @@ const useStyles = makeStyles({
     marginBottom: "20px",
     maxWidth: "80%",
     marginLeft: "auto",
+    flexWrap: "wrap", // Enable wrapping for the images
+  },
+  image: {
+    maxWidth: "20%",
+    height: "auto",
+    ...shorthands.margin("2px") // Add spacing between the images manually
   },
   message: {
     fontSize: "medium",
@@ -26,10 +32,10 @@ const useStyles = makeStyles({
     ...shorthands.borderRadius("2px"),
   },
   icon: {
-    minWidth:"24px",
-    maxWidth:"24px",
-    width:"24px",
-    marginTop:"6px"
+    minWidth: "24px",
+    maxWidth: "24px",
+    width: "24px",
+    marginTop: "6px"
   }
 });
 
@@ -37,10 +43,40 @@ const useStyles = makeStyles({
 export const Message = ({ message }: Props) => {
   const styles = useStyles();
 
+  // write to the console the object type of message.content
+  console.log('message.content is of type: ', typeof message.content);
+
+  const parsedPrompt = message.content ? JSON.parse(message.content) : null;
+
+  if (Array.isArray(parsedPrompt)) {
+    // Find the text content if it exists
+    const textContent = parsedPrompt.find(item => item.type === 'text')?.text;
+
+    // Find all image URLs
+    const imageUrls = parsedPrompt
+      .filter(item => item.type === 'image_url')
+      .map(item => item.imageUrl?.url);
+
+    return (
+      <div className={styles.container}>
+        {textContent && <div className={styles.message}>{textContent}</div>}
+        {imageUrls.map((url, index) => (
+          <img
+            key={index}
+            src={url}
+            alt={`Message image ${index + 1}`}
+            className={styles.image}// Restrict image width
+          />
+        ))}
+        <Person32Regular className={styles.icon} />
+      </div>
+    );
+  }
+
   return (
     <div className={styles.container}>
       <div className={styles.message}>{message.content}</div>
-      <Person32Regular className={styles.icon}/>
+      <Person32Regular className={styles.icon} />
     </div>
   );
 };
diff --git a/src/playground/src/pages/playground/Chat.tsx b/src/playground/src/pages/playground/Chat.tsx
index f26c21cd..54f80290 100644
--- a/src/playground/src/pages/playground/Chat.tsx
+++ b/src/playground/src/pages/playground/Chat.tsx
@@ -70,10 +70,13 @@ export const Chat = () => {
     try {
       const start = Date.now();
 
+      // Parse the JSON string back into an array of message objects
+      const parsedPrompt = JSON.parse(prompt);
+
       const messages: ChatRequestMessage[] = [
         state.systemPrompt,
         ...state.messages.filter((m) => !m.isError).map(mapMessage),
-        { role: "user", content: prompt },
+        { role: "user", content: parsedPrompt },
       ];
 
       const chatCompletions = await client.getChatCompletions(

From 5a54b104147aa621858719399d08064ce9f2f80a Mon Sep 17 00:00:00 2001
From: Dave Glover <dglover@microsoft.com>
Date: Tue, 10 Sep 2024 20:50:46 +1000
Subject: [PATCH 2/3] Update Message.tsx

---
 src/playground/src/components/playground/Message.tsx | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/playground/src/components/playground/Message.tsx b/src/playground/src/components/playground/Message.tsx
index 903bac0d..10a8b43b 100644
--- a/src/playground/src/components/playground/Message.tsx
+++ b/src/playground/src/components/playground/Message.tsx
@@ -43,9 +43,6 @@ const useStyles = makeStyles({
 export const Message = ({ message }: Props) => {
   const styles = useStyles();
 
-  // write to the console the object type of message.content
-  console.log('message.content is of type: ', typeof message.content);
-
   const parsedPrompt = message.content ? JSON.parse(message.content) : null;
 
   if (Array.isArray(parsedPrompt)) {

From 0354a84b0781fc98152dc9a910fc19ff5760483f Mon Sep 17 00:00:00 2001
From: Aaron Powell <me@aaron-powell.com>
Date: Wed, 11 Sep 2024 07:57:52 +0000
Subject: [PATCH 3/3] Fixing TypeScript

---
 .../src/components/playground/ChatCard.tsx    | 518 +++++++++---------
 .../src/pages/playground/Chat.reducers.ts     |  11 +-
 src/playground/src/pages/playground/Chat.tsx  |   8 +-
 3 files changed, 274 insertions(+), 263 deletions(-)

diff --git a/src/playground/src/components/playground/ChatCard.tsx b/src/playground/src/components/playground/ChatCard.tsx
index 45fb3cff..5e0a78c4 100644
--- a/src/playground/src/components/playground/ChatCard.tsx
+++ b/src/playground/src/components/playground/ChatCard.tsx
@@ -1,296 +1,308 @@
 import {
-    makeStyles,
-    Body1,
-    Button,
-    CardFooter,
-    Textarea,
-    Spinner,
-    shorthands
+  makeStyles,
+  Body1,
+  Button,
+  CardFooter,
+  Textarea,
+  Spinner,
+  shorthands,
 } from "@fluentui/react-components";
 import { Dispatch, useEffect, useRef, useState } from "react";
-import { Delete24Regular, SendRegular, Attach24Regular } from "@fluentui/react-icons";
+import {
+  Delete24Regular,
+  SendRegular,
+  Attach24Regular,
+} from "@fluentui/react-icons";
 import { Message } from "./Message";
 import { Response } from "./Response";
 import { useEventDataContext } from "../../providers/EventDataProvider";
 import { Card } from "./Card";
 import { ChatResponseMessageExtended } from "../../pages/playground/Chat.state";
+import {
+  ChatMessageContentItem,
+  ChatMessageImageContentItem,
+} from "@azure/openai";
 
 interface CardProps {
-    onPromptEntered: Dispatch<string>;
-    messageList: ChatResponseMessageExtended[];
-    onClear: () => void;
-    isLoading: boolean;
-    canChat: boolean;
+  onPromptEntered: Dispatch<ChatMessageContentItem[]>;
+  messageList: ChatResponseMessageExtended[];
+  onClear: () => void;
+  isLoading: boolean;
+  canChat: boolean;
 }
 
 const useStyles = makeStyles({
-    dialog: {
-        display: "block",
-    },
-    buttonContainer: {
-        display: "flex",
-        justifyContent: "flex-end",  // Align buttons to the right
-    },
-    smallButton: {
-        marginBottom: "12px",
-        ...shorthands.margin("4px"),
-    },
-    startCard: {
-        display: "flex",
-        maxWidth: "80%",
-        ...shorthands.margin("35%", "20%"),
-        ...shorthands.padding("20px", "0px"),
-    },
-    chatCard: {
-        display: "flex",
-        height: "calc(100vh - 100px)",
-    },
-    wrapper: {
-        display: "flex",
-        flexDirection: "column",
-        justifyContent: "flex-end", // Ensure content sticks to the bottom
-        height: "120px",
-        maxHeight: "120px",
-    },
-    userQuery: {
-        flexGrow: 0,                // Do not grow to fill remaining space
-        marginBottom: "10px",
-        overflowY: "auto",          // Scroll if content overflows
-    }
+  dialog: {
+    display: "block",
+  },
+  buttonContainer: {
+    display: "flex",
+    justifyContent: "flex-end", // Align buttons to the right
+  },
+  smallButton: {
+    marginBottom: "12px",
+    ...shorthands.margin("4px"),
+  },
+  startCard: {
+    display: "flex",
+    maxWidth: "80%",
+    ...shorthands.margin("35%", "20%"),
+    ...shorthands.padding("20px", "0px"),
+  },
+  chatCard: {
+    display: "flex",
+    height: "calc(100vh - 100px)",
+  },
+  wrapper: {
+    display: "flex",
+    flexDirection: "column",
+    justifyContent: "flex-end", // Ensure content sticks to the bottom
+    height: "120px",
+    maxHeight: "120px",
+  },
+  userQuery: {
+    flexGrow: 0, // Do not grow to fill remaining space
+    marginBottom: "10px",
+    overflowY: "auto", // Scroll if content overflows
+  },
 });
 
 export const ChatCard = ({
-    onPromptEntered,
-    messageList,
-    onClear,
-    isLoading,
-    canChat,
+  onPromptEntered,
+  messageList,
+  onClear,
+  isLoading,
+  canChat,
 }: CardProps) => {
-    const chat = useStyles();
-    const chatContainerRef = useRef<HTMLDivElement>(null);
-    const { isAuthorized } = useEventDataContext();
+  const chat = useStyles();
+  const chatContainerRef = useRef<HTMLDivElement>(null);
+  const { isAuthorized } = useEventDataContext();
 
-    useEffect(() => {
-        if (chatContainerRef.current) {
-            chatContainerRef.current.scrollTop =
-                chatContainerRef.current.scrollHeight;
-        }
-    }, [messageList]);
-
-    return (
-        <Card header="Chat session" className={chat.chatCard} >
-
-            {isAuthorized && (
-                <>
-                    <div
-                        id={"chatContainer"}
-                        style={{ overflowY: "auto" }}
-                        ref={chatContainerRef}
-                    >
+  useEffect(() => {
+    if (chatContainerRef.current) {
+      chatContainerRef.current.scrollTop =
+        chatContainerRef.current.scrollHeight;
+    }
+  }, [messageList]);
 
-                        {messageList.length > 1 ? (
-                            messageList.map((message, index) => {
-                                if (message.role === "system") {
-                                    return null;
-                                }
-                                return message.role === "user" ? (
-                                    <Message key={index} message={message} />
-                                ) : (
-                                    <Response key={index} message={message} />
-                                );
-                            })
-                        ) : (
-                            <Card className={chat.startCard}>
-                                <Body1 style={{ textAlign: "center" }}>
-                                    {!canChat && (<h2>Select a model</h2>)}
-                                    {canChat && (
-                                        <>
-                                            <h2>Start chatting</h2>
-                                            Test your assistant by sending queries below. Then adjust your assistant setup to improve the assistant's responses.
-                                        </>
-                                    )}
-                                </Body1>
-                            </Card>
-                        )}
-                    </div>
-                    {isLoading && <Spinner />}
-                    {isAuthorized && (
-                        <ChatInput
-                            promptSubmitted={onPromptEntered}
-                            onClear={onClear}
-                            canChat={canChat}
-                        />
-                    )}
-                    {!isAuthorized && (
-                        <>
-                            <CardFooter style={{ height: "10vh" }}>
-                                <p>Please enter your event code to start the chat session.</p>
-                            </CardFooter>
-                        </>
-                    )}
-                </>
+  return (
+    <Card header="Chat session" className={chat.chatCard}>
+      {isAuthorized && (
+        <>
+          <div
+            id={"chatContainer"}
+            style={{ overflowY: "auto" }}
+            ref={chatContainerRef}
+          >
+            {messageList.length > 1 ? (
+              messageList.map((message, index) => {
+                if (message.role === "system") {
+                  return null;
+                }
+                return message.role === "user" ? (
+                  <Message key={index} message={message} />
+                ) : (
+                  <Response key={index} message={message} />
+                );
+              })
+            ) : (
+              <Card className={chat.startCard}>
+                <Body1 style={{ textAlign: "center" }}>
+                  {!canChat && <h2>Select a model</h2>}
+                  {canChat && (
+                    <>
+                      <h2>Start chatting</h2>
+                      Test your assistant by sending queries below. Then adjust
+                      your assistant setup to improve the assistant's responses.
+                    </>
+                  )}
+                </Body1>
+              </Card>
             )}
-        </Card>
-    );
+          </div>
+          {isLoading && <Spinner />}
+          {isAuthorized && (
+            <ChatInput
+              promptSubmitted={onPromptEntered}
+              onClear={onClear}
+              canChat={canChat}
+            />
+          )}
+          {!isAuthorized && (
+            <>
+              <CardFooter style={{ height: "10vh" }}>
+                <p>Please enter your event code to start the chat session.</p>
+              </CardFooter>
+            </>
+          )}
+        </>
+      )}
+    </Card>
+  );
 };
 
 const hasPrompt = (prompt: string) => {
-    const regex = /^\s*$/;
-    return !regex.test(prompt);
+  const regex = /^\s*$/;
+  return !regex.test(prompt);
 };
 
 function ChatInput({
-    promptSubmitted,
-    onClear,
-    canChat,
+  promptSubmitted,
+  onClear,
+  canChat,
 }: {
-    promptSubmitted: Dispatch<string>;
-    onClear: () => void;
-    canChat: boolean;
+  promptSubmitted: Dispatch<ChatMessageContentItem[]>;
+  onClear: () => void;
+  canChat: boolean;
 }) {
-    const [userPrompt, setPrompt] = useState("");
-    const [files, setFiles] = useState<{ name: string, dataUrl: string }[]>([]);
-    const maxFiles = 10;
+  const [userPrompt, setPrompt] = useState("");
+  const [files, setFiles] = useState<{ name: string; dataUrl: string }[]>([]);
+  const maxFiles = 10;
 
-    const chat = useStyles();
+  const chat = useStyles();
 
-    // Handle file selection and convert files to base64 data URLs
-    const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
-        const selectedFiles = Array.from(e.target.files || []);
+  // Handle file selection and convert files to base64 data URLs
+  const handleFileChange = async (e: React.ChangeEvent<HTMLInputElement>) => {
+    const selectedFiles = Array.from(e.target.files || []);
 
-        if (selectedFiles.length + files.length > maxFiles) {
-            alert(`You can only upload up to ${maxFiles} files.`);
-            return;
-        }
+    if (selectedFiles.length + files.length > maxFiles) {
+      alert(`You can only upload up to ${maxFiles} files.`);
+      return;
+    }
 
-        try {
-            const newFiles = await Promise.all(
-                selectedFiles.map(file => new Promise<{ name: string, dataUrl: string }>((resolve, reject) => {
-                    const reader = new FileReader();
+    try {
+      const newFiles = await Promise.all(
+        selectedFiles.map(
+          (file) =>
+            new Promise<{ name: string; dataUrl: string }>(
+              (resolve, reject) => {
+                const reader = new FileReader();
 
-                    reader.onload = () => {
-                        const dataUrl = reader.result?.toString();
-                        if (dataUrl) {
-                            resolve({ name: file.name, dataUrl });
-                        } else {
-                            reject(new Error('Failed to read file data'));
-                        }
-                    };
+                reader.onload = () => {
+                  const dataUrl = reader.result?.toString();
+                  if (dataUrl) {
+                    resolve({ name: file.name, dataUrl });
+                  } else {
+                    reject(new Error("Failed to read file data"));
+                  }
+                };
 
-                    reader.onerror = () => {
-                        console.error(`Error reading file: ${file.name}`);
-                        reject(new Error(`Error reading file: ${file.name}`));
-                    };
+                reader.onerror = () => {
+                  console.error(`Error reading file: ${file.name}`);
+                  reject(new Error(`Error reading file: ${file.name}`));
+                };
 
-                    reader.readAsDataURL(file);
-                }))
-            );
+                reader.readAsDataURL(file);
+              }
+            )
+        )
+      );
 
-            setFiles(prevFiles => [...prevFiles, ...newFiles]);
-        } catch (error) {
-            console.error('Error encoding files to base64:', error);
-            alert('Error uploading files, please try again.');
-        } finally {
-            // Clear the file input value to allow re-uploading the same file
-            if (fileInputRef.current) {
-                fileInputRef.current.value = '';
-            }
-        }
-    };
+      setFiles((prevFiles) => [...prevFiles, ...newFiles]);
+    } catch (error) {
+      console.error("Error encoding files to base64:", error);
+      alert("Error uploading files, please try again.");
+    } finally {
+      // Clear the file input value to allow re-uploading the same file
+      if (fileInputRef.current) {
+        fileInputRef.current.value = "";
+      }
+    }
+  };
 
-    const fileInputRef = useRef<HTMLInputElement | null>(null);
+  const fileInputRef = useRef<HTMLInputElement | null>(null);
 
-    const triggerFileInput = () => {
-        if (fileInputRef.current) {
-            fileInputRef.current.click();
-        }
-    };
+  const triggerFileInput = () => {
+    if (fileInputRef.current) {
+      fileInputRef.current.click();
+    }
+  };
 
-    const handleSend = () => {
-        if (files.length > 0) {
-            const promptImageData = [
-                { type: "text", text: userPrompt },
-                ...files.map(file => ({
-                    type: "image_url",
-                    imageUrl: { url: file.dataUrl }
-                }))
-            ];
-            promptSubmitted(JSON.stringify(promptImageData));
-        }
-        else {
-            const promptTextData = userPrompt;
-            promptSubmitted(JSON.stringify(promptTextData));
-        }
-        setPrompt(""); // Clear the prompt
-    };
+  const handleSend = () => {
+    if (files.length > 0) {
+      const promptImageData: ChatMessageContentItem[] = [
+        { type: "text", text: userPrompt },
+        ...files.map(
+          (file): ChatMessageImageContentItem => ({
+            type: "image_url",
+            imageUrl: { url: file.dataUrl },
+          })
+        ),
+      ];
+      promptSubmitted(promptImageData);
+    } else {
+      promptSubmitted([{ type: "text", text: userPrompt }]);
+    }
+    setPrompt(""); // Clear the prompt
+  };
 
-    const clearAll = () => {
-        setFiles([]);  // Clear all uploaded files
-        onClear();
-        setPrompt("");
-    };
+  const clearAll = () => {
+    setFiles([]); // Clear all uploaded files
+    onClear();
+    setPrompt("");
+  };
 
-    return (
-        <div className={chat.wrapper}>
-            <Textarea className={chat.wrapper}
-                value={userPrompt}
-                placeholder="Type user query here (Shift + Enter for new line)"
-                disabled={!canChat}
-                onChange={(event) => setPrompt(event.target.value)}
-                onKeyDown={(event) => {
-                    if (
-                        event.key === "Enter" &&
-                        !event.shiftKey &&
-                        hasPrompt(userPrompt)
-                    ) {
-                        handleSend();
-                        setPrompt("");
-                        event.preventDefault();
-                    }
-                }}
-            />
-            <div className={chat.buttonContainer}>
-                {/* Hidden file input */}
-                <input
-                    ref={fileInputRef}
-                    id="upload-button"
-                    type="file"
-                    accept="image/png, image/jpeg, image/jpg, image/gif, image/webp"
-                    onChange={handleFileChange}
-                    style={{ display: 'none' }} // This hides the file input
-                    multiple // Enable multiple file selection
-                />
+  return (
+    <div className={chat.wrapper}>
+      <Textarea
+        className={chat.wrapper}
+        value={userPrompt}
+        placeholder="Type user query here (Shift + Enter for new line)"
+        disabled={!canChat}
+        onChange={(event) => setPrompt(event.target.value)}
+        onKeyDown={(event) => {
+          if (
+            event.key === "Enter" &&
+            !event.shiftKey &&
+            hasPrompt(userPrompt)
+          ) {
+            handleSend();
+            setPrompt("");
+            event.preventDefault();
+          }
+        }}
+      />
+      <div className={chat.buttonContainer}>
+        {/* Hidden file input */}
+        <input
+          ref={fileInputRef}
+          id="upload-button"
+          type="file"
+          accept="image/png, image/jpeg, image/jpg, image/gif, image/webp"
+          onChange={handleFileChange}
+          style={{ display: "none" }} // This hides the file input
+          multiple // Enable multiple file selection
+        />
 
-                {/* Button to trigger the file input */}
-                <Button
-                    className={chat.smallButton}
-                    id="upload-button"
-                    icon={<Attach24Regular />}
-                    iconPosition="before"
-                    disabled={!canChat}
-                    onClick={triggerFileInput}>
-                    {files?.length} of {maxFiles}
-                </Button>
-                <Button
-                    className={chat.smallButton}
-                    id="clear-button"
-                    disabled={!canChat}
-                    icon={<Delete24Regular />}
-                    iconPosition="before"
-                    onClick={clearAll}
-                >
-                </Button>
-                <Button
-                    className={chat.smallButton}
-                    id={"send-button"}
-                    icon={<SendRegular />}
-                    iconPosition="before"
-                    appearance="primary"
-                    onClick={handleSend} // Use handleSend function to check for file or prompt
-                    disabled={!canChat || !hasPrompt(userPrompt)} // Disable if no input or file
-                >
-                </Button>
-            </div>
-        </div>
-    );
+        {/* Button to trigger the file input */}
+        <Button
+          className={chat.smallButton}
+          id="upload-button"
+          icon={<Attach24Regular />}
+          iconPosition="before"
+          disabled={!canChat}
+          onClick={triggerFileInput}
+        >
+          {files?.length} of {maxFiles}
+        </Button>
+        <Button
+          className={chat.smallButton}
+          id="clear-button"
+          disabled={!canChat}
+          icon={<Delete24Regular />}
+          iconPosition="before"
+          onClick={clearAll}
+        ></Button>
+        <Button
+          className={chat.smallButton}
+          id={"send-button"}
+          icon={<SendRegular />}
+          iconPosition="before"
+          appearance="primary"
+          onClick={handleSend} // Use handleSend function to check for file or prompt
+          disabled={!canChat || !hasPrompt(userPrompt)} // Disable if no input or file
+        ></Button>
+      </div>
+    </div>
+  );
 }
diff --git a/src/playground/src/pages/playground/Chat.reducers.ts b/src/playground/src/pages/playground/Chat.reducers.ts
index 215e3a65..c3e6ece0 100644
--- a/src/playground/src/pages/playground/Chat.reducers.ts
+++ b/src/playground/src/pages/playground/Chat.reducers.ts
@@ -3,8 +3,9 @@ import {
   FunctionDefinition,
   GetChatCompletionsOptions,
   ChatResponseMessage,
+  ChatMessageContentItem,
 } from "@azure/openai";
-import { ChatState, INITIAL_STATE } from "./Chat.state";
+import { ChatResponseMessageExtended, ChatState, INITIAL_STATE } from "./Chat.state";
 
 const findRAIError = (
   error: Record<string, { filtered: boolean; severity: number }>
@@ -53,7 +54,7 @@ const createErrorMessage = (error: any): string => {
 type ChatAction =
   | {
       type: "chatStart";
-      payload: string;
+      payload: ChatMessageContentItem[];
     }
   | {
       type: "chatComplete";
@@ -104,12 +105,12 @@ export function reducer(state: ChatState, action: ChatAction): ChatState {
         isLoading: true,
         messages: [
           ...state.messages,
-          {
+          ...action.payload.map((p): ChatResponseMessageExtended => ({
             role: "user",
-            content: action.payload,
+            content: p.type === "text" ? p.text : p.imageUrl?.url,
             toolCalls: [],
             isError: false,
-          },
+          })),
         ],
       };
 
diff --git a/src/playground/src/pages/playground/Chat.tsx b/src/playground/src/pages/playground/Chat.tsx
index 54f80290..d0d36520 100644
--- a/src/playground/src/pages/playground/Chat.tsx
+++ b/src/playground/src/pages/playground/Chat.tsx
@@ -4,6 +4,7 @@ import { SystemCard } from "../../components/playground/SystemCard";
 import { ChatParamsCard } from "../../components/playground/ChatParamsCard";
 import { useOpenAIClientContext } from "../../providers/OpenAIProvider";
 import type {
+  ChatMessageContentItem,
   ChatRequestMessage,
   ChatResponseMessage,
   FunctionDefinition,
@@ -61,7 +62,7 @@ export const Chat = () => {
 
   const { client } = useOpenAIClientContext();
 
-  const onPromptEntered = async (prompt: string) => {
+  const onPromptEntered = async (prompt: ChatMessageContentItem[]) => {
     if (!client || !state.model) {
       return;
     }
@@ -70,13 +71,10 @@ export const Chat = () => {
     try {
       const start = Date.now();
 
-      // Parse the JSON string back into an array of message objects
-      const parsedPrompt = JSON.parse(prompt);
-
       const messages: ChatRequestMessage[] = [
         state.systemPrompt,
         ...state.messages.filter((m) => !m.isError).map(mapMessage),
-        { role: "user", content: parsedPrompt },
+        { role: "user", content: prompt },
       ];
 
       const chatCompletions = await client.getChatCompletions(