Testing merge

.gitattributes

@@ -5,3 +5,4 @@
 *.mp4 filter=lfs diff=lfs merge=lfs -text
 *.unity filter=lfs diff=lfs merge=lfs -text
 *.max filter=lfs diff=lfs merge=lfs -text
+

.gitignore

@@ -26,6 +26,9 @@
 # Visual Studio cache directory
 .vs/
 
+Assets/StreamingAssets/*.json
+
+
 # Gradle cache directory
 .gradle/
 
@@ -70,3 +73,4 @@ crashlytics-build.properties
 # Temporary auto-generated Android Assets
 /[Aa]ssets/[Ss]treamingAssets/aa.meta
 /[Aa]ssets/[Ss]treamingAssets/aa/*
+Assets/StreamingAssets/*.json 

Assets/GeminiManager/Example/Speaking.cs

@@ -0,0 +1,63 @@
+using UnityEngine;
+using Google.Cloud.TextToSpeech.V1;
+using System;
+using System.IO;
+
+public class Speaking : MonoBehaviour
+{
+
+    void Start()
+    {
+        // Setup authentication
+        string credentialPath = Path.Combine(Application.streamingAssetsPath, "sentimental-sonar-55e824d61f3b.json");
+        Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialPath);
+
+        // Create the client
+        var client = TextToSpeechClient.Create();
+
+        // Input text to synthesize
+        var input = new SynthesisInput
+        {
+            Text = "Hello! This is Google Cloud Text to Speech working in Unity!"
+        };
+
+        // Select the voice and language
+        var voice = new VoiceSelectionParams
+        {
+            LanguageCode = "en-US",
+            SsmlGender = SsmlVoiceGender.Female
+        };
+
+        // Audio output configuration
+        var config = new AudioConfig
+        {
+            AudioEncoding = AudioEncoding.Linear16  // WAV format
+        };
+
+        // Synthesize speech
+        var response = client.SynthesizeSpeech(input, voice, config);
+
+        // Save audio to a file
+        string outputPath = Path.Combine(Application.persistentDataPath, "tts_output.wav");
+        File.WriteAllBytes(outputPath, response.AudioContent.ToByteArray());
+        Debug.Log("TTS audio saved to: " + outputPath);
+
+        // Play the audio
+        StartCoroutine(PlayAudio(outputPath));
+        Debug.Log("We are playing audio now!");
+    }
+
+    // Play audio from file
+    System.Collections.IEnumerator PlayAudio(string path)
+    {
+        using (var www = new WWW("file://" + path))
+        {
+            yield return www;
+            AudioClip clip = www.GetAudioClip(false, true);
+            var audioSource = GetComponent<AudioSource>();
+            audioSource.clip = clip;
+            audioSource.Play();
+        }
+    }
+
+}

Assets/GeminiManager/Scripts/UnityAndGeminiV3.cs

@@ -5,6 +5,8 @@
 using TMPro;
 using System.IO; 
 using System;
+using System.Linq;
+using Unity.VisualScripting;
 
 [System.Serializable]
 public class UnityAndGeminiKey
@@ -28,7 +30,6 @@ public class TextPart
     public string text;
 }
 
-// Image-capable part
 [System.Serializable]
 public class ImagePart
 {
@@ -75,7 +76,7 @@ public class ImageResponse
 }
 
 
-// For text requests
+
 [System.Serializable]
 public class ChatRequest
 {
@@ -99,6 +100,10 @@ public class UnityAndGeminiV3: MonoBehaviour
     public TMP_Text uiText;
     public string botInstructions;
     private TextContent[] chatHistory;
+    public string reply;
+    public Transcription TranscriptionManager;
+    public SlideshowManager SlidesManager;
+    public TimeManager TimeManager;
 
 
     [Header("Prompt Function")]
@@ -109,7 +114,7 @@ public class UnityAndGeminiV3: MonoBehaviour
     public Material skyboxMaterial; 
 
     [Header("Media Prompt Function")]
-    // Receives files with a maximum of 20 MB
+
     public string mediaFilePath = "";
     public string mediaPrompt = "";
     public enum MediaType
@@ -145,14 +150,35 @@ public string GetMimeTypeString()
 
     void Start()
     {
+
+        Debug.Log("Gemini Scene Functionality is inactive due to migration of command to be run by other classes, to retrieve previous functionality please copy contents of runGemini() into Start() of this .cs file");
+
+    }
+
+    //The below method is what I want you guys to test 
+
+    public void runGemini()
+    {
+
         UnityAndGeminiKey jsonApiKey = JsonUtility.FromJson<UnityAndGeminiKey>(jsonApi.text);
-        apiKey = jsonApiKey.key;   
+        apiKey = jsonApiKey.key;
+        TranscriptionManager = GetComponent<Transcription>();
         chatHistory = new TextContent[] { };
-        if (prompt != ""){StartCoroutine( SendPromptRequestToGemini(prompt));};
-        if (imagePrompt != ""){StartCoroutine( SendPromptRequestToGeminiImageGenerator(imagePrompt));};
-        if (mediaPrompt != "" && mediaFilePath != ""){StartCoroutine(SendPromptMediaRequestToGemini(mediaPrompt, mediaFilePath));};
+
+        // comment just the line below this comment to remove the performance data eval section if its buggy and just wanting to run Gemini feedback mode from user to ask questions and use runGemini() after presentation is over
+        StartCoroutine(SendPerformanceDataToGemini());
+
+        if (prompt != "") { StartCoroutine(SendPromptRequestToGemini(prompt)); }
+        ;
+        if (imagePrompt != "") { StartCoroutine(SendPromptRequestToGeminiImageGenerator(imagePrompt)); }
+        ;
+        if (mediaPrompt != "" && mediaFilePath != "") { StartCoroutine(SendPromptMediaRequestToGemini(mediaPrompt, mediaFilePath)); }
+        ;
+
     }
 
+
+
     private IEnumerator SendPromptRequestToGemini(string promptText)
     {
         string url = $"{apiEndpoint}?key={apiKey}";
@@ -176,14 +202,96 @@ private IEnumerator SendPromptRequestToGemini(string promptText)
                 TextResponse response = JsonUtility.FromJson<TextResponse>(www.downloadHandler.text);
                 if (response.candidates.Length > 0 && response.candidates[0].content.parts.Length > 0)
                     {
-                        //This is the response to your request
+
                         string text = response.candidates[0].content.parts[0].text;
                         Debug.Log(text);
                     }
                 else
                 {
                     Debug.Log("No text found.");
+
+         url = $"{apiEndpoint}?key={apiKey}";                }
+            }
+        }
+    }
+
+    public string TimeManagerDataConcatenate()
+    {
+
+        string result = "";
+
+        Dictionary<string, string> data = TimeManager.CollectedData();
+        foreach(KeyValuePair<string, string> entry in data)
+        {
+
+            result += entry.Key + ":" + entry.Value;
+
+        }
+
+        return result;
+
+    }
+
+    private IEnumerator SendPerformanceDataToGemini()
+    {
+
+        float[] slideTimes = SlidesManager.getSlideTime();
+        string promptText = "";
+
+        promptText += "Using the inputted data regarding the user's performance while public speaking: please evaluate their performance. Do not include any symbols, when you read an s, that means seconds. ";
+
+
+        promptText += "For Presentation, here are multiple time values regarding user's eye contact: " + TimeManagerDataConcatenate() + " elapsedTime is time spent total during the presentation, audienceTime is time spent total looking at the audience, projectorTime is time spent looking at the projector, lookingAtNothing_Num is the amount of times the user looks at nothing for more than 6 seconds, and disengagedHands_Num is the number of times the user did nothing with their hands during presentation for 10+ seconds straight";
+
+           promptText += "Please proceed and go over each of the criteria separately and explain what they did well, but do not type any symbols. please say what could be improved, if anything. " +
+            "Finally, at the end of it ask them if they have any questions about your evaluation or if they'd like any advice. Keep your response short.";
+
+        string url = $"{apiEndpoint}?key={apiKey}";
+
+
+        string jsonData = "{\"contents\": [{\"parts\": [{\"text\": \"{" + promptText + "}\"}]}]}";
+
+        byte[] jsonToSend = new System.Text.UTF8Encoding().GetBytes(jsonData);
+
+        // Create a UnityWebRequest with the JSON data
+        using (UnityWebRequest www = new UnityWebRequest(url, "POST"))
+        {
+            www.uploadHandler = new UploadHandlerRaw(jsonToSend);
+            www.downloadHandler = new DownloadHandlerBuffer();
+            www.SetRequestHeader("Content-Type", "application/json");
+
+            yield return www.SendWebRequest();
+
+            if (www.result != UnityWebRequest.Result.Success)
+            {
+                Debug.LogError(www.error);
+            }
+            else
+            {
+                Debug.Log("Request complete!");
+                TextResponse response = JsonUtility.FromJson<TextResponse>(www.downloadHandler.text);
+                if (response.candidates.Length > 0 && response.candidates[0].content.parts.Length > 0)
+                {
+                    //This is the response to your request
+                    string text = response.candidates[0].content.parts[0].text;
+                    if (text == "")
+                    {
+                        Debug.Log("error");
+                    }
+                    else
+                    {
+                        StartCoroutine(TranscriptionManager.SynthesizeSpeech(text));
+                    }
+                        Debug.Log(text);
+                }
+                else
+                {
+                    Debug.Log("No text found.");
+
+                    url = $"{apiEndpoint}?key={apiKey}";
                 }
+
+
             }
         }
     }
@@ -194,6 +302,11 @@ public void SendChat()
         StartCoroutine( SendChatRequestToGemini(userMessage));
     }
 
+    public void SendUserMessage(string message)
+    {
+        StartCoroutine(SendChatRequestToGemini(message));
+    }
+
     private IEnumerator SendChatRequestToGemini(string newMessage)
     {
 
@@ -241,9 +354,14 @@ private IEnumerator SendChatRequestToGemini(string newMessage)
                 TextResponse response = JsonUtility.FromJson<TextResponse>(www.downloadHandler.text);
                 if (response.candidates.Length > 0 && response.candidates[0].content.parts.Length > 0)
                     {
-                        //This is the response to your request
-                        string reply = response.candidates[0].content.parts[0].text;
-                        TextContent botContent = new TextContent
+
+                         reply = response.candidates[0].content.parts[0].text;
+                    if (TranscriptionManager != null)
+                    {
+                        StartCoroutine(TranscriptionManager.SynthesizeSpeech(reply));
+                    }
+
+                    TextContent botContent = new TextContent
                         {
                             role = "model",
                             parts = new TextPart[]
@@ -272,7 +390,7 @@ private IEnumerator SendPromptRequestToGeminiImageGenerator(string promptText)
     {
         string url = $"{imageEndpoint}?key={apiKey}";
 
-        // Create the proper JSON structure with model specification
+
         string jsonData = $@"{{
             ""contents"": [{{
                 ""parts"": [{{
@@ -286,7 +404,7 @@ private IEnumerator SendPromptRequestToGeminiImageGenerator(string promptText)
 
         byte[] jsonToSend = new System.Text.UTF8Encoding().GetBytes(jsonData);
 
-        // Create a UnityWebRequest with the JSON data
+
         using (UnityWebRequest www = new UnityWebRequest(url, "POST"))
         {
             www.uploadHandler = new UploadHandlerRaw(jsonToSend);
@@ -302,9 +420,9 @@ private IEnumerator SendPromptRequestToGeminiImageGenerator(string promptText)
             else 
             {
                 Debug.Log("Request complete!");
-                Debug.Log("Full response: " + www.downloadHandler.text); // Log full response for debugging
+                Debug.Log("Full response: " + www.downloadHandler.text); 
 
-                // Parse the JSON response
+
                 try 
                 {
                     ImageResponse response = JsonUtility.FromJson<ImageResponse>(www.downloadHandler.text);
@@ -321,10 +439,10 @@ private IEnumerator SendPromptRequestToGeminiImageGenerator(string promptText)
                             }
                             else if (part.inlineData != null && !string.IsNullOrEmpty(part.inlineData.data))
                             {
-                                // This is the base64 encoded image data
+
                                 byte[] imageBytes = System.Convert.FromBase64String(part.inlineData.data);
 
-                                // Create a texture from the bytes
+
                                 Texture2D tex = new Texture2D(2, 2);
                                 tex.LoadImage(imageBytes);
                                 byte[] pngBytes = tex.EncodeToPNG();
@@ -333,18 +451,18 @@ private IEnumerator SendPromptRequestToGeminiImageGenerator(string promptText)
                                 Debug.Log("Saved to: " + path);
                                 Debug.Log("Image received successfully!");
 
-                                // Load the saved image back as Texture2D
+
                                 string imagePath = Path.Combine(Application.persistentDataPath, "gemini-image.png");
 
                                 Texture2D panoramaTex = new Texture2D(2, 2);
                                 panoramaTex.LoadImage(File.ReadAllBytes(imagePath));
 
                                 Texture2D properlySizedTex = ResizeTexture(panoramaTex, 1024, 512);
 
-                                // Apply to a panoramic skybox material
+
                                 if (skyboxMaterial != null)
                                 {
-                                    // Switch to panoramic shader
+
                                     skyboxMaterial.shader = Shader.Find("Skybox/Panoramic");
                                     skyboxMaterial.SetTexture("_MainTex", properlySizedTex);
                                     DynamicGI.UpdateEnvironment();
@@ -408,7 +526,7 @@ Texture2D ResizeTexture(Texture2D source, int newWidth, int newHeight)
 
     private IEnumerator SendPromptMediaRequestToGemini(string promptText, string mediaPath)
     {
-        // Read video file and convert to base64
+
         byte[] mediaBytes = File.ReadAllBytes(mediaPath);
         string base64Media = System.Convert.ToBase64String(mediaBytes);
 
@@ -438,11 +556,9 @@ private IEnumerator SendPromptMediaRequestToGemini(string promptText, string med
         }}";
 
 
-        // Serialize the request into JSON
-        // string jsonData = JsonUtility.ToJson(jsonBody);
+
         Debug.Log("Sending JSON: " + jsonBody); // For debugging
 
-        // byte[] jsonToSend = new System.Text.UTF8Encoding().GetBytes(jsonData);
 
         byte[] jsonToSend = new System.Text.UTF8Encoding().GetBytes(jsonBody);