# voice_emotion_classification.py import os import subprocess import sys import pkg_resources import time import tempfile import numpy as np import warnings from pathlib import Path warnings.filterwarnings("ignore") def install_package(package, version=None): package_spec = f"{package}=={version}" if version else package print(f"Installing {package_spec}...") try: subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-cache-dir", package_spec]) except subprocess.CalledProcessError as e: print(f"Failed to install {package_spec}: {e}") raise # Required packages (you may add version pins if necessary) required_packages = { "gradio": None, "torch": None, "torchaudio": None, "transformers": None, "librosa": None, "scipy": None, "matplotlib": None, "pydub": None } installed_packages = {pkg.key for pkg in pkg_resources.working_set} for package, version in required_packages.items(): if package not in installed_packages: install_package(package, version) # Now import all necessary packages import gradio as gr import torch import torchaudio import librosa import matplotlib.pyplot as plt from matplotlib.colors import LinearSegmentedColormap from pydub import AudioSegment import scipy import io from transformers import pipeline, AutoFeatureExtractor, AutoModelForAudioClassification from pathlib import Path import matplotlib matplotlib.use('Agg') # Use non-interactive backend # Define emotion labels, tone mapping, and descriptions EMOTION_DESCRIPTIONS = { "angry": "Voice shows irritation, hostility, or aggression. Tone may be harsh, loud, or intense.", "disgust": "Voice expresses revulsion or strong disapproval. Tone may sound repulsed or contemptuous.", "fear": "Voice reveals anxiety, worry, or dread. Tone may be shaky, hesitant, or tense.", "happy": "Voice conveys joy, pleasure, or positive emotions. Tone is often bright, energetic, and uplifted.", "neutral": "Voice lacks strong emotional signals. Tone is even, moderate, and relatively flat.", "sad": "Voice expresses sorrow, unhappiness, or melancholy. Tone may be quiet, heavy, or subdued.", "surprise": "Voice reflects unexpected reactions. Tone may be higher pitched, quick, or energetic." } # Here we map emotion to a generalized tone (for example, negative or positive) TONE_MAPPING = { "positive": ["happy", "surprise"], "neutral": ["neutral"], "negative": ["angry", "sad", "fear", "disgust"] } # Some Hugging Face models return short labels (e.g., "hap", "ang", etc.). # This mapping will ensure they're translated into our full canonical labels. MODEL_TO_EMOTION_MAP = { "hap": "happy", "ang": "angry", "sad": "sad", "dis": "disgust", "fea": "fear", "neu": "neutral", "sur": "surprise" } # Global variable for the emotion classifier audio_emotion_classifier = None def load_emotion_model(): """Load the emotion classification model once and cache it.""" global audio_emotion_classifier if audio_emotion_classifier is None: try: print("Loading emotion classification model...") # Using the Hugging Face pipeline with the new model that classifies speech emotion model_name = "superb/hubert-large-superb-er" audio_emotion_classifier = pipeline("audio-classification", model=model_name) print("Emotion classification model loaded successfully") return True except Exception as e: print(f"Error loading emotion model: {e}") return False return True def convert_audio_to_wav(audio_file): """Convert the uploaded audio to WAV format.""" try: audio = AudioSegment.from_file(audio_file) with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_wav: wav_path = temp_wav.name audio.export(wav_path, format="wav") return wav_path except Exception as e: print(f"Error converting audio: {e}") return None def analyze_audio_emotions(audio_file, progress=gr.Progress(), chunk_duration=5): """ Analyze emotions in an audio file by processing it in chunks. Returns a visualization, processed audio path, summary, and detailed results. """ if not load_emotion_model(): return None, "Failed to load emotion classification model. Please check console for details." # If the file is already a WAV, use it directly; else convert it. if audio_file.endswith('.wav'): audio_path = audio_file else: audio_path = convert_audio_to_wav(audio_file) if not audio_path: return None, "Failed to process audio file. Unsupported format or corrupted file." try: # Load the audio using librosa audio_data, sample_rate = librosa.load(audio_path, sr=16000) duration = len(audio_data) / sample_rate # Process in chunks for long files chunk_samples = int(chunk_duration * sample_rate) num_chunks = max(1, int(np.ceil(len(audio_data) / chunk_samples))) all_emotions = [] time_points = [] for i in range(num_chunks): progress((i + 1) / num_chunks, "Analyzing audio emotions...") start_idx = i * chunk_samples end_idx = min(start_idx + chunk_samples, len(audio_data)) chunk = audio_data[start_idx:end_idx] # Skip too-short chunks (<0.5 seconds) if len(chunk) < 0.5 * sample_rate: continue # Create a temporary file for this audio chunk with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_chunk: chunk_path = temp_chunk.name scipy.io.wavfile.write(chunk_path, sample_rate, (chunk * 32767).astype(np.int16)) # Get emotion classification results on this chunk results = audio_emotion_classifier(chunk_path) os.unlink(chunk_path) # Remove the temporary file all_emotions.append(results) time_points.append((start_idx / sample_rate, end_idx / sample_rate)) # Generate visualization and summary fig, detailed_results = generate_emotion_timeline(all_emotions, time_points, duration) with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_img: img_path = temp_img.name fig.savefig(img_path, dpi=100, bbox_inches='tight') plt.close(fig) summary = generate_emotion_summary(all_emotions, time_points) return img_path, audio_path, summary, detailed_results except Exception as e: print(f"Error analyzing audio: {e}") import traceback traceback.print_exc() return None, None, f"Error analyzing audio: {str(e)}", None def generate_emotion_timeline(all_emotions, time_points, duration): """ Generate a bar chart visualization of emotion percentages with tone analysis. Returns the matplotlib figure and a list of detailed results. """ # All possible emotion labels from our dictionary emotion_labels = list(EMOTION_DESCRIPTIONS.keys()) # We'll accumulate counts based on our canonical labels (e.g., "happy", "angry"). emotion_counts = {} for emotions in all_emotions: if not emotions: continue # The pipeline returns items like {"label": "Hap", "score": 0.95}, etc. top_emotion = max(emotions, key=lambda x: x['score']) # Normalize the label from the model to a canonical label used in EMOTION_DESCRIPTIONS raw_label = top_emotion['label'].lower().strip() # e.g., "hap", "ang", ... canonical_label = MODEL_TO_EMOTION_MAP.get(raw_label, raw_label) # If there's no mapping, we leave it as raw_label. # But typically, it should be one of "happy", "angry", "disgust", "fear", "sad", "neutral", "surprise". # Count how many times each canonical label appears emotion_counts[canonical_label] = emotion_counts.get(canonical_label, 0) + 1 total_chunks = len(all_emotions) emotion_percentages = { e: (count / total_chunks * 100) for e, count in emotion_counts.items() } # Create empty percentages for emotions that didn't appear for label in emotion_labels: if label not in emotion_percentages: emotion_percentages[label] = 0.0 # Sort emotions by percentage sorted_emotions = sorted(emotion_percentages.items(), key=lambda x: x[1], reverse=True) # Create the bar chart with subplots: one for emotions and one for tone fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10), height_ratios=[3, 1], gridspec_kw={'hspace': 0.3}) # Capitalize each label for a nice display emotions = [item[0].capitalize() for item in sorted_emotions] percentages = [item[1] for item in sorted_emotions] # Custom colors for emotions (enough for 7 emotions) colors = ['red', 'brown', 'purple', 'green', 'gray', 'blue', 'orange'] if len(emotions) <= len(colors): bar_colors = colors[:len(emotions)] else: # fallback if there's more emotions than colors bar_colors = colors + ['#666666'] * (len(emotions) - len(colors)) # Plot emotion bars bars = ax1.bar(emotions, percentages, color=bar_colors) # Add percentage labels on top of each bar for bar in bars: height = bar.get_height() ax1.annotate(f'{height:.1f}%', xy=(bar.get_x() + bar.get_width() / 2, height), xytext=(0, 3), # 3 points vertical offset textcoords="offset points", ha='center', va='bottom') ax1.set_ylim(0, 100) # Fixed 100% scale ax1.set_ylabel('Percentage (%)') ax1.set_title('Emotion Distribution') ax1.grid(axis='y', linestyle='--', alpha=0.7) # Calculate tone percentages based on the canonical labels we found tone_percentages = {"positive": 0, "neutral": 0, "negative": 0} for emotion_label, percentage in emotion_percentages.items(): for tone, emotions_list in TONE_MAPPING.items(): if emotion_label in emotions_list: tone_percentages[tone] += percentage # Plot tone bars tones = list(tone_percentages.keys()) tone_values = list(tone_percentages.values()) tone_colors = {'positive': 'green', 'neutral': 'gray', 'negative': 'red'} tone_bars = ax2.bar(tones, tone_values, color=[tone_colors[t] for t in tones]) # Add percentage labels on tone bars for bar in tone_bars: height = bar.get_height() if height > 0: # Only add label if there's a visible bar ax2.annotate(f'{height:.1f}%', xy=(bar.get_x() + bar.get_width() / 2, height), xytext=(0, 3), textcoords="offset points", ha='center', va='bottom') ax2.set_ylim(0, 100) ax2.set_ylabel('Percentage (%)') ax2.set_title('Tone Analysis') ax2.grid(axis='y', linestyle='--', alpha=0.7) plt.tight_layout() # Generate a more detailed time-segmented result detailed_results = [] for idx, (emotions, (start_time, end_time)) in enumerate(zip(all_emotions, time_points)): if not emotions: continue top_emotion = max(emotions, key=lambda x: x['score']) raw_label = top_emotion['label'].lower().strip() canonical_label = MODEL_TO_EMOTION_MAP.get(raw_label, raw_label) # Determine the tone for this emotion # (based on canonical_label rather than the raw model label) tone = next((t for t, e_list in TONE_MAPPING.items() if canonical_label in e_list), "unknown") detailed_results.append({ 'Time Range': f"{start_time:.1f}s - {end_time:.1f}s", 'Emotion': canonical_label, 'Tone': tone.capitalize(), 'Confidence': f"{top_emotion['score']:.2f}", 'Description': EMOTION_DESCRIPTIONS.get(canonical_label, "") }) return fig, detailed_results def generate_emotion_summary(all_emotions, time_points): """ Create a summary text from the emotion analysis. Counts occurrences and computes percentages of the dominant emotion. """ if not all_emotions: return "No emotional content detected." emotion_counts = {} total_chunks = len(all_emotions) for emotions in all_emotions: if not emotions: continue top_emotion = max(emotions, key=lambda x: x['score']) # Normalize the label raw_label = top_emotion['label'].lower().strip() canonical_label = MODEL_TO_EMOTION_MAP.get(raw_label, raw_label) emotion_counts[canonical_label] = emotion_counts.get(canonical_label, 0) + 1 emotion_percentages = { e: (count / total_chunks * 100) for e, count in emotion_counts.items() } if not emotion_percentages: return "No emotional content detected." # Find the dominant emotion (highest percentage) dominant_emotion = max(emotion_percentages.items(), key=lambda x: x[1])[0] summary = f"### Voice Emotion Analysis Summary\n\n" summary += f"**Dominant emotion:** {dominant_emotion.capitalize()} ({emotion_percentages[dominant_emotion]:.1f}%)\n\n" summary += f"**Description:** {EMOTION_DESCRIPTIONS.get(dominant_emotion, '')}\n\n" summary += "**Emotion distribution:**\n" for emotion, percentage in sorted(emotion_percentages.items(), key=lambda x: x[1], reverse=True): summary += f"- {emotion.capitalize()}: {percentage:.1f}%\n" summary += "\n**Interpretation:** The voice predominantly expresses {0} emotion".format(dominant_emotion) return summary def record_audio(audio): """Save recorded audio and analyze emotions.""" try: with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_file: audio_path = temp_file.name with open(audio_path, 'wb') as f: f.write(audio) return audio_path except Exception as e: print(f"Error saving recorded audio: {e}") return None def process_audio(audio_file, progress=gr.Progress()): """Process the audio file and analyze emotions.""" if audio_file is None: return None, None, "No audio file provided.", None img_path, processed_audio, summary, results = analyze_audio_emotions(audio_file, progress) if img_path is None: return None, None, "Failed to analyze audio emotions.", None return img_path, processed_audio, summary, results # Create Gradio interface with gr.Blocks(title="Voice Emotion Analysis System") as demo: gr.Markdown(""" # 🎙️ Voice Emotion Analysis System This app analyzes the emotional content of voice recordings. It detects emotions including: * 😡 **Anger** * 🤢 **Disgust** * 😨 **Fear** * 😊 **Happiness** * 😐 **Neutral** * 😢 **Sadness** * 😲 **Surprise** And provides a detailed analysis and timeline. """) with gr.Tabs(): with gr.TabItem("Upload Audio"): with gr.Row(): with gr.Column(scale=1): audio_input = gr.Audio( label="Upload Audio File", type="filepath", sources=["upload"] ) process_btn = gr.Button("Analyze Voice Emotions") with gr.Column(scale=2): emotion_timeline = gr.Image(label="Emotion Timeline", show_label=True) with gr.Row(): audio_playback = gr.Audio(label="Processed Audio", show_label=True) emotion_summary = gr.Markdown(label="Emotion Summary") with gr.Row(): emotion_results = gr.DataFrame( headers=["Time Range", "Emotion", "Tone", "Confidence", "Description"], label="Detailed Emotion Analysis" ) process_btn.click( fn=process_audio, inputs=[audio_input], outputs=[emotion_timeline, audio_playback, emotion_summary, emotion_results] ) with gr.TabItem("Record Voice"): with gr.Row(): with gr.Column(scale=1): record_input = gr.Audio( label="Record Your Voice", sources=["microphone"], type="filepath" ) analyze_btn = gr.Button("Analyze Recording") with gr.Column(scale=2): rec_emotion_timeline = gr.Image(label="Emotion Timeline", show_label=True) with gr.Row(): rec_audio_playback = gr.Audio(label="Processed Audio", show_label=True) rec_emotion_summary = gr.Markdown(label="Emotion Summary") with gr.Row(): rec_emotion_results = gr.DataFrame( headers=["Time Range", "Emotion", "Tone", "Confidence", "Description"], label="Detailed Emotion Analysis" ) analyze_btn.click( fn=process_audio, inputs=[record_input], outputs=[rec_emotion_timeline, rec_audio_playback, rec_emotion_summary, rec_emotion_results] ) gr.Markdown(""" ### How to Use 1. **Upload Audio Tab:** Upload an audio file and click "Analyze Voice Emotions". 2. **Record Voice Tab:** Record your voice and click "Analyze Recording". **Tips:** - Use clear recordings with minimal background noise. - Longer recordings yield more consistent results. """) def initialize_app(): print("Initializing voice emotion analysis app...") if load_emotion_model(): print("Emotion model loaded successfully!") else: print("Failed to load emotion model.") if __name__ == "__main__": initialize_app() demo.launch()