import gradio as gr import torch from transformers import ( AutoTokenizer, AutoModelForSequenceClassification ) import os from pdf_generator import ReportGenerator from news_checker import NewsChecker from dotenv import load_dotenv from spellchecker import SpellChecker import re load_dotenv() CONTRACTIONS = { # With straight apostrophe "ain't", "aren't", "can't", "couldn't", "didn't", "doesn't", "don't", "hadn't", "hasn't", "haven't", "he'd", "he'll", "he's", "i'd", "i'll", "i'm", "i've", "isn't", "let's", "mightn't", "mustn't", "shan't", "she'd", "she'll", "she's", "shouldn't", "that's", "there's", "they'd", "they'll", "they're", "they've", "we'd", "we're", "we've", "weren't", "what'll", "what're", "what's", "what've", "where's", "who'd", "who'll", "who're", "who's", "who've", "won't", "wouldn't", "you'd", "you'll", "you're", "you've", # With curly apostrophe "ain’t", "aren’t", "can’t", "couldn’t", "didn’t", "doesn’t", "don’t", "hadn’t", "hasn’t", "haven’t", "he’d", "he’ll", "he’s", "i’d", "i’ll", "i’m", "i’ve", "isn’t", "let’s", "mightn’t", "mustn’t", "shan’t", "she’d", "she’ll", "she’s", "shouldn’t", "that’s", "there’s", "they’d", "they’ll", "they’re", "they’ve", "we’d", "we’re", "we’ve", "weren’t", "what’ll", "what’re", "what’s", "what’ve", "where’s", "who’d", "who’ll", "who’re", "who’s", "who’ve", "won’t", "wouldn’t", "you’d", "you’ll", "you’re", "you’ve" } # Initialize models and tokenizers def load_models(): # Hate speech and bias detection model model_name = "facebook/roberta-hate-speech-dynabench-r4-target" hate_tokenizer = AutoTokenizer.from_pretrained(model_name) hate_model = AutoModelForSequenceClassification.from_pretrained(model_name) # Initialize spell checker spell = SpellChecker() return { 'hate_speech': (hate_model, hate_tokenizer), 'spell_check': spell } # Initialize news checker news_checker = NewsChecker() def check_text_length(text): """Check if text length is within the 1000 character limit and return character count""" char_count = len(text) if char_count > 1000: return { 'status': 'fail', 'message': f'Text length: {char_count}/1000 characters (exceeds maximum limit)' } return { 'status': 'pass', 'message': f'Text length: {char_count}/1000 characters' } def check_hate_speech_and_bias(text, model, tokenizer): try: # List of potentially problematic words and phrases bias_terms = { 'political_bias': [ 'woke', 'snowflake', 'libtard', 'conservatard', 'trumptard', 'leftist agenda', 'right-wing agenda', 'radical left', 'radical right' ], 'discriminatory': [ 'crazy', 'insane', 'psycho', 'retarded', 'schizo', 'ghetto', 'thug', 'illegal', 'normal people', 'regular people', 'third-world', 'primitive', 'savage' ], 'gender_bias': [ 'mankind', 'chairman', 'policeman', 'fireman', 'stewardess', 'manpower', 'man-made', 'guys', 'hysterical', 'drama queen' ], 'ageist': [ 'boomer', 'millennial', 'ok boomer', 'zoomer', 'gen z', 'old-timer', 'geezer', 'young people these days', 'kids these days' ], 'cultural_insensitivity': [ 'exotic', 'oriental', 'ethnic', 'colored', 'urban', 'tribal', 'backwards', 'uncivilized' ] } # Check for problematic terms found_terms = {} lower_text = text.lower() for category, terms in bias_terms.items(): found = [term for term in terms if term.lower() in lower_text] if found: found_terms[category] = found # Run the model for hate speech detection inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512) outputs = model(**inputs) predictions = torch.nn.functional.softmax(outputs.logits, dim=-1) model_score = predictions[0][1].item() # Determine the result based on both checks if model_score > 0.3 or len(found_terms) > 0: message = "Content contains potential hate speech or bias:\n\n" if found_terms: message += "Problematic language found:\n" for category, terms in found_terms.items(): category_name = category.replace('_', ' ').title() message += f"- {category_name}: {', '.join(terms)}\n" message += "\nSuggestions:\n" message += "- Consider using more inclusive and neutral language\n" message += "- Avoid stereotypes and discriminatory terms\n" message += "- Focus on specific behaviors or facts rather than generalizations\n" if model_score > 0.3: message += "\nThe content has been flagged by our AI model as potentially containing hate speech or strong bias." return { 'status': 'fail', 'message': message } elif model_score > 0.1 or any(term in lower_text for terms in bias_terms.values() for term in terms): message = "Content may contain subtle bias:\n\n" if found_terms: message += "Consider reviewing these terms:\n" for category, terms in found_terms.items(): category_name = category.replace('_', ' ').title() message += f"- {category_name}: {', '.join(terms)}\n" message += "\nSuggestions:\n" message += "- Review the flagged terms for potential unintended bias\n" message += "- Consider using more inclusive alternatives\n" return { 'status': 'warning', 'message': message } return { 'status': 'pass', 'message': 'No significant bias or hate speech detected' } except Exception as e: return { 'status': 'error', 'message': f'Error in hate speech/bias detection: {str(e)}' } def normalize_apostrophes(text): """Normalize different types of apostrophes and quotes to standard straight apostrophe""" # Replace various types of apostrophes and quotes with standard straight apostrophe return text.replace(''', "'").replace(''', "'").replace('`', "'").replace('´', "'") def check_spelling(text, spell_checker): try: # Normalize apostrophes in the entire text text = normalize_apostrophes(text) # Split text into words words = text.split() # Process words misspelled = set() for word in words: # Normalize apostrophes in the word word = normalize_apostrophes(word) # Remove surrounding punctuation but keep internal apostrophes and hyphens cleaned = re.sub(r'^[^\w\'\-]+|[^\w\'\-]+$', '', word) # Skip empty strings if not cleaned: continue # Skip if the word is in our contractions list if cleaned.lower() in CONTRACTIONS: continue # Handle hyphenated words if '-' in cleaned: parts = cleaned.split('-') # Check if each part is valid all_parts_valid = all( part.lower() in spell_checker.word_frequency for part in parts if part # Skip empty parts ) if all_parts_valid: continue # Skip special cases if (cleaned.isdigit() or # Skip numbers any(char.isdigit() for char in cleaned) or # Skip words with numbers cleaned.startswith('@') or # Skip mentions cleaned.startswith('#') or # Skip hashtags cleaned.startswith('http') or # Skip URLs cleaned.isupper() or # Skip acronyms len(cleaned) <= 1): # Skip single letters continue # Check if word is misspelled if cleaned.lower() not in spell_checker.word_frequency: misspelled.add(cleaned) if misspelled: return { 'status': 'warning', 'message': 'Misspelled words found:\n- ' + '\n- '.join(sorted(misspelled)) } return { 'status': 'pass', 'message': 'No spelling errors detected' } except Exception as e: return { 'status': 'error', 'message': f'Error in spell check: {str(e)}' } def analyze_content(text): try: # Initialize report generator report_gen = ReportGenerator() report_gen.add_header() report_gen.add_input_text(text) # Load models models = load_models() # Run all checks results = {} # 1. Length Check length_result = check_text_length(text) results['Length Check'] = length_result report_gen.add_check_result("Length Check", length_result['status'], length_result['message']) if length_result['status'] == 'fail': report_path = report_gen.save_report() return results, report_path # 2. Hate Speech / Involuntary Bias Check hate_result = check_hate_speech_and_bias(text, models['hate_speech'][0], models['hate_speech'][1]) results['Hate Speech / Involuntary Bias Check'] = hate_result report_gen.add_check_result("Hate Speech / Involuntary Bias Check", hate_result['status'], hate_result['message']) # 3. Spelling Check spell_result = check_spelling(text, models['spell_check']) results['Spelling Check'] = spell_result report_gen.add_check_result("Spelling Check", spell_result['status'], spell_result['message']) # 4. News Context Check if os.getenv('NEWS_API_KEY'): news_result = news_checker.check_content_against_news(text) else: news_result = { 'status': 'warning', 'message': 'News API key not configured. Skipping current events check.' } results['Current Events Context'] = news_result report_gen.add_check_result("Current Events Context", news_result['status'], news_result['message']) # Generate and save report report_path = report_gen.save_report() return results, report_path except Exception as e: print(f"Error in analyze_content: {str(e)}") return { 'Length Check': {'status': 'error', 'message': 'Analysis failed'}, 'Hate Speech / Involuntary Bias Check': {'status': 'error', 'message': 'Analysis failed'}, 'Spelling Check': {'status': 'error', 'message': 'Analysis failed'}, 'Current Events Context': {'status': 'error', 'message': 'Analysis failed'} }, None def format_results(results): status_symbols = { 'pass': '✅', 'fail': '❌', 'warning': '⚠️', 'error': '⚠️' } formatted_output = "" for check, result in results.items(): symbol = status_symbols.get(result['status'], '❓') formatted_output += f"{check}: {symbol}\n" if result['message']: formatted_output += f"Details: {result['message']}\n\n" return formatted_output # Gradio Interface def create_interface(): with gr.Blocks(title="Marketing Content Validator") as interface: gr.Markdown("# Marketing Content Validator") gr.Markdown ("-------------------") gr.Markdown ("Current limitations: Able to use a basic RAG model for news context check with a small dataset due to GPU limitations. Binary classification (positive/negative) might miss nuanced concerns.") gr.Markdown ("-------------------") gr.Markdown("Paste your marketing content below to check for potential issues.") valid_sample = "Introducing our vibrant collection of iPhone cases designed for young adults who love to stand out! These cases come in a variety of eye-catching colors and patterns that add a splash of personality to your device. Made from durable, sustainable materials, they offer robust protection against everyday bumps and scratches while being kind to the planet. Best of all, they're priced competitively, so you don't have to break the bank to accessorize your phone. Elevate your style and safeguard your phone with our eco-friendly, affordable iPhone cases today!" problematic_sample = "Introducing our new daily face mask, perfect for those seeking a quick skincare boost! In just 10 minutes, this mask rejuvenates your skin, leaving it looking healthier and more radiant. Formulated with natural, nourishing ingredients, it's gentle enough for everyday use and suitable for all skin types, so, perfect for all the drama queens out there! Say goodbye to dullness and hello to a refreshed complexion. Plus, our mask is eco-friendly and cruelty-free, aligning with a conscious lifestyle. Elevate your daily routine with this simple step toward glowing skin. Try our face mask today and unveil a more confident you!" with gr.Row(): valid_btn = gr.Button( "Prefill with valid marketing text sample", variant="primary", elem_classes="valid-sample-btn" ) problem_btn = gr.Button( "Prefill with a problematic marketing text sample", variant="secondary", elem_classes="problem-sample-btn" ) # Add custom CSS for the buttons gr.Markdown(""" """) with gr.Row(): with gr.Column(): input_text = gr.TextArea( label="Marketing Content", placeholder="Enter your marketing content here (max 1000 characters)...", lines=10 ) analyze_btn = gr.Button("Analyze Content") with gr.Column(): output_text = gr.TextArea( label="Analysis Results", lines=10, interactive=False ) report_output = gr.File(label="Download Report") valid_btn.click( fn=lambda: valid_sample, outputs=input_text ) problem_btn.click( fn=lambda: problematic_sample, outputs=input_text ) analyze_btn.click( fn=lambda text: ( format_results(analyze_content(text)[0]), analyze_content(text)[1] ), inputs=input_text, outputs=[output_text, report_output] ) gr.Markdown(""" ### Notes: - Maximum text length: 1000 characters - Analysis may take up to 2 minutes - Results include checks for: - Text length - Hate speech and involuntary bias - Spelling - Negative news context """) return interface # Launch the application if __name__ == "__main__": interface = create_interface() interface.launch()