Voice Interface Testing: QA for the Conversational Era

from voice_testing import SpeechRecognitionTester
import pytest

class TestSpeechRecognition:
    def setup_method(self):
        self.tester = SpeechRecognitionTester(
            service='alexa',
            locale='en-US'
        )

    def test_clear_speech_recognition(self):
        """Test recognition with studio-quality audio"""
        result = self.tester.recognize_audio(
            audio_file='test_data/clear_speech/turn_on_lights.wav',
            expected_text='turn on the lights'
        )

        assert result.accuracy >= 0.95
        assert result.word_error_rate <= 0.05

    @pytest.mark.parametrize('noise_type,snr', [
        ('white_noise', 10),
        ('traffic', 5),
        ('restaurant', 0),
        ('music', -5)
    ])
    def test_noisy_environment_recognition(self, noise_type, snr):
        """Test recognition with background noise at various SNR levels"""
        result = self.tester.recognize_with_noise(
            clean_audio='test_data/commands/set_timer.wav',
            noise_type=noise_type,
            signal_to_noise_ratio=snr,
            expected_text='set a timer for five minutes'
        )

        # Acceptance criteria varies by SNR
        if snr >= 5:
            assert result.accuracy >= 0.85
        elif snr >= 0:
            assert result.accuracy >= 0.70
        else:
            assert result.accuracy >= 0.50

const VoiceTester = require('voice-qa-framework');

describe('Accent Recognition Tests', () => {
  const tester = new VoiceTester({
    platform: 'google-assistant',
    language: 'en'
  });

  const accents = [
    { name: 'General American', audio: 'test_data/accents/gen_am.wav' },
    { name: 'British RP', audio: 'test_data/accents/british_rp.wav' },
    { name: 'Indian English', audio: 'test_data/accents/indian.wav' },
    { name: 'Australian', audio: 'test_data/accents/australian.wav' },
    { name: 'Scottish', audio: 'test_data/accents/scottish.wav' }
  ];

  accents.forEach(accent => {
    it(`should recognize "${accent.name}" accent`, async () => {
      const result = await tester.recognizeSpeech({
        audioFile: accent.audio,
        expectedTranscript: 'what is the weather today',
        tolerance: 0.15  // Allow 15% word error rate
      });

      expect(result.recognized).toBe(true);
      expect(result.wordErrorRate).toBeLessThan(0.15);

      // Log for accent performance tracking
      await tester.logMetric({
        metric: 'accent_accuracy',
        accent: accent.name,
        wer: result.wordErrorRate
      });
    });
  });
});

from nlu_testing import IntentTester

class TestIntentRecognition:
    def setup_method(self):
        self.tester = IntentTester(
            nlu_model='skill_handler_v2',
            confidence_threshold=0.75
        )

    def test_single_intent_variations(self):
        """Test intent recognition across natural language variations"""
        test_cases = [
            # Intent: set_timer
            ("set a timer for 5 minutes", "set_timer", {"duration": "5 minutes"}),
            ("start a 5 minute timer", "set_timer", {"duration": "5 minutes"}),
            ("timer for five minutes please", "set_timer", {"duration": "5 minutes"}),
            ("remind me in 5 minutes", "set_timer", {"duration": "5 minutes"}),

            # Intent: play_music
            ("play some jazz", "play_music", {"genre": "jazz"}),
            ("I want to hear jazz music", "play_music", {"genre": "jazz"}),
            ("put on some jazz", "play_music", {"genre": "jazz"}),
        ]

        for utterance, expected_intent, expected_slots in test_cases:
            result = self.tester.classify_intent(utterance)

            assert result.intent == expected_intent, \
                f"Failed on: '{utterance}' - got {result.intent}"
            assert result.confidence >= 0.75
            assert result.slots == expected_slots

    def test_ambiguous_intent_handling(self):
        """Test handling of ambiguous utterances"""
        result = self.tester.classify_intent("play something")

        # Should either ask for clarification or make reasonable assumption
        assert (
            result.intent == "clarification_needed" or
            (result.intent == "play_music" and result.confidence >= 0.65)
        )

import com.voiceqa.ConversationTester;
import org.junit.jupiter.api.Test;

public class MultiTurnConversationTest {
    private ConversationTester tester = new ConversationTester("alexa-skill-pizza-order");

    @Test
    public void testPizzaOrderingConversation() {
        // Turn 1: Intent initiation
        ConversationState state = tester.startConversation();
        Response response1 = state.sendUtterance("I want to order a pizza");

        assertEquals("order_pizza", response1.getIntent());
        assertTrue(response1.getSpeech().contains("What size"));

        // Turn 2: Provide size
        Response response2 = state.sendUtterance("large");

        assertEquals("order_pizza.provide_size", response2.getIntent());
        assertEquals("large", state.getSlot("size"));
        assertTrue(response2.getSpeech().contains("toppings"));

        // Turn 3: Provide toppings
        Response response3 = state.sendUtterance("pepperoni and mushrooms");

        assertEquals(List.of("pepperoni", "mushrooms"), state.getSlot("toppings"));
        assertTrue(response3.getSpeech().contains("confirm"));

        // Turn 4: Confirm order
        Response response4 = state.sendUtterance("yes confirm");

        assertEquals("order_confirmed", response4.getIntent());
        assertTrue(state.isConversationComplete());

        // Verify conversation context was maintained
        assertEquals("large", state.getFinalSlot("size"));
        assertNotNull(state.getFinalSlot("order_id"));
    }

    @Test
    public void testContextSwitchingInConversation() {
        ConversationState state = tester.startConversation();

        // Start pizza order
        state.sendUtterance("order a pizza");
        state.sendUtterance("large");

        // Context switch - user asks different question
        Response response = state.sendUtterance("what time do you close");

        // Should handle context switch gracefully
        assertEquals("store_hours", response.getIntent());
        assertTrue(response.getSpeech().contains("close"));

        // Return to pizza order
        Response returnResponse = state.sendUtterance("continue my order");

        // Should restore previous context
        assertEquals("large", state.getSlot("size"));
        assertTrue(returnResponse.getSpeech().contains("topping"));
    }
}

import pandas as pd
from voice_testing import MultilingualTester

class TestMultilingualSupport:
    LANGUAGES = ['en-US', 'en-GB', 'es-ES', 'es-MX', 'fr-FR', 'de-DE', 'ja-JP', 'zh-CN']

    def test_command_recognition_all_languages(self):
        """Test core commands across all supported languages"""
        tester = MultilingualTester()

        # Define test commands with translations
        commands = {
            'en-US': 'turn on the lights',
            'en-GB': 'turn on the lights',
            'es-ES': 'enciende las luces',
            'es-MX': 'prende las luces',
            'fr-FR': 'allume les lumières',
            'de-DE': 'schalte das licht ein',
            'ja-JP': '電気をつけて',
            'zh-CN': '打开灯'
        }

        results = []

        for lang, command in commands.items():
            audio_file = f'test_data/multilingual/{lang}/lights_on.wav'
            result = tester.test_command(
                locale=lang,
                audio_file=audio_file,
                expected_intent='turn_on_lights',
                expected_text=command
            )

            results.append({
                'language': lang,
                'accuracy': result.accuracy,
                'latency_ms': result.latency_ms,
                'intent_confidence': result.intent_confidence
            })

        # Generate report
        df = pd.DataFrame(results)
        print(df)

        # Assert minimum quality thresholds
        assert df['accuracy'].min() >= 0.85, "Some languages below accuracy threshold"
        assert df['latency_ms'].mean() <= 500, "Average latency too high"

describe('Code-Switching Tests', () => {
  const tester = new MultilingualVoiceTester();

  it('should handle Spanish-English code-switching', async () => {
    // "Play mi canción favorita" (Play my favorite song)
    const result = await tester.processUtterance({
      audio: 'test_data/code_switching/spanglish_play.wav',
      primaryLanguage: 'es-US',
      expectedIntent: 'play_music',
      expectedSlots: {
        playlist: 'favorites'
      }
    });

    expect(result.intentMatched).toBe(true);
    expect(result.handledCodeSwitch).toBe(true);
  });

  it('should handle Hinglish (Hindi-English) code-switching', async () => {
    // "Alarm set karo for 7 AM"
    const result = await tester.processUtterance({
      audio: 'test_data/code_switching/hinglish_alarm.wav',
      primaryLanguage: 'hi-IN',
      expectedIntent: 'set_alarm',
      expectedSlots: {
        time: '07:00'
      }
    });

    expect(result.intentMatched).toBe(true);
  });
});

# Voice Testing Architecture
Components:
  Speech Synthesis:

    - Google Cloud TTS
    - Amazon Polly
    - Azure Speech Services
    Purpose: Generate test audio with controlled parameters

  Speech Recognition Services:

    - Alexa Voice Service (AVS)
    - Google Cloud Speech-to-Text
    - Azure Speech SDK
    Purpose: Test STT accuracy

  NLU Testing:

    - Rasa NLU Test
    - Dialogflow Test Console
    - Custom NLU validators
    Purpose: Intent and entity validation

  Acoustic Testing:

    - Audio manipulation libraries (pydub, sox)
    - Noise injection
    - Reverberation simulation
    Purpose: Environmental condition testing

  Conversation Management:

    - State machine testing
    - Context tracking
    - Session management validation
    Purpose: Multi-turn conversation testing

# voice_testing_framework.py
from dataclasses import dataclass
from typing import List, Dict, Optional
import boto3
from google.cloud import speech, texttospeech
import numpy as np
from pydub import AudioSegment

@dataclass
class VoiceTestResult:
    transcript: str
    expected_transcript: str
    accuracy: float
    intent: str
    intent_confidence: float
    latency_ms: int
    audio_quality_score: float

class VoiceTestingFramework:
    def __init__(self, platform: str, locale: str):
        self.platform = platform
        self.locale = locale
        self.tts_client = texttospeech.TextToSpeechClient()
        self.stt_client = speech.SpeechClient()

    def synthesize_test_audio(
        self,
        text: str,
        voice_params: Dict
    ) -> bytes:
        """Generate synthetic speech for testing"""
        synthesis_input = texttospeech.SynthesisInput(text=text)

        voice = texttospeech.VoiceSelectionParams(
            language_code=self.locale,
            name=voice_params.get('name'),
            ssml_gender=texttospeech.SsmlVoiceGender.NEUTRAL
        )

        audio_config = texttospeech.AudioConfig(
            audio_encoding=texttospeech.AudioEncoding.LINEAR16,
            sample_rate_hertz=16000
        )

        response = self.tts_client.synthesize_speech(
            input=synthesis_input,
            voice=voice,
            audio_config=audio_config
        )

        return response.audio_content

    def add_noise_to_audio(
        self,
        clean_audio: bytes,
        noise_type: str,
        snr_db: float
    ) -> bytes:
        """Add controlled noise to audio for environmental testing"""
        audio = AudioSegment.from_wav(clean_audio)

        # Load or generate noise
        noise = self._get_noise_sample(noise_type, len(audio))

        # Calculate noise level based on SNR
        signal_power = audio.dBFS
        noise_power = signal_power - snr_db

        # Adjust noise level and overlay
        adjusted_noise = noise + (noise_power - noise.dBFS)
        noisy_audio = audio.overlay(adjusted_noise)

        return noisy_audio.raw_data

    def test_voice_command(
        self,
        audio: bytes,
        expected_intent: str,
        expected_transcript: str
    ) -> VoiceTestResult:
        """Execute complete voice command test"""
        import time

        start_time = time.time()

        # Step 1: Speech recognition
        transcript = self._recognize_speech(audio)

        # Step 2: Intent classification
        intent, confidence = self._classify_intent(transcript)

        latency_ms = int((time.time() - start_time) * 1000)

        # Step 3: Calculate accuracy
        accuracy = self._calculate_wer(expected_transcript, transcript)

        return VoiceTestResult(
            transcript=transcript,
            expected_transcript=expected_transcript,
            accuracy=1 - accuracy,  # Convert WER to accuracy
            intent=intent,
            intent_confidence=confidence,
            latency_ms=latency_ms,
            audio_quality_score=self._assess_audio_quality(audio)
        )

    def _calculate_wer(self, reference: str, hypothesis: str) -> float:
        """Calculate Word Error Rate"""
        ref_words = reference.lower().split()
        hyp_words = hypothesis.lower().split()

        # Implement Levenshtein distance at word level
        d = np.zeros((len(ref_words) + 1, len(hyp_words) + 1))

        for i in range(len(ref_words) + 1):
            d[i][0] = i
        for j in range(len(hyp_words) + 1):
            d[0][j] = j

        for i in range(1, len(ref_words) + 1):
            for j in range(1, len(hyp_words) + 1):
                if ref_words[i-1] == hyp_words[j-1]:
                    d[i][j] = d[i-1][j-1]
                else:
                    d[i][j] = min(
                        d[i-1][j] + 1,      # deletion
                        d[i][j-1] + 1,      # insertion
                        d[i-1][j-1] + 1     # substitution
                    )

        return d[len(ref_words)][len(hyp_words)] / len(ref_words)

# CI/CD integration example
voice-test-suite run \
  --platform alexa \
  --test-suite regression \
  --locales en-US,en-GB,es-ES \
  --parallel 10 \
  --report junit \
  --threshold accuracy=0.90

# Production monitoring
voice_metrics.track({
    'intent_accuracy': intent_match_rate,
    'average_wer': weekly_wer,
    'p95_latency': latency_p95,
    'user_satisfaction': explicit_feedback_score
})

def test_error_scenarios():
    """Test graceful handling of edge cases"""
    # Mumbled speech
    result = tester.process_audio('test_data/unintelligible.wav')
    assert result.response_type == 'clarification_request'

    # Unsupported language
    result = tester.process_audio('test_data/swahili.wav')
    assert result.response_type == 'language_not_supported'

    # Timeout
    result = tester.process_long_silence(duration_sec=10)
    assert result.response_type == 'timeout'