Feature Flags Testing Strategy: LaunchDarkly, Flagsmith, and A/B Testing for QA

# app/feature_flags.py
import ldclient
from ldclient.config import Config
import os

class FeatureFlagManager:
    def __init__(self):
        sdk_key = os.getenv('LAUNCHDARKLY_SDK_KEY')
        ldclient.set_config(Config(sdk_key))
        self.client = ldclient.get()

        # Wait for client initialization
        if self.client.is_initialized():
            print("✓ LaunchDarkly client initialized")
        else:
            print("⚠ LaunchDarkly client failed to initialize")

    def is_feature_enabled(self, flag_key: str, user: dict, default: bool = False) -> bool:
        """Check if feature is enabled for user"""
        return self.client.variation(flag_key, user, default)

    def get_flag_value(self, flag_key: str, user: dict, default):
        """Get feature flag value (supports multiple types)"""
        return self.client.variation(flag_key, user, default)

    def track_event(self, event_name: str, user: dict, data: dict = None):
        """Track custom event for analytics"""
        self.client.track(event_name, user, data)

    def close(self):
        """Shutdown client"""
        self.client.close()

# Usage in application
feature_flags = FeatureFlagManager()

def process_payment(user_id: str, amount: float):
    user = {
        "key": user_id,
        "email": f"{user_id}@example.com",
        "custom": {
            "subscription_tier": "premium"
        }
    }

    # Check feature flag
    if feature_flags.is_feature_enabled('new-payment-processor', user):
        result = new_payment_processor(amount)
    else:
        result = legacy_payment_processor(amount)

    # Track conversion event
    if result.success:
        feature_flags.track_event('payment-completed', user, {'amount': amount})

    return result

# tests/test_feature_flags.py
import pytest
from unittest.mock import Mock, patch
from app.feature_flags import FeatureFlagManager

class TestFeatureFlags:
    @pytest.fixture
    def mock_ld_client(self):
        """Mock LaunchDarkly client"""
        with patch('ldclient.get') as mock_get:
            mock_client = Mock()
            mock_client.is_initialized.return_value = True
            mock_get.return_value = mock_client
            yield mock_client

    def test_feature_enabled_for_user(self, mock_ld_client):
        """Test feature flag enabled for specific user"""
        mock_ld_client.variation.return_value = True

        ff_manager = FeatureFlagManager()
        user = {"key": "user123", "email": "user@example.com"}

        result = ff_manager.is_feature_enabled('new-payment-processor', user)

        assert result == True
        mock_ld_client.variation.assert_called_once_with(
            'new-payment-processor',
            user,
            False
        )

    def test_feature_disabled_by_default(self, mock_ld_client):
        """Test feature flag returns default when disabled"""
        mock_ld_client.variation.return_value = False

        ff_manager = FeatureFlagManager()
        user = {"key": "user456"}

        result = ff_manager.is_feature_enabled('experimental-feature', user, default=False)

        assert result == False

    def test_multivariate_flag(self, mock_ld_client):
        """Test multivariate feature flag"""
        mock_ld_client.variation.return_value = "variation-b"

        ff_manager = FeatureFlagManager()
        user = {"key": "user789"}

        result = ff_manager.get_flag_value('checkout-flow', user, "control")

        assert result == "variation-b"

    def test_flag_targeting_by_attribute(self, mock_ld_client):
        """Test flag targeting based on user attributes"""
        def variation_side_effect(flag_key, user, default):
            # Enable for premium users only
            if user.get('custom', {}).get('subscription_tier') == 'premium':
                return True
            return False

        mock_ld_client.variation.side_effect = variation_side_effect

        ff_manager = FeatureFlagManager()

        # Premium user
        premium_user = {
            "key": "premium1",
            "custom": {"subscription_tier": "premium"}
        }
        assert ff_manager.is_feature_enabled('premium-feature', premium_user) == True

        # Free user
        free_user = {
            "key": "free1",
            "custom": {"subscription_tier": "free"}
        }
        assert ff_manager.is_feature_enabled('premium-feature', free_user) == False

# app/flagsmith_manager.py
from flagsmith import Flagsmith
import os

class FlagsmithManager:
    def __init__(self):
        self.flagsmith = Flagsmith(
            environment_key=os.getenv('FLAGSMITH_ENVIRONMENT_KEY'),
            api_url='https://api.flagsmith.com/api/v1/'
        )

    def get_flags(self, identifier: str = None, traits: dict = None):
        """Get all flags for user/identity"""
        if identifier:
            return self.flagsmith.get_identity_flags(identifier, traits)
        return self.flagsmith.get_environment_flags()

    def is_feature_enabled(self, feature_name: str, identifier: str = None) -> bool:
        """Check if feature is enabled"""
        flags = self.get_flags(identifier)
        return flags.is_feature_enabled(feature_name)

    def get_feature_value(self, feature_name: str, identifier: str = None, default=None):
        """Get feature value"""
        flags = self.get_flags(identifier)
        value = flags.get_feature_value(feature_name)
        return value if value is not None else default

# Usage
flagsmith = FlagsmithManager()

def render_homepage(user_id: str):
    # Get flags for user
    show_new_nav = flagsmith.is_feature_enabled('new-navigation', user_id)
    hero_variant = flagsmith.get_feature_value('hero-variant', user_id, 'control')

    return render_template(
        'homepage.html',
        show_new_nav=show_new_nav,
        hero_variant=hero_variant
    )

# tests/test_flagsmith.py
import pytest
from unittest.mock import Mock, patch
from app.flagsmith_manager import FlagsmithManager

class TestFlagsmith:
    @pytest.fixture
    def mock_flagsmith(self):
        """Mock Flagsmith SDK"""
        with patch('flagsmith.Flagsmith') as mock_fs:
            mock_instance = Mock()
            mock_fs.return_value = mock_instance

            mock_flags = Mock()
            mock_instance.get_identity_flags.return_value = mock_flags

            yield mock_flags

    def test_feature_enabled_for_identity(self, mock_flagsmith):
        """Test feature enabled for specific identity"""
        mock_flagsmith.is_feature_enabled.return_value = True

        manager = FlagsmithManager()
        result = manager.is_feature_enabled('new-navigation', 'user123')

        assert result == True

    def test_feature_value_retrieval(self, mock_flagsmith):
        """Test retrieving feature value"""
        mock_flagsmith.get_feature_value.return_value = 'variant-a'

        manager = FlagsmithManager()
        result = manager.get_feature_value('hero-variant', 'user456')

        assert result == 'variant-a'

    def test_default_value_when_not_set(self, mock_flagsmith):
        """Test default value when feature not set"""
        mock_flagsmith.get_feature_value.return_value = None

        manager = FlagsmithManager()
        result = manager.get_feature_value('nonexistent-flag', 'user789', default='fallback')

        assert result == 'fallback'

# tests/test_flag_combinations.py
import pytest
from itertools import product

class FlagCombinationTester:
    """Test all possible feature flag combinations"""

    def __init__(self, flags: dict):
        """
        flags: dict of flag_name -> possible values
        Example: {'new-checkout': [True, False], 'payment-v2': [True, False]}
        """
        self.flags = flags

    def get_all_combinations(self):
        """Generate all possible flag combinations"""
        flag_names = list(self.flags.keys())
        flag_values = [self.flags[name] for name in flag_names]

        for combination in product(*flag_values):
            yield dict(zip(flag_names, combination))

    def test_all_combinations(self):
        """Test application behavior with all flag combinations"""
        for combination in self.get_all_combinations():
            try:
                self._test_with_flags(combination)
                print(f"✓ Tested combination: {combination}")
            except Exception as e:
                pytest.fail(f"✗ Failed with combination {combination}: {str(e)}")

    def _test_with_flags(self, flags: dict):
        """Test specific flag combination"""
        # Set flags
        with patch_feature_flags(flags):
            # Run critical user flows
            response = self.client.post('/checkout', data={
                'product_id': '123',
                'quantity': 1
            })

            assert response.status_code == 200
            assert 'order_id' in response.json()

# Usage
def test_checkout_flag_combinations():
    """Test checkout with all flag combinations"""
    flags = {
        'new-checkout-ui': [True, False],
        'express-payment': [True, False],
        'save-for-later': [True, False]
    }

    tester = FlagCombinationTester(flags)
    tester.test_all_combinations()

# Optimized combination testing (test critical paths only)
@pytest.mark.parametrize("new_checkout,express_payment", [
    (True, True),   # Both new features enabled
    (True, False),  # Only new checkout
    (False, True),  # Only express payment
    (False, False)  # All legacy
])
def test_critical_checkout_paths(new_checkout, express_payment):
    """Test critical checkout paths with key flag combinations"""
    with patch_feature_flags({
        'new-checkout-ui': new_checkout,
        'express-payment': express_payment
    }):
        # Test checkout flow
        response = complete_checkout()
        assert response.success == True

# tests/test_ab_testing.py
import pytest
from scipy import stats
import numpy as np

class ABTestValidator:
    """Validate A/B test statistical significance"""

    def __init__(self, alpha: float = 0.05):
        self.alpha = alpha  # Significance level

    def calculate_sample_size(self, baseline_rate: float, minimum_detectable_effect: float, power: float = 0.8):
        """Calculate required sample size for A/B test"""
        # Using simplified formula for proportions
        z_alpha = stats.norm.ppf(1 - self.alpha / 2)
        z_beta = stats.norm.ppf(power)

        p1 = baseline_rate
        p2 = baseline_rate * (1 + minimum_detectable_effect)
        p_avg = (p1 + p2) / 2

        n = (2 * p_avg * (1 - p_avg) * (z_alpha + z_beta) ** 2) / ((p2 - p1) ** 2)

        return int(np.ceil(n))

    def test_statistical_significance(self, control_conversions: int, control_total: int,
                                     treatment_conversions: int, treatment_total: int):
        """Test if difference between variants is statistically significant"""
        # Create contingency table
        observed = np.array([
            [control_conversions, control_total - control_conversions],
            [treatment_conversions, treatment_total - treatment_conversions]
        ])

        # Perform chi-square test
        chi2, p_value, dof, expected = stats.chi2_contingency(observed)

        is_significant = p_value < self.alpha

        control_rate = control_conversions / control_total
        treatment_rate = treatment_conversions / treatment_total
        lift = ((treatment_rate - control_rate) / control_rate) * 100

        return {
            'is_significant': is_significant,
            'p_value': p_value,
            'control_rate': control_rate,
            'treatment_rate': treatment_rate,
            'lift_percentage': lift,
            'confidence_level': (1 - self.alpha) * 100
        }

# Usage in tests
def test_ab_test_new_checkout():
    """Test A/B experiment for new checkout flow"""
    validator = ABTestValidator(alpha=0.05)

    # Simulate A/B test data collection
    control_conversions = 450
    control_total = 5000

    treatment_conversions = 520
    treatment_total = 5000

    # Validate statistical significance
    result = validator.test_statistical_significance(
        control_conversions, control_total,
        treatment_conversions, treatment_total
    )

    print(f"Control conversion rate: {result['control_rate']:.2%}")
    print(f"Treatment conversion rate: {result['treatment_rate']:.2%}")
    print(f"Lift: {result['lift_percentage']:.2f}%")
    print(f"P-value: {result['p_value']:.4f}")
    print(f"Statistically significant: {result['is_significant']}")

    # Assert treatment is significantly better
    assert result['is_significant'] == True
    assert result['lift_percentage'] > 0

def test_minimum_sample_size():
    """Test that A/B test has sufficient sample size"""
    validator = ABTestValidator()

    baseline_rate = 0.09  # 9% conversion rate
    mde = 0.10  # Detect 10% relative improvement

    required_n = validator.calculate_sample_size(baseline_rate, mde)

    print(f"Required sample size per variant: {required_n}")

    # Verify test has enough samples
    actual_samples = 5000
    assert actual_samples >= required_n, f"Need {required_n} samples, only have {actual_samples}"

# tests/test_flag_lifecycle.py
import pytest
from datetime import datetime, timedelta

class FlagLifecycleTester:
    """Test feature flag lifecycle management"""

    def test_flag_age_tracking(self):
        """Test flags are not left in code indefinitely"""
        flags = self.get_all_flags()

        old_flags = []
        max_age_days = 90

        for flag in flags:
            created_date = datetime.fromisoformat(flag['created_at'])
            age = datetime.now() - created_date

            if age > timedelta(days=max_age_days):
                old_flags.append({
                    'key': flag['key'],
                    'age_days': age.days,
                    'permanent': flag.get('permanent', False)
                })

        # Filter out permanent flags (kill switches, operational toggles)
        temporary_old_flags = [f for f in old_flags if not f['permanent']]

        assert len(temporary_old_flags) == 0, \
            f"Found {len(temporary_old_flags)} flags older than {max_age_days} days: {temporary_old_flags}"

    def test_unused_flags_detection(self):
        """Detect flags that are no longer referenced in code"""
        all_flags = self.get_all_flags()
        code_references = self.scan_codebase_for_flag_references()

        unused_flags = []

        for flag in all_flags:
            if flag['key'] not in code_references:
                unused_flags.append(flag['key'])

        assert len(unused_flags) == 0, \
            f"Found {len(unused_flags)} unreferenced flags: {unused_flags}"

    def test_flag_cleanup_on_100_percent_rollout(self):
        """Test flags are marked for cleanup when at 100% rollout"""
        flags = self.get_all_flags()

        fully_rolled_out = []

        for flag in flags:
            if not flag.get('permanent', False):
                rollout = flag.get('rollout_percentage', 0)
                age_days = (datetime.now() - datetime.fromisoformat(flag['created_at'])).days

                # Flag at 100% for more than 30 days should be cleaned up
                if rollout == 100 and age_days > 30:
                    fully_rolled_out.append({
                        'key': flag['key'],
                        'age_days': age_days
                    })

        assert len(fully_rolled_out) == 0, \
            f"Found {len(fully_rolled_out)} flags ready for cleanup: {fully_rolled_out}"

# .github/workflows/feature-flags-test.yml
name: Feature Flag Testing

on:
  pull_request:
  push:
    branches: [main]

jobs:
  test-flag-combinations:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          pip install pytest scipy numpy launchdarkly-server-sdk flagsmith

      - name: Run flag combination tests
        env:
          LAUNCHDARKLY_SDK_KEY: ${{ secrets.LAUNCHDARKLY_SDK_KEY }}
        run: |
          pytest tests/test_flag_combinations.py -v

      - name: Validate A/B tests
        run: |
          pytest tests/test_ab_testing.py -v

      - name: Check flag lifecycle
        run: |
          pytest tests/test_flag_lifecycle.py -v

      - name: Generate flag coverage report
        run: |
          python scripts/generate_flag_coverage.py > flag-coverage.txt

      - name: Upload flag coverage
        uses: actions/upload-artifact@v3
        with:
          name: flag-coverage-report
          path: flag-coverage.txt