"""
Centralized Metadata Enrichment Service.

Implements profile-driven metadata enrichment with shared key variants approach.
This is the core service that applies business theme metadata to resources
through tags, K8s labels, resource IDs, and namespace modifications.
"""

import logging
from typing import List

import polars as pl

from ..config.profiles import ProfileConfig
from .random_service import DeterministicRandomService


logger = logging.getLogger(__name__)


class SelectedFeature:
    """Container for a selected feature with its variants."""
    
    def __init__(self, name: str, value_variant: str, weight: float):
        self.name = name
        self.value_variant = value_variant
        self.weight = weight
    
    def __repr__(self):
        return f"SelectedFeature(name='{self.name}', value_variant='{self.value_variant}', weight={self.weight})"


class FeatureVariantSelector:
    """Handles deterministic selection of features and their variants."""
    
    def __init__(self, profile_config: ProfileConfig, random_service: DeterministicRandomService):
        """
        Initialize feature variant selector.
        
        Args:
            profile_config: Profile configuration with feature definitions
            random_service: Deterministic random service for consistent selection
        """
        self.profile_config = profile_config
        self.random_service = random_service
    
    def select_features_for_resource(
        self, 
        resource_fingerprint: str, 
        base_allocation: str
    ) -> List[SelectedFeature]:
        """
        Select features based on base_allocation theme from profiles.
        
        Args:
            resource_fingerprint: Unique resource identifier for deterministic seeding
            base_allocation: Theme name from config (e.g., 'analytics_data')
            
        Returns:
            List of selected features with their value variants
        """
        # Get features for the specified base allocation theme
        available_features = self.profile_config.features.get_features(base_allocation)
        
        if not available_features:
            error_msg = f"No features found for base_allocation: {base_allocation}. This should have been caught by validation."
            logger.error(error_msg)
            raise RuntimeError(error_msg)
        
        selected_features = []
        
        for feature_config in available_features:
            # Extract feature metadata (fixed field mapping to match YAML structure)
            feature_name = feature_config.get('value', 'unknown')  # YAML has 'value' not 'name'
            value_variants = feature_config.get('variants', [])   # YAML has 'variants' not 'value_variants'
            weight = feature_config.get('weight', 1.0)
            
            if not value_variants:
                error_msg = f"No value variants found for feature: {feature_name}. This should have been caught by validation."
                logger.error(error_msg)
                raise RuntimeError(error_msg)
            
            # Deterministically select value variant for this resource
            value_variant = self.select_value_variant(
                value_variants, 
                resource_fingerprint, 
                feature_name
            )
            
            selected_features.append(SelectedFeature(
                name=feature_name,
                value_variant=value_variant,
                weight=weight
            ))
        
        logger.debug(f"Selected {len(selected_features)} features for {base_allocation}: {selected_features}")
        return selected_features
    
    def select_value_variant(
        self, 
        value_variants: List[str], 
        resource_fingerprint: str, 
        feature_name: str
    ) -> str:
        """
        Deterministically select value variant for feature.
        
        Args:
            value_variants: Available value variants for the feature
            resource_fingerprint: Unique resource identifier for seeding
            feature_name: Feature name for seeding context
            
        Returns:
            Selected value variant
        """
        if not value_variants:
            return ""
        
        if len(value_variants) == 1:
            return value_variants[0]
        
        # Create deterministic seed for value variant selection
        seed = f"{resource_fingerprint}:feature_value:{feature_name}"
        
        # Use random service for consistent selection
        selected = self.random_service.vectorized_choice(
            value_variants,
            size=1,
            stage=seed
        )[0]
        
        return selected
    
    def add_feature_columns_vectorized(
        self,
        df: pl.DataFrame,
        base_allocation: str
    ) -> pl.DataFrame:
        """
        Add feature columns using vectorized operations.
        
        Args:
            df: DataFrame with resources
            base_allocation: Theme name for feature selection
            
        Returns:
            DataFrame with feature_name and feature_variant columns added
        """
        
        if df.height == 0:
            return df
            
        # Get feature values for the specified theme (fixed to use all features, not just first)
        feature_theme_values = self.profile_config.features.values.get(base_allocation, [])
        
        if not feature_theme_values:
            error_msg = f"No feature values found for base_allocation: {base_allocation}. This should have been caught by validation."
            logger.error(error_msg)
            raise RuntimeError(error_msg)
        
        # Collect all feature variants from ALL features in the theme (not just first)
        all_feature_variants = []
        for feature_value_config in feature_theme_values:
            feature_variants = feature_value_config.get('variants', [])
            all_feature_variants.extend(feature_variants)
        
        if not all_feature_variants:
            error_msg = f"No feature variants found for base_allocation: {base_allocation}. This should have been caught by validation."
            logger.error(error_msg)
            raise RuntimeError(error_msg)
        
        logger.info(f"Using {len(all_feature_variants)} feature variants from {len(feature_theme_values)} features for shared columns: {all_feature_variants}")
        
        # For shared columns, we need a representative feature name (use first feature's name)
        representative_feature_name = feature_theme_values[0].get('value', 'unknown')
        value_variants = all_feature_variants
        
        # Add feature name column (constant) - using representative name since we have multiple features
        result = df.with_columns([
            pl.lit(representative_feature_name).alias('feature_name')
        ])
        
        # Add feature variant column (deterministic selection)
        if len(value_variants) == 1:
            # Single variant - all resources get the same value
            result = result.with_columns([
                pl.lit(value_variants[0]).alias('feature_variant')
            ])
        else:
            # Multiple variants - deterministic selection per resource (vectorized)
            # Create seed strings for deterministic hashing
            # Handle null values by filling with empty strings before concatenation
            seed_column = pl.concat_str([
                pl.col('cloud_provider').fill_null(''),
                pl.lit(':'),
                pl.col('service').fill_null(''),
                pl.lit(':'), 
                pl.col('resource_id').fill_null(''),
                pl.lit(':shared_feature_selection:'),
                pl.lit(base_allocation)
            ])
            
            # Use modulo arithmetic with hash to select variants
            result = result.with_columns([
                (seed_column.hash() % len(value_variants)).alias('variant_index')
            ])
            
            # Create when/then chains to map indices to variants (vectorized)
            variant_mapping = pl.col('variant_index')
            for i, variant in enumerate(value_variants):
                if i == 0:
                    variant_mapping = pl.when(pl.col('variant_index') == i).then(pl.lit(variant))
                else:
                    variant_mapping = variant_mapping.when(pl.col('variant_index') == i).then(pl.lit(variant))
            
            result = result.with_columns([
                variant_mapping.otherwise(pl.lit(value_variants[0])).alias('feature_variant')
            ]).drop('variant_index')
        
        return result


class MetadataEnrichmentService:
    """Centralized service for applying profile-driven metadata enrichment."""
    
    def __init__(
        self, 
        profile_config: ProfileConfig, 
        random_service: DeterministicRandomService
    ):
        """
        Initialize metadata enrichment service.
        
        Args:
            profile_config: Profile configuration with shared key variants
            random_service: Deterministic random service for consistent enrichment
        """
        self.profile_config = profile_config
        self.random_service = random_service
        self.feature_selector = FeatureVariantSelector(profile_config, random_service)
        
        # Fail-fast validation of critical profile configuration
        self._validate_profile_configuration()
        
        logger.info("Initialized MetadataEnrichmentService with shared key variants approach")
    
    def enrich_tags(
        self, 
        df: pl.DataFrame, 
        base_allocation: str
    ) -> pl.DataFrame:
        """
        Create multiple resource/tag columns with ALL key variants for each feature.
        
        For feature "analytics" with key_variants ["feature", "Features", "feat"]:
        Creates: resource/tag:feature = "analytics"
                resource/tag:Features = "analytics"  
                resource/tag:feat = "analytics"
        
        Args:
            df: DataFrame with resources to enrich (must include environment_variant column)
            base_allocation: Theme name for feature selection
            
        Returns:
            DataFrame with multiple tag columns added
        """
        if df.height == 0:
            return df
        
        logger.info(f"Enriching {df.height:,} resources with metadata tags (base_allocation: {base_allocation})")
        
        # Add feature columns using vectorized operations
        result = self.feature_selector.add_feature_columns_vectorized(df, base_allocation)
        
        # Get shared key variants from profiles
        feature_key_variants = self.profile_config.features.key_variants
        environment_key_variants = self.profile_config.environments.key_variants
        
        if not feature_key_variants:
            error_msg = "No feature key variants found in profile config. This should have been caught by validation."
            logger.error(error_msg)
            raise RuntimeError(error_msg)
        
        # Add feature tags with ALL key variants
        result = self._add_feature_tags(result, base_allocation, feature_key_variants)
        
        # Add environment tags with ALL key variants (if environment_variant column exists)
        if environment_key_variants and 'environment_variant' in df.columns:
            result = self._add_environment_tags_vectorized(
                result, 
                environment_key_variants
            )
        
        logger.info(f"Created {len(feature_key_variants)} feature tag columns and {len(environment_key_variants)} environment tag columns")
        
        # Clean up intermediate environment_variant column early
        if 'environment_variant' in result.columns:
            result = result.drop('environment_variant')
            logger.info("Cleaned up intermediate environment_variant column after tag creation")
        
        return result
    
    def enrich_k8s_labels(
        self, 
        df: pl.DataFrame, 
        base_allocation: str
    ) -> pl.DataFrame:
        """
        Update k8s_labels JSON with feature key variants for default namespaces only.
        
        - Default namespaces (default, kube-system, etc.) get feature labels
        - __UNALLOCATED__ namespace gets no feature enrichment (remains clean)
        - Custom namespaces should not reach this method (they get namespace injection instead)
        
        Args:
            df: DataFrame with K8s resources to enrich
            base_allocation: Theme name for feature selection
            
        Returns:
            DataFrame with enhanced k8s_labels JSON for default namespaces
        """
        
        if df.height == 0:
            return df
        
        if 'k8s_labels' not in df.columns:
            error_msg = "No k8s_labels column found for K8s label enrichment. This indicates incorrect resource classification."
            logger.error(error_msg)
            raise RuntimeError(error_msg)
        
        # Define default K8s namespaces that should get feature labels
        default_namespaces = ['default', 'kube-system', 'kube-public', 'kube-node-lease']
        
        # Split resources: default namespaces get feature enrichment, __UNALLOCATED__ gets none
        default_resources = df.filter(pl.col('k8s_namespace').is_in(default_namespaces))
        idle_resources = df.filter(pl.col('k8s_namespace') == '__UNALLOCATED__')
        other_resources = df.filter(
            ~pl.col('k8s_namespace').is_in(default_namespaces) & 
            (pl.col('k8s_namespace') != '__UNALLOCATED__')
        )
        
        logger.info(f"K8s label enrichment split: {default_resources.height:,} default namespace resources, {idle_resources.height:,} __UNALLOCATED__ resources, {other_resources.height:,} other resources")
        
        # Process default namespaces with feature labels
        if default_resources.height > 0:
            
            # Add feature columns using vectorized operations
            enriched_default = self.feature_selector.add_feature_columns_vectorized(default_resources, base_allocation)
            
            # Get shared key variants from profiles
            feature_key_variants = self.profile_config.features.key_variants
            environment_key_variants = self.profile_config.environments.key_variants
            
            # Use vectorized K8s label enhancement for default namespaces only
            enriched_default = self._enhance_k8s_labels_vectorized(
                enriched_default,
                feature_key_variants,
                environment_key_variants
            )
        else:
            enriched_default = default_resources.clone()
        
        # Combine results: enriched default namespaces + clean __UNALLOCATED__ + other resources
        result_parts = []
        if enriched_default.height > 0:
            result_parts.append(enriched_default)
        if idle_resources.height > 0:
            result_parts.append(idle_resources)  # __UNALLOCATED__ remains unchanged
        if other_resources.height > 0:
            result_parts.append(other_resources)  # Other resources remain unchanged
            
        if result_parts:
            result = pl.concat(result_parts)
        else:
            result = df.clone()
        
        
        logger.info(f"Enhanced K8s labels with {len(feature_key_variants)} feature variants and {len(environment_key_variants)} environment variants")
        
        # Add empty tag columns to match schema with tag enrichment
        # This ensures schema compatibility when concatenating with tag-enriched resources
        tag_columns_to_add = []
        
        # Add feature tag columns with empty strings (preferred over nulls for persistence)
        for key_variant in feature_key_variants:
            tag_column = f"resource/tag:{key_variant}"
            if tag_column not in result.columns:
                tag_columns_to_add.append(pl.lit("").alias(tag_column))
        
        # Add environment tag columns with empty strings (preferred over nulls for persistence)
        for key_variant in environment_key_variants:
            tag_column = f"resource/tag:{key_variant}"
            if tag_column not in result.columns:
                tag_columns_to_add.append(pl.lit("").alias(tag_column))
        
        if tag_columns_to_add:
            result = result.with_columns(tag_columns_to_add)
            logger.info(f"Added {len(tag_columns_to_add)} null tag columns for schema compatibility with tag enrichment")
        
        # Clean up intermediate environment_variant column early
        if 'environment_variant' in result.columns:
            result = result.drop('environment_variant')
            logger.info("Cleaned up intermediate environment_variant column after K8s label enhancement")
        
        return result
    
    def enrich_resource_ids(
        self, 
        df: pl.DataFrame, 
        base_allocation: str
    ) -> pl.DataFrame:
        """
        Inject feature value variants into resource IDs with per-resource deterministic selection.
        
        /subscriptions/123/resourceGroups/prod-rg → /subscriptions/123/resourceGroups/pulse-analytics-rg
        
        Args:
            df: DataFrame with resources to enrich
            base_allocation: Theme name for feature selection
            
        Returns:
            DataFrame with modified resource_ids and same schema as tag enrichment
        """
        if df.height == 0:
            return df
        
        
        logger.info(f"Enriching {df.height:,} resources with resource ID injection (base_allocation: {base_allocation})")
        
        # Use new per-resource variant injection (no longer need intermediate columns)
        result = self._inject_resource_id_feature_variants(df, base_allocation)
        
        # Add empty tag columns to match schema with tag enrichment
        # This ensures schema compatibility when concatenating with tag-enriched resources
        feature_key_variants = self.profile_config.features.key_variants
        environment_key_variants = self.profile_config.environments.key_variants
        
        tag_columns_to_add = []
        
        # Add feature tag columns with empty strings (preferred over nulls for persistence)
        for key_variant in feature_key_variants:
            tag_column = f"resource/tag:{key_variant}"
            if tag_column not in result.columns:
                tag_columns_to_add.append(pl.lit("").alias(tag_column))
        
        # Add environment tag columns with empty strings (preferred over nulls for persistence)
        for key_variant in environment_key_variants:
            tag_column = f"resource/tag:{key_variant}"
            if tag_column not in result.columns:
                tag_columns_to_add.append(pl.lit("").alias(tag_column))
        
        if tag_columns_to_add:
            result = result.with_columns(tag_columns_to_add)
            logger.info(f"Added {len(tag_columns_to_add)} null tag columns for schema compatibility with tag enrichment")
        
        logger.info("Completed resource ID variant injection")
        
        # Clean up intermediate environment_variant column early
        if 'environment_variant' in result.columns:
            result = result.drop('environment_variant')
            logger.info("Cleaned up intermediate environment_variant column after resource ID enrichment")
        
        return result
    
    def _inject_resource_id_feature_variants(self, df: pl.DataFrame, base_allocation: str) -> pl.DataFrame:
        """
        Inject feature value variants into resource IDs using per-resource deterministic selection.
        
        This method implements the corrected approach where:
        - Each resource gets ONE feature value variant (not all variants)
        - Selection is deterministic per resource for cross-month consistency
        - Only feature variants are used (not environment variants)
        - Resource fingerprint used for seeding: provider:service:resource_id:feature_injection
        
        Args:
            df: DataFrame with resources to process
            base_allocation: Theme name for feature selection (e.g., "analytics_data")
            
        Returns:
            DataFrame with modified resource_id column
        """
        if df.height == 0:
            return df
            
        # Get feature values for the specified theme
        feature_theme_values = self.profile_config.features.values.get(base_allocation, [])
        if not feature_theme_values:
            logger.warning(f"No feature values found for theme '{base_allocation}' - skipping resource ID injection")
            return df
            
        # Collect all feature variants from all values in the theme
        all_feature_variants = []
        for feature_value_config in feature_theme_values:
            all_feature_variants.extend(feature_value_config['variants'])
        
        if not all_feature_variants:
            logger.warning(f"No feature variants found for theme '{base_allocation}' - skipping resource ID injection")
            return df
            
        logger.info(f"Using {len(all_feature_variants)} feature variants for resource ID injection: {all_feature_variants}")
        
        # Create resource fingerprint for deterministic selection
        # Handle null values by filling with empty strings before concatenation
        resource_fingerprint = pl.concat_str([
            pl.col('cloud_provider').fill_null(''),
            pl.lit(':'),
            pl.col('service').fill_null(''),  
            pl.lit(':'),
            pl.col('resource_id').fill_null(''),
            pl.lit(':feature_injection')
        ])
        
        # Select ONE feature variant per resource using hash modulo
        variant_index = (resource_fingerprint.hash() % len(all_feature_variants)).cast(pl.UInt32)
        
        # Create vectorized feature variant selection using when/then mapping
        selected_variant = pl.when(variant_index == 0).then(pl.lit(all_feature_variants[0]))
        for i in range(1, len(all_feature_variants)):
            selected_variant = selected_variant.when(variant_index == i).then(pl.lit(all_feature_variants[i]))
        
        # Add fallback to first variant (should not be needed with proper modulo)
        selected_variant = selected_variant.otherwise(pl.lit(all_feature_variants[0]))
        
        # Apply feature variant injection to resource_id
        # Strategy: Replace a common part of the resource ID with the feature variant
        result = df.with_columns([
            # Simple injection strategy: replace common patterns
            pl.when(pl.col('resource_id').str.contains('prod'))
            .then(pl.col('resource_id').str.replace_all('prod', selected_variant))
            .when(pl.col('resource_id').str.contains('dev'))
            .then(pl.col('resource_id').str.replace_all('dev', selected_variant))
            .when(pl.col('resource_id').str.contains('test'))
            .then(pl.col('resource_id').str.replace_all('test', selected_variant))
            .otherwise(
                # If no common pattern, append the variant
                pl.concat_str([pl.col('resource_id'), pl.lit('-'), selected_variant])
            )
            .alias('resource_id')
        ])
        
        logger.info(f"Applied per-resource feature variant injection to {df.height:,} resource IDs")
        return result
    
    def enrich_k8s_namespaces(
        self, 
        df: pl.DataFrame, 
        base_allocation: str
    ) -> pl.DataFrame:
        """
        Inject feature value variants into namespace names using workload-based deterministic selection.
        
        This method implements the corrected approach where:
        - Each workload gets ONE feature value variant (not all variants)
        - Selection is deterministic per workload for cross-month consistency
        - Only feature variants are used (not environment variants)
        - Workload fingerprint used for seeding: k8s_cluster:k8s_namespace:k8s_workload:namespace_injection
        
        Args:
            df: DataFrame with K8s resources to enrich
            base_allocation: Theme name for feature selection (e.g., "analytics_data")
            
        Returns:
            DataFrame with modified k8s_namespace column and same schema as label enrichment
        """
        if df.height == 0:
            return df
        
        if 'k8s_namespace' not in df.columns:
            error_msg = "No k8s_namespace column found for namespace enrichment. This indicates incorrect resource classification."
            logger.error(error_msg)
            raise RuntimeError(error_msg)
        
        logger.info(f"Enriching {df.height:,} K8s resources with namespace injection (base_allocation: {base_allocation})")
        
        # Use new per-workload variant injection (no longer need intermediate columns)
        result = self._inject_namespace_feature_variants(df, base_allocation)
        
        # Add empty label column to match schema with label enrichment
        # This ensures schema compatibility when concatenating with label-enriched resources
        if 'k8s_labels' not in result.columns:
            result = result.with_columns([pl.lit(None).alias('k8s_labels')])
            logger.info("Added null k8s_labels column for schema compatibility with label enrichment")
        
        # Add empty tag columns to match schema with tag enrichment
        # This ensures schema compatibility when concatenating with tag-enriched resources
        feature_key_variants = self.profile_config.features.key_variants
        environment_key_variants = self.profile_config.environments.key_variants
        
        tag_columns_to_add = []
        
        # Add feature tag columns with empty strings (preferred over nulls for persistence)
        for key_variant in feature_key_variants:
            tag_column = f"resource/tag:{key_variant}"
            if tag_column not in result.columns:
                tag_columns_to_add.append(pl.lit("").alias(tag_column))
        
        # Add environment tag columns with empty strings (preferred over nulls for persistence)
        for key_variant in environment_key_variants:
            tag_column = f"resource/tag:{key_variant}"
            if tag_column not in result.columns:
                tag_columns_to_add.append(pl.lit("").alias(tag_column))
        
        if tag_columns_to_add:
            result = result.with_columns(tag_columns_to_add)
            logger.info(f"Added {len(tag_columns_to_add)} null tag columns for schema compatibility with tag enrichment")
        
        logger.info("Completed K8s namespace variant injection")
        
        # Clean up intermediate environment_variant column early
        if 'environment_variant' in result.columns:
            result = result.drop('environment_variant')
            logger.info("Cleaned up intermediate environment_variant column after K8s namespace enhancement")
        
        return result
    
    def _inject_namespace_feature_variants(self, df: pl.DataFrame, base_allocation: str) -> pl.DataFrame:
        """
        Inject feature value variants into K8s namespaces using per-workload deterministic selection.
        
        This method implements the corrected approach where:
        - Each workload gets ONE feature value variant (not all variants)
        - Selection is deterministic per workload for cross-month consistency
        - Only feature variants are used (not environment variants)
        - Workload fingerprint used for seeding: k8s_cluster:k8s_namespace:k8s_workload:namespace_injection
        
        Args:
            df: DataFrame with K8s resources to process
            base_allocation: Theme name for feature selection (e.g., "analytics_data")
            
        Returns:
            DataFrame with modified k8s_namespace column
        """
        if df.height == 0:
            return df
        
        # Debug: Check available columns
        logger.info(f"Available columns for namespace injection: {df.columns}")
        
        # Validate required columns for workload fingerprint
        required_columns = ['k8s_cluster', 'k8s_namespace', 'k8s_workload']
        missing_columns = [col for col in required_columns if col not in df.columns]
        
        if missing_columns:
            logger.warning(f"Missing required columns for workload fingerprint: {missing_columns}")
            logger.warning("Falling back to simple namespace-based fingerprint")
            # Fallback to namespace-only fingerprint if workload info not available
            return self._inject_namespace_feature_variants_fallback(df, base_allocation)
            
        # Get feature values for the specified theme
        feature_theme_values = self.profile_config.features.values.get(base_allocation, [])
        if not feature_theme_values:
            logger.warning(f"No feature values found for theme '{base_allocation}' - skipping namespace injection")
            return df
            
        # Collect all feature variants from all values in the theme
        all_feature_variants = []
        for feature_value_config in feature_theme_values:
            all_feature_variants.extend(feature_value_config['variants'])
        
        if not all_feature_variants:
            logger.warning(f"No feature variants found for theme '{base_allocation}' - skipping namespace injection")
            return df
            
        logger.info(f"Using {len(all_feature_variants)} feature variants for namespace injection: {all_feature_variants}")
        
        # Create namespace fingerprint for deterministic selection (prevents namespace splitting)
        # All resources in same namespace get same variant by using namespace-only fingerprint
        namespace_fingerprint = pl.concat_str([
            pl.col('k8s_cluster').fill_null(''),
            pl.lit(':'),
            pl.col('k8s_namespace').fill_null(''),
            pl.lit(':namespace_injection')
        ])
        
        
        try:
            # Select ONE feature variant per namespace using hash modulo
            # All resources in same namespace get same variant
            df_with_index = df.with_columns([
                (namespace_fingerprint.hash() % len(all_feature_variants)).cast(pl.UInt32).alias('variant_index')
            ])
            
            # Create feature variant selection using map_elements for proper column handling
            def select_variant(index: int) -> str:
                return all_feature_variants[index]
            
            selected_variant = pl.col('variant_index').map_elements(select_variant, return_dtype=pl.Utf8)
        except Exception as e:
            logger.error(f"Error in variant selection: {e}")
            raise
        
        
        # Define namespaces that should NOT get injection (only custom namespaces should get injection)
        default_namespaces = {'default', 'kube-system', 'kube-public', 'kube-node-lease'}
        
        # Create condition: only inject into custom namespaces (not default or __UNALLOCATED__)
        injection_condition = ~(
            pl.col('k8s_namespace').is_in(list(default_namespaces)) |
            (pl.col('k8s_namespace') == '__UNALLOCATED__')
        )
        
        # Apply feature variant injection to k8s_namespace ONLY for custom namespaces
        # Strategy: Replace common patterns or append the variant, but only for custom namespaces
        injection_logic = (
            pl.when(pl.col('k8s_namespace').str.contains('prod'))
            .then(pl.col('k8s_namespace').str.replace_all('prod', selected_variant))
            .when(pl.col('k8s_namespace').str.contains('dev'))
            .then(pl.col('k8s_namespace').str.replace_all('dev', selected_variant))
            .when(pl.col('k8s_namespace').str.contains('test'))
            .then(pl.col('k8s_namespace').str.replace_all('test', selected_variant))
            .otherwise(
                # If no common pattern, append the variant
                pl.concat_str([pl.col('k8s_namespace'), pl.lit('-'), selected_variant])
            )
        )
        
        result = df_with_index.with_columns([
            pl.when(injection_condition)
            .then(injection_logic)
            .otherwise(pl.col('k8s_namespace'))  # Keep original name for default and __UNALLOCATED__
            .alias('k8s_namespace')
        ])
        
        # Drop the temporary variant_index column
        result = result.drop('variant_index')
        
        logger.info(f"Applied per-workload feature variant injection to {df.height:,} namespaces")
        return result
    
    def _inject_namespace_feature_variants_fallback(self, df: pl.DataFrame, base_allocation: str) -> pl.DataFrame:
        """
        Fallback method for namespace injection when workload columns are missing.
        
        Uses namespace-only fingerprint for deterministic selection when k8s_cluster
        or k8s_workload columns are not available.
        
        Args:
            df: DataFrame with K8s resources to process
            base_allocation: Theme name for feature selection
            
        Returns:
            DataFrame with modified k8s_namespace column
        """
        # Get feature values for the specified theme
        feature_theme_values = self.profile_config.features.values.get(base_allocation, [])
        if not feature_theme_values:
            logger.warning(f"No feature values found for theme '{base_allocation}' - skipping namespace injection")
            return df
            
        # Collect all feature variants from all values in the theme
        all_feature_variants = []
        for feature_value_config in feature_theme_values:
            all_feature_variants.extend(feature_value_config['variants'])
        
        if not all_feature_variants:
            logger.warning(f"No feature variants found for theme '{base_allocation}' - skipping namespace injection")
            return df
            
        logger.info(f"Using {len(all_feature_variants)} feature variants for namespace injection (fallback): {all_feature_variants}")
        
        # Create namespace-only fingerprint for deterministic selection
        # Handle null values by filling with empty strings before concatenation
        namespace_fingerprint = pl.concat_str([
            pl.col('k8s_namespace').fill_null(''),
            pl.lit(':namespace_injection_fallback')
        ])
        
        # Select ONE feature variant per namespace using hash modulo
        variant_index = (namespace_fingerprint.hash() % len(all_feature_variants)).cast(pl.UInt32)
        
        # Create vectorized feature variant selection using when/then mapping
        selected_variant = pl.when(variant_index == 0).then(pl.lit(all_feature_variants[0]))
        for i in range(1, len(all_feature_variants)):
            selected_variant = selected_variant.when(variant_index == i).then(pl.lit(all_feature_variants[i]))
        
        # Add fallback to first variant (should not be needed with proper modulo)
        selected_variant = selected_variant.otherwise(pl.lit(all_feature_variants[0]))
        
        # Define namespaces that should NOT get injection (only custom namespaces should get injection)
        default_namespaces = {'default', 'kube-system', 'kube-public', 'kube-node-lease'}
        
        # Create condition: only inject into custom namespaces (not default or __UNALLOCATED__)
        injection_condition = ~(
            pl.col('k8s_namespace').is_in(list(default_namespaces)) |
            (pl.col('k8s_namespace') == '__UNALLOCATED__')
        )
        
        # Apply feature variant injection to k8s_namespace ONLY for custom namespaces
        # Strategy: Replace common patterns or append the variant, but only for custom namespaces
        injection_logic = (
            pl.when(pl.col('k8s_namespace').str.contains('prod'))
            .then(pl.col('k8s_namespace').str.replace_all('prod', selected_variant))
            .when(pl.col('k8s_namespace').str.contains('dev'))
            .then(pl.col('k8s_namespace').str.replace_all('dev', selected_variant))
            .when(pl.col('k8s_namespace').str.contains('test'))
            .then(pl.col('k8s_namespace').str.replace_all('test', selected_variant))
            .otherwise(
                # If no common pattern, append the variant
                pl.concat_str([pl.col('k8s_namespace'), pl.lit('-'), selected_variant])
            )
        )
        
        result = df.with_columns([
            pl.when(injection_condition)
            .then(injection_logic)
            .otherwise(pl.col('k8s_namespace'))  # Keep original name for default and __UNALLOCATED__
            .alias('k8s_namespace')
        ])
        
        logger.info(f"Applied per-namespace feature variant injection (fallback) to {df.height:,} namespaces")
        return result
    
    def _add_feature_tags(
        self, 
        df: pl.DataFrame, 
        base_allocation: str, 
        key_variants: List[str]
    ) -> pl.DataFrame:
        """Add feature tags with ONE key variant per resource using vectorized operations."""
        result = df.clone()
        
        if not key_variants:
            return result
        
        # Create deterministic key variant selection based on resource fingerprint
        # Handle null values by filling with empty strings before concatenation
        resource_fingerprint = pl.concat_str([
            pl.col('cloud_provider').fill_null(''),
            pl.lit(':'),
            pl.col('service').fill_null(''), 
            pl.lit(':'),
            pl.col('resource_id').fill_null(''),
            pl.lit(':feature_key')
        ])
        
        # Select ONE key variant per resource using hash modulo
        key_variant_index = (resource_fingerprint.hash() % len(key_variants)).cast(pl.UInt32)
        
        # Create tag column name dynamically based on selected index
        # Use when/then chain to map index to key variant
        selected_key_variant = key_variant_index
        for i, key_variant in enumerate(key_variants):
            if i == 0:
                selected_key_variant = pl.when(key_variant_index == i).then(pl.lit(key_variant))
            else:
                selected_key_variant = selected_key_variant.when(key_variant_index == i).then(pl.lit(key_variant))
        
        # Add prefix to create full column name
        tag_column_name = pl.concat_str([pl.lit("resource/tag:"), selected_key_variant])
        
        # Since Polars doesn't support dynamic column names in with_columns,
        # we'll create separate columns for each possible key variant and use nulls
        tag_columns = []
        for key_variant in key_variants:
            tag_column = f"resource/tag:{key_variant}"
            # Only add value if this is the selected key variant for this resource
            tag_columns.append(
                pl.when(selected_key_variant == key_variant)
                .then(pl.col('feature_variant'))
                .otherwise(None)
                .alias(tag_column)
            )
        
        result = result.with_columns(tag_columns)
        
        # Clean up intermediate columns used for tag creation
        columns_to_drop = ['feature_name', 'feature_variant']
        existing_columns_to_drop = [col for col in columns_to_drop if col in result.columns]
        if existing_columns_to_drop:
            result = result.drop(existing_columns_to_drop)
            logger.info(f"Cleaned up intermediate feature columns: {existing_columns_to_drop}")
        
        return result
    
    
    def _add_environment_tags_vectorized(
        self, 
        df: pl.DataFrame, 
        key_variants: List[str]
    ) -> pl.DataFrame:
        """Add environment tags with ONE key variant per resource using vectorized operations."""
        result = df.clone()
        
        if not key_variants or 'environment_variant' not in df.columns:
            return result
        
        # Create deterministic key variant selection based on resource fingerprint
        # Handle null values by filling with empty strings before concatenation
        resource_fingerprint = pl.concat_str([
            pl.col('cloud_provider').fill_null(''),
            pl.lit(':'),
            pl.col('service').fill_null(''), 
            pl.lit(':'),
            pl.col('resource_id').fill_null(''),
            pl.lit(':env_key')
        ])
        
        # Select ONE key variant per resource using hash modulo
        key_variant_index = (resource_fingerprint.hash() % len(key_variants)).cast(pl.UInt32)
        
        # Create mapping from index to key variant
        selected_key_variant = key_variant_index
        for i, key_variant in enumerate(key_variants):
            if i == 0:
                selected_key_variant = pl.when(key_variant_index == i).then(pl.lit(key_variant))
            else:
                selected_key_variant = selected_key_variant.when(key_variant_index == i).then(pl.lit(key_variant))
        
        # Create separate columns for each possible key variant and use nulls
        tag_columns = []
        for key_variant in key_variants:
            tag_column = f"resource/tag:{key_variant}"
            # Only add value if this is the selected key variant for this resource
            tag_columns.append(
                pl.when(selected_key_variant == key_variant)
                .then(pl.col('environment_variant'))
                .otherwise(None)
                .alias(tag_column)
            )
        
        result = result.with_columns(tag_columns)
        return result
    
    
    def _enhance_k8s_labels_vectorized(
        self,
        df: pl.DataFrame,
        feature_key_variants: List[str],
        environment_key_variants: List[str]
    ) -> pl.DataFrame:
        """Enhance K8s labels JSON using vectorized operations."""
        
        if df.height == 0:
            return df
        
        logger.info(f"Processing K8s label enhancement for {df.height:,} resources")
        
        # Parse existing k8s_labels JSON into a structured format
        result = df.with_columns([
            pl.col('k8s_labels')
            .str.replace(r'^\s*$', '{}')  # Replace empty strings with {}
            .str.replace('', '{}')  # Replace None/null with {}
            .alias('k8s_labels_clean')
        ])
        
        # Each default namespace gets exactly one feature key variant deterministically
        
        if feature_key_variants and 'feature_variant' in result.columns:
            # Namespace-based key variant selection (one key per namespace)
            namespace_fingerprint_feature = pl.concat_str([
                pl.col('k8s_cluster').fill_null(''),
                pl.lit(':'),
                pl.col('k8s_namespace').fill_null(''),
                pl.lit(':feature_key')
            ])
            
            feature_key_index = (namespace_fingerprint_feature.hash() % len(feature_key_variants)).cast(pl.UInt32)
            
            # Map index to key variant using map_elements for proper handling
            def select_feature_key(index: int) -> str:
                return feature_key_variants[index]
            
            selected_feature_key = feature_key_index.map_elements(select_feature_key, return_dtype=pl.Utf8)
            
            # Create simple JSON with only the selected feature key variant
            # Each default namespace gets exactly one feature label entry
            result = result.with_columns([
                pl.format('{"{}":"{}"}',
                         selected_feature_key,
                         pl.col('feature_variant'))
                .alias('k8s_labels')
            ])
        else:
            # No enhancements needed
            result = result.with_columns([
                pl.col('k8s_labels_clean').alias('k8s_labels')
            ])
        
        # Clean up temporary and intermediate columns
        columns_to_drop = ['k8s_labels_clean', 'feature_name', 'feature_variant']
        existing_columns_to_drop = [col for col in columns_to_drop if col in result.columns]
        if existing_columns_to_drop:
            result = result.drop(existing_columns_to_drop)
            logger.info(f"Cleaned up temporary/intermediate columns: {existing_columns_to_drop}")
        
        return result
    
    def _inject_resource_id_variants_vectorized(self, df: pl.DataFrame) -> pl.DataFrame:
        """Inject feature value variants into resource IDs using per-resource deterministic selection."""
        if df.height == 0:
            return df
        
        result = df.clone()
        
        # Get available feature value variants from the processed features
        # This requires the base_allocation to get the right feature variants
        if 'feature_variant' not in result.columns:
            logger.warning("No feature_variant column found - cannot inject resource ID variants")
            return result
        
        # We need to get the actual feature variants for deterministic selection
        # For now, we'll work with the current feature_variant value and create per-resource selection
        
        # First, we need to determine available variants for this base allocation
        # Get a sample to determine the feature variants available
        sample_feature_name = result.select(pl.col('feature_name')).head(1).item() if 'feature_name' in result.columns else None
        
        if not sample_feature_name:
            logger.warning("No feature_name available for resource ID variant selection")
            return result
        
        # Get available variants for this feature (this is a simplified approach)
        # In practice, we'd get this from the profile config, but for now we'll work with what we have
        
        # Create deterministic per-resource feature variant selection
        # Each resource gets its own feature variant based on resource fingerprint
        # Handle null values by filling with empty strings before concatenation
        resource_fingerprint = pl.concat_str([
            pl.col('cloud_provider').fill_null(''),
            pl.lit(':'),
            pl.col('service').fill_null(''),
            pl.lit(':'),
            pl.col('resource_id').fill_null(''),
            pl.lit(':resource_id_feature_variant')
        ])
        
        # For demonstration, we'll use a simplified approach with common variants
        # This should be replaced with actual profile-based variant selection
        common_feature_variants = ["pulse-analytics", "pulse_analytics", "pad"]
        
        # Select feature variant per resource using hash modulo
        variant_index = (resource_fingerprint.hash() % len(common_feature_variants)).cast(pl.UInt32)
        
        # Map index to actual variant
        selected_feature_variant = variant_index
        for i, variant in enumerate(common_feature_variants):
            if i == 0:
                selected_feature_variant = pl.when(variant_index == i).then(pl.lit(variant))
            else:
                selected_feature_variant = selected_feature_variant.when(variant_index == i).then(pl.lit(variant))
        
        # Now inject the selected variant into resource IDs
        # Azure resource ID injection (vectorized)
        # Handle null values by filling with empty strings before comparison and string operations
        azure_condition = (
            (pl.col('cloud_provider').fill_null('') == 'Azure') & 
            pl.col('resource_id').fill_null('').str.starts_with('/subscriptions/')
        )
        
        result = result.with_columns([
            pl.when(azure_condition)
            .then(
                # Azure ARM resource ID injection using selected variant
                pl.col('resource_id')
                .fill_null('')
                .str.replace(
                    r'/resourceGroups/([^/]+)',
                    pl.format('/resourceGroups/{}-\\1', selected_feature_variant),
                    literal=False
                )
                .str.replace(
                    r'/([^/]+)$',
                    pl.format('/{}-\\1', selected_feature_variant),
                    literal=False
                )
            )
            .otherwise(
                # Regular resource ID injection using selected variant
                pl.when(pl.col('resource_id').fill_null('').str.contains('-'))
                .then(
                    # Has dashes - insert variant before last element
                    pl.concat_str([
                        pl.col('resource_id').fill_null('').str.split('-').list.slice(0, -1).list.join('-'),
                        pl.lit('-'),
                        selected_feature_variant,
                        pl.lit('-'),
                        pl.col('resource_id').fill_null('').str.split('-').list.get(-1)
                    ])
                )
                .otherwise(
                    # No dashes - append variant
                    pl.concat_str([pl.col('resource_id').fill_null(''), pl.lit('-'), selected_feature_variant])
                )
            )
            .alias('resource_id')
        ])
        
        return result
    
    
    def _inject_namespace_variants_vectorized_v2(self, df: pl.DataFrame) -> pl.DataFrame:
        """Inject value variants into K8s namespace using vectorized operations."""
        if df.height == 0:
            return df
        
        result = df.clone()
        
        # Build variants to inject using vectorized operations
        variants_parts = []
        
        # Add feature variant if available
        if 'feature_variant' in result.columns:
            variants_parts.append(pl.col('feature_variant'))
        
        # Add environment variant if available  
        if 'environment_variant' in result.columns:
            variants_parts.append(pl.col('environment_variant'))
        
        # Inject variants as suffix using vectorized operations
        if variants_parts:
            # Concatenate all variants with dashes
            variant_suffix = pl.concat_str(variants_parts, separator='-')
            
            # Create the new namespace: original-variants
            result = result.with_columns([
                pl.concat_str([
                    pl.col('k8s_namespace'),
                    pl.lit('-'),
                    variant_suffix
                ]).alias('k8s_namespace')
            ])
        
        return result
    
    

    def _validate_profile_configuration(self):
        """
        Validate critical profile configuration and fail fast if missing.
        
        Raises:
            RuntimeError: If critical configuration is missing or invalid
        """
        errors = []
        
        # Validate feature key variants
        if not hasattr(self.profile_config, "features") or not self.profile_config.features:
            errors.append("Missing features profile configuration")
        elif not self.profile_config.features.key_variants:
            errors.append("Missing feature key_variants in profile configuration")
        
        # Validate environment key variants  
        if not hasattr(self.profile_config, "environments") or not self.profile_config.environments:
            errors.append("Missing environments profile configuration")
        elif not self.profile_config.environments.key_variants:
            errors.append("Missing environment key_variants in profile configuration")
        
        # Validate that we have at least one valid base allocation
        valid_allocations = ["analytics_data", "project_management", "customer_service", "marketing", "security"]
        feature_allocations = []
        
        if hasattr(self.profile_config, "features") and self.profile_config.features:
            if hasattr(self.profile_config.features, "values"):
                feature_allocations = list(self.profile_config.features.values.keys())
        
        if not feature_allocations:
            errors.append("No feature allocations found in profile configuration")
        else:
            missing_allocations = [alloc for alloc in valid_allocations if alloc not in feature_allocations]
            if missing_allocations:
                errors.append(f"Missing feature allocations in profile: {missing_allocations}")
        
        # Validate that each allocation has features with proper structure
        for allocation in feature_allocations:
            if allocation in valid_allocations:  # Only check expected allocations
                features = self.profile_config.features.get_features(allocation)
                if not features:
                    errors.append(f"No features found for allocation \"{allocation}\"")
                else:
                    # Check each feature has required fields
                    for i, feature_config in enumerate(features):
                        if not isinstance(feature_config, dict):
                            errors.append(f"Feature {i} in \"{allocation}\" is not a dictionary")
                            continue
                            
                        # Check for required fields with correct names
                        if "value" not in feature_config:
                            errors.append(f"Feature {i} in \"{allocation}\" missing \"value\" field")
                        if "variants" not in feature_config:
                            errors.append(f"Feature {i} in \"{allocation}\" missing \"variants\" field")
                        elif not feature_config.get("variants"):
                            errors.append(f"Feature {i} in \"{allocation}\" has empty \"variants\" list")
        
        # Fail fast if any errors found
        if errors:
            error_msg = "Critical metadata profile configuration errors:\n" + "\n".join(f"  - {error}" for error in errors)
            logger.error(error_msg)
            raise RuntimeError(error_msg)
    

