"""
Configuration skeleton generation for generate configs.

Automates the tedious first pass of writing generate configs by analyzing
reference data and creating YAML skeleton files with smart defaults.
"""

import hashlib
import math
import random
import re
from datetime import date
from pathlib import Path
from typing import Any, Optional

import polars as pl
import typer
import yaml
from dateutil.relativedelta import relativedelta
from rich.console import Console
from rich.progress import (
    BarColumn,
    Progress,
    SpinnerColumn,
    TextColumn,
    TimeElapsedColumn,
)

from ...cli.common import BaseCommand
from ...cli.data.explore import DataExploreCommand
from ...cli.data.reference_loader import load_reference_data
from ...services.environment_service import env_service

console = Console()

# Configuration constants
COMBINATION_THRESHOLD = 50  # Threshold for smart defaults
MAX_USAGE_FAMILIES = 20  # Maximum usage families to include
MIN_RESOURCE_TYPE_COMBINATIONS = 5  # Minimum combinations for meaningful resource types
YAML_RESOURCE_TYPE_INDENT_LEVEL = 8  # Expected YAML indent level for resource types


class ReferenceDataSession:
    """Session-scoped cache for reference data to avoid multiple S3 reads."""
    
    def __init__(self):
        """Load reference data once during session creation."""
        # Validate AWS credentials once upfront
        env_service.validate_aws_credentials_and_access()
        
        # Load reference data once for the entire session
        self._data = load_reference_data()
        console.print("📚 Reference data cached for session - no additional S3 reads required")
    
    def get_filtered_data(self, cloud_provider=None, service=None, usage_family=None, 
                         resource_type=None, operation=None, usage_type=None, 
                         has_pricing=False, exact_match=False, limit=None) -> pl.DataFrame:
        """Filter cached data with same API as explorer.get_structured_data()."""
        
        filtered_df = self._data
        
        # Apply filters progressively (same logic as DataExploreCommand._apply_filters)
        filters = {
            "cloud_provider": cloud_provider,
            "service": service,
            "usage_family": usage_family,
            "resource_type": resource_type,
            "operation": operation,
            "usage_type": usage_type
        }
        
        for column, value in filters.items():
            if value is not None:
                if exact_match:
                    filtered_df = filtered_df.filter(pl.col(column) == value)
                else:
                    # Case-insensitive contains matching, handling nulls
                    filtered_df = filtered_df.filter(
                        pl.col(column).is_not_null() & 
                        pl.col(column).str.to_lowercase().str.contains(value.lower())
                    )
        
        # Apply pricing filter
        if has_pricing:
            filtered_df = filtered_df.filter(pl.col("has_pricing"))
        
        return filtered_df.head(limit) if limit else filtered_df


class GenerateInitConfigsCommand(BaseCommand):
    """Generate config skeleton files from reference data analysis."""
    
    def __init__(self):
        self.explorer = DataExploreCommand()
    
    def init_configs(
        self,
        provider: str,
        output_dir: str = "configs/generate/prototype",
        category: str = "base",
        top_services: int = 50,
        verbose: bool = False
    ):
        """Generate skeleton config files for top services."""
        
        # Create session with cached reference data (single S3 read)
        session = ReferenceDataSession()
        
        console.print(f"🚀 Generating config skeletons for {provider}")
        console.print(f"📁 Output directory: {output_dir}")
        console.print(f"📊 Top services: {top_services}")
        console.print(f"🏷️ Category: {category}")
        
        # Create output directory structure
        provider_dir = Path(output_dir) / category / provider.lower()
        provider_dir.mkdir(parents=True, exist_ok=True)
        
        # Discover top services using cached data
        console.print(f"\n🔍 Discovering top services for {provider}...")
        top_services_data = self._discover_top_services(provider, top_services, verbose, session)
        
        if not top_services_data:
            console.print(f"❌ No services found for provider {provider}")
            return
            
        console.print(f"✅ Found {len(top_services_data)} services to generate")
        
        # Generate config file for each service using cached data
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            BarColumn(),
            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
            TimeElapsedColumn(),
            console=console,
        ) as progress:
            
            task = progress.add_task("Processing services...", total=len(top_services_data))
            
            for service_info in top_services_data:
                service_name = service_info['service']
                combinations = service_info['combinations']
                
                progress.update(task, description=f"Analyzing {service_name} ({combinations:,} combinations)...")
                
                # Generate skeleton for this service using cached session data (heavy work)
                config_data = self._generate_service_config(
                    provider, service_name, category, combinations, verbose, session
                )
                
                progress.update(task, description=f"Writing {service_name} config...")
                
                # Write YAML file (light work)
                filename = f"{provider.lower()}.{self._normalize_service_name(service_name)}.yaml"
                file_path = provider_dir / filename
                
                self._write_yaml_file(file_path, config_data)
                
                progress.advance(task)
        
        console.print(f"\n✅ Generated {len(top_services_data)} config files in {provider_dir}")
        console.print("📝 Next steps:")
        console.print("   1. Review generated files and customize as needed")
        console.print(f"   2. Run: czbillgen generate validate {provider_dir}")
        console.print(f"   3. Run: czbillgen analyze config {provider_dir}")
    
    def _discover_top_services(self, provider: str, top_services: int, verbose: bool, session: ReferenceDataSession) -> list[dict[str, Any]]:
        """Discover top services for a provider using cached session data."""
        
        # Get service-level data from cached session (no S3 read)
        service_df = session.get_filtered_data(cloud_provider=provider, has_pricing=False, exact_match=False)
        
        # If we got multiple providers, let's see what we have and potentially filter to the one with actual services
        if not service_df.is_empty() and verbose:
            unique_providers_df = service_df.select("cloud_provider").unique()
            if unique_providers_df.height > 1:
                unique_providers = unique_providers_df.to_series()
                console.print(f"[cyan]Multiple providers matched '{provider}': {list(unique_providers)}[/cyan]")
                
                # Check which provider has the most non-null services using vectorized operations
                provider_stats = (
                    service_df
                    .filter(pl.col("service").is_not_null() & (pl.col("service").str.len_chars() > 0))
                    .group_by("cloud_provider")
                    .agg(pl.len().alias("non_null_services"))
                )
                
                for prov, count in zip(
                    provider_stats.select("cloud_provider").to_series(),
                    provider_stats.select("non_null_services").to_series()
                ):
                    console.print(f"[cyan]  {prov}: {count} records with non-null services[/cyan]")
        
        if service_df.is_empty():
            console.print(f"[yellow]No reference data found for provider {provider}[/yellow]")
            # Debug: let's see what providers are available using cached data
            if verbose:
                all_data = session.get_filtered_data(has_pricing=False)
                if not all_data.is_empty():
                    available_providers = all_data.select("cloud_provider").unique().to_series()
                    console.print(f"[yellow]Available providers: {list(available_providers)}[/yellow]")
            return []
        
        console.print(f"[green]Found {service_df.height} records for {provider}[/green]")
        
        # Debug: Show data statistics if verbose
        if verbose:
            # Simple debug info without performance-heavy operations
            provider_stats = (
                service_df
                .group_by("cloud_provider")
                .agg([
                    pl.col("service").filter(pl.col("service").is_not_null()).n_unique().alias("unique_services"),
                    pl.len().alias("total_records")
                ])
            )
            
            for prov, services, records in zip(
                provider_stats.select("cloud_provider").to_series(),
                provider_stats.select("unique_services").to_series(),
                provider_stats.select("total_records").to_series()
            ):
                console.print(f"[cyan]{prov}: {services} unique services, {records} total records[/cyan]")
        
        # Group by service and count combinations and usage data
        service_summary = (
            service_df
            .group_by("service")
            .agg([
                pl.len().alias("combinations"),
                pl.col("has_pricing").sum().alias("with_usage")  # Count of records with pricing data
            ])
            .filter(pl.col("service").is_not_null())  # Filter out null services at the polars level
        )
        
        # Apply provider-specific service filtering
        if provider == "AWS Marketplace":
            # For AWS Marketplace, only include services containing "(Amazon Bedrock Edition)"
            service_summary = service_summary.filter(
                pl.col("service").str.contains(r"\(Amazon Bedrock Edition\)", literal=False)
            )
            if verbose:
                console.print(f"[cyan]Applied AWS Marketplace filter: only services containing '(Amazon Bedrock Edition)'[/cyan]")
        
        service_summary = (
            service_summary
            .sort("combinations", descending=True)
            .head(top_services)
        )
        
        if verbose:
            console.print(f"[cyan]Service summary rows: {service_summary.height}[/cyan]")
            if service_summary.height > 0:
                console.print(service_summary.head(3))
        
        if service_summary.is_empty():
            console.print(f"[yellow]No services found after filtering for provider {provider}[/yellow]")
            return []
        
        # Convert to list of dictionaries using vectorized operations
        # Filter out empty/null service names at DataFrame level
        valid_services_df = service_summary.filter(
            pl.col("service").is_not_null() & 
            (pl.col("service").str.len_chars() > 0)
        )
        
        # Convert to list efficiently using zip
        services_data = [
            {
                "service": service,
                "combinations": combinations, 
                "with_usage": with_usage
            }
            for service, combinations, with_usage in zip(
                valid_services_df.select("service").to_series(),
                valid_services_df.select("combinations").to_series(),
                valid_services_df.select("with_usage").to_series()
            )
        ]
        
        return services_data
    
    def _generate_service_config(
        self, 
        provider: str, 
        service_name: str, 
        category: str, 
        combinations: int,
        verbose: bool,
        session: ReferenceDataSession
    ) -> dict[str, Any]:
        """Generate YAML config structure for a service with discovered usage families."""
        
        # Calculate date range (11 months past, 4 months future)
        # Start date: first day of month 11 months ago
        # End date: last day of month 4 months from now
        today = date.today()
        start_month = today - relativedelta(months=11)
        start_date = start_month.replace(day=1)
        
        end_month = today + relativedelta(months=4)
        # Get last day of the end month
        end_date = (end_month.replace(day=1) + relativedelta(months=1) - relativedelta(days=1))
        
        # Generate job name
        job_name = f"{category.title()}: {provider.upper()} {self._format_service_display_name(service_name)}"
        
        # Determine placeholder cost based on provider tier system
        placeholder_cost = self._get_provider_tier_cost(provider)
        
        # Discover usage families for this service using cached session data
        usage_families = self._discover_usage_families(provider, service_name, verbose, session)
        
        # Calculate intelligent resource count based on reference data matches
        resource_count = self._calculate_resource_count(provider, service_name, usage_families, verbose, session)
        
        config = {
            "generation_jobs": [
                {
                    "name": job_name,
                    "start_date": start_date.strftime("%Y-%m-%d"),
                    "end_date": end_date.strftime("%Y-%m-%d"),
                    "total_cost": placeholder_cost,
                    "resource_count": resource_count,
                    "filters": [
                        {
                            "cloud_provider": provider,
                            "variance_profile": "linear_growth",
                            "anomaly_profile": "",
                            "services": [
                                {
                                    "name": service_name,
                                    "usage_families": usage_families
                                }
                            ]
                        }
                    ]
                }
            ]
        }
        
        # Add comments about what needs manual attention
        comments = []
        if combinations > COMBINATION_THRESHOLD:
            comments.append(f"Service has {combinations} combinations - manual selection needed for operations/usage_types")
        if not usage_families:
            comments.append("TODO: Populate usage_families with top families from reference data")
        comments.extend([
            "TODO: Review and exclude 'service-usage' resource types if present",
            "TODO: Calibrate total_cost using 'czbillgen analyze config'"
        ])
        
        if comments:
            config["_comments"] = comments
        
        return config
    
    def _calculate_resource_count(self, provider: str, service_name: str, usage_families: list[dict[str, Any]], verbose: bool, session: ReferenceDataSession) -> int:
        """Calculate resource count to achieve target total resource count per job ≥ 50.
        
        Formula: resource_count × ref_matches ≥ 50
        Therefore: resource_count = max(1, ceil(50 / max(1, ref_matches)))
        
        Counts logical combinations (6 key columns) not physical rows.
        """
        # Count total logical reference matches for the service configuration
        total_ref_matches = self._count_logical_combinations(provider, service_name, usage_families, verbose, session)
        
        if total_ref_matches == 0:
            if verbose:
                console.print(f"[yellow]No reference data found for {service_name}, using resource_count=1[/yellow]")
            return 1
        
        # Calculate resource count using the target formula
        # target: resource_count × ref_matches ≥ 50
        # therefore: resource_count = max(1, ceil(50 / max(1, ref_matches)))
        calculated_resource_count = max(1, math.ceil(50 / max(1, total_ref_matches)))
        
        if verbose:
            console.print(f"[green]Resource count calculation for {service_name}:[/green]")
            console.print(f"[green]  Total reference matches: {total_ref_matches}[/green]")
            console.print(f"[green]  Target total per job: ≥50[/green]")
            console.print(f"[green]  Calculated resource_count: {calculated_resource_count}[/green]")
            console.print(f"[green]  Expected total per job: {calculated_resource_count * total_ref_matches}[/green]")
        
        return calculated_resource_count
    
    def _count_logical_combinations(self, provider: str, service_name: str, usage_families: list[dict[str, Any]], verbose: bool, session: ReferenceDataSession) -> int:
        """Count logical combinations (6 key columns only) for the service configuration.
        
        This matches the behavior of the fixed analyze command by counting unique combinations
        of: cloud_provider, service, usage_family, resource_type, operation, usage_type
        
        Ignores instance_type and other dimensions that create row multiplication.
        """
        # Get all data for this service from cached session
        service_df = session.get_filtered_data(
            cloud_provider=provider,
            service=service_name,
            has_pricing=False
        )
        
        if service_df.is_empty():
            return 0
        
        total_ref_matches = 0
        key_columns = ["cloud_provider", "service", "usage_family", "resource_type", "operation", "usage_type"]
        
        # If no usage families configured (wildcard case), count unique combinations at service level
        if not usage_families:
            unique_combinations = service_df.select(key_columns).unique()
            total_ref_matches = unique_combinations.height
            if verbose:
                console.print(f"[cyan]Wildcard config: {total_ref_matches} logical combinations for {service_name}[/cyan]")
        else:
            # Count logical matches for each configured usage family
            for usage_family_cfg in usage_families:
                family_name = usage_family_cfg.get("name")
                resource_types_cfg = usage_family_cfg.get("resource_types", [])
                
                # Filter by usage family
                family_df = service_df.filter(pl.col("usage_family") == family_name) if family_name else service_df
                
                if family_df.is_empty():
                    continue
                
                # If no resource types configured, count unique combinations at family level
                if not resource_types_cfg:
                    unique_combinations = family_df.select(key_columns).unique()
                    family_matches = unique_combinations.height
                    total_ref_matches += family_matches
                    if verbose:
                        console.print(f"[cyan]  Wildcard resource_types for {family_name}: +{family_matches} matches[/cyan]")
                else:
                    # Count logical matches for each configured resource type
                    for resource_type_cfg in resource_types_cfg:
                        resource_type_name = resource_type_cfg.get("name")
                        operations_cfg = resource_type_cfg.get("operations", [])
                        
                        # Filter by resource type
                        resource_df = family_df.filter(pl.col("resource_type") == resource_type_name) if resource_type_name else family_df
                        
                        if resource_df.is_empty():
                            continue
                        
                        # If no operations configured, count unique combinations at resource type level
                        # This handles the GCP case where operations are None/empty
                        if not operations_cfg:
                            unique_combinations = resource_df.select(key_columns).unique()
                            resource_matches = unique_combinations.height
                            total_ref_matches += resource_matches
                            if verbose:
                                console.print(f"[cyan]    Wildcard operations for {family_name}.{resource_type_name}: +{resource_matches} matches[/cyan]")
                        else:
                            # Count logical matches for each configured operation
                            for operation_cfg in operations_cfg:
                                operation_name = operation_cfg.get("name")
                                usage_types_cfg = operation_cfg.get("usage_types", [])
                                
                                # Filter by operation
                                operation_df = resource_df.filter(pl.col("operation") == operation_name) if operation_name else resource_df
                                
                                if operation_df.is_empty():
                                    continue
                                
                                # If no usage types configured, count unique combinations at operation level
                                if not usage_types_cfg:
                                    unique_combinations = operation_df.select(key_columns).unique()
                                    operation_matches = unique_combinations.height
                                    total_ref_matches += operation_matches
                                    if verbose:
                                        console.print(f"[cyan]      Wildcard usage_types for {family_name}.{resource_type_name}.{operation_name}: +{operation_matches} matches[/cyan]")
                                else:
                                    # Count logical matches for specific usage types
                                    for usage_type in usage_types_cfg:
                                        usage_type_df = operation_df.filter(pl.col("usage_type") == usage_type)
                                        unique_combinations = usage_type_df.select(key_columns).unique()
                                        usage_type_matches = unique_combinations.height
                                        total_ref_matches += usage_type_matches
                                        if verbose:
                                            console.print(f"[cyan]        Specific usage_type {usage_type}: +{usage_type_matches} matches[/cyan]")
        
        return total_ref_matches
    
    def _discover_usage_families(self, provider: str, service_name: str, verbose: bool, session: ReferenceDataSession) -> list[dict[str, Any]]:
        """Discover top usage families for a service using cached session data."""
        
        # Get usage family data from cached session (no S3 read)
        family_df = session.get_filtered_data(
            cloud_provider=provider, 
            service=service_name,
            has_pricing=False
        )
        
        if family_df.is_empty():
            if verbose:
                console.print(f"[yellow]No usage families found for {service_name}[/yellow]")
            return []
        
        # Group by usage family and count combinations
        family_summary = (
            family_df
            .group_by("usage_family")
            .agg([
                pl.len().alias("combinations"),
                pl.col("has_pricing").sum().alias("with_usage")
            ])
            .sort("combinations", descending=True)
        )
        
        # Filter out usage families containing exclusion terms using vectorized operations
        excluded_terms = ["tax", "discount", "fee", "credit"]
        
        # Create exclusion filter using vectorized string operations
        exclusion_filter = pl.lit(False)
        for term in excluded_terms:
            exclusion_filter = exclusion_filter | pl.col("usage_family").str.to_lowercase().str.contains(term, literal=True)
        
        # Apply filters and get top families
        filtered_families_df = (
            family_summary
            .filter(pl.col("usage_family").is_not_null())  # Remove nulls
            .filter(~exclusion_filter)  # Remove excluded terms
            .head(MAX_USAGE_FAMILIES)  # Get top families
        )
        
        # Debug excluded families if verbose
        if verbose:
            excluded_families_df = (
                family_summary
                .filter(pl.col("usage_family").is_not_null())
                .filter(exclusion_filter)
            )
            for family_name in excluded_families_df.select("usage_family").to_series():
                console.print(f"[dim]Excluding usage family: {family_name} (contains excluded term)[/dim]")
        
        # Process families using vectorized operations
        usage_families = []
        for family_name, combinations in zip(
            filtered_families_df.select("usage_family").to_series(),
            filtered_families_df.select("combinations").to_series()
        ):
            if verbose:
                console.print(f"  Processing usage family: {family_name} ({combinations} combinations)")
            
            # Discover resource types for this usage family using cached session data
            resource_types = self._discover_resource_types(
                provider, service_name, family_name, combinations, verbose, session
            )
            
            usage_families.append({
                "name": family_name,
                "resource_types": resource_types
            })
        
        return usage_families
    
    def _discover_resource_types(self, provider: str, service_name: str, usage_family: str, family_combinations: int, verbose: bool, session: ReferenceDataSession) -> list[dict[str, Any]]:
        """Discover resource types for a usage family using cached session data."""
        
        # Get resource type data from cached session (no S3 read)
        resource_df = session.get_filtered_data(
            cloud_provider=provider,
            service=service_name,
            usage_family=usage_family,
            has_pricing=False
        )
        
        if resource_df.is_empty():
            return []
        
        # Group by resource type and count combinations
        resource_summary = (
            resource_df
            .group_by("resource_type")
            .agg([
                pl.len().alias("combinations"),
                pl.col("has_pricing").sum().alias("with_usage")
            ])
            .filter(pl.col("resource_type") != "service-usage")  # Exclude service-usage
            .filter(pl.col("combinations") >= MIN_RESOURCE_TYPE_COMBINATIONS)  # Only include resource types with meaningful data
            .sort("combinations", descending=True)
            # Removed .head(3) limit - include all resource types with meaningful data
        )
        
        # Process resource types using vectorized operations
        resource_types = []
        for resource_type_name, resource_combinations in zip(
            resource_summary.select("resource_type").to_series(),
            resource_summary.select("combinations").to_series()
        ):
            if verbose:
                console.print(f"    Processing resource type: {resource_type_name} ({resource_combinations} combinations)")
            
            # Determine if we should populate operations/usage_types based on combination threshold
            operations = []
            smart_defaults_comment = None
            
            if resource_combinations <= COMBINATION_THRESHOLD:
                operations = self._discover_operations(
                    provider, service_name, usage_family, resource_type_name, verbose, session
                )
            else:
                # Apply smart defaults for >50 combinations using cached session data
                operations, smart_defaults_comment = self._apply_smart_defaults(
                    provider, service_name, usage_family, resource_type_name, 
                    resource_combinations, verbose, session
                )
            
            resource_type_data = {
                "name": resource_type_name,
                "operations": operations
            }
            
            # Add comment if smart defaults were applied
            if smart_defaults_comment:
                resource_type_data["_comment"] = smart_defaults_comment
            
            resource_types.append(resource_type_data)
        
        return resource_types
    
    def _discover_operations(self, provider: str, service_name: str, usage_family: str, resource_type: str, verbose: bool, session: ReferenceDataSession) -> list[dict[str, Any]]:
        """Discover operations and usage types using cached session data."""
        
        try:
            # Get operation data from cached session (no S3 read)
            operation_df = session.get_filtered_data(
                cloud_provider=provider,
                service=service_name,
                usage_family=usage_family,
                resource_type=resource_type,
                has_pricing=False
            )
            
            if operation_df.is_empty():
                return []
            
            # Group by operation and collect usage types using vectorized operations
            operations_summary = (
                operation_df
                .filter(pl.col("operation").is_not_null())  # Remove null operations
                .group_by("operation")
                .agg([
                    pl.col("usage_type").filter(pl.col("usage_type").is_not_null()).unique().slice(0, 10).alias("usage_types")
                ])
                .filter(pl.col("usage_types").list.len() > 0)  # Only keep operations with valid usage types
            )
            
            # Convert to operations list efficiently
            operations = [
                {
                    "name": operation_name,
                    "usage_types": usage_types_list.to_list() if hasattr(usage_types_list, 'to_list') else list(usage_types_list)
                }
                for operation_name, usage_types_list in zip(
                    operations_summary.select("operation").to_series(),
                    operations_summary.select("usage_types").to_series()
                )
            ]
            
            # Limit to top 3 operations to keep configs manageable
            return operations[:3]
            
        except Exception as e:
            if verbose:
                console.print(f"[red]Error discovering operations: {e}[/red]")
            return []
    
    def _apply_smart_defaults(
        self, 
        provider: str, 
        service_name: str, 
        usage_family: str, 
        resource_type: str, 
        resource_combinations: int,
        verbose: bool,
        session: ReferenceDataSession
    ) -> tuple[list[dict[str, Any]], str]:
        """Apply smart defaults for resource types with >50 combinations using cached session data.
        
        Returns:
            Tuple of (operations_list, comment_string)
        """
        try:
            # Get all operation and usage type data from cached session (no S3 read)
            operation_df = session.get_filtered_data(
                cloud_provider=provider,
                service=service_name,
                usage_family=usage_family,
                resource_type=resource_type,
                has_pricing=False
            )
            
            if operation_df.is_empty():
                return [], f"Resource type '{resource_type}' has {resource_combinations} combinations but no data available"
            
            # Check if operations are None or missing - this indicates natural hierarchy termination
            operations_with_data = operation_df.filter(pl.col("operation").is_not_null())
            
            # If no non-null operations exist, this is a natural stopping point, not an error
            if operations_with_data.is_empty():
                # Natural hierarchy termination - operations are None/missing in reference data
                return [], None  # No error message - this is expected for some providers
            
            # Get operations and usage types with pricing info using vectorized operations
            operations_summary = (
                operations_with_data
                .filter(pl.col("usage_type").is_not_null())  # Still filter null usage types
                .group_by("operation")
                .agg([
                    # Get usage types with pricing
                    pl.col("usage_type").filter(pl.col("has_pricing")).unique().alias("usage_types_with_pricing"),
                    # Get usage types without pricing
                    pl.col("usage_type").filter(~pl.col("has_pricing")).unique().alias("usage_types_without_pricing"),
                ])
                .with_columns([
                    (pl.col("usage_types_with_pricing").list.len() + 
                     pl.col("usage_types_without_pricing").list.len()).alias("total_usage_types")
                ])
                .filter(pl.col("total_usage_types") > 0)  # Only operations with valid usage types
            )
            
            # Convert to operations data efficiently - ensure list columns are converted to Python lists
            operations_data = [
                {
                    "name": operation_name,
                    "usage_types_with_pricing": with_pricing.to_list() if hasattr(with_pricing, 'to_list') else list(with_pricing),
                    "usage_types_without_pricing": without_pricing.to_list() if hasattr(without_pricing, 'to_list') else list(without_pricing),
                    "total_usage_types": total
                }
                for operation_name, with_pricing, without_pricing, total in zip(
                    operations_summary.select("operation").to_series(),
                    operations_summary.select("usage_types_with_pricing").to_series(),
                    operations_summary.select("usage_types_without_pricing").to_series(), 
                    operations_summary.select("total_usage_types").to_series()
                )
            ]
            
            # Create deterministic seed for reproducible results
            context = f"{provider}:{service_name}:{usage_family}:{resource_type}"
            seed = int(hashlib.md5(context.encode()).hexdigest()[:8], 16) % (2**31)
            rng = random.Random(seed)
            
            # Apply Case 2a or Case 2b logic
            num_operations = len(operations_data)
            selected_operations = []
            
            if num_operations <= COMBINATION_THRESHOLD:
                # Case 2a: ≤50 operations available - include all operations, distribute 50 usage types
                usage_types_per_operation = 50 // num_operations
                remainder = 50 % num_operations
                
                usage_with_pricing_count = 0
                usage_without_pricing_count = 0
                
                for i, op_data in enumerate(operations_data):
                    # Calculate how many usage types this operation should get
                    target_count = usage_types_per_operation
                    if i < remainder:  # Distribute remainder across first operations
                        target_count += 1
                    
                    # Prioritize usage types with pricing data
                    available_with_pricing = op_data["usage_types_with_pricing"][:]
                    available_without_pricing = op_data["usage_types_without_pricing"][:]
                    
                    rng.shuffle(available_with_pricing)
                    rng.shuffle(available_without_pricing)
                    
                    selected_usage_types = []
                    
                    # First, add usage types with pricing up to target count
                    selected_usage_types.extend(available_with_pricing[:target_count])
                    usage_with_pricing_count += len(available_with_pricing[:target_count])
                    
                    # If we need more, add usage types without pricing
                    remaining_needed = target_count - len(selected_usage_types)
                    if remaining_needed > 0:
                        selected_usage_types.extend(available_without_pricing[:remaining_needed])
                        usage_without_pricing_count += len(available_without_pricing[:remaining_needed])
                    
                    if selected_usage_types:
                        selected_operations.append({
                            "name": op_data["name"],
                            "usage_types": selected_usage_types
                        })
                
                comment = f"Resource type '{resource_type}' has {resource_combinations} combinations - applied smart defaults\nSelected 50 combinations across {num_operations} operations: {usage_with_pricing_count} with usage data + {usage_without_pricing_count} without usage data\nReview and customize as needed for your demo scenario"
                
            else:
                # Case 2b: >50 operations available - select 50 operations with 1 usage type each
                rng.shuffle(operations_data)
                selected_ops_data = operations_data[:50]
                
                usage_with_pricing_count = 0
                usage_without_pricing_count = 0
                
                for op_data in selected_ops_data:
                    # Prioritize usage types with pricing
                    if op_data["usage_types_with_pricing"]:
                        selected_usage_type = rng.choice(op_data["usage_types_with_pricing"])
                        usage_with_pricing_count += 1
                    elif op_data["usage_types_without_pricing"]:
                        selected_usage_type = rng.choice(op_data["usage_types_without_pricing"])
                        usage_without_pricing_count += 1
                    else:
                        continue  # Skip operations with no valid usage types
                    
                    selected_operations.append({
                        "name": op_data["name"],
                        "usage_types": [selected_usage_type]
                    })
                
                comment = f"Resource type '{resource_type}' has {resource_combinations} combinations - applied smart defaults\nSelected 50 operations from {num_operations} available: {usage_with_pricing_count} with usage data + {usage_without_pricing_count} without usage data\nReview and customize as needed for your demo scenario"
            
            if verbose:
                console.print(f"[cyan]Applied smart defaults to {resource_type}: {len(selected_operations)} operations selected[/cyan]")
            
            return selected_operations, comment
            
        except Exception as e:
            if verbose:
                console.print(f"[red]Error applying smart defaults for {resource_type}: {e}[/red]")
            return [], f"Resource type '{resource_type}' has {resource_combinations} combinations - manual selection needed (error in smart defaults)"
    
    def _normalize_service_name(self, service_name: str) -> str:
        """Normalize service name for filename."""
        # Convert to lowercase, replace dots and special chars with nothing/hyphens
        normalized = re.sub(r'[^a-zA-Z0-9]', '', service_name.lower())
        return normalized
    
    def _format_service_display_name(self, service_name: str) -> str:
        """Format service name for display in job names."""
        # Clean up service names for better readability
        if service_name.startswith("Amazon"):
            return service_name.replace("Amazon", "").strip()
        elif service_name.startswith("AWS"):
            return service_name.replace("AWS", "").strip()
        elif service_name.startswith("microsoft."):
            return service_name.replace("microsoft.", "").title()
        return service_name
    
    def _write_yaml_file(self, file_path: Path, config_data: dict[str, Any]) -> None:
        """Write configuration data to YAML file with proper formatting and inline comments."""
        
        with open(file_path, 'w') as f:
            # Write top-level comments if present
            if '_comments' in config_data:
                for comment in config_data['_comments']:
                    f.write(f"# {comment}\n")
                f.write("\n")
                # Remove comments from data before YAML serialization
                config_data = {k: v for k, v in config_data.items() if k != '_comments'}
            
            # First pass: write YAML without embedded comments
            clean_config = self._strip_embedded_comments(config_data)
            yaml_content = yaml.dump(clean_config, default_flow_style=False, sort_keys=False, indent=2)
            
            # Second pass: add embedded comments back in appropriate places
            yaml_lines = yaml_content.split('\n')
            enhanced_lines = self._add_embedded_comments(yaml_lines, config_data)
            
            f.write('\n'.join(enhanced_lines))
    
    def _strip_embedded_comments(self, data: Any) -> Any:
        """Recursively remove embedded comments from data structure."""
        if isinstance(data, dict):
            return {k: self._strip_embedded_comments(v) for k, v in data.items() if k != '_comment'}
        elif isinstance(data, list):
            return [self._strip_embedded_comments(item) for item in data]
        else:
            return data
    
    def _add_embedded_comments(self, yaml_lines: list[str], original_data: Any) -> list[str]:
        """Add embedded comments back to YAML lines where appropriate."""
        enhanced_lines = []
        
        for line in yaml_lines:
            enhanced_lines.append(line)
            
            # Check if this line represents a resource type with a comment
            if 'name:' in line and 'resource_types' not in line:
                # Extract the name value and check if we have a comment for it
                indent_level = len(line) - len(line.lstrip())
                if indent_level >= YAML_RESOURCE_TYPE_INDENT_LEVEL:  # This is likely a resource type name
                    try:
                        name_part = line.split('name:')[1].strip()
                        resource_name = name_part
                        
                        # Look for this resource type in the original data structure
                        comment = self._find_resource_type_comment(original_data, resource_name)
                        if comment:
                            # Add comment lines after the resource type name
                            comment_lines = comment.split('\n')
                            for comment_line in comment_lines:
                                enhanced_lines.append(' ' * (indent_level + 2) + f"# {comment_line}")
                    except (IndexError, AttributeError):
                        pass  # Skip if we can't parse the line
        
        return enhanced_lines
    
    def _find_resource_type_comment(self, data: Any, resource_name: str) -> Optional[str]:
        """Find comment for a specific resource type in the data structure."""
        if isinstance(data, dict):
            for key, value in data.items():
                if key == 'resource_types' and isinstance(value, list):
                    for resource_type in value:
                        if (isinstance(resource_type, dict) and 
                            resource_type.get('name') == resource_name and 
                            '_comment' in resource_type):
                            return resource_type['_comment']
                
                # Recurse into nested structures
                comment = self._find_resource_type_comment(value, resource_name)
                if comment:
                    return comment
        elif isinstance(data, list):
            for item in data:
                comment = self._find_resource_type_comment(item, resource_name)
                if comment:
                    return comment
        
        return None
    
    def _get_provider_tier_cost(self, provider: str) -> float:
        """Get total cost based on provider tier system.
        
        Cost Tier System:
        - Tier 1 - Major Cloud Providers ($20M): AWS, Azure, GCP
        - Tier 2 - Premium AI/Specialized ($15M): AWS Marketplace, Anthropic, OpenAI  
        - Tier 3 - Data Platforms ($10M): Databricks, Snowflake, MongoDB
        - Tier 4 - Everything Else ($5M): All other providers
        """
        
        # Tier 1 - Major Cloud Providers ($20M)
        tier_1_providers = {"AWS", "Azure", "GCP"}
        
        # Tier 2 - Premium AI/Specialized ($15M)
        tier_2_providers = {"AWS Marketplace", "Anthropic", "OpenAI"}
        
        # Tier 3 - Data Platforms ($10M)
        tier_3_providers = {"Databricks", "Snowflake", "MongoDB"}
        
        # Check exact matches first, then case-insensitive matches
        if provider in tier_1_providers:
            return 20000000.00  # $20M
        elif provider in tier_2_providers:
            return 15000000.00  # $15M
        elif provider in tier_3_providers:
            return 10000000.00  # $10M
        else:
            # Check case-insensitive matches for robustness
            provider_upper = provider.upper()
            
            if provider_upper in {"AWS", "AZURE", "GCP"}:
                return 20000000.00  # $20M
            elif provider_upper in {"AWS MARKETPLACE", "ANTHROPIC", "OPENAI"}:
                return 15000000.00  # $15M
            elif provider_upper in {"DATABRICKS", "SNOWFLAKE", "MONGODB"}:
                return 10000000.00  # $10M
            else:
                # Tier 4 - Everything Else ($5M)
                return 5000000.00  # $5M


def create_init_configs_command():
    """Create the init-configs command function for the CLI."""
    
    def init_configs(
        provider: str = typer.Option(
            ...,
            "--provider",
            help="Cloud provider (AWS, Azure, GCP)"
        ),
        output_dir: str = typer.Option(
            "configs/generate/prototype",
            "--output-dir",
            help="Output directory for generated config files"
        ),
        category: str = typer.Option(
            "base",
            "--category", 
            help="Demo story category (base, ai, anomalies, etc.)"
        ),
        top_services: int = typer.Option(
            50,
            "--top-services",
            help="Number of top services to generate configs for"
        ),
        verbose: bool = typer.Option(
            False,
            "--verbose",
            help="Enable verbose output"
        )
    ):
        """Generate skeleton configuration files for top services."""
        
        command = GenerateInitConfigsCommand()
        command.init_configs(provider, output_dir, category, top_services, verbose)
    
    return init_configs