"""
CLI interface for the generate package.

Provides commands for generating billing datasets using folder-based configurations.
"""

from datetime import datetime
from pathlib import Path
from typing import Optional, List, Dict
from dateutil.relativedelta import relativedelta

import typer
from rich.console import Console
from rich.table import Table
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn

from ..config.models import (
    GenerateConfig,
    load_generation_configs,
    merge_generation_configs,
    validate_generation_config
)
from ...services.s3_data_service import S3DataService
from ..engine import StreamingGenerationEngine, GenerationError
from ..core.filter_reference_data import apply_filters, estimate_record_count_after_filtering
from ...services.environment_service import EnvironmentService
from .init_configs import create_init_configs_command
from .profiles import profiles_cmd


app = typer.Typer(help="Generate billing datasets using shape-based profiles")
console = Console()


@app.command()
def run(
    config_path: Path = typer.Option(
        ...,
        "--config-path",
        help="Path to configuration file or folder containing YAML configs"
    ),
    dataset_name: Optional[str] = typer.Option(
        None,
        "--dataset-name",
        help="Custom dataset name (default: timestamp)"
    ),
    dry_run: bool = typer.Option(
        False,
        "--dry-run",
        help="Validate configuration and show what would be generated"
    ),
    verbose: bool = typer.Option(
        False,
        "--verbose",
        help="Show detailed progress information"
    ),
    force: bool = typer.Option(
        False,
        "--force",
        help="Force reprocessing: deletes existing generate data and reprocesses"
    )
):
    """
    Generate billing datasets from configuration files.
    
    Supports both single files and folders containing multiple configurations.
    Processes all .yaml/.yml files recursively when given a folder.
    """
    # FAIL FAST: Validate AWS credentials before any work
    env_service = EnvironmentService()
    env_service.validate_aws_credentials_and_access()
    
    # Load configurations with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Loading configurations...", total=None)
        configs = load_generation_configs(config_path)
        progress.update(task, completed=True)
    
    console.print(f"[green]✓ Loaded {len(configs)} configuration(s) from: {config_path}[/green]")
    
    # Merge configurations if multiple
    if len(configs) > 1:
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            console=console,
            transient=True
        ) as progress:
            task = progress.add_task(f"Merging {len(configs)} configurations...", total=None)
            merged_config = merge_generation_configs(configs)
            progress.update(task, completed=True)
        console.print(f"[green]✓ Merged {len(configs)} configurations[/green]")
    else:
        merged_config = configs[0]
    
    # Validate configuration with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Validating configuration...", total=None)
        validation_errors = validate_generation_config(merged_config)
        progress.update(task, completed=True)
    
    if validation_errors:
        console.print("[red]❌ Configuration validation failed:[/red]")
        for error in validation_errors:
            console.print(f"  [red]• {error}[/red]")
        raise typer.Exit(1)
    
    console.print("[green]✓ Configuration validation passed[/green]")
    
    # Generate dataset name if not provided
    if dataset_name is None:
        dataset_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
    
    # Check for existing success marker (skip for dry-run)
    if not dry_run:
        s3_service = S3DataService()
        if _check_and_handle_existing_data(s3_service, dataset_name, force):
            raise typer.Exit(0)
    
    # Show summary
    _show_generation_summary(merged_config, dataset_name, dry_run, verbose)
    
    if dry_run:
        # Perform analysis instead of generation
        _perform_dry_run_analysis(merged_config, dataset_name, verbose)
        return
    
    # Initialize S3 service with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Initializing S3 service...", total=None)
        s3_service = S3DataService()
        progress.update(task, completed=True)
    
    console.print("[green]✓ S3 service initialized[/green]")
    
    # Generate datasets using streaming engine
    console.print(f"[blue]Starting dataset generation: {dataset_name}[/blue]")
    
    # Initialize streaming generation engine
    engine = StreamingGenerationEngine(
        s3_service=s3_service,
        console=console,
        org_name=dataset_name  # Use dataset name as org for deterministic UIDs
    )
    
    # Generate the dataset
    results = engine.generate_dataset(merged_config, dataset_name, verbose)
    
    # Show results
    console.print(f"[green]✓ Dataset generation completed: {dataset_name}[/green]")
    console.print(f"[blue]Generated {results['generation']['total_records_generated']:,} records[/blue]")
    
    # Show line item tracking summary
    console.print("\n[blue]Line Item Tracking Summary:[/blue]")
    console.print(results['line_items']['tracking_table'])
    
    if verbose:
        console.print("\n[blue]Detailed Statistics:[/blue]")
        for provider, stats in results['line_items']['providers'].items():
            console.print(f"  [cyan]{provider}[/cyan]: {stats['total_line_items']:,} items across {stats['month_count']} months")
            console.print(f"    Date range: {stats['min_month']} to {stats['max_month']}")
            console.print(f"    Average per month: {stats['avg_per_month']:,.0f}")


@app.command()
def validate(
    config_path: Path = typer.Option(
        ...,
        "--config-path",
        help="Path to configuration file or folder containing YAML configs"
    ),
    verbose: bool = typer.Option(
        False,
        "--verbose",
        help="Show detailed validation information"
    )
):
    """
    Validate configuration files without generating data.
    
    Checks syntax, required fields, and logical consistency.
    """
    # Load configurations with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Loading configurations...", total=None)
        configs = load_generation_configs(config_path)
        progress.update(task, completed=True)
    
    console.print(f"[green]✓ Loaded {len(configs)} configuration(s) from: {config_path}[/green]")
    
    # Validate each configuration with progress bar
    total_errors = 0
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        BarColumn(),
        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
        console=console,
        transient=False if verbose else True
    ) as progress:
        task = progress.add_task("Validating configurations...", total=len(configs))
        
        for i, config in enumerate(configs):
            if verbose:
                progress.update(task, description=f"Validating configuration {i+1}/{len(configs)}...")
            
            errors = validate_generation_config(config)
            if errors:
                console.print(f"[red]❌ Configuration {i+1} has {len(errors)} error(s):[/red]")
                for error in errors:
                    console.print(f"  [red]• {error}[/red]")
                total_errors += len(errors)
            elif verbose:
                console.print(f"[green]✓ Configuration {i+1} is valid[/green]")
            
            progress.update(task, advance=1)
    
    # Summary
    if total_errors > 0:
        console.print(f"[red]❌ Validation failed: {total_errors} error(s) found[/red]")
        raise typer.Exit(1)
    else:
        console.print("[green]✓ All configurations are valid[/green]")


@app.command()
def list_configs(
    config_path: Path = typer.Option(
        ...,
        "--config-path",
        help="Path to configuration file or folder containing YAML configs"
    )
):
    """
    List all configuration files and their generation jobs.
    
    Shows a summary of what would be generated from each configuration.
    """
    # Load configurations with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Loading configurations...", total=None)
        configs = load_generation_configs(config_path)
        progress.update(task, completed=True)
        
        # Create summary table
        table = Table(title="Generation Configurations", show_header=True, header_style="bold magenta")
        table.add_column("Config File", style="cyan", min_width=15)
        table.add_column("Jobs", justify="right", style="green")
        table.add_column("Total Cost", justify="right", style="yellow")
        table.add_column("Date Range", style="blue", min_width=20)
        
        for config in configs:
            job_count = len(config.generation_jobs)
            total_cost = sum(job.total_cost for job in config.generation_jobs)
            
            # Get date range
            start_dates = [job.start_date for job in config.generation_jobs]
            end_dates = [job.end_date for job in config.generation_jobs]
            date_range = f"{min(start_dates)} to {max(end_dates)}"
            
            # Get config file name
            config_name = config.source_file.name if config.source_file else "Merged"
            
            table.add_row(
                config_name,
                str(job_count),
                f"${total_cost:,.2f}",
                date_range
            )
        
        console.print(table)
        console.print(f"[green]Found {len(configs)} configuration(s)[/green]")




@app.command()
def delete_dataset(
    dataset_name: str = typer.Argument(..., help="Dataset name to delete"),
    confirm: bool = typer.Option(
        False,
        "--yes",
        help="Skip confirmation prompt"
    )
):
    """
    Delete a dataset from S3.
    
    Removes all files associated with the dataset.
    """
    if not confirm:
        # Show what will be deleted
        console.print(f"[yellow]This will delete dataset: {dataset_name}[/yellow]")
        
        # Initialize S3 service and show files with progress indicator
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            console=console,
            transient=True
        ) as progress:
            task = progress.add_task("Connecting to S3...", total=None)
            s3_service = S3DataService()
            progress.update(task, description="Listing dataset files...")
            files = s3_service.list_dataset_files(dataset_name)
            progress.update(task, completed=True)
        
        if files:
            console.print(f"[yellow]Files to be deleted ({len(files)}):[/yellow]")
            for provider, month in files:
                console.print(f"  [yellow]• {provider}/{month}.parquet[/yellow]")
        else:
            console.print("[yellow]No files found for this dataset[/yellow]")
            return
        
        # Confirm deletion
        confirm_delete = typer.confirm("Are you sure you want to delete this dataset?")
        if not confirm_delete:
            console.print("[blue]Deletion cancelled[/blue]")
            return
    
    # Initialize S3 service and delete dataset with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Connecting to S3...", total=None)
        s3_service = S3DataService()
        progress.update(task, description=f"Deleting dataset: {dataset_name}...")
        s3_service.delete_dataset(dataset_name)
        progress.update(task, completed=True)
    
    console.print(f"[green]✓ Dataset deleted: {dataset_name}[/green]")


# Add the init-configs command to the app
app.command(name="init-configs")(create_init_configs_command())


def _show_generation_summary(
    config: GenerateConfig,
    dataset_name: str,
    dry_run: bool,
    verbose: bool
):
    """Show summary of what will be generated."""
    console.print(f"[blue]Generation Summary for dataset: {dataset_name}[/blue]")
    
    # Create job summary table
    table = Table(title="Generation Jobs", show_header=True, header_style="bold magenta")
    table.add_column("Job Name", style="cyan", min_width=15)
    table.add_column("Provider", style="yellow", min_width=8)
    table.add_column("Date Range", style="blue", min_width=20)
    table.add_column("Total Cost", justify="right", style="green")
    table.add_column("Resources", justify="right", style="white")
    table.add_column("Profiles", style="magenta", min_width=25)
    
    total_cost = 0
    total_resources = 0
    
    for job in config.generation_jobs:
        total_cost += job.total_cost
        total_resources += job.resource_count
        
        # Get providers from filters
        providers = sorted(set(f.cloud_provider for f in job.filters))
        
        # Get profiles from filters
        variance_profiles = sorted(set(f.variance_profile for f in job.filters if f.variance_profile))
        anomaly_profiles = sorted(set(f.anomaly_profile for f in job.filters if f.anomaly_profile))
        
        # Format profile display
        profile_display = []
        if variance_profiles:
            v_display = ', '.join(variance_profiles[:2])
            if len(variance_profiles) > 2:
                v_display += f" +{len(variance_profiles) - 2} more"
            profile_display.append(f"V: {v_display}")
        
        if anomaly_profiles:
            a_display = ', '.join(anomaly_profiles[:2])
            if len(anomaly_profiles) > 2:
                a_display += f" +{len(anomaly_profiles) - 2} more"
            profile_display.append(f"A: {a_display}")
        
        profiles_text = "\n".join(profile_display) if profile_display else "None"
        
        table.add_row(
            job.name,
            ", ".join(providers),
            f"{job.start_date} to {job.end_date}",
            f"${job.total_cost:,.2f}",
            str(job.resource_count),
            profiles_text
        )
    
    console.print(table)
    
    # Summary statistics
    console.print(f"[blue]Total Jobs: {len(config.generation_jobs)}[/blue]")
    console.print(f"[blue]Total Cost: ${total_cost:,.2f}[/blue]")
    console.print(f"[blue]Total Resources: {total_resources}[/blue]")
    
    if verbose:
        # Show detailed filter information
        console.print("\n[blue]Detailed Filter Information:[/blue]")
        for i, job in enumerate(config.generation_jobs):
            console.print(f"[cyan]Job {i+1}: {job.name}[/cyan]")
            for j, filter_obj in enumerate(job.filters):
                console.print(f"  [yellow]Filter {j+1}: {filter_obj.cloud_provider}[/yellow]")
                console.print(f"    Variance: {filter_obj.variance_profile}")
                console.print(f"    Anomaly: {filter_obj.anomaly_profile}")
                console.print(f"    Services: {len(filter_obj.services)}")
    
    if dry_run:
        console.print("\n[yellow]🔍 DRY RUN MODE - No data will be generated[/yellow]")


def _show_provider_breakdown_table(
    config: GenerateConfig,
    job_results: List[Dict],
    console: Console
):
    """Show provider breakdown table matching the actual generation output format."""
    from rich.table import Table
    from calendar import monthrange
    
    # Step 1: Create month-based accumulator (like real LineItemTracker)
    provider_month_accumulator = {}  # {(provider, "2024-07"): total_line_items}
    provider_annual_costs = {}  # {provider: total_annual_cost}
    
    # Step 2: Process jobs correctly (filters first, then months) - matching real generation
    for job_result in job_results:
        job = job_result['job']
        filtered_results = job_result.get('filtered_results', [])
        months = job_result['months']
        
        # Calculate total resources across ALL filters in this job (like actual generation)
        total_job_resources = sum(len(filtered_df) * job.resource_count 
                                 for filtered_df, _ in filtered_results)
        
        if total_job_resources == 0:
            continue  # Skip jobs with no resources
        
        # Process all filters for this job together (like real generation)
        for filtered_df, filter_config in filtered_results:
            provider = filter_config["cloud_provider"]
            combinations = len(filtered_df)
            provider_resources = combinations * job.resource_count
            
            # Initialize provider annual cost tracking
            if provider not in provider_annual_costs:
                provider_annual_costs[provider] = 0.0
            
            # Distribute job cost proportionally by resources (like actual generation)
            provider_job_cost = (provider_resources / total_job_resources) * job.total_cost
            provider_annual_cost = (provider_job_cost / months) * 12 if months > 0 else provider_job_cost
            provider_annual_costs[provider] += provider_annual_cost
            
            # Calculate monthly line items for this job's entire period
            monthly_items = _calculate_monthly_line_items(job, combinations)
            
            # Generate month keys for this job's period
            start_dt = datetime.fromisoformat(str(job.start_date))
            end_dt = datetime.fromisoformat(str(job.end_date))
            current_date = start_dt.replace(day=1)
            month_keys = []
            
            while current_date <= end_dt:
                month_key = current_date.strftime("%Y-%m")
                month_keys.append(month_key)
                current_date += relativedelta(months=1)
            
            # Accumulate into month buckets (like real track_batch)
            for month_key, month_items in zip(month_keys, monthly_items):
                key = (provider, month_key)
                if key not in provider_month_accumulator:
                    provider_month_accumulator[key] = 0
                provider_month_accumulator[key] += month_items
    
    # Step 3: Extract final monthly values per provider (like real get_provider_stats)
    provider_data = {}
    
    # Get all unique month keys across all jobs
    all_month_keys = sorted(set(month_key for (provider, month_key) in provider_month_accumulator.keys()))
    
    for provider in provider_annual_costs.keys():
        # Get monthly values for this provider
        provider_months = []
        for month_key in all_month_keys:
            key = (provider, month_key)
            if key in provider_month_accumulator:
                provider_months.append(provider_month_accumulator[key])
        
        # Calculate peak and low from accumulated monthly values
        provider_data[provider] = {
            'annual_cost': provider_annual_costs[provider],
            'peak_month_line_items': max(provider_months) if provider_months else 0,
            'low_month_line_items': min(provider_months) if provider_months else 0,
        }
    
    # Create table
    table = Table(title="Provider Breakdown", show_header=True, header_style="bold magenta")
    table.add_column("Provider", style="cyan", min_width=11)
    table.add_column("Cost/Year", style="green", min_width=11) 
    table.add_column("Peak Month\nLine Items", style="yellow", min_width=11)
    table.add_column("Low Month\nLine Items", style="blue", min_width=11)
    
    # Sort by annual cost descending
    sorted_providers = sorted(provider_data.items(), key=lambda x: x[1]['annual_cost'], reverse=True)
    
    for provider, stats in sorted_providers:
        # Format cost
        annual_cost = stats['annual_cost']
        if annual_cost >= 1_000_000_000_000:  # Trillions
            cost_str = f"${annual_cost/1_000_000_000_000:.1f}T"
        elif annual_cost >= 1_000_000_000:  # Billions
            cost_str = f"${annual_cost/1_000_000_000:.1f}B"
        elif annual_cost >= 1_000_000:  # Millions
            cost_str = f"${annual_cost/1_000_000:.1f}M"
        elif annual_cost >= 1_000:  # Thousands
            cost_str = f"${annual_cost/1_000:.0f}K"
        else:
            cost_str = f"${annual_cost:.0f}"
        
        # Format line item counts
        peak_items = stats['peak_month_line_items']
        if peak_items >= 1_000_000:
            peak_str = f"{peak_items/1_000_000:.1f}M items"
        elif peak_items >= 1_000:
            peak_str = f"{peak_items/1_000:.0f}K items"
        else:
            peak_str = f"{peak_items} items"
        
        low_items = stats['low_month_line_items']
        if low_items >= 1_000_000:
            low_str = f"{low_items/1_000_000:.1f}M items"
        elif low_items >= 1_000:
            low_str = f"{low_items/1_000:.0f}K items"
        else:
            low_str = f"{low_items} items"
        
        table.add_row(provider, cost_str, peak_str, low_str)
    
    console.print(f"\n")
    console.print(table)


def _calculate_monthly_line_items(job: 'GenerationJob', combinations: int) -> List[int]:
    """Calculate line items for each month in the job period (matching real generation logic)."""
    from calendar import monthrange
    
    start_dt = datetime.fromisoformat(str(job.start_date))
    end_dt = datetime.fromisoformat(str(job.end_date))
    
    monthly_counts = []
    current_date = start_dt.replace(day=1)  # Start at beginning of month
    
    while current_date <= end_dt:
        # Get actual days in this month
        year = current_date.year
        month = current_date.month
        days_in_month = monthrange(year, month)[1]
        hours_in_month = days_in_month * 24
        
        # Calculate line items for this specific month
        month_line_items = combinations * job.resource_count * hours_in_month
        monthly_counts.append(month_line_items)
        
        # Move to next month
        current_date += relativedelta(months=1)
    
    return monthly_counts


def _perform_dry_run_analysis(
    config: GenerateConfig,
    dataset_name: str,
    verbose: bool
):
    """Perform comprehensive analysis of what would be generated."""
    import polars as pl
    from pathlib import Path
    from datetime import datetime, timedelta
    
    console.print("\n📊 [blue]Generation Analysis Report[/blue]")
    console.print("━" * 80)
    
    # Load reference data using S3DataService
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Ensuring reference data availability...", total=None)
        s3_service = S3DataService()
        ref_path = s3_service.ensure_reference_data()
        progress.update(task, description="Loading reference data...")
        ref_df = pl.read_parquet(ref_path)
        progress.update(task, completed=True)
        
        console.print(f"📋 [cyan]Configuration:[/cyan] {config.source_file.name if config.source_file else 'Merged'}")
        
        # Analyze each job
        total_combinations = 0
        total_line_items = 0
        total_peak_month_items = 0
        wildcard_usage_families = 0
        job_results = []  # Store results for Provider Breakdown
        
        for job in config.generation_jobs:
            # Extract providers and services from filters
            providers = sorted(set(filter_obj.cloud_provider for filter_obj in job.filters))
            all_services = []
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    all_services.append(service_obj.name)
            services = sorted(set(all_services))
            
            console.print(f"\n🏢 [cyan]Job:[/cyan] {job.name}")
            console.print(f"├── 🏅 Providers: {', '.join(providers)}")
            console.print(f"├── 🔧 Services: {len(services)} ({', '.join(services[:3])}{'...' if len(services) > 3 else ''})")
            
            # Calculate time metrics
            start_dt = datetime.fromisoformat(str(job.start_date))
            end_dt = datetime.fromisoformat(str(job.end_date))
            
            # Calculate months and hours
            months = ((end_dt.year - start_dt.year) * 12 + end_dt.month - start_dt.month) + 1
            hours_total = int(((end_dt - start_dt).total_seconds() / 3600) + 24)
            hours_per_month = hours_total // months if months > 0 else hours_total
            
            console.print(f"├── 📅 Period: {job.start_date} to {job.end_date} ({months} months, {hours_total:,} hours)")
            console.print(f"├── 💰 Total Cost: ${job.total_cost:,.2f}")
            console.print(f"└── 📊 Resource Count: {job.resource_count} per combination")
            
            # Convert Filter dataclass objects to dictionaries for apply_filters()
            filter_dicts = []
            for filter_obj in job.filters:
                filter_dict = {
                    "cloud_provider": filter_obj.cloud_provider,
                    "variance_profile": filter_obj.variance_profile,
                    "anomaly_profile": filter_obj.anomaly_profile,
                    "services": []
                }
                
                for service_obj in filter_obj.services:
                    service_dict = {
                        "name": service_obj.name,
                        "usage_families": []
                    }
                    
                    for uf_obj in service_obj.usage_families:
                        uf_dict = {
                            "name": uf_obj.name,
                            "resource_types": []
                        }
                        
                        for rt_obj in uf_obj.resource_types:
                            rt_dict = {
                                "name": rt_obj.name,
                                "operations": []
                            }
                            
                            for op_obj in rt_obj.operations:
                                op_dict = {
                                    "name": op_obj.name,
                                    "usage_types": op_obj.usage_types
                                }
                                rt_dict["operations"].append(op_dict)
                            
                            uf_dict["resource_types"].append(rt_dict)
                        
                        service_dict["usage_families"].append(uf_dict)
                    
                    filter_dict["services"].append(service_dict)
                
                filter_dicts.append(filter_dict)
            
            # Analyze filters
            with Progress(
                SpinnerColumn(),
                TextColumn("[progress.description]{task.description}"),
                console=console,
                transient=True
            ) as progress:
                task = progress.add_task("Analyzing filters...", total=None)
                filtered_results = apply_filters(ref_df, filter_dicts)
                progress.update(task, completed=True)
            
            # Calculate metrics
            job_combinations = sum(len(filtered_df) for filtered_df, _ in filtered_results)
            
            # Use ACCURATE month-by-month calculation for peak month items
            monthly_line_items = _calculate_monthly_line_items(job, job_combinations)
            job_peak_month_items = max(monthly_line_items) if monthly_line_items else 0
            job_total_items = sum(monthly_line_items) if monthly_line_items else 0
            
            # Count wildcard usage families
            job_wildcards = 0
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    for uf_obj in service_obj.usage_families:
                        if not uf_obj.resource_types:
                            job_wildcards += 1
            
            console.print(f"\n🎯 [cyan]Reference Data Analysis[/cyan]")
            console.print(f"├── Filtered combinations: {job_combinations:,}")
            console.print(f"├── Peak month line items: {job_peak_month_items:,} (calculated month-by-month)")
            console.print(f"├── Total line items: {job_total_items:,} (sum of all months)")
            console.print(f"└── Est. processing time: ~{max(1, job_total_items // 100000)}-{max(2, job_total_items // 50000)} minutes")
            
            # Store job results for Provider Breakdown
            job_results.append({
                'job': job,
                'providers': providers,
                'combinations': job_combinations,
                'peak_month_items': job_peak_month_items,
                'total_items': job_total_items,
                'months': months,
                'filtered_results': filtered_results  # Include filtered data for accurate provider calculations
            })
            
            # Accumulate totals
            total_combinations += job_combinations
            total_line_items += job_total_items
            total_peak_month_items += job_peak_month_items
            wildcard_usage_families += job_wildcards
        
        # Show overall summary
        console.print(f"\n📊 [cyan]Overall Summary[/cyan]")
        console.print(f"├── Total combinations: {total_combinations:,}")
        console.print(f"├── Total peak month line items: {total_peak_month_items:,}")
        console.print(f"├── Total line items across all periods: {total_line_items:,}")
        console.print(f"└── Est. dataset size: ~{max(1, total_line_items // 1000000)} MB parquet")
        
        # Configuration validation  
        status_icon = "✅" if wildcard_usage_families == 0 else "⚠️"
        console.print(f"\n{status_icon} [cyan]Configuration Status:[/cyan] Valid")
        if wildcard_usage_families > 0:
            console.print(f"   ({wildcard_usage_families} usage families use wildcards)")
        
        # Provider Breakdown Table
        _show_provider_breakdown_table(config, job_results, console)
        
        console.print("\n[yellow]Dry run analysis complete - no data generated[/yellow]")


if __name__ == "__main__":
    app()


def _show_provider_breakdown_table_old(
    config: GenerateConfig,
    job_results: List[Dict],
    console: Console
):
    """OLD VERSION - Show provider breakdown table matching the actual generation output format."""
    from rich.table import Table
    from calendar import monthrange
    
    # Calculate provider stats using ACCURATE month-by-month analysis
    provider_data = {}
    
    for job_result in job_results:
        job = job_result['job']
        providers = job_result['providers']
        job_combinations = job_result['combinations']
        
        # Calculate annual cost for this job's period (normalize to yearly)
        months = job_result['months']
        annual_cost = (job.total_cost / months) * 12 if months > 0 else job.total_cost
        
        # Generate month-by-month line item counts (like real generation)
        monthly_line_items = _calculate_monthly_line_items(job, job_combinations)
        
        for provider in providers:
            if provider not in provider_data:
                provider_data[provider] = {
                    'annual_cost': 0.0,
                    'monthly_line_items': [],  # Store monthly values for peak/low calculation
                }
            
            # Accumulate costs and monthly line items
            provider_data[provider]['annual_cost'] += annual_cost
            provider_data[provider]['monthly_line_items'].extend(monthly_line_items)
    
    # Calculate peak and low from ACTUAL monthly variations (like real generation)
    for provider in provider_data:
        monthly_items = provider_data[provider]['monthly_line_items']
        provider_data[provider]['peak_month_line_items'] = max(monthly_items) if monthly_items else 0
        provider_data[provider]['low_month_line_items'] = min(monthly_items) if monthly_items else 0


def _calculate_monthly_line_items(job: 'GenerationJob', combinations: int) -> List[int]:
    """Calculate line items for each month in the job period (matching real generation logic)."""
    from calendar import monthrange
    
    start_dt = datetime.fromisoformat(str(job.start_date))
    end_dt = datetime.fromisoformat(str(job.end_date))
    
    monthly_counts = []
    current_date = start_dt.replace(day=1)  # Start at beginning of month
    
    while current_date <= end_dt:
        # Get actual days in this month
        year = current_date.year
        month = current_date.month
        days_in_month = monthrange(year, month)[1]
        hours_in_month = days_in_month * 24
        
        # Calculate line items for this specific month
        month_line_items = combinations * job.resource_count * hours_in_month
        monthly_counts.append(month_line_items)
        
        # Move to next month
        current_date += relativedelta(months=1)
    
    return monthly_counts


    # Create table
    table = Table(title="Provider Breakdown", show_header=True, header_style="bold magenta")
    table.add_column("Provider", style="cyan", min_width=11)
    table.add_column("Cost/Year", style="green", min_width=11) 
    table.add_column("Peak Month\nLine Items", style="yellow", min_width=11)
    table.add_column("Low Month\nLine Items", style="blue", min_width=11)
    
    # Sort by annual cost descending
    sorted_providers = sorted(provider_data.items(), key=lambda x: x[1]['annual_cost'], reverse=True)
    
    for provider, stats in sorted_providers:
        # Format cost
        annual_cost = stats['annual_cost']
        if annual_cost >= 1_000_000_000_000:  # Trillions
            cost_str = f"${annual_cost/1_000_000_000_000:.1f}T"
        elif annual_cost >= 1_000_000_000:  # Billions
            cost_str = f"${annual_cost/1_000_000_000:.1f}B"
        elif annual_cost >= 1_000_000:  # Millions
            cost_str = f"${annual_cost/1_000_000:.1f}M"
        elif annual_cost >= 1_000:  # Thousands
            cost_str = f"${annual_cost/1_000:.0f}K"
        else:
            cost_str = f"${annual_cost:.0f}"
        
        # Format line item counts
        peak_items = stats['peak_month_line_items']
        if peak_items >= 1_000_000:
            peak_str = f"{peak_items/1_000_000:.1f}M items"
        elif peak_items >= 1_000:
            peak_str = f"{peak_items/1_000:.0f}K items"
        else:
            peak_str = f"{peak_items} items"
        
        low_items = stats['low_month_line_items']
        if low_items >= 1_000_000:
            low_str = f"{low_items/1_000_000:.1f}M items"
        elif low_items >= 1_000:
            low_str = f"{low_items/1_000:.0f}K items"
        else:
            low_str = f"{low_items} items"
        
        table.add_row(provider, cost_str, peak_str, low_str)
    
    console.print(f"\n")
    console.print(table)


def _perform_dry_run_analysis(
    config: GenerateConfig,
    dataset_name: str,
    verbose: bool
):
    """Perform comprehensive analysis of what would be generated."""
    import polars as pl
    from pathlib import Path
    from datetime import datetime, timedelta
    
    console.print("\n📊 [blue]Generation Analysis Report[/blue]")
    console.print("━" * 80)
    
    # Load reference data using S3DataService
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Ensuring reference data availability...", total=None)
        s3_service = S3DataService()
        ref_path = s3_service.ensure_reference_data()
        progress.update(task, description="Loading reference data...")
        ref_df = pl.read_parquet(ref_path)
        progress.update(task, completed=True)
        
        console.print(f"📋 [cyan]Configuration:[/cyan] {config.source_file.name if config.source_file else 'Merged'}")
        
        # Analyze each job
        total_combinations = 0
        total_line_items = 0
        total_peak_month_items = 0
        wildcard_usage_families = 0
        job_results = []  # Store results for Provider Breakdown
        
        for job in config.generation_jobs:
            # Extract providers and services from filters
            providers = sorted(set(filter_obj.cloud_provider for filter_obj in job.filters))
            all_services = []
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    all_services.append(service_obj.name)
            services = sorted(set(all_services))
            
            console.print(f"\n🏢 [cyan]Job:[/cyan] {job.name}")
            console.print(f"├── 🏅 Providers: {', '.join(providers)}")
            console.print(f"├── 🔧 Services: {len(services)} ({', '.join(services[:3])}{'...' if len(services) > 3 else ''})")
            
            # Calculate time metrics
            start_dt = datetime.fromisoformat(str(job.start_date))
            end_dt = datetime.fromisoformat(str(job.end_date))
            
            # Calculate months and hours
            months = ((end_dt.year - start_dt.year) * 12 + end_dt.month - start_dt.month) + 1
            hours_total = int(((end_dt - start_dt).total_seconds() / 3600) + 24)
            hours_per_month = hours_total // months if months > 0 else hours_total
            
            console.print(f"├── 📅 Period: {job.start_date} to {job.end_date} ({months} months, {hours_total:,} hours)")
            console.print(f"├── 💰 Total Cost: ${job.total_cost:,.2f}")
            console.print(f"└── 📊 Resource Count: {job.resource_count} per combination")
            
            # Convert Filter dataclass objects to dictionaries for apply_filters()
            filter_dicts = []
            for filter_obj in job.filters:
                filter_dict = {
                    "cloud_provider": filter_obj.cloud_provider,
                    "variance_profile": filter_obj.variance_profile,
                    "anomaly_profile": filter_obj.anomaly_profile,
                    "services": []
                }
                
                for service_obj in filter_obj.services:
                    service_dict = {
                        "name": service_obj.name,
                        "usage_families": []
                    }
                    
                    for uf_obj in service_obj.usage_families:
                        uf_dict = {
                            "name": uf_obj.name,
                            "resource_types": []
                        }
                        
                        for rt_obj in uf_obj.resource_types:
                            rt_dict = {
                                "name": rt_obj.name,
                                "operations": []
                            }
                            
                            for op_obj in rt_obj.operations:
                                op_dict = {
                                    "name": op_obj.name,
                                    "usage_types": op_obj.usage_types
                                }
                                rt_dict["operations"].append(op_dict)
                            
                            uf_dict["resource_types"].append(rt_dict)
                        
                        service_dict["usage_families"].append(uf_dict)
                    
                    filter_dict["services"].append(service_dict)
                
                filter_dicts.append(filter_dict)
            
            # Analyze filters
            with Progress(
                SpinnerColumn(),
                TextColumn("[progress.description]{task.description}"),
                console=console,
                transient=True
            ) as progress:
                task = progress.add_task("Analyzing filters...", total=None)
                filtered_results = apply_filters(ref_df, filter_dicts)
                progress.update(task, completed=True)
            
            # Calculate metrics
            job_combinations = sum(len(filtered_df) for filtered_df, _ in filtered_results)
            
            # Use ACCURATE month-by-month calculation for peak month items
            monthly_line_items = _calculate_monthly_line_items(job, job_combinations)
            job_peak_month_items = max(monthly_line_items) if monthly_line_items else 0
            job_total_items = sum(monthly_line_items) if monthly_line_items else 0
            
            # Count wildcard usage families
            job_wildcards = 0
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    for uf_obj in service_obj.usage_families:
                        if not uf_obj.resource_types:
                            job_wildcards += 1
            
            console.print(f"\n🎯 [cyan]Reference Data Analysis[/cyan]")
            console.print(f"├── Filtered combinations: {job_combinations:,}")
            console.print(f"├── Peak month line items: {job_peak_month_items:,} (calculated month-by-month)")
            console.print(f"├── Total line items: {job_total_items:,} (sum of all months)")
            console.print(f"└── Est. processing time: ~{max(1, job_total_items // 100000)}-{max(2, job_total_items // 50000)} minutes")
            
            # Store job results for Provider Breakdown
            job_results.append({
                'job': job,
                'providers': providers,
                'combinations': job_combinations,
                'peak_month_items': job_peak_month_items,
                'total_items': job_total_items,
                'months': months,
                'filtered_results': filtered_results  # Include filtered data for accurate provider calculations
            })
            
            # Accumulate totals
            total_combinations += job_combinations
            total_line_items += job_total_items
            total_peak_month_items += job_peak_month_items
            wildcard_usage_families += job_wildcards
        
        # Show overall summary
        console.print(f"\n📊 [cyan]Overall Summary[/cyan]")
        console.print(f"├── Total combinations: {total_combinations:,}")
        console.print(f"├── Total peak month line items: {total_peak_month_items:,}")
        console.print(f"├── Total line items across all periods: {total_line_items:,}")
        console.print(f"└── Est. dataset size: ~{max(1, total_line_items // 1000000)} MB parquet")
        
        # Configuration validation  
        status_icon = "✅" if wildcard_usage_families == 0 else "⚠️"
        console.print(f"\n{status_icon} [cyan]Configuration Status:[/cyan] Valid")
        if wildcard_usage_families > 0:
            console.print(f"   ({wildcard_usage_families} usage families use wildcards)")
        
        # Provider Breakdown Table
        _show_provider_breakdown_table(config, job_results, console)
        
        console.print("\n[yellow]Dry run analysis complete - no data generated[/yellow]")


def _check_and_handle_existing_data(s3_service: S3DataService, dataset_name: str, force: bool) -> bool:
    """
    Check for existing success marker and handle force flag logic.
    
    Args:
        s3_service: S3 data service instance
        dataset_name: Dataset name to check
        force: Whether force flag was provided
        
    Returns:
        True if should exit (data exists and not forced), False if should continue
    """
    if _generate_success_marker_exists(s3_service, dataset_name):
        if not force:
            console.print(f"✅ Generate stage already completed for dataset '{dataset_name}'. Use --force to reprocess.")
            return True
        
        # Force flag provided - confirm and delete existing data
        console.print(f"⚠️  This will delete ALL existing generate data for dataset '{dataset_name}' and reprocess.")
        
        # Confirmation prompt
        confirm = typer.confirm("❓ Continue?")
        if not confirm:
            console.print("Operation cancelled.")
            return True
        
        # Delete existing generate data
        console.print(f"🗑️  Deleting existing generate data for dataset '{dataset_name}'...")
        _delete_generate_stage_data(s3_service, dataset_name)
        console.print(f"✅ Existing generate data deleted for dataset '{dataset_name}'")
    
    return False


def _generate_success_marker_exists(s3_service: S3DataService, dataset_name: str) -> bool:
    """
    Check if generate stage success marker exists for a dataset.
    
    Args:
        s3_service: S3 data service instance
        dataset_name: Dataset name to check
        
    Returns:
        True if success marker exists, False otherwise
    """
    from ...services.success_marker_service import SuccessMarkerService
    success_marker_service = SuccessMarkerService(s3_service)
    
    return success_marker_service.check_success_marker_exists(dataset_name, "generate")


def _delete_generate_stage_data(s3_service: S3DataService, dataset_name: str) -> None:
    """
    Delete all generate stage data for a dataset.
    
    Deletes entire generate folder: s3://bucket/czbillgen/datasets/{dataset_name}/generate/
    This ensures no mixed configuration data remains.
    
    Args:
        s3_service: S3 data service instance
        dataset_name: Dataset name to delete data for
    """
    stage_prefix = f"czbillgen/datasets/{dataset_name}/generate/"
    
    # List all objects in generate stage folder
    paginator = s3_service.s3_client.get_paginator('list_objects_v2')
    page_iterator = paginator.paginate(
        Bucket=s3_service.bucket,
        Prefix=stage_prefix
    )
    
    objects_to_delete = []
    for page in page_iterator:
        if 'Contents' in page:
            objects_to_delete.extend([{'Key': obj['Key']} for obj in page['Contents']])
    
    if objects_to_delete:
        # Delete objects in batches (S3 delete_objects limit is 1000)
        for i in range(0, len(objects_to_delete), 1000):
            batch = objects_to_delete[i:i+1000]
            s3_service.s3_client.delete_objects(
                Bucket=s3_service.bucket,
                Delete={'Objects': batch}
            )
            
            console.print(f"🗑️  Deleted {len(objects_to_delete)} objects from generate stage")
        else:
            console.print("📂 No existing generate data found to delete")
        
        # Also delete success marker using standardized service
        from ...services.success_marker_service import SuccessMarkerService
        success_marker_service = SuccessMarkerService(s3_service)
        success_marker_service.delete_success_marker(dataset_name, "generate")


@app.command("list-profiles")
def list_profiles(
    profile_type: Optional[str] = typer.Option(
        None, 
        "--type", 
        help="Profile type to list (variance or anomaly)"
    )
):
    """List available variance and anomaly profiles for generate configs."""
    profiles_cmd.list_profiles(profile_type)


@app.command("show-profile")
def show_profile(
    profile_name: str = typer.Argument(..., help="Profile name to show details for")
):
    """Show detailed information about a specific profile."""
    profiles_cmd.show_profile(profile_name)


if __name__ == "__main__":
    app()