"""
CLI interface for the generate package.

Provides commands for generating billing datasets using folder-based configurations.
"""

from datetime import datetime
from pathlib import Path
from typing import Optional, List, Dict
from dateutil.relativedelta import relativedelta

import typer
from rich.console import Console
from rich.table import Table
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn

from ..config.models import (
    GenerateConfig,
    load_generation_configs,
    merge_generation_configs,
    validate_generation_config
)
from src.services.s3_data_service import S3DataService
from ..engine import StreamingGenerationEngine, GenerationError
from ..core.filter_reference_data import apply_filters, estimate_record_count_after_filtering
from src.services.environment_service import EnvironmentService
from src.services.dataset_identification import DatasetMetadataService
from .init_configs import create_init_configs_command
# Temporarily disabled due to profiles system removal
# from .profiles import profiles_cmd


app = typer.Typer(help="Generate billing datasets using shape-based profiles")
console = Console()


@app.command()
def run(
    config_path: Path = typer.Option(
        ...,
        "--config-path",
        help="Path to configuration file or folder containing YAML configs"
    ),
    dataset_name: Optional[str] = typer.Option(
        None,
        "--dataset-name",
        help="Custom dataset name (default: timestamp)"
    ),
    description: Optional[str] = typer.Option(
        None,
        "--description",
        help="Description for the dataset (for identification and management)"
    ),
    dry_run: bool = typer.Option(
        False,
        "--dry-run",
        help="Validate configuration and show what would be generated"
    ),
    verbose: bool = typer.Option(
        False,
        "--verbose",
        help="Show detailed progress information"
    ),
    force: bool = typer.Option(
        False,
        "--force",
        help="Force reprocessing: deletes existing generate data and reprocesses"
    ),
    api: bool = typer.Option(
        False,
        "--api",
        help="Execute via API - same as default (kept for backward compatibility)"
    ),
    local: bool = typer.Option(
        False,
        "--local",
        help="Execute locally instead of via API (for development/debugging)"
    )
):
    """
    Generate billing datasets from configuration files.

    Supports both single files and folders containing multiple configurations.
    Processes all .yaml/.yml files recursively when given a folder.

    Execution modes:
    - Default: Submit job to Lambda API for processing on ECS workers (scalable, observable)
    - --local: Execute locally for development, debugging, or small configs
    - --api: Explicit API execution (same as default, kept for backward compatibility)
    """
    # Validate mutually exclusive flags
    if api and local:
        console.print("[red]❌ Cannot specify both --api and --local[/red]")
        raise typer.Exit(1)

    # Execute locally if --local flag provided
    if local:
        console.print("[dim]⚙️  Executing locally...[/dim]")
        _execute_locally(
            config_path=config_path,
            dataset_name=dataset_name,
            description=description,
            dry_run=dry_run,
            verbose=verbose,
            force=force
        )
        return

    # Default: Execute via API
    console.print("[cyan]☁️  Executing via API (Lambda + ECS workers)...[/cyan]")
    _execute_via_api(
        config_path=config_path,
        dataset_name=dataset_name,
        description=description,
        dry_run=dry_run,
        force=force
    )


def _execute_locally(
    config_path: Path,
    dataset_name: Optional[str],
    description: Optional[str],
    dry_run: bool,
    verbose: bool,
    force: bool
):
    """
    Execute generate locally (not via API).

    Single responsibility: Local execution logic.
    Method size: <50 lines per CLAUDE.md.
    """
    # FAIL FAST: Validate AWS credentials before any work
    env_service = EnvironmentService()
    env_service.validate_aws_credentials_and_access()

    # Load configurations with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Loading configurations...", total=None)
        configs = load_generation_configs(config_path)
        progress.update(task, completed=True)
    
    console.print(f"[green]✓ Loaded {len(configs)} configuration(s) from: {config_path}[/green]")
    
    # Merge configurations if multiple
    if len(configs) > 1:
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            console=console,
            transient=True
        ) as progress:
            task = progress.add_task(f"Merging {len(configs)} configurations...", total=None)
            merged_config = merge_generation_configs(configs)
            progress.update(task, completed=True)
        console.print(f"[green]✓ Merged {len(configs)} configurations[/green]")
    else:
        merged_config = configs[0]
    
    # Validate configuration with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Validating configuration...", total=None)
        validation_errors = validate_generation_config(merged_config)
        progress.update(task, completed=True)
    
    if validation_errors:
        console.print("[red]❌ Configuration validation failed:[/red]")
        for error in validation_errors:
            console.print(f"  [red]• {error}[/red]")
        raise typer.Exit(1)
    
    console.print("[green]✓ Configuration validation passed[/green]")
    
    # Generate dataset name if not provided
    if dataset_name is None:
        dataset_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
    
    # Check for existing success marker (skip for dry-run)
    if not dry_run:
        s3_service = S3DataService()
        if _check_and_handle_existing_data(s3_service, dataset_name, force):
            raise typer.Exit(0)
    
    # Show summary
    _show_generation_summary(merged_config, dataset_name, dry_run, verbose)
    
    if dry_run:
        # Perform enhanced dry run analysis with automatic anomaly detection
        _perform_enhanced_dry_run_analysis(merged_config, dataset_name, verbose)
        return
    
    # Initialize S3 service with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Initializing S3 service...", total=None)
        s3_service = S3DataService()
        progress.update(task, completed=True)
    
    console.print("[green]✓ S3 service initialized[/green]")
    
    # Generate datasets using streaming engine
    console.print(f"[blue]Starting dataset generation: {dataset_name}[/blue]")
    
    # Initialize streaming generation engine
    engine = StreamingGenerationEngine(
        s3_service=s3_service,
        console=console,
        org_name=dataset_name  # Use dataset name as org for deterministic UIDs
    )
    
    # Generate the dataset
    results = engine.generate_dataset(merged_config, dataset_name, verbose)

    # Create initial dataset metadata if description provided
    if description:
        try:
            metadata_service = DatasetMetadataService(s3_service)

            # Try to get the reference data version from the engine or service
            reference_version = "Unknown"
            try:
                # This would need to be added to the engine results if we want accurate versioning
                reference_version = getattr(engine, 'reference_data_version', "Unknown")
            except:
                pass

            metadata_service.create_initial_metadata(
                dataset_id=dataset_name,
                description=description,
                reference_data_version=reference_version
            )
            console.print(f"[green]✓ Dataset metadata created with description: {description}[/green]")
        except Exception as e:
            console.print(f"[yellow]⚠️  Warning: Could not create dataset metadata: {e}[/yellow]")

    # Show results
    console.print(f"[green]✓ Dataset generation completed: {dataset_name}[/green]")
    console.print(f"[blue]Generated {results['generation']['total_records_generated']:,} records[/blue]")
    
    # Show line item tracking summary
    console.print("\n[blue]Line Item Tracking Summary:[/blue]")
    console.print(results['line_items']['tracking_table'])
    
    if verbose:
        console.print("\n[blue]Detailed Statistics:[/blue]")
        for provider, stats in results['line_items']['providers'].items():
            console.print(f"  [cyan]{provider}[/cyan]: {stats['total_line_items']:,} items across {stats['month_count']} months")
            console.print(f"    Date range: {stats['min_month']} to {stats['max_month']}")
            console.print(f"    Average per month: {stats['avg_per_month']:,.0f}")


@app.command()
def validate(
    config_path: Path = typer.Option(
        ...,
        "--config-path",
        help="Path to configuration file or folder containing YAML configs"
    ),
    verbose: bool = typer.Option(
        False,
        "--verbose",
        help="Show detailed validation information"
    )
):
    """
    Validate configuration files without generating data.
    
    Checks syntax, required fields, and logical consistency.
    """
    # Load configurations with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Loading configurations...", total=None)
        configs = load_generation_configs(config_path)
        progress.update(task, completed=True)
    
    console.print(f"[green]✓ Loaded {len(configs)} configuration(s) from: {config_path}[/green]")
    
    # Validate each configuration with progress bar
    total_errors = 0
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        BarColumn(),
        TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
        console=console,
        transient=False if verbose else True
    ) as progress:
        task = progress.add_task("Validating configurations...", total=len(configs))
        
        for i, config in enumerate(configs):
            if verbose:
                progress.update(task, description=f"Validating configuration {i+1}/{len(configs)}...")
            
            errors = validate_generation_config(config)
            if errors:
                console.print(f"[red]❌ Configuration {i+1} has {len(errors)} error(s):[/red]")
                for error in errors:
                    console.print(f"  [red]• {error}[/red]")
                total_errors += len(errors)
            elif verbose:
                console.print(f"[green]✓ Configuration {i+1} is valid[/green]")
            
            progress.update(task, advance=1)
    
    # Summary
    if total_errors > 0:
        console.print(f"[red]❌ Validation failed: {total_errors} error(s) found[/red]")
        raise typer.Exit(1)
    else:
        console.print("[green]✓ All configurations are valid[/green]")


@app.command()
def list_configs(
    config_path: Path = typer.Option(
        ...,
        "--config-path",
        help="Path to configuration file or folder containing YAML configs"
    )
):
    """
    List all configuration files and their generation jobs.
    
    Shows a summary of what would be generated from each configuration.
    """
    # Load configurations with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Loading configurations...", total=None)
        configs = load_generation_configs(config_path)
        progress.update(task, completed=True)
        
        # Create summary table
        table = Table(title="Generation Configurations", show_header=True, header_style="bold magenta")
        table.add_column("Config File", style="cyan", min_width=15)
        table.add_column("Jobs", justify="right", style="green")
        table.add_column("Total Cost", justify="right", style="yellow")
        table.add_column("Date Range", style="blue", min_width=20)
        
        for config in configs:
            job_count = len(config.generation_jobs)
            total_cost = sum(job.total_cost for job in config.generation_jobs)
            
            # Get date range
            start_dates = [job.start_date for job in config.generation_jobs]
            end_dates = [job.end_date for job in config.generation_jobs]
            date_range = f"{min(start_dates)} to {max(end_dates)}"
            
            # Get config file name
            config_name = config.source_file.name if config.source_file else "Merged"
            
            table.add_row(
                config_name,
                str(job_count),
                f"${total_cost:,.2f}",
                date_range
            )
        
        console.print(table)
        console.print(f"[green]Found {len(configs)} configuration(s)[/green]")




@app.command()
def delete_dataset(
    dataset_name: str = typer.Argument(..., help="Dataset name to delete"),
    confirm: bool = typer.Option(
        False,
        "--yes",
        help="Skip confirmation prompt"
    )
):
    """
    Delete a dataset from S3.
    
    Removes all files associated with the dataset.
    """
    if not confirm:
        # Show what will be deleted
        console.print(f"[yellow]This will delete dataset: {dataset_name}[/yellow]")
        
        # Initialize S3 service and show files with progress indicator
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            console=console,
            transient=True
        ) as progress:
            task = progress.add_task("Connecting to S3...", total=None)
            s3_service = S3DataService()
            progress.update(task, description="Listing dataset files...")
            files = s3_service.list_dataset_files(dataset_name)
            progress.update(task, completed=True)
        
        if files:
            console.print(f"[yellow]Files to be deleted ({len(files)}):[/yellow]")
            for provider, month in files:
                console.print(f"  [yellow]• {provider}/{month}.parquet[/yellow]")
        else:
            console.print("[yellow]No files found for this dataset[/yellow]")
            return
        
        # Confirm deletion
        confirm_delete = typer.confirm("Are you sure you want to delete this dataset?")
        if not confirm_delete:
            console.print("[blue]Deletion cancelled[/blue]")
            return
    
    # Initialize S3 service and delete dataset with progress indicator
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Connecting to S3...", total=None)
        s3_service = S3DataService()
        progress.update(task, description=f"Deleting dataset: {dataset_name}...")
        s3_service.delete_dataset(dataset_name)
        progress.update(task, completed=True)
    
    console.print(f"[green]✓ Dataset deleted: {dataset_name}[/green]")


# Add the init-configs command to the app
app.command(name="init-configs")(create_init_configs_command())


def _show_generation_summary(
    config: GenerateConfig,
    dataset_name: str,
    dry_run: bool,
    verbose: bool
):
    """Show summary of what will be generated."""
    console.print(f"[blue]Generation Summary for dataset: {dataset_name}[/blue]")
    
    # Create job summary table
    table = Table(title="Generation Jobs", show_header=True, header_style="bold magenta")
    table.add_column("Job Name", style="cyan", min_width=15)
    table.add_column("Provider", style="yellow", min_width=8)
    table.add_column("Date Range", style="blue", min_width=20)
    table.add_column("Total Cost", justify="right", style="green")
    table.add_column("Resources", justify="right", style="white")
    table.add_column("Profiles", style="magenta", min_width=25)
    
    total_cost = 0
    total_resources = 0
    
    for job in config.generation_jobs:
        total_cost += job.total_cost
        total_resources += job.resource_count
        
        # Get providers from filters
        providers = sorted(set(f.cloud_provider for f in job.filters))
        
        # Get profiles from filters
        variance_profiles = sorted(set(f.variance_profile for f in job.filters if f.variance_profile))
        
        # Handle anomaly profiles (can be string or dict)
        anomaly_types = []
        for f in job.filters:
            if f.anomaly_profile:
                if isinstance(f.anomaly_profile, dict):
                    anomaly_types.append(f.anomaly_profile.get('type', 'unknown'))
                else:
                    anomaly_types.append(f.anomaly_profile)
        anomaly_profiles = sorted(set(anomaly_types))
        
        # Format profile display
        profile_display = []
        if variance_profiles:
            v_display = ', '.join(variance_profiles[:2])
            if len(variance_profiles) > 2:
                v_display += f" +{len(variance_profiles) - 2} more"
            profile_display.append(f"V: {v_display}")
        
        if anomaly_profiles:
            a_display = ', '.join(anomaly_profiles[:2])
            if len(anomaly_profiles) > 2:
                a_display += f" +{len(anomaly_profiles) - 2} more"
            profile_display.append(f"A: {a_display}")
        
        profiles_text = "\n".join(profile_display) if profile_display else "None"
        
        table.add_row(
            job.name,
            ", ".join(providers),
            f"{job.start_date} to {job.end_date}",
            f"${job.total_cost:,.2f}",
            str(job.resource_count),
            profiles_text
        )
    
    console.print(table)
    
    # Summary statistics
    console.print(f"[blue]Total Jobs: {len(config.generation_jobs)}[/blue]")
    console.print(f"[blue]Total Cost: ${total_cost:,.2f}[/blue]")
    console.print(f"[blue]Total Resources: {total_resources}[/blue]")
    
    if verbose:
        # Show detailed filter information
        console.print("\n[blue]Detailed Filter Information:[/blue]")
        for i, job in enumerate(config.generation_jobs):
            console.print(f"[cyan]Job {i+1}: {job.name}[/cyan]")
            for j, filter_obj in enumerate(job.filters):
                console.print(f"  [yellow]Filter {j+1}: {filter_obj.cloud_provider}[/yellow]")
                console.print(f"    Variance: {filter_obj.variance_profile}")
                anomaly_display = filter_obj.anomaly_profile
                if isinstance(anomaly_display, dict):
                    anomaly_type = anomaly_display.get('type', 'unknown')
                    intensity = anomaly_display.get('intensity', 'N/A')
                    anomaly_display = f"{anomaly_type} (intensity: {intensity})"
                console.print(f"    Anomaly: {anomaly_display}")
                console.print(f"    Services: {len(filter_obj.services)}")
    
    if dry_run:
        console.print("\n[yellow]🔍 DRY RUN MODE - No data will be generated[/yellow]")


def _show_provider_breakdown_table(
    config: GenerateConfig,
    job_results: List[Dict],
    console: Console
):
    """Show provider breakdown table matching the actual generation output format."""
    from rich.table import Table
    from calendar import monthrange
    
    # Step 1: Create month-based accumulator (like real LineItemTracker)
    provider_month_accumulator = {}  # {(provider, "2024-07"): total_line_items}
    provider_annual_costs = {}  # {provider: total_annual_cost}
    
    # Step 2: Process jobs correctly (filters first, then months) - matching real generation
    for job_result in job_results:
        job = job_result['job']
        filtered_results = job_result.get('filtered_results', [])
        months = job_result['months']
        
        # Calculate total resources across ALL filters in this job (like actual generation)
        total_job_resources = sum(len(filtered_df) * job.resource_count 
                                 for filtered_df, _ in filtered_results)
        
        if total_job_resources == 0:
            continue  # Skip jobs with no resources
        
        # Process all filters for this job together (like real generation)
        for filtered_df, filter_config in filtered_results:
            provider = filter_config["cloud_provider"]
            combinations = len(filtered_df)
            provider_resources = combinations * job.resource_count
            
            # Initialize provider annual cost tracking
            if provider not in provider_annual_costs:
                provider_annual_costs[provider] = 0.0
            
            # Distribute job cost proportionally by resources (like actual generation)
            provider_job_cost = (provider_resources / total_job_resources) * job.total_cost
            provider_annual_cost = (provider_job_cost / months) * 12 if months > 0 else provider_job_cost
            provider_annual_costs[provider] += provider_annual_cost
            
            # Calculate monthly line items for this job's entire period
            monthly_items = _calculate_monthly_line_items(job, combinations)
            
            # Generate month keys for this job's period
            start_dt = datetime.fromisoformat(str(job.start_date))
            end_dt = datetime.fromisoformat(str(job.end_date))
            current_date = start_dt.replace(day=1)
            month_keys = []
            
            while current_date <= end_dt:
                month_key = current_date.strftime("%Y-%m")
                month_keys.append(month_key)
                current_date += relativedelta(months=1)
            
            # Accumulate into month buckets (like real track_batch)
            for month_key, month_items in zip(month_keys, monthly_items):
                key = (provider, month_key)
                if key not in provider_month_accumulator:
                    provider_month_accumulator[key] = 0
                provider_month_accumulator[key] += month_items
    
    # Step 3: Extract final monthly values per provider (like real get_provider_stats)
    provider_data = {}
    
    # Get all unique month keys across all jobs
    all_month_keys = sorted(set(month_key for (provider, month_key) in provider_month_accumulator.keys()))
    
    for provider in provider_annual_costs.keys():
        # Get monthly values for this provider
        provider_months = []
        for month_key in all_month_keys:
            key = (provider, month_key)
            if key in provider_month_accumulator:
                provider_months.append(provider_month_accumulator[key])
        
        # Calculate peak and low from accumulated monthly values
        provider_data[provider] = {
            'annual_cost': provider_annual_costs[provider],
            'peak_month_line_items': max(provider_months) if provider_months else 0,
            'low_month_line_items': min(provider_months) if provider_months else 0,
        }
    
    # Create table
    table = Table(title="Provider Breakdown", show_header=True, header_style="bold magenta")
    table.add_column("Provider", style="cyan", min_width=11)
    table.add_column("Cost/Year", style="green", min_width=11) 
    table.add_column("Peak Month\nLine Items", style="yellow", min_width=11)
    table.add_column("Low Month\nLine Items", style="blue", min_width=11)
    
    # Sort by annual cost descending
    sorted_providers = sorted(provider_data.items(), key=lambda x: x[1]['annual_cost'], reverse=True)
    
    for provider, stats in sorted_providers:
        # Format cost
        annual_cost = stats['annual_cost']
        if annual_cost >= 1_000_000_000_000:  # Trillions
            cost_str = f"${annual_cost/1_000_000_000_000:.1f}T"
        elif annual_cost >= 1_000_000_000:  # Billions
            cost_str = f"${annual_cost/1_000_000_000:.1f}B"
        elif annual_cost >= 1_000_000:  # Millions
            cost_str = f"${annual_cost/1_000_000:.1f}M"
        elif annual_cost >= 1_000:  # Thousands
            cost_str = f"${annual_cost/1_000:.0f}K"
        else:
            cost_str = f"${annual_cost:.0f}"
        
        # Format line item counts
        peak_items = stats['peak_month_line_items']
        if peak_items >= 1_000_000:
            peak_str = f"{peak_items/1_000_000:.1f}M items"
        elif peak_items >= 1_000:
            peak_str = f"{peak_items/1_000:.0f}K items"
        else:
            peak_str = f"{peak_items} items"
        
        low_items = stats['low_month_line_items']
        if low_items >= 1_000_000:
            low_str = f"{low_items/1_000_000:.1f}M items"
        elif low_items >= 1_000:
            low_str = f"{low_items/1_000:.0f}K items"
        else:
            low_str = f"{low_items} items"
        
        table.add_row(provider, cost_str, peak_str, low_str)
    
    console.print(f"\n")
    console.print(table)


def _calculate_monthly_line_items(job: 'GenerationJob', combinations: int) -> List[int]:
    """Calculate line items for each month in the job period (matching real generation logic)."""
    from calendar import monthrange
    
    start_dt = datetime.fromisoformat(str(job.start_date))
    end_dt = datetime.fromisoformat(str(job.end_date))
    
    monthly_counts = []
    current_date = start_dt.replace(day=1)  # Start at beginning of month
    
    while current_date <= end_dt:
        # Get actual days in this month
        year = current_date.year
        month = current_date.month
        days_in_month = monthrange(year, month)[1]
        hours_in_month = days_in_month * 24
        
        # Calculate line items for this specific month
        month_line_items = combinations * job.resource_count * hours_in_month
        monthly_counts.append(month_line_items)
        
        # Move to next month
        current_date += relativedelta(months=1)
    
    return monthly_counts


def _perform_dry_run_analysis(
    config: GenerateConfig,
    dataset_name: str,
    verbose: bool
):
    """Perform comprehensive analysis of what would be generated."""
    import polars as pl
    from pathlib import Path
    from datetime import datetime, timedelta
    
    console.print("\n📊 [blue]Generation Analysis Report[/blue]")
    console.print("━" * 80)
    
    # Load reference data using S3DataService
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Ensuring reference data availability...", total=None)
        s3_service = S3DataService()
        ref_path = s3_service.ensure_reference_data()
        progress.update(task, description="Loading reference data...")
        ref_df = pl.read_parquet(ref_path)
        progress.update(task, completed=True)
        
        console.print(f"📋 [cyan]Configuration:[/cyan] {config.source_file.name if config.source_file else 'Merged'}")
        
        # Analyze each job
        total_combinations = 0
        total_line_items = 0
        total_peak_month_items = 0
        wildcard_usage_families = 0
        job_results = []  # Store results for Provider Breakdown
        
        for job in config.generation_jobs:
            # Extract providers and services from filters
            providers = sorted(set(filter_obj.cloud_provider for filter_obj in job.filters))
            all_services = []
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    all_services.append(service_obj.name)
            services = sorted(set(all_services))
            
            console.print(f"\n🏢 [cyan]Job:[/cyan] {job.name}")
            console.print(f"├── 🏅 Providers: {', '.join(providers)}")
            console.print(f"├── 🔧 Services: {len(services)} ({', '.join(services[:3])}{'...' if len(services) > 3 else ''})")
            
            # Calculate time metrics
            start_dt = datetime.fromisoformat(str(job.start_date))
            end_dt = datetime.fromisoformat(str(job.end_date))
            
            # Calculate months and hours
            months = ((end_dt.year - start_dt.year) * 12 + end_dt.month - start_dt.month) + 1
            hours_total = int(((end_dt - start_dt).total_seconds() / 3600) + 24)
            hours_per_month = hours_total // months if months > 0 else hours_total
            
            console.print(f"├── 📅 Period: {job.start_date} to {job.end_date} ({months} months, {hours_total:,} hours)")
            console.print(f"├── 💰 Total Cost: ${job.total_cost:,.2f}")
            console.print(f"└── 📊 Resource Count: {job.resource_count} per combination")
            
            # Convert Filter dataclass objects to dictionaries for apply_filters()
            filter_dicts = []
            for filter_obj in job.filters:
                filter_dict = {
                    "cloud_provider": filter_obj.cloud_provider,
                    "variance_profile": filter_obj.variance_profile,
                    "anomaly_profile": filter_obj.anomaly_profile,
                    "services": []
                }
                
                for service_obj in filter_obj.services:
                    service_dict = {
                        "name": service_obj.name,
                        "usage_families": []
                    }
                    
                    for uf_obj in service_obj.usage_families:
                        uf_dict = {
                            "name": uf_obj.name,
                            "resource_types": []
                        }
                        
                        for rt_obj in uf_obj.resource_types:
                            rt_dict = {
                                "name": rt_obj.name,
                                "operations": []
                            }
                            
                            for op_obj in rt_obj.operations:
                                op_dict = {
                                    "name": op_obj.name,
                                    "usage_types": op_obj.usage_types
                                }
                                rt_dict["operations"].append(op_dict)
                            
                            uf_dict["resource_types"].append(rt_dict)
                        
                        service_dict["usage_families"].append(uf_dict)
                    
                    filter_dict["services"].append(service_dict)
                
                filter_dicts.append(filter_dict)
            
            # Analyze filters
            with Progress(
                SpinnerColumn(),
                TextColumn("[progress.description]{task.description}"),
                console=console,
                transient=True
            ) as progress:
                task = progress.add_task("Analyzing filters...", total=None)
                filtered_results = apply_filters(ref_df, filter_dicts)
                progress.update(task, completed=True)
            
            # Calculate metrics
            job_combinations = sum(len(filtered_df) for filtered_df, _ in filtered_results)
            
            # Use ACCURATE month-by-month calculation for peak month items
            monthly_line_items = _calculate_monthly_line_items(job, job_combinations)
            job_peak_month_items = max(monthly_line_items) if monthly_line_items else 0
            job_total_items = sum(monthly_line_items) if monthly_line_items else 0
            
            # Count wildcard usage families
            job_wildcards = 0
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    for uf_obj in service_obj.usage_families:
                        if not uf_obj.resource_types:
                            job_wildcards += 1
            
            console.print(f"\n🎯 [cyan]Reference Data Analysis[/cyan]")
            console.print(f"├── Filtered combinations: {job_combinations:,}")
            console.print(f"├── Peak month line items: {job_peak_month_items:,} (calculated month-by-month)")
            console.print(f"├── Total line items: {job_total_items:,} (sum of all months)")
            console.print(f"└── Est. processing time: ~{max(1, job_total_items // 100000)}-{max(2, job_total_items // 50000)} minutes")
            
            # Store job results for Provider Breakdown
            job_results.append({
                'job': job,
                'providers': providers,
                'combinations': job_combinations,
                'peak_month_items': job_peak_month_items,
                'total_items': job_total_items,
                'months': months,
                'filtered_results': filtered_results  # Include filtered data for accurate provider calculations
            })
            
            # Accumulate totals
            total_combinations += job_combinations
            total_line_items += job_total_items
            total_peak_month_items += job_peak_month_items
            wildcard_usage_families += job_wildcards
        
        # Show overall summary
        console.print(f"\n📊 [cyan]Overall Summary[/cyan]")
        console.print(f"├── Total combinations: {total_combinations:,}")
        console.print(f"├── Total peak month line items: {total_peak_month_items:,}")
        console.print(f"├── Total line items across all periods: {total_line_items:,}")
        console.print(f"└── Est. dataset size: ~{max(1, total_line_items // 1000000)} MB parquet")
        
        # Configuration validation  
        status_icon = "✅" if wildcard_usage_families == 0 else "⚠️"
        console.print(f"\n{status_icon} [cyan]Configuration Status:[/cyan] Valid")
        if wildcard_usage_families > 0:
            console.print(f"   ({wildcard_usage_families} usage families use wildcards)")
        
        # Provider Breakdown Table
        _show_provider_breakdown_table(config, job_results, console)
        
        console.print("\n[yellow]Dry run analysis complete - no data generated[/yellow]")


def _perform_enhanced_dry_run_analysis(
    config: GenerateConfig,
    dataset_name: str,
    verbose: bool
):
    """
    Perform enhanced dry run analysis with automatic anomaly detection.

    Uses full pipeline execution for jobs with anomaly profiles and fast
    estimation for jobs without anomaly profiles.
    """
    from ..engine.dry_run_engine import EnhancedDryRunEngine
    from ..processing.cost_analysis import has_anomaly_profiles

    console.print("\n📊 [blue]Enhanced Generation Analysis Report[/blue]")
    console.print("━" * 80)

    # Separate jobs by analysis method needed
    enhanced_jobs = []
    estimation_jobs = []

    for job in config.generation_jobs:
        if has_anomaly_profiles(job):
            enhanced_jobs.append(job)
        else:
            estimation_jobs.append(job)

    # Show analysis strategy
    if enhanced_jobs and estimation_jobs:
        console.print(f"📈 [cyan]Analysis Strategy:[/cyan]")
        console.print(f"├── Enhanced analysis: {len(enhanced_jobs)} jobs with anomaly profiles")
        console.print(f"└── Fast estimation: {len(estimation_jobs)} jobs without anomaly profiles")
    elif enhanced_jobs:
        console.print(f"📈 [cyan]Analysis Strategy:[/cyan] Enhanced analysis for all jobs (anomaly profiles detected)")
    else:
        console.print(f"📈 [cyan]Analysis Strategy:[/cyan] Fast estimation for all jobs (no anomaly profiles)")

    # Run enhanced analysis for anomaly profile jobs
    if enhanced_jobs:
        console.print(f"\n🔥 [bold]Enhanced Cost Analysis (Anomaly Profiles)[/bold]")
        console.print("━" * 50)

        # Use a separate console instance to avoid Rich Progress conflicts
        from rich.console import Console
        enhanced_console = Console()
        enhanced_engine = EnhancedDryRunEngine(console=enhanced_console)

        for job in enhanced_jobs:
            try:
                console.print(f"\n📊 [cyan]Analyzing:[/cyan] {job.name}")

                # Run full pipeline analysis
                analysis_result = enhanced_engine.analyze_job_cost(job)

                # Display enhanced analysis results (will be formatted in Phase 6)
                _display_enhanced_cost_analysis(analysis_result)

            except Exception as e:
                console.print(f"[red]❌ Enhanced analysis failed for '{job.name}': {e}[/red]")
                console.print(f"[dim]   Falling back to estimation for this job[/dim]")
                estimation_jobs.append(job)

    # Run fast estimation for non-anomaly jobs
    if estimation_jobs:
        if enhanced_jobs:
            console.print(f"\n📋 [bold]Fast Estimation (No Anomaly Profiles)[/bold]")
            console.print("━" * 50)

        # Create temporary config with only estimation jobs
        estimation_config = GenerateConfig(
            generation_jobs=estimation_jobs,
            source_file=config.source_file
        )

        # Skip detailed analysis for non-anomaly jobs to avoid Progress conflicts
        console.print(f"[dim]Skipping detailed analysis for {len(estimation_jobs)} jobs without anomaly profiles[/dim]")
        console.print(f"[dim]Use regular dry run (without enhanced analysis) for detailed estimation of these jobs[/dim]")

    console.print(f"\n✅ [green]Enhanced dry run analysis complete[/green]")


def _display_enhanced_cost_analysis(analysis_result):
    """
    Display enhanced cost analysis results with rich formatting and actionable guidance.

    Provides clear, user-friendly output that helps users understand cost impact
    and take appropriate action to adjust their configurations.
    """
    from rich.table import Table
    from rich.panel import Panel

    # Display time period information above the table
    if analysis_result.start_date and analysis_result.end_date:
        start_str = analysis_result.start_date.strftime("%Y-%m-%d")
        end_str = analysis_result.end_date.strftime("%Y-%m-%d")
        duration_days = (analysis_result.end_date - analysis_result.start_date).days

        console.print(f"[bold]📅 Analysis Period:[/bold] {start_str} to {end_str} [dim]({duration_days:,} days)[/dim]")
        console.print()

    # Create cost summary table
    cost_table = Table(show_header=True, header_style="bold cyan")
    cost_table.add_column("Metric", style="cyan", width=20)
    cost_table.add_column("Value", style="white", width=25)
    cost_table.add_column("Status", style="white", width=30)

    # Add cost metrics
    cost_table.add_row(
        "Configured Cost",
        f"${analysis_result.configured_total_cost:,.2f}",
        "[dim]From YAML configuration[/dim]"
    )

    cost_table.add_row(
        "Actual Final Cost",
        f"${analysis_result.actual_final_cost:,.2f}",
        "[dim]After anomaly/variance effects[/dim]"
    )

    # Color-code multiplier based on significance
    multiplier_color = "red" if analysis_result.cost_multiplier > 5.0 or analysis_result.cost_multiplier < 0.2 else "yellow" if analysis_result.cost_multiplier > 2.0 or analysis_result.cost_multiplier < 0.5 else "green"

    cost_table.add_row(
        "Cost Multiplier",
        f"[{multiplier_color}]{analysis_result.cost_multiplier:.2f}x[/{multiplier_color}]",
        _get_multiplier_interpretation(analysis_result.cost_multiplier)
    )

    # Add monthly cost breakdown if available
    if analysis_result.max_monthly_cost is not None:
        cost_table.add_row(
            "Max Monthly Cost",
            f"${analysis_result.max_monthly_cost:,.2f}",
            "[dim]Highest cost month[/dim]"
        )

    if analysis_result.median_monthly_cost is not None:
        cost_table.add_row(
            "Median Monthly Cost",
            f"${analysis_result.median_monthly_cost:,.2f}",
            "[dim]Middle cost month[/dim]"
        )

    # Display the table
    console.print(cost_table)

    # Anomaly information panel (if present)
    if analysis_result.anomaly_type:
        anomaly_panel = _create_anomaly_info_panel(analysis_result)
        console.print()
        console.print(anomaly_panel)

    # Cost deviation analysis and guidance
    if analysis_result.is_significant_deviation():
        guidance_panel = _create_guidance_panel(analysis_result)
        console.print()
        console.print(guidance_panel)
    else:
        console.print("\n✅ [bold green]Cost Impact: Within Expected Range[/bold green]")
        console.print("[dim]   No configuration adjustment needed[/dim]")

    # Generation statistics
    stats_table = Table(show_header=False, box=None, padding=(0, 2))
    stats_table.add_column("Label", style="dim")
    stats_table.add_column("Value", style="white")

    stats_table.add_row("📋 Line Items Generated:", f"{analysis_result.line_item_count:,}")
    stats_table.add_row("⏱️  Analysis Duration:", f"{analysis_result.generation_duration_seconds:.1f}s")
    stats_table.add_row("🗑️  Temporary Dataset:", f"[dim]{analysis_result.temp_dataset_name} (cleaned up)[/dim]")

    console.print(f"\n[bold]Generation Statistics[/bold]")
    console.print(stats_table)


def _get_multiplier_interpretation(multiplier: float) -> str:
    """Get user-friendly interpretation of cost multiplier."""
    if multiplier > 10.0:
        return "[red]Extreme increase[/red]"
    elif multiplier > 5.0:
        return "[red]Very high increase[/red]"
    elif multiplier > 2.0:
        return "[yellow]Significant increase[/yellow]"
    elif multiplier > 1.5:
        return "[yellow]Moderate increase[/yellow]"
    elif multiplier > 0.8:
        return "[green]Normal range[/green]"
    elif multiplier > 0.5:
        return "[yellow]Moderate decrease[/yellow]"
    elif multiplier > 0.2:
        return "[yellow]Significant decrease[/yellow]"
    else:
        return "[red]Extreme decrease[/red]"


def _create_anomaly_info_panel(analysis_result):
    """Create rich panel with anomaly information."""
    from rich.panel import Panel

    anomaly_type = analysis_result.anomaly_type
    anomaly_details = analysis_result.anomaly_details or {}

    # Format anomaly type for display
    display_type = anomaly_type.replace("_", " ").title() if anomaly_type else "Unknown"

    content = f"[bold cyan]Type:[/bold cyan] {display_type}\n"

    # Add specific details based on anomaly type
    if "spike" in anomaly_type.lower():
        content += f"[cyan]Effect:[/cyan] Temporary cost increases during spike periods\n"
        content += f"[cyan]Impact:[/cyan] Time-weighted based on spike frequency and duration"
    elif "step_up" in anomaly_type.lower():
        content += f"[cyan]Effect:[/cyan] Permanent cost increases at step intervals\n"
        content += f"[cyan]Impact:[/cyan] Exponential growth over time (2x per step)"
    elif "step_down" in anomaly_type.lower():
        content += f"[cyan]Effect:[/cyan] Permanent cost reductions at step intervals\n"
        content += f"[cyan]Impact:[/cyan] Exponential reduction over time (÷2 per step)"

    # Add intensity information if available
    if anomaly_details and "profiles" in anomaly_details:
        profiles = anomaly_details["profiles"]
        if profiles:
            profile = profiles[0]  # Show first profile
            if profile.get("intensity"):
                content += f"\n[cyan]Intensity:[/cyan] {profile['intensity']}x"
            if profile.get("auto_calculated_intensity"):
                content += f" [dim](auto-calculated)[/dim]"

    return Panel(
        content,
        title="🔥 Anomaly Profile Impact",
        border_style="yellow",
        padding=(1, 2)
    )


def _create_guidance_panel(analysis_result):
    """Create rich panel with cost adjustment guidance."""
    from rich.panel import Panel

    guidance = analysis_result.get_adjustment_guidance()
    multiplier = analysis_result.cost_multiplier

    # Create specific recommendations based on multiplier
    if multiplier > 5.0:
        urgency = "[bold red]HIGH PRIORITY[/bold red]"
        icon = "🚨"
        recommendation = "Strong recommendation to adjust total_cost before generating large datasets"
    elif multiplier > 2.0 or multiplier < 0.5:
        urgency = "[bold yellow]MODERATE PRIORITY[/bold yellow]"
        icon = "⚠️"
        recommendation = "Consider adjusting total_cost to better match expectations"
    else:
        urgency = "[bold green]LOW PRIORITY[/bold green]"
        icon = "💡"
        recommendation = "Minor adjustment may be helpful for precision"

    content = f"{icon} [bold]Cost Adjustment Recommended[/bold]\n\n"
    content += f"[cyan]Priority:[/cyan] {urgency}\n"
    content += f"[cyan]Guidance:[/cyan] {guidance}\n"
    content += f"[cyan]Next Steps:[/cyan] {recommendation}"

    # Add workflow reminder
    content += "\n\n[dim]Workflow:[/dim]\n"
    content += "[dim]1. Update total_cost in your YAML configuration[/dim]\n"
    content += "[dim]2. Run dry run again to verify adjustment[/dim]\n"
    content += "[dim]3. Generate dataset when satisfied with predicted cost[/dim]"

    return Panel(
        content,
        title="💡 Configuration Adjustment Guidance",
        border_style="yellow",
        padding=(1, 2)
    )


def _has_any_anomaly_profiles(config: GenerateConfig) -> bool:
    """Check if any job in config has anomaly profiles."""
    from ..processing.cost_analysis import has_anomaly_profiles

    return any(has_anomaly_profiles(job) for job in config.generation_jobs)


if __name__ == "__main__":
    app()


def _show_provider_breakdown_table_old(
    config: GenerateConfig,
    job_results: List[Dict],
    console: Console
):
    """OLD VERSION - Show provider breakdown table matching the actual generation output format."""
    from rich.table import Table
    from calendar import monthrange
    
    # Calculate provider stats using ACCURATE month-by-month analysis
    provider_data = {}
    
    for job_result in job_results:
        job = job_result['job']
        providers = job_result['providers']
        job_combinations = job_result['combinations']
        
        # Calculate annual cost for this job's period (normalize to yearly)
        months = job_result['months']
        annual_cost = (job.total_cost / months) * 12 if months > 0 else job.total_cost
        
        # Generate month-by-month line item counts (like real generation)
        monthly_line_items = _calculate_monthly_line_items(job, job_combinations)
        
        for provider in providers:
            if provider not in provider_data:
                provider_data[provider] = {
                    'annual_cost': 0.0,
                    'monthly_line_items': [],  # Store monthly values for peak/low calculation
                }
            
            # Accumulate costs and monthly line items
            provider_data[provider]['annual_cost'] += annual_cost
            provider_data[provider]['monthly_line_items'].extend(monthly_line_items)
    
    # Calculate peak and low from ACTUAL monthly variations (like real generation)
    for provider in provider_data:
        monthly_items = provider_data[provider]['monthly_line_items']
        provider_data[provider]['peak_month_line_items'] = max(monthly_items) if monthly_items else 0
        provider_data[provider]['low_month_line_items'] = min(monthly_items) if monthly_items else 0


def _calculate_monthly_line_items(job: 'GenerationJob', combinations: int) -> List[int]:
    """Calculate line items for each month in the job period (matching real generation logic)."""
    from calendar import monthrange
    
    start_dt = datetime.fromisoformat(str(job.start_date))
    end_dt = datetime.fromisoformat(str(job.end_date))
    
    monthly_counts = []
    current_date = start_dt.replace(day=1)  # Start at beginning of month
    
    while current_date <= end_dt:
        # Get actual days in this month
        year = current_date.year
        month = current_date.month
        days_in_month = monthrange(year, month)[1]
        hours_in_month = days_in_month * 24
        
        # Calculate line items for this specific month
        month_line_items = combinations * job.resource_count * hours_in_month
        monthly_counts.append(month_line_items)
        
        # Move to next month
        current_date += relativedelta(months=1)
    
    return monthly_counts


    # Create table
    table = Table(title="Provider Breakdown", show_header=True, header_style="bold magenta")
    table.add_column("Provider", style="cyan", min_width=11)
    table.add_column("Cost/Year", style="green", min_width=11) 
    table.add_column("Peak Month\nLine Items", style="yellow", min_width=11)
    table.add_column("Low Month\nLine Items", style="blue", min_width=11)
    
    # Sort by annual cost descending
    sorted_providers = sorted(provider_data.items(), key=lambda x: x[1]['annual_cost'], reverse=True)
    
    for provider, stats in sorted_providers:
        # Format cost
        annual_cost = stats['annual_cost']
        if annual_cost >= 1_000_000_000_000:  # Trillions
            cost_str = f"${annual_cost/1_000_000_000_000:.1f}T"
        elif annual_cost >= 1_000_000_000:  # Billions
            cost_str = f"${annual_cost/1_000_000_000:.1f}B"
        elif annual_cost >= 1_000_000:  # Millions
            cost_str = f"${annual_cost/1_000_000:.1f}M"
        elif annual_cost >= 1_000:  # Thousands
            cost_str = f"${annual_cost/1_000:.0f}K"
        else:
            cost_str = f"${annual_cost:.0f}"
        
        # Format line item counts
        peak_items = stats['peak_month_line_items']
        if peak_items >= 1_000_000:
            peak_str = f"{peak_items/1_000_000:.1f}M items"
        elif peak_items >= 1_000:
            peak_str = f"{peak_items/1_000:.0f}K items"
        else:
            peak_str = f"{peak_items} items"
        
        low_items = stats['low_month_line_items']
        if low_items >= 1_000_000:
            low_str = f"{low_items/1_000_000:.1f}M items"
        elif low_items >= 1_000:
            low_str = f"{low_items/1_000:.0f}K items"
        else:
            low_str = f"{low_items} items"
        
        table.add_row(provider, cost_str, peak_str, low_str)
    
    console.print(f"\n")
    console.print(table)


def _perform_dry_run_analysis(
    config: GenerateConfig,
    dataset_name: str,
    verbose: bool
):
    """Perform comprehensive analysis of what would be generated."""
    import polars as pl
    from pathlib import Path
    from datetime import datetime, timedelta
    
    console.print("\n📊 [blue]Generation Analysis Report[/blue]")
    console.print("━" * 80)
    
    # Load reference data using S3DataService
    with Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        console=console,
        transient=True
    ) as progress:
        task = progress.add_task("Ensuring reference data availability...", total=None)
        s3_service = S3DataService()
        ref_path = s3_service.ensure_reference_data()
        progress.update(task, description="Loading reference data...")
        ref_df = pl.read_parquet(ref_path)
        progress.update(task, completed=True)
        
        console.print(f"📋 [cyan]Configuration:[/cyan] {config.source_file.name if config.source_file else 'Merged'}")
        
        # Analyze each job
        total_combinations = 0
        total_line_items = 0
        total_peak_month_items = 0
        wildcard_usage_families = 0
        job_results = []  # Store results for Provider Breakdown
        
        for job in config.generation_jobs:
            # Extract providers and services from filters
            providers = sorted(set(filter_obj.cloud_provider for filter_obj in job.filters))
            all_services = []
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    all_services.append(service_obj.name)
            services = sorted(set(all_services))
            
            console.print(f"\n🏢 [cyan]Job:[/cyan] {job.name}")
            console.print(f"├── 🏅 Providers: {', '.join(providers)}")
            console.print(f"├── 🔧 Services: {len(services)} ({', '.join(services[:3])}{'...' if len(services) > 3 else ''})")
            
            # Calculate time metrics
            start_dt = datetime.fromisoformat(str(job.start_date))
            end_dt = datetime.fromisoformat(str(job.end_date))
            
            # Calculate months and hours
            months = ((end_dt.year - start_dt.year) * 12 + end_dt.month - start_dt.month) + 1
            hours_total = int(((end_dt - start_dt).total_seconds() / 3600) + 24)
            hours_per_month = hours_total // months if months > 0 else hours_total
            
            console.print(f"├── 📅 Period: {job.start_date} to {job.end_date} ({months} months, {hours_total:,} hours)")
            console.print(f"├── 💰 Total Cost: ${job.total_cost:,.2f}")
            console.print(f"└── 📊 Resource Count: {job.resource_count} per combination")
            
            # Convert Filter dataclass objects to dictionaries for apply_filters()
            filter_dicts = []
            for filter_obj in job.filters:
                filter_dict = {
                    "cloud_provider": filter_obj.cloud_provider,
                    "variance_profile": filter_obj.variance_profile,
                    "anomaly_profile": filter_obj.anomaly_profile,
                    "services": []
                }
                
                for service_obj in filter_obj.services:
                    service_dict = {
                        "name": service_obj.name,
                        "usage_families": []
                    }
                    
                    for uf_obj in service_obj.usage_families:
                        uf_dict = {
                            "name": uf_obj.name,
                            "resource_types": []
                        }
                        
                        for rt_obj in uf_obj.resource_types:
                            rt_dict = {
                                "name": rt_obj.name,
                                "operations": []
                            }
                            
                            for op_obj in rt_obj.operations:
                                op_dict = {
                                    "name": op_obj.name,
                                    "usage_types": op_obj.usage_types
                                }
                                rt_dict["operations"].append(op_dict)
                            
                            uf_dict["resource_types"].append(rt_dict)
                        
                        service_dict["usage_families"].append(uf_dict)
                    
                    filter_dict["services"].append(service_dict)
                
                filter_dicts.append(filter_dict)
            
            # Analyze filters
            with Progress(
                SpinnerColumn(),
                TextColumn("[progress.description]{task.description}"),
                console=console,
                transient=True
            ) as progress:
                task = progress.add_task("Analyzing filters...", total=None)
                filtered_results = apply_filters(ref_df, filter_dicts)
                progress.update(task, completed=True)
            
            # Calculate metrics
            job_combinations = sum(len(filtered_df) for filtered_df, _ in filtered_results)
            
            # Use ACCURATE month-by-month calculation for peak month items
            monthly_line_items = _calculate_monthly_line_items(job, job_combinations)
            job_peak_month_items = max(monthly_line_items) if monthly_line_items else 0
            job_total_items = sum(monthly_line_items) if monthly_line_items else 0
            
            # Count wildcard usage families
            job_wildcards = 0
            for filter_obj in job.filters:
                for service_obj in filter_obj.services:
                    for uf_obj in service_obj.usage_families:
                        if not uf_obj.resource_types:
                            job_wildcards += 1
            
            console.print(f"\n🎯 [cyan]Reference Data Analysis[/cyan]")
            console.print(f"├── Filtered combinations: {job_combinations:,}")
            console.print(f"├── Peak month line items: {job_peak_month_items:,} (calculated month-by-month)")
            console.print(f"├── Total line items: {job_total_items:,} (sum of all months)")
            console.print(f"└── Est. processing time: ~{max(1, job_total_items // 100000)}-{max(2, job_total_items // 50000)} minutes")
            
            # Store job results for Provider Breakdown
            job_results.append({
                'job': job,
                'providers': providers,
                'combinations': job_combinations,
                'peak_month_items': job_peak_month_items,
                'total_items': job_total_items,
                'months': months,
                'filtered_results': filtered_results  # Include filtered data for accurate provider calculations
            })
            
            # Accumulate totals
            total_combinations += job_combinations
            total_line_items += job_total_items
            total_peak_month_items += job_peak_month_items
            wildcard_usage_families += job_wildcards
        
        # Show overall summary
        console.print(f"\n📊 [cyan]Overall Summary[/cyan]")
        console.print(f"├── Total combinations: {total_combinations:,}")
        console.print(f"├── Total peak month line items: {total_peak_month_items:,}")
        console.print(f"├── Total line items across all periods: {total_line_items:,}")
        console.print(f"└── Est. dataset size: ~{max(1, total_line_items // 1000000)} MB parquet")
        
        # Configuration validation  
        status_icon = "✅" if wildcard_usage_families == 0 else "⚠️"
        console.print(f"\n{status_icon} [cyan]Configuration Status:[/cyan] Valid")
        if wildcard_usage_families > 0:
            console.print(f"   ({wildcard_usage_families} usage families use wildcards)")
        
        # Provider Breakdown Table
        _show_provider_breakdown_table(config, job_results, console)
        
        console.print("\n[yellow]Dry run analysis complete - no data generated[/yellow]")


def _check_and_handle_existing_data(s3_service: S3DataService, dataset_name: str, force: bool) -> bool:
    """
    Check for existing success marker and handle force flag logic.
    
    Args:
        s3_service: S3 data service instance
        dataset_name: Dataset name to check
        force: Whether force flag was provided
        
    Returns:
        True if should exit (data exists and not forced), False if should continue
    """
    if _generate_success_marker_exists(s3_service, dataset_name):
        if not force:
            console.print(f"✅ Generate stage already completed for dataset '{dataset_name}'. Use --force to reprocess.")
            return True
        
        # Force flag provided - confirm and delete existing data
        console.print(f"⚠️  This will delete ALL existing generate data for dataset '{dataset_name}' and reprocess.")
        
        # Confirmation prompt
        confirm = typer.confirm("❓ Continue?")
        if not confirm:
            console.print("Operation cancelled.")
            return True
        
        # Delete existing generate data
        console.print(f"🗑️  Deleting existing generate data for dataset '{dataset_name}'...")
        _delete_generate_stage_data(s3_service, dataset_name)
        console.print(f"✅ Existing generate data deleted for dataset '{dataset_name}'")
    
    return False


def _generate_success_marker_exists(s3_service: S3DataService, dataset_name: str) -> bool:
    """
    Check if generate stage success marker exists for a dataset.
    
    Args:
        s3_service: S3 data service instance
        dataset_name: Dataset name to check
        
    Returns:
        True if success marker exists, False otherwise
    """
    from src.services.success_marker_service import SuccessMarkerService
    success_marker_service = SuccessMarkerService(s3_service)
    
    return success_marker_service.check_success_marker_exists(dataset_name, "generate")


def _delete_generate_stage_data(s3_service: S3DataService, dataset_name: str) -> None:
    """
    Delete all generate stage data for a dataset.
    
    Deletes entire generate folder: s3://bucket/czbillgen/datasets/{dataset_name}/generate/
    This ensures no mixed configuration data remains.
    
    Args:
        s3_service: S3 data service instance
        dataset_name: Dataset name to delete data for
    """
    stage_prefix = f"czbillgen/datasets/{dataset_name}/generate/"
    
    # List all objects in generate stage folder
    paginator = s3_service.s3_client.get_paginator('list_objects_v2')
    page_iterator = paginator.paginate(
        Bucket=s3_service.bucket,
        Prefix=stage_prefix
    )
    
    objects_to_delete = []
    for page in page_iterator:
        if 'Contents' in page:
            objects_to_delete.extend([{'Key': obj['Key']} for obj in page['Contents']])
    
    if objects_to_delete:
        # Delete objects in batches (S3 delete_objects limit is 1000)
        for i in range(0, len(objects_to_delete), 1000):
            batch = objects_to_delete[i:i+1000]
            s3_service.s3_client.delete_objects(
                Bucket=s3_service.bucket,
                Delete={'Objects': batch}
            )
            
            console.print(f"🗑️  Deleted {len(objects_to_delete)} objects from generate stage")
        else:
            console.print("📂 No existing generate data found to delete")
        
        # Also delete success marker using standardized service
        from src.services.success_marker_service import SuccessMarkerService
        success_marker_service = SuccessMarkerService(s3_service)
        success_marker_service.delete_success_marker(dataset_name, "generate")


def _execute_via_api(
    config_path: Path,
    dataset_name: Optional[str],
    description: Optional[str],
    dry_run: bool,
    force: bool
) -> None:
    """
    Execute generate job via API.

    Single responsibility: Route generate command to API execution.
    Method size: <50 lines (CLAUDE.md compliant).
    """
    from rich.console import Console
    from src.api.router import ExecutionRouter

    console_api = Console()

    # Dry run not supported in API mode
    if dry_run:
        console_api.print(
            "[yellow]⚠️  Dry run is not supported in API mode.[/yellow]"
        )
        console_api.print(
            "[yellow]   Please use --local for dry run analysis.[/yellow]"
        )
        raise typer.Exit(1)

    try:
        # Initialize execution router
        router = ExecutionRouter(console_api)

        # Build kwargs for API submission
        kwargs = {}
        if force:
            kwargs["force"] = force

        # Execute via API
        router.execute_api(
            job_type="generate",
            config_path=config_path,
            dataset_name=dataset_name,
            description=description,
            **kwargs,
        )

    except Exception as e:
        console_api.print(f"[red]✗ API execution failed: {e}[/red]")
        raise typer.Exit(1)


# Temporarily disabled due to profiles system removal
# @app.command("list-profiles")
# def list_profiles(
#     profile_type: Optional[str] = typer.Option(
#         None, 
#         "--type", 
#         help="Profile type to list (variance or anomaly)"
#     )
# ):
#     """List available variance and anomaly profiles for generate configs."""
#     profiles_cmd.list_profiles(profile_type)
# 
# 
# @app.command("show-profile")
# def show_profile(
#     profile_name: str = typer.Argument(..., help="Profile name to show details for")
# ):
#     """Show detailed information about a specific profile."""
#     profiles_cmd.show_profile(profile_name)


if __name__ == "__main__":
    app()