DATA_INTEGRATION_QUICK_START.sh

#!/bin/bash
# ChuckkleNet Data Integration Quick Start Script
# Automated execution of Stage 1 data integration

set -e  # Exit on error
set -o pipefail  # Catch pipe failures

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Configuration
PROJECT_DIR="/Users/Subho/autonomous_laughter_prediction_essential"
DATA_DIR="$PROJECT_DIR/data"
EXTERNAL_DIR="$DATA_DIR/external"
PROCESSED_DIR="$DATA_DIR/processed"
STAGE1_DIR="$PROCESSED_DIR/stage1"

# Create directories
log_info "Creating directory structure..."
mkdir -p "$STAGE1_DIR"
mkdir -p "$PROJECT_DIR/training/enhanced"
mkdir -p "$PROJECT_DIR/checkpoints/stage1"
mkdir -p "$PROJECT_DIR/logs/stage1"

# Change to project directory
cd "$PROJECT_DIR"

log_info "Working directory: $(pwd)"

# Step 1: Verify data availability
log_info "Step 1: Verifying data availability..."

if [ ! -f "$EXTERNAL_DIR/synthetic_50k.jsonl" ]; then
    log_error "Synthetic 50k data not found at $EXTERNAL_DIR/synthetic_50k.jsonl"
    exit 1
fi

if [ ! -f "$DATA_DIR/standup_transcript_examples.jsonl" ]; then
    log_error "Original standup data not found at $DATA_DIR/standup_transcript_examples.jsonl"
    exit 1
fi

log_success "Data files verified"

# Step 2: Create quality validator
log_info "Step 2: Creating quality validation framework..."

cat > "$PROJECT_DIR/training/enhanced/quality_validator.py" << 'EOF'
#!/usr/bin/env python3
"""Quick Quality Validation for Stage 1 Integration"""

import json
import logging
import random
from pathlib import Path
from typing import Dict, List, Tuple
import numpy as np

logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
logger = logging.getLogger(__name__)

def quick_validate_synthetic(data_path: str, sample_size: int = 1000) -> Dict:
    """Quick validation of synthetic data quality"""
    logger.info(f"Validating {data_path}")

    # Load and sample data
    data = []
    with open(data_path, 'r') as f:
        for line in f:
            data.append(json.loads(line))

    if len(data) > sample_size:
        sample = random.sample(data, sample_size)
    else:
        sample = data

    logger.info(f"Sampled {len(sample)} examples for validation")

    # Quality checks
    passed = 0
    failed = 0
    issues = []

    for example in sample:
        example_passed = True

        # Check required fields
        if 'text' not in example or 'label' not in example:
            example_passed = False
            issues.append("Missing required fields")

        # Check label validity
        if example.get('label') not in [0, 1]:
            example_passed = False
            issues.append("Invalid label")

        # Check text quality
        text = example.get('text', '')
        if len(text) < 5 or len(text) > 500:
            example_passed = False
            issues.append("Text length issue")

        if example_passed:
            passed += 1
        else:
            failed += 1

    quality_score = passed / len(sample) if sample else 0

    results = {
        'total_sampled': len(sample),
        'passed': passed,
        'failed': failed,
        'quality_score': quality_score,
        'sample_issues': issues[:10]  # First 10 issues
    }

    logger.info(f"Quality score: {quality_score:.2%}")
    logger.info(f"Passed: {passed}/{len(sample)}")

    return results

def filter_high_quality(data_path: str, output_path: str, min_quality: float = 0.7) -> int:
    """Filter high-quality examples from synthetic data"""
    logger.info(f"Filtering high-quality examples from {data_path}")

    high_quality = []
    low_quality = []

    with open(data_path, 'r') as f:
        for line in f:
            example = json.loads(line)

            # Simple quality check
            text = example.get('text', '')
            label = example.get('label')

            quality_score = 1.0

            # Check basic quality
            if 'text' not in example or 'label' not in example:
                quality_score -= 0.3

            if label not in [0, 1]:
                quality_score -= 0.3

            if len(text) < 5 or len(text) > 500:
                quality_score -= 0.2

            # Check for reasonable content
            words = text.split()
            if len(words) < 2:
                quality_score -= 0.2

            if quality_score >= min_quality:
                high_quality.append(example)
            else:
                low_quality.append(example)

    # Save high-quality data
    with open(output_path, 'w') as f:
        for example in high_quality:
            f.write(json.dumps(example) + '\n')

    logger.info(f"Saved {len(high_quality)} high-quality examples to {output_path}")
    logger.info(f"Rejected {len(low_quality)} low-quality examples")

    return len(high_quality)

def main():
    """Main execution"""
    logger.info("Starting quick quality validation")

    # Paths
    synthetic_path = "data/external/synthetic_50k.jsonl"
    output_path = "data/processed/stage1/high_quality_synthetic.jsonl"

    # Quick validation
    results = quick_validate_synthetic(synthetic_path)

    print(f"\n{'='*50}")
    print("QUALITY VALIDATION RESULTS")
    print(f"{'='*50}")
    print(f"Sampled: {results['total_sampled']} examples")
    print(f"Passed: {results['passed']} ({results['passed']/results['total_sampled']:.1%})")
    print(f"Failed: {results['failed']} ({results['failed']/results['total_sampled']:.1%})")
    print(f"Quality Score: {results['quality_score']:.2%}")

    if results['sample_issues']:
        print(f"\nSample Issues:")
        for issue in results['sample_issues'][:5]:
            print(f"  - {issue}")

    # Filter high-quality examples
    count = filter_high_quality(synthetic_path, output_path)

    print(f"\n{'='*50}")
    print("HIGH-QUALITY DATA EXTRACTION")
    print(f"{'='*50}")
    print(f"High-quality examples: {count}")
    print(f"Saved to: {output_path}")

    if count > 10000:
        print("✅ Sufficient high-quality data for Stage 1")
    else:
        print("⚠️  Limited high-quality data - consider adjusting quality threshold")

if __name__ == "__main__":
    main()
EOF

chmod +x "$PROJECT_DIR/training/enhanced/quality_validator.py"
log_success "Quality validator created"

# Step 3: Run quality validation
log_info "Step 3: Running quality validation..."
python3 "$PROJECT_DIR/training/enhanced/quality_validator.py"

# Check if validation was successful
if [ ! -f "$STAGE1_DIR/high_quality_synthetic.jsonl" ]; then
    log_error "Quality validation failed - no output file created"
    exit 1
fi

log_success "Quality validation completed"

# Step 4: Create Stage 1 integration script
log_info "Step 4: Creating Stage 1 integration script..."

cat > "$PROJECT_DIR/training/enhanced/stage1_integration.py" << 'EOF'
#!/usr/bin/env python3
"""Stage 1 Data Integration - Quick Implementation"""

import json
import random
import logging
from pathlib import Path

logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
logger = logging.getLogger(__name__)

def create_stage1_dataset():
    """Create Stage 1 dataset with original + synthetic data"""
    logger.info("Creating Stage 1 dataset")

    # Paths
    original_path = "data/standup_transcript_examples.jsonl"
    synthetic_path = "data/processed/stage1/high_quality_synthetic.jsonl"
    output_dir = Path("data/processed/stage1/")
    output_dir.mkdir(parents=True, exist_ok=True)

    # Load original data
    logger.info("Loading original standup data...")
    original_data = []
    with open(original_path, 'r') as f:
        for line in f:
            original_data.append(json.loads(line))

    logger.info(f"Loaded {len(original_data)} original examples")

    # Load synthetic data
    logger.info("Loading high-quality synthetic data...")
    synthetic_data = []
    with open(synthetic_path, 'r') as f:
        for line in f:
            synthetic_data.append(json.loads(line))

    logger.info(f"Loaded {len(synthetic_data)} synthetic examples")

    # Sample synthetic data (target: 10,000 for Stage 1)
    target_synthetic = 10000
    if len(synthetic_data) > target_synthetic:
        sampled_synthetic = random.sample(synthetic_data, target_synthetic)
    else:
        sampled_synthetic = synthetic_data

    logger.info(f"Sampled {len(sampled_synthetic)} synthetic examples")

    # Combine datasets
    stage1_data = original_data + sampled_synthetic
    random.shuffle(stage1_data)

    logger.info(f"Stage 1 dataset: {len(stage1_data)} total examples")

    # Analyze labels
    labels = [ex.get('label', 0) for ex in stage1_data]
    positive_count = labels.count(1)
    negative_count = labels.count(0)

    logger.info(f"Label distribution:")
    logger.info(f"  Positive (1): {positive_count} ({positive_count/len(stage1_data):.1%})")
    logger.info(f"  Negative (0): {negative_count} ({negative_count/len(stage1_data):.1%})")

    # Create train/val split (90/10)
    random.shuffle(stage1_data)
    split_idx = int(len(stage1_data) * 0.9)

    train_data = stage1_data[:split_idx]
    val_data = stage1_data[split_idx:]

    logger.info(f"Train/val split: {len(train_data)}/{len(val_data)}")

    # Save datasets
    train_path = output_dir / 'train.jsonl'
    val_path = output_dir / 'val.jsonl'

    with open(train_path, 'w') as f:
        for example in train_data:
            f.write(json.dumps(example) + '\n')

    with open(val_path, 'w') as f:
        for example in val_data:
            f.write(json.dumps(example) + '\n')

    logger.info(f"Saved training data to {train_path}")
    logger.info(f"Saved validation data to {val_path}")

    # Save metadata
    metadata = {
        'stage': 1,
        'total_examples': len(stage1_data),
        'train_examples': len(train_data),
        'val_examples': len(val_data),
        'original_examples': len(original_data),
        'synthetic_examples': len(sampled_synthetic),
        'label_distribution': {
            'positive': positive_count,
            'negative': negative_count,
            'positive_ratio': positive_count / len(stage1_data)
        }
    }

    metadata_path = output_dir / 'metadata.json'
    with open(metadata_path, 'w') as f:
        json.dump(metadata, f, indent=2)

    logger.info(f"Saved metadata to {metadata_path}")

    return metadata

def main():
    """Main execution"""
    logger.info("="*50)
    logger.info("STAGE 1 DATA INTEGRATION")
    logger.info("="*50)

    metadata = create_stage1_dataset()

    print(f"\n{'='*50}")
    print("STAGE 1 INTEGRATION COMPLETE")
    print(f"{'='*50}")
    print(f"Total examples: {metadata['total_examples']:,}")
    print(f"Training: {metadata['train_examples']:,}")
    print(f"Validation: {metadata['val_examples']:,}")
    print(f"\nData composition:")
    print(f"  Original: {metadata['original_examples']:,}")
    print(f"  Synthetic: {metadata['synthetic_examples']:,}")
    print(f"\nLabel balance:")
    print(f"  Positive: {metadata['label_distribution']['positive']:,} ({metadata['label_distribution']['positive_ratio']:.1%})")
    print(f"  Negative: {metadata['label_distribution']['negative']:,}")

    print(f"\n✅ Stage 1 dataset ready for training!")

if __name__ == "__main__":
    main()
EOF

chmod +x "$PROJECT_DIR/training/enhanced/stage1_integration.py"
log_success "Stage 1 integration script created"

# Step 5: Run Stage 1 integration
log_info "Step 5: Running Stage 1 data integration..."
python3 "$PROJECT_DIR/training/enhanced/stage1_integration.py"

# Verify integration success
if [ ! -f "$STAGE1_DIR/train.jsonl" ] || [ ! -f "$STAGE1_DIR/val.jsonl" ]; then
    log_error "Stage 1 integration failed"
    exit 1
fi

log_success "Stage 1 integration completed"

# Step 6: Display results
log_info "Stage 1 integration results:"
echo ""
echo "📊 DATASET STATISTICS"
echo "===================="

if [ -f "$STAGE1_DIR/metadata.json" ]; then
    python3 << 'PYTHON_SCRIPT'
import json
with open('data/processed/stage1/metadata.json') as f:
    metadata = json.load(f)

print(f"Total examples: {metadata['total_examples']:,}")
print(f"Training: {metadata['train_examples']:,}")
print(f"Validation: {metadata['val_examples']:,}")
print(f"\nData Sources:")
print(f"  Original standup: {metadata['original_examples']:,}")
print(f"  Synthetic: {metadata['synthetic_examples']:,}")
print(f"\nLabel Distribution:")
print(f"  Positive: {metadata['label_distribution']['positive']:,} ({metadata['label_distribution']['positive_ratio']:.1%})")
print(f"  Negative: {metadata['label_distribution']['negative']:,} ({1-metadata['label_distribution']['positive_ratio']:.1%})")
PYTHON_SCRIPT
fi

# Step 7: Next steps guidance
echo ""
log_success "🎉 Stage 1 data integration completed successfully!"
echo ""
echo "📁 Generated files:"
echo "  - $STAGE1_DIR/train.jsonl ($(wc -l < "$STAGE1_DIR/train.jsonl") training examples)"
echo "  - $STAGE1_DIR/val.jsonl ($(wc -l < "$STAGE1_DIR/val.jsonl") validation examples)"
echo "  - $STAGE1_DIR/metadata.json (integration metadata)"
echo ""
echo "🚀 Next Steps:"
echo "  1. Review the generated datasets"
echo "  2. Run enhanced training: python3 training/enhanced/stage1_train.py"
echo "  3. Monitor performance in logs/stage1/"
echo ""
echo "📈 Expected Performance: 73.5% F1 ± 0.5%"
echo ""

# Verification
log_info "Performing final verification..."

TRAIN_COUNT=$(wc -l < "$STAGE1_DIR/train.jsonl")
VAL_COUNT=$(wc -l < "$STAGE1_DIR/val.jsonl")
TOTAL_COUNT=$((TRAIN_COUNT + VAL_COUNT))

if [ $TOTAL_COUNT -ge 10000 ] && [ $TOTAL_COUNT -le 11000 ]; then
    log_success "✅ Dataset size within expected range (10,000-11,000 examples)"
else
    log_warning "⚠️  Dataset size outside expected range: $TOTAL_COUNT examples"
fi

log_info "Quick Start completed successfully!"
echo ""
echo "💡 Pro tip: Monitor training progress with:"
echo "   tail -f logs/stage1/training.log"
echo ""