#!/bin/bash

# Optimized AWS S3 sync for millions of files

# Configure AWS CLI for maximum performance (only needs to be done once)
echo "Configuring AWS CLI for maximum performance..."
aws configure set default.s3.max_concurrent_requests 100
aws configure set default.s3.max_queue_size 100000
aws configure set default.s3.multipart_threshold 64MB
aws configure set default.s3.multipart_chunksize 16MB
aws configure set default.s3.use_threads True

# Increase system file descriptor limit
ulimit -n 65536

# Set retry behavior for resilience
export AWS_MAX_ATTEMPTS=10
export AWS_RETRY_MODE=adaptive

# Source and destination
LOCAL_PATH="/app2/suno/data/sara/sfx_get_beats/aligned_audio"
S3_PATH="s3://suno-data/datasets/bundles/v5/sfx_all/aligned_audio"

# Run the sync
echo "Starting sync at $(date)"
echo "Source: $LOCAL_PATH"
echo "Destination: $S3_PATH"

aws s3 sync "$LOCAL_PATH" "$S3_PATH" --only-show-errors

echo "Sync completed at $(date)"