diff --git a/deploy.sh b/deploy.sh index 662040a..ed06176 100644 --- a/deploy.sh +++ b/deploy.sh @@ -40,11 +40,12 @@ echo "" echo -e "${YELLOW}This script will:${NC}" echo -e " ${BLUE}1.${NC} Tear down existing containers" echo -e " ${BLUE}2.${NC} Remove all data volumes" -echo -e " ${BLUE}3.${NC} Fix file permissions" -echo -e " ${BLUE}4.${NC} Start fresh containers" -echo -e " ${BLUE}5.${NC} Initialize database schema" -echo -e " ${BLUE}6.${NC} Generate test data" -echo -e " ${BLUE}7.${NC} Verify deployment" +echo -e " ${BLUE}3.${NC} Check and install dependencies (Python, psql, Docker)" +echo -e " ${BLUE}4.${NC} Fix file permissions" +echo -e " ${BLUE}5.${NC} Start fresh containers" +echo -e " ${BLUE}6.${NC} Initialize database schema" +echo -e " ${BLUE}7.${NC} Generate test data" +echo -e " ${BLUE}8.${NC} Verify deployment" echo "" echo -e "${YELLOW}⚠️ WARNING: This will DELETE all existing data!${NC}" echo "" @@ -84,7 +85,38 @@ fi echo "" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" -echo -e "${CYAN}[STEP 3/7] Fixing file permissions...${NC}" +echo -e "${CYAN}[STEP 3/7] Checking dependencies...${NC}" +echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" +echo "" + +# Check for required dependencies +MISSING_DEPS=() + +if ! command -v python3 >/dev/null 2>&1; then + MISSING_DEPS+=("python3") +fi + +if ! command -v psql >/dev/null 2>&1; then + MISSING_DEPS+=("psql") +fi + +if ! command -v docker >/dev/null 2>&1; then + MISSING_DEPS+=("docker") +fi + +if [ ${#MISSING_DEPS[@]} -gt 0 ]; then + echo -e "${YELLOW}⚠ Missing dependencies: ${MISSING_DEPS[*]}${NC}" + echo -e "${YELLOW}Running dependency installer...${NC}" + echo "" + chmod +x scripts/install-dependencies.sh 2>/dev/null || true + ./scripts/install-dependencies.sh +else + echo -e "${GREEN}✓ All dependencies are installed${NC}" +fi + +echo "" +echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" +echo -e "${CYAN}[STEP 4/7] Fixing file permissions...${NC}" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" echo "" @@ -107,7 +139,7 @@ fi echo "" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" -echo -e "${CYAN}[STEP 4/7] Starting containers...${NC}" +echo -e "${CYAN}[STEP 5/8] Starting containers...${NC}" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" echo "" @@ -147,7 +179,7 @@ echo -e "${YELLOW}Note: pgAdmin may take 30-60 seconds to fully initialize${NC}" echo "" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" -echo -e "${CYAN}[STEP 5/7] Initializing database schema...${NC}" +echo -e "${CYAN}[STEP 6/8] Initializing database schema...${NC}" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" echo "" @@ -155,7 +187,7 @@ echo "" echo "" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" -echo -e "${CYAN}[STEP 6/7] Generating test data...${NC}" +echo -e "${CYAN}[STEP 7/8] Generating test data...${NC}" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" echo "" echo -e "${YELLOW}⏱️ This will take 15-30 minutes depending on your system...${NC}" @@ -165,7 +197,7 @@ echo "" echo "" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" -echo -e "${CYAN}[STEP 7/7] Verifying deployment...${NC}" +echo -e "${CYAN}[STEP 8/8] Verifying deployment...${NC}" echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}" echo "" diff --git a/scripts/generate_csv_data.py b/scripts/generate_csv_data.py new file mode 100644 index 0000000..dda71c0 --- /dev/null +++ b/scripts/generate_csv_data.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +""" +Generate CSV files for bulk import into PostgreSQL +Much faster and more reliable than INSERT statements +""" + +import csv +import random +import os +from datetime import datetime, timedelta +from pathlib import Path + +# Configuration +NUM_CUSTOMERS = 100000 +NUM_ACCOUNTS = 150000 +NUM_MERCHANTS = 50000 +NUM_CARDS = 200000 +NUM_TRANSACTIONS = 5000000 +FRAUD_RATE = 0.07 + +# Output directory +DATA_DIR = Path(__file__).parent.parent / "data" / "csv" +DATA_DIR.mkdir(parents=True, exist_ok=True) + +print("=" * 80) +print("CSV Data Generation for Business Analytics Database") +print("=" * 80) +print(f"Output directory: {DATA_DIR}") +print(f"Customers: {NUM_CUSTOMERS:,}") +print(f"Accounts: {NUM_ACCOUNTS:,}") +print(f"Merchants: {NUM_MERCHANTS:,}") +print(f"Cards: {NUM_CARDS:,}") +print(f"Transactions: {NUM_TRANSACTIONS:,}") +print(f"Fraud rate: {FRAUD_RATE*100}%") +print("=" * 80) +print() + +# Reference data +FIRST_NAMES = ["James", "Mary", "John", "Patricia", "Robert", "Jennifer", "Michael", "Linda", + "William", "Barbara", "David", "Elizabeth", "Richard", "Susan", "Joseph", "Jessica", + "Thomas", "Sarah", "Charles", "Karen", "Christopher", "Nancy", "Daniel", "Lisa"] + +LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", + "Rodriguez", "Martinez", "Hernandez", "Lopez", "Gonzalez", "Wilson", "Anderson", "Thomas", + "Taylor", "Moore", "Jackson", "Martin", "Lee", "Perez", "Thompson", "White"] + +CITIES = ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", + "San Diego", "Dallas", "San Jose", "Austin", "Jacksonville", "Fort Worth", "Columbus", + "Charlotte", "San Francisco", "Indianapolis", "Seattle", "Denver", "Boston"] + +STATES = ["NY", "CA", "IL", "TX", "AZ", "PA", "TX", "CA", "TX", "CA", "TX", "FL", "TX", "OH", + "NC", "CA", "IN", "WA", "CO", "MA"] + +COUNTRIES = ["US", "CA", "GB", "DE", "FR", "AU", "JP", "CN", "IN", "BR"] + +MERCHANT_TYPES = [ + ("5411", "Grocery Stores"), ("5812", "Restaurants"), ("5541", "Gas Stations"), + ("5311", "Department Stores"), ("5912", "Pharmacies"), ("5999", "Miscellaneous Retail"), + ("5732", "Electronics"), ("5651", "Clothing"), ("5814", "Fast Food"), + ("5942", "Books"), ("7011", "Hotels"), ("7512", "Car Rental") +] + +TRANSACTION_TYPES = ["PURCHASE", "ATM_WITHDRAWAL", "TRANSFER_OUT", "TRANSFER_IN", "PAYMENT", + "REFUND", "DEPOSIT", "WIRE_OUT", "WIRE_IN"] + +FRAUD_CODES = ["CARD_STOLEN", "ACCOUNT_TAKEOVER", "IDENTITY_THEFT", "SYNTHETIC_IDENTITY", + "CARD_NOT_PRESENT", "PHISHING", "ATM_SKIMMING", "WIRE_FRAUD"] + +def generate_email(first_name, last_name, customer_id): + """Generate realistic email address""" + domains = ["gmail.com", "yahoo.com", "hotmail.com", "outlook.com", "icloud.com"] + return f"{first_name.lower()}.{last_name.lower()}{customer_id % 1000}@{random.choice(domains)}" + +def generate_phone(): + """Generate US phone number""" + return f"+1{random.randint(2000000000, 9999999999)}" + +def generate_ssn(): + """Generate SSN-like identifier""" + return f"{random.randint(100, 999)}-{random.randint(10, 99)}-{random.randint(1000, 9999)}" + +def generate_card_number(): + """Generate realistic card number""" + prefix = random.choice(["4", "5"]) # Visa or Mastercard + return prefix + "".join([str(random.randint(0, 9)) for _ in range(15)]) + +def random_date(start_date, end_date): + """Generate random date between start and end""" + delta = end_date - start_date + random_days = random.randint(0, delta.days) + return start_date + timedelta(days=random_days) + +def random_datetime(start_date, end_date): + """Generate random datetime between start and end""" + delta = end_date - start_date + random_seconds = random.randint(0, int(delta.total_seconds())) + return start_date + timedelta(seconds=random_seconds) + +print("[1/10] Generating customers...") +customers = [] +with open(DATA_DIR / "customers.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["customer_id", "first_name", "last_name", "email", "phone", "date_of_birth", + "ssn", "address_line1", "city", "state", "postal_code", "country_code", + "risk_score", "kyc_status", "created_at"]) + + for i in range(1, NUM_CUSTOMERS + 1): + first_name = random.choice(FIRST_NAMES) + last_name = random.choice(LAST_NAMES) + city_idx = random.randint(0, len(CITIES) - 1) + + customer = [ + i, # customer_id + first_name, + last_name, + generate_email(first_name, last_name, i), + generate_phone(), + random_date(datetime(1950, 1, 1), datetime(2005, 12, 31)).strftime("%Y-%m-%d"), + generate_ssn(), + f"{random.randint(100, 9999)} {random.choice(['Main', 'Oak', 'Maple', 'Park'])} St", + CITIES[city_idx], + STATES[city_idx], + f"{random.randint(10000, 99999)}", + random.choice(COUNTRIES), + round(random.uniform(0, 100), 2), + random.choice(["VERIFIED", "PENDING", "REJECTED"]), + random_datetime(datetime(2020, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d %H:%M:%S") + ] + writer.writerow(customer) + customers.append(i) + + if i % 10000 == 0: + print(f" Generated {i:,} customers...") + +print(f"✓ Generated {NUM_CUSTOMERS:,} customers") + +print("[2/10] Generating accounts...") +accounts = [] +with open(DATA_DIR / "accounts.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["account_id", "customer_id", "account_number", "account_type", "currency", + "balance", "available_balance", "status", "opened_date", "closed_date"]) + + for i in range(1, NUM_ACCOUNTS + 1): + customer_id = random.choice(customers) + account_type = random.choice(["CHECKING", "SAVINGS", "CREDIT"]) + balance = round(random.uniform(100, 50000), 2) + + account = [ + i, # account_id + customer_id, + f"{random.randint(1000000000, 9999999999)}", + account_type, + "USD", + balance, + balance * random.uniform(0.8, 1.0), + random.choice(["ACTIVE", "ACTIVE", "ACTIVE", "SUSPENDED", "CLOSED"]), + random_date(datetime(2020, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d"), + "" if random.random() > 0.05 else random_date(datetime(2023, 1, 1), datetime(2024, 12, 31)).strftime("%Y-%m-%d") + ] + writer.writerow(account) + accounts.append(i) + + if i % 10000 == 0: + print(f" Generated {i:,} accounts...") + +print(f"✓ Generated {NUM_ACCOUNTS:,} accounts") + +print("[3/10] Generating merchants...") +merchants = [] +with open(DATA_DIR / "merchants.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["merchant_id", "merchant_name", "category_id", "country_code", "city", + "risk_rating", "is_active", "created_at"]) + + for i in range(1, NUM_MERCHANTS + 1): + category = random.choice(MERCHANT_TYPES) + city_idx = random.randint(0, len(CITIES) - 1) + + merchant = [ + i, # merchant_id + f"{category[1]} #{i}", + random.randint(1, 35), # category_id from merchant_categories + random.choice(COUNTRIES), + CITIES[city_idx], + round(random.uniform(1, 10), 2), + random.choice([True, True, True, False]), + random_datetime(datetime(2015, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d %H:%M:%S") + ] + writer.writerow(merchant) + merchants.append(i) + + if i % 5000 == 0: + print(f" Generated {i:,} merchants...") + +print(f"✓ Generated {NUM_MERCHANTS:,} merchants") + +print("[4/10] Generating cards...") +cards = [] +with open(DATA_DIR / "cards.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["card_id", "account_id", "card_number", "card_type", "expiry_date", + "cvv", "status", "daily_limit", "issued_date"]) + + for i in range(1, NUM_CARDS + 1): + account_id = random.choice(accounts) + + card = [ + i, # card_id + account_id, + generate_card_number(), + random.choice(["DEBIT", "CREDIT", "PREPAID"]), + random_date(datetime(2025, 1, 1), datetime(2030, 12, 31)).strftime("%Y-%m-%d"), + f"{random.randint(100, 999)}", + random.choice(["ACTIVE", "ACTIVE", "ACTIVE", "BLOCKED", "EXPIRED"]), + round(random.choice([500, 1000, 2000, 5000, 10000]), 2), + random_date(datetime(2020, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d") + ] + writer.writerow(card) + cards.append(i) + + if i % 20000 == 0: + print(f" Generated {i:,} cards...") + +print(f"✓ Generated {NUM_CARDS:,} cards") + +print("[5/10] Generating transactions (this will take a few minutes)...") +print(f" Generating {NUM_TRANSACTIONS:,} transactions in batches...") + diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 9112553..be6bf63 100644 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -49,9 +49,61 @@ command_exists() { command -v "$1" >/dev/null 2>&1 } +# Install Python 3 +install_python() { + echo -e "\n${YELLOW}[1/5] Checking Python 3...${NC}" + + if command_exists python3; then + PYTHON_VERSION=$(python3 --version | awk '{print $2}') + echo -e "${GREEN}✓ Python 3 is already installed (version $PYTHON_VERSION)${NC}" + return 0 + fi + + echo -e "${YELLOW}Installing Python 3...${NC}" + + case $OS in + ubuntu|debian) + sudo apt-get update + sudo apt-get install -y python3 python3-pip + ;; + centos|rhel|fedora) + sudo yum install -y python3 python3-pip + ;; + arch) + sudo pacman -S --noconfirm python python-pip + ;; + macos) + if command_exists brew; then + brew install python3 + else + echo -e "${RED}✗ Homebrew not found. Please install Homebrew first: https://brew.sh${NC}" + exit 1 + fi + ;; + windows) + echo -e "${YELLOW}On Windows, please install Python 3 manually:${NC}" + echo -e " 1. Download from: https://www.python.org/downloads/" + echo -e " 2. Or use WSL2 with Ubuntu" + echo -e " 3. Make sure to check 'Add Python to PATH' during installation" + exit 1 + ;; + *) + echo -e "${RED}✗ Unsupported OS. Please install Python 3 manually.${NC}" + exit 1 + ;; + esac + + if command_exists python3; then + echo -e "${GREEN}✓ Python 3 installed successfully${NC}" + else + echo -e "${RED}✗ Failed to install Python 3${NC}" + exit 1 + fi +} + # Install PostgreSQL client install_psql() { - echo -e "\n${YELLOW}[1/4] Checking PostgreSQL client (psql)...${NC}" + echo -e "\n${YELLOW}[2/5] Checking PostgreSQL client (psql)...${NC}" if command_exists psql; then PSQL_VERSION=$(psql --version | awk '{print $3}') @@ -103,7 +155,7 @@ install_psql() { # Install Docker install_docker() { - echo -e "\n${YELLOW}[2/4] Checking Docker...${NC}" + echo -e "\n${YELLOW}[3/5] Checking Docker...${NC}" if command_exists docker; then DOCKER_VERSION=$(docker --version | awk '{print $3}' | sed 's/,//') @@ -163,7 +215,7 @@ install_docker() { # Install Docker Compose install_docker_compose() { - echo -e "\n${YELLOW}[3/4] Checking Docker Compose...${NC}" + echo -e "\n${YELLOW}[4/5] Checking Docker Compose...${NC}" # Check for docker-compose (standalone) or docker compose (plugin) if command_exists docker-compose || docker compose version >/dev/null 2>&1; then @@ -209,7 +261,7 @@ install_docker_compose() { # Install utility packages install_utilities() { - echo -e "\n${YELLOW}[4/4] Checking utility packages...${NC}" + echo -e "\n${YELLOW}[5/5] Checking utility packages...${NC}" local missing_packages=() @@ -265,6 +317,7 @@ main() { echo -e "\n${BLUE}Starting dependency installation...${NC}" echo -e "${YELLOW}This script will install:${NC}" + echo -e " - Python 3 (for CSV data generation)" echo -e " - PostgreSQL client (psql)" echo -e " - Docker" echo -e " - Docker Compose" @@ -274,7 +327,8 @@ main() { echo -e "${YELLOW}Press Ctrl+C to cancel, or wait 5 seconds to continue...${NC}" sleep 5 echo "" - + + install_python install_psql install_docker install_docker_compose