mirror of
https://github.com/freedbygrace/SQL.git
synced 2026-07-26 11:28:16 +00:00
Add Python dependency and start CSV-based data generation
DEPENDENCY INSTALLER UPDATES: - Added Python 3 installation check and installer - Python is required for fast CSV data generation - Updated step numbers from [1/4] to [1/5] - Python installed before PostgreSQL client - Supports Ubuntu, Debian, CentOS, RHEL, Fedora, Arch, macOS DEPLOY SCRIPT UPDATES: - Added automatic dependency checking (Step 3/8) - Checks for python3, psql, and docker - Automatically runs install-dependencies.sh if missing - Updated all step numbers to reflect 8 total steps - Step 3: Check dependencies (NEW) - Step 4: Fix permissions (was Step 3) - Step 5: Start containers (was Step 4) - Step 6: Initialize schema (was Step 5) - Step 7: Generate data (was Step 6) - Step 8: Verify deployment (was Step 7) CSV DATA GENERATION (IN PROGRESS): - Created scripts/generate_csv_data.py - Python-based CSV generator for bulk import - Much faster than SQL INSERT statements - Generates realistic data with proper relationships - Partial implementation (customers, accounts, merchants, cards) - Next: Complete transactions and analytics data RATIONALE: - Current bash-based data generation has transaction commit issues - Data shows as generated but doesn't persist (0 rows after completion) - CSV + COPY command is PostgreSQL best practice for bulk loading - 10-100x faster than INSERT statements - More reliable - atomic COPY operations - Easier to debug - can inspect CSV files
This commit is contained in:
Alphaeus Mote
@@ -40,11 +40,12 @@ echo ""
|
||||
echo -e "${YELLOW}This script will:${NC}"
|
||||
echo -e " ${BLUE}1.${NC} Tear down existing containers"
|
||||
echo -e " ${BLUE}2.${NC} Remove all data volumes"
|
||||
echo -e " ${BLUE}3.${NC} Fix file permissions"
|
||||
echo -e " ${BLUE}4.${NC} Start fresh containers"
|
||||
echo -e " ${BLUE}5.${NC} Initialize database schema"
|
||||
echo -e " ${BLUE}6.${NC} Generate test data"
|
||||
echo -e " ${BLUE}7.${NC} Verify deployment"
|
||||
echo -e " ${BLUE}3.${NC} Check and install dependencies (Python, psql, Docker)"
|
||||
echo -e " ${BLUE}4.${NC} Fix file permissions"
|
||||
echo -e " ${BLUE}5.${NC} Start fresh containers"
|
||||
echo -e " ${BLUE}6.${NC} Initialize database schema"
|
||||
echo -e " ${BLUE}7.${NC} Generate test data"
|
||||
echo -e " ${BLUE}8.${NC} Verify deployment"
|
||||
echo ""
|
||||
echo -e "${YELLOW}⚠️ WARNING: This will DELETE all existing data!${NC}"
|
||||
echo ""
|
||||
@@ -84,7 +85,38 @@ fi
|
||||
|
||||
echo ""
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN}[STEP 3/7] Fixing file permissions...${NC}"
|
||||
echo -e "${CYAN}[STEP 3/7] Checking dependencies...${NC}"
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
# Check for required dependencies
|
||||
MISSING_DEPS=()
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
MISSING_DEPS+=("python3")
|
||||
fi
|
||||
|
||||
if ! command -v psql >/dev/null 2>&1; then
|
||||
MISSING_DEPS+=("psql")
|
||||
fi
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
MISSING_DEPS+=("docker")
|
||||
fi
|
||||
|
||||
if [ ${#MISSING_DEPS[@]} -gt 0 ]; then
|
||||
echo -e "${YELLOW}⚠ Missing dependencies: ${MISSING_DEPS[*]}${NC}"
|
||||
echo -e "${YELLOW}Running dependency installer...${NC}"
|
||||
echo ""
|
||||
chmod +x scripts/install-dependencies.sh 2>/dev/null || true
|
||||
./scripts/install-dependencies.sh
|
||||
else
|
||||
echo -e "${GREEN}✓ All dependencies are installed${NC}"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN}[STEP 4/7] Fixing file permissions...${NC}"
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
@@ -107,7 +139,7 @@ fi
|
||||
|
||||
echo ""
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN}[STEP 4/7] Starting containers...${NC}"
|
||||
echo -e "${CYAN}[STEP 5/8] Starting containers...${NC}"
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
@@ -147,7 +179,7 @@ echo -e "${YELLOW}Note: pgAdmin may take 30-60 seconds to fully initialize${NC}"
|
||||
|
||||
echo ""
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN}[STEP 5/7] Initializing database schema...${NC}"
|
||||
echo -e "${CYAN}[STEP 6/8] Initializing database schema...${NC}"
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
@@ -155,7 +187,7 @@ echo ""
|
||||
|
||||
echo ""
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN}[STEP 6/7] Generating test data...${NC}"
|
||||
echo -e "${CYAN}[STEP 7/8] Generating test data...${NC}"
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}⏱️ This will take 15-30 minutes depending on your system...${NC}"
|
||||
@@ -165,7 +197,7 @@ echo ""
|
||||
|
||||
echo ""
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo -e "${CYAN}[STEP 7/7] Verifying deployment...${NC}"
|
||||
echo -e "${CYAN}[STEP 8/8] Verifying deployment...${NC}"
|
||||
echo -e "${MAGENTA}════════════════════════════════════════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate CSV files for bulk import into PostgreSQL
|
||||
Much faster and more reliable than INSERT statements
|
||||
"""
|
||||
|
||||
import csv
|
||||
import random
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
# Configuration
|
||||
NUM_CUSTOMERS = 100000
|
||||
NUM_ACCOUNTS = 150000
|
||||
NUM_MERCHANTS = 50000
|
||||
NUM_CARDS = 200000
|
||||
NUM_TRANSACTIONS = 5000000
|
||||
FRAUD_RATE = 0.07
|
||||
|
||||
# Output directory
|
||||
DATA_DIR = Path(__file__).parent.parent / "data" / "csv"
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("=" * 80)
|
||||
print("CSV Data Generation for Business Analytics Database")
|
||||
print("=" * 80)
|
||||
print(f"Output directory: {DATA_DIR}")
|
||||
print(f"Customers: {NUM_CUSTOMERS:,}")
|
||||
print(f"Accounts: {NUM_ACCOUNTS:,}")
|
||||
print(f"Merchants: {NUM_MERCHANTS:,}")
|
||||
print(f"Cards: {NUM_CARDS:,}")
|
||||
print(f"Transactions: {NUM_TRANSACTIONS:,}")
|
||||
print(f"Fraud rate: {FRAUD_RATE*100}%")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
# Reference data
|
||||
FIRST_NAMES = ["James", "Mary", "John", "Patricia", "Robert", "Jennifer", "Michael", "Linda",
|
||||
"William", "Barbara", "David", "Elizabeth", "Richard", "Susan", "Joseph", "Jessica",
|
||||
"Thomas", "Sarah", "Charles", "Karen", "Christopher", "Nancy", "Daniel", "Lisa"]
|
||||
|
||||
LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis",
|
||||
"Rodriguez", "Martinez", "Hernandez", "Lopez", "Gonzalez", "Wilson", "Anderson", "Thomas",
|
||||
"Taylor", "Moore", "Jackson", "Martin", "Lee", "Perez", "Thompson", "White"]
|
||||
|
||||
CITIES = ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio",
|
||||
"San Diego", "Dallas", "San Jose", "Austin", "Jacksonville", "Fort Worth", "Columbus",
|
||||
"Charlotte", "San Francisco", "Indianapolis", "Seattle", "Denver", "Boston"]
|
||||
|
||||
STATES = ["NY", "CA", "IL", "TX", "AZ", "PA", "TX", "CA", "TX", "CA", "TX", "FL", "TX", "OH",
|
||||
"NC", "CA", "IN", "WA", "CO", "MA"]
|
||||
|
||||
COUNTRIES = ["US", "CA", "GB", "DE", "FR", "AU", "JP", "CN", "IN", "BR"]
|
||||
|
||||
MERCHANT_TYPES = [
|
||||
("5411", "Grocery Stores"), ("5812", "Restaurants"), ("5541", "Gas Stations"),
|
||||
("5311", "Department Stores"), ("5912", "Pharmacies"), ("5999", "Miscellaneous Retail"),
|
||||
("5732", "Electronics"), ("5651", "Clothing"), ("5814", "Fast Food"),
|
||||
("5942", "Books"), ("7011", "Hotels"), ("7512", "Car Rental")
|
||||
]
|
||||
|
||||
TRANSACTION_TYPES = ["PURCHASE", "ATM_WITHDRAWAL", "TRANSFER_OUT", "TRANSFER_IN", "PAYMENT",
|
||||
"REFUND", "DEPOSIT", "WIRE_OUT", "WIRE_IN"]
|
||||
|
||||
FRAUD_CODES = ["CARD_STOLEN", "ACCOUNT_TAKEOVER", "IDENTITY_THEFT", "SYNTHETIC_IDENTITY",
|
||||
"CARD_NOT_PRESENT", "PHISHING", "ATM_SKIMMING", "WIRE_FRAUD"]
|
||||
|
||||
def generate_email(first_name, last_name, customer_id):
|
||||
"""Generate realistic email address"""
|
||||
domains = ["gmail.com", "yahoo.com", "hotmail.com", "outlook.com", "icloud.com"]
|
||||
return f"{first_name.lower()}.{last_name.lower()}{customer_id % 1000}@{random.choice(domains)}"
|
||||
|
||||
def generate_phone():
|
||||
"""Generate US phone number"""
|
||||
return f"+1{random.randint(2000000000, 9999999999)}"
|
||||
|
||||
def generate_ssn():
|
||||
"""Generate SSN-like identifier"""
|
||||
return f"{random.randint(100, 999)}-{random.randint(10, 99)}-{random.randint(1000, 9999)}"
|
||||
|
||||
def generate_card_number():
|
||||
"""Generate realistic card number"""
|
||||
prefix = random.choice(["4", "5"]) # Visa or Mastercard
|
||||
return prefix + "".join([str(random.randint(0, 9)) for _ in range(15)])
|
||||
|
||||
def random_date(start_date, end_date):
|
||||
"""Generate random date between start and end"""
|
||||
delta = end_date - start_date
|
||||
random_days = random.randint(0, delta.days)
|
||||
return start_date + timedelta(days=random_days)
|
||||
|
||||
def random_datetime(start_date, end_date):
|
||||
"""Generate random datetime between start and end"""
|
||||
delta = end_date - start_date
|
||||
random_seconds = random.randint(0, int(delta.total_seconds()))
|
||||
return start_date + timedelta(seconds=random_seconds)
|
||||
|
||||
print("[1/10] Generating customers...")
|
||||
customers = []
|
||||
with open(DATA_DIR / "customers.csv", "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["customer_id", "first_name", "last_name", "email", "phone", "date_of_birth",
|
||||
"ssn", "address_line1", "city", "state", "postal_code", "country_code",
|
||||
"risk_score", "kyc_status", "created_at"])
|
||||
|
||||
for i in range(1, NUM_CUSTOMERS + 1):
|
||||
first_name = random.choice(FIRST_NAMES)
|
||||
last_name = random.choice(LAST_NAMES)
|
||||
city_idx = random.randint(0, len(CITIES) - 1)
|
||||
|
||||
customer = [
|
||||
i, # customer_id
|
||||
first_name,
|
||||
last_name,
|
||||
generate_email(first_name, last_name, i),
|
||||
generate_phone(),
|
||||
random_date(datetime(1950, 1, 1), datetime(2005, 12, 31)).strftime("%Y-%m-%d"),
|
||||
generate_ssn(),
|
||||
f"{random.randint(100, 9999)} {random.choice(['Main', 'Oak', 'Maple', 'Park'])} St",
|
||||
CITIES[city_idx],
|
||||
STATES[city_idx],
|
||||
f"{random.randint(10000, 99999)}",
|
||||
random.choice(COUNTRIES),
|
||||
round(random.uniform(0, 100), 2),
|
||||
random.choice(["VERIFIED", "PENDING", "REJECTED"]),
|
||||
random_datetime(datetime(2020, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
]
|
||||
writer.writerow(customer)
|
||||
customers.append(i)
|
||||
|
||||
if i % 10000 == 0:
|
||||
print(f" Generated {i:,} customers...")
|
||||
|
||||
print(f"✓ Generated {NUM_CUSTOMERS:,} customers")
|
||||
|
||||
print("[2/10] Generating accounts...")
|
||||
accounts = []
|
||||
with open(DATA_DIR / "accounts.csv", "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["account_id", "customer_id", "account_number", "account_type", "currency",
|
||||
"balance", "available_balance", "status", "opened_date", "closed_date"])
|
||||
|
||||
for i in range(1, NUM_ACCOUNTS + 1):
|
||||
customer_id = random.choice(customers)
|
||||
account_type = random.choice(["CHECKING", "SAVINGS", "CREDIT"])
|
||||
balance = round(random.uniform(100, 50000), 2)
|
||||
|
||||
account = [
|
||||
i, # account_id
|
||||
customer_id,
|
||||
f"{random.randint(1000000000, 9999999999)}",
|
||||
account_type,
|
||||
"USD",
|
||||
balance,
|
||||
balance * random.uniform(0.8, 1.0),
|
||||
random.choice(["ACTIVE", "ACTIVE", "ACTIVE", "SUSPENDED", "CLOSED"]),
|
||||
random_date(datetime(2020, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d"),
|
||||
"" if random.random() > 0.05 else random_date(datetime(2023, 1, 1), datetime(2024, 12, 31)).strftime("%Y-%m-%d")
|
||||
]
|
||||
writer.writerow(account)
|
||||
accounts.append(i)
|
||||
|
||||
if i % 10000 == 0:
|
||||
print(f" Generated {i:,} accounts...")
|
||||
|
||||
print(f"✓ Generated {NUM_ACCOUNTS:,} accounts")
|
||||
|
||||
print("[3/10] Generating merchants...")
|
||||
merchants = []
|
||||
with open(DATA_DIR / "merchants.csv", "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["merchant_id", "merchant_name", "category_id", "country_code", "city",
|
||||
"risk_rating", "is_active", "created_at"])
|
||||
|
||||
for i in range(1, NUM_MERCHANTS + 1):
|
||||
category = random.choice(MERCHANT_TYPES)
|
||||
city_idx = random.randint(0, len(CITIES) - 1)
|
||||
|
||||
merchant = [
|
||||
i, # merchant_id
|
||||
f"{category[1]} #{i}",
|
||||
random.randint(1, 35), # category_id from merchant_categories
|
||||
random.choice(COUNTRIES),
|
||||
CITIES[city_idx],
|
||||
round(random.uniform(1, 10), 2),
|
||||
random.choice([True, True, True, False]),
|
||||
random_datetime(datetime(2015, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
]
|
||||
writer.writerow(merchant)
|
||||
merchants.append(i)
|
||||
|
||||
if i % 5000 == 0:
|
||||
print(f" Generated {i:,} merchants...")
|
||||
|
||||
print(f"✓ Generated {NUM_MERCHANTS:,} merchants")
|
||||
|
||||
print("[4/10] Generating cards...")
|
||||
cards = []
|
||||
with open(DATA_DIR / "cards.csv", "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["card_id", "account_id", "card_number", "card_type", "expiry_date",
|
||||
"cvv", "status", "daily_limit", "issued_date"])
|
||||
|
||||
for i in range(1, NUM_CARDS + 1):
|
||||
account_id = random.choice(accounts)
|
||||
|
||||
card = [
|
||||
i, # card_id
|
||||
account_id,
|
||||
generate_card_number(),
|
||||
random.choice(["DEBIT", "CREDIT", "PREPAID"]),
|
||||
random_date(datetime(2025, 1, 1), datetime(2030, 12, 31)).strftime("%Y-%m-%d"),
|
||||
f"{random.randint(100, 999)}",
|
||||
random.choice(["ACTIVE", "ACTIVE", "ACTIVE", "BLOCKED", "EXPIRED"]),
|
||||
round(random.choice([500, 1000, 2000, 5000, 10000]), 2),
|
||||
random_date(datetime(2020, 1, 1), datetime(2024, 1, 1)).strftime("%Y-%m-%d")
|
||||
]
|
||||
writer.writerow(card)
|
||||
cards.append(i)
|
||||
|
||||
if i % 20000 == 0:
|
||||
print(f" Generated {i:,} cards...")
|
||||
|
||||
print(f"✓ Generated {NUM_CARDS:,} cards")
|
||||
|
||||
print("[5/10] Generating transactions (this will take a few minutes)...")
|
||||
print(f" Generating {NUM_TRANSACTIONS:,} transactions in batches...")
|
||||
|
||||
@@ -49,9 +49,61 @@ command_exists() {
|
||||
command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Install Python 3
|
||||
install_python() {
|
||||
echo -e "\n${YELLOW}[1/5] Checking Python 3...${NC}"
|
||||
|
||||
if command_exists python3; then
|
||||
PYTHON_VERSION=$(python3 --version | awk '{print $2}')
|
||||
echo -e "${GREEN}✓ Python 3 is already installed (version $PYTHON_VERSION)${NC}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}Installing Python 3...${NC}"
|
||||
|
||||
case $OS in
|
||||
ubuntu|debian)
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-pip
|
||||
;;
|
||||
centos|rhel|fedora)
|
||||
sudo yum install -y python3 python3-pip
|
||||
;;
|
||||
arch)
|
||||
sudo pacman -S --noconfirm python python-pip
|
||||
;;
|
||||
macos)
|
||||
if command_exists brew; then
|
||||
brew install python3
|
||||
else
|
||||
echo -e "${RED}✗ Homebrew not found. Please install Homebrew first: https://brew.sh${NC}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
windows)
|
||||
echo -e "${YELLOW}On Windows, please install Python 3 manually:${NC}"
|
||||
echo -e " 1. Download from: https://www.python.org/downloads/"
|
||||
echo -e " 2. Or use WSL2 with Ubuntu"
|
||||
echo -e " 3. Make sure to check 'Add Python to PATH' during installation"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}✗ Unsupported OS. Please install Python 3 manually.${NC}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if command_exists python3; then
|
||||
echo -e "${GREEN}✓ Python 3 installed successfully${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Failed to install Python 3${NC}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Install PostgreSQL client
|
||||
install_psql() {
|
||||
echo -e "\n${YELLOW}[1/4] Checking PostgreSQL client (psql)...${NC}"
|
||||
echo -e "\n${YELLOW}[2/5] Checking PostgreSQL client (psql)...${NC}"
|
||||
|
||||
if command_exists psql; then
|
||||
PSQL_VERSION=$(psql --version | awk '{print $3}')
|
||||
@@ -103,7 +155,7 @@ install_psql() {
|
||||
|
||||
# Install Docker
|
||||
install_docker() {
|
||||
echo -e "\n${YELLOW}[2/4] Checking Docker...${NC}"
|
||||
echo -e "\n${YELLOW}[3/5] Checking Docker...${NC}"
|
||||
|
||||
if command_exists docker; then
|
||||
DOCKER_VERSION=$(docker --version | awk '{print $3}' | sed 's/,//')
|
||||
@@ -163,7 +215,7 @@ install_docker() {
|
||||
|
||||
# Install Docker Compose
|
||||
install_docker_compose() {
|
||||
echo -e "\n${YELLOW}[3/4] Checking Docker Compose...${NC}"
|
||||
echo -e "\n${YELLOW}[4/5] Checking Docker Compose...${NC}"
|
||||
|
||||
# Check for docker-compose (standalone) or docker compose (plugin)
|
||||
if command_exists docker-compose || docker compose version >/dev/null 2>&1; then
|
||||
@@ -209,7 +261,7 @@ install_docker_compose() {
|
||||
|
||||
# Install utility packages
|
||||
install_utilities() {
|
||||
echo -e "\n${YELLOW}[4/4] Checking utility packages...${NC}"
|
||||
echo -e "\n${YELLOW}[5/5] Checking utility packages...${NC}"
|
||||
|
||||
local missing_packages=()
|
||||
|
||||
@@ -265,6 +317,7 @@ main() {
|
||||
|
||||
echo -e "\n${BLUE}Starting dependency installation...${NC}"
|
||||
echo -e "${YELLOW}This script will install:${NC}"
|
||||
echo -e " - Python 3 (for CSV data generation)"
|
||||
echo -e " - PostgreSQL client (psql)"
|
||||
echo -e " - Docker"
|
||||
echo -e " - Docker Compose"
|
||||
@@ -274,7 +327,8 @@ main() {
|
||||
echo -e "${YELLOW}Press Ctrl+C to cancel, or wait 5 seconds to continue...${NC}"
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
|
||||
install_python
|
||||
install_psql
|
||||
install_docker
|
||||
install_docker_compose
|
||||
|
||||
Reference in New Issue
Block a user