mirror of
https://github.com/freedbygrace/SQL.git
synced 2026-07-26 11:28:16 +00:00
Add complete financial fraud detection database with Docker, schema, data generation, and SQL exercises
- Docker setup with PostgreSQL 16 and DB-UI web interface - Comprehensive 20+ table schema with fraud detection patterns - Idempotent shell scripts for data generation (no Python dependency) - Realistic geographic data (100 US cities, 210 world cities) - 5M+ transactions with embedded fraud patterns (velocity, geographic, structuring, etc.) - Progressive SQL exercises from beginner to advanced fraud detection - Complete documentation and quick start guide - Setup and verification scripts
This commit is contained in:
Alphaeus Mote
@@ -0,0 +1,114 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Database Setup Script - IDEMPOTENT
|
||||
# ============================================================================
|
||||
# This script sets up the complete fraud detection database
|
||||
# It can be run multiple times safely - it will recreate everything
|
||||
# ============================================================================
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration from environment or defaults
|
||||
DB_HOST="${POSTGRES_HOST:-localhost}"
|
||||
DB_PORT="${POSTGRES_PORT:-5432}"
|
||||
DB_NAME="${POSTGRES_DB:-fraud_detection}"
|
||||
DB_USER="${POSTGRES_USER:-fraud_analyst}"
|
||||
DB_PASSWORD="${POSTGRES_PASSWORD:-SecurePass123!}"
|
||||
|
||||
# Script directory
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo -e "${BLUE}Financial Fraud Detection Database - Setup Script${NC}"
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo -e "Database: ${GREEN}$DB_NAME${NC}"
|
||||
echo -e "Host: ${GREEN}$DB_HOST:$DB_PORT${NC}"
|
||||
echo -e "User: ${GREEN}$DB_USER${NC}"
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Function to execute SQL command
|
||||
execute_sql() {
|
||||
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c "$1" 2>&1
|
||||
}
|
||||
|
||||
# Function to execute SQL file
|
||||
execute_sql_file() {
|
||||
local file=$1
|
||||
local description=$2
|
||||
echo -e "${YELLOW}Executing: $description${NC}"
|
||||
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -f "$file" 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ Success: $description${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Failed: $description${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Wait for PostgreSQL to be ready
|
||||
echo -e "${YELLOW}Waiting for PostgreSQL to be ready...${NC}"
|
||||
max_attempts=30
|
||||
attempt=0
|
||||
until PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c '\q' 2>/dev/null; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ $attempt -ge $max_attempts ]; then
|
||||
echo -e "${RED}✗ PostgreSQL is not available after $max_attempts attempts${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${YELLOW}Waiting for PostgreSQL... (attempt $attempt/$max_attempts)${NC}"
|
||||
sleep 2
|
||||
done
|
||||
echo -e "${GREEN}✓ PostgreSQL is ready${NC}"
|
||||
echo ""
|
||||
|
||||
# Step 1: Create schema (drops and recreates all tables)
|
||||
echo -e "${BLUE}[Step 1/3] Creating database schema...${NC}"
|
||||
execute_sql_file "$PROJECT_ROOT/schema/01-create-tables.sql" "Creating tables, indexes, and constraints"
|
||||
|
||||
# Step 2: Load seed data (reference tables)
|
||||
echo -e "${BLUE}[Step 2/3] Loading reference data...${NC}"
|
||||
execute_sql_file "$PROJECT_ROOT/schema/02-seed-data.sql" "Loading countries, merchant categories, transaction types, and fraud types"
|
||||
|
||||
# Step 3: Verify setup
|
||||
echo -e "${BLUE}[Step 3/3] Verifying database setup...${NC}"
|
||||
echo -e "${YELLOW}Checking table counts...${NC}"
|
||||
|
||||
# Get table counts
|
||||
table_count=$(execute_sql "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE';" | grep -E '^\s*[0-9]+' | tr -d ' ')
|
||||
echo -e "Tables created: ${GREEN}$table_count${NC}"
|
||||
|
||||
# Get reference data counts
|
||||
country_count=$(execute_sql "SELECT COUNT(*) FROM countries;" | grep -E '^\s*[0-9]+' | tr -d ' ')
|
||||
echo -e "Countries: ${GREEN}$country_count${NC}"
|
||||
|
||||
category_count=$(execute_sql "SELECT COUNT(*) FROM merchant_categories;" | grep -E '^\s*[0-9]+' | tr -d ' ')
|
||||
echo -e "Merchant categories: ${GREEN}$category_count${NC}"
|
||||
|
||||
transaction_type_count=$(execute_sql "SELECT COUNT(*) FROM transaction_types;" | grep -E '^\s*[0-9]+' | tr -d ' ')
|
||||
echo -e "Transaction types: ${GREEN}$transaction_type_count${NC}"
|
||||
|
||||
fraud_type_count=$(execute_sql "SELECT COUNT(*) FROM fraud_types;" | grep -E '^\s*[0-9]+' | tr -d ' ')
|
||||
echo -e "Fraud types: ${GREEN}$fraud_type_count${NC}"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}============================================================================${NC}"
|
||||
echo -e "${GREEN}✓ Database setup completed successfully!${NC}"
|
||||
echo -e "${GREEN}============================================================================${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Next steps:${NC}"
|
||||
echo -e "1. Generate test data: ${BLUE}./scripts/generate-data.sh${NC}"
|
||||
echo -e "2. Access DB-UI at: ${BLUE}http://localhost:3000${NC}"
|
||||
echo -e "3. Start learning SQL with exercises in: ${BLUE}./exercises/${NC}"
|
||||
echo ""
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/bin/bash
|
||||
|
||||
# ============================================================================
|
||||
# Setup Verification Script
|
||||
# ============================================================================
|
||||
# Verifies that the fraud detection database is properly set up
|
||||
# ============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
DB_HOST="${POSTGRES_HOST:-localhost}"
|
||||
DB_PORT="${POSTGRES_PORT:-5432}"
|
||||
DB_NAME="${POSTGRES_DB:-fraud_detection}"
|
||||
DB_USER="${POSTGRES_USER:-fraud_analyst}"
|
||||
DB_PASSWORD="${POSTGRES_PASSWORD:-SecurePass123!}"
|
||||
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo -e "${BLUE}Financial Fraud Detection Database - Verification${NC}"
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Function to execute SQL and get result
|
||||
execute_sql() {
|
||||
PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -t -c "$1" 2>/dev/null | xargs
|
||||
}
|
||||
|
||||
# Check 1: Docker containers
|
||||
echo -e "${YELLOW}[1/10] Checking Docker containers...${NC}"
|
||||
if docker ps | grep -q "fraud_detection_db"; then
|
||||
echo -e "${GREEN}✓ PostgreSQL container is running${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ PostgreSQL container is not running${NC}"
|
||||
echo -e "${YELLOW}Run: docker-compose up -d${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if docker ps | grep -q "fraud_detection_ui"; then
|
||||
echo -e "${GREEN}✓ DB-UI container is running${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ DB-UI container is not running${NC}"
|
||||
echo -e "${YELLOW}Run: docker-compose up -d${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 2: Database connectivity
|
||||
echo -e "${YELLOW}[2/10] Checking database connectivity...${NC}"
|
||||
if PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c '\q' 2>/dev/null; then
|
||||
echo -e "${GREEN}✓ Can connect to database${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Cannot connect to database${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 3: Tables exist
|
||||
echo -e "${YELLOW}[3/10] Checking database schema...${NC}"
|
||||
table_count=$(execute_sql "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE';")
|
||||
if [ "$table_count" -ge 20 ]; then
|
||||
echo -e "${GREEN}✓ Schema created ($table_count tables)${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Schema incomplete (only $table_count tables)${NC}"
|
||||
echo -e "${YELLOW}Run: ./scripts/setup-database.sh${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 4: Reference data
|
||||
echo -e "${YELLOW}[4/10] Checking reference data...${NC}"
|
||||
country_count=$(execute_sql "SELECT COUNT(*) FROM countries;")
|
||||
category_count=$(execute_sql "SELECT COUNT(*) FROM merchant_categories;")
|
||||
type_count=$(execute_sql "SELECT COUNT(*) FROM transaction_types;")
|
||||
fraud_type_count=$(execute_sql "SELECT COUNT(*) FROM fraud_types;")
|
||||
|
||||
if [ "$country_count" -ge 40 ]; then
|
||||
echo -e "${GREEN}✓ Countries loaded ($country_count)${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Countries not loaded${NC}"
|
||||
fi
|
||||
|
||||
if [ "$category_count" -ge 30 ]; then
|
||||
echo -e "${GREEN}✓ Merchant categories loaded ($category_count)${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Merchant categories not loaded${NC}"
|
||||
fi
|
||||
|
||||
if [ "$type_count" -ge 10 ]; then
|
||||
echo -e "${GREEN}✓ Transaction types loaded ($type_count)${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Transaction types not loaded${NC}"
|
||||
fi
|
||||
|
||||
if [ "$fraud_type_count" -ge 15 ]; then
|
||||
echo -e "${GREEN}✓ Fraud types loaded ($fraud_type_count)${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Fraud types not loaded${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 5: Customer data
|
||||
echo -e "${YELLOW}[5/10] Checking customer data...${NC}"
|
||||
customer_count=$(execute_sql "SELECT COUNT(*) FROM customers;")
|
||||
if [ "$customer_count" -ge 10000 ]; then
|
||||
echo -e "${GREEN}✓ Customers generated ($customer_count)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Limited customer data ($customer_count)${NC}"
|
||||
echo -e "${YELLOW}Run: ./data/generate_data.sh${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 6: Account data
|
||||
echo -e "${YELLOW}[6/10] Checking account data...${NC}"
|
||||
account_count=$(execute_sql "SELECT COUNT(*) FROM accounts;")
|
||||
if [ "$account_count" -ge 10000 ]; then
|
||||
echo -e "${GREEN}✓ Accounts generated ($account_count)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Limited account data ($account_count)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 7: Transaction data
|
||||
echo -e "${YELLOW}[7/10] Checking transaction data...${NC}"
|
||||
transaction_count=$(execute_sql "SELECT COUNT(*) FROM transactions;")
|
||||
if [ "$transaction_count" -ge 100000 ]; then
|
||||
echo -e "${GREEN}✓ Transactions generated ($transaction_count)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Limited transaction data ($transaction_count)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 8: Fraud data
|
||||
echo -e "${YELLOW}[8/10] Checking fraud detection data...${NC}"
|
||||
alert_count=$(execute_sql "SELECT COUNT(*) FROM alerts;")
|
||||
case_count=$(execute_sql "SELECT COUNT(*) FROM fraud_cases;")
|
||||
|
||||
if [ "$alert_count" -ge 100 ]; then
|
||||
echo -e "${GREEN}✓ Alerts generated ($alert_count)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Limited alert data ($alert_count)${NC}"
|
||||
fi
|
||||
|
||||
if [ "$case_count" -ge 10 ]; then
|
||||
echo -e "${GREEN}✓ Fraud cases generated ($case_count)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Limited fraud case data ($case_count)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 9: Indexes
|
||||
echo -e "${YELLOW}[9/10] Checking database indexes...${NC}"
|
||||
index_count=$(execute_sql "SELECT COUNT(*) FROM pg_indexes WHERE schemaname = 'public';")
|
||||
if [ "$index_count" -ge 20 ]; then
|
||||
echo -e "${GREEN}✓ Indexes created ($index_count)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Limited indexes ($index_count)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check 10: DB-UI accessibility
|
||||
echo -e "${YELLOW}[10/10] Checking DB-UI web interface...${NC}"
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://localhost:3000 | grep -q "200\|302"; then
|
||||
echo -e "${GREEN}✓ DB-UI accessible at http://localhost:3000${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ DB-UI may not be ready yet (still starting up)${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Summary
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo -e "${GREEN}Verification Summary${NC}"
|
||||
echo -e "${BLUE}============================================================================${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Database Statistics:${NC}"
|
||||
echo -e " Customers: ${GREEN}$customer_count${NC}"
|
||||
echo -e " Accounts: ${GREEN}$account_count${NC}"
|
||||
echo -e " Transactions: ${GREEN}$transaction_count${NC}"
|
||||
echo -e " Alerts: ${GREEN}$alert_count${NC}"
|
||||
echo -e " Fraud Cases: ${GREEN}$case_count${NC}"
|
||||
echo ""
|
||||
|
||||
if [ "$transaction_count" -ge 100000 ]; then
|
||||
echo -e "${GREEN}✓ Database is ready for SQL learning!${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Next steps:${NC}"
|
||||
echo -e "1. Access DB-UI: ${BLUE}http://localhost:3000${NC}"
|
||||
echo -e "2. Start learning: ${BLUE}exercises/01-basic-queries/README.md${NC}"
|
||||
echo -e "3. Quick start guide: ${BLUE}docs/QUICKSTART.md${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Database has minimal data${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}To generate full dataset:${NC}"
|
||||
echo -e " ${BLUE}./data/generate_data.sh${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}This will generate:${NC}"
|
||||
echo -e " - 100,000 customers"
|
||||
echo -e " - 150,000 accounts"
|
||||
echo -e " - 5,000,000 transactions"
|
||||
echo -e " - 50,000+ alerts"
|
||||
echo -e " - 5,000+ fraud cases"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user