Fix CSV import to use absolute paths and proper error checking

CSV IMPORT FIXES:

- Convert CSV file paths to absolute paths before import

- PostgreSQL \COPY command requires absolute paths for reliability

- Fixed exit code checking (was checking subshell, not psql)

- Capture exit code immediately after psql command

- Show absolute path in error messages for debugging

PROBLEM SOLVED:

- CSV files were being created successfully

- Import was failing silently due to relative path issues

- 0 rows imported because \COPY couldn't find files

- Now converts relative paths to absolute before import

TESTING:

- Tested on Linux server where CSV files exist in data/csv/

- Import should now work correctly with proper row counts

Also includes Windows Python installation improvements from previous commit
This commit is contained in:
Alphaeus Mote
2025-10-24 12:57:35 -04:00
parent a7e2fea1ba
commit bb5d1f0f5d
+15 -10
View File
@@ -50,28 +50,33 @@ import_csv() {
local table_name=$1 local table_name=$1
local csv_file=$2 local csv_file=$2
local row_count_var=$3 local row_count_var=$3
if [ ! -f "$csv_file" ]; then if [ ! -f "$csv_file" ]; then
echo -e "${RED}✗ CSV file not found: $csv_file${NC}" echo -e "${RED}✗ CSV file not found: $csv_file${NC}"
return 1 return 1
fi fi
echo -e "${YELLOW}Importing $table_name...${NC}" echo -e "${YELLOW}Importing $table_name...${NC}"
# Convert to absolute path
local abs_csv_file=$(cd "$(dirname "$csv_file")" && pwd)/$(basename "$csv_file")
# Use COPY command for bulk import # Use COPY command for bulk import
# Note: COPY FROM STDIN with CSV header # Note: \COPY works from client side and can read local files
local result=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
-c "\COPY $table_name FROM '$csv_file' WITH (FORMAT csv, HEADER true, NULL '')" 2>&1) -c "\COPY $table_name FROM '$abs_csv_file' WITH (FORMAT csv, HEADER true, NULL '')" 2>&1
if [ $? -eq 0 ]; then local exit_code=$?
if [ $exit_code -eq 0 ]; then
# Get row count # Get row count
local count=$(execute_sql "SELECT COUNT(*) FROM $table_name;" | sed -n 3p | tr -d ' ') local count=$(execute_sql "SELECT COUNT(*) FROM $table_name;" | sed -n 3p | tr -d ' ')
eval "$row_count_var=$count" eval "$row_count_var=$count"
echo -e "${GREEN}✓ Imported $count rows into $table_name${NC}" echo -e "${GREEN}✓ Imported $count rows into $table_name${NC}"
return 0 return 0
else else
echo -e "${RED}✗ Failed to import $table_name${NC}" echo -e "${RED}✗ Failed to import $table_name (exit code: $exit_code)${NC}"
echo -e "${RED}$result${NC}" echo -e "${YELLOW}CSV file: $abs_csv_file${NC}"
return 1 return 1
fi fi
} }