ci: add race detection, fix test race conditions

.github/workflows/hive.yml

@@ -57,6 +57,42 @@ jobs:
       - name: Check Diff Coverage
         run: cd hive && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
+  race_detection:
+    name: Hive Race Detection
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout Hive
+        uses: actions/checkout@v6
+        with:
+          path: hive
+
+      - name: Checkout Viperblock
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ github.ref_name }}
+          path: viperblock
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v6
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ github.ref_name }}
+          path: predastore
+
+      - name: Setup Go Workspace
+        run: go work init ./hive ./viperblock ./predastore
+
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          cache-dependency-path: "hive/go.sum"
+          go-version-file: "hive/go.mod"
+
+      - name: Run Tests with Race Detector
+        run: make -C hive test-race
+
   lint_and_security:
     name: Hive Lint and Security
     runs-on: ubuntu-24.04

Makefile

@@ -52,13 +52,13 @@ go_run:
 
 # Preflight — runs the same checks as GitHub Actions (format + lint + security + tests).
 # Use this before committing to catch CI failures locally.
-preflight: check-format check-modernize vet security-check test
+preflight: check-format check-modernize vet security-check test-cover diff-coverage test-race
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	LOG_IGNORE=1 go test -v -timeout 120s ./...
+	LOG_IGNORE=1 go test -v -timeout 120s ./hive/...
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
@@ -69,6 +69,11 @@ test-cover:
 	@echo "=== Total Coverage ==="
 	@go tool cover -func=$(COVERPROFILE) | tail -1
 
+# Run unit tests with race detector
+test-race:
+	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
+	LOG_IGNORE=1 go test -race -timeout 300s ./hive/...
+
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@scripts/diff-coverage.sh $(COVERPROFILE)
@@ -205,7 +210,7 @@ test-docker: test-docker-build
 	@echo -e "\n....Running Multi-Node E2E Tests...."
 	docker run --privileged --rm -v /dev/kvm:/dev/kvm --cap-add=NET_ADMIN $(E2E_IMAGE) ./tests/e2e/run-multinode-e2e.sh
 
-.PHONY: build build-ui go_build go_run preflight test test-cover diff-coverage bench run clean \
+.PHONY: build build-ui go_build go_run preflight test test-cover test-race diff-coverage bench run clean \
 	install-system install-go install-aws quickinstall \
 	format check-format modernize check-modernize vet security-check \
 	test-docker-build test-docker-single test-docker-multi test-docker

hive/utils/utils_test.go

@@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -540,33 +541,19 @@ func TestKillProcess(t *testing.T) {
 	// Create a test process
 	cmd := exec.Command("sleep", "60")
 	err := cmd.Start()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	pid := cmd.Process.Pid
 
-	// Wait for the process to actually terminate
-	// On macOS, the process cleanup can take longer
+	// Reap in background so KillProcess can detect termination
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		_ = cmd.Wait()
+	})
 
-	go func() {
-		time.Sleep(500 * time.Millisecond)
-		// Kill the process
-		err = KillProcess(pid)
-		assert.NoError(t, err)
-
-	}()
-
-	// Reap the process so it does not stay DEFUNCT
-	if err := cmd.Wait(); err != nil {
-		// often you expect a non nil error here since it was terminated by a signal
-		t.Logf("process exited after SIGTERM: %v", err)
-	}
-
-	// Wait for process to be reaped
-	//_ = cmd.Wait()
-
-	// On macOS, after Wait(), Signal(0) may not reliably detect dead process
-	// So we skip this check as it's platform-dependent behavior
-	// The fact that KillProcess returned no error is sufficient
+	err = KillProcess(pid)
+	assert.NoError(t, err)
+	wg.Wait()
 
 	// Test killing non-existent process
 	err = KillProcess(999999)
@@ -577,33 +564,26 @@ func TestStopProcess(t *testing.T) {
 	// Create and start a test process
 	cmd := exec.Command("sleep", "60")
 	err := cmd.Start()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// Write PID file
 	testName := "stopprocess-test"
-
 	err = WritePidFile(testName, cmd.Process.Pid)
+	require.NoError(t, err)
 
+	// Reap in background so StopProcess can detect termination
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		_ = cmd.Wait()
+	})
+
+	err = StopProcess(testName)
 	assert.NoError(t, err)
+	wg.Wait()
 
-	go func() {
-		time.Sleep(500 * time.Millisecond)
-
-		// Stop the process
-		err = StopProcess(testName)
-		assert.NoError(t, err)
-
-		// Verify PID file was removed
-		_, err = ReadPidFile(testName)
-		assert.Error(t, err, "PID file should be removed")
-
-	}()
-
-	// Reap the process so it does not stay DEFUNCT
-	if err := cmd.Wait(); err != nil {
-		// often you expect a non nil error here since it was terminated by a signal
-		t.Logf("process exited after SIGTERM: %v", err)
-	}
+	// Verify PID file was removed
+	_, err = ReadPidFile(testName)
+	assert.Error(t, err, "PID file should be removed")
 
 	// Test stopping non-existent process
 	err = StopProcess("nonexistent-process")
@@ -877,20 +857,19 @@ func TestStopProcessAt(t *testing.T) {
 	err := WritePidFileTo(dir, "stopat-test", cmd.Process.Pid)
 	require.NoError(t, err)
 
-	go func() {
-		time.Sleep(500 * time.Millisecond)
-		err := StopProcessAt(dir, "stopat-test")
-		assert.NoError(t, err)
+	// Reap in background so StopProcessAt can detect termination
+	var wg sync.WaitGroup
+	wg.Go(func() {
+		_ = cmd.Wait()
+	})
 
-		// Verify PID file was removed
-		_, err = ReadPidFileFrom(dir, "stopat-test")
-		assert.Error(t, err, "PID file should be removed")
-	}()
+	err = StopProcessAt(dir, "stopat-test")
+	assert.NoError(t, err)
+	wg.Wait()
 
-	// Reap the process
-	if err := cmd.Wait(); err != nil {
-		t.Logf("process exited after SIGTERM: %v", err)
-	}
+	// Verify PID file was removed
+	_, err = ReadPidFileFrom(dir, "stopat-test")
+	assert.Error(t, err, "PID file should be removed")
 }
 
 func TestStopProcessAt_StaleProcess(t *testing.T) {