gsadmin/flowfish
1
0
Fork 0
mirror of https://github.com/taylanbakircioglu/flowfish.git synced 2026-09-12 05:48:55 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
flowfish/deployment/docker-compose/docker-compose.local-test.yml
T
taylanbakircioglu 6e503368f7 feat: L7 (Application Level) observability — Service Map, Trace Explorer, APM, Beyla 
- Grafana Beyla DaemonSet for kernel-level HTTP/gRPC/DNS capture (passive,
  zero application changes, W3C traceparent header propagation)
- flowfish-l7-collector in-cluster bridge: OTLP receiver + buffered pull API
- L7 Ingestion Service: K8s service-proxy poll → enrich → RabbitMQ
- ClickHouse l7_http_flows / l7_grpc_flows / l7_dns_flows + APM RED MVs
- Neo4j L7Workload nodes + SAME_WORKLOAD cross-cluster bridges
- New pages: Service Map, Trace Explorer, APM Services List, APM Service Detail
- Analysis Wizard now supports L4 / L7 / Both modes with HTTP/gRPC/DNS picks
- Integration Hub gains L7 dependency summary + tree-summary integrations
- Multi-Cluster Management: dual-agent install (Inspector Gadget L4 + Beyla L7),
  runtime OpenShift detection so SCCs auto-install with kubectl too
- ServiceMap edge → Trace Explorer drill-down with virtual_trace_id correlation
- Docs: new L7 architecture diagram, README L7 sections, 3 new screenshots
2026-05-14 10:09:15 +03:00

454 lines
15 KiB
YAML
Raw Permalink Blame History

version: '3.8'
# Flowfish Local Test Environment (Docker Compose)
# Usage: docker-compose -f deployment/docker-compose/docker-compose.local-test.yml up -d
# Requires: schemas/ directory at workspace root
services:
# ============================================================================
# DATABASES
# ============================================================================
postgres:
image: postgres:15-alpine
container_name: flowfish-postgres
environment:
POSTGRES_USER: flowfish
POSTGRES_PASSWORD: flowfish123
POSTGRES_DB: flowfish
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
- ../../schemas/postgresql-schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U flowfish"]
interval: 5s
timeout: 5s
retries: 5
networks:
- flowfish-network
redis:
image: redis:7-alpine
container_name: flowfish-redis
command: redis-server --requirepass redis123
ports:
- "6379:6379"
healthcheck:
test: ["CMD", "redis-cli", "-a", "redis123", "ping"]
interval: 5s
timeout: 3s
retries: 5
networks:
- flowfish-network
neo4j:
image: neo4j:5.15-community
container_name: flowfish-neo4j
environment:
NEO4J_AUTH: neo4j/flowfish123
NEO4J_server_memory_heap_initial__size: 256m
NEO4J_server_memory_heap_max__size: 512m
ports:
- "7474:7474"
- "7687:7687"
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7474"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- flowfish-network
clickhouse:
image: clickhouse/clickhouse-server:23-alpine
container_name: flowfish-clickhouse
environment:
CLICKHOUSE_DB: flowfish
CLICKHOUSE_USER: flowfish
CLICKHOUSE_PASSWORD: flowfish123
ports:
- "8123:8123"
- "9000:9000"
volumes:
- clickhouse_data:/var/lib/clickhouse
- ../../schemas/clickhouse-events-schema.sql:/docker-entrypoint-initdb.d/01-events-schema.sql:ro
- ../../schemas/clickhouse-change-events.sql:/docker-entrypoint-initdb.d/02-change-events.sql:ro
- ../../schemas/clickhouse-l7-schema.sql:/docker-entrypoint-initdb.d/03-l7-schema.sql:ro
# Production migrations are also applied to local-test so the dev
# environment matches what the Azure DevOps job builds in OpenShift.
# Without these, local APM endpoints return 404 (no RED MVs) and
# the PID-temporal correlator silently drops events (no
# virtual_trace_id column on l7_http_flows / l7_grpc_flows). See
# deployment/kubernetes-manifests/03-migrations-job.yaml for the
# production-side counterparts. ClickHouse runs each /docker-
# entrypoint-initdb.d/*.sql alphabetically on first boot only.
- ../../schemas/migrations/clickhouse_004_add_analysis_id.sql:/docker-entrypoint-initdb.d/04-add-analysis-id.sql:ro
- ../../schemas/migrations/clickhouse_005_add_apm_red_mvs.sql:/docker-entrypoint-initdb.d/05-add-apm-red-mvs.sql:ro
- ../../schemas/migrations/clickhouse_006_add_apm_indexes.sql:/docker-entrypoint-initdb.d/06-add-apm-indexes.sql:ro
- ../../schemas/migrations/clickhouse_007_add_l7_pid.sql:/docker-entrypoint-initdb.d/07-add-l7-pid.sql:ro
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:8123/ping"]
interval: 10s
timeout: 5s
retries: 10
start_period: 60s
networks:
- flowfish-network
rabbitmq:
image: rabbitmq:3-management-alpine
container_name: flowfish-rabbitmq
environment:
RABBITMQ_DEFAULT_USER: flowfish
RABBITMQ_DEFAULT_PASS: flowfish123
ports:
- "5672:5672"
- "15672:15672"
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 5s
retries: 5
networks:
- flowfish-network
# ============================================================================
# BACKEND
# ============================================================================
backend:
image: taylanbakircioglu/flowfish:backend-latest
# To build locally instead of pulling from Docker Hub, uncomment below:
# build:
# context: ../..
# dockerfile: backend/Dockerfile
# volumes:
# - ../../backend:/app
container_name: flowfish-backend
environment:
# Database connections
DATABASE_URL: postgresql+asyncpg://flowfish:flowfish123@postgres:5432/flowfish
REDIS_URL: redis://:redis123@redis:6379/0
CLICKHOUSE_URL: http://flowfish:flowfish123@clickhouse:8123
CLICKHOUSE_USER: flowfish
CLICKHOUSE_PASSWORD: flowfish123
CLICKHOUSE_DATABASE: flowfish
NEO4J_BOLT_URI: bolt://neo4j:7687
NEO4J_HTTP_URI: http://neo4j:7474
NEO4J_USER: neo4j
NEO4J_PASSWORD: flowfish123
NEO4J_DATABASE: neo4j
RABBITMQ_HOST: rabbitmq
RABBITMQ_PORT: "5672"
RABBITMQ_USER: flowfish
RABBITMQ_PASSWORD: flowfish123
# Application settings
SECRET_KEY: local-dev-secret-key
JWT_EXPIRATION_HOURS: 24
CORS_ORIGINS: http://localhost:3000
LOG_LEVEL: DEBUG
# Build metadata
IMAGE_TAG: ${IMAGE_TAG:-latest}
# Cluster Manager gRPC endpoint
CLUSTER_MANAGER_GRPC: cluster-manager:5001
# Encryption key for cluster credentials (shared with cluster-manager)
FLOWFISH_ENCRYPTION_KEY: "Zo6CfvKB6y4IIPMAUJermGJ7UQm8wgjnvuehRGKY-sA="
ports:
- "8000:8000"
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
clickhouse:
condition: service_healthy
neo4j:
condition: service_healthy
rabbitmq:
condition: service_healthy
cluster-manager:
condition: service_started
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/v1/health')"]
interval: 10s
timeout: 10s
retries: 5
start_period: 30s
networks:
- flowfish-network
# ============================================================================
# CLUSTER MANAGER (gRPC gateway for Kubernetes API access)
# ============================================================================
cluster-manager:
image: taylanbakircioglu/flowfish:cluster-manager-latest
# To build locally instead of pulling from Docker Hub, uncomment below:
# build:
# context: ../..
# dockerfile: services/cluster-manager/Dockerfile
# volumes:
# - ../../services/cluster-manager/app:/app/app
container_name: flowfish-cluster-manager
environment:
SERVICE_NAME: cluster-manager
GRPC_PORT: "5001"
DATABASE_URL: postgresql://flowfish:flowfish123@postgres:5432/flowfish
REDIS_URL: redis://:redis123@redis:6379/0
LOG_LEVEL: DEBUG
FLOWFISH_ENCRYPTION_KEY: "Zo6CfvKB6y4IIPMAUJermGJ7UQm8wgjnvuehRGKY-sA="
ports:
- "5001:5001"
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
test: ["CMD", "python", "-c", "import grpc; ch=grpc.insecure_channel('localhost:5001'); ch.close()"]
interval: 30s
timeout: 10s
retries: 3
start_period: 15s
networks:
- flowfish-network
# ============================================================================
# FRONTEND
# ============================================================================
frontend:
image: taylanbakircioglu/flowfish:frontend-latest
# To build locally instead of pulling from Docker Hub, uncomment below:
# build:
# context: ../..
# dockerfile: frontend/Dockerfile.production
# volumes:
# - ../../frontend/src:/app/src
container_name: flowfish-frontend
user: "0:0"
environment:
REACT_APP_API_URL: http://localhost:8000
ports:
- "3000:3000"
depends_on:
backend:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3000"]
interval: 30s
timeout: 10s
retries: 3
networks:
- flowfish-network
rabbitmq-l7-init:
image: curlimages/curl:latest
depends_on:
rabbitmq:
condition: service_healthy
restart: "no"
entrypoint: ["/bin/sh", "-c"]
command:
- |
echo "Creating L7 exchanges..."
for EX in flowfish.l7.http_flows flowfish.l7.grpc_flows flowfish.l7.dns_flows flowfish.l7.dlx; do
curl -s -u flowfish:flowfish123 -X PUT \
"http://rabbitmq:15672/api/exchanges/%2F/$${EX}" \
-H 'content-type: application/json' \
-d '{"type":"topic","durable":true}'
done
echo "Creating L7 queues..."
for Q in flowfish.queue.l7_http_flows.graph flowfish.queue.l7_grpc_flows.graph flowfish.queue.l7_dns_flows.graph \
flowfish.queue.l7_http_flows.timeseries flowfish.queue.l7_grpc_flows.timeseries flowfish.queue.l7_dns_flows.timeseries; do
curl -s -u flowfish:flowfish123 -X PUT \
"http://rabbitmq:15672/api/queues/%2F/$${Q}" \
-H 'content-type: application/json' \
-d "{\"durable\":true,\"arguments\":{\"x-dead-letter-exchange\":\"flowfish.l7.dlx\",\"x-message-ttl\":86400000,\"x-max-length\":1000000}}"
done
echo "Creating L7 DLQ..."
curl -s -u flowfish:flowfish123 -X PUT \
"http://rabbitmq:15672/api/queues/%2F/flowfish.queue.l7.dlq" \
-H 'content-type: application/json' \
-d "{\"durable\":true,\"arguments\":{\"x-message-ttl\":604800000,\"x-max-length\":100000}}"
curl -s -u flowfish:flowfish123 -X POST \
"http://rabbitmq:15672/api/bindings/%2F/e/flowfish.l7.dlx/q/flowfish.queue.l7.dlq" \
-H 'content-type: application/json' \
-d '{"routing_key":"#"}'
echo "Binding L7 queues to exchanges..."
for PROTO in http grpc dns; do
for SUFFIX in graph timeseries; do
curl -s -u flowfish:flowfish123 -X POST \
"http://rabbitmq:15672/api/bindings/%2F/e/flowfish.l7.$${PROTO}_flows/q/flowfish.queue.l7_$${PROTO}_flows.$${SUFFIX}" \
-H 'content-type: application/json' \
-d '{"routing_key":"#"}'
done
done
echo "L7 RabbitMQ topology ready."
networks:
- flowfish-network
flowfish-l7-collector:
build:
context: ../../services/flowfish-l7-collector
environment:
- MOCK_MODE=true
- BUFFER_MAX_SIZE=100000
- LOG_LEVEL=INFO
ports:
- "4318:8080"
- "18080:8080"
networks:
- flowfish-network
l7-ingestion-service:
build:
context: ../../
dockerfile: services/l7-ingestion-service/Dockerfile
environment:
- GRPC_PORT=5006
- RABBITMQ_HOST=rabbitmq
- RABBITMQ_PORT=5672
- RABBITMQ_USER=flowfish
- RABBITMQ_PASSWORD=flowfish123
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_PASSWORD=redis123
- FLOWFISH_ENCRYPTION_KEY=Zo6CfvKB6y4IIPMAUJermGJ7UQm8wgjnvuehRGKY-sA=
- SAMPLING_RATE=1.0
- MAX_EVENTS_PER_SECOND=5000
- COLLECTOR_POLL_INTERVAL=2
depends_on:
- rabbitmq
- redis
ports:
- "5006:5006"
networks:
- flowfish-network
graph-writer:
image: taylanbakircioglu/flowfish:graph-writer-latest
environment:
- L7_ENABLED=true
- SERVICE_NAME=graph-writer
- LOG_LEVEL=INFO
- RABBITMQ_HOST=rabbitmq
- RABBITMQ_PORT=5672
- RABBITMQ_USER=flowfish
- RABBITMQ_PASSWORD=flowfish123
- RABBITMQ_VHOST=/
- NEO4J_BOLT_URI=bolt://neo4j:7687
- NEO4J_HTTP_URI=http://neo4j:7474
- NEO4J_USER=neo4j
- NEO4J_PASSWORD=flowfish123
- NEO4J_DATABASE=neo4j
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_PASSWORD=redis123
- REDIS_URL=redis://:redis123@redis:6379/0
depends_on:
- rabbitmq
- redis
- neo4j
networks:
- flowfish-network
timeseries-writer:
image: taylanbakircioglu/flowfish:timeseries-writer-latest
environment:
- L7_ENABLED=true
- SERVICE_NAME=timeseries-writer
- LOG_LEVEL=INFO
- RABBITMQ_HOST=rabbitmq
- RABBITMQ_PORT=5672
- RABBITMQ_USER=flowfish
- RABBITMQ_PASSWORD=flowfish123
- CLICKHOUSE_HOST=clickhouse
- CLICKHOUSE_PORT=9000
- CLICKHOUSE_USER=flowfish
- CLICKHOUSE_PASSWORD=flowfish123
- CLICKHOUSE_DATABASE=flowfish
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_PASSWORD=redis123
- REDIS_URL=redis://:redis123@redis:6379/0
- POSTGRES_HOST=postgres
- POSTGRES_PORT=5432
- POSTGRES_USER=flowfish
- POSTGRES_PASSWORD=flowfish123
- POSTGRES_DATABASE=flowfish
depends_on:
- rabbitmq
- redis
- clickhouse
- postgres
networks:
- flowfish-network
analysis-orchestrator:
image: taylanbakircioglu/flowfish:analysis-orchestrator-latest
environment:
- SERVICE_NAME=analysis-orchestrator
- GRPC_PORT=5002
- LOG_LEVEL=INFO
- POSTGRES_HOST=postgres
- POSTGRES_PORT=5432
- POSTGRES_USER=flowfish
- POSTGRES_PASSWORD=flowfish123
- POSTGRES_DATABASE=flowfish
- NEO4J_BOLT_URI=bolt://neo4j:7687
- NEO4J_HTTP_URI=http://neo4j:7474
- NEO4J_USER=neo4j
- NEO4J_PASSWORD=flowfish123
- NEO4J_DATABASE=neo4j
- CLICKHOUSE_HOST=clickhouse
- CLICKHOUSE_PORT=9000
- CLICKHOUSE_USER=flowfish
- CLICKHOUSE_PASSWORD=flowfish123
- CLICKHOUSE_DATABASE=flowfish
- CLUSTER_MANAGER_HOST=cluster-manager
- CLUSTER_MANAGER_PORT=5001
- L7_INGESTION_HOST=l7-ingestion-service
- L7_INGESTION_PORT=5006
depends_on:
- postgres
- neo4j
- clickhouse
- cluster-manager
- l7-ingestion-service
ports:
- "5002:5002"
networks:
- flowfish-network
# ============================================================================
# VOLUMES & NETWORKS
# ============================================================================
# ============================================================================
# NOTE: Inspektor Gadget (eBPF Data Collection)
# ============================================================================
# Inspektor Gadget is a Kubernetes-native DaemonSet and CANNOT run in Docker
# Compose. It requires the Kubernetes API for pod/container discovery and
# metadata enrichment. To collect eBPF network/DNS/process events:
# - Use the Kubernetes deployment (deployment/local-test/ or deployment/kubernetes-manifests/)
# - Or run Flowfish here and add a remote Kubernetes cluster via Token method
# in the UI (Clusters > Add Cluster > Token)
# ============================================================================
volumes:
postgres_data:
clickhouse_data:
networks:
flowfish-network:
driver: bridge
Reference in New Issue View Git Blame Copy Permalink