mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
52f12586f5
predastore pinned MULGA_ROOT_TRACE_RATIO=0.05 so guest block I/O, which is locally rooted and has no caller trace, would not drown request traces. That worked, but TraceIDRatioBased decides when the root span is created, before any error exists, so it discarded 95% of predastore failures along with the successes. An RCA on a rare locally-rooted failure had a 1-in-20 chance of having the span at all. The sink now runs a tail-sampling tier in front of APM Server that keeps every errored trace, keeps every trace slower than 2s, and samples the rest at 10%. It judges a trace once it is whole, so it can shed the same volume while keeping the traces worth having. Dropping the head ratio is what lets it see them: at 0.05 the tail sampler only ever saw 5% of predastore roots, so its error rule could keep at most 5% of the failures. Sized before changing: predastore's true local root rate is ~434 spans/s across six active CI environments, against 21.7/s reaching the sink today. Driving the tier at 508 spans/s held it at 88 MiB of a 1.5 GiB cap and 1.8% CPU with zero drops, and trace storage goes from 14.9 to 22.5 GB per 7 days on a volume with 735 GB free.
71 lines
2.4 KiB
Desktop File
71 lines
2.4 KiB
Desktop File
# spinifex-unit-version: 3
|
|
[Unit]
|
|
Description=Spinifex Predastore (S3-compatible Object Storage)
|
|
PartOf=spinifex.target
|
|
After=spinifex-nats.service
|
|
Wants=spinifex-nats.service
|
|
|
|
[Service]
|
|
EnvironmentFile=-/etc/spinifex/telemetry.env
|
|
# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
|
|
# instead, and it can see whether a trace errored. A head ratio decides before
|
|
# the error exists, so it discarded failures at the same rate as successes.
|
|
Type=simple
|
|
Slice=spinifex-system.slice
|
|
User=spinifex-storage
|
|
Group=spinifex
|
|
ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
|
ExecStart=/usr/local/bin/spx service predastore start
|
|
Restart=on-failure
|
|
RestartSec=5
|
|
# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
|
|
# predastore, but After= already stops viperblock first (within its own 150s
|
|
# budget); this is the backstop if that ordering is ever bypassed.
|
|
TimeoutStopSec=30
|
|
OOMScoreAdjust=-500
|
|
# Legitimately the most memory-hungry infra service (S3 buffering/caching);
|
|
# High/Max stay well above idle so normal load isn't throttled, while still
|
|
# bounding a leak short of the slice-wide cap.
|
|
MemoryHigh=40%
|
|
MemoryMax=55%
|
|
Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
|
|
Environment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml
|
|
Environment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/
|
|
Environment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem
|
|
Environment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key
|
|
Environment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key
|
|
|
|
# Filesystem access
|
|
NoNewPrivileges=yes
|
|
ProtectSystem=strict
|
|
PrivateDevices=yes
|
|
ReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
|
|
ReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex
|
|
|
|
# Hardening
|
|
ProtectHome=yes
|
|
PrivateTmp=yes
|
|
RestrictSUIDSGID=yes
|
|
LockPersonality=yes
|
|
RestrictRealtime=yes
|
|
RestrictNamespaces=yes
|
|
ProtectClock=yes
|
|
ProtectKernelTunables=yes
|
|
ProtectKernelModules=yes
|
|
ProtectKernelLogs=yes
|
|
ProtectControlGroups=yes
|
|
ProtectProc=invisible
|
|
SystemCallArchitectures=native
|
|
MemoryDenyWriteExecute=yes
|
|
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
|
|
CapabilityBoundingSet=
|
|
ProtectHostname=yes
|
|
KeyringMode=private
|
|
RemoveIPC=yes
|
|
LimitCORE=0
|
|
LimitNOFILE=65536
|
|
LimitNPROC=4096
|
|
|
|
[Install]
|
|
WantedBy=spinifex.target
|