diff --git a/.docker/observability/README.md b/.docker/observability/README.md index e69de29bb..f0bd03fcb 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -0,0 +1,101 @@ +# Observability + +This directory contains the observability stack for the application. The stack is composed of the following components: + +- Prometheus +- Grafana +- Loki +- Jaeger +- Otel Collector + +## Prometheus + +Prometheus is a monitoring and alerting toolkit. It scrapes metrics from instrumented jobs, either directly or via an +intermediary push gateway for short-lived jobs. It stores all scraped samples locally and runs rules over this data to +either aggregate and record new time series from existing data or generate alerts. Grafana or other API consumers can be +used to visualize the collected data. + +## Grafana + +Grafana is a multi-platform open-source analytics and interactive visualization web application. It provides charts, +graphs, and alerts for the web when connected to supported data sources. + +## Loki + +Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is +designed to be very cost-effective and easy to operate. It does not index the contents of the logs, but rather a set of +labels for each log stream. + +## Jaeger + +Jaeger is a distributed tracing system released as open source by Uber Technologies. It is used for monitoring and +troubleshooting microservices-based distributed systems, including: + +- Distributed context propagation +- Distributed transaction monitoring +- Root cause analysis +- Service dependency analysis +- Performance / latency optimization + +## Otel Collector + +The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry +data. It removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source +observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or more open-source or commercial back-ends. + +## How to use + +To deploy the observability stack, run the following command: + +```bash +docker-compose -f docker-compose.yml -f docker-compose.override.yml up -d +``` + +To access the Grafana dashboard, navigate to `http://localhost:3000` in your browser. The default username and password +are `admin` and `admin`, respectively. + +To access the Jaeger dashboard, navigate to `http://localhost:16686` in your browser. + +To access the Prometheus dashboard, navigate to `http://localhost:9090` in your browser. + +## How to stop + +To stop the observability stack, run the following command: + +```bash +docker-compose -f docker-compose.yml -f docker-compose.override.yml down +``` + +## How to remove data + +To remove the data generated by the observability stack, run the following command: + +```bash +docker-compose -f docker-compose.yml -f docker-compose.override.yml down -v +``` + +## How to configure + +To configure the observability stack, modify the `docker-compose.override.yml` file. The file contains the following + +```yaml +services: + prometheus: + environment: + - PROMETHEUS_CONFIG_FILE=/etc/prometheus/prometheus.yml + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + + grafana: + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning +``` + +The `prometheus` service mounts the `prometheus.yml` file to `/etc/prometheus/prometheus.yml`. The `grafana` service +mounts the `grafana/provisioning` directory to `/etc/grafana/provisioning`. You can modify these files to configure the +observability stack. + + + diff --git a/README.md b/README.md index dbb7e9f6e..217ab3c14 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ export RUSTFS_VOLUMES="./target/volume/test" export RUSTFS_ADDRESS="0.0.0.0:9000" export RUSTFS_CONSOLE_ENABLE=true export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001" +export RUSTFS_OBS_CONFIG="config/obs.toml" ``` You need replace your real data folder: @@ -47,3 +48,47 @@ You need replace your real data folder: ``` ./rustfs /data/rustfs ``` + +## How to deploy the observability stack + +The OpenTelemetry Collector offers a vendor-agnostic implementation on how to receive, process, and export telemetry +data. It removes the need to run, operate, and maintain multiple agents/collectors in order to support open-source +observability data formats (e.g. Jaeger, Prometheus, etc.) sending to one or more open-source or commercial back-ends. + +1. Enter the `.docker/observability` directory, +2. Run the following command: + +```bash +docker-compose -f docker-compose.yml up -d +``` + +3. Access the Grafana dashboard by navigating to `http://localhost:3000` in your browser. The default username and + password are `admin` and `admin`, respectively. + +4. Access the Jaeger dashboard by navigating to `http://localhost:16686` in your browser. + +5. Access the Prometheus dashboard by navigating to `http://localhost:9090` in your browser. + +## Create a new Observability configuration file + +#### 1. Enter the `config` directory, + +#### 2. Copy `obs.toml.example` to `obs.toml` + +#### 3. Modify the `obs.toml` configuration file + +##### 3.1. Modify the `endpoint` value to the address of the OpenTelemetry Collector + +##### 3.2. Modify the `service_name` value to the name of the service + +##### 3.3. Modify the `service_version` value to the version of the service + +##### 3.4. Modify the `deployment_environment` value to the environment of the service + +##### 3.5. Modify the `meter_interval` value to export interval + +##### 3.6. Modify the `sample_ratio` value to the sample ratio + +##### 3.7. Modify the `use_stdout` value to export to stdout + + diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index cd8f238fd..d6552af83 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -1,11 +1,32 @@ use clap::Parser; use const_str::concat; use ecstore::global::DEFAULT_PORT; +use std::string::ToString; shadow_rs::shadow!(build); +/// Default Access Key +/// Default value: rustfsadmin +/// Environment variable: RUSTFS_ACCESS_KEY +/// Command line argument: --access-key +/// Example: RUSTFS_ACCESS_KEY=rustfsadmin +/// Example: --access-key rustfsadmin pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; +/// Default Secret Key +/// Default value: rustfsadmin +/// Environment variable: RUSTFS_SECRET_KEY +/// Command line argument: --secret-key +/// Example: RUSTFS_SECRET_KEY=rustfsadmin +/// Example: --secret-key rustfsadmin pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; +/// Default configuration file for observability +/// Default value: config/obs.toml +/// Environment variable: RUSTFS_OBS_CONFIG +/// Command line argument: --obs-config +/// Example: RUSTFS_OBS_CONFIG=config/obs.toml +/// Example: --obs-config config/obs.toml +/// Example: --obs-config /etc/rustfs/obs.toml +pub const DEFAULT_OBS_CONFIG: &str = "config/obs.toml"; #[allow(clippy::const_is_empty)] const SHORT_VERSION: &str = { @@ -31,7 +52,7 @@ const LONG_VERSION: &str = concat!( concat!("git status :\n", build::GIT_STATUS_FILE), ); -#[derive(Debug, Parser)] +#[derive(Debug, Parser, Clone)] #[command(version = SHORT_VERSION, long_version = LONG_VERSION)] pub struct Opt { /// DIR points to a directory on a filesystem. @@ -62,4 +83,9 @@ pub struct Opt { #[arg(long, default_value_t = format!("127.0.0.1:{}", 9002), env = "RUSTFS_CONSOLE_ADDRESS")] pub console_address: String, + + /// Observability configuration file + /// Default value: config/obs.toml + #[arg(long, default_value_t = DEFAULT_OBS_CONFIG.to_string(), env = "RUSTFS_OBS_CONFIG")] + pub obs_config: String, } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index b87b16ca3..1c1351768 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -92,9 +92,7 @@ fn main() -> Result<()> { //设置 trace // setup_tracing(); - let config = load_config(Some("packages/obs/examples/config".to_string())); - // Initialize the logger asynchronously - let (_logger, _guard) = tokio::runtime::Runtime::new()?.block_on(async { init_obs(config).await }); + //运行参数 run(opt) } @@ -102,6 +100,9 @@ fn main() -> Result<()> { #[tokio::main] async fn run(opt: config::Opt) -> Result<()> { debug!("opt: {:?}", &opt); + let config = load_config(Some(opt.clone().obs_config)); + // Initialize Observability + init_obs(config).await; let mut server_addr = net::check_local_server_addr(opt.address.as_str()).unwrap(); @@ -277,7 +278,7 @@ async fn run(opt: config::Opt) -> Result<()> { init_iam_sys(store.clone()).await.unwrap(); new_global_notification_sys(endpoint_pools.clone()).await.map_err(|err| { - error!("new_global_notification_sys faild {:?}", &err); + error!("new_global_notification_sys failed {:?}", &err); Error::from_string(err.to_string()) })?;