gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(targets): make target removal and reload transactional; prevent reappearing entries (#662)

Browse Source 
* feat: improve code for notify

* upgrade starshard version

* upgrade version

* Fix ETag format to comply with HTTP standards by wrapping with quotes (#592)

* Initial plan

* Fix ETag format to comply with HTTP standards by wrapping with quotes

Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com>

* bufigx

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>

* Improve lock (#596)

* improve lock

Signed-off-by: Mu junxiang <1948535941@qq.com>

* feat(tests): add wait_for_object_absence helper and improve lifecycle test reliability

Signed-off-by: Mu junxiang <1948535941@qq.com>

* chore: remove dirty docs

Signed-off-by: Mu junxiang <1948535941@qq.com>

---------

Signed-off-by: Mu junxiang <1948535941@qq.com>

* feat(append): implement object append operations with state tracking (#599)

* feat(append): implement object append operations with state tracking

Signed-off-by: junxiang Mu <1948535941@qq.com>

* chore: rebase

Signed-off-by: junxiang Mu <1948535941@qq.com>

---------

Signed-off-by: junxiang Mu <1948535941@qq.com>

* build(deps): upgrade s3s (#595)

Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>

* fix: validate mqtt broker

* improve code for `import`

* fix

* improve

* remove logger from `rustfs-obs` crate

* remove code for config Observability

* fix

* improve code

* fix comment

* up

* up

* upgrade version

* fix

* fmt

* upgrade tokio version to 1.48.0

* upgrade `datafusion` and `reed-solomon-simd` version

* fix

* fmt

* improve code for notify webhook example

* improve code

* fix

* fix

* fmt

---------

Signed-off-by: Mu junxiang <1948535941@qq.com>
Signed-off-by: junxiang Mu <1948535941@qq.com>
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: overtrue <1472352+overtrue@users.noreply.github.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>
Co-authored-by: guojidan <63799833+guojidan@users.noreply.github.com>
Co-authored-by: Nugine <nugine@foxmail.com>
Co-authored-by: loverustfs <155562731+loverustfs@users.noreply.github.com>
This commit is contained in:
houseme
2025-10-17 15:34:53 +08:00
committed by GitHub
parent 30e7f00b02
commit 42d3645d6f
42 changed files with 514 additions and 3219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/config.rs
+4 -174
View File
@@ -13,16 +13,9 @@
// limitations under the License.
use rustfs_config::observability::{
DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, DEFAULT_SINKS_FILE_BUFFER_SIZE, DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS,
DEFAULT_SINKS_FILE_FLUSH_THRESHOLD, DEFAULT_SINKS_KAFKA_BATCH_SIZE, DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS,
DEFAULT_SINKS_KAFKA_BROKERS, DEFAULT_SINKS_KAFKA_TOPIC, DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN, DEFAULT_SINKS_WEBHOOK_ENDPOINT,
DEFAULT_SINKS_WEBHOOK_MAX_RETRIES, DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS, ENV_AUDIT_LOGGER_QUEUE_CAPACITY, ENV_OBS_ENDPOINT,
ENV_OBS_ENVIRONMENT, ENV_OBS_LOCAL_LOGGING_ENABLED, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_KEEP_FILES,
ENV_OBS_LOG_ROTATION_SIZE_MB, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOGGER_LEVEL, ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO,
ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT, ENV_SINKS_FILE_BUFFER_SIZE,
ENV_SINKS_FILE_FLUSH_INTERVAL_MS, ENV_SINKS_FILE_FLUSH_THRESHOLD, ENV_SINKS_FILE_PATH, ENV_SINKS_KAFKA_BATCH_SIZE,
ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS, ENV_SINKS_KAFKA_BROKERS, ENV_SINKS_KAFKA_TOPIC, ENV_SINKS_WEBHOOK_AUTH_TOKEN,
ENV_SINKS_WEBHOOK_ENDPOINT, ENV_SINKS_WEBHOOK_MAX_RETRIES, ENV_SINKS_WEBHOOK_RETRY_DELAY_MS,
ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, ENV_OBS_LOCAL_LOGGING_ENABLED, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_FILENAME,
ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_ROTATION_SIZE_MB, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOGGER_LEVEL,
ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT,
};
use rustfs_config::{
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_SIZE_MB, DEFAULT_LOG_ROTATION_TIME,
@@ -145,167 +138,10 @@ impl Default for OtelConfig {
}
}
/// Kafka Sink Configuration - Add batch parameters
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct KafkaSinkConfig {
pub brokers: String,
pub topic: String,
pub batch_size: Option<usize>, // Batch size, default 100
pub batch_timeout_ms: Option<u64>, // Batch timeout time, default 1000ms
}
impl KafkaSinkConfig {
pub fn new() -> Self {
Self::default()
}
}
impl Default for KafkaSinkConfig {
fn default() -> Self {
Self {
brokers: env::var(ENV_SINKS_KAFKA_BROKERS)
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| DEFAULT_SINKS_KAFKA_BROKERS.to_string()),
topic: env::var(ENV_SINKS_KAFKA_TOPIC)
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| DEFAULT_SINKS_KAFKA_TOPIC.to_string()),
batch_size: env::var(ENV_SINKS_KAFKA_BATCH_SIZE)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_KAFKA_BATCH_SIZE)),
batch_timeout_ms: env::var(ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS)),
}
}
}
/// Webhook Sink Configuration - Add Retry Parameters
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WebhookSinkConfig {
pub endpoint: String,
pub auth_token: String,
pub max_retries: Option<usize>, // Maximum number of retry times, default 3
pub retry_delay_ms: Option<u64>, // Retry the delay cardinality, default 100ms
}
impl WebhookSinkConfig {
pub fn new() -> Self {
Self::default()
}
}
impl Default for WebhookSinkConfig {
fn default() -> Self {
Self {
endpoint: env::var(ENV_SINKS_WEBHOOK_ENDPOINT)
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| DEFAULT_SINKS_WEBHOOK_ENDPOINT.to_string()),
auth_token: env::var(ENV_SINKS_WEBHOOK_AUTH_TOKEN)
.ok()
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN.to_string()),
max_retries: env::var(ENV_SINKS_WEBHOOK_MAX_RETRIES)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_WEBHOOK_MAX_RETRIES)),
retry_delay_ms: env::var(ENV_SINKS_WEBHOOK_RETRY_DELAY_MS)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS)),
}
}
}
/// File Sink Configuration - Add buffering parameters
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct FileSinkConfig {
pub path: String,
pub buffer_size: Option<usize>, // Write buffer size, default 8192
pub flush_interval_ms: Option<u64>, // Refresh interval time, default 1000ms
pub flush_threshold: Option<usize>, // Refresh threshold, default 100 logs
}
impl FileSinkConfig {
pub fn new() -> Self {
Self::default()
}
}
impl Default for FileSinkConfig {
fn default() -> Self {
Self {
path: get_log_directory_to_string(ENV_SINKS_FILE_PATH),
buffer_size: env::var(ENV_SINKS_FILE_BUFFER_SIZE)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_FILE_BUFFER_SIZE)),
flush_interval_ms: env::var(ENV_SINKS_FILE_FLUSH_INTERVAL_MS)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS)),
flush_threshold: env::var(ENV_SINKS_FILE_FLUSH_THRESHOLD)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_SINKS_FILE_FLUSH_THRESHOLD)),
}
}
}
/// Sink configuration collection
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum SinkConfig {
File(FileSinkConfig),
Kafka(KafkaSinkConfig),
Webhook(WebhookSinkConfig),
}
impl SinkConfig {
pub fn new() -> Self {
Self::File(FileSinkConfig::new())
}
}
impl Default for SinkConfig {
fn default() -> Self {
Self::new()
}
}
///Logger Configuration
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct LoggerConfig {
pub queue_capacity: Option<usize>,
}
impl LoggerConfig {
pub fn new() -> Self {
Self {
queue_capacity: env::var(ENV_AUDIT_LOGGER_QUEUE_CAPACITY)
.ok()
.and_then(|v| v.parse().ok())
.or(Some(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY)),
}
}
}
impl Default for LoggerConfig {
fn default() -> Self {
Self::new()
}
}
/// Overall application configuration
/// Add observability, sinks, and logger configuration
/// Add observability configuration
///
/// Observability: OpenTelemetry configuration
/// Sinks: Kafka, Webhook, File sink configuration
/// Logger: Logger configuration
///
/// # Example
/// ```
@@ -316,8 +152,6 @@ impl Default for LoggerConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct AppConfig {
pub observability: OtelConfig,
pub sinks: Vec<SinkConfig>,
pub logger: Option<LoggerConfig>,
}
impl AppConfig {
@@ -328,16 +162,12 @@ impl AppConfig {
pub fn new() -> Self {
Self {
observability: OtelConfig::default(),
sinks: vec![SinkConfig::default()],
logger: Some(LoggerConfig::default()),
}
}
pub fn new_with_endpoint(endpoint: Option<String>) -> Self {
Self {
observability: OtelConfig::extract_otel_config_from_env(endpoint),
sinks: vec![SinkConfig::new()],
logger: Some(LoggerConfig::new()),
}
}
}