mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-06 13:27:43 +00:00
f403ed1c2e
* feat(embedded): allow multiple embedded servers to coexist in one process
backlog#1052 S5: turn the embedded startup guard into a sequential lock
and make every write-once shared cell tolerate a second embedded start.
Two RustFSServers on different ports and volumes now start, run, and
shut down independently in the same process.
- EMBEDDED_SERVER_STARTED is released once each startup hands off; a
second startup that runs after the first is no longer rejected.
- The second embedded server constructs its own InstanceContext instead
of adopting the process bootstrap one, so region/endpoints/deployment
id land on that context (the first server keeps adopting bootstrap to
keep single-instance ambient facades unchanged).
- Startup-time tolerant paths:
- action_credentials publish treats AlreadyInitialized as success
(per-server ActionCredentialHandle already holds the real creds).
- GLOBAL_RUSTFS_PORT warns instead of panicking on a second set.
- Observability install returns Ok when the process subscriber is
already set — the second server reuses it.
- New acceptance test proves two servers start, respond on their own
ports, and one can be shut down without disturbing the other; the
survivor keeps serving S3 requests. IAM and root-credential lookup
still share a process domain (a second server whose creds differ from
the first will fail signature validation), tracked as a follow-up.
- Embedded doc rewritten: 'Limitations' → 'Multi-instance status'; the
AlreadyStarted error is now scoped to concurrent startups only.
The remaining work in #1052 is the auth path per-server dispatch and the
matching data-plane routing so two servers with different credentials
serve independent buckets end-to-end.
* feat(app): per-server auth and application context for multiple embedded servers (#4633)
backlog#1052 S6: each embedded server now authenticates against — and
its request path resolves — its OWN application context, so two servers
with different root credentials each accept their own access key and
reject the other's.
- AppContext is per-server: ensure_startup_after_iam constructs a fresh
context around this server's store + IAM + KMS and installs it into the
server's own ServerContextSlot, then publishes it as the process
default first-writer-wins (publish_global_app_context) for legacy
ambient readers. The old 'reuse the global if present' path is gone.
- FS::check (the S3 data-plane access gate) resolves auth against
self.server_ctx's context: check_key_valid gains a _with_context
variant that takes the root credentials and IAM system from an explicit
context (None = ambient, unchanged for all 140+ existing callers). The
region and the server context slot are published into the request
extensions for downstream handlers.
- Each embedded server seeds its own root credentials into its context
(ActionCredentialHandle.publish) at startup, so credential validation
no longer falls back to the first server's process-global identity.
- The bucket/object/multipart use-cases resolve their store from the
server's context (bucket_usecase_for/object_usecase_for/... take &FS).
New acceptance test: two servers with distinct credentials each
authenticate with their own key and reject the other's.
KNOWN FOLLOW-UP: full bucket-namespace isolation still requires threading
the instance context through the lower ecstore data plane (peer_sys /
disk registry / bucket-metadata reads still resolve via the process
GLOBAL_OBJECT_API), so the two servers do not yet present independent
bucket listings even though each holds its own store. That deeper pass —
a continuation of the #939 object-graph ctx threading — is the remaining
work on #1052.
Stacked on the S5 guard change.
187 lines
5.9 KiB
Rust
187 lines
5.9 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use crate::license::license_status;
|
|
use crate::startup_runtime_sources;
|
|
use rustls::crypto::aws_lc_rs::default_provider;
|
|
use std::future::Future;
|
|
use std::io::Result;
|
|
use tracing::{debug, info};
|
|
|
|
const LOG_COMPONENT_EMBEDDED: &str = "embedded";
|
|
const LOG_COMPONENT_MAIN: &str = "main";
|
|
const LOG_SUBSYSTEM_EMBEDDED: &str = "embedded";
|
|
const LOG_SUBSYSTEM_LICENSE: &str = "license";
|
|
const LOG_SUBSYSTEM_STARTUP: &str = "startup";
|
|
const EVENT_CRYPTO_PROVIDER_STATE: &str = "crypto_provider_state";
|
|
const EVENT_DIAL9_RUNTIME_STATUS: &str = "dial9_runtime_status";
|
|
const EVENT_RUNTIME_LICENSE_STATUS: &str = "runtime_license_status";
|
|
|
|
pub(crate) fn log_startup_runtime_diagnostics() {
|
|
log_dial9_runtime_status();
|
|
log_runtime_license_status();
|
|
debug!("{}", crate::server::LOGO);
|
|
}
|
|
|
|
fn log_dial9_runtime_status() {
|
|
if rustfs_obs::dial9::is_enabled() {
|
|
info!(
|
|
target: "rustfs::main",
|
|
event = EVENT_DIAL9_RUNTIME_STATUS,
|
|
component = LOG_COMPONENT_MAIN,
|
|
subsystem = LOG_SUBSYSTEM_STARTUP,
|
|
enabled = true,
|
|
"Dial9 Tokio runtime telemetry is enabled"
|
|
);
|
|
} else {
|
|
debug!(
|
|
target: "rustfs::main",
|
|
event = EVENT_DIAL9_RUNTIME_STATUS,
|
|
component = LOG_COMPONENT_MAIN,
|
|
subsystem = LOG_SUBSYSTEM_STARTUP,
|
|
enabled = false,
|
|
"Dial9 Tokio runtime telemetry is disabled"
|
|
);
|
|
}
|
|
}
|
|
|
|
fn log_runtime_license_status() {
|
|
info!(
|
|
target: "rustfs::main",
|
|
event = EVENT_RUNTIME_LICENSE_STATUS,
|
|
component = LOG_COMPONENT_MAIN,
|
|
subsystem = LOG_SUBSYSTEM_LICENSE,
|
|
license_status = %license_status(),
|
|
"Initialized runtime license state"
|
|
);
|
|
}
|
|
|
|
pub(crate) async fn init_profiling_runtime() {
|
|
init_profiling_runtime_with(crate::profiling::init_from_env).await;
|
|
}
|
|
|
|
async fn init_profiling_runtime_with<InitFn, InitFuture>(init: InitFn)
|
|
where
|
|
InitFn: FnOnce() -> InitFuture,
|
|
InitFuture: Future<Output = ()>,
|
|
{
|
|
init().await;
|
|
}
|
|
|
|
pub(crate) fn shutdown_profiling_runtime() {
|
|
shutdown_profiling_runtime_with(crate::profiling::shutdown_profiling);
|
|
}
|
|
|
|
fn shutdown_profiling_runtime_with<ShutdownFn>(shutdown: ShutdownFn)
|
|
where
|
|
ShutdownFn: FnOnce(),
|
|
{
|
|
shutdown();
|
|
}
|
|
|
|
pub(crate) fn install_default_crypto_provider() {
|
|
if default_provider().install_default().is_err() {
|
|
debug!(
|
|
target: "rustfs::main",
|
|
event = EVENT_CRYPTO_PROVIDER_STATE,
|
|
component = LOG_COMPONENT_MAIN,
|
|
subsystem = LOG_SUBSYSTEM_STARTUP,
|
|
provider = "aws_lc_rs",
|
|
state = "already_installed",
|
|
"Rustls crypto provider state checked"
|
|
);
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn init_embedded_runtime_hooks(obs_endpoint: String) -> Result<()> {
|
|
// Observability is a process-global tracing subscriber and can only be
|
|
// installed once. A second embedded server in the same process reuses the
|
|
// first server's subscriber — try to publish anyway (first server wins),
|
|
// and treat both an init failure and a set failure as "already set" so a
|
|
// second startup does not abort (backlog#1052 S5).
|
|
match startup_runtime_sources::init_observability_guard(obs_endpoint).await {
|
|
Ok(guard) => {
|
|
if let Err(err) = startup_runtime_sources::set_observability_guard(guard) {
|
|
debug!(
|
|
component = LOG_COMPONENT_EMBEDDED,
|
|
subsystem = LOG_SUBSYSTEM_EMBEDDED,
|
|
event = "observability_guard_reused",
|
|
error = %err,
|
|
"process already has an observability guard installed; second server reuses it"
|
|
);
|
|
}
|
|
}
|
|
Err(err) => {
|
|
debug!(
|
|
component = LOG_COMPONENT_EMBEDDED,
|
|
subsystem = LOG_SUBSYSTEM_EMBEDDED,
|
|
event = "observability_init_skipped",
|
|
error = %err,
|
|
"observability already initialized; second server reuses the process subscriber"
|
|
);
|
|
}
|
|
}
|
|
|
|
install_embedded_default_crypto_provider();
|
|
rustfs_trusted_proxies::init();
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn install_embedded_default_crypto_provider() {
|
|
if let Err(err) = default_provider().install_default() {
|
|
debug!(
|
|
component = LOG_COMPONENT_EMBEDDED,
|
|
subsystem = LOG_SUBSYSTEM_EMBEDDED,
|
|
event = EVENT_CRYPTO_PROVIDER_STATE,
|
|
state = "already_installed",
|
|
error = ?err,
|
|
"Embedded crypto provider state changed"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{init_profiling_runtime_with, shutdown_profiling_runtime_with};
|
|
use std::sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, Ordering},
|
|
};
|
|
|
|
#[tokio::test]
|
|
async fn init_profiling_runtime_invokes_registered_hook() {
|
|
let called = Arc::new(AtomicBool::new(false));
|
|
let hook_called = called.clone();
|
|
|
|
init_profiling_runtime_with(move || async move {
|
|
hook_called.store(true, Ordering::SeqCst);
|
|
})
|
|
.await;
|
|
|
|
assert!(called.load(Ordering::SeqCst));
|
|
}
|
|
|
|
#[test]
|
|
fn shutdown_profiling_runtime_invokes_registered_hook() {
|
|
let called = AtomicBool::new(false);
|
|
|
|
shutdown_profiling_runtime_with(|| {
|
|
called.store(true, Ordering::SeqCst);
|
|
});
|
|
|
|
assert!(called.load(Ordering::SeqCst));
|
|
}
|
|
}
|