spinifex/cmd/rds-agent/parameters.go

package main

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
)

// The generated parameter files and what it takes to install one. They live on
// the data volume rather than under /etc, at the same paths rds-init writes them
// to: a class change is a VM replacement, so a set under /etc would be discarded
// with the boot volume and the instance would silently revert to catalog
// defaults while the API still reported the customer's values.
type parameterStore struct {
	dir string
	// The include the resolved set is rendered to.
	installed string
	// The copy of the last set the engine accepted, which a boot loop is rolled
	// back to. Deliberately not a name the engine's own include glob reads, so it
	// is never parsed as a second set of settings.
	lastGood string
	// What the engine actually started on, for an engine that cannot be asked
	// which of its settings are still pending a restart. Empty for one that can.
	serving string
	// Prepended to every file written here, for a configuration format that has
	// no setting outside a group.
	header string
	// The engine parses its own configuration, so a root-owned file would be
	// unreadable to it.
	osUser string
}

func (s parameterStore) installedPath() string { return filepath.Join(s.dir, s.installed) }
func (s parameterStore) lastGoodPath() string  { return filepath.Join(s.dir, s.lastGood) }
func (s parameterStore) servingPath() string   { return filepath.Join(s.dir, s.serving) }

// Byte-for-byte what rds-init writes for the same set, so the copies compared
// against it are comparing values rather than formatting.
func (s parameterStore) render(params []handlers_rds.Parameter) []byte {
	return []byte(s.header + renderParameters(params))
}

// Installs the resolved set at the path rds-init uses, so a later boot
// overwrites rather than shadows it. Returns a func putting back whatever was
// there, so the caller can undo the install without re-rendering anything.
func (s parameterStore) install(params []handlers_rds.Parameter) (restore func() error, err error) {
	path := s.installedPath()
	// A missing file is the first-ever apply, and its rollback is a removal.
	previous, readErr := os.ReadFile(path)
	if readErr != nil && !os.IsNotExist(readErr) {
		return nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
	}
	existed := readErr == nil

	if err := s.write(path, s.render(params)); err != nil {
		return nil, err
	}
	return func() error {
		if !existed {
			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
			}
			return nil
		}
		return s.write(path, previous)
	}, nil
}

// Atomic: a temp file in the same directory, renamed over the target. The temp
// name deliberately does not end in the engine's include suffix, so a crash
// between write and rename cannot leave the engine reading a half-written file.
func (s parameterStore) write(path string, content []byte) error {
	tmp := path + ".new"
	if err := os.WriteFile(tmp, content, 0o600); err != nil {
		return fmt.Errorf("write %s: %w", tmp, err)
	}
	if err := s.chownToEngine(tmp); err != nil {
		os.Remove(tmp)
		return err
	}
	if err := os.Rename(tmp, path); err != nil {
		os.Remove(tmp)
		return fmt.Errorf("install %s: %w", path, err)
	}
	return nil
}

// A guest with no engine user is a broken image, not something to work around.
func (s parameterStore) chownToEngine(path string) error {
	credential, err := lookupCredential(s.osUser)
	if err != nil {
		return err
	}
	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
	}
	return nil
}

// Snapshots the installed set as the rollback target, but only once the engine
// is running all of it. A set with a setting still pending a restart has not
// been shown to serve, and promoting it would leave the rollback pointing at the
// configuration that is about to fail.
func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
	if _, err := os.Stat(store.installedPath()); err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return fmt.Errorf("inspect the serving parameters: %w", err)
	}
	names, err := pending(ctx)
	if err != nil {
		return err
	}
	if len(names) > 0 {
		return nil
	}

	content, err := os.ReadFile(store.installedPath())
	if err != nil {
		return fmt.Errorf("read the serving parameters: %w", err)
	}
	lastGood, err := os.ReadFile(store.lastGoodPath())
	if err == nil && bytes.Equal(content, lastGood) {
		return nil
	}
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("read the last known good parameters: %w", err)
	}
	return store.write(store.lastGoodPath(), content)
}

// Puts the last set the engine accepted back in place, for a restart that failed
// after a parameter change. The probe is checked again under the caller's
// parameter lock, so a repair that just brought the engine back is not reversed.
func restoreLastGood(ctx context.Context, store parameterStore, probe *engineProbe) (bool, error) {
	if state, _ := probe.state(ctx); state != engineAbsent {
		return false, nil
	}
	lastGood, err := os.ReadFile(store.lastGoodPath())
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, fmt.Errorf("read the last known good parameters: %w", err)
	}
	current, err := os.ReadFile(store.installedPath())
	if err != nil && !os.IsNotExist(err) {
		return false, fmt.Errorf("read the installed parameters: %w", err)
	}
	if bytes.Equal(current, lastGood) {
		return false, nil
	}
	if err := store.write(store.installedPath(), lastGood); err != nil {
		return false, err
	}
	return true, nil
}

// The settings the engine has accepted but will not honour until it restarts.
// Each implementation answers it its own way, and both the apply and the serving
// snapshot are gated on the same answer.
type pendingRestartFn func(ctx context.Context) ([]string, error)

const (
	parameterRepairTimeout = 90 * time.Second
	parameterRepairPoll    = time.Second
)

// Starts a down engine on the set that was just installed and waits for it to
// serve. Reached when the engine went away during an apply: the installed set is
// the newer of the two, so starting on it beats restoring one that already left
// the engine unable to serve.
func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
	repairCtx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	if err := restart(repairCtx); err != nil {
		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
	}

	timer := time.NewTimer(0)
	defer timer.Stop()
	lastMessage := "engine did not respond"
	for {
		select {
		case <-repairCtx.Done():
			if ctx.Err() != nil {
				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
			}
			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
		case <-timer.C:
		}

		state, message := probe.state(repairCtx)
		if state == engineServing {
			names, err := pending(repairCtx)
			if err == nil {
				return names, nil
			}
			lastMessage = err.Error()
		} else if message != "" {
			lastMessage = message
		}
		timer.Reset(poll)
	}
}