spinifex/tests/integration/request_conformance.go

//go:build integration

package integration

import (
	"cmp"
	"fmt"
	"maps"
	"slices"
	"strings"
	"sync"

	"github.com/mulgadc/spinifex/internal/awsmodel"
)

// requestFinding is one generated request whose outcome disagrees with the
// model: an acceptance request rejected as invalid, a rejection request that
// succeeded, or one refused with an error the operation does not declare.
type requestFinding struct {
	service awsmodel.Service
	request awsmodel.RequestCase
	result  awsmodel.RequestResult
	verdict awsmodel.Verdict
}

type requestServiceCounts struct {
	operations     int
	requests       int
	verdicts       map[awsmodel.Verdict]int
	inconclusive   map[string]int
	skippedMembers int
	unbroken       int
	// untested maps each operation that produced no request to the reason.
	untested map[string]string
}

// requestCollector gathers the verdicts of the generated request sweep.
type requestCollector struct {
	mu       sync.Mutex
	services map[awsmodel.Service]*requestServiceCounts
	findings []requestFinding
}

func newRequestCollector() *requestCollector {
	return &requestCollector{services: map[awsmodel.Service]*requestServiceCounts{}}
}

var suiteRequestConformance = newRequestCollector()

func (c *requestCollector) serviceLocked(service awsmodel.Service) *requestServiceCounts {
	counts := c.services[service]
	if counts == nil {
		counts = &requestServiceCounts{verdicts: map[awsmodel.Verdict]int{}, inconclusive: map[string]int{}, untested: map[string]string{}}
		c.services[service] = counts
	}
	return counts
}

func (c *requestCollector) recordOperation(service awsmodel.Service, operation string, plan awsmodel.RequestPlan) {
	c.mu.Lock()
	defer c.mu.Unlock()
	counts := c.serviceLocked(service)
	counts.operations++
	counts.skippedMembers += len(plan.Skipped)
	counts.unbroken += len(plan.Unbroken)
	if len(plan.Cases) == 0 {
		reasons := make([]string, len(plan.Skipped))
		for i, skip := range plan.Skipped {
			reasons[i] = skip.Path + ": " + skip.Reason
		}
		counts.untested[operation] = strings.Join(reasons, "; ")
	}
}

func (c *requestCollector) record(service awsmodel.Service, request awsmodel.RequestCase, result awsmodel.RequestResult, verdict awsmodel.Verdict) {
	c.mu.Lock()
	defer c.mu.Unlock()
	counts := c.serviceLocked(service)
	counts.requests++
	counts.verdicts[verdict]++
	switch verdict {
	case awsmodel.VerdictFinding, awsmodel.VerdictUndeclaredError:
		c.findings = append(c.findings, requestFinding{service: service, request: request, result: result, verdict: verdict})
	case awsmodel.VerdictInconclusive:
		code := result.Code
		if code == "" {
			code = fmt.Sprintf("status %d", result.Status)
		}
		counts.inconclusive[code]++
	}
}

func (c *requestCollector) blocking(policy conformancePolicy, mode conformanceMode) int {
	if mode != conformanceModeFail {
		return 0
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	blocking := 0
	for _, finding := range c.findings {
		if policy.isRequestPromoted(finding.service) {
			blocking++
		}
	}
	return blocking
}

func (c *requestCollector) report(policy conformancePolicy, mode conformanceMode) string {
	c.mu.Lock()
	defer c.mu.Unlock()

	var report strings.Builder
	total := requestServiceCounts{verdicts: map[awsmodel.Verdict]int{}}
	untested := 0
	for _, counts := range c.services {
		total.operations += counts.operations
		total.requests += counts.requests
		total.skippedMembers += counts.skippedMembers
		total.unbroken += counts.unbroken
		untested += len(counts.untested)
		for verdict, count := range counts.verdicts {
			total.verdicts[verdict] += count
		}
	}
	fmt.Fprintf(&report, "AWS request conformance (%s): operations=%d untested_operations=%d requests=%d %s\n",
		mode, total.operations, untested, total.requests, verdictSummary(&total))

	for _, service := range slices.Sorted(maps.Keys(c.services)) {
		counts := c.services[service]
		fmt.Fprintf(&report, "REQUESTS %s operations=%d untested_operations=%d requests=%d %s promoted=%t\n",
			service, counts.operations, len(counts.untested), counts.requests, verdictSummary(counts), policy.isRequestPromoted(service))
		for _, operation := range slices.Sorted(maps.Keys(counts.untested)) {
			fmt.Fprintf(&report, "UNTESTED %s %s: %s\n", service, operation, counts.untested[operation])
		}
		codes := slices.SortedFunc(maps.Keys(counts.inconclusive), func(a, b string) int {
			return cmp.Or(cmp.Compare(counts.inconclusive[b], counts.inconclusive[a]), strings.Compare(a, b))
		})
		if len(codes) > 0 {
			parts := make([]string, len(codes))
			for i, code := range codes {
				parts[i] = fmt.Sprintf("%s=%d", code, counts.inconclusive[code])
			}
			fmt.Fprintf(&report, "INCONCLUSIVE %s %s\n", service, strings.Join(parts, " "))
		}
	}

	findings := slices.Clone(c.findings)
	slices.SortFunc(findings, func(a, b requestFinding) int {
		return cmp.Or(cmp.Compare(a.service, b.service), strings.Compare(a.request.Operation, b.request.Operation),
			strings.Compare(a.request.Path, b.request.Path), strings.Compare(a.request.Detail, b.request.Detail))
	})
	for _, finding := range findings {
		severity := "WARN"
		if mode == conformanceModeFail && policy.isRequestPromoted(finding.service) {
			severity = "FAIL"
		}
		request := finding.request
		switch {
		case finding.verdict == awsmodel.VerdictUndeclaredError:
			fmt.Fprintf(&report, "%s %s %s %s %s (%s) refused with undeclared %s: %s\n",
				severity, finding.service, request.Operation, request.Path, request.Constraint, request.Detail, finding.result.Code, finding.result.Message)
		case request.Acceptance():
			with := "with only required members"
			if request.Member != "" {
				with = "with " + request.Path + " set"
			}
			fmt.Fprintf(&report, "%s %s %s model-valid request %s rejected: %s: %s\n",
				severity, finding.service, request.Operation, with, finding.result.Code, finding.result.Message)
		default:
			fmt.Fprintf(&report, "%s %s %s %s %s (%s) accepted\n",
				severity, finding.service, request.Operation, request.Path, request.Constraint, request.Detail)
		}
	}
	return strings.TrimSuffix(report.String(), "\n")
}

func verdictSummary(counts *requestServiceCounts) string {
	return fmt.Sprintf("pass=%d findings=%d undeclared_errors=%d inconclusive=%d skipped_members=%d unbroken_constraints=%d",
		counts.verdicts[awsmodel.VerdictPass], counts.verdicts[awsmodel.VerdictFinding], counts.verdicts[awsmodel.VerdictUndeclaredError],
		counts.verdicts[awsmodel.VerdictInconclusive], counts.skippedMembers, counts.unbroken)
}