mirror of
https://github.com/PerpetualSoftware/pad.git
synced 2026-09-10 23:15:40 +00:00
cfda4463e8
* feat(cmdhelp): tests + golden contract + drift validator (TASK-938)
The verification layer that turns cmdhelp v0.1 from "implementation"
into "stable contract." Three categories of tests, all running in
`go test ./...`:
1. Schema validation (cmdhelp.schema.json as CI gate)
- internal/cmdhelp/schema_test.go — synthetic tree's emitted JSON
validates after static walk, after dynamic resolution, and after
a no-workspace fallback.
- cmd/pad/cmdhelp_real_test.go — the REAL pad cobra tree's emitted
JSON validates against the published schema. Future regressions
caught: types outside the closed vocabulary, non-numeric exit_code
keys, flag names violating propertyNames, malformed cmdhelp_version.
2. Drift-prevention contract (spec §6 / §11 Q5)
- internal/cmdhelp/example_validation.go — ValidateExamples walks
every example's `cmd` string, tokenizes with shellSplit, resolves
non-flag tokens against the live cobra tree, and asserts every
--flag exists on the resolved command (or any ancestor for
persistent / inherited flags). Negate-flag form (`--no-cache`)
is recognized via the negation rule from spec §5.3.
- shellSplit handles double/single quotes, backslash escape, and
stops at unquoted pipeline boundaries (|, ;, &, >, <) so the
validator only checks the first command in a pipeline.
- ValidateBoolArity asserts no bool flag appears in valued form
(--flag=value) anywhere in its examples (spec §5.3).
- cmd/pad/cmdhelp_real_test.go runs both validators against the
real pad tree as CI gates.
- Negative tests in internal/cmdhelp/example_validation_test.go
prove the validator catches: typo'd flag (--priorty), unknown
command path, valued-form bool flag.
3. Capabilities form equivalence (spec §8)
- cmd/pad/cmdhelp_real_test.go — both forms (help --capabilities
and --cmdhelp-capabilities fallback) produce byte-identical
output. Side-effect-free guarantee verified by passing garbage
args alongside the fallback flag.
Refactors enabling the tests:
- cmd/pad/main.go: extract newRootCmd() so tests can build the real
cobra tree without running it. main() body shrinks to two lines.
- cmd/pad/main.go: extract handleCmdhelpCapabilitiesFallback() so the
fallback's side-effect-free contract is directly assertable instead
of requiring a subprocess.
Parser improvements driven by real-pad-tree drift findings:
- parseExamplesFromLong: strip same-line `# comment` annotations so
`pad foo --bar # one item's attachments` doesn't pollute Examples.
stripCommentIndex is quote-aware (# inside "..." or '...' is literal).
- main.go (github cmd): the Long had annotations on example lines
separated only by spaces (no `#`), which was malformed input. Fixed
to use `#` separators — caught by the drift validator on first run.
New deps:
- github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 — Go JSON Schema
validator supporting draft 2020-12 (matches the cmdhelp schema's $schema).
New helpers in internal/cmdhelp/:
- FindAndCompileSchema(startDir) walks up to locate
schema/cmdhelp.schema.json and returns a compiled schema. Reusable
by any consumer that wants to validate cmdhelp documents.
End-to-end on real binary:
- pad help --format json → 100 commands, schema-valid.
- All examples in pad's emitted output resolve against the live tree
(zero drift findings).
- pad help --capabilities byte-identical to pad --cmdhelp-capabilities.
- Adding a typo'd flag in any cobra Long block in cmd/pad MUST break
TestRealPadTree_ExampleDriftValidator. Verified by the negative
TestValidateExamples_DetectsTypoFlag.
make check clean. All 53 cmdhelp + cmd/pad tests pass.
Parent: PLAN-930.
* fix(cmdhelp): pass full token stream to cobra.Find per Codex review (round 1)
Codex round 1 caught: ValidateExamples stopped collecting the command
path at the first flag, so an example like
pad --workspace foo item create task --priority high
resolved to root, not `item create`. That meant `--priority` was
checked against root's flag set (where it doesn't exist) — false
positive — AND the validator silently missed any command-path drift
after a leading root flag.
Cobra's own Find walks the full token stream and uses each command's
flag definitions to skip flag/value pairs while matching subcommand
names. Pass tokens[1:] directly to root.Find — let cobra handle the
interleaving correctly.
New test:
- TestValidateExamples_FlagBeforeSubcommandResolvesToCorrectTarget —
flag-before-subcommand resolves to the leaf and accepts leaf flags.
116 lines
3.5 KiB
Go
116 lines
3.5 KiB
Go
package cmdhelp
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
|
|
"github.com/santhosh-tekuri/jsonschema/v6"
|
|
)
|
|
|
|
// loadSchemaForTest is a test-only wrapper around FindAndCompileSchema
|
|
// that fails the test on error and starts the walk from the test's CWD.
|
|
func loadSchemaForTest(t *testing.T) *jsonschema.Schema {
|
|
t.Helper()
|
|
cwd, err := os.Getwd()
|
|
if err != nil {
|
|
t.Fatalf("getwd: %v", err)
|
|
}
|
|
schema, err := FindAndCompileSchema(cwd)
|
|
if err != nil {
|
|
t.Fatalf("load schema: %v", err)
|
|
}
|
|
return schema
|
|
}
|
|
|
|
// validateAgainstSchema runs the compiled schema against the given JSON
|
|
// document and returns the validation error (nil on success).
|
|
func validateAgainstSchema(t *testing.T, schema *jsonschema.Schema, jsonBytes []byte) error {
|
|
t.Helper()
|
|
var doc interface{}
|
|
if err := json.Unmarshal(jsonBytes, &doc); err != nil {
|
|
t.Fatalf("unmarshal emitted JSON: %v", err)
|
|
}
|
|
return schema.Validate(doc)
|
|
}
|
|
|
|
func TestEmittedJSON_ValidatesAgainstSchema_StaticTree(t *testing.T) {
|
|
// Lock the static-tree path: emit the synthetic tree (no resolver,
|
|
// no dynamic enums) and assert the result satisfies every contract
|
|
// in cmdhelp.schema.json. This protects every TASK-934/935 emitter
|
|
// change going forward — break the schema contract and CI fails.
|
|
root := buildSyntheticTree()
|
|
var buf bytes.Buffer
|
|
if err := EmitJSON(root, root, Options{
|
|
Binary: "padtest",
|
|
Version: "test",
|
|
Homepage: "https://example.test",
|
|
MaxDepth: -1,
|
|
}, &buf); err != nil {
|
|
t.Fatalf("EmitJSON: %v", err)
|
|
}
|
|
|
|
schema := loadSchemaForTest(t)
|
|
if err := validateAgainstSchema(t, schema, buf.Bytes()); err != nil {
|
|
t.Errorf("synthetic tree's emitted JSON fails schema validation:\n%v\n--- output ---\n%s", err, buf.String())
|
|
}
|
|
}
|
|
|
|
func TestEmittedJSON_ValidatesAgainstSchema_AfterDynamicResolution(t *testing.T) {
|
|
// Same lock, post-resolver: dynamic enum injection must not break
|
|
// schema validity. Common regressions this guards against:
|
|
// - dynamic enum_source not matching `^dynamic:.+$`
|
|
// - non-numeric exit_code keys after some future templating
|
|
// - flag-name keys violating the propertyNames pattern
|
|
root := buildSyntheticTree()
|
|
r := &Resolver{
|
|
Workspace: "fixture",
|
|
ArgEnumSources: map[string]string{
|
|
"collection": EnumSourceCollections,
|
|
},
|
|
Sources: map[string]DynamicEnum{
|
|
EnumSourceCollections: func() ([]interface{}, error) {
|
|
return []interface{}{"tasks", "ideas"}, nil
|
|
},
|
|
},
|
|
}
|
|
var buf bytes.Buffer
|
|
if err := EmitJSON(root, root, Options{
|
|
Binary: "padtest",
|
|
MaxDepth: -1,
|
|
Resolver: r,
|
|
}, &buf); err != nil {
|
|
t.Fatalf("EmitJSON: %v", err)
|
|
}
|
|
|
|
schema := loadSchemaForTest(t)
|
|
if err := validateAgainstSchema(t, schema, buf.Bytes()); err != nil {
|
|
t.Errorf("post-resolver JSON fails schema validation:\n%v\n--- output ---\n%s", err, buf.String())
|
|
}
|
|
}
|
|
|
|
func TestEmittedJSON_ValidatesAgainstSchema_NoWorkspaceFallback(t *testing.T) {
|
|
// When dynamic resolution would fail (no workspace, no live values),
|
|
// the document must STILL validate against the schema. This is the
|
|
// graceful-fallback guarantee from TASK-936.
|
|
root := buildSyntheticTree()
|
|
r := &Resolver{
|
|
// No workspace, no sources — resolver is essentially a no-op
|
|
// shape but with the path set up for testing the fallback.
|
|
}
|
|
var buf bytes.Buffer
|
|
if err := EmitJSON(root, root, Options{
|
|
Binary: "padtest",
|
|
MaxDepth: -1,
|
|
Resolver: r,
|
|
}, &buf); err != nil {
|
|
t.Fatalf("EmitJSON: %v", err)
|
|
}
|
|
|
|
schema := loadSchemaForTest(t)
|
|
if err := validateAgainstSchema(t, schema, buf.Bytes()); err != nil {
|
|
t.Errorf("no-workspace fallback JSON fails schema validation:\n%v", err)
|
|
}
|
|
}
|