From 645236c1577d6639bd6d94bd84eb7812443a3861 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Tue, 15 Aug 2023 09:58:20 -0400
Subject: [PATCH 01/31] `assert.out.none` and `assert.err.none` to exec

Makes the assertion fields of the exec plugin's `assert` struct more
flexible and adds a new `assert.out.none` and `assert.err.none` field
that contains a string or strings that should *not* appear in either
stdout or stderr pipes from the exec command.

The `plugin/exec.PipeExpect` struct now contains three fields, each of
which are `types.FlexStrings` that can contain either a string or a list
of strings. There is now a custom YAML unmarshaler for `PipeExpect`
which allows a flexible naming convention to be used for specifying
assertions:

All of these examples are valid and represent human-readable assertions:

```yaml
tests:
  # Variants of contains
  - exec: ls -l
    assert:
      out:
        contains:
         - parse.go
  - exec: ls -l
    assert:
      out:
        is: parse.go
  - exec: ls -l
    assert:
      out:
        is:
         - parse.go
  - exec: ls -l
    assert:
      out:
        contains: parse.go
  - exec: ls -l
    assert:
      out:
        contains-all:
         - parse.go
  - exec: ls -l
    assert:
      out:
        all: parse.go
  # Variants of contains-any
  - exec: ls -l
    assert:
      out:
        any:
         - thisdoesnotexist
         - neitherdoesthisexist
         - parse.go
  - exec: ls -l
    assert:
      out:
        contains_one_of:
         - thisdoesnotexist
         - neitherdoesthisexist
         - parse.go
  - exec: ls -l
    assert:
      out:
        contains-any:
         - parse.go
  - exec: ls -l
    assert:
      out:
        any: parse.go
  # Variants of contains-none-of
  - exec: ls -l
    assert:
      out:
        none:
         - notexisting.go
  - exec: ls -l
    assert:
      out:
        none-of: notexisting.go
  - exec: ls -l
    assert:
      out:
        contains_none_of:
         - notexisting.go
  - exec: ls -l
    assert:
      out:
        none-of: notexisting.go
```

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 README.md                                     | 14 +++--
 errors/failure.go                             | 11 ++++
 plugin/exec/assertions.go                     | 54 +++++++++++--------
 plugin/exec/eval_test.go                      | 19 +++++++
 plugin/exec/parse.go                          | 50 ++++++++++++++++-
 plugin/exec/testdata/ls-contains-none-of.yaml | 30 +++++++++++
 plugin/exec/testdata/ls-contains-one-of.yaml  | 19 ++++++-
 plugin/exec/testdata/ls-contains.yaml         | 23 ++++++++
 plugin/exec/testdata/ls-with-exit-code.yaml   |  2 +-
 .../exec/testdata/mac-ls-with-exit-code.yaml  |  2 +-
 .../exec/testdata/windows-sleep-timeout.yaml  |  2 +-
 11 files changed, 194 insertions(+), 32 deletions(-)
 create mode 100644 plugin/exec/testdata/ls-contains-none-of.yaml

diff --git a/README.md b/README.md
index f658daf..62e8eeb 100644
--- a/README.md
+++ b/README.md
@@ -491,25 +491,29 @@ the base `Spec` fields listed above):
   instead the operating system's `exec` family of calls is used.
 * `assert`: (optional) an object describing the conditions that will be
   asserted about the test action.
-* `assert.exit_code`: (optional) an integer with the expected exit code from the
+* `assert.exit-code`: (optional) an integer with the expected exit code from the
   executed command. The default successful exit code is 0 and therefore you do
   not need to specify this if you expect a successful exit code.
 * `assert.out`: (optional) a [`PipeExpect`][pipeexpect] object containing
   assertions about content in `stdout`.
 * `assert.out.is`: (optional) a string with the exact contents of `stdout` you expect
   to get.
-* `assert.out.contains`: (optional) a list of one or more strings that *all* must be
+* `assert.out.all`: (optional) a string or list of strings that *all* must be
   present in `stdout`.
-* `assert.out.contains_one_of`: (optional) a list of one or more strings of which *at
+* `assert.out.any`: (optional) a string or list of strings of which *at
   least one* must be present in `stdout`.
+* `assert.out.none`: (optional) a string or list of strings of which *none
+  should be present* in `stdout`.
 * `assert.err`: (optional) a [`PipeAssertions`][pipeexpect] object containing
   assertions about content in `stderr`.
 * `assert.err.is`: (optional) a string with the exact contents of `stderr` you expect
   to get.
-* `assert.err.contains`: (optional) a list of one or more strings that *all* must be
+* `assert.err.all`: (optional) a string or list of strings that *all* must be
   present in `stderr`.
-* `assert.err.contains_one_of`: (optional) a list of one or more strings of which *at
+* `assert.err.any`: (optional) a string or list of strings of which *at
   least one* must be present in `stderr`.
+* `assert.err.none`: (optional) a string or list of strings of which *none
+  should be present* in `stderr`.
 * `on`: (optional) an object describing actions to take upon certain
   conditions.
 * `on.fail`: (optional) an object describing an action to take when any
diff --git a/errors/failure.go b/errors/failure.go
index a16d42c..10b12f9 100644
--- a/errors/failure.go
+++ b/errors/failure.go
@@ -19,6 +19,9 @@ var (
 	// ErrNotEqual is an ErrFailure when an expected thing doesn't equal an
 	// observed thing.
 	ErrNotEqual = fmt.Errorf("%w: not equal", ErrFailure)
+	// ErrIn is an ErrFailure when a thing unexpectedly appears in an
+	// container.
+	ErrIn = fmt.Errorf("%w: in", ErrFailure)
 	// ErrNotIn is an ErrFailure when an expected thing doesn't appear in an
 	// expected container.
 	ErrNotIn = fmt.Errorf("%w: not in", ErrFailure)
@@ -58,6 +61,14 @@ func NotEqual(exp, got interface{}) error {
 	return fmt.Errorf("%w: expected %v but got %v", ErrNotEqual, exp, got)
 }
 
+// In returns an ErrIn when a thing unexpectedly appears in a container.
+func In(element, container interface{}) error {
+	return fmt.Errorf(
+		"%w: expected %v not to contain %v",
+		ErrIn, container, element,
+	)
+}
+
 // NotIn returns an ErrNotIn when an expected thing doesn't appear in an
 // expected container.
 func NotIn(element, container interface{}) error {
diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go
index b6738bc..330b34c 100644
--- a/plugin/exec/assertions.go
+++ b/plugin/exec/assertions.go
@@ -17,7 +17,7 @@ type Expect struct {
 	// ExitCode is the expected exit code for the executed command. The default
 	// (0) is the universal successful exit code, so you only need to set this
 	// if you expect a non-successful result from executing the command.
-	ExitCode int `yaml:"exit_code,omitempty"`
+	ExitCode int `yaml:"exit-code,omitempty"`
 	// Out has things that are expected in the stdout response
 	Out *PipeExpect `yaml:"out,omitempty"`
 	// Err has things that are expected in the stderr response
@@ -26,15 +26,15 @@ type Expect struct {
 
 // PipeExpect contains assertions about the contents of a pipe
 type PipeExpect struct {
-	// Is contains the exact match (minus whitespace) of the contents of the
-	// pipe
-	Is *string `yaml:"is,omitempty"`
-	// Contains is one or more strings that *all* must be present in the
+	// ContainsAll is one or more strings that *all* must be present in the
 	// contents of the pipe
-	Contains []string `yaml:"contains,omitempty"`
+	ContainsAll *gdttypes.FlexStrings `yaml:"contains,omitempty"`
+	// ContainsNone is one or more strings, *none of which* should be present in
+	// the contents of the pipe
+	ContainsNone *gdttypes.FlexStrings `yaml:"contains-none-of,omitempty"`
 	// ContainsOneOf is one or more strings of which *at least one* must be
 	// present in the contents of the pipe
-	ContainsOneOf []string `yaml:"contains_one_of,omitempty"`
+	ContainsAny *gdttypes.FlexStrings `yaml:"contains-one-of,omitempty"`
 }
 
 // pipeAssertions contains assertions about the contents of a pipe
@@ -83,35 +83,45 @@ func (a *pipeAssertions) OK() bool {
 
 	res := true
 	contents := strings.TrimSpace(a.pipe.String())
-	if a.Is != nil {
-		exp := *a.Is
-		got := contents
-		if exp != got {
-			a.Fail(errors.NotEqual(exp, got))
-			res = false
-		}
-	}
-	if len(a.Contains) > 0 {
-		for _, find := range a.Contains {
-			if !strings.Contains(contents, find) {
-				a.Fail(errors.NotIn(find, a.name))
+	if a.ContainsAll != nil {
+		// When there is just a single value, we use the NotEqual error,
+		// otherwise we use the NotIn error
+		vals := a.ContainsAll.Values()
+		if len(vals) == 1 {
+			if !strings.Contains(contents, vals[0]) {
+				a.Fail(errors.NotEqual(vals[0], contents))
 				res = false
 			}
+		} else {
+			for _, find := range vals {
+				if !strings.Contains(contents, find) {
+					a.Fail(errors.NotIn(find, a.name))
+					res = false
+				}
+			}
 		}
 	}
-	if len(a.ContainsOneOf) > 0 {
+	if a.ContainsAny != nil {
 		found := false
-		for _, find := range a.ContainsOneOf {
+		for _, find := range a.ContainsAny.Values() {
 			if idx := strings.Index(contents, find); idx > -1 {
 				found = true
 				break
 			}
 		}
 		if !found {
-			a.Fail(errors.NoneIn(a.ContainsOneOf, a.name))
+			a.Fail(errors.NoneIn(a.ContainsAny.Values(), a.name))
 			res = false
 		}
 	}
+	if a.ContainsNone != nil {
+		for _, find := range a.ContainsNone.Values() {
+			if strings.Contains(contents, find) {
+				a.Fail(errors.In(find, a.name))
+				res = false
+			}
+		}
+	}
 	return res
 }
 
diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index 96ed19c..b598aaf 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -138,6 +138,25 @@ func TestContainsOneOf(t *testing.T) {
 	require.Nil(err)
 }
 
+func TestContainsNoneOf(t *testing.T) {
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "ls-contains-none-of.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := context.TODO()
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
 func TestSleepTimeout(t *testing.T) {
 	require := require.New(t)
 
diff --git a/plugin/exec/parse.go b/plugin/exec/parse.go
index ff83016..939104e 100644
--- a/plugin/exec/parse.go
+++ b/plugin/exec/parse.go
@@ -116,7 +116,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
-		case "exit_code":
+		case "exit_code", "exit-code":
 			if valNode.Kind != yaml.ScalarNode {
 				return errors.ExpectedScalarAt(valNode)
 			}
@@ -149,3 +149,51 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 	}
 	return nil
 }
+
+func (e *PipeExpect) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return errors.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return errors.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "all", "is", "contains", "contains-all", "contains_all":
+			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
+				return errors.ExpectedScalarOrSequenceAt(valNode)
+			}
+			var v gdttypes.FlexStrings
+			if err := valNode.Decode(&v); err != nil {
+				return err
+			}
+			e.ContainsAll = &v
+		case "any", "contains-one-of", "contains-any", "contains_one_of", "contains_any":
+			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
+				return errors.ExpectedScalarOrSequenceAt(valNode)
+			}
+			var v gdttypes.FlexStrings
+			if err := valNode.Decode(&v); err != nil {
+				return err
+			}
+			e.ContainsAny = &v
+		case "none", "none-of", "contains-none-of", "contains-none", "none_of", "contains_none_of", "contains_none":
+			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
+				return errors.ExpectedScalarOrSequenceAt(valNode)
+			}
+			var v gdttypes.FlexStrings
+			if err := valNode.Decode(&v); err != nil {
+				return err
+			}
+			e.ContainsNone = &v
+		default:
+			return errors.UnknownFieldAt(key, keyNode)
+		}
+	}
+	return nil
+}
diff --git a/plugin/exec/testdata/ls-contains-none-of.yaml b/plugin/exec/testdata/ls-contains-none-of.yaml
new file mode 100644
index 0000000..d517f0c
--- /dev/null
+++ b/plugin/exec/testdata/ls-contains-none-of.yaml
@@ -0,0 +1,30 @@
+name: ls-contains-none-of
+description: a scenario that runs the `ls` command and checks the output does not contain a string
+tests:
+  - exec: ls -l
+    assert:
+      out:
+        none:
+         - notexisting.go
+  # Variants of contains-none-of
+  - exec: ls -l
+    assert:
+      out:
+        none-of: notexisting.go
+  - exec: ls -l
+    assert:
+      out:
+        contains_none_of:
+         - notexisting.go
+  - exec: ls -l
+    assert:
+      out:
+        none-of: notexisting.go
+  # To test the stderr assertions, we redirect stdout to stderr in a shell
+  # command...
+  - exec: "ls -l 1>&2"
+    shell: sh
+    assert:
+      err:
+        none:
+         - notexisting.go
diff --git a/plugin/exec/testdata/ls-contains-one-of.yaml b/plugin/exec/testdata/ls-contains-one-of.yaml
index 559af9a..37ef84a 100644
--- a/plugin/exec/testdata/ls-contains-one-of.yaml
+++ b/plugin/exec/testdata/ls-contains-one-of.yaml
@@ -1,6 +1,14 @@
 name: ls-contains-one-of
 description: a scenario that runs the `ls` command and checks the output contains one of a set of strings
 tests:
+  - exec: ls -l
+    assert:
+      out:
+        any:
+         - thisdoesnotexist
+         - neitherdoesthisexist
+         - parse.go
+  # Variants of contains-any
   - exec: ls -l
     assert:
       out:
@@ -8,13 +16,22 @@ tests:
          - thisdoesnotexist
          - neitherdoesthisexist
          - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        contains-any:
+         - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        any: parse.go
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "ls -l 1>&2"
     shell: sh
     assert:
       err:
-        contains_one_of:
+        any:
          - thisdoesnotexist
          - neitherdoesthisexist
          - parse.go
diff --git a/plugin/exec/testdata/ls-contains.yaml b/plugin/exec/testdata/ls-contains.yaml
index 1181fbf..2f6d3d8 100644
--- a/plugin/exec/testdata/ls-contains.yaml
+++ b/plugin/exec/testdata/ls-contains.yaml
@@ -6,6 +6,29 @@ tests:
       out:
         contains:
          - parse.go
+  # Variants of contains-all
+  - exec: ls -l
+    assert:
+      out:
+        is: parse.go
+  - exec: ls -l
+    assert:
+      out:
+        is:
+         - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        contains: parse.go
+  - exec: ls -l
+    assert:
+      out:
+        contains-all:
+         - parse.go
+  - exec: ls -l
+    assert:
+      out:
+        all: parse.go
   # To test the stderr assertions, we redirect stdout to stderr in a shell
   # command...
   - exec: "ls -l 1>&2"
diff --git a/plugin/exec/testdata/ls-with-exit-code.yaml b/plugin/exec/testdata/ls-with-exit-code.yaml
index 3fac80f..7fe4679 100644
--- a/plugin/exec/testdata/ls-with-exit-code.yaml
+++ b/plugin/exec/testdata/ls-with-exit-code.yaml
@@ -3,4 +3,4 @@ description: a scenario that runs the `ls` command expecting a non-0 exit code
 tests:
   - exec: ls /this/dir/does/not/exist
     assert:
-      exit_code: 2
+      exit-code: 2
diff --git a/plugin/exec/testdata/mac-ls-with-exit-code.yaml b/plugin/exec/testdata/mac-ls-with-exit-code.yaml
index 8cee8ac..d1915b5 100644
--- a/plugin/exec/testdata/mac-ls-with-exit-code.yaml
+++ b/plugin/exec/testdata/mac-ls-with-exit-code.yaml
@@ -3,4 +3,4 @@ description: a scenario that runs the `ls` command expecting a non-0 exit code o
 tests:
   - exec: ls /this/dir/does/not/exist
     assert:
-      exit_code: 1
+      exit-code: 1
diff --git a/plugin/exec/testdata/windows-sleep-timeout.yaml b/plugin/exec/testdata/windows-sleep-timeout.yaml
index 2dce257..d85b8ca 100644
--- a/plugin/exec/testdata/windows-sleep-timeout.yaml
+++ b/plugin/exec/testdata/windows-sleep-timeout.yaml
@@ -11,4 +11,4 @@ tests:
     # the context's deadline cancels the pipe and results in a 1 result
     # code on Windows...
     assert:
-      exit_code: 1
+      exit-code: 1

From c1bed53f824da3113b8f24bf58a4eed2bcdf8733 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Tue, 15 Aug 2023 11:00:28 -0400
Subject: [PATCH 02/31] remove unused Assertions.Terminal() interface

Now that the `Scenario.Run()` and `Evaluable.Eval()` split has happened,
the need for checking a terminal condition in the assertions is gone, so
removing this now-unneeded interface method.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 assertion/json/json.go      | 19 -------------------
 assertion/json/json_test.go | 10 ----------
 types/assertions.go         |  4 ----
 3 files changed, 33 deletions(-)

diff --git a/assertion/json/json.go b/assertion/json/json.go
index 1f84182..b614c21 100644
--- a/assertion/json/json.go
+++ b/assertion/json/json.go
@@ -152,10 +152,6 @@ func New(
 type assertions struct {
 	// failures contains the set of error messages for failed assertions
 	failures []error
-	// terminal indicates there was a failure in evaluating the assertions that
-	// should be considered a terminal condition (and therefore the test action
-	// should not be retried).
-	terminal bool
 	// exp contains the expected conditions for to be asserted
 	exp *Expect
 	// content is the JSON content we will check
@@ -173,13 +169,6 @@ func (a *assertions) Failures() []error {
 	return a.failures
 }
 
-// Terminal returns true if re-executing the assertions against the same result
-// would be pointless. This allows assertions to inform the Spec that retrying
-// the same operation would not be necessary.
-func (a *assertions) Terminal() bool {
-	return a.terminal
-}
-
 // OK returns true if all contained assertions pass successfully
 func (a *assertions) OK() bool {
 	if a == nil || a.exp == nil {
@@ -229,7 +218,6 @@ func (a *assertions) pathsOK() bool {
 	v := interface{}(nil)
 	if err := json.Unmarshal(a.content, &v); err != nil {
 		a.Fail(JSONUnmarshalError(err, nil))
-		a.terminal = true
 		return false
 	}
 	for path, expVal := range a.exp.Paths {
@@ -250,7 +238,6 @@ func (a *assertions) pathsOK() bool {
 			expValInt, err := strconv.Atoi(expVal)
 			if err != nil {
 				a.Fail(JSONPathConversionError(path, expVal, got))
-				a.terminal = true
 				return false
 			}
 			if expValInt != got.(int) {
@@ -261,7 +248,6 @@ func (a *assertions) pathsOK() bool {
 			expValFloat, err := strconv.ParseFloat(expVal, 64)
 			if err != nil {
 				a.Fail(JSONPathConversionError(path, expVal, got))
-				a.terminal = true
 				return false
 			}
 			if expValFloat != got.(float64) {
@@ -272,7 +258,6 @@ func (a *assertions) pathsOK() bool {
 			expValBool, err := strconv.ParseBool(expVal)
 			if err != nil {
 				a.Fail(JSONPathConversionError(path, expVal, got))
-				a.terminal = true
 				return false
 			}
 			if expValBool != got.(bool) {
@@ -281,7 +266,6 @@ func (a *assertions) pathsOK() bool {
 			}
 		default:
 			a.Fail(JSONPathConversionError(path, expVal, got))
-			a.terminal = true
 			return false
 		}
 	}
@@ -300,7 +284,6 @@ func (a *assertions) pathFormatsOK() bool {
 	v := interface{}(nil)
 	if e := json.Unmarshal(a.content, &v); e != nil {
 		a.Fail(JSONUnmarshalError(e, nil))
-		a.terminal = true
 		return false
 	}
 	for path, format := range a.exp.PathFormats {
@@ -314,7 +297,6 @@ func (a *assertions) pathFormatsOK() bool {
 		ok, err := isFormatted(format, got)
 		if err != nil {
 			a.Fail(JSONFormatError(format, err))
-			a.terminal = true
 			return false
 		}
 		if !ok {
@@ -342,7 +324,6 @@ func (a *assertions) schemaOK() bool {
 	res, err := gjs.Validate(schemaLoader, docLoader)
 	if err != nil {
 		a.Fail(JSONSchemaValidateError(schemaPath, err))
-		a.terminal = true
 		return false
 	}
 
diff --git a/assertion/json/json_test.go b/assertion/json/json_test.go
index dbd5755..f914568 100644
--- a/assertion/json/json_test.go
+++ b/assertion/json/json_test.go
@@ -107,13 +107,11 @@ func TestLength(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.True(a.OK())
-	require.False(a.Terminal())
 	require.Empty(a.Failures())
 
 	expLen = 0
 	a = gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.False(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdterrors.ErrNotEqual)
@@ -132,7 +130,6 @@ func TestJSONUnmarshalError(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.True(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONUnmarshalError)
@@ -151,7 +148,6 @@ func TestJSONPathError(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.False(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathNotFound)
@@ -170,7 +166,6 @@ func TestJSONPathConversionError(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.True(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathConversionError)
@@ -189,7 +184,6 @@ func TestJSONPathNotEqual(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.True(a.OK())
-	require.False(a.Terminal())
 	require.Empty(a.Failures())
 
 	exp = gdtjson.Expect{
@@ -200,7 +194,6 @@ func TestJSONPathNotEqual(t *testing.T) {
 
 	a = gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.False(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathNotEqual)
@@ -219,7 +212,6 @@ func TestJSONPathFormatNotFound(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.False(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathNotFound)
@@ -238,7 +230,6 @@ func TestJSONPathFormatNotEqual(t *testing.T) {
 
 	a := gdtjson.New(&exp, c)
 	require.True(a.OK())
-	require.False(a.Terminal())
 	require.Empty(a.Failures())
 
 	exp = gdtjson.Expect{
@@ -249,7 +240,6 @@ func TestJSONPathFormatNotEqual(t *testing.T) {
 
 	a = gdtjson.New(&exp, c)
 	require.False(a.OK())
-	require.False(a.Terminal())
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONFormatNotEqual)
diff --git a/types/assertions.go b/types/assertions.go
index 787260f..dbb51f7 100644
--- a/types/assertions.go
+++ b/types/assertions.go
@@ -15,8 +15,4 @@ type Assertions interface {
 	// Failures returns a slice of failure messages indicating which assertions
 	// did not succeed.
 	Failures() []error
-	// Terminal returns true if re-executing the Assertions against the same
-	// result would be pointless. This allows Assertions to inform the Spec
-	// that retrying the same operation would not be necessary.
-	Terminal() bool
 }

From 22000f746472418f7fd2e5022009da8d41012fc6 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sun, 18 Feb 2024 18:27:21 -0500
Subject: [PATCH 03/31] exec: fix debug handler

For the exec plugin, we were writing debug output even when the user had
not called `gdtcontext.WithDebug()`. In addition, stdout and stderr
contents were only being written to debug writers when executed during
`on.fail.exec`. These two problems are fixed in this commit in addition
to correcting an erroneous `$LOCATION` in the README.md that needed to
be `$$LOCATION`.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 .github/workflows/gate-tests.yml |  9 +++++++--
 README.md                        |  2 +-
 debug/print.go                   | 10 ++--------
 plugin/exec/action.go            |  6 ++++++
 plugin/exec/eval.go              | 10 ----------
 plugin/exec/eval_test.go         |  2 ++
 scenario/run_test.go             | 13 ++++++++++---
 7 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/gate-tests.yml b/.github/workflows/gate-tests.yml
index 6553a6c..dcc1c66 100644
--- a/.github/workflows/gate-tests.yml
+++ b/.github/workflows/gate-tests.yml
@@ -13,7 +13,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x]
+        go: ['1.19', '1.20', '1.21']
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -24,11 +24,16 @@ jobs:
          disable-sudo: true
          allowed-endpoints: >
            github.com:443
+           api.github.com:443
+           proxy.github.com:443
+           raw.githubusercontent.com:443
+           objects.githubusercontent.com:443
            proxy.golang.org:443
      - name: checkout code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
      - name: setup go
-       uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+       uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
          go-version: ${{ matrix.go }}
+         check-latest: true
      - run: go test -v ./...
diff --git a/README.md b/README.md
index 62e8eeb..016933a 100644
--- a/README.md
+++ b/README.md
@@ -402,7 +402,7 @@ tests:
      headers:
       - Location
  - name: look up that created book
-   GET: $LOCATION
+   GET: $$LOCATION
    response:
      status: 200
      json:
diff --git a/debug/print.go b/debug/print.go
index c26c2bf..87567de 100644
--- a/debug/print.go
+++ b/debug/print.go
@@ -23,13 +23,10 @@ func Printf(
 ) {
 	t.Helper()
 	writers := gdtcontext.Debug(ctx)
-	if writers == nil {
-		return
-	}
-	t.Logf(format, args...)
 	if len(writers) == 0 {
 		return
 	}
+	t.Logf(format, args...)
 
 	if !strings.HasPrefix(format, "[gdt] ") {
 		format = "[gdt] " + t.Name() + " " + format
@@ -50,14 +47,11 @@ func Println(
 ) {
 	t.Helper()
 	writers := gdtcontext.Debug(ctx)
-	if writers == nil {
+	if len(writers) == 0 {
 		return
 	}
 	// NOTE(jaypipes): T.Logf() automatically adds newlines...
 	t.Logf(format, args...)
-	if len(writers) == 0 {
-		return
-	}
 
 	if !strings.HasPrefix(format, "[gdt] ") {
 		format = "[gdt] " + t.Name() + " " + format
diff --git a/plugin/exec/action.go b/plugin/exec/action.go
index 9884972..cadd4b8 100644
--- a/plugin/exec/action.go
+++ b/plugin/exec/action.go
@@ -79,9 +79,15 @@ func (a *Action) Do(
 	}
 	if outbuf != nil {
 		outbuf.ReadFrom(outpipe)
+		if outbuf.Len() > 0 {
+			debug.Println(ctx, t, "exec: stdout: %s", outbuf.String())
+		}
 	}
 	if errbuf != nil {
 		errbuf.ReadFrom(errpipe)
+		if errbuf.Len() > 0 {
+			debug.Println(ctx, t, "exec: stderr: %s", errbuf.String())
+		}
 	}
 
 	err = cmd.Wait()
diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go
index 243680f..34c7fd7 100644
--- a/plugin/exec/eval.go
+++ b/plugin/exec/eval.go
@@ -42,16 +42,6 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 				if err != nil {
 					debug.Println(ctx, t, "error in on.fail.exec: %s", err)
 				}
-				if outbuf.Len() > 0 {
-					debug.Println(
-						ctx, t, "on.fail.exec: stdout: %s", outbuf.String(),
-					)
-				}
-				if errbuf.Len() > 0 {
-					debug.Println(
-						ctx, t, "on.fail.exec: stderr: %s", errbuf.String(),
-					)
-				}
 			}
 		}
 	}
diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index b598aaf..4fc024a 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -199,7 +199,9 @@ func TestDebugWriter(t *testing.T) {
 	require.NotEqual(b.Len(), 0)
 	debugout := b.String()
 	require.Contains(debugout, "exec: echo [cat]")
+	require.Contains(debugout, "exec: stdout: cat")
 	require.Contains(debugout, "exec: sh [-c echo cat 1>&2]")
+	require.Contains(debugout, "exec: stderr: cat")
 }
 
 func TestWait(t *testing.T) {
diff --git a/scenario/run_test.go b/scenario/run_test.go
index a114602..722313e 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -5,6 +5,8 @@
 package scenario_test
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"os"
 	"path/filepath"
@@ -72,9 +74,9 @@ func TestDebugFlushing(t *testing.T) {
 	f, err := os.Open(fp)
 	require.Nil(err)
 
-	ctx := gdtcontext.New(
-		gdtcontext.WithDebug(),
-	)
+	var b bytes.Buffer
+	w := bufio.NewWriter(&b)
+	ctx := gdtcontext.New(gdtcontext.WithDebug(w))
 
 	s, err := scenario.FromReader(f, scenario.WithPath(fp))
 	require.Nil(err)
@@ -82,6 +84,11 @@ func TestDebugFlushing(t *testing.T) {
 
 	err = s.Run(ctx, t)
 	require.Nil(err)
+	require.False(t.Failed())
+	w.Flush()
+	require.NotEqual(b.Len(), 0)
+	debugout := b.String()
+	require.Contains(debugout, "TestDebugFlushing/foo-debug-wait-flush wait: 250ms before")
 }
 
 func TestTimeoutCascade(t *testing.T) {

From 32058882f24c2a0adca7d3cb7b7ebc8ca717d90c Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Mon, 27 May 2024 08:20:03 -0400
Subject: [PATCH 04/31] add context.Context arg to assertions.OK()

**BREAKING API CHANGE for plugin developers**

This changes the `gdt.Assertions.OK()` interface signature to accept a
`context.Context` object. This context object is required for debug
output/inspection in the kube plugin.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 assertion/json/json.go      |  3 ++-
 assertion/json/json_test.go | 28 ++++++++++++++++++----------
 plugin/exec/assertions.go   |  9 +++++----
 plugin/exec/eval.go         |  2 +-
 types/assertions.go         |  4 +++-
 5 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/assertion/json/json.go b/assertion/json/json.go
index b614c21..7a60584 100644
--- a/assertion/json/json.go
+++ b/assertion/json/json.go
@@ -5,6 +5,7 @@
 package json
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -170,7 +171,7 @@ func (a *assertions) Failures() []error {
 }
 
 // OK returns true if all contained assertions pass successfully
-func (a *assertions) OK() bool {
+func (a *assertions) OK(ctx context.Context) bool {
 	if a == nil || a.exp == nil {
 		return true
 	}
diff --git a/assertion/json/json_test.go b/assertion/json/json_test.go
index f914568..3eed0e2 100644
--- a/assertion/json/json_test.go
+++ b/assertion/json/json_test.go
@@ -5,6 +5,7 @@
 package json_test
 
 import (
+	"context"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -98,6 +99,7 @@ func content() []byte {
 func TestLength(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := content()
 	expLen := len(c)
 
@@ -106,12 +108,12 @@ func TestLength(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.True(a.OK())
+	require.True(a.OK(ctx))
 	require.Empty(a.Failures())
 
 	expLen = 0
 	a = gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdterrors.ErrNotEqual)
@@ -120,6 +122,7 @@ func TestLength(t *testing.T) {
 func TestJSONUnmarshalError(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := []byte(`not { value } json`)
 
 	exp := gdtjson.Expect{
@@ -129,7 +132,7 @@ func TestJSONUnmarshalError(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONUnmarshalError)
@@ -138,6 +141,7 @@ func TestJSONUnmarshalError(t *testing.T) {
 func TestJSONPathError(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := content()
 
 	exp := gdtjson.Expect{
@@ -147,7 +151,7 @@ func TestJSONPathError(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathNotFound)
@@ -156,6 +160,7 @@ func TestJSONPathError(t *testing.T) {
 func TestJSONPathConversionError(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := content()
 
 	exp := gdtjson.Expect{
@@ -165,7 +170,7 @@ func TestJSONPathConversionError(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathConversionError)
@@ -174,6 +179,7 @@ func TestJSONPathConversionError(t *testing.T) {
 func TestJSONPathNotEqual(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := content()
 
 	exp := gdtjson.Expect{
@@ -183,7 +189,7 @@ func TestJSONPathNotEqual(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.True(a.OK())
+	require.True(a.OK(ctx))
 	require.Empty(a.Failures())
 
 	exp = gdtjson.Expect{
@@ -193,7 +199,7 @@ func TestJSONPathNotEqual(t *testing.T) {
 	}
 
 	a = gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathNotEqual)
@@ -202,6 +208,7 @@ func TestJSONPathNotEqual(t *testing.T) {
 func TestJSONPathFormatNotFound(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := content()
 
 	exp := gdtjson.Expect{
@@ -211,7 +218,7 @@ func TestJSONPathFormatNotFound(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONPathNotFound)
@@ -220,6 +227,7 @@ func TestJSONPathFormatNotFound(t *testing.T) {
 func TestJSONPathFormatNotEqual(t *testing.T) {
 	require := require.New(t)
 
+	ctx := context.TODO()
 	c := content()
 
 	exp := gdtjson.Expect{
@@ -229,7 +237,7 @@ func TestJSONPathFormatNotEqual(t *testing.T) {
 	}
 
 	a := gdtjson.New(&exp, c)
-	require.True(a.OK())
+	require.True(a.OK(ctx))
 	require.Empty(a.Failures())
 
 	exp = gdtjson.Expect{
@@ -239,7 +247,7 @@ func TestJSONPathFormatNotEqual(t *testing.T) {
 	}
 
 	a = gdtjson.New(&exp, c)
-	require.False(a.OK())
+	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
 	require.ErrorIs(failures[0], gdtjson.ErrJSONFormatNotEqual)
diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go
index 330b34c..b72553a 100644
--- a/plugin/exec/assertions.go
+++ b/plugin/exec/assertions.go
@@ -6,6 +6,7 @@ package exec
 
 import (
 	"bytes"
+	"context"
 	"strings"
 
 	"github.com/gdt-dev/gdt/errors"
@@ -76,7 +77,7 @@ func (a *pipeAssertions) Terminal() bool {
 
 // OK checks all the assertions in the pipeAssertions against the supplied pipe
 // contents and returns true if all assertions pass.
-func (a *pipeAssertions) OK() bool {
+func (a *pipeAssertions) OK(ctx context.Context) bool {
 	if a == nil || a.pipe == nil {
 		return true
 	}
@@ -167,17 +168,17 @@ func (a *assertions) Terminal() bool {
 
 // OK checks all the assertions against the supplied arguments and returns true
 // if all assertions pass.
-func (a *assertions) OK() bool {
+func (a *assertions) OK(ctx context.Context) bool {
 	res := true
 	if a.expExitCode != a.exitCode {
 		a.Fail(errors.NotEqual(a.expExitCode, a.exitCode))
 		res = false
 	}
-	if !a.expOutPipe.OK() {
+	if !a.expOutPipe.OK(ctx) {
 		a.failures = append(a.failures, a.expOutPipe.Failures()...)
 		res = false
 	}
-	if !a.expErrPipe.OK() {
+	if !a.expErrPipe.OK(ctx) {
 		a.failures = append(a.failures, a.expErrPipe.Failures()...)
 		res = false
 	}
diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go
index 34c7fd7..565ab16 100644
--- a/plugin/exec/eval.go
+++ b/plugin/exec/eval.go
@@ -30,7 +30,7 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
 	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
-	if !a.OK() {
+	if !a.OK(ctx) {
 		for _, fail := range a.Failures() {
 			t.Error(fail)
 		}
diff --git a/types/assertions.go b/types/assertions.go
index dbb51f7..510d436 100644
--- a/types/assertions.go
+++ b/types/assertions.go
@@ -4,12 +4,14 @@
 
 package types
 
+import "context"
+
 // Assertions track zero or more assertions about some result
 type Assertions interface {
 	// OK returns true if all contained assertions pass successfully, false
 	// otherwise. If false is returned, Failures() is guaranteed to be
 	// non-empty.
-	OK() bool
+	OK(context.Context) bool
 	// Fail appends a supplied error to the set of failed assertions
 	Fail(error)
 	// Failures returns a slice of failure messages indicating which assertions

From 1886a278353d4a794e1daae2a96f7955a35dc601 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sat, 1 Jun 2024 07:50:23 -0400
Subject: [PATCH 05/31] rework debug/print into a tracing context

Changes the way that `gdtcontext.WithDebug()` and `gdtdebug.Printf/ln`
work to remove the dependence on a `*testing.T` argument and using a
stack concept for tracing where in the test evaluation we are.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 context/context.go            | 34 +++++++++++++++++++++++++++++-----
 context/getter.go             | 31 +++++++++++++++++++++++++++++--
 debug/print.go                | 16 ++++++----------
 plugin/exec/action.go         |  6 +++---
 plugin/exec/assertions.go     | 26 --------------------------
 plugin/exec/eval.go           |  2 +-
 scenario/run.go               | 12 ++++++++----
 scenario/run_test.go          |  2 +-
 scenario/stub_plugins_test.go |  6 +++++-
 9 files changed, 82 insertions(+), 53 deletions(-)

diff --git a/context/context.go b/context/context.go
index d666f45..6d7da7c 100644
--- a/context/context.go
+++ b/context/context.go
@@ -16,6 +16,7 @@ type ContextKey string
 
 var (
 	debugKey    = ContextKey("gdt.debug")
+	traceKey    = ContextKey("gdt.trace")
 	pluginsKey  = ContextKey("gdt.plugins")
 	fixturesKey = ContextKey("gdt.fixtures")
 	priorRunKey = ContextKey("gdt.run.prior")
@@ -41,11 +42,11 @@ type ContextModifier func(context.Context) context.Context
 //		   require.Nil(err)
 //
 //		   ctx := gdtcontext.New(gdtcontext.WithDebug())
-//	    s, err := scenario.FromReader(
-//	        f,
-//	        scenario.WithPath(fp),
-//	        scenario.WithContext(ctx),
-//	    )
+//	       s, err := scenario.FromReader(
+//	          f,
+//	          scenario.WithPath(fp),
+//	          scenario.WithContext(ctx),
+//	       )
 //		   require.Nil(err)
 //		   require.NotNil(s)
 //
@@ -154,6 +155,29 @@ func StorePriorRun(
 	return context.WithValue(ctx, priorRunKey, merged)
 }
 
+// PushTrace pushes a debug/trace name onto the debug/trace stack. It is used
+// by plugins to track where in the processing of a test or assertion the
+// plugin is and gets output at the start of a debug.Printf/Println message.
+func PushTrace(
+	ctx context.Context,
+	name string,
+) context.Context {
+	stack := TraceStack(ctx)
+	stack = append(stack, name)
+	return context.WithValue(ctx, traceKey, stack)
+}
+
+// PopTrace pops the last name off the debug/trace stack. It is used by plugins
+// to track where in the processing of a test or assertion the plugin is and
+// gets output at the start of a debug.Printf/Println message.
+func PopTrace(
+	ctx context.Context,
+) context.Context {
+	stack := TraceStack(ctx)
+	stack = stack[:len(stack)-1]
+	return context.WithValue(ctx, traceKey, stack)
+}
+
 // New returns a new Context
 func New(mods ...ContextModifier) context.Context {
 	ctx := context.TODO()
diff --git a/context/getter.go b/context/getter.go
index 019b93c..f4f666a 100644
--- a/context/getter.go
+++ b/context/getter.go
@@ -7,19 +7,46 @@ package context
 import (
 	"context"
 	"io"
+	"strings"
 
 	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
+const (
+	traceDelimiter = "/"
+)
+
+// Trace gets a context's trace name stack joined together with
+func Trace(ctx context.Context) string {
+	if ctx == nil {
+		return ""
+	}
+	if v := ctx.Value(traceKey); v != nil {
+		return strings.Join(v.([]string), traceDelimiter)
+	}
+	return ""
+}
+
+// TraceStack gets a context's trace name stack
+func TraceStack(ctx context.Context) []string {
+	if ctx == nil {
+		return []string{}
+	}
+	if v := ctx.Value(traceKey); v != nil {
+		return v.([]string)
+	}
+	return []string{}
+}
+
 // Debug gets a context's Debug writer
 func Debug(ctx context.Context) []io.Writer {
 	if ctx == nil {
-		return nil
+		return []io.Writer{}
 	}
 	if v := ctx.Value(debugKey); v != nil {
 		return v.([]io.Writer)
 	}
-	return nil
+	return []io.Writer{}
 }
 
 // Plugins gets a context's Plugins
diff --git a/debug/print.go b/debug/print.go
index 87567de..8aa9d28 100644
--- a/debug/print.go
+++ b/debug/print.go
@@ -8,7 +8,6 @@ import (
 	"context"
 	"fmt"
 	"strings"
-	"testing"
 
 	gdtcontext "github.com/gdt-dev/gdt/context"
 )
@@ -17,19 +16,18 @@ import (
 // Debug output.
 func Printf(
 	ctx context.Context,
-	t *testing.T,
 	format string,
 	args ...interface{},
 ) {
-	t.Helper()
 	writers := gdtcontext.Debug(ctx)
 	if len(writers) == 0 {
 		return
 	}
-	t.Logf(format, args...)
+
+	trace := gdtcontext.Trace(ctx)
 
 	if !strings.HasPrefix(format, "[gdt] ") {
-		format = "[gdt] " + t.Name() + " " + format
+		format = "[gdt] [" + trace + "] " + format
 	}
 	msg := fmt.Sprintf(format, args...)
 	for _, w := range writers {
@@ -41,20 +39,18 @@ func Printf(
 // Debug output, ensuring there is a newline in the message line.
 func Println(
 	ctx context.Context,
-	t *testing.T,
 	format string,
 	args ...interface{},
 ) {
-	t.Helper()
 	writers := gdtcontext.Debug(ctx)
 	if len(writers) == 0 {
 		return
 	}
-	// NOTE(jaypipes): T.Logf() automatically adds newlines...
-	t.Logf(format, args...)
+
+	trace := gdtcontext.Trace(ctx)
 
 	if !strings.HasPrefix(format, "[gdt] ") {
-		format = "[gdt] " + t.Name() + " " + format
+		format = "[gdt] [" + trace + "] " + format
 	}
 	if !strings.HasSuffix(format, "\n") {
 		format += "\n"
diff --git a/plugin/exec/action.go b/plugin/exec/action.go
index cadd4b8..da3f753 100644
--- a/plugin/exec/action.go
+++ b/plugin/exec/action.go
@@ -56,7 +56,7 @@ func (a *Action) Do(
 		args = []string{"-c", a.Exec}
 	}
 
-	debug.Println(ctx, t, "exec: %s %s", target, args)
+	debug.Println(ctx, "exec: %s %s", target, args)
 
 	var cmd *exec.Cmd
 	cmd = exec.CommandContext(ctx, target, args...)
@@ -80,13 +80,13 @@ func (a *Action) Do(
 	if outbuf != nil {
 		outbuf.ReadFrom(outpipe)
 		if outbuf.Len() > 0 {
-			debug.Println(ctx, t, "exec: stdout: %s", outbuf.String())
+			debug.Println(ctx, "exec: stdout: %s", outbuf.String())
 		}
 	}
 	if errbuf != nil {
 		errbuf.ReadFrom(errpipe)
 		if errbuf.Len() > 0 {
-			debug.Println(ctx, t, "exec: stderr: %s", errbuf.String())
+			debug.Println(ctx, "exec: stderr: %s", errbuf.String())
 		}
 	}
 
diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go
index b72553a..6e361c2 100644
--- a/plugin/exec/assertions.go
+++ b/plugin/exec/assertions.go
@@ -47,10 +47,6 @@ type pipeAssertions struct {
 	name string
 	// failures contains the set of error messages for failed assertions.
 	failures []error
-	// terminal indicates there was a failure in evaluating the assertions that
-	// should be considered a terminal condition (and therefore the test action
-	// should not be retried).
-	terminal bool
 }
 
 // Fail appends a supplied error to the set of failed assertions
@@ -66,15 +62,6 @@ func (a *pipeAssertions) Failures() []error {
 	return a.failures
 }
 
-// Terminal returns a bool indicating the assertions failed in a way that is
-// not retryable.
-func (a *pipeAssertions) Terminal() bool {
-	if a == nil {
-		return false
-	}
-	return a.terminal
-}
-
 // OK checks all the assertions in the pipeAssertions against the supplied pipe
 // contents and returns true if all assertions pass.
 func (a *pipeAssertions) OK(ctx context.Context) bool {
@@ -130,10 +117,6 @@ func (a *pipeAssertions) OK(ctx context.Context) bool {
 type assertions struct {
 	// failures contains the set of error messages for failed assertions
 	failures []error
-	// terminal indicates there was a failure in evaluating the assertions that
-	// should be considered a terminal condition (and therefore the test action
-	// should not be retried).
-	terminal bool
 	// expExitCode contains the expected exit code
 	expExitCode int
 	// exitCode is the exit code we got from the execution
@@ -157,15 +140,6 @@ func (a *assertions) Failures() []error {
 	return a.failures
 }
 
-// Terminal returns a bool indicating the assertions failed in a way that is
-// not retryable.
-func (a *assertions) Terminal() bool {
-	if a == nil {
-		return false
-	}
-	return a.terminal
-}
-
 // OK checks all the assertions against the supplied arguments and returns true
 // if all assertions pass.
 func (a *assertions) OK(ctx context.Context) bool {
diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go
index 565ab16..36040cc 100644
--- a/plugin/exec/eval.go
+++ b/plugin/exec/eval.go
@@ -40,7 +40,7 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 				errbuf.Reset()
 				err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
 				if err != nil {
-					debug.Println(ctx, t, "error in on.fail.exec: %s", err)
+					debug.Println(ctx, "error in on.fail.exec: %s", err)
 				}
 			}
 		}
diff --git a/scenario/run.go b/scenario/run.go
index c1fec8a..7baa5d6 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -59,6 +59,10 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 		}
 	}
 	t.Run(s.Title(), func(t *testing.T) {
+		ctx = gdtcontext.PushTrace(ctx, s.Title())
+		defer func() {
+			ctx = gdtcontext.PopTrace(ctx)
+		}()
 		for _, spec := range s.Tests {
 			// Create a brand new context that inherits the top-level context's
 			// cancel func. We want to set deadlines for each test spec and if
@@ -70,7 +74,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			sb := spec.Base()
 			wait := sb.Wait
 			if wait != nil && wait.Before != "" {
-				debug.Println(ctx, t, "wait: %s before", wait.Before)
+				debug.Println(ctx, "wait: %s before", wait.Before)
 				time.Sleep(wait.BeforeDuration())
 			}
 
@@ -93,7 +97,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				ctx = gdtcontext.StorePriorRun(ctx, res.Data())
 			}
 			if wait != nil && wait.After != "" {
-				debug.Println(ctx, t, "wait: %s after", wait.After)
+				debug.Println(ctx, "wait: %s after", wait.After)
 				time.Sleep(wait.AfterDuration())
 			}
 		}
@@ -112,14 +116,14 @@ func specTimeout(
 ) *gdttypes.Timeout {
 	if specTimeout != nil {
 		debug.Println(
-			ctx, t, "using timeout of %s (expected: %t)",
+			ctx, "using timeout of %s (expected: %t)",
 			specTimeout.After, specTimeout.Expected,
 		)
 		return specTimeout
 	}
 	if scenDefaults != nil && scenDefaults.Timeout != nil {
 		debug.Println(
-			ctx, t, "using timeout of %s (expected: %t) [scenario default]",
+			ctx, "using timeout of %s (expected: %t) [scenario default]",
 			scenDefaults.Timeout.After, scenDefaults.Timeout.Expected,
 		)
 		return scenDefaults.Timeout
diff --git a/scenario/run_test.go b/scenario/run_test.go
index 722313e..842461d 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -88,7 +88,7 @@ func TestDebugFlushing(t *testing.T) {
 	w.Flush()
 	require.NotEqual(b.Len(), 0)
 	debugout := b.String()
-	require.Contains(debugout, "TestDebugFlushing/foo-debug-wait-flush wait: 250ms before")
+	require.Contains(debugout, "[gdt] [foo-debug-wait-flush] wait: 250ms before")
 }
 
 func TestTimeoutCascade(t *testing.T) {
diff --git a/scenario/stub_plugins_test.go b/scenario/stub_plugins_test.go
index e48fba1..70287cb 100644
--- a/scenario/stub_plugins_test.go
+++ b/scenario/stub_plugins_test.go
@@ -223,7 +223,11 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 func (s *fooSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
 	fails := []error{}
 	t.Run(s.Title(), func(t *testing.T) {
-		debug.Printf(ctx, t, "in %s Foo=%s", s.Title(), s.Foo)
+		ctx = gdtcontext.PushTrace(ctx, s.Title())
+		defer func() {
+			ctx = gdtcontext.PopTrace(ctx)
+		}()
+		debug.Println(ctx, "in %s Foo=%s", s.Title(), s.Foo)
 		// This is just a silly test to demonstrate how to write Eval() methods
 		// for plugin Spec specialization classes.
 		if s.Name == "bar" && s.Foo != "bar" {

From 6ce5715456776834fdb151cb5b9f0a74dc1ba9d4 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sat, 1 Jun 2024 09:07:52 -0400
Subject: [PATCH 06/31] uplift Go and Github action runners

Uplifts Go to 1.21 and hardens our Github action runners.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 .github/workflows/fmtcheck.yml                | 36 ++++++++++++++++
 .github/workflows/lint.yml                    | 41 +++++++++++++++++++
 .../workflows/{gate-tests.yml => test.yml}    |  2 +-
 README.md                                     |  2 +-
 go.mod                                        |  2 +-
 5 files changed, 80 insertions(+), 3 deletions(-)
 create mode 100644 .github/workflows/fmtcheck.yml
 create mode 100644 .github/workflows/lint.yml
 rename .github/workflows/{gate-tests.yml => test.yml} (98%)

diff --git a/.github/workflows/fmtcheck.yml b/.github/workflows/fmtcheck.yml
new file mode 100644
index 0000000..242acc1
--- /dev/null
+++ b/.github/workflows/fmtcheck.yml
@@ -0,0 +1,36 @@
+name: fmtcheck
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+
+jobs:
+  fmtcheck:
+    runs-on: ubuntu-latest
+    steps:
+     - name: harden runner
+       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       with:
+         egress-policy: block
+         disable-sudo: true
+         allowed-endpoints: >
+           github.com:443
+           api.github.com:443
+           proxy.github.com:443
+           proxy.golang.org:443
+           raw.githubusercontent.com:443
+           objects.githubusercontent.com:443
+           proxy.golang.org:443
+     - name: checkout code
+       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+     - name: setup go
+       uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+       with:
+         go-version: 1.21
+     - name: check fmt
+       run: 'bash -c "diff -u <(echo -n) <(gofmt -d .)"'
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 0000000..2ad167d
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,41 @@
+name: lint
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+  pull-requests: read # needed for only-new-issues option below
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+     - name: harden runner
+       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       with:
+         egress-policy: block
+         disable-sudo: true
+         allowed-endpoints: >
+           github.com:443
+           api.github.com:443
+           proxy.github.com:443
+           proxy.golang.org:443
+           raw.githubusercontent.com:443
+           objects.githubusercontent.com:443
+           proxy.golang.org:443
+     - name: checkout code
+       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+     - name: setup go
+       uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+       with:
+         go-version: 1.21
+     - name: lint
+       uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299 # v3.6.0
+       with:
+         version: v1.53
+         args: --timeout=5m0s --verbose
+         only-new-issues: true
diff --git a/.github/workflows/gate-tests.yml b/.github/workflows/test.yml
similarity index 98%
rename from .github/workflows/gate-tests.yml
rename to .github/workflows/test.yml
index dcc1c66..b984717 100644
--- a/.github/workflows/gate-tests.yml
+++ b/.github/workflows/test.yml
@@ -1,4 +1,4 @@
-name: gate tests
+name: test
 
 on:
   push:
diff --git a/README.md b/README.md
index 016933a..aaef261 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 [![Go Reference](https://pkg.go.dev/badge/github.com/gdt-dev/gdt.svg)](https://pkg.go.dev/github.com/gdt-dev/gdt)
 [![Go Report Card](https://goreportcard.com/badge/github.com/gdt-dev/gdt)](https://goreportcard.com/report/github.com/gdt-dev/gdt)
-[![Build Status](https://github.com/gdt-dev/gdt/actions/workflows/gate-tests.yml/badge.svg?branch=main)](https://github.com/gdt-dev/gdt/actions)
+[![Build Status](https://github.com/gdt-dev/gdt/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/gdt-dev/gdt/actions)
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md)
 
 <div style="float: left">
diff --git a/go.mod b/go.mod
index 6d57e4b..f1e8df8 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/gdt-dev/gdt
 
-go 1.19
+go 1.21
 
 require (
 	github.com/PaesslerAG/jsonpath v0.1.1

From 8b27fb617705a5f49c5337d3470a6e22f26e26d8 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sat, 8 Jun 2024 20:00:32 -0400
Subject: [PATCH 07/31] add context argument to fixture Stop/Start

In order to integrate fixtures into debug printing, we needed to add
context arguments to the `fixture.Start()` and `fixture.Stop()` methods.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 context/context_test.go |  2 +-
 fixture/generic.go      | 17 +++++++++--------
 fixture/generic_test.go |  9 +++++----
 fixture/json/json.go    |  5 +++--
 scenario/run.go         |  4 ++--
 types/fixture.go        |  6 ++++--
 6 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/context/context_test.go b/context/context_test.go
index 0c92ee3..82fe57f 100644
--- a/context/context_test.go
+++ b/context/context_test.go
@@ -16,7 +16,7 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func fooStart() {}
+func fooStart(_ context.Context) {}
 
 type fooDefaults struct {
 	Foo string `yaml:"foo"`
diff --git a/fixture/generic.go b/fixture/generic.go
index 40b1a3f..84c455b 100644
--- a/fixture/generic.go
+++ b/fixture/generic.go
@@ -5,6 +5,7 @@
 package fixture
 
 import (
+	"context"
 	"strings"
 
 	gdttypes "github.com/gdt-dev/gdt/types"
@@ -12,22 +13,22 @@ import (
 
 // genericFixture adapts functions and state dicts into the Fixture type
 type genericFixture struct {
-	starter func()
-	stopper func()
+	starter func(context.Context)
+	stopper func(context.Context)
 	state   map[string]interface{}
 }
 
 // Start sets up any resources the fixture uses
-func (f *genericFixture) Start() {
+func (f *genericFixture) Start(ctx context.Context) {
 	if f.starter != nil {
-		f.starter()
+		f.starter(ctx)
 	}
 }
 
 // Stop cleans up any resources the fixture uses
-func (f *genericFixture) Stop() {
+func (f *genericFixture) Stop(ctx context.Context) {
 	if f.stopper != nil {
-		f.stopper()
+		f.stopper(ctx)
 	}
 }
 
@@ -54,14 +55,14 @@ func (f *genericFixture) State(key string) interface{} {
 type genericFixtureModifier func(s *genericFixture)
 
 // WithStarter allows a starter functor to be adapted into a fixture
-func WithStarter(starter func()) genericFixtureModifier {
+func WithStarter(starter func(context.Context)) genericFixtureModifier {
 	return func(f *genericFixture) {
 		f.starter = starter
 	}
 }
 
 // WithStopper allows a stopper functor to be adapted into a fixture
-func WithStopper(stopper func()) genericFixtureModifier {
+func WithStopper(stopper func(context.Context)) genericFixtureModifier {
 	return func(f *genericFixture) {
 		f.stopper = stopper
 	}
diff --git a/fixture/generic_test.go b/fixture/generic_test.go
index 5867476..48c73ac 100644
--- a/fixture/generic_test.go
+++ b/fixture/generic_test.go
@@ -5,6 +5,7 @@
 package fixture_test
 
 import (
+	"context"
 	"testing"
 
 	"github.com/gdt-dev/gdt/fixture"
@@ -35,7 +36,7 @@ func TestStarter(t *testing.T) {
 
 	started := false
 
-	starter := func() {
+	starter := func(_ context.Context) {
 		started = true
 	}
 
@@ -45,7 +46,7 @@ func TestStarter(t *testing.T) {
 
 	assert.False(started)
 
-	f.Start()
+	f.Start(context.TODO())
 
 	assert.True(started)
 }
@@ -55,7 +56,7 @@ func TestStopper(t *testing.T) {
 
 	stopped := false
 
-	stopper := func() {
+	stopper := func(_ context.Context) {
 		stopped = true
 	}
 
@@ -65,7 +66,7 @@ func TestStopper(t *testing.T) {
 
 	assert.False(stopped)
 
-	f.Stop()
+	f.Stop(context.TODO())
 
 	assert.True(stopped)
 }
diff --git a/fixture/json/json.go b/fixture/json/json.go
index 8829f5c..1a4629f 100644
--- a/fixture/json/json.go
+++ b/fixture/json/json.go
@@ -5,6 +5,7 @@
 package json
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 	"io/ioutil"
@@ -18,9 +19,9 @@ type jsonFixture struct {
 	data interface{}
 }
 
-func (f *jsonFixture) Start() {}
+func (f *jsonFixture) Start(_ context.Context) {}
 
-func (f *jsonFixture) Stop() {}
+func (f *jsonFixture) Stop(_ context.Context) {}
 
 // HasState returns true if the supplied JSONPath expression results in a found
 // value in the fixture's data
diff --git a/scenario/run.go b/scenario/run.go
index 7baa5d6..c31f796 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -32,8 +32,8 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			if !found {
 				return gdterrors.RequiredFixtureMissing(fname)
 			}
-			fix.Start()
-			defer fix.Stop()
+			fix.Start(ctx)
+			defer fix.Stop(ctx)
 		}
 	}
 	var rterr error
diff --git a/types/fixture.go b/types/fixture.go
index c4c0371..d421d78 100644
--- a/types/fixture.go
+++ b/types/fixture.go
@@ -4,12 +4,14 @@
 
 package types
 
+import "context"
+
 // A Fixture allows state to be passed from setups
 type Fixture interface {
 	// Start sets up the fixture
-	Start()
+	Start(context.Context)
 	// Stop tears down the fixture, cleaning up any owned resources
-	Stop()
+	Stop(context.Context)
 	// HasState returns true if the fixture contains some state with the given
 	// key
 	HasState(string) bool

From 6ed56ae397477427d5f36ea57784108a5030da00 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sat, 8 Jun 2024 20:30:06 -0400
Subject: [PATCH 08/31] gdtcontext.WithDebug() no args to stdout

Changes the way that the `gdtcontext.WithDebug()` method works when
passed no arguments. When called with no arguments, we now output
debugging messages to stdout instead of doing the thing with an
io.Discard that was used to call t.Logf().

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 context/context.go | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/context/context.go b/context/context.go
index 6d7da7c..4f830a6 100644
--- a/context/context.go
+++ b/context/context.go
@@ -7,6 +7,7 @@ package context
 import (
 	"context"
 	"io"
+	"os"
 
 	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/samber/lo"
@@ -100,18 +101,15 @@ func WithFixtures(fixtures map[string]gdttypes.Fixture) ContextModifier {
 // SetDebug sets gdt's debug logging to the supplied `io.Writer`.
 //
 // The `writers` parameters is optional. If no `io.Writer` objects are
-// supplied, gdt will output debug messages using the `testing.T.Log[f]()`
-// function. This means that you will only get these debug messages if you call
-// the `go test` tool with the `-v` option (either as `go test -v` or with `go
-// test -v=test2json`.
+// supplied, gdt will output debug messages to stdout.
 func SetDebug(
 	ctx context.Context,
 	writers ...io.Writer,
 ) context.Context {
 	if len(writers) == 0 {
-		// This simply triggers a call to t.Logf() when WithDebug() is
-		// called with no parameters...
-		writers = []io.Writer{io.Discard}
+		// This triggers writes to stdout when WithDebug() is called with no
+		// parameters...
+		writers = []io.Writer{os.Stdout}
 	}
 	return context.WithValue(ctx, debugKey, writers)
 }

From 98ba1460651fdec18f61bfc3c04b96a0d6d47989 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sun, 16 Jun 2024 19:45:55 -0400
Subject: [PATCH 09/31] actually write to stdout for Debug() no params

We were creating an io.Writer that discarded instead of creating an
io.Writer to stdout when no parameters were passed to the
`context.Debug()` function. Doh.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 context/context.go | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/context/context.go b/context/context.go
index 4f830a6..23469ed 100644
--- a/context/context.go
+++ b/context/context.go
@@ -30,9 +30,13 @@ type ContextModifier func(context.Context) context.Context
 // zero or more `io.Writer` objects to the function.
 //
 // If no `io.Writer` objects are supplied, gdt will output debug messages using
-// the `testing.T.Log[f]()` function. This means that you will only get these
-// debug messages if you call the `go test` tool with the `-v` option (either
-// as `go test -v` or with `go test -v=test2json`.
+// the `fmt.Printf` function. The `fmt.Printf` function is *unbuffered* however
+// unless you call `go test` with the `-v` argument, `go test` swallows output
+// to stdout and does not display it unless a test fails.
+//
+// This means that you will only get these debug messages if you call the `go
+// test` tool with the `-v` option (either as `go test -v` or with `go test
+// -v=test2json`.
 //
 // ```go
 //
@@ -76,9 +80,8 @@ type ContextModifier func(context.Context) context.Context
 func WithDebug(writers ...io.Writer) ContextModifier {
 	return func(ctx context.Context) context.Context {
 		if len(writers) == 0 {
-			// This simply triggers a call to t.Logf() when WithDebug() is
-			// called with no parameters...
-			writers = []io.Writer{io.Discard}
+			// Write to stdout when WithDebug() is called with no parameters
+			writers = []io.Writer{os.Stdout}
 		}
 		return context.WithValue(ctx, debugKey, writers)
 	}

From 442bd52013f2ffd86877ae58512f6756e8316336 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Tue, 18 Jun 2024 10:51:00 -0400
Subject: [PATCH 10/31] add `error` return to `Fixture.Start()`

In order to better handle setup failures in fixtures, the
`fixture.Start()` interface method should return `error`.

Issue #28

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 context/context_test.go                    |  2 +-
 fixture/generic.go                         |  9 +++++----
 fixture/generic_test.go                    |  3 ++-
 fixture/json/json.go                       |  2 +-
 scenario/run.go                            |  4 +++-
 scenario/run_test.go                       | 20 ++++++++++++++++++++
 scenario/stub_fixtures_test.go             | 22 ++++++++++++++++++++++
 scenario/testdata/fixture-start-error.yaml |  7 +++++++
 types/fixture.go                           |  2 +-
 9 files changed, 62 insertions(+), 9 deletions(-)
 create mode 100644 scenario/stub_fixtures_test.go
 create mode 100644 scenario/testdata/fixture-start-error.yaml

diff --git a/context/context_test.go b/context/context_test.go
index 82fe57f..6259926 100644
--- a/context/context_test.go
+++ b/context/context_test.go
@@ -16,7 +16,7 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func fooStart(_ context.Context) {}
+func fooStart(_ context.Context) error { return nil }
 
 type fooDefaults struct {
 	Foo string `yaml:"foo"`
diff --git a/fixture/generic.go b/fixture/generic.go
index 84c455b..f1102ff 100644
--- a/fixture/generic.go
+++ b/fixture/generic.go
@@ -13,16 +13,17 @@ import (
 
 // genericFixture adapts functions and state dicts into the Fixture type
 type genericFixture struct {
-	starter func(context.Context)
+	starter func(context.Context) error
 	stopper func(context.Context)
 	state   map[string]interface{}
 }
 
 // Start sets up any resources the fixture uses
-func (f *genericFixture) Start(ctx context.Context) {
+func (f *genericFixture) Start(ctx context.Context) error {
 	if f.starter != nil {
-		f.starter(ctx)
+		return f.starter(ctx)
 	}
+	return nil
 }
 
 // Stop cleans up any resources the fixture uses
@@ -55,7 +56,7 @@ func (f *genericFixture) State(key string) interface{} {
 type genericFixtureModifier func(s *genericFixture)
 
 // WithStarter allows a starter functor to be adapted into a fixture
-func WithStarter(starter func(context.Context)) genericFixtureModifier {
+func WithStarter(starter func(context.Context) error) genericFixtureModifier {
 	return func(f *genericFixture) {
 		f.starter = starter
 	}
diff --git a/fixture/generic_test.go b/fixture/generic_test.go
index 48c73ac..95e2b37 100644
--- a/fixture/generic_test.go
+++ b/fixture/generic_test.go
@@ -36,8 +36,9 @@ func TestStarter(t *testing.T) {
 
 	started := false
 
-	starter := func(_ context.Context) {
+	starter := func(_ context.Context) error {
 		started = true
+		return nil
 	}
 
 	f := fixture.New(
diff --git a/fixture/json/json.go b/fixture/json/json.go
index 1a4629f..b034106 100644
--- a/fixture/json/json.go
+++ b/fixture/json/json.go
@@ -19,7 +19,7 @@ type jsonFixture struct {
 	data interface{}
 }
 
-func (f *jsonFixture) Start(_ context.Context) {}
+func (f *jsonFixture) Start(_ context.Context) error { return nil }
 
 func (f *jsonFixture) Stop(_ context.Context) {}
 
diff --git a/scenario/run.go b/scenario/run.go
index c31f796..7d0948b 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -32,7 +32,9 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			if !found {
 				return gdterrors.RequiredFixtureMissing(fname)
 			}
-			fix.Start(ctx)
+			if err := fix.Start(ctx); err != nil {
+				return err
+			}
 			defer fix.Stop(ctx)
 		}
 	}
diff --git a/scenario/run_test.go b/scenario/run_test.go
index 842461d..7d98d23 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -67,6 +67,26 @@ func TestMissingFixtures(t *testing.T) {
 	assert.ErrorIs(err, gdterrors.RuntimeError)
 }
 
+func TestFixtureStartError(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	fp := filepath.Join("testdata", "fixture-start-error.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New()
+	ctx = gdtcontext.RegisterFixture(ctx, "start-error", errStarterFixture)
+
+	err = s.Run(ctx, t)
+	assert.NotNil(err)
+	assert.ErrorContains(err, "error starting fixture!")
+}
+
 func TestDebugFlushing(t *testing.T) {
 	require := require.New(t)
 
diff --git a/scenario/stub_fixtures_test.go b/scenario/stub_fixtures_test.go
new file mode 100644
index 0000000..4875b46
--- /dev/null
+++ b/scenario/stub_fixtures_test.go
@@ -0,0 +1,22 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package scenario_test
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gdt-dev/gdt/fixture"
+)
+
+var (
+	errStarter = func(_ context.Context) error {
+		return fmt.Errorf("error starting fixture!")
+	}
+
+	errStarterFixture = fixture.New(
+		fixture.WithStarter(errStarter),
+	)
+)
diff --git a/scenario/testdata/fixture-start-error.yaml b/scenario/testdata/fixture-start-error.yaml
new file mode 100644
index 0000000..2c625a0
--- /dev/null
+++ b/scenario/testdata/fixture-start-error.yaml
@@ -0,0 +1,7 @@
+name: fixture-start-error
+description: a scenario with a fixture that errors in start
+fixtures:
+  - start-error
+tests:
+  - foo: bar
+
diff --git a/types/fixture.go b/types/fixture.go
index d421d78..8cb4b7a 100644
--- a/types/fixture.go
+++ b/types/fixture.go
@@ -9,7 +9,7 @@ import "context"
 // A Fixture allows state to be passed from setups
 type Fixture interface {
 	// Start sets up the fixture
-	Start(context.Context)
+	Start(context.Context) error
 	// Stop tears down the fixture, cleaning up any owned resources
 	Stop(context.Context)
 	// HasState returns true if the fixture contains some state with the given

From 74db3341f16bb548a614830a6fc7d33d9e04e0b5 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 19 Jun 2024 16:44:57 -0400
Subject: [PATCH 11/31] add custom retry configuration

Adds the ability to configure retry behaviour for any test spec, using
either a constant interval or an exponential backoff.

In doing so, reworks the framework to have all calls to
testing.T.Errorf() happen in the Scenario.Run() method, as well as have
that method handle all retries. gdt-kube will need to be refactored to
remove the retry behaviour implemented in that plugin.

Also, this finally fixes the way that failures are tested by using the
strategy outlined in https://github.com/golang/go/issues/39903.

Issue #26

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 README.md                                     | 102 +++++++-----
 errors/parse.go                               |  32 +++-
 go.mod                                        |   1 +
 go.sum                                        |   2 +
 plugin/exec/eval.go                           |   3 -
 plugin/exec/eval_test.go                      | 107 +++++++-----
 plugin/exec/testdata/on-fail-exec.yaml        |  23 +--
 scenario/defaults.go                          |  28 +++-
 scenario/parse.go                             |  28 ++--
 scenario/parse_test.go                        |  39 +++++
 scenario/run.go                               | 152 ++++++++++++++++--
 scenario/run_test.go                          |  56 ++++++-
 scenario/scenario.go                          |   3 +
 scenario/stub_plugins_test.go                 |   3 +
 scenario/testdata/no-retry.yaml               |   6 +
 .../parse/fail/bad-retry-attempts.yaml        |   6 +
 .../fail/bad-retry-interval-duration.yaml     |   6 +
 scenario/testdata/parse/fail/bad-retry.yaml   |   5 +
 scenario/testdata/retry-test-override.yaml    |   9 ++
 types/plugin.go                               |   3 +
 types/retry.go                                |  46 ++++++
 types/spec.go                                 |  24 +++
 22 files changed, 550 insertions(+), 134 deletions(-)
 create mode 100644 scenario/testdata/no-retry.yaml
 create mode 100644 scenario/testdata/parse/fail/bad-retry-attempts.yaml
 create mode 100644 scenario/testdata/parse/fail/bad-retry-interval-duration.yaml
 create mode 100644 scenario/testdata/parse/fail/bad-retry.yaml
 create mode 100644 scenario/testdata/retry-test-override.yaml
 create mode 100644 types/retry.go

diff --git a/README.md b/README.md
index aaef261..da837d1 100644
--- a/README.md
+++ b/README.md
@@ -11,16 +11,16 @@
 
 `gdt` is a testing library that allows test authors to cleanly describe tests
 in a YAML file. `gdt` reads YAML files that describe a test's assertions and
-then builds a set of Golang structures that the standard Golang
+then builds a set of Go structures that the standard Go
 [`testing`](https://golang.org/pkg/testing/) package and standard `go test`
 tool can execute.
 
 ## Introduction
 
-Writing functional tests in Golang can be overly verbose and tedious. When the
-code that tests some part of an application is verbose or tedious, then it
-becomes difficult to read the tests and quickly understand the assertions the
-test is making.
+Writing functional tests in Go can be overly verbose and tedious. When the code
+that tests some part of an application is verbose or tedious, then it becomes
+difficult to read the tests and quickly understand the assertions the test is
+making.
 
 The more difficult it is to understand the test assertions or the test setups
 and assumptions, the greater the chance that the test improperly validates the
@@ -34,7 +34,7 @@ describe a functional test's **assumptions** and **assertions** in a
 declarative format.
 
 Separating the *description* of a test's assumptions (setup) and assertions
-from the Golang code that actually performs the test assertions leads to tests
+from the Go code that actually performs the test assertions leads to tests
 that are easier to read and understand. This allows developers to spend *more
 time writing code* and less time copy/pasting boilerplate test code. Due to the
 easier test comprehension, `gdt` also encourages writing greater quality and
@@ -138,9 +138,9 @@ var _ = Describe("Books API Types", func() {
 ```
 
 
-This is perfectly fine for simple unit tests of Golang code. However, once the
-tests begin to call multiple APIs or packages, the Ginkgo Golang tests start to
-get cumbersome. Consider the following example of *functionally* testing the
+This is perfectly fine for simple unit tests of Go code. However, once the
+tests begin to call multiple APIs or packages, the Ginkgo Go tests start to get
+cumbersome. Consider the following example of *functionally* testing the
 failure modes for a simple HTTP REST API endpoint
 ([`failure_test.go`](https://github.com/gdt-dev/gdt-examples/blob/main/http/api/failure_test.go)):
 
@@ -256,7 +256,7 @@ var _ = Describe("Books API - GET /books failures", func() {
 ```
 
 The above test code obscures what is being tested by cluttering the test
-assertions with the Golang closures and accessor code. Compare the above with
+assertions with the Go closures and accessor code. Compare the above with
 how `gdt` allows the test author to describe the same assertions
 ([`failures.yaml`](https://github.com/gdt-dev/gdt-examples/blob/main/http/tests/api/failures.yaml)):
 
@@ -284,7 +284,7 @@ No more closures and boilerplate function code getting in the way of expressing
 the assertions, which should be the focus of the test.
 
 The more intricate the assertions being verified by the test, generally the
-more verbose and cumbersome the Golang test code tends to become. First and
+more verbose and cumbersome the Go test code tends to become. First and
 foremost, tests should be *readable*. If they are not readable, then the test's
 assertions are not *understandable*. And tests that cannot easily be understood
 are often the source of bit rot and technical debt. Worse, tests that aren't
@@ -300,7 +300,7 @@ Consider a Ginkgo test case that checks the following behaviour:
 * The newly-created book's ID field is a valid UUID
 * The newly-created book's publisher has an address containing a known state code
 
-A typical implementation of a Ginkgo Golang test might look like this
+A typical implementation of a Ginkgo test might look like this
 ([`create_then_get_test.go`](https://github.com/gdt-dev/gdt-examples/blob/main/http/api/create_then_get_test.go)):
 
 ```go
@@ -423,8 +423,7 @@ All `gdt` scenarios have the following fields:
   missing or empty, the filename is used as the name
 * `description`: (optional) string with longer description of the test file
   contents
-* `defaults`: (optional) is a map, keyed by a plugin name, of default options
-  and configuration values for that plugin.
+* `defaults`: (optional) is a map of default options and configuration values
 * `fixtures`: (optional) list of strings indicating named fixtures that will be
   started before any of the tests in the file are run
 * `skip-if`: (optional) list of [`Spec`][basespec] specializations that will be
@@ -433,26 +432,29 @@ All `gdt` scenarios have the following fields:
 * `tests`: list of [`Spec`][basespec] specializations that represent the
   runnable test units in the test scenario.
 
-[basespec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/spec.go#L27-L44
+[basespec]: https://github.com/gdt-dev/gdt/blob/ecee17249e1fa10147cf9191be0358923da44094/types/spec.go#L30
 
 The scenario's `tests` field is the most important and the [`Spec`][basespec]
 objects that it contains are the meat of a test scenario.
 
-## `gdt` test spec structure
+### `gdt` test spec structure
 
 A spec represents a single *action* that is taken and zero or more
 *assertions* that represent what you expect to see resulting from that action.
 
-Each spec is a specialized class of the base [`Spec`][basespec] that deals with
-a particular type of test. For example, there is a `Spec` class called `exec`
-that allows you to execute arbitrary commands and assert expected result codes
-and output. There is a `Spec` class called `http` that allows you to call an
-HTTP URL and assert that the response looks like what you expect. Depending on
-how you define your test units, `gdt` will parse the YAML definition into one
-of these specialized `Spec` classes.
+`gdt` plugins each define a specialized subclass of the base [`Spec`][basespec]
+that contains fields that are specific to that type of test.
 
-The base `Spec` class has the following fields (and thus all `Spec` specialized
-classes inherit these fields):
+For example, there is an [`exec`][exec-plugin] plugin that allows you to
+execute arbitrary commands and assert expected result codes and output. There
+is an [`http`][http-plugin] that allows you to call an HTTP URL and assert that
+the response looks like what you expect. There is a [`kube`][kube-plugin]
+plugin that allows you to interact with a Kubernetes API, etc.
+
+`gdt` examines the YAML file that defines your test scenario and uses these
+plugins to parse individual test specs.
+
+All test specs have the following fields:
 
 * `name`: (optional) string describing the test unit.
 * `description`: (optional) string with longer description of the test unit.
@@ -462,24 +464,49 @@ classes inherit these fields):
   complete within.
 * `timeout.expected`: a bool indicating that the test unit is expected to not
   complete before `timeout.after`. This is really only useful in unit testing.
+* `retry`: (optional) an object containing retry configurationu for the test
+  unit. Some plugins will automatically attempt to retry the test action when
+  an assertion fails. This field allows you to control this retry behaviour for
+  each individual test.
+* `retry.interval`: (optional) a string duration of time that the test plugin
+  will retry the test action in the event assertions fail. The default interval
+  for retries is plugin-dependent.
+* `retry.attempts`: (optional) an integer indicating the number of times that a
+  plugin will retry the test action in the event assertions fail. The default
+  number of attempts for retries is plugin-dependent.
+* `retry.exponential`: (optional) a boolean indicating an exponential backoff
+  should be applied to the retry interval. The default is is plugin-dependent.
 * `wait` (optional) an object containing [wait information][wait] for the test
   unit.
 * `wait.before`: a string duration of time that gdt should wait before
   executing the test unit's action.
 * `wait.after`: a string duration of time that gdt should wait after executing
   the test unit's action.
+* `on`: (optional) an object describing actions to take upon certain
+  conditions.
+* `on.fail`: (optional) an object describing an action to take when any
+  assertion fails for the test action.
+* `on.fail.exec`: a string with the exact command to execute upon test
+  assertion failure. You may execute more than one command but must include the
+  `on.fail.shell` field to indicate that the command should be run in a shell.
+* `on.fail.shell`: (optional) a string with the specific shell to use in executing the
+  command to run upon test assertion failure. If empty (the default), no shell
+  is used to execute the command and instead the operating system's `exec` family
+  of calls is used.
 
+[exec-plugin]: https://github.com/gdt-dev/gdt/tree/ecee17249e1fa10147cf9191be0358923da44094/plugin/exec
+[http-plugin]: https://github.com/gdt-dev/http
+[kube-plugin]: https://github.com/gdt-dev/kube
 [timeout]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/timeout.go#L11-L22
 [wait]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/wait.go#L11-L25
 
-### `exec` test spec structure
+#### `exec` test spec structure
 
-An exec spec is a specialization of the base [`Spec`][basespec] that allows
-test authors to execute arbitrary commands and assert that the command results
-in an expected result code or output.
+The `exec` plugin's test spec allows test authors to execute arbitrary commands and
+assert that the command results in an expected result code or output.
 
-The [exec `Spec`][execspec] class has the following fields (in addition to all
-the base `Spec` fields listed above):
+In addition to all the base `Spec` fields listed above, the `exec` plugin's
+test spec also contains these fields:
 
 * `exec`: a string with the exact command to execute. You may execute more than
   one command but must include the `shell` field to indicate that the command
@@ -514,17 +541,6 @@ the base `Spec` fields listed above):
   least one* must be present in `stderr`.
 * `assert.err.none`: (optional) a string or list of strings of which *none
   should be present* in `stderr`.
-* `on`: (optional) an object describing actions to take upon certain
-  conditions.
-* `on.fail`: (optional) an object describing an action to take when any
-  assertion fails for the test action.
-* `on.fail.exec`: a string with the exact command to execute upon test
-  assertion failure. You may execute more than one command but must include the
-  `on.fail.shell` field to indicate that the command should be run in a shell.
-* `on.fail.shell`: (optional) a string with the specific shell to use in executing the
-  command to run upon test assertion failure. If empty (the default), no shell
-  is used to execute the command and instead the operating system's `exec` family
-  of calls is used.
 
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26
@@ -533,7 +549,7 @@ the base `Spec` fields listed above):
 
 `gdt` was inspired by [Gabbi](https://github.com/cdent/gabbi), the excellent
 Python declarative testing framework. `gdt` tries to bring the same clear,
-concise test definitions to the world of Golang functional testing.
+concise test definitions to the world of Go functional testing.
 
 The Go gopher logo, from which gdt's logo was derived, was created by Renee
 French.
diff --git a/errors/parse.go b/errors/parse.go
index 867d13e..3761389 100644
--- a/errors/parse.go
+++ b/errors/parse.go
@@ -53,14 +53,24 @@ var (
 	ErrExpectedScalarOrSequence = fmt.Errorf(
 		"%w: expected scalar or sequence of scalars field", ErrParse,
 	)
-	// ErrParseTimeout indicates that the timeout specification was not valid.
+	// ErrExpectedTimeout indicates that the timeout specification was not
+	// valid.
 	ErrExpectedTimeout = fmt.Errorf(
 		"%w: expected timeout specification", ErrParse,
 	)
-	// ErrParseWait indicates that the wait specification was not valid.
+	// ErrExpectedWait indicates that the wait specification was not valid.
 	ErrExpectedWait = fmt.Errorf(
 		"%w: expected wait specification", ErrParse,
 	)
+	// ErrExpectedRetry indicates that the retry specification was not valid.
+	ErrExpectedRetry = fmt.Errorf(
+		"%w: expected retry specification", ErrParse,
+	)
+	// ErrInvalidRetryAttempts indicates that the retry attempts was not
+	// positive.
+	ErrInvalidRetryAttempts = fmt.Errorf(
+		"%w: invalid retry attempts", ErrParse,
+	)
 	// ErrFileNotFound is returned when a file path does not exist for a
 	// create/apply/delete target.
 	ErrFileNotFound = fmt.Errorf(
@@ -158,6 +168,24 @@ func ExpectedWaitAt(node *yaml.Node) error {
 	)
 }
 
+// ExpectedRetryAt returns an ErrExpectedRetry error annotated with the
+// line/column of the supplied YAML node.
+func ExpectedRetryAt(node *yaml.Node) error {
+	return fmt.Errorf(
+		"%w at line %d, column %d",
+		ErrExpectedRetry, node.Line, node.Column,
+	)
+}
+
+// InvalidRetryAttempts returns an ErrInvalidRetryAttempts error annotated with
+// the line/column of the supplied YAML node.
+func InvalidRetryAttempts(node *yaml.Node, attempts int) error {
+	return fmt.Errorf(
+		"%w of %d at line %d, column %d",
+		ErrInvalidRetryAttempts, attempts, node.Line, node.Column,
+	)
+}
+
 // UnknownSourceType returns an ErrUnknownSourceType error describing the
 // supplied parameter type.
 func UnknownSourceType(source interface{}) error {
diff --git a/go.mod b/go.mod
index f1e8df8..dc2f74c 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/PaesslerAG/jsonpath v0.1.1
+	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.3.0
 	github.com/samber/lo v1.38.1
diff --git a/go.sum b/go.sum
index 22835ec..e4b94e9 100644
--- a/go.sum
+++ b/go.sum
@@ -3,6 +3,8 @@ github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
 github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
 github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go
index 36040cc..8897972 100644
--- a/plugin/exec/eval.go
+++ b/plugin/exec/eval.go
@@ -31,9 +31,6 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
 	if !a.OK(ctx) {
-		for _, fail := range a.Failures() {
-			t.Error(fail)
-		}
 		if s.On != nil {
 			if s.On.Fail != nil {
 				outbuf.Reset()
diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index 4fc024a..b3e9bc9 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -8,7 +8,10 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"flag"
+	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"testing"
@@ -18,6 +21,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var failFlag = flag.Bool("fail", false, "run tests expected to fail")
+
 func TestNoExitCodeSimpleCommand(t *testing.T) {
 	require := require.New(t)
 
@@ -157,7 +162,10 @@ func TestContainsNoneOf(t *testing.T) {
 	require.Nil(err)
 }
 
-func TestSleepTimeout(t *testing.T) {
+func TestFailExecSleepTimeout(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "sleep-timeout.yaml")
@@ -176,6 +184,22 @@ func TestSleepTimeout(t *testing.T) {
 	require.Nil(err)
 }
 
+func TestExecSleepTimeout(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailExecSleepTimeout",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+	debugout := fmt.Sprintf("%s", outerr)
+	require.Contains(debugout, "assertion failed: timeout exceeded")
+}
+
 func TestDebugWriter(t *testing.T) {
 	require := require.New(t)
 
@@ -230,7 +254,10 @@ func TestWait(t *testing.T) {
 	require.Contains(debugout, "wait: 20ms after")
 }
 
-func TestTimeoutCascade(t *testing.T) {
+func TestFailExecTimeoutCascade(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "timeout-cascade.yaml")
@@ -244,47 +271,32 @@ func TestTimeoutCascade(t *testing.T) {
 	require.Nil(err)
 	require.NotNil(s)
 
-	var b bytes.Buffer
-	w := bufio.NewWriter(&b)
-	ctx := gdtcontext.New(gdtcontext.WithDebug(w))
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
 	err = s.Run(ctx, t)
-	require.Nil(err)
-	require.False(t.Failed())
-	w.Flush()
-	require.NotEqual(b.Len(), 0)
-	debugout := b.String()
+}
+
+func TestExecTimeoutCascade(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailExecTimeoutCascade",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+
+	debugout := fmt.Sprintf("%s", outerr)
 	require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]")
 	require.Contains(debugout, "using timeout of 20ms (expected: true)")
 }
 
-// Unfortunately there's not really any good way of testing things like this
-// except by manually causing an assertion to fail in the test case and
-// checking to see if the `on.fail` action was taken and debug output emitted
-// to the console.
-//
-// When I change the `testdata/on-fail-exec.yaml` file to have a failed
-// assertion by changing `assert.out.is` to "dat" instead of "cat", I get the
-// correct behaviour:
-//
-// === RUN   TestOnFail
-// === RUN   TestOnFail/on-fail-exec
-//
-//	action.go:59: exec: echo [cat]
-//	eval.go:35: assertion failed: not equal: expected dat but got cat
-//	action.go:59: exec: echo [bad kitty]
-//	eval.go:46: on.fail.exec: stdout: bad kitty
-//
-// === NAME  TestOnFail
-//
-//	eval_test.go:256:
-//	    	Error Trace:	/home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
-//	    	Error:      	Should be false
-//	    	Test:       	TestOnFail
-//
-// --- FAIL: TestOnFail (0.00s)
-//
-//	--- FAIL: TestOnFail/on-fail-exec (0.00s)
-func TestOnFail(t *testing.T) {
+func TestFailExecOnFail(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
 	require := require.New(t)
 
 	fp := filepath.Join("testdata", "on-fail-exec.yaml")
@@ -301,5 +313,22 @@ func TestOnFail(t *testing.T) {
 	ctx := gdtcontext.New(gdtcontext.WithDebug())
 	err = s.Run(ctx, t)
 	require.Nil(err)
-	require.False(t.Failed())
+}
+
+func TestExecOnFail(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailExecOnFail",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+
+	debugout := fmt.Sprintf("%s", outerr)
+	require.Contains(debugout, "assertion failed: not equal: expected dat but got cat")
+	require.Contains(debugout, "echo [bad kitty]")
 }
diff --git a/plugin/exec/testdata/on-fail-exec.yaml b/plugin/exec/testdata/on-fail-exec.yaml
index 5cfc00f..3ebeced 100644
--- a/plugin/exec/testdata/on-fail-exec.yaml
+++ b/plugin/exec/testdata/on-fail-exec.yaml
@@ -4,28 +4,7 @@ tests:
   - exec: echo "cat"
     assert:
       out:
-        is: cat
-    # Unfortunately there's not really any good way of testing things like this
-    # except by manually causing an assertion to fail in the test case and checking
-    # to see if the `on.fail` action was taken and debug output emitted to the
-    # console.
-    # 
-    # When I change `assert.out.is` above to "dat" instead of "cat", I get the
-    # correct behaviour:
-    # 
-    # === RUN   TestOnFail
-    # === RUN   TestOnFail/on-fail-exec
-    #     action.go:59: exec: echo [cat]
-    #     eval.go:35: assertion failed: not equal: expected dat but got cat
-    #     action.go:59: exec: echo [bad kitty]
-    #     eval.go:46: on.fail.exec: stdout: bad kitty
-    # === NAME  TestOnFail
-    #     eval_test.go:256:
-    #         	Error Trace:	/home/jaypipes/src/github.com/gdt-dev/gdt/plugin/exec/eval_test.go:256
-    #         	Error:      	Should be false
-    #         	Test:       	TestOnFail
-    # --- FAIL: TestOnFail (0.00s)
-    #     --- FAIL: TestOnFail/on-fail-exec (0.00s)
+        is: dat
     on:
       fail:
         exec: echo "bad kitty"
diff --git a/scenario/defaults.go b/scenario/defaults.go
index efeec9e..68fff91 100644
--- a/scenario/defaults.go
+++ b/scenario/defaults.go
@@ -14,7 +14,9 @@ import (
 
 const (
 	// DefaultsKey is the key within the Defaults collection for
-	// scenario defaults.
+	// scenario defaults. Note that this isn't exposed in the YAML schema for a
+	// scenario. It's just used as a way of indicating to the scenario runner
+	// what was found in the scenario YAML's `defaults` top-level field.
 	DefaultsKey = "gdt.scenario"
 )
 
@@ -23,6 +25,9 @@ type Defaults struct {
 	// Timeout has fields that represent the default timeout behaviour and
 	// expectations to use for test specs in the scenario.
 	Timeout *gdttypes.Timeout `yaml:"timeout,omitempty"`
+	// Retry has fields that represent the default retry behaviour for test
+	// specs in the scenario.
+	Retry *gdttypes.Retry `yaml:"retry,omitempty"`
 }
 
 func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
@@ -52,6 +57,27 @@ func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
 				return err
 			}
 			d.Timeout = to
+		case "retry":
+			if valNode.Kind != yaml.MappingNode {
+				return errors.ExpectedMapAt(valNode)
+			}
+			var r *gdttypes.Retry
+			if err := valNode.Decode(&r); err != nil {
+				return errors.ExpectedRetryAt(valNode)
+			}
+			if r.Attempts != nil {
+				attempts := *r.Attempts
+				if attempts < 1 {
+					return errors.InvalidRetryAttempts(valNode, attempts)
+				}
+			}
+			if r.Interval != "" {
+				_, err := time.ParseDuration(r.Interval)
+				if err != nil {
+					return err
+				}
+			}
+			d.Retry = r
 		default:
 			continue
 		}
diff --git a/scenario/parse.go b/scenario/parse.go
index 4972ae3..753432e 100644
--- a/scenario/parse.go
+++ b/scenario/parse.go
@@ -83,6 +83,8 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 			s.Defaults = defaults
 		}
 	}
+	// We store a lookup to the parsing plugin for each parsed test spec
+	evalPlugins := map[int]gdttypes.Plugin{}
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
@@ -103,21 +105,24 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 				}
 				base.Index = idx
 				base.Defaults = &defaults
-				specs := []gdttypes.Evaluable{}
+				pluginSpecs := map[gdttypes.Plugin][]gdttypes.Evaluable{}
 				for _, p := range plugins {
-					specs = append(specs, p.Specs()...)
+					pluginSpecs[p] = p.Specs()
 				}
-				for _, sp := range specs {
-					if err := testNode.Decode(sp); err != nil {
-						if errors.Is(err, gdterrors.ErrUnknownField) {
-							continue
+				for plugin, specs := range pluginSpecs {
+					for _, sp := range specs {
+						if err := testNode.Decode(sp); err != nil {
+							if errors.Is(err, gdterrors.ErrUnknownField) {
+								continue
+							}
+							return err
 						}
-						return err
+						sp.SetBase(base)
+						s.Tests = append(s.Tests, sp)
+						parsed = true
+						evalPlugins[idx] = plugin
+						break
 					}
-					sp.SetBase(base)
-					s.Tests = append(s.Tests, sp)
-					parsed = true
-					break
 				}
 				if !parsed {
 					return gdterrors.UnknownSpecAt(s.Path, valNode)
@@ -157,5 +162,6 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 			}
 		}
 	}
+	s.evalPlugins = evalPlugins
 	return nil
 }
diff --git a/scenario/parse_test.go b/scenario/parse_test.go
index c27cec1..8991ea9 100644
--- a/scenario/parse_test.go
+++ b/scenario/parse_test.go
@@ -131,6 +131,45 @@ func TestBadTimeoutDurationScenario(t *testing.T) {
 	assert.Nil(s)
 }
 
+func TestBadRetry(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "parse", "fail", "bad-retry.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	assert.ErrorIs(err, errors.ErrExpectedMap)
+	assert.Nil(s)
+}
+
+func TestBadRetryAttempts(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "parse", "fail", "bad-retry-attempts.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	assert.ErrorIs(err, errors.ErrInvalidRetryAttempts)
+	assert.Nil(s)
+}
+
+func TestBadRetryIntervalDuration(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "parse", "fail", "bad-retry-interval-duration.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	assert.ErrorContains(err, "invalid duration")
+	assert.Nil(s)
+}
+
 func TestKnownSpec(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
diff --git a/scenario/run.go b/scenario/run.go
index 7d0948b..877035d 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -6,13 +6,17 @@ package scenario
 
 import (
 	"context"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/cenkalti/backoff"
+
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
 	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/result"
 	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
@@ -65,7 +69,11 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 		defer func() {
 			ctx = gdtcontext.PopTrace(ctx)
 		}()
-		for _, spec := range s.Tests {
+		for idx, spec := range s.Tests {
+			plugin := s.evalPlugins[idx]
+			pinfo := plugin.Info()
+			pretry := pinfo.Retry
+
 			// Create a brand new context that inherits the top-level context's
 			// cancel func. We want to set deadlines for each test spec and if
 			// we mutate the single supplied top-level context, then only the
@@ -80,17 +88,91 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				time.Sleep(wait.BeforeDuration())
 			}
 
-			to := specTimeout(ctx, t, sb.Timeout, scDefaults)
+			to := getTimeout(ctx, sb.Timeout, scDefaults)
 			if to != nil {
 				var cancel context.CancelFunc
 				specCtx, cancel = context.WithTimeout(specCtx, to.Duration())
 				defer cancel()
 			}
-			res := spec.Eval(specCtx, t)
-			if res.HasRuntimeError() {
-				rterr = res.RuntimeError()
-				t.Fatal(rterr)
-				break
+
+			var res *result.Result
+			rt := getRetry(ctx, sb.Retry, scDefaults, pretry)
+			if rt == nil {
+				// Just evaluate the test spec once
+				res = spec.Eval(specCtx, t)
+				if res.HasRuntimeError() {
+					rterr = res.RuntimeError()
+					t.Fatal(rterr)
+				}
+				debug.Println(
+					ctx, "run: single-shot (no retries) ok: %v",
+					!res.Failed(),
+				)
+			} else {
+				// retry the action and test the assertions until they succeed,
+				// there is a terminal failure, or the timeout expires.
+				var bo backoff.BackOff
+
+				if rt.Exponential {
+					bo = backoff.WithContext(
+						backoff.NewExponentialBackOff(),
+						ctx,
+					)
+				} else {
+					interval := gdttypes.DefaultRetryConstantInterval
+					if rt.Interval != "" {
+						interval = rt.IntervalDuration()
+					}
+					bo = backoff.WithContext(
+						backoff.NewConstantBackOff(interval),
+						ctx,
+					)
+				}
+				ticker := backoff.NewTicker(bo)
+				maxAttempts := 0
+				if rt.Attempts != nil {
+					maxAttempts = *rt.Attempts
+				}
+				attempts := 1
+				start := time.Now().UTC()
+				success := false
+				for tick := range ticker.C {
+					if (maxAttempts > 0) && (attempts > maxAttempts) {
+						debug.Println(
+							ctx, "run: exceeded max attempts %d. stopping.",
+							maxAttempts,
+						)
+						ticker.Stop()
+						break
+					}
+					after := tick.Sub(start)
+
+					res = spec.Eval(specCtx, t)
+					if res.HasRuntimeError() {
+						rterr = res.RuntimeError()
+						t.Fatal(rterr)
+						break
+					}
+					success = !res.Failed()
+					debug.Println(
+						ctx, "run: attempt %d after %s ok: %v",
+						attempts, after, success,
+					)
+					if success {
+						ticker.Stop()
+						break
+					}
+					for _, f := range res.Failures() {
+						debug.Println(
+							ctx, "run: attempt %d after %s failure: %s",
+							attempts, after, f,
+						)
+					}
+					attempts++
+				}
+			}
+			for _, fail := range res.Failures() {
+				t.Error(fail)
 			}
 			// Results can have arbitrary run data stored in them and we
 			// save this prior run data in the top-level context (and pass
@@ -107,12 +189,11 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 	return rterr
 }
 
-// specTimeout returns the timeout value for the test spec. If the spec has a
+// getTimeout returns the timeout value for the test spec. If the spec has a
 // timeout override, we use that. Otherwise, we inspect the scenario's defaults
 // and, if present, use that timeout.
-func specTimeout(
+func getTimeout(
 	ctx context.Context,
-	t *testing.T,
 	specTimeout *gdttypes.Timeout,
 	scenDefaults *Defaults,
 ) *gdttypes.Timeout {
@@ -132,3 +213,54 @@ func specTimeout(
 	}
 	return nil
 }
+
+// getRetry returns the retry configuration for the test spec. If the spec has a
+// retry override, we use that. Otherwise, we inspect the scenario's defaults
+// and, if present, use that timeout. If the scenario's defaults do not
+// indicate a retry configuration, we ask the plugin if it has retry defaults
+// and use that.
+func getRetry(
+	ctx context.Context,
+	specRetry *gdttypes.Retry,
+	scenDefaults *Defaults,
+	pluginRetry *gdttypes.Retry,
+) *gdttypes.Retry {
+	if specRetry != nil {
+		msg := "using retry"
+		if specRetry.Attempts != nil {
+			msg += fmt.Sprintf(" (attempts: %d)", *specRetry.Attempts)
+		}
+		if specRetry.Interval != "" {
+			msg += fmt.Sprintf(" (interval: %s)", specRetry.Interval)
+		}
+		msg += fmt.Sprintf(" (exponential: %t)", specRetry.Exponential)
+		debug.Println(ctx, msg)
+		return specRetry
+	}
+	if scenDefaults != nil && scenDefaults.Retry != nil {
+		scenRetry := scenDefaults.Retry
+		msg := "using retry"
+		if scenRetry.Attempts != nil {
+			msg += fmt.Sprintf(" (attempts: %d)", *scenRetry.Attempts)
+		}
+		if scenRetry.Interval != "" {
+			msg += fmt.Sprintf(" (interval: %s)", scenRetry.Interval)
+		}
+		msg += fmt.Sprintf(" (exponential: %t) [scenario default]", scenRetry.Exponential)
+		debug.Println(ctx, msg)
+		return scenRetry
+	}
+	if pluginRetry != nil {
+		msg := "using retry"
+		if pluginRetry.Attempts != nil {
+			msg += fmt.Sprintf(" (attempts: %d)", *pluginRetry.Attempts)
+		}
+		if pluginRetry.Interval != "" {
+			msg += fmt.Sprintf(" (interval: %s)", pluginRetry.Interval)
+		}
+		msg += fmt.Sprintf(" (exponential: %t) [plugin default]", pluginRetry.Exponential)
+		debug.Println(ctx, msg)
+		return pluginRetry
+	}
+	return nil
+}
diff --git a/scenario/run_test.go b/scenario/run_test.go
index 7d98d23..f2585fa 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -8,7 +8,10 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"flag"
+	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 
@@ -19,6 +22,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var failFlag = flag.Bool("fail", false, "run tests expected to fail")
+
 func TestRun(t *testing.T) {
 	require := require.New(t)
 
@@ -111,19 +116,64 @@ func TestDebugFlushing(t *testing.T) {
 	require.Contains(debugout, "[gdt] [foo-debug-wait-flush] wait: 250ms before")
 }
 
-func TestTimeoutCascade(t *testing.T) {
+func TestNoRetry(t *testing.T) {
 	require := require.New(t)
 
-	fp := filepath.Join("testdata", "foo-timeout.yaml")
+	fp := filepath.Join("testdata", "no-retry.yaml")
 	f, err := os.Open(fp)
 	require.Nil(err)
 
+	var b bytes.Buffer
+	w := bufio.NewWriter(&b)
+	ctx := gdtcontext.New(gdtcontext.WithDebug(w))
+
 	s, err := scenario.FromReader(f, scenario.WithPath(fp))
 	require.Nil(err)
 	require.NotNil(s)
 
-	err = s.Run(context.TODO(), t)
+	err = s.Run(ctx, t)
 	require.Nil(err)
+	require.False(t.Failed())
+	w.Flush()
+	require.NotEqual(b.Len(), 0)
+	debugout := b.String()
+	require.Contains(debugout, "[gdt] [no-retry] run: single-shot (no retries) ok: true")
+}
+
+func TestFailRetryTestOverride(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "retry-test-override.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
+func TestRetryTestOverride(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailRetryTestOverride",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+
+	debugout := fmt.Sprintf("%s", outerr)
+	require.Contains(debugout, "[gdt] [retry-test-override] run: exceeded max attempts 2. stopping.")
 }
 
 func TestSkipIf(t *testing.T) {
diff --git a/scenario/scenario.go b/scenario/scenario.go
index 8d2a270..18716e2 100644
--- a/scenario/scenario.go
+++ b/scenario/scenario.go
@@ -13,6 +13,9 @@ import (
 // Scenario is a generalized gdt test case file. It contains a set of Runnable
 // test units.
 type Scenario struct {
+	// evalPlugins stores the plugin that will evaluate the test spec at a
+	// particular index
+	evalPlugins map[int]gdttypes.Plugin
 	// Path is the filepath to the test case.
 	Path string `yaml:"-"`
 	// Name is the short name for the test case. If empty, defaults to the base
diff --git a/scenario/stub_plugins_test.go b/scenario/stub_plugins_test.go
index 70287cb..f702bda 100644
--- a/scenario/stub_plugins_test.go
+++ b/scenario/stub_plugins_test.go
@@ -238,6 +238,9 @@ func (s *fooSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
 			fails = append(fails, fail)
 		}
 	})
+	for _, fail := range fails {
+		t.Error(fail)
+	}
 	return result.New(result.WithFailures(fails...))
 }
 
diff --git a/scenario/testdata/no-retry.yaml b/scenario/testdata/no-retry.yaml
new file mode 100644
index 0000000..4628858
--- /dev/null
+++ b/scenario/testdata/no-retry.yaml
@@ -0,0 +1,6 @@
+name: no-retry
+description: a scenario using a plugin with no retries
+tests:
+  # The foo plugin does not use retries
+  - foo: bar
+    name: bar
diff --git a/scenario/testdata/parse/fail/bad-retry-attempts.yaml b/scenario/testdata/parse/fail/bad-retry-attempts.yaml
new file mode 100644
index 0000000..f44b9ab
--- /dev/null
+++ b/scenario/testdata/parse/fail/bad-retry-attempts.yaml
@@ -0,0 +1,6 @@
+name: bad-retry-attempts
+description: a scenario with an invalid retry attempts
+tests:
+  - foo: baz
+    retry:
+      attempts: -1
diff --git a/scenario/testdata/parse/fail/bad-retry-interval-duration.yaml b/scenario/testdata/parse/fail/bad-retry-interval-duration.yaml
new file mode 100644
index 0000000..086c62c
--- /dev/null
+++ b/scenario/testdata/parse/fail/bad-retry-interval-duration.yaml
@@ -0,0 +1,6 @@
+name: bad-retry-interval-duration
+description: a scenario with an invalid duration string in a spec retry interval
+tests:
+  - foo: baz
+    retry:
+      interval: notaduration
diff --git a/scenario/testdata/parse/fail/bad-retry.yaml b/scenario/testdata/parse/fail/bad-retry.yaml
new file mode 100644
index 0000000..f68cf13
--- /dev/null
+++ b/scenario/testdata/parse/fail/bad-retry.yaml
@@ -0,0 +1,5 @@
+name: bad-retry
+description: a scenario with an invalid retry spec
+tests:
+  - foo: baz
+    retry: notaretry
diff --git a/scenario/testdata/retry-test-override.yaml b/scenario/testdata/retry-test-override.yaml
new file mode 100644
index 0000000..41c6747
--- /dev/null
+++ b/scenario/testdata/retry-test-override.yaml
@@ -0,0 +1,9 @@
+name: retry-test-override
+description: a scenario using a test spec override for a retry
+tests:
+  # The foo plugin fails if foo == bar but name != bar
+  - foo: bar
+    name: baz
+    retry:
+      attempts: 2
+      interval: .25s
diff --git a/types/plugin.go b/types/plugin.go
index a182d29..ff4043b 100644
--- a/types/plugin.go
+++ b/types/plugin.go
@@ -15,6 +15,9 @@ type PluginInfo struct {
 	Aliases []string
 	// Description describes what types of tests the plugin can handle.
 	Description string
+	// Retry is a Retry that should be used by default for test specs of this
+	// plugin.
+	Retry *Retry
 }
 
 // Plugin is the driver interface for different types of gdt tests.
diff --git a/types/retry.go b/types/retry.go
new file mode 100644
index 0000000..d8df33d
--- /dev/null
+++ b/types/retry.go
@@ -0,0 +1,46 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package types
+
+import (
+	"time"
+)
+
+const (
+	// DefaultRetryAttempts indicates the default number of times to retry
+	// retries when the plugin uses retries but has not specified a number of
+	// attempts.
+	DefaultRetryAttempts = 3 * time.Second
+	// DefaultRetryConstantInterval indicates the default interval to use for
+	// retries when the plugin uses retries but does not use exponential
+	// backoff.
+	DefaultRetryConstantInterval = 3 * time.Second
+)
+
+// Retry contains information about the number of attempts and interval
+// duration with which a Plugin should re-run a Spec's action if the Spec's
+// assertions fail.
+type Retry struct {
+	// Attempts is the number of  times that the test unit should be retried in
+	// the event of assertion failure.
+	Attempts *int `yaml:"attempts,omitempty"`
+	// Interval is the amount of time that the plugin should wait before
+	// retrying the test unit in the event of assertion failure.
+	// Specify a duration using Go's time duration string.
+	// See https://pkg.go.dev/time#ParseDuration
+	Interval string `yaml:"interval,omitempty"`
+	// Exponential indicates that an exponential backoff should be applied to
+	// the retry. When true, the value of Interval, if any, is used as the
+	// initial interval for the backoff algoritm.
+	Exponential bool `yaml:"exponential,omitempty"`
+}
+
+// IntervalDuration returns the time duration of the Retry.Interval
+func (r *Retry) IntervalDuration() time.Duration {
+	// Parsing already validated the duration string so no need to check again
+	// here
+	dur, _ := time.ParseDuration(r.Interval)
+	return dur
+}
diff --git a/types/spec.go b/types/spec.go
index 9bc5998..6d090ec 100644
--- a/types/spec.go
+++ b/types/spec.go
@@ -21,6 +21,7 @@ var (
 		"description",
 		"timeout",
 		"wait",
+		"retry",
 	}
 )
 
@@ -41,6 +42,8 @@ type Spec struct {
 	Timeout *Timeout `yaml:"timeout,omitempty"`
 	// Wait contains the wait configuration for the Spec
 	Wait *Wait `yaml:"wait,omitempty"`
+	// Retry contains the retry configuration for the Spec
+	Retry *Retry `yaml:"retry,omitempty"`
 }
 
 // Title returns the Name of the scenario or the Path's file/base name if there
@@ -135,6 +138,27 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 				}
 			}
 			s.Wait = w
+		case "retry":
+			if valNode.Kind != yaml.MappingNode {
+				return errors.ExpectedMapAt(valNode)
+			}
+			var r *Retry
+			if err := valNode.Decode(&r); err != nil {
+				return errors.ExpectedRetryAt(valNode)
+			}
+			if r.Attempts != nil {
+				attempts := *r.Attempts
+				if attempts < 1 {
+					return errors.InvalidRetryAttempts(valNode, attempts)
+				}
+			}
+			if r.Interval != "" {
+				_, err := time.ParseDuration(r.Interval)
+				if err != nil {
+					return err
+				}
+			}
+			s.Retry = r
 		}
 	}
 	return nil

From b870af3b637eaa3f9e1f56828110ec5e3ecbdcaf Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Thu, 20 Jun 2024 13:38:13 -0400
Subject: [PATCH 12/31] clean up any linter failures

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 .github/workflows/lint.yml  |  8 ++++----
 .golangci.yaml              |  6 ++++++
 assertion/json/errors.go    |  3 +--
 assertion/json/json.go      | 14 +++++++-------
 assertion/json/json_test.go |  4 ++--
 debug/print.go              |  2 ++
 fixture/generic_test.go     |  2 +-
 fixture/json/json.go        | 16 +++++++---------
 plugin/exec/action.go       | 11 +++++++----
 plugin/exec/defaults.go     | 13 -------------
 plugin/exec/eval_test.go    |  8 ++++----
 scenario/from.go            |  3 +--
 scenario/run_test.go        |  6 +++---
 suite/from.go               |  2 +-
 14 files changed, 46 insertions(+), 52 deletions(-)
 create mode 100644 .golangci.yaml

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 2ad167d..50c153f 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
      - name: harden runner
-       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          egress-policy: block
          disable-sudo: true
@@ -28,14 +28,14 @@ jobs:
            objects.githubusercontent.com:443
            proxy.golang.org:443
      - name: checkout code
-       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
          go-version: 1.21
      - name: lint
-       uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299 # v3.6.0
+       uses: golangci/golangci-lint-action@a4f60bb28d35aeee14e6880718e0c85ff1882e64 # v6.0.1
        with:
-         version: v1.53
+         version: v1.59.1
          args: --timeout=5m0s --verbose
          only-new-issues: true
diff --git a/.golangci.yaml b/.golangci.yaml
new file mode 100644
index 0000000..1425afb
--- /dev/null
+++ b/.golangci.yaml
@@ -0,0 +1,6 @@
+linters-settings:
+  staticcheck:
+    checks:
+      - all
+      - '-ST1000' # ignore at least one file in package should have comment
+      - '-EXC0003' # ignore the stuttered name error
diff --git a/assertion/json/errors.go b/assertion/json/errors.go
index 0f580b0..6bdb521 100644
--- a/assertion/json/errors.go
+++ b/assertion/json/errors.go
@@ -103,9 +103,8 @@ func JSONUnmarshalError(err error, node *yaml.Node) error {
 			"%w: %s at line %d, column %d",
 			ErrJSONUnmarshalError, err, node.Line, node.Column,
 		)
-	} else {
-		return fmt.Errorf("%w: %s", ErrJSONUnmarshalError, err)
 	}
+	return fmt.Errorf("%w: %s", ErrJSONUnmarshalError, err)
 }
 
 // JSONPathInvalid returns an ErrParse when a JSONPath expression could not be
diff --git a/assertion/json/json.go b/assertion/json/json.go
index 7a60584..8a23615 100644
--- a/assertion/json/json.go
+++ b/assertion/json/json.go
@@ -104,7 +104,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			if err := valNode.Decode(&paths); err != nil {
 				return err
 			}
-			for path, _ := range paths {
+			for path := range paths {
 				if len(path) == 0 || path[0] != '$' {
 					return JSONPathInvalidNoRoot(path, valNode)
 				}
@@ -121,7 +121,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			if err := valNode.Decode(&pathFormats); err != nil {
 				return err
 			}
-			for pathFormat, _ := range pathFormats {
+			for pathFormat := range pathFormats {
 				if len(pathFormat) == 0 || pathFormat[0] != '$' {
 					return JSONPathInvalidNoRoot(pathFormat, valNode)
 				}
@@ -229,9 +229,9 @@ func (a *assertions) pathsOK() bool {
 			a.Fail(JSONPathNotFound(path, err))
 			return false
 		}
-		switch got.(type) {
+		switch got := got.(type) {
 		case string:
-			if expVal != got.(string) {
+			if expVal != got {
 				a.Fail(JSONPathNotEqual(path, expVal, got))
 				return false
 			}
@@ -241,7 +241,7 @@ func (a *assertions) pathsOK() bool {
 				a.Fail(JSONPathConversionError(path, expVal, got))
 				return false
 			}
-			if expValInt != got.(int) {
+			if expValInt != got {
 				a.Fail(JSONPathNotEqual(path, expVal, got))
 				return false
 			}
@@ -251,7 +251,7 @@ func (a *assertions) pathsOK() bool {
 				a.Fail(JSONPathConversionError(path, expVal, got))
 				return false
 			}
-			if expValFloat != got.(float64) {
+			if expValFloat != got {
 				a.Fail(JSONPathNotEqual(path, expVal, got))
 				return false
 			}
@@ -261,7 +261,7 @@ func (a *assertions) pathsOK() bool {
 				a.Fail(JSONPathConversionError(path, expVal, got))
 				return false
 			}
-			if expValBool != got.(bool) {
+			if expValBool != got {
 				a.Fail(JSONPathNotEqual(path, expVal, got))
 				return false
 			}
diff --git a/assertion/json/json_test.go b/assertion/json/json_test.go
index 3eed0e2..6b2fdbd 100644
--- a/assertion/json/json_test.go
+++ b/assertion/json/json_test.go
@@ -6,7 +6,7 @@ package json_test
 
 import (
 	"context"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -92,7 +92,7 @@ paths:
 }
 
 func content() []byte {
-	b, _ := ioutil.ReadFile(filepath.Join("testdata", "books.json"))
+	b, _ := os.ReadFile(filepath.Join("testdata", "books.json"))
 	return b
 }
 
diff --git a/debug/print.go b/debug/print.go
index 8aa9d28..19eb949 100644
--- a/debug/print.go
+++ b/debug/print.go
@@ -31,6 +31,7 @@ func Printf(
 	}
 	msg := fmt.Sprintf(format, args...)
 	for _, w := range writers {
+		//nolint:errcheck
 		w.Write([]byte(msg))
 	}
 }
@@ -57,6 +58,7 @@ func Println(
 	}
 	msg := fmt.Sprintf(format, args...)
 	for _, w := range writers {
+		//nolint:errcheck
 		w.Write([]byte(msg))
 	}
 }
diff --git a/fixture/generic_test.go b/fixture/generic_test.go
index 95e2b37..2fe567c 100644
--- a/fixture/generic_test.go
+++ b/fixture/generic_test.go
@@ -47,7 +47,7 @@ func TestStarter(t *testing.T) {
 
 	assert.False(started)
 
-	f.Start(context.TODO())
+	_ = f.Start(context.TODO())
 
 	assert.True(started)
 }
diff --git a/fixture/json/json.go b/fixture/json/json.go
index b034106..69f6b3a 100644
--- a/fixture/json/json.go
+++ b/fixture/json/json.go
@@ -8,7 +8,6 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"strconv"
 
 	"github.com/PaesslerAG/jsonpath"
@@ -49,11 +48,11 @@ func (f *jsonFixture) State(path string) interface{} {
 	if err != nil {
 		return nil
 	}
-	switch got.(type) {
+	switch got := got.(type) {
 	case string:
-		return got.(string)
+		return got
 	case float64:
-		return strconv.FormatFloat(got.(float64), 'f', 0, 64)
+		return strconv.FormatFloat(got, 'f', 0, 64)
 	default:
 		return nil
 	}
@@ -64,17 +63,16 @@ func (f *jsonFixture) State(path string) interface{} {
 func New(data interface{}) (gdttypes.Fixture, error) {
 	var err error
 	var b []byte
-	switch data.(type) {
+	switch data := data.(type) {
 	case io.Reader:
-		r := data.(io.Reader)
-		b, err = ioutil.ReadAll(r)
+		b, err = io.ReadAll(data)
 		if err != nil {
 			return nil, err
 		}
 	case []byte:
-		b = data.([]byte)
+		b = data
 	case string:
-		b = []byte(data.(string))
+		b = []byte(data)
 	}
 	f := jsonFixture{
 		data: interface{}(nil),
diff --git a/plugin/exec/action.go b/plugin/exec/action.go
index da3f753..f1b8ef2 100644
--- a/plugin/exec/action.go
+++ b/plugin/exec/action.go
@@ -58,8 +58,7 @@ func (a *Action) Do(
 
 	debug.Println(ctx, "exec: %s %s", target, args)
 
-	var cmd *exec.Cmd
-	cmd = exec.CommandContext(ctx, target, args...)
+	cmd := exec.CommandContext(ctx, target, args...)
 
 	outpipe, err := cmd.StdoutPipe()
 	if err != nil {
@@ -78,13 +77,17 @@ func (a *Action) Do(
 		return err
 	}
 	if outbuf != nil {
-		outbuf.ReadFrom(outpipe)
+		if _, err = outbuf.ReadFrom(outpipe); err != nil {
+			debug.Println(ctx, "exec: error reading from stdout: %s", err)
+		}
 		if outbuf.Len() > 0 {
 			debug.Println(ctx, "exec: stdout: %s", outbuf.String())
 		}
 	}
 	if errbuf != nil {
-		errbuf.ReadFrom(errpipe)
+		if _, err = errbuf.ReadFrom(errpipe); err != nil {
+			debug.Println(ctx, "exec: error reading from stderr: %s", err)
+		}
 		if errbuf.Len() > 0 {
 			debug.Println(ctx, "exec: stderr: %s", errbuf.String())
 		}
diff --git a/plugin/exec/defaults.go b/plugin/exec/defaults.go
index ecf6e32..5be9bd9 100644
--- a/plugin/exec/defaults.go
+++ b/plugin/exec/defaults.go
@@ -8,7 +8,6 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/gdt-dev/gdt/errors"
-	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
 type execDefaults struct{}
@@ -47,15 +46,3 @@ func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
 	}
 	return nil
 }
-
-// fromBaseDefaults returns an exec plugin-specific Defaults from a Spec
-func fromBaseDefaults(base *gdttypes.Defaults) *Defaults {
-	if base == nil {
-		return nil
-	}
-	d := base.For(pluginName)
-	if d == nil {
-		return nil
-	}
-	return d.(*Defaults)
-}
diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index b3e9bc9..62820e1 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -9,7 +9,6 @@ import (
 	"bytes"
 	"context"
 	"flag"
-	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -196,7 +195,7 @@ func TestExecSleepTimeout(t *testing.T) {
 
 	// The test should have failed...
 	require.NotNil(err)
-	debugout := fmt.Sprintf("%s", outerr)
+	debugout := string(outerr)
 	require.Contains(debugout, "assertion failed: timeout exceeded")
 }
 
@@ -273,6 +272,7 @@ func TestFailExecTimeoutCascade(t *testing.T) {
 
 	ctx := gdtcontext.New(gdtcontext.WithDebug())
 	err = s.Run(ctx, t)
+	require.Nil(err)
 }
 
 func TestExecTimeoutCascade(t *testing.T) {
@@ -288,7 +288,7 @@ func TestExecTimeoutCascade(t *testing.T) {
 	// The test should have failed...
 	require.NotNil(err)
 
-	debugout := fmt.Sprintf("%s", outerr)
+	debugout := string(outerr)
 	require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]")
 	require.Contains(debugout, "using timeout of 20ms (expected: true)")
 }
@@ -328,7 +328,7 @@ func TestExecOnFail(t *testing.T) {
 	// The test should have failed...
 	require.NotNil(err)
 
-	debugout := fmt.Sprintf("%s", outerr)
+	debugout := string(outerr)
 	require.Contains(debugout, "assertion failed: not equal: expected dat but got cat")
 	require.Contains(debugout, "echo [bad kitty]")
 }
diff --git a/scenario/from.go b/scenario/from.go
index 4a71bee..44cc7bc 100644
--- a/scenario/from.go
+++ b/scenario/from.go
@@ -6,7 +6,6 @@ package scenario
 
 import (
 	"io"
-	"io/ioutil"
 
 	"gopkg.in/yaml.v3"
 
@@ -20,7 +19,7 @@ func FromReader(
 	r io.Reader,
 	mods ...ScenarioModifier,
 ) (*Scenario, error) {
-	contents, err := ioutil.ReadAll(r)
+	contents, err := io.ReadAll(r)
 	if err != nil {
 		return nil, err
 	}
diff --git a/scenario/run_test.go b/scenario/run_test.go
index f2585fa..e3a1432 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -9,7 +9,6 @@ import (
 	"bytes"
 	"context"
 	"flag"
-	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -35,7 +34,8 @@ func TestRun(t *testing.T) {
 	require.Nil(err)
 	require.NotNil(s)
 
-	s.Run(context.TODO(), t)
+	err = s.Run(context.TODO(), t)
+	require.Nil(err)
 }
 
 func TestPriorRun(t *testing.T) {
@@ -172,7 +172,7 @@ func TestRetryTestOverride(t *testing.T) {
 	// The test should have failed...
 	require.NotNil(err)
 
-	debugout := fmt.Sprintf("%s", outerr)
+	debugout := string(outerr)
 	require.Contains(debugout, "[gdt] [retry-test-override] run: exceeded max attempts 2. stopping.")
 }
 
diff --git a/suite/from.go b/suite/from.go
index b8b0c0c..d094452 100644
--- a/suite/from.go
+++ b/suite/from.go
@@ -31,7 +31,7 @@ func FromDir(
 
 	if err := filepath.Walk(
 		dirPath,
-		func(path string, info os.FileInfo, err error) error {
+		func(path string, info os.FileInfo, _ error) error {
 			if info.IsDir() {
 				return nil
 			}

From 1ac476c4b2884e4e551591071c822999d6784845 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Thu, 20 Jun 2024 15:03:50 -0400
Subject: [PATCH 13/31] allow plugin-defaulted timeout values

Allows plugins to specify their own default timeout values. In addition,
removes the `timeout.expected` flag which was only a crutch for before
we figured out how to test assertion failures in a clean manner.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 README.md                                     |  7 +--
 plugin/exec/eval_test.go                      | 48 ++++++++++++++++++-
 plugin/exec/plugin.go                         | 12 +++++
 plugin/exec/testdata/sleep-timeout.yaml       |  1 -
 plugin/exec/testdata/timeout-cascade.yaml     |  1 -
 .../exec/testdata/timeout-plugin-default.yaml |  5 ++
 scenario/parse_test.go                        | 14 +++++-
 scenario/run.go                               | 23 ++++++---
 scenario/testdata/parse/fail/bad-timeout.yaml |  3 +-
 .../testdata/parse/timeout-scalar-or-map.yaml |  8 ++++
 types/plugin.go                               |  3 ++
 types/spec.go                                 | 18 +++++--
 types/timeout.go                              |  3 --
 13 files changed, 120 insertions(+), 26 deletions(-)
 create mode 100644 plugin/exec/testdata/timeout-plugin-default.yaml
 create mode 100644 scenario/testdata/parse/timeout-scalar-or-map.yaml

diff --git a/README.md b/README.md
index da837d1..72da0a7 100644
--- a/README.md
+++ b/README.md
@@ -458,12 +458,8 @@ All test specs have the following fields:
 
 * `name`: (optional) string describing the test unit.
 * `description`: (optional) string with longer description of the test unit.
-* `timeout`: (optional) an object containing [timeout information][timeout] for the test
-  unit.
-* `timeout.after`: a string duration of time the test unit is expected to
+* `timeout`: (optional) a string duration of time the test unit is expected to
   complete within.
-* `timeout.expected`: a bool indicating that the test unit is expected to not
-  complete before `timeout.after`. This is really only useful in unit testing.
 * `retry`: (optional) an object containing retry configurationu for the test
   unit. Some plugins will automatically attempt to retry the test action when
   an assertion fails. This field allows you to control this retry behaviour for
@@ -497,7 +493,6 @@ All test specs have the following fields:
 [exec-plugin]: https://github.com/gdt-dev/gdt/tree/ecee17249e1fa10147cf9191be0358923da44094/plugin/exec
 [http-plugin]: https://github.com/gdt-dev/http
 [kube-plugin]: https://github.com/gdt-dev/kube
-[timeout]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/timeout.go#L11-L22
 [wait]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/types/wait.go#L11-L25
 
 #### `exec` test spec structure
diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index 62820e1..f433ea2 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -16,10 +16,15 @@ import (
 	"testing"
 
 	gdtcontext "github.com/gdt-dev/gdt/context"
+	execplugin "github.com/gdt-dev/gdt/plugin/exec"
 	"github.com/gdt-dev/gdt/scenario"
 	"github.com/stretchr/testify/require"
 )
 
+func init() {
+	execplugin.OverrideDefaultTimeout("0.5s")
+}
+
 var failFlag = flag.Bool("fail", false, "run tests expected to fail")
 
 func TestNoExitCodeSimpleCommand(t *testing.T) {
@@ -161,6 +166,45 @@ func TestContainsNoneOf(t *testing.T) {
 	require.Nil(err)
 }
 
+func TestFailExecTimeoutPluginDefault(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "timeout-plugin-default.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
+func TestExecTimeoutPluginDefault(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailExecTimeoutPluginDefault",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+	debugout := string(outerr)
+	require.Contains(debugout, "using timeout of 0.5s [plugin default]")
+	require.Contains(debugout, "assertion failed: timeout exceeded")
+}
+
 func TestFailExecSleepTimeout(t *testing.T) {
 	if !*failFlag {
 		t.Skip("skipping without -fail flag")
@@ -289,8 +333,8 @@ func TestExecTimeoutCascade(t *testing.T) {
 	require.NotNil(err)
 
 	debugout := string(outerr)
-	require.Contains(debugout, "using timeout of 500ms (expected: false) [scenario default]")
-	require.Contains(debugout, "using timeout of 20ms (expected: true)")
+	require.Contains(debugout, "using timeout of 500ms [scenario default]")
+	require.Contains(debugout, "using timeout of 20ms")
 }
 
 func TestFailExecOnFail(t *testing.T) {
diff --git a/plugin/exec/plugin.go b/plugin/exec/plugin.go
index 6fb0c9d..222c86d 100644
--- a/plugin/exec/plugin.go
+++ b/plugin/exec/plugin.go
@@ -11,6 +11,15 @@ import (
 	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
+var (
+	DefaultTimeout = "10s"
+)
+
+// OverrideDefaultTimeout is only used in testing...
+func OverrideDefaultTimeout(d string) {
+	DefaultTimeout = d
+}
+
 func init() {
 	gdtplugin.Register(Plugin())
 }
@@ -24,6 +33,9 @@ type plugin struct{}
 func (p *plugin) Info() gdttypes.PluginInfo {
 	return gdttypes.PluginInfo{
 		Name: pluginName,
+		Timeout: &gdttypes.Timeout{
+			After: DefaultTimeout,
+		},
 	}
 }
 
diff --git a/plugin/exec/testdata/sleep-timeout.yaml b/plugin/exec/testdata/sleep-timeout.yaml
index 5880423..d0d9725 100644
--- a/plugin/exec/testdata/sleep-timeout.yaml
+++ b/plugin/exec/testdata/sleep-timeout.yaml
@@ -7,4 +7,3 @@ tests:
   - exec: sleep 5
     timeout:
       after: 50ms
-      expected: true
diff --git a/plugin/exec/testdata/timeout-cascade.yaml b/plugin/exec/testdata/timeout-cascade.yaml
index 6a0cc5c..af243c7 100644
--- a/plugin/exec/testdata/timeout-cascade.yaml
+++ b/plugin/exec/testdata/timeout-cascade.yaml
@@ -10,4 +10,3 @@ tests:
     exec: sleep .25
     timeout:
       after: 20ms
-      expected: true
diff --git a/plugin/exec/testdata/timeout-plugin-default.yaml b/plugin/exec/testdata/timeout-plugin-default.yaml
new file mode 100644
index 0000000..499bda4
--- /dev/null
+++ b/plugin/exec/testdata/timeout-plugin-default.yaml
@@ -0,0 +1,5 @@
+name: timeout-plugin-default
+description: a scenario that tests the plugin default timeout
+tests:
+  - name: uses plugin default timeout
+    exec: sleep 1
diff --git a/scenario/parse_test.go b/scenario/parse_test.go
index 8991ea9..14bb855 100644
--- a/scenario/parse_test.go
+++ b/scenario/parse_test.go
@@ -92,6 +92,18 @@ func TestUnknownSpec(t *testing.T) {
 	assert.Nil(s)
 }
 
+func TestTimeoutScalarOrMap(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "parse", "timeout-scalar-or-map.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	_, err = scenario.FromReader(f, scenario.WithPath(fp))
+	assert.Nil(err)
+}
+
 func TestBadTimeout(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
@@ -101,7 +113,7 @@ func TestBadTimeout(t *testing.T) {
 	require.Nil(err)
 
 	s, err := scenario.FromReader(f, scenario.WithPath(fp))
-	assert.ErrorIs(err, errors.ErrExpectedMap)
+	assert.ErrorIs(err, errors.ErrExpectedScalarOrMap)
 	assert.Nil(s)
 }
 
diff --git a/scenario/run.go b/scenario/run.go
index 877035d..36bb477 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -73,6 +73,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			plugin := s.evalPlugins[idx]
 			pinfo := plugin.Info()
 			pretry := pinfo.Retry
+			ptimeout := pinfo.Timeout
 
 			// Create a brand new context that inherits the top-level context's
 			// cancel func. We want to set deadlines for each test spec and if
@@ -88,7 +89,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				time.Sleep(wait.BeforeDuration())
 			}
 
-			to := getTimeout(ctx, sb.Timeout, scDefaults)
+			to := getTimeout(ctx, sb.Timeout, ptimeout, scDefaults)
 			if to != nil {
 				var cancel context.CancelFunc
 				specCtx, cancel = context.WithTimeout(specCtx, to.Duration())
@@ -191,26 +192,36 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 
 // getTimeout returns the timeout value for the test spec. If the spec has a
 // timeout override, we use that. Otherwise, we inspect the scenario's defaults
-// and, if present, use that timeout.
+// and, if present, use that timeout. If the scenario's defaults for not
+// indicate a timeout configuration, we ask the plugin if it has timeout
+// defaults and use that.
 func getTimeout(
 	ctx context.Context,
 	specTimeout *gdttypes.Timeout,
+	pluginTimeout *gdttypes.Timeout,
 	scenDefaults *Defaults,
 ) *gdttypes.Timeout {
 	if specTimeout != nil {
 		debug.Println(
-			ctx, "using timeout of %s (expected: %t)",
-			specTimeout.After, specTimeout.Expected,
+			ctx, "using timeout of %s",
+			specTimeout.After,
 		)
 		return specTimeout
 	}
 	if scenDefaults != nil && scenDefaults.Timeout != nil {
 		debug.Println(
-			ctx, "using timeout of %s (expected: %t) [scenario default]",
-			scenDefaults.Timeout.After, scenDefaults.Timeout.Expected,
+			ctx, "using timeout of %s [scenario default]",
+			scenDefaults.Timeout.After,
 		)
 		return scenDefaults.Timeout
 	}
+	if pluginTimeout != nil {
+		debug.Println(
+			ctx, "using timeout of %s [plugin default]",
+			pluginTimeout.After,
+		)
+		return pluginTimeout
+	}
 	return nil
 }
 
diff --git a/scenario/testdata/parse/fail/bad-timeout.yaml b/scenario/testdata/parse/fail/bad-timeout.yaml
index 220ecd9..1c56936 100644
--- a/scenario/testdata/parse/fail/bad-timeout.yaml
+++ b/scenario/testdata/parse/fail/bad-timeout.yaml
@@ -2,4 +2,5 @@ name: bad-timeout
 description: a scenario with an invalid timeout spec
 tests:
   - foo: baz
-    timeout: notatimeout
+    timeout:
+      - one
diff --git a/scenario/testdata/parse/timeout-scalar-or-map.yaml b/scenario/testdata/parse/timeout-scalar-or-map.yaml
new file mode 100644
index 0000000..06ac989
--- /dev/null
+++ b/scenario/testdata/parse/timeout-scalar-or-map.yaml
@@ -0,0 +1,8 @@
+name: timeout-scalar-or-map
+description: a scenario with both scalar and object timeout spec
+tests:
+  - foo: baz
+    timeout: 1s
+  - foo: bar
+    timeout:
+      after: 1s
diff --git a/types/plugin.go b/types/plugin.go
index ff4043b..151d7b5 100644
--- a/types/plugin.go
+++ b/types/plugin.go
@@ -15,6 +15,9 @@ type PluginInfo struct {
 	Aliases []string
 	// Description describes what types of tests the plugin can handle.
 	Description string
+	// Timeout is a Timeout that should be used by default for test specs of
+	// this plugin.
+	Timeout *Timeout
 	// Retry is a Retry that should be used by default for test specs of this
 	// plugin.
 	Retry *Retry
diff --git a/types/spec.go b/types/spec.go
index 6d090ec..ebd6762 100644
--- a/types/spec.go
+++ b/types/spec.go
@@ -105,12 +105,20 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			}
 			s.Description = valNode.Value
 		case "timeout":
-			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
-			}
 			var to *Timeout
-			if err := valNode.Decode(&to); err != nil {
-				return errors.ExpectedTimeoutAt(valNode)
+			switch valNode.Kind {
+			case yaml.MappingNode:
+				// We support the old-style timeout:after
+				if err := valNode.Decode(&to); err != nil {
+					return errors.ExpectedTimeoutAt(valNode)
+				}
+			case yaml.ScalarNode:
+				// We also support a straight string duration
+				to = &Timeout{
+					After: valNode.Value,
+				}
+			default:
+				return errors.ExpectedScalarOrMapAt(valNode)
 			}
 			_, err := time.ParseDuration(to.After)
 			if err != nil {
diff --git a/types/timeout.go b/types/timeout.go
index d14e889..bdd32e9 100644
--- a/types/timeout.go
+++ b/types/timeout.go
@@ -16,9 +16,6 @@ type Timeout struct {
 	// Specify a duration using Go's time duration string.
 	// See https://pkg.go.dev/time#ParseDuration
 	After string `yaml:"after,omitempty"`
-	// Expected indicates whether the timeout is expected to be exceeded. This
-	// is mostly useful for unit testing of the timeout functionality itself.
-	Expected bool `yaml:"expected,omitempty"`
 }
 
 // Duration returns the time duration of the Timeout

From 36b25699476d5bbbdc4c5c171f3662a7adae0d64 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Fri, 21 Jun 2024 09:41:25 -0400
Subject: [PATCH 14/31] simplify Evaluable interface and Scenario.Run

No longer pass the `*testing.T` down into Plugin's Spec.Eval() method,
freeing up plugins to focus on actions and assertions. Scenario.Run()
now handles the `*testing.T` and the trace context centrally instead of
plugins needing to do this.

Also simplifies the Scenario.Run() interface and Evaluable.Eval()
interfaces to return (result.Result, error) to more cleanly pass back
RuntimeErrors instead of needing a Result.RuntimeError() method.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 README.md                     | 102 +++++++++++++++++
 context/context_test.go       |   4 +-
 plugin/exec/action.go         |   2 -
 plugin/exec/eval.go           |  17 +--
 plugin/registry_test.go       |   4 +-
 result/result.go              |  34 ------
 scenario/run.go               | 204 +++++++++++++++++++---------------
 scenario/run_test.go          |   4 +-
 scenario/stub_plugins_test.go |  80 ++++++-------
 types/evaluable.go            |   6 +-
 10 files changed, 272 insertions(+), 185 deletions(-)

diff --git a/README.md b/README.md
index 72da0a7..107fc84 100644
--- a/README.md
+++ b/README.md
@@ -540,6 +540,108 @@ test spec also contains these fields:
 [execspec]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/spec.go#L11-L34
 [pipeexpect]: https://github.com/gdt-dev/gdt/blob/2791e11105fd3c36d1f11a7d111e089be7cdc84c/exec/assertions.go#L15-L26
 
+### Timeouts and retrying assertions
+
+When evaluating assertions for a test spec, `gdt` inspects the test's
+`timeout` value to determine how long to retry the `get` call and recheck
+the assertions.
+
+If a test's `timeout` is empty, `gdt` inspects the scenario's
+`defaults.timeout` value. If both of those values are empty, `gdt` will look
+for any default `timeout` value that the plugin uses.
+
+If you're interested in seeing the individual results of `gdt`'s
+assertion-checks for a single `get` call, you can use the `gdt.WithDebug()`
+function, like this test function demonstrates:
+
+file: `testdata/matches.yaml`:
+
+```yaml
+name: matches
+description: create a deployment and check the matches condition succeeds
+fixtures:
+  - kind
+tests:
+  - name: create-deployment
+    kube:
+      create: testdata/manifests/nginx-deployment.yaml
+  - name: deployment-exists
+    kube:
+      get: deployments/nginx
+    assert:
+      matches:
+        spec:
+          replicas: 2
+          template:
+            metadata:
+              labels:
+                app: nginx
+        status:
+          readyReplicas: 2
+  - name: delete-deployment
+    kube:
+      delete: deployments/nginx
+```
+
+file: `matches_test.go`
+
+```go
+import (
+    "github.com/gdt-dev/gdt"
+    _ "github.com/gdt-dev/kube"
+    kindfix "github.com/gdt-dev/kube/fixture/kind"
+)
+
+func TestMatches(t *testing.T) {
+	fp := filepath.Join("testdata", "matches.yaml")
+
+	kfix := kindfix.New()
+
+	s, err := gdt.From(fp)
+
+	ctx := gdt.NewContext(gdt.WithDebug())
+	ctx = gdt.RegisterFixture(ctx, "kind", kfix)
+	s.Run(ctx, t)
+}
+```
+
+Here's what running `go test -v matches_test.go` would look like:
+
+```
+$ go test -v matches_test.go
+=== RUN   TestMatches
+=== RUN   TestMatches/matches
+=== RUN   TestMatches/matches/create-deployment
+=== RUN   TestMatches/matches/deployment-exists
+deployment-exists (try 1 after 1.303µs) ok: false, terminal: false
+deployment-exists (try 1 after 1.303µs) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 2 after 595.62786ms) ok: false, terminal: false
+deployment-exists (try 2 after 595.62786ms) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 3 after 1.020003807s) ok: false, terminal: false
+deployment-exists (try 3 after 1.020003807s) failure: assertion failed: match field not equal: $.status.readyReplicas not present in subject
+deployment-exists (try 4 after 1.760006109s) ok: false, terminal: false
+deployment-exists (try 4 after 1.760006109s) failure: assertion failed: match field not equal: $.status.readyReplicas had different values. expected 2 but found 1
+deployment-exists (try 5 after 2.772416449s) ok: true, terminal: false
+=== RUN   TestMatches/matches/delete-deployment
+--- PASS: TestMatches (3.32s)
+    --- PASS: TestMatches/matches (3.30s)
+        --- PASS: TestMatches/matches/create-deployment (0.01s)
+        --- PASS: TestMatches/matches/deployment-exists (2.78s)
+        --- PASS: TestMatches/matches/delete-deployment (0.02s)
+PASS
+ok  	command-line-arguments	3.683s
+```
+
+You can see from the debug output above that `gdt` created the Deployment and
+then did a `kube.get` for the `deployments/nginx` Deployment. Initially
+(attempt 1), the `assert.matches` assertion failed because the
+`status.readyReplicas` field was not present in the returned resource. `gdt`
+retried the `kube.get` call 4 more times (attempts 2-5), with attempts 2 and 3
+failed the existence check for the `status.readyReplicas` field and attempt 4
+failing the *value* check for the `status.readyReplicas` field being `1`
+instead of the expected `2`. Finally, when the Deployment was completely rolled
+out, attempt 5 succeeded in all the `assert.matches` assertions.
+
 ## Contributing and acknowledgements
 
 `gdt` was inspired by [Gabbi](https://github.com/cdent/gabbi), the excellent
diff --git a/context/context_test.go b/context/context_test.go
index 6259926..7644948 100644
--- a/context/context_test.go
+++ b/context/context_test.go
@@ -43,8 +43,8 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-func (s *fooSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
-	return nil
+func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
+	return nil, nil
 }
 
 type fooPlugin struct{}
diff --git a/plugin/exec/action.go b/plugin/exec/action.go
index f1b8ef2..d172ef0 100644
--- a/plugin/exec/action.go
+++ b/plugin/exec/action.go
@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"context"
 	"os/exec"
-	"testing"
 
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
@@ -38,7 +37,6 @@ type Action struct {
 // respectively.
 func (a *Action) Do(
 	ctx context.Context,
-	t *testing.T,
 	outbuf *bytes.Buffer,
 	errbuf *bytes.Buffer,
 	exitcode *int,
diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go
index 8897972..a02d3cf 100644
--- a/plugin/exec/eval.go
+++ b/plugin/exec/eval.go
@@ -7,7 +7,6 @@ package exec
 import (
 	"bytes"
 	"context"
-	"testing"
 
 	"github.com/gdt-dev/gdt/debug"
 	gdterrors "github.com/gdt-dev/gdt/errors"
@@ -17,17 +16,21 @@ import (
 // Eval performs an action and evaluates the results of that action, returning
 // a Result that informs the Scenario about what failed or succeeded about the
 // Evaluable's conditions.
-func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
+//
+// Errors returned by Eval() are **RuntimeErrors**, not failures in assertions.
+func (s *Spec) Eval(
+	ctx context.Context,
+) (*result.Result, error) {
 	outbuf := &bytes.Buffer{}
 	errbuf := &bytes.Buffer{}
 
 	var ec int
 
-	if err := s.Do(ctx, t, outbuf, errbuf, &ec); err != nil {
+	if err := s.Do(ctx, outbuf, errbuf, &ec); err != nil {
 		if err == gdterrors.ErrTimeoutExceeded {
-			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded))
+			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)), nil
 		}
-		return result.New(result.WithRuntimeError(ExecRuntimeError(err)))
+		return nil, ExecRuntimeError(err)
 	}
 	a := newAssertions(s.Assert, ec, outbuf, errbuf)
 	if !a.OK(ctx) {
@@ -35,12 +38,12 @@ func (s *Spec) Eval(ctx context.Context, t *testing.T) *result.Result {
 			if s.On.Fail != nil {
 				outbuf.Reset()
 				errbuf.Reset()
-				err := s.On.Fail.Do(ctx, t, outbuf, errbuf, nil)
+				err := s.On.Fail.Do(ctx, outbuf, errbuf, nil)
 				if err != nil {
 					debug.Println(ctx, "error in on.fail.exec: %s", err)
 				}
 			}
 		}
 	}
-	return result.New(result.WithFailures(a.Failures()...))
+	return result.New(result.WithFailures(a.Failures()...)), nil
 }
diff --git a/plugin/registry_test.go b/plugin/registry_test.go
index 24db5a7..b778f90 100644
--- a/plugin/registry_test.go
+++ b/plugin/registry_test.go
@@ -36,8 +36,8 @@ func (s *fooSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
-func (s *fooSpec) Eval(context.Context, *testing.T) *result.Result {
-	return nil
+func (s *fooSpec) Eval(context.Context) (*result.Result, error) {
+	return nil, nil
 }
 
 func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
diff --git a/result/result.go b/result/result.go
index c281f25..8182460 100644
--- a/result/result.go
+++ b/result/result.go
@@ -4,13 +4,6 @@
 
 package result
 
-import (
-	"errors"
-	"fmt"
-
-	gdterrors "github.com/gdt-dev/gdt/errors"
-)
-
 // Result is returned from a `Evaluable.Eval` execution. It serves two
 // purposes:
 //
@@ -23,9 +16,6 @@ import (
 // returned in the Result and the `Scenario.Run` method injects that
 // information into the context that is supplied to the next Spec's `Run`.
 type Result struct {
-	// err is any error that was returned from the Evaluable's execution. This
-	// is guaranteed to be a `gdterrors.RuntimeError`.
-	err error
 	// failures is the collection of error messages from assertion failures
 	// that occurred during Eval(). These are *not* `gdterrors.RuntimeError`.
 	failures []error
@@ -36,17 +26,6 @@ type Result struct {
 	data map[string]interface{}
 }
 
-// HasRuntimeError returns true if the Eval() returned a runtime error, false
-// otherwise.
-func (r *Result) HasRuntimeError() bool {
-	return r.err != nil
-}
-
-// RuntimeError returns the runtime error
-func (r *Result) RuntimeError() error {
-	return r.err
-}
-
 // HasData returns true if any of the run data has been set, false otherwise.
 func (r *Result) HasData() bool {
 	return r.data != nil
@@ -86,19 +65,6 @@ func (r *Result) SetFailures(failures ...error) {
 
 type ResultModifier func(*Result)
 
-// WithRuntimeError modifies the Result with the supplied error
-func WithRuntimeError(err error) ResultModifier {
-	if !errors.Is(err, gdterrors.RuntimeError) {
-		msg := fmt.Sprintf("expected %s to be a gdterrors.RuntimeError", err)
-		// panic here because a plugin author incorrectly implemented their
-		// plugin Spec's Eval() method...
-		panic(msg)
-	}
-	return func(r *Result) {
-		r.err = err
-	}
-}
-
 // WithData modifies the Result with the supplied run data key and value
 func WithData(key string, val interface{}) ResultModifier {
 	return func(r *Result) {
diff --git a/scenario/run.go b/scenario/run.go
index 36bb477..79152d6 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -7,6 +7,7 @@ package scenario
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -52,9 +53,9 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 	// collection, evaluate those first and if any failed, skip the scenario's
 	// tests.
 	for _, skipIf := range s.SkipIf {
-		res := skipIf.Eval(ctx, t)
-		if res.HasRuntimeError() {
-			return res.RuntimeError()
+		res, err := skipIf.Eval(ctx)
+		if err != nil {
+			return err
 		}
 		if len(res.Failures()) == 0 {
 			t.Skipf(
@@ -70,11 +71,19 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			ctx = gdtcontext.PopTrace(ctx)
 		}()
 		for idx, spec := range s.Tests {
+			sb := spec.Base()
+			wait := sb.Wait
+			if wait != nil && wait.Before != "" {
+				debug.Println(ctx, "wait: %s before", wait.Before)
+				time.Sleep(wait.BeforeDuration())
+			}
 			plugin := s.evalPlugins[idx]
 			pinfo := plugin.Info()
 			pretry := pinfo.Retry
 			ptimeout := pinfo.Timeout
 
+			rt := getRetry(ctx, sb.Retry, scDefaults, pretry)
+
 			// Create a brand new context that inherits the top-level context's
 			// cancel func. We want to set deadlines for each test spec and if
 			// we mutate the single supplied top-level context, then only the
@@ -82,13 +91,6 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			specCtx, specCancel := context.WithCancel(ctx)
 			defer specCancel()
 
-			sb := spec.Base()
-			wait := sb.Wait
-			if wait != nil && wait.Before != "" {
-				debug.Println(ctx, "wait: %s before", wait.Before)
-				time.Sleep(wait.BeforeDuration())
-			}
-
 			to := getTimeout(ctx, sb.Timeout, ptimeout, scDefaults)
 			if to != nil {
 				var cancel context.CancelFunc
@@ -96,84 +98,13 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				defer cancel()
 			}
 
-			var res *result.Result
-			rt := getRetry(ctx, sb.Retry, scDefaults, pretry)
-			if rt == nil {
-				// Just evaluate the test spec once
-				res = spec.Eval(specCtx, t)
-				if res.HasRuntimeError() {
-					rterr = res.RuntimeError()
-					t.Fatal(rterr)
-				}
-				debug.Println(
-					ctx, "run: single-shot (no retries) ok: %v",
-					!res.Failed(),
-				)
-			} else {
-				// retry the action and test the assertions until they succeed,
-				// there is a terminal failure, or the timeout expires.
-				var bo backoff.BackOff
-
-				if rt.Exponential {
-					bo = backoff.WithContext(
-						backoff.NewExponentialBackOff(),
-						ctx,
-					)
-				} else {
-					interval := gdttypes.DefaultRetryConstantInterval
-					if rt.Interval != "" {
-						interval = rt.IntervalDuration()
-					}
-					bo = backoff.WithContext(
-						backoff.NewConstantBackOff(interval),
-						ctx,
-					)
-				}
-				ticker := backoff.NewTicker(bo)
-				maxAttempts := 0
-				if rt.Attempts != nil {
-					maxAttempts = *rt.Attempts
-				}
-				attempts := 1
-				start := time.Now().UTC()
-				success := false
-				for tick := range ticker.C {
-					if (maxAttempts > 0) && (attempts > maxAttempts) {
-						debug.Println(
-							ctx, "run: exceeded max attempts %d. stopping.",
-							maxAttempts,
-						)
-						ticker.Stop()
-						break
-					}
-					after := tick.Sub(start)
-
-					res = spec.Eval(specCtx, t)
-					if res.HasRuntimeError() {
-						rterr = res.RuntimeError()
-						t.Fatal(rterr)
-						break
-					}
-					success = !res.Failed()
-					debug.Println(
-						ctx, "run: attempt %d after %s ok: %v",
-						attempts, after, success,
-					)
-					if success {
-						ticker.Stop()
-						break
-					}
-					for _, f := range res.Failures() {
-						debug.Println(
-							ctx, "run: attempt %d after %s failure: %s",
-							attempts, after, f,
-						)
-					}
-					attempts++
-				}
+			res, rterr := s.runSpec(specCtx, rt, to, idx, spec)
+			if rterr != nil {
+				break
 			}
-			for _, fail := range res.Failures() {
-				t.Error(fail)
+			if wait != nil && wait.After != "" {
+				debug.Println(ctx, "wait: %s after", wait.After)
+				time.Sleep(wait.AfterDuration())
 			}
 			// Results can have arbitrary run data stored in them and we
 			// save this prior run data in the top-level context (and pass
@@ -181,15 +112,108 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			if res.HasData() {
 				ctx = gdtcontext.StorePriorRun(ctx, res.Data())
 			}
-			if wait != nil && wait.After != "" {
-				debug.Println(ctx, "wait: %s after", wait.After)
-				time.Sleep(wait.AfterDuration())
+			for _, fail := range res.Failures() {
+				t.Error(fail)
 			}
 		}
 	})
 	return rterr
 }
 
+// runSpec executes an individual test spec
+func (s *Scenario) runSpec(
+	ctx context.Context,
+	retry *gdttypes.Retry,
+	timeout *gdttypes.Timeout,
+	idx int,
+	spec gdttypes.Evaluable,
+) (*result.Result, error) {
+	sb := spec.Base()
+	specTraceMsg := strconv.Itoa(idx)
+	if sb.Name != "" {
+		specTraceMsg += ":" + sb.Name
+	}
+	ctx = gdtcontext.PushTrace(ctx, specTraceMsg)
+	defer func() {
+		ctx = gdtcontext.PopTrace(ctx)
+	}()
+	if retry == nil {
+		// Just evaluate the test spec once
+		res, err := spec.Eval(ctx)
+		if err != nil {
+			return nil, err
+		}
+		debug.Println(
+			ctx, "run: single-shot (no retries) ok: %v",
+			!res.Failed(),
+		)
+		return res, nil
+	}
+
+	// retry the action and test the assertions until they succeed,
+	// there is a terminal failure, or the timeout expires.
+	var bo backoff.BackOff
+	var res *result.Result
+	var err error
+
+	if retry.Exponential {
+		bo = backoff.WithContext(
+			backoff.NewExponentialBackOff(),
+			ctx,
+		)
+	} else {
+		interval := gdttypes.DefaultRetryConstantInterval
+		if retry.Interval != "" {
+			interval = retry.IntervalDuration()
+		}
+		bo = backoff.WithContext(
+			backoff.NewConstantBackOff(interval),
+			ctx,
+		)
+	}
+	ticker := backoff.NewTicker(bo)
+	maxAttempts := 0
+	if retry.Attempts != nil {
+		maxAttempts = *retry.Attempts
+	}
+	attempts := 1
+	start := time.Now().UTC()
+	success := false
+	for tick := range ticker.C {
+		if (maxAttempts > 0) && (attempts > maxAttempts) {
+			debug.Println(
+				ctx, "run: exceeded max attempts %d. stopping.",
+				maxAttempts,
+			)
+			ticker.Stop()
+			break
+		}
+		after := tick.Sub(start)
+
+		res, err = spec.Eval(ctx)
+		if err != nil {
+			return nil, err
+		}
+		success = !res.Failed()
+		debug.Println(
+			ctx, "run: attempt %d after %s ok: %v",
+			attempts, after, success,
+		)
+		if success {
+			ticker.Stop()
+			break
+		}
+		for _, f := range res.Failures() {
+			debug.Println(
+				ctx, "run: attempt %d after %s failure: %s",
+				attempts, after, f,
+			)
+		}
+		attempts++
+	}
+	return res, nil
+}
+
 // getTimeout returns the timeout value for the test spec. If the spec has a
 // timeout override, we use that. Otherwise, we inspect the scenario's defaults
 // and, if present, use that timeout. If the scenario's defaults for not
diff --git a/scenario/run_test.go b/scenario/run_test.go
index e3a1432..a1030f1 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -137,7 +137,7 @@ func TestNoRetry(t *testing.T) {
 	w.Flush()
 	require.NotEqual(b.Len(), 0)
 	debugout := b.String()
-	require.Contains(debugout, "[gdt] [no-retry] run: single-shot (no retries) ok: true")
+	require.Contains(debugout, "[gdt] [no-retry/0:bar] run: single-shot (no retries) ok: true")
 }
 
 func TestFailRetryTestOverride(t *testing.T) {
@@ -173,7 +173,7 @@ func TestRetryTestOverride(t *testing.T) {
 	require.NotNil(err)
 
 	debugout := string(outerr)
-	require.Contains(debugout, "[gdt] [retry-test-override] run: exceeded max attempts 2. stopping.")
+	require.Contains(debugout, "[gdt] [retry-test-override/0:baz] run: exceeded max attempts 2. stopping.")
 }
 
 func TestSkipIf(t *testing.T) {
diff --git a/scenario/stub_plugins_test.go b/scenario/stub_plugins_test.go
index f702bda..f49b336 100644
--- a/scenario/stub_plugins_test.go
+++ b/scenario/stub_plugins_test.go
@@ -8,7 +8,6 @@ import (
 	"context"
 	"fmt"
 	"strconv"
-	"testing"
 
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
@@ -18,7 +17,6 @@ import (
 	"github.com/gdt-dev/gdt/result"
 	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/samber/lo"
-	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
 
@@ -84,12 +82,8 @@ func (s *failSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
-func (s *failSpec) Eval(context.Context, *testing.T) *result.Result {
-	return result.New(
-		result.WithRuntimeError(
-			fmt.Errorf("%w: Indy, bad dates!", gdterrors.RuntimeError),
-		),
-	)
+func (s *failSpec) Eval(context.Context) (*result.Result, error) {
+	return nil, fmt.Errorf("%w: Indy, bad dates!", gdterrors.RuntimeError)
 }
 
 func (s *failSpec) UnmarshalYAML(node *yaml.Node) error {
@@ -220,28 +214,19 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-func (s *fooSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
+func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
 	fails := []error{}
-	t.Run(s.Title(), func(t *testing.T) {
-		ctx = gdtcontext.PushTrace(ctx, s.Title())
-		defer func() {
-			ctx = gdtcontext.PopTrace(ctx)
-		}()
-		debug.Println(ctx, "in %s Foo=%s", s.Title(), s.Foo)
-		// This is just a silly test to demonstrate how to write Eval() methods
-		// for plugin Spec specialization classes.
-		if s.Name == "bar" && s.Foo != "bar" {
-			fail := fmt.Errorf("expected s.Foo = 'bar', got %s", s.Foo)
-			fails = append(fails, fail)
-		} else if s.Name != "bar" && s.Foo != "baz" {
-			fail := fmt.Errorf("expected s.Foo = 'baz', got %s", s.Foo)
-			fails = append(fails, fail)
-		}
-	})
-	for _, fail := range fails {
-		t.Error(fail)
+	debug.Println(ctx, "in %s Foo=%s", s.Title(), s.Foo)
+	// This is just a silly test to demonstrate how to write Eval() methods
+	// for plugin Spec specialization classes.
+	if s.Name == "bar" && s.Foo != "bar" {
+		fail := fmt.Errorf("expected s.Foo = 'bar', got %s", s.Foo)
+		fails = append(fails, fail)
+	} else if s.Name != "bar" && s.Foo != "baz" {
+		fail := fmt.Errorf("expected s.Foo = 'baz', got %s", s.Foo)
+		fails = append(fails, fail)
 	}
-	return result.New(result.WithFailures(fails...))
+	return result.New(result.WithFailures(fails...)), nil
 }
 
 type fooPlugin struct{}
@@ -281,8 +266,8 @@ func (s *barSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
-func (s *barSpec) Eval(context.Context, *testing.T) *result.Result {
-	return result.New()
+func (s *barSpec) Eval(context.Context) (*result.Result, error) {
+	return result.New(), nil
 }
 
 func (s *barSpec) UnmarshalYAML(node *yaml.Node) error {
@@ -390,21 +375,28 @@ func (s *priorRunSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-func (s *priorRunSpec) Eval(ctx context.Context, t *testing.T) *result.Result {
-	t.Run(s.Title(), func(t *testing.T) {
-		assert := assert.New(t)
-		// Here we test that the prior run data that we save at the end of each
-		// Run() is showing up properly in the next Run()'s context.
-		prData := gdtcontext.PriorRun(ctx)
-		if s.Index == 0 {
-			assert.Empty(prData)
-		} else {
-			assert.Contains(prData, priorRunDataKey)
-			assert.IsType(prData[priorRunDataKey], "")
-			assert.Equal(s.Prior, prData[priorRunDataKey])
+func (s *priorRunSpec) Eval(ctx context.Context) (*result.Result, error) {
+	// Here we test that the prior run data that we save at the end of each
+	// Run() is showing up properly in the next Run()'s context.
+	fails := []error{}
+	prData := gdtcontext.PriorRun(ctx)
+	if s.Index == 0 {
+		if len(prData) != 0 {
+			fails = append(fails, fmt.Errorf("expected prData to be empty"))
+		}
+	} else {
+		data, ok := prData[priorRunDataKey]
+		if !ok {
+			fails = append(fails, fmt.Errorf("expected priorRunDataKey in priorRun map"))
 		}
-	})
-	return result.New(result.WithData(priorRunDataKey, s.State))
+		if s.Prior != data {
+			fails = append(fails, fmt.Errorf("expected priorRunData == data"))
+		}
+	}
+	return result.New(
+		result.WithFailures(fails...),
+		result.WithData(priorRunDataKey, s.State),
+	), nil
 }
 
 type priorRunPlugin struct{}
diff --git a/types/evaluable.go b/types/evaluable.go
index cb2450b..eeca127 100644
--- a/types/evaluable.go
+++ b/types/evaluable.go
@@ -6,7 +6,6 @@ package types
 
 import (
 	"context"
-	"testing"
 
 	"github.com/gdt-dev/gdt/result"
 )
@@ -16,7 +15,10 @@ type Evaluable interface {
 	// Eval performs an action and evaluates the results of that action,
 	// returning a Result that informs the Scenario about what failed or
 	// succeeded about the Evaluable's conditions.
-	Eval(context.Context, *testing.T) *result.Result
+	//
+	// Errors returned by Eval() are **RuntimeErrors**, not failures in
+	// assertions.
+	Eval(context.Context) (*result.Result, error)
 	// SetBase sets the Evaluable's base Spec
 	SetBase(Spec)
 	// Base returns the Evaluable's base Spec

From db376ccf0e5852130f5ba984d666294af98c6267 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Tue, 25 Jun 2024 09:37:06 -0400
Subject: [PATCH 15/31] allow an Evaluable to override retry behaviour

We need to allow a plugin's Evaluable to override the plugin's default
retry behaviour. In the case of gdt-kube, we want to *not* retry in the
case of `kubectl.create`, `kubectl.apply` and `kubectl.delete`, however
we *do* want to retry in the case of `kubectl.get`...

Issue gdt-dev/kube#17

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 context/context_test.go                       |   8 ++
 plugin/exec/spec.go                           |   8 ++
 plugin/registry_test.go                       |   8 ++
 scenario/run.go                               | 112 +++++++++++++-----
 scenario/run_test.go                          |  24 ++++
 scenario/stub_plugins_test.go                 |  32 +++++
 .../testdata/no-retry-evaluable-override.yaml |   6 +
 types/evaluable.go                            |   4 +
 types/retry.go                                |   6 +
 9 files changed, 177 insertions(+), 31 deletions(-)
 create mode 100644 scenario/testdata/no-retry-evaluable-override.yaml

diff --git a/context/context_test.go b/context/context_test.go
index 7644948..d026db2 100644
--- a/context/context_test.go
+++ b/context/context_test.go
@@ -39,6 +39,14 @@ func (s *fooSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
+func (s *fooSpec) Retry() *gdttypes.Retry {
+	return nil
+}
+
+func (s *fooSpec) Timeout() *gdttypes.Timeout {
+	return nil
+}
+
 func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
diff --git a/plugin/exec/spec.go b/plugin/exec/spec.go
index 2589369..fa591da 100644
--- a/plugin/exec/spec.go
+++ b/plugin/exec/spec.go
@@ -26,3 +26,11 @@ func (s *Spec) SetBase(b gdttypes.Spec) {
 func (s *Spec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
+
+func (s *Spec) Retry() *gdttypes.Retry {
+	return nil
+}
+
+func (s *Spec) Timeout() *gdttypes.Timeout {
+	return nil
+}
diff --git a/plugin/registry_test.go b/plugin/registry_test.go
index b778f90..59fe284 100644
--- a/plugin/registry_test.go
+++ b/plugin/registry_test.go
@@ -36,6 +36,14 @@ func (s *fooSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
+func (s *fooSpec) Retry() *gdttypes.Retry {
+	return nil
+}
+
+func (s *fooSpec) Timeout() *gdttypes.Timeout {
+	return nil
+}
+
 func (s *fooSpec) Eval(context.Context) (*result.Result, error) {
 	return nil, nil
 }
diff --git a/scenario/run.go b/scenario/run.go
index 79152d6..35bb607 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -78,11 +78,8 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				time.Sleep(wait.BeforeDuration())
 			}
 			plugin := s.evalPlugins[idx]
-			pinfo := plugin.Info()
-			pretry := pinfo.Retry
-			ptimeout := pinfo.Timeout
 
-			rt := getRetry(ctx, sb.Retry, scDefaults, pretry)
+			rt := getRetry(ctx, scDefaults, plugin, spec)
 
 			// Create a brand new context that inherits the top-level context's
 			// cancel func. We want to set deadlines for each test spec and if
@@ -91,7 +88,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			specCtx, specCancel := context.WithCancel(ctx)
 			defer specCancel()
 
-			to := getTimeout(ctx, sb.Timeout, ptimeout, scDefaults)
+			to := getTimeout(ctx, scDefaults, plugin, spec)
 			if to != nil {
 				var cancel context.CancelFunc
 				specCtx, cancel = context.WithTimeout(specCtx, to.Duration())
@@ -113,7 +110,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				ctx = gdtcontext.StorePriorRun(ctx, res.Data())
 			}
 			for _, fail := range res.Failures() {
-				t.Error(fail)
+				t.Fatal(fail)
 			}
 		}
 	})
@@ -137,7 +134,7 @@ func (s *Scenario) runSpec(
 	defer func() {
 		ctx = gdtcontext.PopTrace(ctx)
 	}()
-	if retry == nil {
+	if retry == nil || retry == gdttypes.NoRetry {
 		// Just evaluate the test spec once
 		res, err := spec.Eval(ctx)
 		if err != nil {
@@ -214,24 +211,38 @@ func (s *Scenario) runSpec(
 	return res, nil
 }
 
-// getTimeout returns the timeout value for the test spec. If the spec has a
-// timeout override, we use that. Otherwise, we inspect the scenario's defaults
-// and, if present, use that timeout. If the scenario's defaults for not
-// indicate a timeout configuration, we ask the plugin if it has timeout
-// defaults and use that.
+// getTimeout returns the timeout configuration for the test spec. We check for
+// overrides in timeout configuration using the following precedence:
+//
+// * Spec (Evaluable) override
+// * Spec's Base override
+// * Scenario's default
+// * Plugin's default
 func getTimeout(
 	ctx context.Context,
-	specTimeout *gdttypes.Timeout,
-	pluginTimeout *gdttypes.Timeout,
 	scenDefaults *Defaults,
+	plugin gdttypes.Plugin,
+	eval gdttypes.Evaluable,
 ) *gdttypes.Timeout {
-	if specTimeout != nil {
+	evalTimeout := eval.Timeout()
+	if evalTimeout != nil {
+		debug.Println(
+			ctx, "using timeout of %s",
+			evalTimeout.After,
+		)
+		return evalTimeout
+	}
+
+	sb := eval.Base()
+	baseTimeout := sb.Timeout
+	if baseTimeout != nil {
 		debug.Println(
 			ctx, "using timeout of %s",
-			specTimeout.After,
+			baseTimeout.After,
 		)
-		return specTimeout
+		return baseTimeout
 	}
+
 	if scenDefaults != nil && scenDefaults.Timeout != nil {
 		debug.Println(
 			ctx, "using timeout of %s [scenario default]",
@@ -239,6 +250,10 @@ func getTimeout(
 		)
 		return scenDefaults.Timeout
 	}
+
+	pluginInfo := plugin.Info()
+	pluginTimeout := pluginInfo.Timeout
+
 	if pluginTimeout != nil {
 		debug.Println(
 			ctx, "using timeout of %s [plugin default]",
@@ -249,31 +264,59 @@ func getTimeout(
 	return nil
 }
 
-// getRetry returns the retry configuration for the test spec. If the spec has a
-// retry override, we use that. Otherwise, we inspect the scenario's defaults
-// and, if present, use that timeout. If the scenario's defaults do not
-// indicate a retry configuration, we ask the plugin if it has retry defaults
-// and use that.
+// getRetry returns the retry configuration for the test spec. We check for
+// overrides in retry configuration using the following precedence:
+//
+// * Spec (Evaluable) override
+// * Spec's Base override
+// * Scenario's default
+// * Plugin's default
 func getRetry(
 	ctx context.Context,
-	specRetry *gdttypes.Retry,
 	scenDefaults *Defaults,
-	pluginRetry *gdttypes.Retry,
+	plugin gdttypes.Plugin,
+	eval gdttypes.Evaluable,
 ) *gdttypes.Retry {
-	if specRetry != nil {
+	evalRetry := eval.Retry()
+	if evalRetry != nil {
+		if evalRetry == gdttypes.NoRetry {
+			return evalRetry
+		}
 		msg := "using retry"
-		if specRetry.Attempts != nil {
-			msg += fmt.Sprintf(" (attempts: %d)", *specRetry.Attempts)
+		if evalRetry.Attempts != nil {
+			msg += fmt.Sprintf(" (attempts: %d)", *evalRetry.Attempts)
 		}
-		if specRetry.Interval != "" {
-			msg += fmt.Sprintf(" (interval: %s)", specRetry.Interval)
+		if evalRetry.Interval != "" {
+			msg += fmt.Sprintf(" (interval: %s)", evalRetry.Interval)
 		}
-		msg += fmt.Sprintf(" (exponential: %t)", specRetry.Exponential)
+		msg += fmt.Sprintf(" (exponential: %t)", evalRetry.Exponential)
 		debug.Println(ctx, msg)
-		return specRetry
+		return evalRetry
 	}
+
+	sb := eval.Base()
+	baseRetry := sb.Retry
+	if baseRetry != nil {
+		if baseRetry == gdttypes.NoRetry {
+			return baseRetry
+		}
+		msg := "using retry"
+		if baseRetry.Attempts != nil {
+			msg += fmt.Sprintf(" (attempts: %d)", *baseRetry.Attempts)
+		}
+		if baseRetry.Interval != "" {
+			msg += fmt.Sprintf(" (interval: %s)", baseRetry.Interval)
+		}
+		msg += fmt.Sprintf(" (exponential: %t)", baseRetry.Exponential)
+		debug.Println(ctx, msg)
+		return baseRetry
+	}
+
 	if scenDefaults != nil && scenDefaults.Retry != nil {
 		scenRetry := scenDefaults.Retry
+		if scenRetry == gdttypes.NoRetry {
+			return scenRetry
+		}
 		msg := "using retry"
 		if scenRetry.Attempts != nil {
 			msg += fmt.Sprintf(" (attempts: %d)", *scenRetry.Attempts)
@@ -285,7 +328,14 @@ func getRetry(
 		debug.Println(ctx, msg)
 		return scenRetry
 	}
+
+	pluginInfo := plugin.Info()
+	pluginRetry := pluginInfo.Retry
+
 	if pluginRetry != nil {
+		if pluginRetry == gdttypes.NoRetry {
+			return pluginRetry
+		}
 		msg := "using retry"
 		if pluginRetry.Attempts != nil {
 			msg += fmt.Sprintf(" (attempts: %d)", *pluginRetry.Attempts)
diff --git a/scenario/run_test.go b/scenario/run_test.go
index a1030f1..92c8ab3 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -140,6 +140,30 @@ func TestNoRetry(t *testing.T) {
 	require.Contains(debugout, "[gdt] [no-retry/0:bar] run: single-shot (no retries) ok: true")
 }
 
+func TestNoRetryEvaluableOverride(t *testing.T) {
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "no-retry-evaluable-override.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	var b bytes.Buffer
+	w := bufio.NewWriter(&b)
+	ctx := gdtcontext.New(gdtcontext.WithDebug(w))
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	require.Nil(err)
+	require.NotNil(s)
+
+	err = s.Run(ctx, t)
+	require.Nil(err)
+	require.False(t.Failed())
+	w.Flush()
+	require.NotEqual(b.Len(), 0)
+	debugout := b.String()
+	require.Contains(debugout, "[gdt] [no-retry-evaluable-override/0:bar] run: single-shot (no retries) ok: true")
+}
+
 func TestFailRetryTestOverride(t *testing.T) {
 	if !*failFlag {
 		t.Skip("skipping without -fail flag")
diff --git a/scenario/stub_plugins_test.go b/scenario/stub_plugins_test.go
index f49b336..181c3db 100644
--- a/scenario/stub_plugins_test.go
+++ b/scenario/stub_plugins_test.go
@@ -82,6 +82,14 @@ func (s *failSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
+func (s *failSpec) Retry() *gdttypes.Retry {
+	return nil
+}
+
+func (s *failSpec) Timeout() *gdttypes.Timeout {
+	return nil
+}
+
 func (s *failSpec) Eval(context.Context) (*result.Result, error) {
 	return nil, fmt.Errorf("%w: Indy, bad dates!", gdterrors.RuntimeError)
 }
@@ -185,6 +193,14 @@ func (s *fooSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
+func (s *fooSpec) Retry() *gdttypes.Retry {
+	return nil
+}
+
+func (s *fooSpec) Timeout() *gdttypes.Timeout {
+	return nil
+}
+
 func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
 		return errors.ExpectedMapAt(node)
@@ -266,6 +282,14 @@ func (s *barSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
+func (s *barSpec) Retry() *gdttypes.Retry {
+	return gdttypes.NoRetry
+}
+
+func (s *barSpec) Timeout() *gdttypes.Timeout {
+	return nil
+}
+
 func (s *barSpec) Eval(context.Context) (*result.Result, error) {
 	return result.New(), nil
 }
@@ -341,6 +365,14 @@ func (s *priorRunSpec) Base() *gdttypes.Spec {
 	return &s.Spec
 }
 
+func (s *priorRunSpec) Retry() *gdttypes.Retry {
+	return nil
+}
+
+func (s *priorRunSpec) Timeout() *gdttypes.Timeout {
+	return nil
+}
+
 func (s *priorRunSpec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
 		return errors.ExpectedMapAt(node)
diff --git a/scenario/testdata/no-retry-evaluable-override.yaml b/scenario/testdata/no-retry-evaluable-override.yaml
new file mode 100644
index 0000000..d6156f2
--- /dev/null
+++ b/scenario/testdata/no-retry-evaluable-override.yaml
@@ -0,0 +1,6 @@
+name: no-retry-evaluable-override
+description: a scenario using a plugin with an evaluable override for no retries
+tests:
+  # The bar plugin's Evaluable has a NoRetry override
+  - bar: 42
+    name: bar
diff --git a/types/evaluable.go b/types/evaluable.go
index eeca127..4288f29 100644
--- a/types/evaluable.go
+++ b/types/evaluable.go
@@ -23,4 +23,8 @@ type Evaluable interface {
 	SetBase(Spec)
 	// Base returns the Evaluable's base Spec
 	Base() *Spec
+	// Retry returns the Evaluable's Retry override, if any
+	Retry() *Retry
+	// Timeout returns the Evaluable's Timeout override, if any
+	Timeout() *Timeout
 }
diff --git a/types/retry.go b/types/retry.go
index d8df33d..d7b46e0 100644
--- a/types/retry.go
+++ b/types/retry.go
@@ -19,6 +19,12 @@ const (
 	DefaultRetryConstantInterval = 3 * time.Second
 )
 
+var (
+	// NoRetry indicates that there should not be any retry attempts. It is
+	// passed from a plugin to indicate a Spec should not be retried.
+	NoRetry = &Retry{}
+)
+
 // Retry contains information about the number of attempts and interval
 // duration with which a Plugin should re-run a Spec's action if the Spec's
 // assertions fail.

From 6edc4f7b1b646b382007bd0353d526d80c31b1e0 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Tue, 25 Jun 2024 10:23:44 -0400
Subject: [PATCH 16/31] consolidate types/, result/ and errors/ into api/

simplify the top-level packages by combining types/, result/ and errors/
into a single `api/` package.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 {types => api}/assertions.go              |   2 +-
 {types => api}/defaults.go                |   2 +-
 errors/parse.go => api/error.go           | 104 +++++++++++++-
 errors/parse_test.go => api/error_test.go |   8 +-
 {types => api}/evaluable.go               |   6 +-
 {types => api}/fixture.go                 |   2 +-
 {types => api}/flexstrings.go             |   8 +-
 {types => api}/flexstrings_test.go        |  11 +-
 {types => api}/plugin.go                  |   2 +-
 {result => api}/result.go                 |   6 +-
 {types => api}/retry.go                   |   2 +-
 {types => api}/spec.go                    |  25 ++--
 {types => api}/timeout.go                 |   2 +-
 {types => api}/wait.go                    |   2 +-
 assertion/json/errors.go                  |  24 ++--
 assertion/json/json.go                    |  23 ++--
 assertion/json/json_test.go               |   6 +-
 context/context.go                        |  11 +-
 context/context_test.go                   |  23 ++--
 context/getter.go                         |  18 +--
 context/timeout.go                        |   4 +-
 errors/failure.go                         |  94 -------------
 errors/runtime.go                         |  28 ----
 fixture/generic.go                        |   4 +-
 fixture/json/json.go                      |   7 +-
 fixture/json/json_test.go                 |   8 +-
 plugin/exec/action.go                     |   6 +-
 plugin/exec/assertions.go                 |  25 ++--
 plugin/exec/defaults.go                   |   8 +-
 plugin/exec/errors.go                     |   8 +-
 plugin/exec/eval.go                       |  11 +-
 plugin/exec/parse.go                      |  51 ++++---
 plugin/exec/parse_test.go                 |  11 +-
 plugin/exec/plugin.go                     |  14 +-
 plugin/exec/spec.go                       |  12 +-
 plugin/registry.go                        |  18 +--
 plugin/registry_test.go                   |  23 ++--
 porcelain.go                              |   6 +-
 porcelain_test.go                         |   4 +-
 scenario/defaults.go                      |  26 ++--
 scenario/parse.go                         |  45 +++---
 scenario/parse_test.go                    |  49 ++++---
 scenario/run.go                           |  42 +++---
 scenario/run_test.go                      |   6 +-
 scenario/scenario.go                      |   8 +-
 scenario/stub_plugins_test.go             | 158 +++++++++++-----------
 46 files changed, 463 insertions(+), 500 deletions(-)
 rename {types => api}/assertions.go (97%)
 rename {types => api}/defaults.go (96%)
 rename errors/parse.go => api/error.go (64%)
 rename errors/parse_test.go => api/error_test.go (72%)
 rename {types => api}/evaluable.go (89%)
 rename {types => api}/fixture.go (97%)
 rename {types => api}/flexstrings.go (85%)
 rename {types => api}/flexstrings_test.go (84%)
 rename {types => api}/plugin.go (98%)
 rename {result => api}/result.go (96%)
 rename {types => api}/retry.go (99%)
 rename {types => api}/spec.go (87%)
 rename {types => api}/timeout.go (98%)
 rename {types => api}/wait.go (98%)
 delete mode 100644 errors/failure.go
 delete mode 100644 errors/runtime.go

diff --git a/types/assertions.go b/api/assertions.go
similarity index 97%
rename from types/assertions.go
rename to api/assertions.go
index 510d436..831ead1 100644
--- a/types/assertions.go
+++ b/api/assertions.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import "context"
 
diff --git a/types/defaults.go b/api/defaults.go
similarity index 96%
rename from types/defaults.go
rename to api/defaults.go
index 3152d3c..6567c6c 100644
--- a/types/defaults.go
+++ b/api/defaults.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 // Defaults are a collection of default configuration values
 type Defaults map[string]interface{}
diff --git a/errors/parse.go b/api/error.go
similarity index 64%
rename from errors/parse.go
rename to api/error.go
index 3761389..4a1c7c0 100644
--- a/errors/parse.go
+++ b/api/error.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package errors
+package api
 
 import (
 	"errors"
@@ -11,6 +11,90 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+var (
+	// ErrFailure is the base error class for all errors that represent failed
+	// assertions when evaluating a test.
+	ErrFailure = errors.New("assertion failed")
+	// ErrTimeoutExceeded is an ErrFailure when a test's execution exceeds a
+	// timeout length.
+	ErrTimeoutExceeded = fmt.Errorf("%s: timeout exceeded", ErrFailure)
+	// ErrNotEqual is an ErrFailure when an expected thing doesn't equal an
+	// observed thing.
+	ErrNotEqual = fmt.Errorf("%w: not equal", ErrFailure)
+	// ErrIn is an ErrFailure when a thing unexpectedly appears in an
+	// container.
+	ErrIn = fmt.Errorf("%w: in", ErrFailure)
+	// ErrNotIn is an ErrFailure when an expected thing doesn't appear in an
+	// expected container.
+	ErrNotIn = fmt.Errorf("%w: not in", ErrFailure)
+	// ErrNoneIn is an ErrFailure when none of a list of elements appears in an
+	// expected container.
+	ErrNoneIn = fmt.Errorf("%w: none in", ErrFailure)
+	// ErrUnexpectedError is an ErrFailure when an unexpected error has
+	// occurred.
+	ErrUnexpectedError = fmt.Errorf("%w: unexpected error", ErrFailure)
+)
+
+// TimeoutExceeded returns an ErrTimeoutExceeded when a test's execution
+// exceeds a timeout length. The optional failure parameter indicates a failed
+// assertion that occurred before a timeout was reached.
+func TimeoutExceeded(duration string, failure error) error {
+	if failure != nil {
+		return fmt.Errorf(
+			"%w: timed out waiting for assertion to succeed (%s)",
+			failure, duration,
+		)
+	}
+	return fmt.Errorf("%s (%s)", ErrTimeoutExceeded, duration)
+}
+
+// NotEqualLength returns an ErrNotEqual when an expected length doesn't
+// equal an observed length.
+func NotEqualLength(exp, got int) error {
+	return fmt.Errorf(
+		"%w: expected length of %d but got %d",
+		ErrNotEqual, exp, got,
+	)
+}
+
+// NotEqual returns an ErrNotEqual when an expected thing doesn't equal an
+// observed thing.
+func NotEqual(exp, got interface{}) error {
+	return fmt.Errorf("%w: expected %v but got %v", ErrNotEqual, exp, got)
+}
+
+// In returns an ErrIn when a thing unexpectedly appears in a container.
+func In(element, container interface{}) error {
+	return fmt.Errorf(
+		"%w: expected %v not to contain %v",
+		ErrIn, container, element,
+	)
+}
+
+// NotIn returns an ErrNotIn when an expected thing doesn't appear in an
+// expected container.
+func NotIn(element, container interface{}) error {
+	return fmt.Errorf(
+		"%w: expected %v to contain %v",
+		ErrNotIn, container, element,
+	)
+}
+
+// NoneIn returns an ErrNoneIn when none of a list of elements appears in an
+// expected container.
+func NoneIn(elements, container interface{}) error {
+	return fmt.Errorf(
+		"%w: expected %v to contain one of %v",
+		ErrNoneIn, container, elements,
+	)
+}
+
+// UnexpectedError returns an ErrUnexpectedError when a supplied error is not
+// expected.
+func UnexpectedError(err error) error {
+	return fmt.Errorf("%w: %s", ErrUnexpectedError, err)
+}
+
 var (
 	// ErrUnknownSourceType indicates that a From() function was called with an
 	// unknown source parameter type.
@@ -199,3 +283,21 @@ func FileNotFound(path string, node *yaml.Node) error {
 		ErrFileNotFound, path, node.Line, node.Column,
 	)
 }
+
+var (
+	// RuntimeError is the base error class for all errors occurring during
+	// runtime (and not during the parsing of a scenario or spec)
+	RuntimeError = errors.New("runtime error")
+	// ErrRequiredFixture is returned when a required fixture has not
+	// been registered with the context.
+	ErrRequiredFixture = fmt.Errorf(
+		"%w: required fixture missing",
+		RuntimeError,
+	)
+)
+
+// RequiredFixtureMissing returns an ErrRequiredFixture with the supplied
+// fixture name
+func RequiredFixtureMissing(name string) error {
+	return fmt.Errorf("%w: %s", ErrRequiredFixture, name)
+}
diff --git a/errors/parse_test.go b/api/error_test.go
similarity index 72%
rename from errors/parse_test.go
rename to api/error_test.go
index 15452cc..69e2afd 100644
--- a/errors/parse_test.go
+++ b/api/error_test.go
@@ -2,22 +2,22 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package errors_test
+package api_test
 
 import (
 	"testing"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestUnknownSourceType(t *testing.T) {
 	assert := assert.New(t)
 
-	err := gdterrors.UnknownSourceType(1)
+	err := api.UnknownSourceType(1)
 	assert.ErrorContains(err, "int")
 
 	source := []string{"foo", "bar"}
-	err = gdterrors.UnknownSourceType(source)
+	err = api.UnknownSourceType(source)
 	assert.ErrorContains(err, "[]string")
 }
diff --git a/types/evaluable.go b/api/evaluable.go
similarity index 89%
rename from types/evaluable.go
rename to api/evaluable.go
index 4288f29..b1cfba6 100644
--- a/types/evaluable.go
+++ b/api/evaluable.go
@@ -2,12 +2,10 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import (
 	"context"
-
-	"github.com/gdt-dev/gdt/result"
 )
 
 // Evaluable represents individual test units in a Scenario
@@ -18,7 +16,7 @@ type Evaluable interface {
 	//
 	// Errors returned by Eval() are **RuntimeErrors**, not failures in
 	// assertions.
-	Eval(context.Context) (*result.Result, error)
+	Eval(context.Context) (*Result, error)
 	// SetBase sets the Evaluable's base Spec
 	SetBase(Spec)
 	// Base returns the Evaluable's base Spec
diff --git a/types/fixture.go b/api/fixture.go
similarity index 97%
rename from types/fixture.go
rename to api/fixture.go
index 8cb4b7a..5f0789a 100644
--- a/types/fixture.go
+++ b/api/fixture.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import "context"
 
diff --git a/types/flexstrings.go b/api/flexstrings.go
similarity index 85%
rename from types/flexstrings.go
rename to api/flexstrings.go
index 5689b84..7d681da 100644
--- a/types/flexstrings.go
+++ b/api/flexstrings.go
@@ -2,12 +2,10 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import (
 	"gopkg.in/yaml.v3"
-
-	gdterrors "github.com/gdt-dev/gdt/errors"
 )
 
 // FlexStrings is a struct used to parse an interface{} that can be either a
@@ -25,7 +23,7 @@ func (f *FlexStrings) Values() []string {
 // FlexStrings can be either a string or a slice of strings.
 func (f *FlexStrings) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.ScalarNode && node.Kind != yaml.SequenceNode {
-		return gdterrors.ExpectedScalarOrSequenceAt(node)
+		return ExpectedScalarOrSequenceAt(node)
 	}
 	var single string
 	if err := node.Decode(&single); err == nil {
@@ -37,5 +35,5 @@ func (f *FlexStrings) UnmarshalYAML(node *yaml.Node) error {
 		f.values = many
 		return nil
 	}
-	return gdterrors.ExpectedScalarOrSequenceAt(node)
+	return ExpectedScalarOrSequenceAt(node)
 }
diff --git a/types/flexstrings_test.go b/api/flexstrings_test.go
similarity index 84%
rename from types/flexstrings_test.go
rename to api/flexstrings_test.go
index ce53790..e8c64d3 100644
--- a/types/flexstrings_test.go
+++ b/api/flexstrings_test.go
@@ -2,24 +2,23 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types_test
+package api_test
 
 import (
 	"testing"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
 
 type foo struct {
-	Foo gdttypes.FlexStrings `yaml:"foo"`
+	Foo api.FlexStrings `yaml:"foo"`
 }
 
 type foop struct {
-	Foo *gdttypes.FlexStrings `yaml:"foo"`
+	Foo *api.FlexStrings `yaml:"foo"`
 }
 
 func TestFlexStringsError(t *testing.T) {
@@ -31,7 +30,7 @@ func TestFlexStringsError(t *testing.T) {
 	err := yaml.Unmarshal(contents, &f)
 
 	require.NotNil(err)
-	assert.ErrorIs(err, gdterrors.ErrExpectedScalarOrSequence)
+	assert.ErrorIs(err, api.ErrExpectedScalarOrSequence)
 }
 
 func TestFlexStrings(t *testing.T) {
diff --git a/types/plugin.go b/api/plugin.go
similarity index 98%
rename from types/plugin.go
rename to api/plugin.go
index 151d7b5..b0b3e21 100644
--- a/types/plugin.go
+++ b/api/plugin.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import "gopkg.in/yaml.v3"
 
diff --git a/result/result.go b/api/result.go
similarity index 96%
rename from result/result.go
rename to api/result.go
index 8182460..b9741e3 100644
--- a/result/result.go
+++ b/api/result.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package result
+package api
 
 // Result is returned from a `Evaluable.Eval` execution. It serves two
 // purposes:
@@ -80,8 +80,8 @@ func WithFailures(failures ...error) ResultModifier {
 	}
 }
 
-// New returns a new Result
-func New(mods ...ResultModifier) *Result {
+// NewResult returns a new Result
+func NewResult(mods ...ResultModifier) *Result {
 	r := &Result{}
 	for _, mod := range mods {
 		mod(r)
diff --git a/types/retry.go b/api/retry.go
similarity index 99%
rename from types/retry.go
rename to api/retry.go
index d7b46e0..a6b717a 100644
--- a/types/retry.go
+++ b/api/retry.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import (
 	"time"
diff --git a/types/spec.go b/api/spec.go
similarity index 87%
rename from types/spec.go
rename to api/spec.go
index ebd6762..9a2bdc0 100644
--- a/types/spec.go
+++ b/api/spec.go
@@ -2,14 +2,13 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import (
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/gdt-dev/gdt/errors"
 	"gopkg.in/yaml.v3"
 )
 
@@ -82,26 +81,26 @@ func slugify(s string) string {
 // the associated struct field from that value node.
 func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "name":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return ExpectedScalarAt(valNode)
 			}
 			s.Name = valNode.Value
 		case "description":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return ExpectedScalarAt(valNode)
 			}
 			s.Description = valNode.Value
 		case "timeout":
@@ -110,7 +109,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			case yaml.MappingNode:
 				// We support the old-style timeout:after
 				if err := valNode.Decode(&to); err != nil {
-					return errors.ExpectedTimeoutAt(valNode)
+					return ExpectedTimeoutAt(valNode)
 				}
 			case yaml.ScalarNode:
 				// We also support a straight string duration
@@ -118,7 +117,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 					After: valNode.Value,
 				}
 			default:
-				return errors.ExpectedScalarOrMapAt(valNode)
+				return ExpectedScalarOrMapAt(valNode)
 			}
 			_, err := time.ParseDuration(to.After)
 			if err != nil {
@@ -127,11 +126,11 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			s.Timeout = to
 		case "wait":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return ExpectedMapAt(valNode)
 			}
 			var w *Wait
 			if err := valNode.Decode(&w); err != nil {
-				return errors.ExpectedWaitAt(valNode)
+				return ExpectedWaitAt(valNode)
 			}
 			if w.Before != "" {
 				_, err := time.ParseDuration(w.Before)
@@ -148,16 +147,16 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			s.Wait = w
 		case "retry":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return ExpectedMapAt(valNode)
 			}
 			var r *Retry
 			if err := valNode.Decode(&r); err != nil {
-				return errors.ExpectedRetryAt(valNode)
+				return ExpectedRetryAt(valNode)
 			}
 			if r.Attempts != nil {
 				attempts := *r.Attempts
 				if attempts < 1 {
-					return errors.InvalidRetryAttempts(valNode, attempts)
+					return InvalidRetryAttempts(valNode, attempts)
 				}
 			}
 			if r.Interval != "" {
diff --git a/types/timeout.go b/api/timeout.go
similarity index 98%
rename from types/timeout.go
rename to api/timeout.go
index bdd32e9..e38e7c4 100644
--- a/types/timeout.go
+++ b/api/timeout.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import (
 	"time"
diff --git a/types/wait.go b/api/wait.go
similarity index 98%
rename from types/wait.go
rename to api/wait.go
index c5fca16..3b9642b 100644
--- a/types/wait.go
+++ b/api/wait.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package types
+package api
 
 import (
 	"time"
diff --git a/assertion/json/errors.go b/assertion/json/errors.go
index 6bdb521..3ccd3af 100644
--- a/assertion/json/errors.go
+++ b/assertion/json/errors.go
@@ -7,7 +7,7 @@ package json
 import (
 	"fmt"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 	"gopkg.in/yaml.v3"
 )
 
@@ -15,7 +15,7 @@ var (
 	// ErrJSONPathInvalid returns an ErrParse when a JSONPath expression could
 	// not be parsed.
 	ErrJSONPathInvalid = fmt.Errorf(
-		"%w: JSONPath invalid", gdterrors.ErrParse,
+		"%w: JSONPath invalid", api.ErrParse,
 	)
 	// ErrJSONPathInvalidNoRoot returns an ErrParse when a JSONPath expression
 	// does not start with '$'
@@ -24,56 +24,56 @@ var (
 	)
 	// ErrJSONUnmarshalError is returned when JSON content cannot be decoded
 	ErrJSONUnmarshalError = fmt.Errorf(
-		"%w: failed to unmarshal JSON", gdterrors.ErrFailure,
+		"%w: failed to unmarshal JSON", api.ErrFailure,
 	)
 	// ErrJSONPathNotFound returns an ErrFailure when a JSONPath expression
 	// could not evaluate to a found element.
 	ErrJSONPathNotFound = fmt.Errorf(
-		"%w: failed to find element at JSONPath", gdterrors.ErrFailure,
+		"%w: failed to find element at JSONPath", api.ErrFailure,
 	)
 	// ErrJSONPathConversionError returns an ErrFailure when a JSONPath
 	// expression evaluated to a found element but could not be converted to a
 	// string.
 	ErrJSONPathConversionError = fmt.Errorf(
-		"%w: JSONPath value could not be compared", gdterrors.ErrFailure,
+		"%w: JSONPath value could not be compared", api.ErrFailure,
 	)
 	// ErrJSONPathNotEqual returns an ErrFailure when a JSONPath
 	// expression evaluated to a found element but the value did not match an
 	// expected string.
 	ErrJSONPathNotEqual = fmt.Errorf(
-		"%w: JSONPath values not equal", gdterrors.ErrFailure,
+		"%w: JSONPath values not equal", api.ErrFailure,
 	)
 	// ErrJSONSchemaValidateError returns an ErrFailure when a JSONSchema could
 	// not be parsed.
 	ErrJSONSchemaValidateError = fmt.Errorf(
-		"%w: failed to parse JSONSchema", gdterrors.ErrFailure,
+		"%w: failed to parse JSONSchema", api.ErrFailure,
 	)
 	// ErrJSONSchemaInvalid returns an ErrFailure when some content could not
 	// be validated with a JSONSchema.
 	ErrJSONSchemaInvalid = fmt.Errorf(
-		"%w: JSON content did not adhere to JSONSchema", gdterrors.ErrFailure,
+		"%w: JSON content did not adhere to JSONSchema", api.ErrFailure,
 	)
 	// ErrJSONFormatError returns an ErrFailure when a JSONFormat expression
 	// could not evaluate to a found element.
 	ErrJSONFormatError = fmt.Errorf(
-		"%w: failed to determine JSON format", gdterrors.ErrFailure,
+		"%w: failed to determine JSON format", api.ErrFailure,
 	)
 	// ErrJSONFormatNotEqual returns an ErrFailure when a an element at a
 	// JSONPath was not in the expected format.
 	ErrJSONFormatNotEqual = fmt.Errorf(
-		"%w: JSON format not equal", gdterrors.ErrFailure,
+		"%w: JSON format not equal", api.ErrFailure,
 	)
 	// ErrJSONSchemaFileNotFound indicates a specified JSONSchema file could
 	// not be found.
 	ErrJSONSchemaFileNotFound = fmt.Errorf(
 		"%w: unable to find JSONSchema file",
-		gdterrors.ErrParse,
+		api.ErrParse,
 	)
 	// ErrUnsupportedJSONSchemaReference indicates that a specified JSONSchema
 	// file is referenced as an HTTP(S) URL instead of a file URI.
 	ErrUnsupportedJSONSchemaReference = fmt.Errorf(
 		"%w: unsupported JSONSchema reference",
-		gdterrors.ErrParse,
+		api.ErrParse,
 	)
 )
 
diff --git a/assertion/json/json.go b/assertion/json/json.go
index 8a23615..779d1a6 100644
--- a/assertion/json/json.go
+++ b/assertion/json/json.go
@@ -18,8 +18,7 @@ import (
 	gjs "github.com/xeipuuv/gojsonschema"
 	"gopkg.in/yaml.v3"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 var (
@@ -47,21 +46,21 @@ type Expect struct {
 // contained in the Expect are valid.
 func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return gdterrors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return gdterrors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "len":
 			if valNode.Kind != yaml.ScalarNode {
-				return gdterrors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			var v *int
 			if err := valNode.Decode(&v); err != nil {
@@ -70,7 +69,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			e.Len = v
 		case "schema":
 			if valNode.Kind != yaml.ScalarNode {
-				return gdterrors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			// Ensure any JSONSchema URL specified in exponse.json.schema exists
 			schemaURL := valNode.Value
@@ -98,7 +97,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			}
 		case "paths":
 			if valNode.Kind != yaml.MappingNode {
-				return gdterrors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			paths := map[string]string{}
 			if err := valNode.Decode(&paths); err != nil {
@@ -115,7 +114,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			e.Paths = paths
 		case "path_formats", "path-formats":
 			if valNode.Kind != yaml.MappingNode {
-				return gdterrors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			pathFormats := map[string]string{}
 			if err := valNode.Decode(&pathFormats); err != nil {
@@ -135,12 +134,12 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-// New returns a `gdttypes.Assertions` that asserts various conditions about
+// New returns a `api.Assertions` that asserts various conditions about
 // JSON content
 func New(
 	exp *Expect,
 	content []byte,
-) gdttypes.Assertions {
+) api.Assertions {
 	return &assertions{
 		failures: []error{},
 		exp:      exp,
@@ -149,7 +148,7 @@ func New(
 }
 
 // assertions represents one or more assertions about JSON data responses and
-// implements the gdttypes.Assertions interface
+// implements the api.Assertions interface
 type assertions struct {
 	// failures contains the set of error messages for failed assertions
 	failures []error
@@ -200,7 +199,7 @@ func (a *assertions) lenOK() bool {
 		exp := *a.exp.Len
 		got := len(a.content)
 		if exp != got {
-			a.Fail(gdterrors.NotEqualLength(exp, got))
+			a.Fail(api.NotEqualLength(exp, got))
 			return false
 		}
 	}
diff --git a/assertion/json/json_test.go b/assertion/json/json_test.go
index 6b2fdbd..5da39ed 100644
--- a/assertion/json/json_test.go
+++ b/assertion/json/json_test.go
@@ -10,8 +10,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtjson "github.com/gdt-dev/gdt/assertion/json"
-	gdterrors "github.com/gdt-dev/gdt/errors"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
@@ -61,7 +61,7 @@ paths: notamap
 `)
 	err = yaml.Unmarshal(content, &exp)
 	require.NotNil(err)
-	require.ErrorIs(err, gdterrors.ErrExpectedMap)
+	require.ErrorIs(err, api.ErrExpectedMap)
 
 	content = []byte(`
 len: 1
@@ -116,7 +116,7 @@ func TestLength(t *testing.T) {
 	require.False(a.OK(ctx))
 	failures := a.Failures()
 	require.Len(failures, 1)
-	require.ErrorIs(failures[0], gdterrors.ErrNotEqual)
+	require.ErrorIs(failures[0], api.ErrNotEqual)
 }
 
 func TestJSONUnmarshalError(t *testing.T) {
diff --git a/context/context.go b/context/context.go
index 23469ed..bf9ed38 100644
--- a/context/context.go
+++ b/context/context.go
@@ -9,8 +9,9 @@ import (
 	"io"
 	"os"
 
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/samber/lo"
+
+	"github.com/gdt-dev/gdt/api"
 )
 
 type ContextKey string
@@ -88,14 +89,14 @@ func WithDebug(writers ...io.Writer) ContextModifier {
 }
 
 // WithPlugins sets a context's Plugins
-func WithPlugins(plugins []gdttypes.Plugin) ContextModifier {
+func WithPlugins(plugins []api.Plugin) ContextModifier {
 	return func(ctx context.Context) context.Context {
 		return context.WithValue(ctx, pluginsKey, plugins)
 	}
 }
 
 // WithFixtures sets a context's Fixtures
-func WithFixtures(fixtures map[string]gdttypes.Fixture) ContextModifier {
+func WithFixtures(fixtures map[string]api.Fixture) ContextModifier {
 	return func(ctx context.Context) context.Context {
 		return context.WithValue(ctx, fixturesKey, fixtures)
 	}
@@ -121,7 +122,7 @@ func SetDebug(
 func RegisterFixture(
 	ctx context.Context,
 	name string,
-	f gdttypes.Fixture,
+	f api.Fixture,
 ) context.Context {
 	fixtures := Fixtures(ctx)
 	fixtures[name] = f
@@ -131,7 +132,7 @@ func RegisterFixture(
 // RegisterPlugin registers a plugin with the context
 func RegisterPlugin(
 	ctx context.Context,
-	p gdttypes.Plugin,
+	p api.Plugin,
 ) context.Context {
 	plugins := Plugins(ctx)
 	for _, plug := range plugins {
diff --git a/context/context_test.go b/context/context_test.go
index d026db2..a568b90 100644
--- a/context/context_test.go
+++ b/context/context_test.go
@@ -8,10 +8,9 @@ import (
 	"context"
 	"testing"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/fixture"
-	"github.com/gdt-dev/gdt/result"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
@@ -27,23 +26,23 @@ func (d *fooDefaults) UnmarshalYAML(node *yaml.Node) error {
 }
 
 type fooSpec struct {
-	gdttypes.Spec
+	api.Spec
 	Foo string `yaml:"foo"`
 }
 
-func (s *fooSpec) SetBase(b gdttypes.Spec) {
+func (s *fooSpec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *fooSpec) Base() *gdttypes.Spec {
+func (s *fooSpec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *fooSpec) Retry() *gdttypes.Retry {
+func (s *fooSpec) Retry() *api.Retry {
 	return nil
 }
 
-func (s *fooSpec) Timeout() *gdttypes.Timeout {
+func (s *fooSpec) Timeout() *api.Timeout {
 	return nil
 }
 
@@ -51,14 +50,14 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
+func (s *fooSpec) Eval(ctx context.Context) (*api.Result, error) {
 	return nil, nil
 }
 
 type fooPlugin struct{}
 
-func (p *fooPlugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *fooPlugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: "foo",
 	}
 }
@@ -67,8 +66,8 @@ func (p *fooPlugin) Defaults() yaml.Unmarshaler {
 	return &fooDefaults{}
 }
 
-func (p *fooPlugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&fooSpec{}}
+func (p *fooPlugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&fooSpec{}}
 }
 
 func TestContext(t *testing.T) {
diff --git a/context/getter.go b/context/getter.go
index f4f666a..84c1d80 100644
--- a/context/getter.go
+++ b/context/getter.go
@@ -9,7 +9,7 @@ import (
 	"io"
 	"strings"
 
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 const (
@@ -50,25 +50,25 @@ func Debug(ctx context.Context) []io.Writer {
 }
 
 // Plugins gets a context's Plugins
-func Plugins(ctx context.Context) []gdttypes.Plugin {
+func Plugins(ctx context.Context) []api.Plugin {
 	if ctx == nil {
-		return []gdttypes.Plugin{}
+		return []api.Plugin{}
 	}
 	if v := ctx.Value(pluginsKey); v != nil {
-		return v.([]gdttypes.Plugin)
+		return v.([]api.Plugin)
 	}
-	return []gdttypes.Plugin{}
+	return []api.Plugin{}
 }
 
 // Fixtures gets a context's Fixtures
-func Fixtures(ctx context.Context) map[string]gdttypes.Fixture {
+func Fixtures(ctx context.Context) map[string]api.Fixture {
 	if ctx == nil {
-		return map[string]gdttypes.Fixture{}
+		return map[string]api.Fixture{}
 	}
 	if v := ctx.Value(fixturesKey); v != nil {
-		return v.(map[string]gdttypes.Fixture)
+		return v.(map[string]api.Fixture)
 	}
-	return map[string]gdttypes.Fixture{}
+	return map[string]api.Fixture{}
 }
 
 // PriorRun gets a context's prior run data
diff --git a/context/timeout.go b/context/timeout.go
index 62b1beb..b9d599d 100644
--- a/context/timeout.go
+++ b/context/timeout.go
@@ -9,7 +9,7 @@ import (
 	"errors"
 	"strings"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 )
 
 // TimedOut evaluates a context and an arbitrary other error and returns true
@@ -25,7 +25,7 @@ func TimedOut(
 		return errors.Is(cerr, context.DeadlineExceeded)
 	}
 	if err != nil {
-		if errors.Is(err, gdterrors.ErrTimeoutExceeded) {
+		if errors.Is(err, api.ErrTimeoutExceeded) {
 			return true
 		}
 		return strings.Contains(err.Error(), "signal: killed")
diff --git a/errors/failure.go b/errors/failure.go
deleted file mode 100644
index 10b12f9..0000000
--- a/errors/failure.go
+++ /dev/null
@@ -1,94 +0,0 @@
-// Use and distribution licensed under the Apache license version 2.
-//
-// See the COPYING file in the root project directory for full text.
-
-package errors
-
-import (
-	"errors"
-	"fmt"
-)
-
-var (
-	// ErrFailure is the base error class for all errors that represent failed
-	// assertions when evaluating a test.
-	ErrFailure = errors.New("assertion failed")
-	// ErrTimeoutExceeded is an ErrFailure when a test's execution exceeds a
-	// timeout length.
-	ErrTimeoutExceeded = fmt.Errorf("%s: timeout exceeded", ErrFailure)
-	// ErrNotEqual is an ErrFailure when an expected thing doesn't equal an
-	// observed thing.
-	ErrNotEqual = fmt.Errorf("%w: not equal", ErrFailure)
-	// ErrIn is an ErrFailure when a thing unexpectedly appears in an
-	// container.
-	ErrIn = fmt.Errorf("%w: in", ErrFailure)
-	// ErrNotIn is an ErrFailure when an expected thing doesn't appear in an
-	// expected container.
-	ErrNotIn = fmt.Errorf("%w: not in", ErrFailure)
-	// ErrNoneIn is an ErrFailure when none of a list of elements appears in an
-	// expected container.
-	ErrNoneIn = fmt.Errorf("%w: none in", ErrFailure)
-	// ErrUnexpectedError is an ErrFailure when an unexpected error has
-	// occurred.
-	ErrUnexpectedError = fmt.Errorf("%w: unexpected error", ErrFailure)
-)
-
-// TimeoutExceeded returns an ErrTimeoutExceeded when a test's execution
-// exceeds a timeout length. The optional failure parameter indicates a failed
-// assertion that occurred before a timeout was reached.
-func TimeoutExceeded(duration string, failure error) error {
-	if failure != nil {
-		return fmt.Errorf(
-			"%w: timed out waiting for assertion to succeed (%s)",
-			failure, duration,
-		)
-	}
-	return fmt.Errorf("%s (%s)", ErrTimeoutExceeded, duration)
-}
-
-// NotEqualLength returns an ErrNotEqual when an expected length doesn't
-// equal an observed length.
-func NotEqualLength(exp, got int) error {
-	return fmt.Errorf(
-		"%w: expected length of %d but got %d",
-		ErrNotEqual, exp, got,
-	)
-}
-
-// NotEqual returns an ErrNotEqual when an expected thing doesn't equal an
-// observed thing.
-func NotEqual(exp, got interface{}) error {
-	return fmt.Errorf("%w: expected %v but got %v", ErrNotEqual, exp, got)
-}
-
-// In returns an ErrIn when a thing unexpectedly appears in a container.
-func In(element, container interface{}) error {
-	return fmt.Errorf(
-		"%w: expected %v not to contain %v",
-		ErrIn, container, element,
-	)
-}
-
-// NotIn returns an ErrNotIn when an expected thing doesn't appear in an
-// expected container.
-func NotIn(element, container interface{}) error {
-	return fmt.Errorf(
-		"%w: expected %v to contain %v",
-		ErrNotIn, container, element,
-	)
-}
-
-// NoneIn returns an ErrNoneIn when none of a list of elements appears in an
-// expected container.
-func NoneIn(elements, container interface{}) error {
-	return fmt.Errorf(
-		"%w: expected %v to contain one of %v",
-		ErrNoneIn, container, elements,
-	)
-}
-
-// UnexpectedError returns an ErrUnexpectedError when a supplied error is not
-// expected.
-func UnexpectedError(err error) error {
-	return fmt.Errorf("%w: %s", ErrUnexpectedError, err)
-}
diff --git a/errors/runtime.go b/errors/runtime.go
deleted file mode 100644
index 734e9c5..0000000
--- a/errors/runtime.go
+++ /dev/null
@@ -1,28 +0,0 @@
-// Use and distribution licensed under the Apache license version 2.
-//
-// See the COPYING file in the root project directory for full text.
-
-package errors
-
-import (
-	"errors"
-	"fmt"
-)
-
-var (
-	// RuntimeError is the base error class for all errors occurring during
-	// runtime (and not during the parsing of a scenario or spec)
-	RuntimeError = errors.New("runtime error")
-	// ErrRequiredFixture is returned when a required fixture has not
-	// been registered with the context.
-	ErrRequiredFixture = fmt.Errorf(
-		"%w: required fixture missing",
-		RuntimeError,
-	)
-)
-
-// RequiredFixtureMissing returns an ErrRequiredFixture with the supplied
-// fixture name
-func RequiredFixtureMissing(name string) error {
-	return fmt.Errorf("%w: %s", ErrRequiredFixture, name)
-}
diff --git a/fixture/generic.go b/fixture/generic.go
index f1102ff..ee3ba5d 100644
--- a/fixture/generic.go
+++ b/fixture/generic.go
@@ -8,7 +8,7 @@ import (
 	"context"
 	"strings"
 
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 // genericFixture adapts functions and state dicts into the Fixture type
@@ -77,7 +77,7 @@ func WithState(state map[string]interface{}) genericFixtureModifier {
 }
 
 // New returns a new generic Fixture
-func New(mods ...genericFixtureModifier) gdttypes.Fixture {
+func New(mods ...genericFixtureModifier) api.Fixture {
 	f := &genericFixture{}
 	for _, mod := range mods {
 		mod(f)
diff --git a/fixture/json/json.go b/fixture/json/json.go
index 69f6b3a..c1a421d 100644
--- a/fixture/json/json.go
+++ b/fixture/json/json.go
@@ -11,7 +11,8 @@ import (
 	"strconv"
 
 	"github.com/PaesslerAG/jsonpath"
-	gdttypes "github.com/gdt-dev/gdt/types"
+
+	"github.com/gdt-dev/gdt/api"
 )
 
 type jsonFixture struct {
@@ -59,8 +60,8 @@ func (f *jsonFixture) State(path string) interface{} {
 }
 
 // New takes a string, some bytes or an io.Reader and returns a new
-// gdttypes.Fixture that can have its state queried via JSONPath
-func New(data interface{}) (gdttypes.Fixture, error) {
+// api.Fixture that can have its state queried via JSONPath
+func New(data interface{}) (api.Fixture, error) {
 	var err error
 	var b []byte
 	switch data := data.(type) {
diff --git a/fixture/json/json_test.go b/fixture/json/json_test.go
index 9402757..9f04770 100644
--- a/fixture/json/json_test.go
+++ b/fixture/json/json_test.go
@@ -8,8 +8,8 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/gdt-dev/gdt/api"
 	jsonfix "github.com/gdt-dev/gdt/fixture/json"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +23,7 @@ func TestNewFromString(t *testing.T) {
 
 	require.Nil(err)
 	require.NotNil(f)
-	require.Implements((*gdttypes.Fixture)(nil), f)
+	require.Implements((*api.Fixture)(nil), f)
 
 	assert.True(f.HasState("$.book.year"))
 	assert.Equal("1957", f.State("$.book.year"))
@@ -40,7 +40,7 @@ func TestNewFromBytes(t *testing.T) {
 
 	require.Nil(err)
 	require.NotNil(f)
-	require.Implements((*gdttypes.Fixture)(nil), f)
+	require.Implements((*api.Fixture)(nil), f)
 
 	assert.True(f.HasState("$.book.year"))
 	assert.Equal("1957", f.State("$.book.year"))
@@ -58,7 +58,7 @@ func TestNewFromReader(t *testing.T) {
 
 	require.Nil(err)
 	require.NotNil(f)
-	require.Implements((*gdttypes.Fixture)(nil), f)
+	require.Implements((*api.Fixture)(nil), f)
 
 	assert.True(f.HasState("$.book.year"))
 	assert.Equal("1957", f.State("$.book.year"))
diff --git a/plugin/exec/action.go b/plugin/exec/action.go
index d172ef0..687f5ef 100644
--- a/plugin/exec/action.go
+++ b/plugin/exec/action.go
@@ -9,9 +9,9 @@ import (
 	"context"
 	"os/exec"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
-	gdterrors "github.com/gdt-dev/gdt/errors"
 	"github.com/google/shlex"
 )
 
@@ -69,7 +69,7 @@ func (a *Action) Do(
 
 	err = cmd.Start()
 	if gdtcontext.TimedOut(ctx, err) {
-		return gdterrors.ErrTimeoutExceeded
+		return api.ErrTimeoutExceeded
 	}
 	if err != nil {
 		return err
@@ -93,7 +93,7 @@ func (a *Action) Do(
 
 	err = cmd.Wait()
 	if gdtcontext.TimedOut(ctx, err) {
-		return gdterrors.ErrTimeoutExceeded
+		return api.ErrTimeoutExceeded
 	}
 	if err != nil && exitcode != nil {
 		eerr, _ := err.(*exec.ExitError)
diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go
index 6e361c2..defebb0 100644
--- a/plugin/exec/assertions.go
+++ b/plugin/exec/assertions.go
@@ -9,8 +9,7 @@ import (
 	"context"
 	"strings"
 
-	"github.com/gdt-dev/gdt/errors"
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 // Expect contains the assertions about an Exec Spec's actions
@@ -29,13 +28,13 @@ type Expect struct {
 type PipeExpect struct {
 	// ContainsAll is one or more strings that *all* must be present in the
 	// contents of the pipe
-	ContainsAll *gdttypes.FlexStrings `yaml:"contains,omitempty"`
+	ContainsAll *api.FlexStrings `yaml:"contains,omitempty"`
 	// ContainsNone is one or more strings, *none of which* should be present in
 	// the contents of the pipe
-	ContainsNone *gdttypes.FlexStrings `yaml:"contains-none-of,omitempty"`
+	ContainsNone *api.FlexStrings `yaml:"contains-none-of,omitempty"`
 	// ContainsOneOf is one or more strings of which *at least one* must be
 	// present in the contents of the pipe
-	ContainsAny *gdttypes.FlexStrings `yaml:"contains-one-of,omitempty"`
+	ContainsAny *api.FlexStrings `yaml:"contains-one-of,omitempty"`
 }
 
 // pipeAssertions contains assertions about the contents of a pipe
@@ -54,7 +53,7 @@ func (a *pipeAssertions) Fail(err error) {
 	a.failures = append(a.failures, err)
 }
 
-// Failures returns a slice of errors for all failed assertions
+// Failures returns a slice of api for all failed assertions
 func (a *pipeAssertions) Failures() []error {
 	if a == nil {
 		return []error{}
@@ -77,13 +76,13 @@ func (a *pipeAssertions) OK(ctx context.Context) bool {
 		vals := a.ContainsAll.Values()
 		if len(vals) == 1 {
 			if !strings.Contains(contents, vals[0]) {
-				a.Fail(errors.NotEqual(vals[0], contents))
+				a.Fail(api.NotEqual(vals[0], contents))
 				res = false
 			}
 		} else {
 			for _, find := range vals {
 				if !strings.Contains(contents, find) {
-					a.Fail(errors.NotIn(find, a.name))
+					a.Fail(api.NotIn(find, a.name))
 					res = false
 				}
 			}
@@ -98,14 +97,14 @@ func (a *pipeAssertions) OK(ctx context.Context) bool {
 			}
 		}
 		if !found {
-			a.Fail(errors.NoneIn(a.ContainsAny.Values(), a.name))
+			a.Fail(api.NoneIn(a.ContainsAny.Values(), a.name))
 			res = false
 		}
 	}
 	if a.ContainsNone != nil {
 		for _, find := range a.ContainsNone.Values() {
 			if strings.Contains(contents, find) {
-				a.Fail(errors.In(find, a.name))
+				a.Fail(api.In(find, a.name))
 				res = false
 			}
 		}
@@ -132,7 +131,7 @@ func (a *assertions) Fail(err error) {
 	a.failures = append(a.failures, err)
 }
 
-// Failures returns a slice of errors for all failed assertions
+// Failures returns a slice of api for all failed assertions
 func (a *assertions) Failures() []error {
 	if a == nil {
 		return []error{}
@@ -145,7 +144,7 @@ func (a *assertions) Failures() []error {
 func (a *assertions) OK(ctx context.Context) bool {
 	res := true
 	if a.expExitCode != a.exitCode {
-		a.Fail(errors.NotEqual(a.expExitCode, a.exitCode))
+		a.Fail(api.NotEqual(a.expExitCode, a.exitCode))
 		res = false
 	}
 	if !a.expOutPipe.OK(ctx) {
@@ -166,7 +165,7 @@ func newAssertions(
 	exitCode int,
 	outPipe *bytes.Buffer,
 	errPipe *bytes.Buffer,
-) gdttypes.Assertions {
+) api.Assertions {
 	a := &assertions{
 		failures:    []error{},
 		expExitCode: exitCode,
diff --git a/plugin/exec/defaults.go b/plugin/exec/defaults.go
index 5be9bd9..657e3e1 100644
--- a/plugin/exec/defaults.go
+++ b/plugin/exec/defaults.go
@@ -7,7 +7,7 @@ package exec
 import (
 	"gopkg.in/yaml.v3"
 
-	"github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 )
 
 type execDefaults struct{}
@@ -19,21 +19,21 @@ type Defaults struct {
 
 func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "exec":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			ed := execDefaults{}
 			if err := valNode.Decode(&ed); err != nil {
diff --git a/plugin/exec/errors.go b/plugin/exec/errors.go
index fa7833b..1cb7ba1 100644
--- a/plugin/exec/errors.go
+++ b/plugin/exec/errors.go
@@ -9,18 +9,18 @@ import (
 
 	"gopkg.in/yaml.v3"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 )
 
 var (
 	// ErrExecEmpty indicates that the user specified an empty "exec"
 	// field
 	ErrExecEmpty = fmt.Errorf(
-		"%w: expected non-empty exec field", gdterrors.ErrParse,
+		"%w: expected non-empty exec field", api.ErrParse,
 	)
 	// ErrExecInvalid indicates that the user specified an invalid "exec" field
 	ErrExecInvalid = fmt.Errorf(
-		"%w: invalid exec field", gdterrors.ErrParse,
+		"%w: invalid exec field", api.ErrParse,
 	)
 )
 
@@ -44,5 +44,5 @@ func ExecInvalidShellParse(err error, node *yaml.Node) error {
 
 // ExecRuntimeError returns a RuntimeError with an error from the Exec() call.
 func ExecRuntimeError(err error) error {
-	return fmt.Errorf("%w: %s", gdterrors.RuntimeError, err)
+	return fmt.Errorf("%w: %s", api.RuntimeError, err)
 }
diff --git a/plugin/exec/eval.go b/plugin/exec/eval.go
index a02d3cf..45ec494 100644
--- a/plugin/exec/eval.go
+++ b/plugin/exec/eval.go
@@ -8,9 +8,8 @@ import (
 	"bytes"
 	"context"
 
+	"github.com/gdt-dev/gdt/api"
 	"github.com/gdt-dev/gdt/debug"
-	gdterrors "github.com/gdt-dev/gdt/errors"
-	"github.com/gdt-dev/gdt/result"
 )
 
 // Eval performs an action and evaluates the results of that action, returning
@@ -20,15 +19,15 @@ import (
 // Errors returned by Eval() are **RuntimeErrors**, not failures in assertions.
 func (s *Spec) Eval(
 	ctx context.Context,
-) (*result.Result, error) {
+) (*api.Result, error) {
 	outbuf := &bytes.Buffer{}
 	errbuf := &bytes.Buffer{}
 
 	var ec int
 
 	if err := s.Do(ctx, outbuf, errbuf, &ec); err != nil {
-		if err == gdterrors.ErrTimeoutExceeded {
-			return result.New(result.WithFailures(gdterrors.ErrTimeoutExceeded)), nil
+		if err == api.ErrTimeoutExceeded {
+			return api.NewResult(api.WithFailures(api.ErrTimeoutExceeded)), nil
 		}
 		return nil, ExecRuntimeError(err)
 	}
@@ -45,5 +44,5 @@ func (s *Spec) Eval(
 			}
 		}
 	}
-	return result.New(result.WithFailures(a.Failures()...)), nil
+	return api.NewResult(api.WithFailures(a.Failures()...)), nil
 }
diff --git a/plugin/exec/parse.go b/plugin/exec/parse.go
index 939104e..295d5a0 100644
--- a/plugin/exec/parse.go
+++ b/plugin/exec/parse.go
@@ -14,14 +14,13 @@ import (
 	"github.com/samber/lo"
 	"gopkg.in/yaml.v3"
 
-	"github.com/gdt-dev/gdt/errors"
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 var (
 	// ErrUnknownShell returns an ErrParse when an unknown shell is specified
 	ErrUnknownShell = fmt.Errorf(
-		"%w: unknown shell", errors.ErrParse,
+		"%w: unknown shell", api.ErrParse,
 	)
 )
 
@@ -35,7 +34,7 @@ func UnknownShell(shell string) error {
 
 func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	var execValNode *yaml.Node
 	// maps/structs are stored in a top-level Node.Content field which is a
@@ -43,14 +42,14 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "shell":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.Shell = strings.TrimSpace(valNode.Value)
 			if _, err := exec.LookPath(s.Shell); err != nil {
@@ -58,7 +57,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			}
 		case "exec":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			execValNode = valNode
 			s.Exec = strings.TrimSpace(valNode.Value)
@@ -67,7 +66,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			}
 		case "assert":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			var e *Expect
 			if err := valNode.Decode(&e); err != nil {
@@ -76,7 +75,7 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			s.Assert = e
 		case "on":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			var o *On
 			if err := valNode.Decode(&o); err != nil {
@@ -84,10 +83,10 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 			}
 			s.On = o
 		default:
-			if lo.Contains(gdttypes.BaseSpecFields, key) {
+			if lo.Contains(api.BaseSpecFields, key) {
 				continue
 			}
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	if s.Exec == "" {
@@ -104,21 +103,21 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 
 func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "exit_code", "exit-code":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			ec, err := strconv.Atoi(valNode.Value)
 			if err != nil {
@@ -127,7 +126,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			e.ExitCode = ec
 		case "out":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			var pe *PipeExpect
 			if err := valNode.Decode(&pe); err != nil {
@@ -136,7 +135,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			e.Out = pe
 		case "err":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			var pe *PipeExpect
 			if err := valNode.Decode(&pe); err != nil {
@@ -144,7 +143,7 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 			}
 			e.Err = pe
 		default:
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	return nil
@@ -152,47 +151,47 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 
 func (e *PipeExpect) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "all", "is", "contains", "contains-all", "contains_all":
 			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
-				return errors.ExpectedScalarOrSequenceAt(valNode)
+				return api.ExpectedScalarOrSequenceAt(valNode)
 			}
-			var v gdttypes.FlexStrings
+			var v api.FlexStrings
 			if err := valNode.Decode(&v); err != nil {
 				return err
 			}
 			e.ContainsAll = &v
 		case "any", "contains-one-of", "contains-any", "contains_one_of", "contains_any":
 			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
-				return errors.ExpectedScalarOrSequenceAt(valNode)
+				return api.ExpectedScalarOrSequenceAt(valNode)
 			}
-			var v gdttypes.FlexStrings
+			var v api.FlexStrings
 			if err := valNode.Decode(&v); err != nil {
 				return err
 			}
 			e.ContainsAny = &v
 		case "none", "none-of", "contains-none-of", "contains-none", "none_of", "contains_none_of", "contains_none":
 			if valNode.Kind != yaml.ScalarNode && valNode.Kind != yaml.SequenceNode {
-				return errors.ExpectedScalarOrSequenceAt(valNode)
+				return api.ExpectedScalarOrSequenceAt(valNode)
 			}
-			var v gdttypes.FlexStrings
+			var v api.FlexStrings
 			if err := valNode.Decode(&v); err != nil {
 				return err
 			}
 			e.ContainsNone = &v
 		default:
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	return nil
diff --git a/plugin/exec/parse_test.go b/plugin/exec/parse_test.go
index bc0e697..c79b419 100644
--- a/plugin/exec/parse_test.go
+++ b/plugin/exec/parse_test.go
@@ -9,10 +9,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 	gdtexec "github.com/gdt-dev/gdt/plugin/exec"
 	"github.com/gdt-dev/gdt/scenario"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -31,7 +30,7 @@ func TestUnknownShell(t *testing.T) {
 	)
 	assert.NotNil(err)
 	assert.ErrorIs(err, gdtexec.ErrUnknownShell)
-	assert.ErrorIs(err, errors.ErrParse)
+	assert.ErrorIs(err, api.ErrParse)
 	assert.Nil(s)
 }
 
@@ -51,11 +50,11 @@ func TestSimpleCommand(t *testing.T) {
 	assert.NotNil(s)
 
 	assert.IsType(&scenario.Scenario{}, s)
-	expTests := []gdttypes.Evaluable{
+	expTests := []api.Evaluable{
 		&gdtexec.Spec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    0,
-				Defaults: &gdttypes.Defaults{},
+				Defaults: &api.Defaults{},
 			},
 			Action: gdtexec.Action{
 				Exec: "ls",
diff --git a/plugin/exec/plugin.go b/plugin/exec/plugin.go
index 222c86d..87fac20 100644
--- a/plugin/exec/plugin.go
+++ b/plugin/exec/plugin.go
@@ -7,8 +7,8 @@ package exec
 import (
 	"gopkg.in/yaml.v3"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtplugin "github.com/gdt-dev/gdt/plugin"
-	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
 var (
@@ -30,10 +30,10 @@ const (
 
 type plugin struct{}
 
-func (p *plugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *plugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: pluginName,
-		Timeout: &gdttypes.Timeout{
+		Timeout: &api.Timeout{
 			After: DefaultTimeout,
 		},
 	}
@@ -43,11 +43,11 @@ func (p *plugin) Defaults() yaml.Unmarshaler {
 	return &Defaults{}
 }
 
-func (p *plugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&Spec{}}
+func (p *plugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&Spec{}}
 }
 
 // Plugin returns the HTTP gdt plugin
-func Plugin() gdttypes.Plugin {
+func Plugin() api.Plugin {
 	return &plugin{}
 }
diff --git a/plugin/exec/spec.go b/plugin/exec/spec.go
index fa591da..5cb76c1 100644
--- a/plugin/exec/spec.go
+++ b/plugin/exec/spec.go
@@ -5,13 +5,13 @@
 package exec
 
 import (
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 // Spec describes a single Spec that executes one or more commands via the
 // operating system's `exec` family of functions.
 type Spec struct {
-	gdttypes.Spec
+	api.Spec
 	Action
 	// Assert is an object containing the conditions that the Spec will assert.
 	Assert *Expect `yaml:"assert,omitempty"`
@@ -19,18 +19,18 @@ type Spec struct {
 	On *On `yaml:"on,omitempty"`
 }
 
-func (s *Spec) SetBase(b gdttypes.Spec) {
+func (s *Spec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *Spec) Base() *gdttypes.Spec {
+func (s *Spec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *Spec) Retry() *gdttypes.Retry {
+func (s *Spec) Retry() *api.Retry {
 	return nil
 }
 
-func (s *Spec) Timeout() *gdttypes.Timeout {
+func (s *Spec) Timeout() *api.Timeout {
 	return nil
 }
diff --git a/plugin/registry.go b/plugin/registry.go
index 1354ea1..023ddd4 100644
--- a/plugin/registry.go
+++ b/plugin/registry.go
@@ -8,18 +8,18 @@ import (
 	"strings"
 	"sync"
 
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 // registry stores a set of Plugins and is safe to use in threaded
 // environments.
 type registry struct {
 	sync.RWMutex
-	entries map[string]gdttypes.Plugin
+	entries map[string]api.Plugin
 }
 
 // Remove delists the Plugin with registry. Only really useful for testing.
-func (r *registry) Remove(p gdttypes.Plugin) {
+func (r *registry) Remove(p api.Plugin) {
 	r.Lock()
 	defer r.Unlock()
 	lowered := strings.ToLower(p.Info().Name)
@@ -27,7 +27,7 @@ func (r *registry) Remove(p gdttypes.Plugin) {
 }
 
 // Add registers a Plugin with the registry.
-func (r *registry) Add(p gdttypes.Plugin) {
+func (r *registry) Add(p api.Plugin) {
 	r.Lock()
 	defer r.Unlock()
 	lowered := strings.ToLower(p.Info().Name)
@@ -35,10 +35,10 @@ func (r *registry) Add(p gdttypes.Plugin) {
 }
 
 // List returns a slice of Plugins that are registered with gdt.
-func (r *registry) List() []gdttypes.Plugin {
+func (r *registry) List() []api.Plugin {
 	r.RLock()
 	defer r.RUnlock()
-	res := []gdttypes.Plugin{}
+	res := []api.Plugin{}
 	for _, p := range r.entries {
 		res = append(res, p)
 	}
@@ -47,7 +47,7 @@ func (r *registry) List() []gdttypes.Plugin {
 
 var (
 	knownPlugins = &registry{
-		entries: map[string]gdttypes.Plugin{},
+		entries: map[string]api.Plugin{},
 	}
 )
 
@@ -55,11 +55,11 @@ var (
 //
 // Generally only plugin authors will ever need to call this function. It is
 // not required for normal use of gdt or any known plugin.
-func Register(p gdttypes.Plugin) {
+func Register(p api.Plugin) {
 	knownPlugins.Add(p)
 }
 
 // Registered returns a slice of pointers to gdt's known plugins.
-func Registered() []gdttypes.Plugin {
+func Registered() []api.Plugin {
 	return knownPlugins.List()
 }
diff --git a/plugin/registry_test.go b/plugin/registry_test.go
index 59fe284..769443b 100644
--- a/plugin/registry_test.go
+++ b/plugin/registry_test.go
@@ -8,9 +8,8 @@ import (
 	"context"
 	"testing"
 
+	"github.com/gdt-dev/gdt/api"
 	"github.com/gdt-dev/gdt/plugin"
-	"github.com/gdt-dev/gdt/result"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
@@ -24,27 +23,27 @@ func (d *fooDefaults) UnmarshalYAML(node *yaml.Node) error {
 }
 
 type fooSpec struct {
-	gdttypes.Spec
+	api.Spec
 	Foo string `yaml:"foo"`
 }
 
-func (s *fooSpec) SetBase(b gdttypes.Spec) {
+func (s *fooSpec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *fooSpec) Base() *gdttypes.Spec {
+func (s *fooSpec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *fooSpec) Retry() *gdttypes.Retry {
+func (s *fooSpec) Retry() *api.Retry {
 	return nil
 }
 
-func (s *fooSpec) Timeout() *gdttypes.Timeout {
+func (s *fooSpec) Timeout() *api.Timeout {
 	return nil
 }
 
-func (s *fooSpec) Eval(context.Context) (*result.Result, error) {
+func (s *fooSpec) Eval(context.Context) (*api.Result, error) {
 	return nil, nil
 }
 
@@ -54,8 +53,8 @@ func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 
 type fooPlugin struct{}
 
-func (p *fooPlugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *fooPlugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: "foo",
 	}
 }
@@ -64,8 +63,8 @@ func (p *fooPlugin) Defaults() yaml.Unmarshaler {
 	return &fooDefaults{}
 }
 
-func (p *fooPlugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&fooSpec{}}
+func (p *fooPlugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&fooSpec{}}
 }
 
 func TestRegisterAndList(t *testing.T) {
diff --git a/porcelain.go b/porcelain.go
index 743779b..0d7d7b4 100644
--- a/porcelain.go
+++ b/porcelain.go
@@ -8,8 +8,8 @@ import (
 	"io"
 	"os"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
-	gdterrors "github.com/gdt-dev/gdt/errors"
 	jsonfix "github.com/gdt-dev/gdt/fixture/json"
 	"github.com/gdt-dev/gdt/plugin"
 	_ "github.com/gdt-dev/gdt/plugin/exec"
@@ -85,7 +85,7 @@ var (
 	// with `go test -v=test2json`.
 	SetDebug = gdtcontext.SetDebug
 	// NewJSONFixture takes a string, some bytes or an io.Reader and returns a
-	// new gdttypes.Fixture that can have its state queried via JSONPath
+	// new api.Fixture that can have its state queried via JSONPath
 	NewJSONFixture = jsonfix.New
 )
 
@@ -124,6 +124,6 @@ func From(source interface{}) (*suite.Suite, error) {
 		}
 		return suite.FromScenario(s), nil
 	default:
-		return nil, gdterrors.UnknownSourceType(source)
+		return nil, api.UnknownSourceType(source)
 	}
 }
diff --git a/porcelain_test.go b/porcelain_test.go
index 4f89b94..3edac1a 100644
--- a/porcelain_test.go
+++ b/porcelain_test.go
@@ -11,7 +11,7 @@ import (
 	"testing"
 
 	"github.com/gdt-dev/gdt"
-	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -24,7 +24,7 @@ func TestFromUnknownSourceType(t *testing.T) {
 	require.NotNil(err)
 	require.Nil(s)
 
-	assert.ErrorIs(err, gdterrors.ErrUnknownSourceType)
+	assert.ErrorIs(err, api.ErrUnknownSourceType)
 }
 
 func TestFromFileNotFound(t *testing.T) {
diff --git a/scenario/defaults.go b/scenario/defaults.go
index 68fff91..95bb250 100644
--- a/scenario/defaults.go
+++ b/scenario/defaults.go
@@ -7,9 +7,9 @@ package scenario
 import (
 	"time"
 
-	"github.com/gdt-dev/gdt/errors"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"gopkg.in/yaml.v3"
+
+	"github.com/gdt-dev/gdt/api"
 )
 
 const (
@@ -24,33 +24,33 @@ const (
 type Defaults struct {
 	// Timeout has fields that represent the default timeout behaviour and
 	// expectations to use for test specs in the scenario.
-	Timeout *gdttypes.Timeout `yaml:"timeout,omitempty"`
+	Timeout *api.Timeout `yaml:"timeout,omitempty"`
 	// Retry has fields that represent the default retry behaviour for test
 	// specs in the scenario.
-	Retry *gdttypes.Retry `yaml:"retry,omitempty"`
+	Retry *api.Retry `yaml:"retry,omitempty"`
 }
 
 func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "timeout":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
-			var to *gdttypes.Timeout
+			var to *api.Timeout
 			if err := valNode.Decode(&to); err != nil {
-				return errors.ExpectedTimeoutAt(valNode)
+				return api.ExpectedTimeoutAt(valNode)
 			}
 			_, err := time.ParseDuration(to.After)
 			if err != nil {
@@ -59,16 +59,16 @@ func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
 			d.Timeout = to
 		case "retry":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
-			var r *gdttypes.Retry
+			var r *api.Retry
 			if err := valNode.Decode(&r); err != nil {
-				return errors.ExpectedRetryAt(valNode)
+				return api.ExpectedRetryAt(valNode)
 			}
 			if r.Attempts != nil {
 				attempts := *r.Attempts
 				if attempts < 1 {
-					return errors.InvalidRetryAttempts(valNode, attempts)
+					return api.InvalidRetryAttempts(valNode, attempts)
 				}
 			}
 			if r.Interval != "" {
diff --git a/scenario/parse.go b/scenario/parse.go
index 753432e..4f34698 100644
--- a/scenario/parse.go
+++ b/scenario/parse.go
@@ -9,19 +9,18 @@ import (
 
 	"gopkg.in/yaml.v3"
 
-	gdterrors "github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 	"github.com/gdt-dev/gdt/plugin"
-	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
 // UnmarshalYAML is a custom unmarshaler that asks plugins for their known spec
 // types and attempts to unmarshal test spec contents into those types.
 func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return gdterrors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	plugins := plugin.Registered()
-	defaults := gdttypes.Defaults{}
+	defaults := api.Defaults{}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	//
@@ -32,38 +31,38 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return gdterrors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "name":
 			if valNode.Kind != yaml.ScalarNode {
-				return gdterrors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.Name = valNode.Value
 		case "description":
 			if valNode.Kind != yaml.ScalarNode {
-				return gdterrors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.Description = valNode.Value
 		case "fixtures":
 			if valNode.Kind != yaml.SequenceNode {
-				return gdterrors.ExpectedSequenceAt(valNode)
+				return api.ExpectedSequenceAt(valNode)
 			}
 			var fixtures []string
 			if err := valNode.Decode(&fixtures); err != nil {
-				return gdterrors.ExpectedSequenceAt(valNode)
+				return api.ExpectedSequenceAt(valNode)
 			}
 			s.Fixtures = fixtures
 		case "defaults":
 			if valNode.Kind != yaml.MappingNode {
-				return gdterrors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			// Each plugin can have its own set of default configuration values
 			// under an outer map field keyed to the name of the plugin.
 			// Plugins return a Defaults prototype from
-			// `gdttypes.Plugin.Defaults()` that understands how to parse a
+			// `api.Plugin.Defaults()` that understands how to parse a
 			// `yaml.Node` that represents the top-level defaults object in the
 			// scenario.
 			for _, p := range plugins {
@@ -84,35 +83,35 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 		}
 	}
 	// We store a lookup to the parsing plugin for each parsed test spec
-	evalPlugins := map[int]gdttypes.Plugin{}
+	evalPlugins := map[int]api.Plugin{}
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return gdterrors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "tests":
 			if valNode.Kind != yaml.SequenceNode {
-				return gdterrors.ExpectedSequenceAt(valNode)
+				return api.ExpectedSequenceAt(valNode)
 			}
 			for idx, testNode := range valNode.Content {
 				parsed := false
-				base := gdttypes.Spec{}
+				base := api.Spec{}
 				if err := testNode.Decode(&base); err != nil {
 					return err
 				}
 				base.Index = idx
 				base.Defaults = &defaults
-				pluginSpecs := map[gdttypes.Plugin][]gdttypes.Evaluable{}
+				pluginSpecs := map[api.Plugin][]api.Evaluable{}
 				for _, p := range plugins {
 					pluginSpecs[p] = p.Specs()
 				}
 				for plugin, specs := range pluginSpecs {
 					for _, sp := range specs {
 						if err := testNode.Decode(sp); err != nil {
-							if errors.Is(err, gdterrors.ErrUnknownField) {
+							if errors.Is(err, api.ErrUnknownField) {
 								continue
 							}
 							return err
@@ -125,28 +124,28 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 					}
 				}
 				if !parsed {
-					return gdterrors.UnknownSpecAt(s.Path, valNode)
+					return api.UnknownSpecAt(s.Path, valNode)
 				}
 			}
 		case "skip-if":
 			if valNode.Kind != yaml.SequenceNode {
-				return gdterrors.ExpectedSequenceAt(valNode)
+				return api.ExpectedSequenceAt(valNode)
 			}
 			for idx, testNode := range valNode.Content {
 				parsed := false
-				base := gdttypes.Spec{}
+				base := api.Spec{}
 				if err := testNode.Decode(&base); err != nil {
 					return err
 				}
 				base.Index = idx
 				base.Defaults = &defaults
-				specs := []gdttypes.Evaluable{}
+				specs := []api.Evaluable{}
 				for _, p := range plugins {
 					specs = append(specs, p.Specs()...)
 				}
 				for _, sp := range specs {
 					if err := testNode.Decode(sp); err != nil {
-						if errors.Is(err, gdterrors.ErrUnknownField) {
+						if errors.Is(err, api.ErrUnknownField) {
 							continue
 						}
 						return err
@@ -157,7 +156,7 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 					break
 				}
 				if !parsed {
-					return gdterrors.UnknownSpecAt(s.Path, valNode)
+					return api.UnknownSpecAt(s.Path, valNode)
 				}
 			}
 		}
diff --git a/scenario/parse_test.go b/scenario/parse_test.go
index 14bb855..76de4e0 100644
--- a/scenario/parse_test.go
+++ b/scenario/parse_test.go
@@ -9,9 +9,8 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/gdt-dev/gdt/errors"
+	"github.com/gdt-dev/gdt/api"
 	"github.com/gdt-dev/gdt/scenario"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -113,7 +112,7 @@ func TestBadTimeout(t *testing.T) {
 	require.Nil(err)
 
 	s, err := scenario.FromReader(f, scenario.WithPath(fp))
-	assert.ErrorIs(err, errors.ErrExpectedScalarOrMap)
+	assert.ErrorIs(err, api.ErrExpectedScalarOrMap)
 	assert.Nil(s)
 }
 
@@ -152,7 +151,7 @@ func TestBadRetry(t *testing.T) {
 	require.Nil(err)
 
 	s, err := scenario.FromReader(f, scenario.WithPath(fp))
-	assert.ErrorIs(err, errors.ErrExpectedMap)
+	assert.ErrorIs(err, api.ErrExpectedMap)
 	assert.Nil(s)
 }
 
@@ -165,7 +164,7 @@ func TestBadRetryAttempts(t *testing.T) {
 	require.Nil(err)
 
 	s, err := scenario.FromReader(f, scenario.WithPath(fp))
-	assert.ErrorIs(err, errors.ErrInvalidRetryAttempts)
+	assert.ErrorIs(err, api.ErrInvalidRetryAttempts)
 	assert.Nil(s)
 }
 
@@ -212,7 +211,7 @@ func TestKnownSpec(t *testing.T) {
 		},
 		s.Defaults,
 	)
-	expSpecDefaults := &gdttypes.Defaults{
+	expSpecDefaults := &api.Defaults{
 		"foo": &fooDefaults{
 			fooInnerDefaults{
 				Bar: "barconfig",
@@ -223,9 +222,9 @@ func TestKnownSpec(t *testing.T) {
 		"priorRun":           &priorRunDefaults{},
 		scenario.DefaultsKey: &scenario.Defaults{},
 	}
-	expTests := []gdttypes.Evaluable{
+	expTests := []api.Evaluable{
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    0,
 				Name:     "bar",
 				Defaults: expSpecDefaults,
@@ -233,7 +232,7 @@ func TestKnownSpec(t *testing.T) {
 			Foo: "bar",
 		},
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:       1,
 				Description: "Bazzy Bizzy",
 				Defaults:    expSpecDefaults,
@@ -259,18 +258,18 @@ func TestMultipleSpec(t *testing.T) {
 	assert.IsType(&scenario.Scenario{}, s)
 	assert.Equal("foo-bar", s.Name)
 	assert.Equal(filepath.Join("testdata", "foo-bar.yaml"), s.Path)
-	expTests := []gdttypes.Evaluable{
+	expTests := []api.Evaluable{
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    0,
-				Defaults: &gdttypes.Defaults{},
+				Defaults: &api.Defaults{},
 			},
 			Foo: "bar",
 		},
 		&barSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    1,
-				Defaults: &gdttypes.Defaults{},
+				Defaults: &api.Defaults{},
 			},
 			Bar: 42,
 		},
@@ -312,7 +311,7 @@ func TestEnvExpansion(t *testing.T) {
 		},
 		s.Defaults,
 	)
-	expSpecDefaults := &gdttypes.Defaults{
+	expSpecDefaults := &api.Defaults{
 		"foo": &fooDefaults{
 			fooInnerDefaults{
 				Bar: "barconfig",
@@ -323,9 +322,9 @@ func TestEnvExpansion(t *testing.T) {
 		"priorRun":           &priorRunDefaults{},
 		scenario.DefaultsKey: &scenario.Defaults{},
 	}
-	expTests := []gdttypes.Evaluable{
+	expTests := []api.Evaluable{
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    0,
 				Name:     "$NOT_EXPANDED",
 				Defaults: expSpecDefaults,
@@ -333,7 +332,7 @@ func TestEnvExpansion(t *testing.T) {
 			Foo: "bar",
 		},
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:       1,
 				Description: "Bazzy Bizzy",
 				Defaults:    expSpecDefaults,
@@ -367,37 +366,37 @@ func TestScenarioDefaults(t *testing.T) {
 			"fail":     &failDefaults{failInnerDefaults{}},
 			"priorRun": &priorRunDefaults{},
 			scenario.DefaultsKey: &scenario.Defaults{
-				Timeout: &gdttypes.Timeout{
+				Timeout: &api.Timeout{
 					After: "2s",
 				},
 			},
 		},
 		s.Defaults,
 	)
-	expSpecDefaults := &gdttypes.Defaults{
+	expSpecDefaults := &api.Defaults{
 		"foo":      &fooDefaults{},
 		"bar":      &barDefaults{},
 		"fail":     &failDefaults{failInnerDefaults{}},
 		"priorRun": &priorRunDefaults{},
 		scenario.DefaultsKey: &scenario.Defaults{
-			Timeout: &gdttypes.Timeout{
+			Timeout: &api.Timeout{
 				After: "2s",
 			},
 		},
 	}
-	expTests := []gdttypes.Evaluable{
+	expTests := []api.Evaluable{
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    0,
 				Defaults: expSpecDefaults,
-				Timeout: &gdttypes.Timeout{
+				Timeout: &api.Timeout{
 					After: "1s",
 				},
 			},
 			Foo: "baz",
 		},
 		&fooSpec{
-			Spec: gdttypes.Spec{
+			Spec: api.Spec{
 				Index:    1,
 				Defaults: expSpecDefaults,
 			},
diff --git a/scenario/run.go b/scenario/run.go
index 35bb607..a6e37fb 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -14,15 +14,13 @@ import (
 
 	"github.com/cenkalti/backoff"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
-	gdterrors "github.com/gdt-dev/gdt/errors"
-	"github.com/gdt-dev/gdt/result"
-	gdttypes "github.com/gdt-dev/gdt/types"
 )
 
 // Run executes the scenario. The error that is returned will always be derived
-// from `gdterrors.RuntimeError` and represents an *unrecoverable* error.
+// from `api.RuntimeError` and represents an *unrecoverable* error.
 //
 // Test assertion failures are *not* considered errors. The Scenario.Run()
 // method controls whether `testing.T.Fail()` or `testing.T.Skip()` is called
@@ -35,7 +33,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			lookup := strings.ToLower(fname)
 			fix, found := fixtures[lookup]
 			if !found {
-				return gdterrors.RequiredFixtureMissing(fname)
+				return api.RequiredFixtureMissing(fname)
 			}
 			if err := fix.Start(ctx); err != nil {
 				return err
@@ -120,11 +118,11 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 // runSpec executes an individual test spec
 func (s *Scenario) runSpec(
 	ctx context.Context,
-	retry *gdttypes.Retry,
-	timeout *gdttypes.Timeout,
+	retry *api.Retry,
+	timeout *api.Timeout,
 	idx int,
-	spec gdttypes.Evaluable,
-) (*result.Result, error) {
+	spec api.Evaluable,
+) (*api.Result, error) {
 	sb := spec.Base()
 	specTraceMsg := strconv.Itoa(idx)
 	if sb.Name != "" {
@@ -134,7 +132,7 @@ func (s *Scenario) runSpec(
 	defer func() {
 		ctx = gdtcontext.PopTrace(ctx)
 	}()
-	if retry == nil || retry == gdttypes.NoRetry {
+	if retry == nil || retry == api.NoRetry {
 		// Just evaluate the test spec once
 		res, err := spec.Eval(ctx)
 		if err != nil {
@@ -150,7 +148,7 @@ func (s *Scenario) runSpec(
 	// retry the action and test the assertions until they succeed,
 	// there is a terminal failure, or the timeout expires.
 	var bo backoff.BackOff
-	var res *result.Result
+	var res *api.Result
 	var err error
 
 	if retry.Exponential {
@@ -159,7 +157,7 @@ func (s *Scenario) runSpec(
 			ctx,
 		)
 	} else {
-		interval := gdttypes.DefaultRetryConstantInterval
+		interval := api.DefaultRetryConstantInterval
 		if retry.Interval != "" {
 			interval = retry.IntervalDuration()
 		}
@@ -221,9 +219,9 @@ func (s *Scenario) runSpec(
 func getTimeout(
 	ctx context.Context,
 	scenDefaults *Defaults,
-	plugin gdttypes.Plugin,
-	eval gdttypes.Evaluable,
-) *gdttypes.Timeout {
+	plugin api.Plugin,
+	eval api.Evaluable,
+) *api.Timeout {
 	evalTimeout := eval.Timeout()
 	if evalTimeout != nil {
 		debug.Println(
@@ -274,12 +272,12 @@ func getTimeout(
 func getRetry(
 	ctx context.Context,
 	scenDefaults *Defaults,
-	plugin gdttypes.Plugin,
-	eval gdttypes.Evaluable,
-) *gdttypes.Retry {
+	plugin api.Plugin,
+	eval api.Evaluable,
+) *api.Retry {
 	evalRetry := eval.Retry()
 	if evalRetry != nil {
-		if evalRetry == gdttypes.NoRetry {
+		if evalRetry == api.NoRetry {
 			return evalRetry
 		}
 		msg := "using retry"
@@ -297,7 +295,7 @@ func getRetry(
 	sb := eval.Base()
 	baseRetry := sb.Retry
 	if baseRetry != nil {
-		if baseRetry == gdttypes.NoRetry {
+		if baseRetry == api.NoRetry {
 			return baseRetry
 		}
 		msg := "using retry"
@@ -314,7 +312,7 @@ func getRetry(
 
 	if scenDefaults != nil && scenDefaults.Retry != nil {
 		scenRetry := scenDefaults.Retry
-		if scenRetry == gdttypes.NoRetry {
+		if scenRetry == api.NoRetry {
 			return scenRetry
 		}
 		msg := "using retry"
@@ -333,7 +331,7 @@ func getRetry(
 	pluginRetry := pluginInfo.Retry
 
 	if pluginRetry != nil {
-		if pluginRetry == gdttypes.NoRetry {
+		if pluginRetry == api.NoRetry {
 			return pluginRetry
 		}
 		msg := "using retry"
diff --git a/scenario/run_test.go b/scenario/run_test.go
index 92c8ab3..c458d93 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -14,8 +14,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
-	gdterrors "github.com/gdt-dev/gdt/errors"
 	"github.com/gdt-dev/gdt/scenario"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -68,8 +68,8 @@ func TestMissingFixtures(t *testing.T) {
 	// Pass a context with no fixtures registered...
 	err = s.Run(context.TODO(), t)
 	assert.NotNil(err)
-	assert.ErrorIs(err, gdterrors.ErrRequiredFixture)
-	assert.ErrorIs(err, gdterrors.RuntimeError)
+	assert.ErrorIs(err, api.ErrRequiredFixture)
+	assert.ErrorIs(err, api.RuntimeError)
 }
 
 func TestFixtureStartError(t *testing.T) {
diff --git a/scenario/scenario.go b/scenario/scenario.go
index 18716e2..c12b337 100644
--- a/scenario/scenario.go
+++ b/scenario/scenario.go
@@ -7,7 +7,7 @@ package scenario
 import (
 	gopath "path"
 
-	gdttypes "github.com/gdt-dev/gdt/types"
+	"github.com/gdt-dev/gdt/api"
 )
 
 // Scenario is a generalized gdt test case file. It contains a set of Runnable
@@ -15,7 +15,7 @@ import (
 type Scenario struct {
 	// evalPlugins stores the plugin that will evaluate the test spec at a
 	// particular index
-	evalPlugins map[int]gdttypes.Plugin
+	evalPlugins map[int]api.Plugin
 	// Path is the filepath to the test case.
 	Path string `yaml:"-"`
 	// Name is the short name for the test case. If empty, defaults to the base
@@ -72,10 +72,10 @@ type Scenario struct {
 	//
 	// With the above, if an 'nginx' deployment exists already, the scenario
 	// will skip all the tests.
-	SkipIf []gdttypes.Evaluable `yaml:"skip-if,omitempty"`
+	SkipIf []api.Evaluable `yaml:"skip-if,omitempty"`
 	// Tests is the collection of test units in this test case. These will be
 	// the fully parsed and materialized plugin Spec structs.
-	Tests []gdttypes.Evaluable `yaml:"tests,omitempty"`
+	Tests []api.Evaluable `yaml:"tests,omitempty"`
 }
 
 // Title returns the Name of the scenario or the Path's file/base name if there
diff --git a/scenario/stub_plugins_test.go b/scenario/stub_plugins_test.go
index 181c3db..d11954e 100644
--- a/scenario/stub_plugins_test.go
+++ b/scenario/stub_plugins_test.go
@@ -9,13 +9,11 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/gdt-dev/gdt/api"
+	gdtapi "github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
 	"github.com/gdt-dev/gdt/debug"
-	"github.com/gdt-dev/gdt/errors"
-	gdterrors "github.com/gdt-dev/gdt/errors"
 	"github.com/gdt-dev/gdt/plugin"
-	"github.com/gdt-dev/gdt/result"
-	gdttypes "github.com/gdt-dev/gdt/types"
 	"github.com/samber/lo"
 	"gopkg.in/yaml.v3"
 )
@@ -37,28 +35,28 @@ type failDefaults struct {
 
 func (d *failDefaults) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "fail":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			inner := failInnerDefaults{}
 			if err := valNode.Decode(&inner); err != nil {
 				return err
 			}
 			d.failInnerDefaults = inner
-			// This is just for testing errors when parsing defaults...
+			// This is just for testing api when parsing defaults...
 			if d.Fail {
 				return fmt.Errorf("defaults parsing failed")
 			}
@@ -70,57 +68,57 @@ func (d *failDefaults) UnmarshalYAML(node *yaml.Node) error {
 }
 
 type failSpec struct {
-	gdttypes.Spec
+	api.Spec
 	Fail bool `yaml:"fail"`
 }
 
-func (s *failSpec) SetBase(b gdttypes.Spec) {
+func (s *failSpec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *failSpec) Base() *gdttypes.Spec {
+func (s *failSpec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *failSpec) Retry() *gdttypes.Retry {
+func (s *failSpec) Retry() *api.Retry {
 	return nil
 }
 
-func (s *failSpec) Timeout() *gdttypes.Timeout {
+func (s *failSpec) Timeout() *api.Timeout {
 	return nil
 }
 
-func (s *failSpec) Eval(context.Context) (*result.Result, error) {
-	return nil, fmt.Errorf("%w: Indy, bad dates!", gdterrors.RuntimeError)
+func (s *failSpec) Eval(context.Context) (*api.Result, error) {
+	return nil, fmt.Errorf("%w: Indy, bad dates!", gdtapi.RuntimeError)
 }
 
 func (s *failSpec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "fail":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.Fail, _ = strconv.ParseBool(valNode.Value)
 			if s.Fail {
 				return fmt.Errorf("Indy, bad parse!")
 			}
 		default:
-			if lo.Contains(gdttypes.BaseSpecFields, key) {
+			if lo.Contains(api.BaseSpecFields, key) {
 				continue
 			}
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	return nil
@@ -128,8 +126,8 @@ func (s *failSpec) UnmarshalYAML(node *yaml.Node) error {
 
 type failingPlugin struct{}
 
-func (p *failingPlugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *failingPlugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: "fail",
 	}
 }
@@ -138,8 +136,8 @@ func (p *failingPlugin) Defaults() yaml.Unmarshaler {
 	return &failDefaults{}
 }
 
-func (p *failingPlugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&failSpec{}}
+func (p *failingPlugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&failSpec{}}
 }
 
 type fooInnerDefaults struct {
@@ -152,21 +150,21 @@ type fooDefaults struct {
 
 func (d *fooDefaults) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "foo":
 			if valNode.Kind != yaml.MappingNode {
-				return errors.ExpectedMapAt(valNode)
+				return api.ExpectedMapAt(valNode)
 			}
 			inner := fooInnerDefaults{}
 			if err := valNode.Decode(&inner); err != nil {
@@ -181,56 +179,56 @@ func (d *fooDefaults) UnmarshalYAML(node *yaml.Node) error {
 }
 
 type fooSpec struct {
-	gdttypes.Spec
+	api.Spec
 	Foo string `yaml:"foo"`
 }
 
-func (s *fooSpec) SetBase(b gdttypes.Spec) {
+func (s *fooSpec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *fooSpec) Base() *gdttypes.Spec {
+func (s *fooSpec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *fooSpec) Retry() *gdttypes.Retry {
+func (s *fooSpec) Retry() *api.Retry {
 	return nil
 }
 
-func (s *fooSpec) Timeout() *gdttypes.Timeout {
+func (s *fooSpec) Timeout() *api.Timeout {
 	return nil
 }
 
 func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "foo":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.Foo = valNode.Value
 		default:
-			if lo.Contains(gdttypes.BaseSpecFields, key) {
+			if lo.Contains(api.BaseSpecFields, key) {
 				continue
 			}
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	return nil
 }
 
-func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
+func (s *fooSpec) Eval(ctx context.Context) (*api.Result, error) {
 	fails := []error{}
 	debug.Println(ctx, "in %s Foo=%s", s.Title(), s.Foo)
 	// This is just a silly test to demonstrate how to write Eval() methods
@@ -242,13 +240,13 @@ func (s *fooSpec) Eval(ctx context.Context) (*result.Result, error) {
 		fail := fmt.Errorf("expected s.Foo = 'baz', got %s", s.Foo)
 		fails = append(fails, fail)
 	}
-	return result.New(result.WithFailures(fails...)), nil
+	return api.NewResult(api.WithFailures(fails...)), nil
 }
 
 type fooPlugin struct{}
 
-func (p *fooPlugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *fooPlugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: "foo",
 	}
 }
@@ -257,8 +255,8 @@ func (p *fooPlugin) Defaults() yaml.Unmarshaler {
 	return &fooDefaults{}
 }
 
-func (p *fooPlugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&fooSpec{}}
+func (p *fooPlugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&fooSpec{}}
 }
 
 type barDefaults struct {
@@ -270,58 +268,58 @@ func (d *barDefaults) UnmarshalYAML(node *yaml.Node) error {
 }
 
 type barSpec struct {
-	gdttypes.Spec
+	api.Spec
 	Bar int `yaml:"bar"`
 }
 
-func (s *barSpec) SetBase(b gdttypes.Spec) {
+func (s *barSpec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *barSpec) Base() *gdttypes.Spec {
+func (s *barSpec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *barSpec) Retry() *gdttypes.Retry {
-	return gdttypes.NoRetry
+func (s *barSpec) Retry() *api.Retry {
+	return api.NoRetry
 }
 
-func (s *barSpec) Timeout() *gdttypes.Timeout {
+func (s *barSpec) Timeout() *api.Timeout {
 	return nil
 }
 
-func (s *barSpec) Eval(context.Context) (*result.Result, error) {
-	return result.New(), nil
+func (s *barSpec) Eval(context.Context) (*api.Result, error) {
+	return api.NewResult(), nil
 }
 
 func (s *barSpec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "bar":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			if v, err := strconv.Atoi(valNode.Value); err != nil {
-				return errors.ExpectedIntAt(valNode)
+				return api.ExpectedIntAt(valNode)
 			} else {
 				s.Bar = v
 			}
 		default:
-			if lo.Contains(gdttypes.BaseSpecFields, key) {
+			if lo.Contains(api.BaseSpecFields, key) {
 				continue
 			}
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	return nil
@@ -329,8 +327,8 @@ func (s *barSpec) UnmarshalYAML(node *yaml.Node) error {
 
 type barPlugin struct{}
 
-func (p *barPlugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *barPlugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: "bar",
 	}
 }
@@ -339,8 +337,8 @@ func (p *barPlugin) Defaults() yaml.Unmarshaler {
 	return &barDefaults{}
 }
 
-func (p *barPlugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&barSpec{}}
+func (p *barPlugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&barSpec{}}
 }
 
 const priorRunDataKey = "priorrun"
@@ -352,62 +350,62 @@ func (d *priorRunDefaults) UnmarshalYAML(node *yaml.Node) error {
 }
 
 type priorRunSpec struct {
-	gdttypes.Spec
+	api.Spec
 	State string `yaml:"state"`
 	Prior string `yaml:"prior"`
 }
 
-func (s *priorRunSpec) SetBase(b gdttypes.Spec) {
+func (s *priorRunSpec) SetBase(b api.Spec) {
 	s.Spec = b
 }
 
-func (s *priorRunSpec) Base() *gdttypes.Spec {
+func (s *priorRunSpec) Base() *api.Spec {
 	return &s.Spec
 }
 
-func (s *priorRunSpec) Retry() *gdttypes.Retry {
+func (s *priorRunSpec) Retry() *api.Retry {
 	return nil
 }
 
-func (s *priorRunSpec) Timeout() *gdttypes.Timeout {
+func (s *priorRunSpec) Timeout() *api.Timeout {
 	return nil
 }
 
 func (s *priorRunSpec) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
-		return errors.ExpectedMapAt(node)
+		return api.ExpectedMapAt(node)
 	}
 	// maps/structs are stored in a top-level Node.Content field which is a
 	// concatenated slice of Node pointers in pairs of key/values.
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
-			return errors.ExpectedScalarAt(keyNode)
+			return api.ExpectedScalarAt(keyNode)
 		}
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
 		case "state":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.State = valNode.Value
 		case "prior":
 			if valNode.Kind != yaml.ScalarNode {
-				return errors.ExpectedScalarAt(valNode)
+				return api.ExpectedScalarAt(valNode)
 			}
 			s.Prior = valNode.Value
 		default:
-			if lo.Contains(gdttypes.BaseSpecFields, key) {
+			if lo.Contains(api.BaseSpecFields, key) {
 				continue
 			}
-			return errors.UnknownFieldAt(key, keyNode)
+			return api.UnknownFieldAt(key, keyNode)
 		}
 	}
 	return nil
 }
 
-func (s *priorRunSpec) Eval(ctx context.Context) (*result.Result, error) {
+func (s *priorRunSpec) Eval(ctx context.Context) (*api.Result, error) {
 	// Here we test that the prior run data that we save at the end of each
 	// Run() is showing up properly in the next Run()'s context.
 	fails := []error{}
@@ -425,16 +423,16 @@ func (s *priorRunSpec) Eval(ctx context.Context) (*result.Result, error) {
 			fails = append(fails, fmt.Errorf("expected priorRunData == data"))
 		}
 	}
-	return result.New(
-		result.WithFailures(fails...),
-		result.WithData(priorRunDataKey, s.State),
+	return api.NewResult(
+		api.WithFailures(fails...),
+		api.WithData(priorRunDataKey, s.State),
 	), nil
 }
 
 type priorRunPlugin struct{}
 
-func (p *priorRunPlugin) Info() gdttypes.PluginInfo {
-	return gdttypes.PluginInfo{
+func (p *priorRunPlugin) Info() api.PluginInfo {
+	return api.PluginInfo{
 		Name: "priorRun",
 	}
 }
@@ -443,6 +441,6 @@ func (p *priorRunPlugin) Defaults() yaml.Unmarshaler {
 	return &priorRunDefaults{}
 }
 
-func (p *priorRunPlugin) Specs() []gdttypes.Evaluable {
-	return []gdttypes.Evaluable{&priorRunSpec{}}
+func (p *priorRunPlugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&priorRunSpec{}}
 }

From 4cff4c59fb1f4ce42f4b2d9a30be84a964b5f481 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 26 Jun 2024 07:31:20 -0400
Subject: [PATCH 17/31] catch innermost spec context timeout

In order to prevent the panic() that occurs if a context timeout
cancel() function fires in the Go testing tool, we execute each test
spec in its own goroutine and catch the individual cancel() function
firing in order to properly output a failed assertion instead of a
panic.

Issue #37

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 scenario/run.go | 37 +++++++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/scenario/run.go b/scenario/run.go
index a6e37fb..94b9542 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -93,7 +93,19 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				defer cancel()
 			}
 
-			res, rterr := s.runSpec(specCtx, rt, to, idx, spec)
+			var res *api.Result
+			ch := make(chan runSpecRes, 1)
+
+			go s.runSpec(specCtx, ch, rt, to, idx, spec)
+
+			select {
+			case <-specCtx.Done():
+				t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
+				break
+			case runres := <-ch:
+				res = runres.r
+				rterr = runres.err
+			}
 			if rterr != nil {
 				break
 			}
@@ -115,14 +127,20 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 	return rterr
 }
 
+type runSpecRes struct {
+	r   *api.Result
+	err error
+}
+
 // runSpec executes an individual test spec
 func (s *Scenario) runSpec(
 	ctx context.Context,
+	ch chan runSpecRes,
 	retry *api.Retry,
 	timeout *api.Timeout,
 	idx int,
 	spec api.Evaluable,
-) (*api.Result, error) {
+) {
 	sb := spec.Base()
 	specTraceMsg := strconv.Itoa(idx)
 	if sb.Name != "" {
@@ -136,13 +154,15 @@ func (s *Scenario) runSpec(
 		// Just evaluate the test spec once
 		res, err := spec.Eval(ctx)
 		if err != nil {
-			return nil, err
+			ch <- runSpecRes{nil, err}
+			return
 		}
 		debug.Println(
 			ctx, "run: single-shot (no retries) ok: %v",
 			!res.Failed(),
 		)
-		return res, nil
+		ch <- runSpecRes{res, nil}
+		return
 	}
 
 	// retry the action and test the assertions until they succeed,
@@ -187,7 +207,8 @@ func (s *Scenario) runSpec(
 
 		res, err = spec.Eval(ctx)
 		if err != nil {
-			return nil, err
+			ch <- runSpecRes{nil, err}
+			return
 		}
 		success = !res.Failed()
 		debug.Println(
@@ -200,13 +221,13 @@ func (s *Scenario) runSpec(
 		}
 		for _, f := range res.Failures() {
 			debug.Println(
-				ctx, "run: attempt %d after %s failure: %s",
-				attempts, after, f,
+				ctx, "run: attempt %d failure: %s",
+				attempts, f,
 			)
 		}
 		attempts++
 	}
-	return res, nil
+	ch <- runSpecRes{res, nil}
 }
 
 // getTimeout returns the timeout configuration for the test spec. We check for

From 3090f729d2fdc332b8e65d23a8fe2fd21f7edcd1 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 26 Jun 2024 12:19:33 -0400
Subject: [PATCH 18/31] try manually canceling specCtx

Instead of defer'ing the specCancel, try manually calling the specCancel
on exit points in each spec execution.

Issue #37

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 scenario/run.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/scenario/run.go b/scenario/run.go
index 94b9542..354d89b 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -84,35 +84,36 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			// we mutate the single supplied top-level context, then only the
 			// first deadline/timeout will be used.
 			specCtx, specCancel := context.WithCancel(ctx)
-			defer specCancel()
 
 			to := getTimeout(ctx, scDefaults, plugin, spec)
 			if to != nil {
-				var cancel context.CancelFunc
-				specCtx, cancel = context.WithTimeout(specCtx, to.Duration())
-				defer cancel()
+				specCtx, specCancel = context.WithTimeout(specCtx, to.Duration())
 			}
 
 			var res *api.Result
 			ch := make(chan runSpecRes, 1)
 
-			go s.runSpec(specCtx, ch, rt, to, idx, spec)
+			go s.runSpec(specCtx, ch, rt, idx, spec)
 
 			select {
 			case <-specCtx.Done():
 				t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
+				specCancel()
 				break
 			case runres := <-ch:
 				res = runres.r
 				rterr = runres.err
 			}
 			if rterr != nil {
+				specCancel()
 				break
 			}
+
 			if wait != nil && wait.After != "" {
 				debug.Println(ctx, "wait: %s after", wait.After)
 				time.Sleep(wait.AfterDuration())
 			}
+
 			// Results can have arbitrary run data stored in them and we
 			// save this prior run data in the top-level context (and pass
 			// that context to the next Run invocation).
@@ -122,6 +123,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			for _, fail := range res.Failures() {
 				t.Fatal(fail)
 			}
+			specCancel()
 		}
 	})
 	return rterr
@@ -137,7 +139,6 @@ func (s *Scenario) runSpec(
 	ctx context.Context,
 	ch chan runSpecRes,
 	retry *api.Retry,
-	timeout *api.Timeout,
 	idx int,
 	spec api.Evaluable,
 ) {

From 380c50ad0503d73bca17f9546aec4dcadc641ffb Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 26 Jun 2024 12:33:10 -0400
Subject: [PATCH 19/31] pull PushTrace up into main spec loop

In order to have the wait debug messages have the appropriate context
prefix, moves the PushTrace up into the main loop over specs.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 scenario/run.go      | 42 +++++++++++++++++++++++-------------------
 scenario/run_test.go |  2 +-
 2 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/scenario/run.go b/scenario/run.go
index 354d89b..adc05b2 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -70,14 +70,6 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 		}()
 		for idx, spec := range s.Tests {
 			sb := spec.Base()
-			wait := sb.Wait
-			if wait != nil && wait.Before != "" {
-				debug.Println(ctx, "wait: %s before", wait.Before)
-				time.Sleep(wait.BeforeDuration())
-			}
-			plugin := s.evalPlugins[idx]
-
-			rt := getRetry(ctx, scDefaults, plugin, spec)
 
 			// Create a brand new context that inherits the top-level context's
 			// cancel func. We want to set deadlines for each test spec and if
@@ -85,7 +77,25 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			// first deadline/timeout will be used.
 			specCtx, specCancel := context.WithCancel(ctx)
 
-			to := getTimeout(ctx, scDefaults, plugin, spec)
+			specTraceMsg := strconv.Itoa(idx)
+			if sb.Name != "" {
+				specTraceMsg += ":" + sb.Name
+			}
+			specCtx = gdtcontext.PushTrace(specCtx, specTraceMsg)
+			popTracer := func() {
+				specCtx = gdtcontext.PopTrace(specCtx)
+			}
+
+			wait := sb.Wait
+			if wait != nil && wait.Before != "" {
+				debug.Println(specCtx, "wait: %s before", wait.Before)
+				time.Sleep(wait.BeforeDuration())
+			}
+			plugin := s.evalPlugins[idx]
+
+			rt := getRetry(specCtx, scDefaults, plugin, spec)
+
+			to := getTimeout(specCtx, scDefaults, plugin, spec)
 			if to != nil {
 				specCtx, specCancel = context.WithTimeout(specCtx, to.Duration())
 			}
@@ -98,6 +108,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			select {
 			case <-specCtx.Done():
 				t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
+				popTracer()
 				specCancel()
 				break
 			case runres := <-ch:
@@ -105,12 +116,13 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				rterr = runres.err
 			}
 			if rterr != nil {
+				popTracer()
 				specCancel()
 				break
 			}
 
 			if wait != nil && wait.After != "" {
-				debug.Println(ctx, "wait: %s after", wait.After)
+				debug.Println(specCtx, "wait: %s after", wait.After)
 				time.Sleep(wait.AfterDuration())
 			}
 
@@ -123,6 +135,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			for _, fail := range res.Failures() {
 				t.Fatal(fail)
 			}
+			popTracer()
 			specCancel()
 		}
 	})
@@ -142,15 +155,6 @@ func (s *Scenario) runSpec(
 	idx int,
 	spec api.Evaluable,
 ) {
-	sb := spec.Base()
-	specTraceMsg := strconv.Itoa(idx)
-	if sb.Name != "" {
-		specTraceMsg += ":" + sb.Name
-	}
-	ctx = gdtcontext.PushTrace(ctx, specTraceMsg)
-	defer func() {
-		ctx = gdtcontext.PopTrace(ctx)
-	}()
 	if retry == nil || retry == api.NoRetry {
 		// Just evaluate the test spec once
 		res, err := spec.Eval(ctx)
diff --git a/scenario/run_test.go b/scenario/run_test.go
index c458d93..8b951a9 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -113,7 +113,7 @@ func TestDebugFlushing(t *testing.T) {
 	w.Flush()
 	require.NotEqual(b.Len(), 0)
 	debugout := b.String()
-	require.Contains(debugout, "[gdt] [foo-debug-wait-flush] wait: 250ms before")
+	require.Contains(debugout, "[gdt] [foo-debug-wait-flush/0:bar] wait: 250ms before")
 }
 
 func TestNoRetry(t *testing.T) {

From 3cee83986640851afbcf8e7e6efb7aaa60b1350a Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 26 Jun 2024 13:54:42 -0400
Subject: [PATCH 20/31] calculate timeout before doing wait.before

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 scenario/run.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scenario/run.go b/scenario/run.go
index adc05b2..a2386d4 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -86,11 +86,6 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				specCtx = gdtcontext.PopTrace(specCtx)
 			}
 
-			wait := sb.Wait
-			if wait != nil && wait.Before != "" {
-				debug.Println(specCtx, "wait: %s before", wait.Before)
-				time.Sleep(wait.BeforeDuration())
-			}
 			plugin := s.evalPlugins[idx]
 
 			rt := getRetry(specCtx, scDefaults, plugin, spec)
@@ -103,6 +98,12 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			var res *api.Result
 			ch := make(chan runSpecRes, 1)
 
+			wait := sb.Wait
+			if wait != nil && wait.Before != "" {
+				debug.Println(specCtx, "wait: %s before", wait.Before)
+				time.Sleep(wait.BeforeDuration())
+			}
+
 			go s.runSpec(specCtx, ch, rt, idx, spec)
 
 			select {

From 449bb12a73cec9a23257477ba34132e90ef454c5 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 26 Jun 2024 14:31:11 -0400
Subject: [PATCH 21/31] move WithTimeout after wait.before execution

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 plugin/exec/eval_test.go                    | 19 +++++++++++++++++++
 plugin/exec/testdata/timeout-with-wait.yaml | 10 ++++++++++
 scenario/run.go                             |  7 ++++---
 3 files changed, 33 insertions(+), 3 deletions(-)
 create mode 100644 plugin/exec/testdata/timeout-with-wait.yaml

diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index f433ea2..60f40e8 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -376,3 +376,22 @@ func TestExecOnFail(t *testing.T) {
 	require.Contains(debugout, "assertion failed: not equal: expected dat but got cat")
 	require.Contains(debugout, "echo [bad kitty]")
 }
+
+func TestTimeoutWithWait(t *testing.T) {
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "timeout-with-wait.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
diff --git a/plugin/exec/testdata/timeout-with-wait.yaml b/plugin/exec/testdata/timeout-with-wait.yaml
new file mode 100644
index 0000000..3beafa2
--- /dev/null
+++ b/plugin/exec/testdata/timeout-with-wait.yaml
@@ -0,0 +1,10 @@
+name: timeout-with-wait
+description: a scenario that waits and the wait time should not be included in the timeout
+tests:
+  - exec: sleep .50
+    wait:
+      # The timeout should NOT fire because sleep .50 should be the only thing
+      # that is accounted for in the timeout, not the wait.before
+      before: .5s
+    timeout:
+      after: 1s
diff --git a/scenario/run.go b/scenario/run.go
index a2386d4..5b38588 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -91,9 +91,6 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			rt := getRetry(specCtx, scDefaults, plugin, spec)
 
 			to := getTimeout(specCtx, scDefaults, plugin, spec)
-			if to != nil {
-				specCtx, specCancel = context.WithTimeout(specCtx, to.Duration())
-			}
 
 			var res *api.Result
 			ch := make(chan runSpecRes, 1)
@@ -104,6 +101,10 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				time.Sleep(wait.BeforeDuration())
 			}
 
+			if to != nil {
+				specCtx, specCancel = context.WithTimeout(specCtx, to.Duration())
+			}
+
 			go s.runSpec(specCtx, ch, rt, idx, spec)
 
 			select {

From 560d0ce612c67182977dabbc8f634dbd562d2c74 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Fri, 28 Jun 2024 07:41:03 -0400
Subject: [PATCH 22/31] rename Suite.Require -> Suite.Fixtures

Also adds an `api.Runnable` interface and switches the porcelain
`gdt.From()` function to return that interface instead of a concrete
`*suite.Suite` struct.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 api/runnable.go   | 20 ++++++++++++++++
 porcelain.go      |  4 ++--
 porcelain_test.go | 60 +++++++++++++++++++++++++++++++----------------
 suite/from.go     |  7 +++---
 suite/suite.go    | 12 +++++-----
 5 files changed, 71 insertions(+), 32 deletions(-)
 create mode 100644 api/runnable.go

diff --git a/api/runnable.go b/api/runnable.go
new file mode 100644
index 0000000..9e46f2f
--- /dev/null
+++ b/api/runnable.go
@@ -0,0 +1,20 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package api
+
+import (
+	"context"
+	"testing"
+)
+
+// Runnable are things that Run a `*testing.T`
+type Runnable interface {
+	// Run accepts a context and a `*testing.T` and runs some tests within that
+	// context
+	//
+	// Errors returned by Run() are **RuntimeErrors**, not failures in
+	// assertions.
+	Run(context.Context, *testing.T) error
+}
diff --git a/porcelain.go b/porcelain.go
index 0d7d7b4..d2b6c18 100644
--- a/porcelain.go
+++ b/porcelain.go
@@ -89,10 +89,10 @@ var (
 	NewJSONFixture = jsonfix.New
 )
 
-// From returns a new suite.Suite from an `io.Reader`, a string file or
+// From returns a new `api.Runnable` from an `io.Reader`, a string file or
 // directory path, or the raw bytes of YAML content describing a scenario or
 // suite.
-func From(source interface{}) (*suite.Suite, error) {
+func From(source interface{}) (api.Runnable, error) {
 	switch src := source.(type) {
 	case io.Reader:
 		s, err := scenario.FromReader(src)
diff --git a/porcelain_test.go b/porcelain_test.go
index 3edac1a..ad451f0 100644
--- a/porcelain_test.go
+++ b/porcelain_test.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/gdt-dev/gdt"
 	"github.com/gdt-dev/gdt/api"
+	"github.com/gdt-dev/gdt/suite"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -39,27 +40,35 @@ func TestFromFileNotFound(t *testing.T) {
 	assert.True(os.IsNotExist(err))
 }
 
-func TestFromSuite(t *testing.T) {
+func TestFromSuitePath(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 
 	fp := filepath.Join("suite", "testdata", "exec")
-	suite, err := gdt.From(fp)
+	r, err := gdt.From(fp)
 	require.Nil(err)
-	require.NotNil(suite)
+	require.NotNil(r)
 
-	assert.Equal(fp, suite.Path)
-	assert.Len(suite.Scenarios, 2)
+	assert.IsType(r, &suite.Suite{})
+
+	s := r.(*suite.Suite)
+
+	assert.Equal(fp, s.Path)
+	assert.Len(s.Scenarios, 2)
 }
 
-func TestFromScenarioPath(t *testing.T) {
+func TestFromFilePath(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 
 	fp := filepath.Join("suite", "testdata", "exec", "ls.yaml")
-	s, err := gdt.From(fp)
+	r, err := gdt.From(fp)
 	require.Nil(err)
-	require.NotNil(s)
+	require.NotNil(r)
+
+	assert.IsType(r, &suite.Suite{})
+
+	s := r.(*suite.Suite)
 
 	assert.Equal(fp, s.Path)
 	assert.Len(s.Scenarios, 1)
@@ -74,18 +83,24 @@ func TestFromScenarioReader(t *testing.T) {
 	fp := filepath.Join("suite", "testdata", "exec", "ls.yaml")
 	f, err := os.Open(fp)
 	require.Nil(err)
-	suite, err := gdt.From(f)
-	assert.Nil(err)
-	assert.NotNil(suite)
+
+	r, err := gdt.From(f)
+	require.Nil(err)
+	require.NotNil(r)
+
+	assert.IsType(r, &suite.Suite{})
+
+	s := r.(*suite.Suite)
 
 	// The scenario's path isn't set because we didn't supply a filepath...
-	assert.Equal("", suite.Path)
-	assert.Len(suite.Scenarios, 1)
-	assert.Len(suite.Scenarios[0].Tests, 1)
+	assert.Equal("", s.Path)
+	assert.Len(s.Scenarios, 1)
+	assert.Len(s.Scenarios[0].Tests, 1)
 }
 
 func TestFromScenarioBytes(t *testing.T) {
 	assert := assert.New(t)
+	require := require.New(t)
 
 	raw := `name: foo
 description: simple foo test
@@ -93,14 +108,19 @@ tests:
  - exec: echo foo
 `
 	b := []byte(raw)
-	suite, err := gdt.From(b)
-	assert.Nil(err)
-	assert.NotNil(suite)
+
+	r, err := gdt.From(b)
+	require.Nil(err)
+	require.NotNil(r)
+
+	assert.IsType(r, &suite.Suite{})
+
+	s := r.(*suite.Suite)
 
 	// The scenario's path isn't set because we didn't supply a filepath...
-	assert.Equal("", suite.Path)
-	assert.Len(suite.Scenarios, 1)
-	assert.Len(suite.Scenarios[0].Tests, 1)
+	assert.Equal("", s.Path)
+	assert.Len(s.Scenarios, 1)
+	assert.Len(s.Scenarios[0].Tests, 1)
 }
 
 func TestRunExecSuite(t *testing.T) {
diff --git a/suite/from.go b/suite/from.go
index d094452..e92a78b 100644
--- a/suite/from.go
+++ b/suite/from.go
@@ -70,10 +70,9 @@ func FromScenario(s *scenario.Scenario) *Suite {
 		Path:        s.Path,
 		Name:        suiteNameFromScenarioPath(s.Path),
 		Description: s.Description,
-		// NOTE: require needs to be named to fixture?
-		Require:   s.Fixtures,
-		Defaults:  s.Defaults,
-		Scenarios: []*scenario.Scenario{s},
+		Fixtures:    s.Fixtures,
+		Defaults:    s.Defaults,
+		Scenarios:   []*scenario.Scenario{s},
 	}
 }
 
diff --git a/suite/suite.go b/suite/suite.go
index 38371c7..fcc3060 100644
--- a/suite/suite.go
+++ b/suite/suite.go
@@ -23,9 +23,9 @@ type Suite struct {
 	// During parsing, plugins are handed this raw data and asked to interpret
 	// it into known configuration values for that plugin.
 	Defaults map[string]interface{} `yaml:"defaults,omitempty"`
-	// Require specifies an ordered list of fixtures the test suite's test
-	// cases depends on.
-	Require []string `yaml:"require,omitempty"`
+	// Fixtures specifies an ordered list of fixtures the test suite's test
+	// cases depend on.
+	Fixtures []string `yaml:"fixtures,omitempty"`
 	// Scenarios is a collection of test scenarios in this test suite
 	Scenarios []*scenario.Scenario `yaml:"-"`
 }
@@ -61,10 +61,10 @@ func WithDefaults(defaults map[string]interface{}) SuiteModifier {
 	}
 }
 
-// WithRequires sets a test suite's Requires attribute
-func WithRequires(require []string) SuiteModifier {
+// WithFixtures sets a test suite's Fixtures attribute
+func WithFixtures(fixtures []string) SuiteModifier {
 	return func(s *Suite) {
-		s.Require = require
+		s.Fixtures = fixtures
 	}
 }
 

From 185e695ebaf4a10548e8f00badc53ddcfbb63e64 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Fri, 28 Jun 2024 08:13:18 -0400
Subject: [PATCH 23/31] Move testing plugins into internal/testutil

Moves the stub plugins into a dedicated package for each plugin inside
`internal/testutil/plugin`.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 internal/testutil/plugin/bar/plugin.go      | 101 +++++
 internal/testutil/plugin/failer/plugin.go   | 136 ++++++
 internal/testutil/plugin/foo/plugin.go      | 139 ++++++
 internal/testutil/plugin/priorrun/plugin.go | 124 ++++++
 scenario/parse_test.go                      | 101 +++--
 scenario/stub_plugins_test.go               | 446 --------------------
 6 files changed, 560 insertions(+), 487 deletions(-)
 create mode 100644 internal/testutil/plugin/bar/plugin.go
 create mode 100644 internal/testutil/plugin/failer/plugin.go
 create mode 100644 internal/testutil/plugin/foo/plugin.go
 create mode 100644 internal/testutil/plugin/priorrun/plugin.go
 delete mode 100644 scenario/stub_plugins_test.go

diff --git a/internal/testutil/plugin/bar/plugin.go b/internal/testutil/plugin/bar/plugin.go
new file mode 100644
index 0000000..bfcd43c
--- /dev/null
+++ b/internal/testutil/plugin/bar/plugin.go
@@ -0,0 +1,101 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package bar
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/gdt-dev/gdt/api"
+	"github.com/gdt-dev/gdt/plugin"
+	"github.com/samber/lo"
+	"gopkg.in/yaml.v3"
+)
+
+func init() {
+	plugin.Register(&Plugin{})
+}
+
+type Defaults struct {
+	Foo string `yaml:"bar"`
+}
+
+func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
+	return nil
+}
+
+type Spec struct {
+	api.Spec
+	Bar int `yaml:"bar"`
+}
+
+func (s *Spec) SetBase(b api.Spec) {
+	s.Spec = b
+}
+
+func (s *Spec) Base() *api.Spec {
+	return &s.Spec
+}
+
+func (s *Spec) Retry() *api.Retry {
+	return api.NoRetry
+}
+
+func (s *Spec) Timeout() *api.Timeout {
+	return nil
+}
+
+func (s *Spec) Eval(context.Context) (*api.Result, error) {
+	return api.NewResult(), nil
+}
+
+func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return api.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return api.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "bar":
+			if valNode.Kind != yaml.ScalarNode {
+				return api.ExpectedScalarAt(valNode)
+			}
+			if v, err := strconv.Atoi(valNode.Value); err != nil {
+				return api.ExpectedIntAt(valNode)
+			} else {
+				s.Bar = v
+			}
+		default:
+			if lo.Contains(api.BaseSpecFields, key) {
+				continue
+			}
+			return api.UnknownFieldAt(key, keyNode)
+		}
+	}
+	return nil
+}
+
+type Plugin struct{}
+
+func (p *Plugin) Info() api.PluginInfo {
+	return api.PluginInfo{
+		Name: "bar",
+	}
+}
+
+func (p *Plugin) Defaults() yaml.Unmarshaler {
+	return &Defaults{}
+}
+
+func (p *Plugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&Spec{}}
+}
diff --git a/internal/testutil/plugin/failer/plugin.go b/internal/testutil/plugin/failer/plugin.go
new file mode 100644
index 0000000..af42a9b
--- /dev/null
+++ b/internal/testutil/plugin/failer/plugin.go
@@ -0,0 +1,136 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package failer
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/gdt-dev/gdt/api"
+	gdtapi "github.com/gdt-dev/gdt/api"
+	"github.com/gdt-dev/gdt/plugin"
+	"github.com/samber/lo"
+	"gopkg.in/yaml.v3"
+)
+
+func init() {
+	plugin.Register(&Plugin{})
+}
+
+type InnerDefaults struct {
+	Fail bool `yaml:"fail,omitempty"`
+}
+
+type Defaults struct {
+	InnerDefaults
+}
+
+func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return api.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return api.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "fail":
+			if valNode.Kind != yaml.MappingNode {
+				return api.ExpectedMapAt(valNode)
+			}
+			inner := InnerDefaults{}
+			if err := valNode.Decode(&inner); err != nil {
+				return err
+			}
+			d.InnerDefaults = inner
+			// This is just for testing api when parsing defaults...
+			if d.Fail {
+				return fmt.Errorf("defaults parsing failed")
+			}
+		default:
+			continue
+		}
+	}
+	return nil
+}
+
+type Spec struct {
+	api.Spec
+	Fail bool `yaml:"fail"`
+}
+
+func (s *Spec) SetBase(b api.Spec) {
+	s.Spec = b
+}
+
+func (s *Spec) Base() *api.Spec {
+	return &s.Spec
+}
+
+func (s *Spec) Retry() *api.Retry {
+	return nil
+}
+
+func (s *Spec) Timeout() *api.Timeout {
+	return nil
+}
+
+func (s *Spec) Eval(context.Context) (*api.Result, error) {
+	return nil, fmt.Errorf("%w: Indy, bad dates!", gdtapi.RuntimeError)
+}
+
+func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return api.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return api.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "fail":
+			if valNode.Kind != yaml.ScalarNode {
+				return api.ExpectedScalarAt(valNode)
+			}
+			s.Fail, _ = strconv.ParseBool(valNode.Value)
+			if s.Fail {
+				return fmt.Errorf("Indy, bad parse!")
+			}
+		default:
+			if lo.Contains(api.BaseSpecFields, key) {
+				continue
+			}
+			return api.UnknownFieldAt(key, keyNode)
+		}
+	}
+	return nil
+}
+
+type Plugin struct{}
+
+func (p *Plugin) Info() api.PluginInfo {
+	return api.PluginInfo{
+		Name: "fail",
+	}
+}
+
+func (p *Plugin) Defaults() yaml.Unmarshaler {
+	return &Defaults{}
+}
+
+func (p *Plugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&Spec{}}
+}
diff --git a/internal/testutil/plugin/foo/plugin.go b/internal/testutil/plugin/foo/plugin.go
new file mode 100644
index 0000000..b62c5d5
--- /dev/null
+++ b/internal/testutil/plugin/foo/plugin.go
@@ -0,0 +1,139 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package foo
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gdt-dev/gdt/api"
+	"github.com/gdt-dev/gdt/debug"
+	"github.com/gdt-dev/gdt/plugin"
+	"github.com/samber/lo"
+	"gopkg.in/yaml.v3"
+)
+
+func init() {
+	plugin.Register(&Plugin{})
+}
+
+type InnerDefaults struct {
+	Bar string `yaml:"bar,omitempty"`
+}
+
+type Defaults struct {
+	InnerDefaults `yaml:",inline"`
+}
+
+func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return api.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return api.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "foo":
+			if valNode.Kind != yaml.MappingNode {
+				return api.ExpectedMapAt(valNode)
+			}
+			inner := InnerDefaults{}
+			if err := valNode.Decode(&inner); err != nil {
+				return err
+			}
+			d.InnerDefaults = inner
+		default:
+			continue
+		}
+	}
+	return nil
+}
+
+type Spec struct {
+	api.Spec
+	Foo string `yaml:"foo"`
+}
+
+func (s *Spec) SetBase(b api.Spec) {
+	s.Spec = b
+}
+
+func (s *Spec) Base() *api.Spec {
+	return &s.Spec
+}
+
+func (s *Spec) Retry() *api.Retry {
+	return nil
+}
+
+func (s *Spec) Timeout() *api.Timeout {
+	return nil
+}
+
+func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return api.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return api.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "foo":
+			if valNode.Kind != yaml.ScalarNode {
+				return api.ExpectedScalarAt(valNode)
+			}
+			s.Foo = valNode.Value
+		default:
+			if lo.Contains(api.BaseSpecFields, key) {
+				continue
+			}
+			return api.UnknownFieldAt(key, keyNode)
+		}
+	}
+	return nil
+}
+
+func (s *Spec) Eval(ctx context.Context) (*api.Result, error) {
+	fails := []error{}
+	debug.Println(ctx, "in %s Foo=%s", s.Title(), s.Foo)
+	// This is just a silly test to demonstrate how to write Eval() methods
+	// for plugin Spec specialization classes.
+	if s.Name == "bar" && s.Foo != "bar" {
+		fail := fmt.Errorf("expected s.Foo = 'bar', got %s", s.Foo)
+		fails = append(fails, fail)
+	} else if s.Name != "bar" && s.Foo != "baz" {
+		fail := fmt.Errorf("expected s.Foo = 'baz', got %s", s.Foo)
+		fails = append(fails, fail)
+	}
+	return api.NewResult(api.WithFailures(fails...)), nil
+}
+
+type Plugin struct{}
+
+func (p *Plugin) Info() api.PluginInfo {
+	return api.PluginInfo{
+		Name: "foo",
+	}
+}
+
+func (p *Plugin) Defaults() yaml.Unmarshaler {
+	return &Defaults{}
+}
+
+func (p *Plugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&Spec{}}
+}
diff --git a/internal/testutil/plugin/priorrun/plugin.go b/internal/testutil/plugin/priorrun/plugin.go
new file mode 100644
index 0000000..5d9aa47
--- /dev/null
+++ b/internal/testutil/plugin/priorrun/plugin.go
@@ -0,0 +1,124 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package priorrun
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gdt-dev/gdt/api"
+	gdtcontext "github.com/gdt-dev/gdt/context"
+	"github.com/gdt-dev/gdt/plugin"
+	"github.com/samber/lo"
+	"gopkg.in/yaml.v3"
+)
+
+func init() {
+	plugin.Register(&Plugin{})
+}
+
+const PriorRunDataKey = "priorrun"
+
+type Defaults struct{}
+
+func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
+	return nil
+}
+
+type Spec struct {
+	api.Spec
+	State string `yaml:"state"`
+	Prior string `yaml:"prior"`
+}
+
+func (s *Spec) SetBase(b api.Spec) {
+	s.Spec = b
+}
+
+func (s *Spec) Base() *api.Spec {
+	return &s.Spec
+}
+
+func (s *Spec) Retry() *api.Retry {
+	return nil
+}
+
+func (s *Spec) Timeout() *api.Timeout {
+	return nil
+}
+
+func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.MappingNode {
+		return api.ExpectedMapAt(node)
+	}
+	// maps/structs are stored in a top-level Node.Content field which is a
+	// concatenated slice of Node pointers in pairs of key/values.
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		if keyNode.Kind != yaml.ScalarNode {
+			return api.ExpectedScalarAt(keyNode)
+		}
+		key := keyNode.Value
+		valNode := node.Content[i+1]
+		switch key {
+		case "state":
+			if valNode.Kind != yaml.ScalarNode {
+				return api.ExpectedScalarAt(valNode)
+			}
+			s.State = valNode.Value
+		case "prior":
+			if valNode.Kind != yaml.ScalarNode {
+				return api.ExpectedScalarAt(valNode)
+			}
+			s.Prior = valNode.Value
+		default:
+			if lo.Contains(api.BaseSpecFields, key) {
+				continue
+			}
+			return api.UnknownFieldAt(key, keyNode)
+		}
+	}
+	return nil
+}
+
+func (s *Spec) Eval(ctx context.Context) (*api.Result, error) {
+	// Here we test that the prior run data that we save at the end of each
+	// Run() is showing up properly in the next Run()'s context.
+	fails := []error{}
+	prData := gdtcontext.PriorRun(ctx)
+	if s.Index == 0 {
+		if len(prData) != 0 {
+			fails = append(fails, fmt.Errorf("expected prData to be empty"))
+		}
+	} else {
+		data, ok := prData[PriorRunDataKey]
+		if !ok {
+			fails = append(fails, fmt.Errorf("expected PriorRunDataKey in priorRun map"))
+		}
+		if s.Prior != data {
+			fails = append(fails, fmt.Errorf("expected priorRunData == data"))
+		}
+	}
+	return api.NewResult(
+		api.WithFailures(fails...),
+		api.WithData(PriorRunDataKey, s.State),
+	), nil
+}
+
+type Plugin struct{}
+
+func (p *Plugin) Info() api.PluginInfo {
+	return api.PluginInfo{
+		Name: "priorRun",
+	}
+}
+
+func (p *Plugin) Defaults() yaml.Unmarshaler {
+	return &Defaults{}
+}
+
+func (p *Plugin) Specs() []api.Evaluable {
+	return []api.Evaluable{&Spec{}}
+}
diff --git a/scenario/parse_test.go b/scenario/parse_test.go
index 76de4e0..2cf041e 100644
--- a/scenario/parse_test.go
+++ b/scenario/parse_test.go
@@ -13,6 +13,11 @@ import (
 	"github.com/gdt-dev/gdt/scenario"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/gdt-dev/gdt/internal/testutil/plugin/bar"
+	"github.com/gdt-dev/gdt/internal/testutil/plugin/failer"
+	"github.com/gdt-dev/gdt/internal/testutil/plugin/foo"
+	"github.com/gdt-dev/gdt/internal/testutil/plugin/priorrun"
 )
 
 func TestFailingDefaults(t *testing.T) {
@@ -49,14 +54,16 @@ func TestNoTests(t *testing.T) {
 	assert.Equal([]string{"books_api", "books_data"}, s.Fixtures)
 	assert.Equal(
 		map[string]interface{}{
-			"foo": &fooDefaults{
-				fooInnerDefaults{
+			"foo": &foo.Defaults{
+				InnerDefaults: foo.InnerDefaults{
 					Bar: "barconfig",
 				},
 			},
-			"bar":                &barDefaults{},
-			"fail":               &failDefaults{failInnerDefaults{}},
-			"priorRun":           &priorRunDefaults{},
+			"bar": &bar.Defaults{},
+			"fail": &failer.Defaults{
+				InnerDefaults: failer.InnerDefaults{},
+			},
+			"priorRun":           &priorrun.Defaults{},
 			scenario.DefaultsKey: &scenario.Defaults{},
 		},
 		s.Defaults,
@@ -199,31 +206,35 @@ func TestKnownSpec(t *testing.T) {
 	assert.Empty(s.Fixtures)
 	assert.Equal(
 		map[string]interface{}{
-			"foo": &fooDefaults{
-				fooInnerDefaults{
+			"foo": &foo.Defaults{
+				InnerDefaults: foo.InnerDefaults{
 					Bar: "barconfig",
 				},
 			},
-			"bar":                &barDefaults{},
-			"fail":               &failDefaults{failInnerDefaults{}},
-			"priorRun":           &priorRunDefaults{},
+			"bar": &bar.Defaults{},
+			"fail": &failer.Defaults{
+				InnerDefaults: failer.InnerDefaults{},
+			},
+			"priorRun":           &priorrun.Defaults{},
 			scenario.DefaultsKey: &scenario.Defaults{},
 		},
 		s.Defaults,
 	)
 	expSpecDefaults := &api.Defaults{
-		"foo": &fooDefaults{
-			fooInnerDefaults{
+		"foo": &foo.Defaults{
+			InnerDefaults: foo.InnerDefaults{
 				Bar: "barconfig",
 			},
 		},
-		"bar":                &barDefaults{},
-		"fail":               &failDefaults{failInnerDefaults{}},
-		"priorRun":           &priorRunDefaults{},
+		"bar": &bar.Defaults{},
+		"fail": &failer.Defaults{
+			InnerDefaults: failer.InnerDefaults{},
+		},
+		"priorRun":           &priorrun.Defaults{},
 		scenario.DefaultsKey: &scenario.Defaults{},
 	}
 	expTests := []api.Evaluable{
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:    0,
 				Name:     "bar",
@@ -231,7 +242,7 @@ func TestKnownSpec(t *testing.T) {
 			},
 			Foo: "bar",
 		},
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:       1,
 				Description: "Bazzy Bizzy",
@@ -259,14 +270,14 @@ func TestMultipleSpec(t *testing.T) {
 	assert.Equal("foo-bar", s.Name)
 	assert.Equal(filepath.Join("testdata", "foo-bar.yaml"), s.Path)
 	expTests := []api.Evaluable{
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:    0,
 				Defaults: &api.Defaults{},
 			},
 			Foo: "bar",
 		},
-		&barSpec{
+		&bar.Spec{
 			Spec: api.Spec{
 				Index:    1,
 				Defaults: &api.Defaults{},
@@ -299,31 +310,35 @@ func TestEnvExpansion(t *testing.T) {
 	assert.Empty(s.Fixtures)
 	assert.Equal(
 		map[string]interface{}{
-			"foo": &fooDefaults{
-				fooInnerDefaults{
+			"foo": &foo.Defaults{
+				InnerDefaults: foo.InnerDefaults{
 					Bar: "barconfig",
 				},
 			},
-			"bar":                &barDefaults{},
-			"fail":               &failDefaults{failInnerDefaults{}},
-			"priorRun":           &priorRunDefaults{},
+			"bar": &bar.Defaults{},
+			"fail": &failer.Defaults{
+				InnerDefaults: failer.InnerDefaults{},
+			},
+			"priorRun":           &priorrun.Defaults{},
 			scenario.DefaultsKey: &scenario.Defaults{},
 		},
 		s.Defaults,
 	)
 	expSpecDefaults := &api.Defaults{
-		"foo": &fooDefaults{
-			fooInnerDefaults{
+		"foo": &foo.Defaults{
+			InnerDefaults: foo.InnerDefaults{
 				Bar: "barconfig",
 			},
 		},
-		"bar":                &barDefaults{},
-		"fail":               &failDefaults{failInnerDefaults{}},
-		"priorRun":           &priorRunDefaults{},
+		"bar": &bar.Defaults{},
+		"fail": &failer.Defaults{
+			InnerDefaults: failer.InnerDefaults{},
+		},
+		"priorRun":           &priorrun.Defaults{},
 		scenario.DefaultsKey: &scenario.Defaults{},
 	}
 	expTests := []api.Evaluable{
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:    0,
 				Name:     "$NOT_EXPANDED",
@@ -331,7 +346,7 @@ func TestEnvExpansion(t *testing.T) {
 			},
 			Foo: "bar",
 		},
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:       1,
 				Description: "Bazzy Bizzy",
@@ -361,10 +376,12 @@ func TestScenarioDefaults(t *testing.T) {
 	assert.Empty(s.Fixtures)
 	assert.Equal(
 		map[string]interface{}{
-			"foo":      &fooDefaults{},
-			"bar":      &barDefaults{},
-			"fail":     &failDefaults{failInnerDefaults{}},
-			"priorRun": &priorRunDefaults{},
+			"foo": &foo.Defaults{},
+			"bar": &bar.Defaults{},
+			"fail": &failer.Defaults{
+				InnerDefaults: failer.InnerDefaults{},
+			},
+			"priorRun": &priorrun.Defaults{},
 			scenario.DefaultsKey: &scenario.Defaults{
 				Timeout: &api.Timeout{
 					After: "2s",
@@ -374,10 +391,12 @@ func TestScenarioDefaults(t *testing.T) {
 		s.Defaults,
 	)
 	expSpecDefaults := &api.Defaults{
-		"foo":      &fooDefaults{},
-		"bar":      &barDefaults{},
-		"fail":     &failDefaults{failInnerDefaults{}},
-		"priorRun": &priorRunDefaults{},
+		"foo": &foo.Defaults{},
+		"bar": &bar.Defaults{},
+		"fail": &failer.Defaults{
+			InnerDefaults: failer.InnerDefaults{},
+		},
+		"priorRun": &priorrun.Defaults{},
 		scenario.DefaultsKey: &scenario.Defaults{
 			Timeout: &api.Timeout{
 				After: "2s",
@@ -385,7 +404,7 @@ func TestScenarioDefaults(t *testing.T) {
 		},
 	}
 	expTests := []api.Evaluable{
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:    0,
 				Defaults: expSpecDefaults,
@@ -395,7 +414,7 @@ func TestScenarioDefaults(t *testing.T) {
 			},
 			Foo: "baz",
 		},
-		&fooSpec{
+		&foo.Spec{
 			Spec: api.Spec{
 				Index:    1,
 				Defaults: expSpecDefaults,
diff --git a/scenario/stub_plugins_test.go b/scenario/stub_plugins_test.go
deleted file mode 100644
index d11954e..0000000
--- a/scenario/stub_plugins_test.go
+++ /dev/null
@@ -1,446 +0,0 @@
-// Use and distribution licensed under the Apache license version 2.
-//
-// See the COPYING file in the root project directory for full text.
-
-package scenario_test
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-
-	"github.com/gdt-dev/gdt/api"
-	gdtapi "github.com/gdt-dev/gdt/api"
-	gdtcontext "github.com/gdt-dev/gdt/context"
-	"github.com/gdt-dev/gdt/debug"
-	"github.com/gdt-dev/gdt/plugin"
-	"github.com/samber/lo"
-	"gopkg.in/yaml.v3"
-)
-
-func init() {
-	plugin.Register(&fooPlugin{})
-	plugin.Register(&barPlugin{})
-	plugin.Register(&failingPlugin{})
-	plugin.Register(&priorRunPlugin{})
-}
-
-type failInnerDefaults struct {
-	Fail bool `yaml:"fail,omitempty"`
-}
-
-type failDefaults struct {
-	failInnerDefaults
-}
-
-func (d *failDefaults) UnmarshalYAML(node *yaml.Node) error {
-	if node.Kind != yaml.MappingNode {
-		return api.ExpectedMapAt(node)
-	}
-	// maps/structs are stored in a top-level Node.Content field which is a
-	// concatenated slice of Node pointers in pairs of key/values.
-	for i := 0; i < len(node.Content); i += 2 {
-		keyNode := node.Content[i]
-		if keyNode.Kind != yaml.ScalarNode {
-			return api.ExpectedScalarAt(keyNode)
-		}
-		key := keyNode.Value
-		valNode := node.Content[i+1]
-		switch key {
-		case "fail":
-			if valNode.Kind != yaml.MappingNode {
-				return api.ExpectedMapAt(valNode)
-			}
-			inner := failInnerDefaults{}
-			if err := valNode.Decode(&inner); err != nil {
-				return err
-			}
-			d.failInnerDefaults = inner
-			// This is just for testing api when parsing defaults...
-			if d.Fail {
-				return fmt.Errorf("defaults parsing failed")
-			}
-		default:
-			continue
-		}
-	}
-	return nil
-}
-
-type failSpec struct {
-	api.Spec
-	Fail bool `yaml:"fail"`
-}
-
-func (s *failSpec) SetBase(b api.Spec) {
-	s.Spec = b
-}
-
-func (s *failSpec) Base() *api.Spec {
-	return &s.Spec
-}
-
-func (s *failSpec) Retry() *api.Retry {
-	return nil
-}
-
-func (s *failSpec) Timeout() *api.Timeout {
-	return nil
-}
-
-func (s *failSpec) Eval(context.Context) (*api.Result, error) {
-	return nil, fmt.Errorf("%w: Indy, bad dates!", gdtapi.RuntimeError)
-}
-
-func (s *failSpec) UnmarshalYAML(node *yaml.Node) error {
-	if node.Kind != yaml.MappingNode {
-		return api.ExpectedMapAt(node)
-	}
-	// maps/structs are stored in a top-level Node.Content field which is a
-	// concatenated slice of Node pointers in pairs of key/values.
-	for i := 0; i < len(node.Content); i += 2 {
-		keyNode := node.Content[i]
-		if keyNode.Kind != yaml.ScalarNode {
-			return api.ExpectedScalarAt(keyNode)
-		}
-		key := keyNode.Value
-		valNode := node.Content[i+1]
-		switch key {
-		case "fail":
-			if valNode.Kind != yaml.ScalarNode {
-				return api.ExpectedScalarAt(valNode)
-			}
-			s.Fail, _ = strconv.ParseBool(valNode.Value)
-			if s.Fail {
-				return fmt.Errorf("Indy, bad parse!")
-			}
-		default:
-			if lo.Contains(api.BaseSpecFields, key) {
-				continue
-			}
-			return api.UnknownFieldAt(key, keyNode)
-		}
-	}
-	return nil
-}
-
-type failingPlugin struct{}
-
-func (p *failingPlugin) Info() api.PluginInfo {
-	return api.PluginInfo{
-		Name: "fail",
-	}
-}
-
-func (p *failingPlugin) Defaults() yaml.Unmarshaler {
-	return &failDefaults{}
-}
-
-func (p *failingPlugin) Specs() []api.Evaluable {
-	return []api.Evaluable{&failSpec{}}
-}
-
-type fooInnerDefaults struct {
-	Bar string `yaml:"bar,omitempty"`
-}
-
-type fooDefaults struct {
-	fooInnerDefaults
-}
-
-func (d *fooDefaults) UnmarshalYAML(node *yaml.Node) error {
-	if node.Kind != yaml.MappingNode {
-		return api.ExpectedMapAt(node)
-	}
-	// maps/structs are stored in a top-level Node.Content field which is a
-	// concatenated slice of Node pointers in pairs of key/values.
-	for i := 0; i < len(node.Content); i += 2 {
-		keyNode := node.Content[i]
-		if keyNode.Kind != yaml.ScalarNode {
-			return api.ExpectedScalarAt(keyNode)
-		}
-		key := keyNode.Value
-		valNode := node.Content[i+1]
-		switch key {
-		case "foo":
-			if valNode.Kind != yaml.MappingNode {
-				return api.ExpectedMapAt(valNode)
-			}
-			inner := fooInnerDefaults{}
-			if err := valNode.Decode(&inner); err != nil {
-				return err
-			}
-			d.fooInnerDefaults = inner
-		default:
-			continue
-		}
-	}
-	return nil
-}
-
-type fooSpec struct {
-	api.Spec
-	Foo string `yaml:"foo"`
-}
-
-func (s *fooSpec) SetBase(b api.Spec) {
-	s.Spec = b
-}
-
-func (s *fooSpec) Base() *api.Spec {
-	return &s.Spec
-}
-
-func (s *fooSpec) Retry() *api.Retry {
-	return nil
-}
-
-func (s *fooSpec) Timeout() *api.Timeout {
-	return nil
-}
-
-func (s *fooSpec) UnmarshalYAML(node *yaml.Node) error {
-	if node.Kind != yaml.MappingNode {
-		return api.ExpectedMapAt(node)
-	}
-	// maps/structs are stored in a top-level Node.Content field which is a
-	// concatenated slice of Node pointers in pairs of key/values.
-	for i := 0; i < len(node.Content); i += 2 {
-		keyNode := node.Content[i]
-		if keyNode.Kind != yaml.ScalarNode {
-			return api.ExpectedScalarAt(keyNode)
-		}
-		key := keyNode.Value
-		valNode := node.Content[i+1]
-		switch key {
-		case "foo":
-			if valNode.Kind != yaml.ScalarNode {
-				return api.ExpectedScalarAt(valNode)
-			}
-			s.Foo = valNode.Value
-		default:
-			if lo.Contains(api.BaseSpecFields, key) {
-				continue
-			}
-			return api.UnknownFieldAt(key, keyNode)
-		}
-	}
-	return nil
-}
-
-func (s *fooSpec) Eval(ctx context.Context) (*api.Result, error) {
-	fails := []error{}
-	debug.Println(ctx, "in %s Foo=%s", s.Title(), s.Foo)
-	// This is just a silly test to demonstrate how to write Eval() methods
-	// for plugin Spec specialization classes.
-	if s.Name == "bar" && s.Foo != "bar" {
-		fail := fmt.Errorf("expected s.Foo = 'bar', got %s", s.Foo)
-		fails = append(fails, fail)
-	} else if s.Name != "bar" && s.Foo != "baz" {
-		fail := fmt.Errorf("expected s.Foo = 'baz', got %s", s.Foo)
-		fails = append(fails, fail)
-	}
-	return api.NewResult(api.WithFailures(fails...)), nil
-}
-
-type fooPlugin struct{}
-
-func (p *fooPlugin) Info() api.PluginInfo {
-	return api.PluginInfo{
-		Name: "foo",
-	}
-}
-
-func (p *fooPlugin) Defaults() yaml.Unmarshaler {
-	return &fooDefaults{}
-}
-
-func (p *fooPlugin) Specs() []api.Evaluable {
-	return []api.Evaluable{&fooSpec{}}
-}
-
-type barDefaults struct {
-	Foo string `yaml:"bar"`
-}
-
-func (d *barDefaults) UnmarshalYAML(node *yaml.Node) error {
-	return nil
-}
-
-type barSpec struct {
-	api.Spec
-	Bar int `yaml:"bar"`
-}
-
-func (s *barSpec) SetBase(b api.Spec) {
-	s.Spec = b
-}
-
-func (s *barSpec) Base() *api.Spec {
-	return &s.Spec
-}
-
-func (s *barSpec) Retry() *api.Retry {
-	return api.NoRetry
-}
-
-func (s *barSpec) Timeout() *api.Timeout {
-	return nil
-}
-
-func (s *barSpec) Eval(context.Context) (*api.Result, error) {
-	return api.NewResult(), nil
-}
-
-func (s *barSpec) UnmarshalYAML(node *yaml.Node) error {
-	if node.Kind != yaml.MappingNode {
-		return api.ExpectedMapAt(node)
-	}
-	// maps/structs are stored in a top-level Node.Content field which is a
-	// concatenated slice of Node pointers in pairs of key/values.
-	for i := 0; i < len(node.Content); i += 2 {
-		keyNode := node.Content[i]
-		if keyNode.Kind != yaml.ScalarNode {
-			return api.ExpectedScalarAt(keyNode)
-		}
-		key := keyNode.Value
-		valNode := node.Content[i+1]
-		switch key {
-		case "bar":
-			if valNode.Kind != yaml.ScalarNode {
-				return api.ExpectedScalarAt(valNode)
-			}
-			if v, err := strconv.Atoi(valNode.Value); err != nil {
-				return api.ExpectedIntAt(valNode)
-			} else {
-				s.Bar = v
-			}
-		default:
-			if lo.Contains(api.BaseSpecFields, key) {
-				continue
-			}
-			return api.UnknownFieldAt(key, keyNode)
-		}
-	}
-	return nil
-}
-
-type barPlugin struct{}
-
-func (p *barPlugin) Info() api.PluginInfo {
-	return api.PluginInfo{
-		Name: "bar",
-	}
-}
-
-func (p *barPlugin) Defaults() yaml.Unmarshaler {
-	return &barDefaults{}
-}
-
-func (p *barPlugin) Specs() []api.Evaluable {
-	return []api.Evaluable{&barSpec{}}
-}
-
-const priorRunDataKey = "priorrun"
-
-type priorRunDefaults struct{}
-
-func (d *priorRunDefaults) UnmarshalYAML(node *yaml.Node) error {
-	return nil
-}
-
-type priorRunSpec struct {
-	api.Spec
-	State string `yaml:"state"`
-	Prior string `yaml:"prior"`
-}
-
-func (s *priorRunSpec) SetBase(b api.Spec) {
-	s.Spec = b
-}
-
-func (s *priorRunSpec) Base() *api.Spec {
-	return &s.Spec
-}
-
-func (s *priorRunSpec) Retry() *api.Retry {
-	return nil
-}
-
-func (s *priorRunSpec) Timeout() *api.Timeout {
-	return nil
-}
-
-func (s *priorRunSpec) UnmarshalYAML(node *yaml.Node) error {
-	if node.Kind != yaml.MappingNode {
-		return api.ExpectedMapAt(node)
-	}
-	// maps/structs are stored in a top-level Node.Content field which is a
-	// concatenated slice of Node pointers in pairs of key/values.
-	for i := 0; i < len(node.Content); i += 2 {
-		keyNode := node.Content[i]
-		if keyNode.Kind != yaml.ScalarNode {
-			return api.ExpectedScalarAt(keyNode)
-		}
-		key := keyNode.Value
-		valNode := node.Content[i+1]
-		switch key {
-		case "state":
-			if valNode.Kind != yaml.ScalarNode {
-				return api.ExpectedScalarAt(valNode)
-			}
-			s.State = valNode.Value
-		case "prior":
-			if valNode.Kind != yaml.ScalarNode {
-				return api.ExpectedScalarAt(valNode)
-			}
-			s.Prior = valNode.Value
-		default:
-			if lo.Contains(api.BaseSpecFields, key) {
-				continue
-			}
-			return api.UnknownFieldAt(key, keyNode)
-		}
-	}
-	return nil
-}
-
-func (s *priorRunSpec) Eval(ctx context.Context) (*api.Result, error) {
-	// Here we test that the prior run data that we save at the end of each
-	// Run() is showing up properly in the next Run()'s context.
-	fails := []error{}
-	prData := gdtcontext.PriorRun(ctx)
-	if s.Index == 0 {
-		if len(prData) != 0 {
-			fails = append(fails, fmt.Errorf("expected prData to be empty"))
-		}
-	} else {
-		data, ok := prData[priorRunDataKey]
-		if !ok {
-			fails = append(fails, fmt.Errorf("expected priorRunDataKey in priorRun map"))
-		}
-		if s.Prior != data {
-			fails = append(fails, fmt.Errorf("expected priorRunData == data"))
-		}
-	}
-	return api.NewResult(
-		api.WithFailures(fails...),
-		api.WithData(priorRunDataKey, s.State),
-	), nil
-}
-
-type priorRunPlugin struct{}
-
-func (p *priorRunPlugin) Info() api.PluginInfo {
-	return api.PluginInfo{
-		Name: "priorRun",
-	}
-}
-
-func (p *priorRunPlugin) Defaults() yaml.Unmarshaler {
-	return &priorRunDefaults{}
-}
-
-func (p *priorRunPlugin) Specs() []api.Evaluable {
-	return []api.Evaluable{&priorRunSpec{}}
-}

From 64abc7aa1e5ace96d3f3339780bfdffc249e7a60 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Fri, 28 Jun 2024 08:19:52 -0400
Subject: [PATCH 24/31] move errStarterFixture into internal/testutil

Moves the errStarterFixture into its own package in
`internal/testutil/fixture/errstarter`

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 .../testutil/fixture/errstarter/fixture.go                    | 4 ++--
 scenario/run_test.go                                          | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)
 rename scenario/stub_fixtures_test.go => internal/testutil/fixture/errstarter/fixture.go (86%)

diff --git a/scenario/stub_fixtures_test.go b/internal/testutil/fixture/errstarter/fixture.go
similarity index 86%
rename from scenario/stub_fixtures_test.go
rename to internal/testutil/fixture/errstarter/fixture.go
index 4875b46..b38dd45 100644
--- a/scenario/stub_fixtures_test.go
+++ b/internal/testutil/fixture/errstarter/fixture.go
@@ -2,7 +2,7 @@
 //
 // See the COPYING file in the root project directory for full text.
 
-package scenario_test
+package errstarter
 
 import (
 	"context"
@@ -16,7 +16,7 @@ var (
 		return fmt.Errorf("error starting fixture!")
 	}
 
-	errStarterFixture = fixture.New(
+	Fixture = fixture.New(
 		fixture.WithStarter(errStarter),
 	)
 )
diff --git a/scenario/run_test.go b/scenario/run_test.go
index 8b951a9..ec75a7c 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -19,6 +19,8 @@ import (
 	"github.com/gdt-dev/gdt/scenario"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/gdt-dev/gdt/internal/testutil/fixture/errstarter"
 )
 
 var failFlag = flag.Bool("fail", false, "run tests expected to fail")
@@ -85,7 +87,7 @@ func TestFixtureStartError(t *testing.T) {
 	require.NotNil(s)
 
 	ctx := gdtcontext.New()
-	ctx = gdtcontext.RegisterFixture(ctx, "start-error", errStarterFixture)
+	ctx = gdtcontext.RegisterFixture(ctx, "start-error", errstarter.Fixture)
 
 	err = s.Run(ctx, t)
 	assert.NotNil(err)

From 8fe4561fe2273622379eb95b2fb3211c11ec6115 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Wed, 3 Jul 2024 08:06:53 -0400
Subject: [PATCH 25/31] remove Scenario.evalPlugins

Adds a new `Spec.Plugin` field and stores a pointer to the plugin that
successfully parsed the test spec.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 api/spec.go                            |  2 ++
 internal/testutil/plugin/bar/plugin.go |  7 +++++-
 internal/testutil/plugin/foo/plugin.go |  7 +++++-
 plugin/exec/parse_test.go              |  1 +
 plugin/exec/plugin.go                  | 13 +++++++---
 scenario/parse.go                      |  5 +---
 scenario/parse_test.go                 |  8 ++++++
 scenario/run.go                        | 35 +++++++++++++++-----------
 8 files changed, 54 insertions(+), 24 deletions(-)

diff --git a/api/spec.go b/api/spec.go
index 9a2bdc0..8d52df3 100644
--- a/api/spec.go
+++ b/api/spec.go
@@ -28,6 +28,8 @@ var (
 // output or behaviour. All gdt plugins have their own Spec structs that
 // inherit from this base struct.
 type Spec struct {
+	// Plugin is a pointer to the plugin that successfully parsed the test spec
+	Plugin Plugin `yaml:"-"`
 	// Defaults contains the parsed defaults for the Spec. These are injected
 	// by the scenario during parse.
 	Defaults *Defaults `yaml:"-"`
diff --git a/internal/testutil/plugin/bar/plugin.go b/internal/testutil/plugin/bar/plugin.go
index bfcd43c..9f334a1 100644
--- a/internal/testutil/plugin/bar/plugin.go
+++ b/internal/testutil/plugin/bar/plugin.go
@@ -14,8 +14,13 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+var (
+	// this is just for testing purposes...
+	PluginRef = &Plugin{}
+)
+
 func init() {
-	plugin.Register(&Plugin{})
+	plugin.Register(PluginRef)
 }
 
 type Defaults struct {
diff --git a/internal/testutil/plugin/foo/plugin.go b/internal/testutil/plugin/foo/plugin.go
index b62c5d5..180fd9c 100644
--- a/internal/testutil/plugin/foo/plugin.go
+++ b/internal/testutil/plugin/foo/plugin.go
@@ -15,8 +15,13 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+var (
+	// this is just for testing purposes...
+	PluginRef = &Plugin{}
+)
+
 func init() {
-	plugin.Register(&Plugin{})
+	plugin.Register(PluginRef)
 }
 
 type InnerDefaults struct {
diff --git a/plugin/exec/parse_test.go b/plugin/exec/parse_test.go
index c79b419..fa739bb 100644
--- a/plugin/exec/parse_test.go
+++ b/plugin/exec/parse_test.go
@@ -53,6 +53,7 @@ func TestSimpleCommand(t *testing.T) {
 	expTests := []api.Evaluable{
 		&gdtexec.Spec{
 			Spec: api.Spec{
+				Plugin:   gdtexec.PluginRef,
 				Index:    0,
 				Defaults: &api.Defaults{},
 			},
diff --git a/plugin/exec/plugin.go b/plugin/exec/plugin.go
index 87fac20..3a7d03a 100644
--- a/plugin/exec/plugin.go
+++ b/plugin/exec/plugin.go
@@ -11,6 +11,15 @@ import (
 	gdtplugin "github.com/gdt-dev/gdt/plugin"
 )
 
+var (
+	// this is just for testing purposes...
+	PluginRef = &plugin{}
+)
+
+func init() {
+	gdtplugin.Register(PluginRef)
+}
+
 var (
 	DefaultTimeout = "10s"
 )
@@ -20,10 +29,6 @@ func OverrideDefaultTimeout(d string) {
 	DefaultTimeout = d
 }
 
-func init() {
-	gdtplugin.Register(Plugin())
-}
-
 const (
 	pluginName = "exec"
 )
diff --git a/scenario/parse.go b/scenario/parse.go
index 4f34698..8a1e486 100644
--- a/scenario/parse.go
+++ b/scenario/parse.go
@@ -82,8 +82,6 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 			s.Defaults = defaults
 		}
 	}
-	// We store a lookup to the parsing plugin for each parsed test spec
-	evalPlugins := map[int]api.Plugin{}
 	for i := 0; i < len(node.Content); i += 2 {
 		keyNode := node.Content[i]
 		if keyNode.Kind != yaml.ScalarNode {
@@ -116,10 +114,10 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 							}
 							return err
 						}
+						base.Plugin = plugin
 						sp.SetBase(base)
 						s.Tests = append(s.Tests, sp)
 						parsed = true
-						evalPlugins[idx] = plugin
 						break
 					}
 				}
@@ -161,6 +159,5 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 			}
 		}
 	}
-	s.evalPlugins = evalPlugins
 	return nil
 }
diff --git a/scenario/parse_test.go b/scenario/parse_test.go
index 2cf041e..493fc80 100644
--- a/scenario/parse_test.go
+++ b/scenario/parse_test.go
@@ -236,6 +236,7 @@ func TestKnownSpec(t *testing.T) {
 	expTests := []api.Evaluable{
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:   foo.PluginRef,
 				Index:    0,
 				Name:     "bar",
 				Defaults: expSpecDefaults,
@@ -244,6 +245,7 @@ func TestKnownSpec(t *testing.T) {
 		},
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:      foo.PluginRef,
 				Index:       1,
 				Description: "Bazzy Bizzy",
 				Defaults:    expSpecDefaults,
@@ -272,6 +274,7 @@ func TestMultipleSpec(t *testing.T) {
 	expTests := []api.Evaluable{
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:   foo.PluginRef,
 				Index:    0,
 				Defaults: &api.Defaults{},
 			},
@@ -279,6 +282,7 @@ func TestMultipleSpec(t *testing.T) {
 		},
 		&bar.Spec{
 			Spec: api.Spec{
+				Plugin:   bar.PluginRef,
 				Index:    1,
 				Defaults: &api.Defaults{},
 			},
@@ -340,6 +344,7 @@ func TestEnvExpansion(t *testing.T) {
 	expTests := []api.Evaluable{
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:   foo.PluginRef,
 				Index:    0,
 				Name:     "$NOT_EXPANDED",
 				Defaults: expSpecDefaults,
@@ -348,6 +353,7 @@ func TestEnvExpansion(t *testing.T) {
 		},
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:      foo.PluginRef,
 				Index:       1,
 				Description: "Bazzy Bizzy",
 				Defaults:    expSpecDefaults,
@@ -406,6 +412,7 @@ func TestScenarioDefaults(t *testing.T) {
 	expTests := []api.Evaluable{
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:   foo.PluginRef,
 				Index:    0,
 				Defaults: expSpecDefaults,
 				Timeout: &api.Timeout{
@@ -416,6 +423,7 @@ func TestScenarioDefaults(t *testing.T) {
 		},
 		&foo.Spec{
 			Spec: api.Spec{
+				Plugin:   foo.PluginRef,
 				Index:    1,
 				Defaults: expSpecDefaults,
 			},
diff --git a/scenario/run.go b/scenario/run.go
index 5b38588..d843f75 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -41,12 +41,6 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			defer fix.Stop(ctx)
 		}
 	}
-	var rterr error
-	var scDefaults *Defaults
-	scDefaultsAny, found := s.Defaults[DefaultsKey]
-	if found {
-		scDefaults = scDefaultsAny.(*Defaults)
-	}
 	// If the test author has specified any pre-flight checks in the `skip-if`
 	// collection, evaluate those first and if any failed, skip the scenario's
 	// tests.
@@ -63,6 +57,10 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			return nil
 		}
 	}
+
+	var err error
+	scDefaults := s.getScenarioDefaults()
+
 	t.Run(s.Title(), func(t *testing.T) {
 		ctx = gdtcontext.PushTrace(ctx, s.Title())
 		defer func() {
@@ -86,7 +84,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 				specCtx = gdtcontext.PopTrace(specCtx)
 			}
 
-			plugin := s.evalPlugins[idx]
+			plugin := sb.Plugin
 
 			rt := getRetry(specCtx, scDefaults, plugin, spec)
 
@@ -109,15 +107,14 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 
 			select {
 			case <-specCtx.Done():
-				t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
 				popTracer()
-				specCancel()
+				t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
 				break
 			case runres := <-ch:
 				res = runres.r
-				rterr = runres.err
+				err = runres.err
 			}
-			if rterr != nil {
+			if err != nil {
 				popTracer()
 				specCancel()
 				break
@@ -134,14 +131,14 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			if res.HasData() {
 				ctx = gdtcontext.StorePriorRun(ctx, res.Data())
 			}
+			popTracer()
+			specCancel()
 			for _, fail := range res.Failures() {
 				t.Fatal(fail)
 			}
-			popTracer()
-			specCancel()
 		}
 	})
-	return rterr
+	return err
 }
 
 type runSpecRes struct {
@@ -375,3 +372,13 @@ func getRetry(
 	}
 	return nil
 }
+
+// getScenarioDefaults returns the Defaults parsed from the scenario's YAML
+// file's `defaults` field, or nil if none were specified.
+func (s *Scenario) getScenarioDefaults() *Defaults {
+	scDefaultsAny, found := s.Defaults[DefaultsKey]
+	if found {
+		return scDefaultsAny.(*Defaults)
+	}
+	return nil
+}

From 9eed4db6ec3535583515e4ccb5c27fff8c81ac45 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Sat, 6 Jul 2024 09:40:43 -0400
Subject: [PATCH 26/31] check go test -timeout before running scenario

Adds code that totals up wait times and max timeout values and checks
that the `go test -timeout` value (which defaults to 10m) is smaller
than either the total wait time or largest timeout and returns a
RuntimeError if so, allowing the user to adjust their `go test -timeout`
value or scenario/spec timeouts/waits.

Issue #45

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 .github/workflows/test.yml                    |  2 +-
 Makefile                                      |  2 +-
 api/error.go                                  | 37 ++++++++++
 api/timings.go                                | 71 +++++++++++++++++++
 scenario/defaults.go                          | 18 +++--
 scenario/parse.go                             | 29 +++++++-
 scenario/run.go                               | 36 ++++++++--
 scenario/run_test.go                          | 54 ++++++++++++++
 scenario/scenario.go                          |  7 +-
 scenario/testdata/fixture-start-error.yaml    |  1 -
 .../timeout-conflict-default-timeout.yaml     |  6 ++
 .../timeout-conflict-spec-timeout.yaml        |  5 ++
 .../testdata/timeout-conflict-total-wait.yaml |  6 ++
 13 files changed, 257 insertions(+), 17 deletions(-)
 create mode 100644 api/timings.go
 create mode 100644 scenario/testdata/timeout-conflict-default-timeout.yaml
 create mode 100644 scenario/testdata/timeout-conflict-spec-timeout.yaml
 create mode 100644 scenario/testdata/timeout-conflict-total-wait.yaml

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b984717..a727f30 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,4 +36,4 @@ jobs:
        with:
          go-version: ${{ matrix.go }}
          check-latest: true
-     - run: go test -v ./...
+     - run: go test -timeout 10s -v ./...
diff --git a/Makefile b/Makefile
index 266aaf1..6dd033b 100644
--- a/Makefile
+++ b/Makefile
@@ -3,4 +3,4 @@ VERSION ?= $(shell git describe --tags --always --dirty)
 .PHONY: test
 
 test:
-	go test -v ./...
+	go test -timeout 10s -v ./...
diff --git a/api/error.go b/api/error.go
index 4a1c7c0..bbd4008 100644
--- a/api/error.go
+++ b/api/error.go
@@ -7,6 +7,7 @@ package api
 import (
 	"errors"
 	"fmt"
+	"time"
 
 	"gopkg.in/yaml.v3"
 )
@@ -294,6 +295,12 @@ var (
 		"%w: required fixture missing",
 		RuntimeError,
 	)
+	// ErrTimeoutConflict is returned when the Go test tool's timeout conflicts
+	// with either a total wait time or a timeout in a scenario or test spec
+	ErrTimeoutConflict = fmt.Errorf(
+		"%w: timeout conflict",
+		RuntimeError,
+	)
 )
 
 // RequiredFixtureMissing returns an ErrRequiredFixture with the supplied
@@ -301,3 +308,33 @@ var (
 func RequiredFixtureMissing(name string) error {
 	return fmt.Errorf("%w: %s", ErrRequiredFixture, name)
 }
+
+// TimeoutConflict returns an ErrTimeoutConflict describing how the Go test
+// tool's timeout conflicts with either a total wait time or a timeout value
+// from a scenario or spec.
+func TimeoutConflict(
+	gotestDeadline time.Duration,
+	totalWait time.Duration,
+	maxTimeout time.Duration,
+) error {
+	msg := fmt.Sprintf(
+		"go test -timeout value of %s ",
+		(gotestDeadline + time.Second).Round(time.Second),
+	)
+	if totalWait > 0 {
+		msg += fmt.Sprintf(
+			"is shorter than the total wait time in the scenario: %s. "+
+				"either decrease the wait times or increase the "+
+				"go test -timeout value.",
+			totalWait.Round(time.Second),
+		)
+	} else {
+		msg += fmt.Sprintf(
+			"is shorter than the maximum timeout specified in the "+
+				"scenario: %s. either decrease the scenario or spec "+
+				"timeout or increase the go test -timeout value.",
+			maxTimeout.Round(time.Second),
+		)
+	}
+	return fmt.Errorf("%w: %s", ErrTimeoutConflict, msg)
+}
diff --git a/api/timings.go b/api/timings.go
new file mode 100644
index 0000000..cc10a95
--- /dev/null
+++ b/api/timings.go
@@ -0,0 +1,71 @@
+// Use and distribution licensed under the Apache license version 2.
+//
+// See the COPYING file in the root project directory for full text.
+
+package api
+
+import (
+	"time"
+)
+
+type SetOn int
+
+const (
+	SetOnNone          SetOn = iota
+	SetOnSpec                // a test spec override
+	SetOnPlugin              // a plugin override
+	SetOnPluginDefault       // a plugin default
+	SetOnDefault             // a scenario default
+)
+
+// Timings contains information about a test scenario's maximum wait and
+// timeout duration and what aspect of the scenario (the scenario defaults, a
+// plugin default, a test spec override, etc) had the maximum timeout or wait
+// value.
+//
+// We use this information when initially assessing whether the Go test tool's
+// overall timeout is shorter than this maximum in order to inform the user to
+// increase the Go test tool timeout.
+type Timings struct {
+	// GoTestTimeout will be the duration of the timeout specified (or
+	// defaulted) by the Go test tool
+	GoTestTimeout time.Duration
+	// TotalWait will be non-zero when there is a wait specified for either the
+	// scenario or a test spec and will contain the aggregate duration of all
+	// waits
+	TotalWait time.Duration
+	// MaxTimeout will be non-zero when there is a timeout specified for either
+	// the scenario or a test spec and will contain the duration of the maximum
+	// timeout
+	MaxTimeout time.Duration
+	//TimeoutSetOn indicates where the MaxTimeout value was found
+	MaxTimeoutSetOn SetOn
+	// TimeoutSpecIndex indicates the test spec's index within the scenario where
+	// the max timeout was found
+	MaxTimeoutSpecIndex int
+}
+
+// AddWait adds a wait duration to the Timings and (re)-calculates the Timings'
+// MaxWait attributes
+func (t *Timings) AddWait(
+	d time.Duration,
+) {
+	t.TotalWait += d
+}
+
+// AddTimeout adds a timeout duration to the Timings and (re)-calculates the
+// Timings' MaxTimeout attributes
+func (t *Timings) AddTimeout(
+	d time.Duration,
+	on SetOn,
+	specIndex int,
+) {
+	if d == 0 {
+		return
+	}
+	if d.Abs() > t.MaxTimeout {
+		t.MaxTimeout = d
+		t.MaxTimeoutSetOn = on
+		t.MaxTimeoutSpecIndex = specIndex
+	}
+}
diff --git a/scenario/defaults.go b/scenario/defaults.go
index 95bb250..803055a 100644
--- a/scenario/defaults.go
+++ b/scenario/defaults.go
@@ -45,12 +45,20 @@ func (d *Defaults) UnmarshalYAML(node *yaml.Node) error {
 		valNode := node.Content[i+1]
 		switch key {
 		case "timeout":
-			if valNode.Kind != yaml.MappingNode {
-				return api.ExpectedMapAt(valNode)
-			}
 			var to *api.Timeout
-			if err := valNode.Decode(&to); err != nil {
-				return api.ExpectedTimeoutAt(valNode)
+			switch valNode.Kind {
+			case yaml.MappingNode:
+				// We support the old-style timeout:after
+				if err := valNode.Decode(&to); err != nil {
+					return api.ExpectedTimeoutAt(valNode)
+				}
+			case yaml.ScalarNode:
+				// We also support a straight string duration
+				to = &api.Timeout{
+					After: valNode.Value,
+				}
+			default:
+				return api.ExpectedScalarOrMapAt(valNode)
 			}
 			_, err := time.ParseDuration(to.After)
 			if err != nil {
diff --git a/scenario/parse.go b/scenario/parse.go
index 8a1e486..92b7c07 100644
--- a/scenario/parse.go
+++ b/scenario/parse.go
@@ -19,6 +19,7 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 	if node.Kind != yaml.MappingNode {
 		return api.ExpectedMapAt(node)
 	}
+	s.Timings = &api.Timings{}
 	plugins := plugin.Registered()
 	defaults := api.Defaults{}
 	// maps/structs are stored in a top-level Node.Content field which is a
@@ -78,6 +79,13 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 			if err := valNode.Decode(&scenDefaults); err != nil {
 				return err
 			}
+			if scenDefaults.Timeout != nil {
+				s.Timings.AddTimeout(
+					scenDefaults.Timeout.Duration(),
+					api.SetOnDefault,
+					-1,
+				)
+			}
 			defaults[DefaultsKey] = &scenDefaults
 			s.Defaults = defaults
 		}
@@ -107,7 +115,7 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 					pluginSpecs[p] = p.Specs()
 				}
 				for plugin, specs := range pluginSpecs {
-					for _, sp := range specs {
+					for idx, sp := range specs {
 						if err := testNode.Decode(sp); err != nil {
 							if errors.Is(err, api.ErrUnknownField) {
 								continue
@@ -115,6 +123,25 @@ func (s *Scenario) UnmarshalYAML(node *yaml.Node) error {
 							return err
 						}
 						base.Plugin = plugin
+						if base.Wait != nil {
+							if base.Wait.Before != "" {
+								s.Timings.AddWait(
+									base.Wait.BeforeDuration(),
+								)
+							}
+							if base.Wait.After != "" {
+								s.Timings.AddWait(
+									base.Wait.AfterDuration(),
+								)
+							}
+						}
+						if base.Timeout != nil {
+							s.Timings.AddTimeout(
+								base.Timeout.Duration(),
+								api.SetOnSpec,
+								idx,
+							)
+						}
 						sp.SetBase(base)
 						s.Tests = append(s.Tests, sp)
 						parsed = true
diff --git a/scenario/run.go b/scenario/run.go
index d843f75..89cfa87 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -27,6 +27,38 @@ import (
 // which will mark the test units failed or skipped if a test unit evaluates to
 // false.
 func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
+	ctx = gdtcontext.PushTrace(ctx, s.Title())
+	defer func() {
+		ctx = gdtcontext.PopTrace(ctx)
+	}()
+	now := time.Now()
+	d, ok := t.Deadline()
+	if ok && !d.IsZero() {
+		s.Timings.GoTestTimeout = d.Sub(now)
+		debug.Println(
+			ctx, "scenario/run: go test tool deadline: %s",
+			(s.Timings.GoTestTimeout + time.Second).Round(time.Second),
+		)
+		if s.Timings.TotalWait > 0 {
+			if s.Timings.TotalWait.Abs() > s.Timings.GoTestTimeout.Abs() {
+				return api.TimeoutConflict(
+					s.Timings.GoTestTimeout,
+					s.Timings.TotalWait,
+					s.Timings.MaxTimeout,
+				)
+			}
+		}
+		if s.Timings.MaxTimeout > 0 {
+			if s.Timings.MaxTimeout.Abs() > s.Timings.GoTestTimeout.Abs() {
+				return api.TimeoutConflict(
+					s.Timings.GoTestTimeout,
+					s.Timings.TotalWait,
+					s.Timings.MaxTimeout,
+				)
+			}
+		}
+	}
+
 	if len(s.Fixtures) > 0 {
 		fixtures := gdtcontext.Fixtures(ctx)
 		for _, fname := range s.Fixtures {
@@ -62,10 +94,6 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 	scDefaults := s.getScenarioDefaults()
 
 	t.Run(s.Title(), func(t *testing.T) {
-		ctx = gdtcontext.PushTrace(ctx, s.Title())
-		defer func() {
-			ctx = gdtcontext.PopTrace(ctx)
-		}()
 		for idx, spec := range s.Tests {
 			sb := spec.Base()
 
diff --git a/scenario/run_test.go b/scenario/run_test.go
index ec75a7c..9b8ed35 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -74,6 +74,60 @@ func TestMissingFixtures(t *testing.T) {
 	assert.ErrorIs(err, api.RuntimeError)
 }
 
+func TestTimeoutConflictTotalWait(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	fp := filepath.Join("testdata", "timeout-conflict-total-wait.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	require.Nil(err)
+	require.NotNil(s)
+
+	err = s.Run(context.TODO(), t)
+	assert.NotNil(err)
+	assert.ErrorIs(err, api.ErrTimeoutConflict)
+	assert.ErrorIs(err, api.RuntimeError)
+}
+
+func TestTimeoutConflictSpecTimeout(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	fp := filepath.Join("testdata", "timeout-conflict-spec-timeout.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	require.Nil(err)
+	require.NotNil(s)
+
+	err = s.Run(context.TODO(), t)
+	assert.NotNil(err)
+	assert.ErrorIs(err, api.ErrTimeoutConflict)
+	assert.ErrorIs(err, api.RuntimeError)
+}
+
+func TestTimeoutConflictDefaultTimeout(t *testing.T) {
+	require := require.New(t)
+	assert := assert.New(t)
+
+	fp := filepath.Join("testdata", "timeout-conflict-default-timeout.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(f, scenario.WithPath(fp))
+	require.Nil(err)
+	require.NotNil(s)
+
+	err = s.Run(context.TODO(), t)
+	assert.NotNil(err)
+	assert.ErrorIs(err, api.ErrTimeoutConflict)
+	assert.ErrorIs(err, api.RuntimeError)
+}
+
 func TestFixtureStartError(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)
diff --git a/scenario/scenario.go b/scenario/scenario.go
index c12b337..fd079bf 100644
--- a/scenario/scenario.go
+++ b/scenario/scenario.go
@@ -13,10 +13,9 @@ import (
 // Scenario is a generalized gdt test case file. It contains a set of Runnable
 // test units.
 type Scenario struct {
-	// evalPlugins stores the plugin that will evaluate the test spec at a
-	// particular index
-	evalPlugins map[int]api.Plugin
-	// Path is the filepath to the test case.
+	// Timings is the collection of max wait/timeout values for the scenario.
+	Timings *api.Timings `yaml:"-"`
+	// Path is the filepath to the test scenario YAML file.
 	Path string `yaml:"-"`
 	// Name is the short name for the test case. If empty, defaults to the base
 	// filename in Path.
diff --git a/scenario/testdata/fixture-start-error.yaml b/scenario/testdata/fixture-start-error.yaml
index 2c625a0..b9c023f 100644
--- a/scenario/testdata/fixture-start-error.yaml
+++ b/scenario/testdata/fixture-start-error.yaml
@@ -4,4 +4,3 @@ fixtures:
   - start-error
 tests:
   - foo: bar
-
diff --git a/scenario/testdata/timeout-conflict-default-timeout.yaml b/scenario/testdata/timeout-conflict-default-timeout.yaml
new file mode 100644
index 0000000..499ce6e
--- /dev/null
+++ b/scenario/testdata/timeout-conflict-default-timeout.yaml
@@ -0,0 +1,6 @@
+name: timeout-conflict-default-timeout
+description: a scenario with a default timeout greater than the go test tool timeout
+defaults:
+  timeout: 1m
+tests:
+  - foo: baz
diff --git a/scenario/testdata/timeout-conflict-spec-timeout.yaml b/scenario/testdata/timeout-conflict-spec-timeout.yaml
new file mode 100644
index 0000000..4b20977
--- /dev/null
+++ b/scenario/testdata/timeout-conflict-spec-timeout.yaml
@@ -0,0 +1,5 @@
+name: timeout-conflict-spec-timeout
+description: a scenario with a spec timeout greater than the go test tool timeout
+tests:
+  - timeout: 1m
+    foo: baz
diff --git a/scenario/testdata/timeout-conflict-total-wait.yaml b/scenario/testdata/timeout-conflict-total-wait.yaml
new file mode 100644
index 0000000..f43da4a
--- /dev/null
+++ b/scenario/testdata/timeout-conflict-total-wait.yaml
@@ -0,0 +1,6 @@
+name: timeout-conflict-total-wait
+description: a scenario with total wait times greater than the go test tool timeout
+tests:
+  - wait:
+      before: 1m
+    foo: baz

From a916cff197b05934a2efc1e4e5a36d701c63edc1 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Mon, 8 Jul 2024 07:58:19 -0400
Subject: [PATCH 27/31] break run spec and exec spec into separate methods

Cleans up the (large) Scenario.Run and runSpec methods by separating the
"preparation" phase of the test spec execution from the execution itself
into a different method. This allows us to clean up the manual `defer`s
and make the code more readable.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 api/error.go         |   7 +-
 scenario/run.go      | 231 +++++++++++++++++++++++--------------------
 scenario/run_test.go |   6 +-
 3 files changed, 130 insertions(+), 114 deletions(-)

diff --git a/api/error.go b/api/error.go
index bbd4008..07875c9 100644
--- a/api/error.go
+++ b/api/error.go
@@ -313,10 +313,11 @@ func RequiredFixtureMissing(name string) error {
 // tool's timeout conflicts with either a total wait time or a timeout value
 // from a scenario or spec.
 func TimeoutConflict(
-	gotestDeadline time.Duration,
-	totalWait time.Duration,
-	maxTimeout time.Duration,
+	ti *Timings,
 ) error {
+	gotestDeadline := ti.GoTestTimeout
+	totalWait := ti.TotalWait
+	maxTimeout := ti.MaxTimeout
 	msg := fmt.Sprintf(
 		"go test -timeout value of %s ",
 		(gotestDeadline + time.Second).Round(time.Second),
diff --git a/scenario/run.go b/scenario/run.go
index 89cfa87..cc2514b 100644
--- a/scenario/run.go
+++ b/scenario/run.go
@@ -31,32 +31,9 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 	defer func() {
 		ctx = gdtcontext.PopTrace(ctx)
 	}()
-	now := time.Now()
-	d, ok := t.Deadline()
-	if ok && !d.IsZero() {
-		s.Timings.GoTestTimeout = d.Sub(now)
-		debug.Println(
-			ctx, "scenario/run: go test tool deadline: %s",
-			(s.Timings.GoTestTimeout + time.Second).Round(time.Second),
-		)
-		if s.Timings.TotalWait > 0 {
-			if s.Timings.TotalWait.Abs() > s.Timings.GoTestTimeout.Abs() {
-				return api.TimeoutConflict(
-					s.Timings.GoTestTimeout,
-					s.Timings.TotalWait,
-					s.Timings.MaxTimeout,
-				)
-			}
-		}
-		if s.Timings.MaxTimeout > 0 {
-			if s.Timings.MaxTimeout.Abs() > s.Timings.GoTestTimeout.Abs() {
-				return api.TimeoutConflict(
-					s.Timings.GoTestTimeout,
-					s.Timings.TotalWait,
-					s.Timings.MaxTimeout,
-				)
-			}
-		}
+
+	if s.hasTimeoutConflict(ctx, t) {
+		return api.TimeoutConflict(s.Timings)
 	}
 
 	if len(s.Fixtures) > 0 {
@@ -73,6 +50,7 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 			defer fix.Stop(ctx)
 		}
 	}
+
 	// If the test author has specified any pre-flight checks in the `skip-if`
 	// collection, evaluate those first and if any failed, skip the scenario's
 	// tests.
@@ -90,79 +68,25 @@ func (s *Scenario) Run(ctx context.Context, t *testing.T) error {
 		}
 	}
 
+	var res *api.Result
 	var err error
-	scDefaults := s.getScenarioDefaults()
-
-	t.Run(s.Title(), func(t *testing.T) {
-		for idx, spec := range s.Tests {
-			sb := spec.Base()
 
-			// Create a brand new context that inherits the top-level context's
-			// cancel func. We want to set deadlines for each test spec and if
-			// we mutate the single supplied top-level context, then only the
-			// first deadline/timeout will be used.
-			specCtx, specCancel := context.WithCancel(ctx)
-
-			specTraceMsg := strconv.Itoa(idx)
-			if sb.Name != "" {
-				specTraceMsg += ":" + sb.Name
-			}
-			specCtx = gdtcontext.PushTrace(specCtx, specTraceMsg)
-			popTracer := func() {
-				specCtx = gdtcontext.PopTrace(specCtx)
-			}
-
-			plugin := sb.Plugin
-
-			rt := getRetry(specCtx, scDefaults, plugin, spec)
-
-			to := getTimeout(specCtx, scDefaults, plugin, spec)
-
-			var res *api.Result
-			ch := make(chan runSpecRes, 1)
-
-			wait := sb.Wait
-			if wait != nil && wait.Before != "" {
-				debug.Println(specCtx, "wait: %s before", wait.Before)
-				time.Sleep(wait.BeforeDuration())
-			}
-
-			if to != nil {
-				specCtx, specCancel = context.WithTimeout(specCtx, to.Duration())
-			}
-
-			go s.runSpec(specCtx, ch, rt, idx, spec)
-
-			select {
-			case <-specCtx.Done():
-				popTracer()
-				t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
-				break
-			case runres := <-ch:
-				res = runres.r
-				err = runres.err
-			}
+	t.Run(s.Title(), func(tt *testing.T) {
+		for idx := range s.Tests {
+			res, err = s.runSpec(ctx, tt, idx)
 			if err != nil {
-				popTracer()
-				specCancel()
 				break
 			}
 
-			if wait != nil && wait.After != "" {
-				debug.Println(specCtx, "wait: %s after", wait.After)
-				time.Sleep(wait.AfterDuration())
-			}
-
 			// Results can have arbitrary run data stored in them and we
 			// save this prior run data in the top-level context (and pass
 			// that context to the next Run invocation).
 			if res.HasData() {
 				ctx = gdtcontext.StorePriorRun(ctx, res.Data())
 			}
-			popTracer()
-			specCancel()
+
 			for _, fail := range res.Failures() {
-				t.Fatal(fail)
+				tt.Fatal(fail)
 			}
 		}
 	})
@@ -174,8 +98,71 @@ type runSpecRes struct {
 	err error
 }
 
-// runSpec executes an individual test spec
+// runSpec wraps the execution of a single test spec
 func (s *Scenario) runSpec(
+	ctx context.Context, // this is the overall scenario's context
+	t *testing.T, // T specific to the goroutine running this test spec
+	idx int, // index of the test spec within Scenario.Tests
+) (res *api.Result, err error) {
+	// Create a brand new context that inherits the top-level context's
+	// cancel func. We want to set deadlines for each test spec and if
+	// we mutate the single supplied top-level context, then only the
+	// first deadline/timeout will be used.
+	specCtx, specCancel := context.WithCancel(ctx)
+	defer specCancel()
+
+	defaults := s.getDefaults()
+	spec := s.Tests[idx]
+	sb := spec.Base()
+
+	specTraceMsg := strconv.Itoa(idx)
+	if sb.Name != "" {
+		specTraceMsg += ":" + sb.Name
+	}
+	specCtx = gdtcontext.PushTrace(specCtx, specTraceMsg)
+	defer func() {
+		specCtx = gdtcontext.PopTrace(specCtx)
+	}()
+
+	plugin := sb.Plugin
+	rt := getRetry(specCtx, defaults, plugin, spec)
+	to := getTimeout(specCtx, defaults, plugin, spec)
+	ch := make(chan runSpecRes, 1)
+
+	wait := sb.Wait
+	if wait != nil && wait.Before != "" {
+		debug.Println(specCtx, "wait: %s before", wait.Before)
+		time.Sleep(wait.BeforeDuration())
+	}
+
+	if to != nil {
+		specCtx, specCancel = context.WithTimeout(specCtx, to.Duration())
+		defer specCancel()
+	}
+
+	go s.execSpec(specCtx, ch, rt, idx, spec)
+
+	select {
+	case <-specCtx.Done():
+		t.Fatalf("assertion failed: timeout exceeded (%s)", to.After)
+	case runres := <-ch:
+		res = runres.r
+		err = runres.err
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if wait != nil && wait.After != "" {
+		debug.Println(specCtx, "wait: %s after", wait.After)
+		time.Sleep(wait.AfterDuration())
+	}
+	return res, nil
+}
+
+// execSpec executes an individual test spec, performing any retries as
+// necessary until a timeout is exceeded or the test spec succeeds
+func (s *Scenario) execSpec(
 	ctx context.Context,
 	ch chan runSpecRes,
 	retry *api.Retry,
@@ -190,7 +177,7 @@ func (s *Scenario) runSpec(
 			return
 		}
 		debug.Println(
-			ctx, "run: single-shot (no retries) ok: %v",
+			ctx, "spec/run: single-shot (no retries) ok: %v",
 			!res.Failed(),
 		)
 		ch <- runSpecRes{res, nil}
@@ -229,7 +216,7 @@ func (s *Scenario) runSpec(
 	for tick := range ticker.C {
 		if (maxAttempts > 0) && (attempts > maxAttempts) {
 			debug.Println(
-				ctx, "run: exceeded max attempts %d. stopping.",
+				ctx, "spec/run: exceeded max attempts %d. stopping.",
 				maxAttempts,
 			)
 			ticker.Stop()
@@ -244,7 +231,7 @@ func (s *Scenario) runSpec(
 		}
 		success = !res.Failed()
 		debug.Println(
-			ctx, "run: attempt %d after %s ok: %v",
+			ctx, "spec/run: attempt %d after %s ok: %v",
 			attempts, after, success,
 		)
 		if success {
@@ -253,7 +240,7 @@ func (s *Scenario) runSpec(
 		}
 		for _, f := range res.Failures() {
 			debug.Println(
-				ctx, "run: attempt %d failure: %s",
+				ctx, "spec/run: attempt %d failure: %s",
 				attempts, f,
 			)
 		}
@@ -262,6 +249,34 @@ func (s *Scenario) runSpec(
 	ch <- runSpecRes{res, nil}
 }
 
+// hasTimeoutConflict returns true if the scenario or any of its test specs has
+// a wait or timeout that exceeds the go test tool's specified timeout value
+func (s *Scenario) hasTimeoutConflict(
+	ctx context.Context,
+	t *testing.T,
+) bool {
+	d, ok := t.Deadline()
+	if ok && !d.IsZero() {
+		now := time.Now()
+		s.Timings.GoTestTimeout = d.Sub(now)
+		debug.Println(
+			ctx, "scenario/run: go test tool timeout: %s",
+			(s.Timings.GoTestTimeout + time.Second).Round(time.Second),
+		)
+		if s.Timings.TotalWait > 0 {
+			if s.Timings.TotalWait.Abs() > s.Timings.GoTestTimeout.Abs() {
+				return true
+			}
+		}
+		if s.Timings.MaxTimeout > 0 {
+			if s.Timings.MaxTimeout.Abs() > s.Timings.GoTestTimeout.Abs() {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // getTimeout returns the timeout configuration for the test spec. We check for
 // overrides in timeout configuration using the following precedence:
 //
@@ -271,7 +286,7 @@ func (s *Scenario) runSpec(
 // * Plugin's default
 func getTimeout(
 	ctx context.Context,
-	scenDefaults *Defaults,
+	defaults *Defaults,
 	plugin api.Plugin,
 	eval api.Evaluable,
 ) *api.Timeout {
@@ -294,12 +309,12 @@ func getTimeout(
 		return baseTimeout
 	}
 
-	if scenDefaults != nil && scenDefaults.Timeout != nil {
+	if defaults != nil && defaults.Timeout != nil {
 		debug.Println(
 			ctx, "using timeout of %s [scenario default]",
-			scenDefaults.Timeout.After,
+			defaults.Timeout.After,
 		)
-		return scenDefaults.Timeout
+		return defaults.Timeout
 	}
 
 	pluginInfo := plugin.Info()
@@ -324,7 +339,7 @@ func getTimeout(
 // * Plugin's default
 func getRetry(
 	ctx context.Context,
-	scenDefaults *Defaults,
+	defaults *Defaults,
 	plugin api.Plugin,
 	eval api.Evaluable,
 ) *api.Retry {
@@ -363,21 +378,21 @@ func getRetry(
 		return baseRetry
 	}
 
-	if scenDefaults != nil && scenDefaults.Retry != nil {
-		scenRetry := scenDefaults.Retry
-		if scenRetry == api.NoRetry {
-			return scenRetry
+	if defaults != nil && defaults.Retry != nil {
+		defaultRetry := defaults.Retry
+		if defaultRetry == api.NoRetry {
+			return defaultRetry
 		}
 		msg := "using retry"
-		if scenRetry.Attempts != nil {
-			msg += fmt.Sprintf(" (attempts: %d)", *scenRetry.Attempts)
+		if defaultRetry.Attempts != nil {
+			msg += fmt.Sprintf(" (attempts: %d)", *defaultRetry.Attempts)
 		}
-		if scenRetry.Interval != "" {
-			msg += fmt.Sprintf(" (interval: %s)", scenRetry.Interval)
+		if defaultRetry.Interval != "" {
+			msg += fmt.Sprintf(" (interval: %s)", defaultRetry.Interval)
 		}
-		msg += fmt.Sprintf(" (exponential: %t) [scenario default]", scenRetry.Exponential)
+		msg += fmt.Sprintf(" (exponential: %t) [scenario default]", defaultRetry.Exponential)
 		debug.Println(ctx, msg)
-		return scenRetry
+		return defaultRetry
 	}
 
 	pluginInfo := plugin.Info()
@@ -401,9 +416,9 @@ func getRetry(
 	return nil
 }
 
-// getScenarioDefaults returns the Defaults parsed from the scenario's YAML
+// getDefaults returns the Defaults parsed from the scenario's YAML
 // file's `defaults` field, or nil if none were specified.
-func (s *Scenario) getScenarioDefaults() *Defaults {
+func (s *Scenario) getDefaults() *Defaults {
 	scDefaultsAny, found := s.Defaults[DefaultsKey]
 	if found {
 		return scDefaultsAny.(*Defaults)
diff --git a/scenario/run_test.go b/scenario/run_test.go
index 9b8ed35..440477e 100644
--- a/scenario/run_test.go
+++ b/scenario/run_test.go
@@ -193,7 +193,7 @@ func TestNoRetry(t *testing.T) {
 	w.Flush()
 	require.NotEqual(b.Len(), 0)
 	debugout := b.String()
-	require.Contains(debugout, "[gdt] [no-retry/0:bar] run: single-shot (no retries) ok: true")
+	require.Contains(debugout, "[gdt] [no-retry/0:bar] spec/run: single-shot (no retries) ok: true")
 }
 
 func TestNoRetryEvaluableOverride(t *testing.T) {
@@ -217,7 +217,7 @@ func TestNoRetryEvaluableOverride(t *testing.T) {
 	w.Flush()
 	require.NotEqual(b.Len(), 0)
 	debugout := b.String()
-	require.Contains(debugout, "[gdt] [no-retry-evaluable-override/0:bar] run: single-shot (no retries) ok: true")
+	require.Contains(debugout, "[gdt] [no-retry-evaluable-override/0:bar] spec/run: single-shot (no retries) ok: true")
 }
 
 func TestFailRetryTestOverride(t *testing.T) {
@@ -253,7 +253,7 @@ func TestRetryTestOverride(t *testing.T) {
 	require.NotNil(err)
 
 	debugout := string(outerr)
-	require.Contains(debugout, "[gdt] [retry-test-override/0:baz] run: exceeded max attempts 2. stopping.")
+	require.Contains(debugout, "[gdt] [retry-test-override/0:baz] spec/run: exceeded max attempts 2. stopping.")
 }
 
 func TestSkipIf(t *testing.T) {

From a21aaf4952b34752b2a310c491000d18b0d685d7 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Mon, 8 Jul 2024 08:59:49 -0400
Subject: [PATCH 28/31] fix TimeoutConflict error message

The TimeoutConflict error message was not accurately showing the
conflict if both the wait total and max timeout values were non-zero.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 api/error.go | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/api/error.go b/api/error.go
index 07875c9..5c77294 100644
--- a/api/error.go
+++ b/api/error.go
@@ -315,27 +315,31 @@ func RequiredFixtureMissing(name string) error {
 func TimeoutConflict(
 	ti *Timings,
 ) error {
-	gotestDeadline := ti.GoTestTimeout
+	goTestTimeout := ti.GoTestTimeout
 	totalWait := ti.TotalWait
 	maxTimeout := ti.MaxTimeout
 	msg := fmt.Sprintf(
 		"go test -timeout value of %s ",
-		(gotestDeadline + time.Second).Round(time.Second),
+		(goTestTimeout + time.Second).Round(time.Second),
 	)
 	if totalWait > 0 {
-		msg += fmt.Sprintf(
-			"is shorter than the total wait time in the scenario: %s. "+
-				"either decrease the wait times or increase the "+
-				"go test -timeout value.",
-			totalWait.Round(time.Second),
-		)
+		if totalWait.Abs() > goTestTimeout.Abs() {
+			msg += fmt.Sprintf(
+				"is shorter than the total wait time in the scenario: %s. "+
+					"either decrease the wait times or increase the "+
+					"go test -timeout value.",
+				totalWait.Round(time.Second),
+			)
+		}
 	} else {
-		msg += fmt.Sprintf(
-			"is shorter than the maximum timeout specified in the "+
-				"scenario: %s. either decrease the scenario or spec "+
-				"timeout or increase the go test -timeout value.",
-			maxTimeout.Round(time.Second),
-		)
+		if maxTimeout.Abs() > goTestTimeout.Abs() {
+			msg += fmt.Sprintf(
+				"is shorter than the maximum timeout specified in the "+
+					"scenario: %s. either decrease the scenario or spec "+
+					"timeout or increase the go test -timeout value.",
+				maxTimeout.Round(time.Second),
+			)
+		}
 	}
 	return fmt.Errorf("%w: %s", ErrTimeoutConflict, msg)
 }

From 0d3fecdcd7e38b0a9517d53315ed8bf8360c55d3 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Mon, 8 Jul 2024 11:56:18 -0400
Subject: [PATCH 29/31] trim space from exec plugin out/err

Removes trailing whitespace/newlines from the stdout and stderr debug
output for the exec plugin.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 plugin/exec/action.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/plugin/exec/action.go b/plugin/exec/action.go
index 687f5ef..a4e820f 100644
--- a/plugin/exec/action.go
+++ b/plugin/exec/action.go
@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"context"
 	"os/exec"
+	"strings"
 
 	"github.com/gdt-dev/gdt/api"
 	gdtcontext "github.com/gdt-dev/gdt/context"
@@ -79,7 +80,10 @@ func (a *Action) Do(
 			debug.Println(ctx, "exec: error reading from stdout: %s", err)
 		}
 		if outbuf.Len() > 0 {
-			debug.Println(ctx, "exec: stdout: %s", outbuf.String())
+			debug.Println(
+				ctx, "exec: stdout: %s",
+				strings.TrimSpace(outbuf.String()),
+			)
 		}
 	}
 	if errbuf != nil {
@@ -87,7 +91,10 @@ func (a *Action) Do(
 			debug.Println(ctx, "exec: error reading from stderr: %s", err)
 		}
 		if errbuf.Len() > 0 {
-			debug.Println(ctx, "exec: stderr: %s", errbuf.String())
+			debug.Println(
+				ctx, "exec: stderr: %s",
+				strings.TrimSpace(errbuf.String()),
+			)
 		}
 	}
 

From f942ccf798667450d5df1c4fe9e7248e0b11e026 Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Mon, 8 Jul 2024 13:26:00 -0400
Subject: [PATCH 30/31] correct exit code assertion in exec plugin

The constructor for assertions in the exec plugin was incorrectly
setting the expected exit code equal to the exit code it received from
the test spec, resulting in script or command executions returning
non-0 exit codes from causing a test.FailNow().

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 plugin/exec/assertions.go                     |  6 ++-
 plugin/exec/eval_test.go                      | 47 +++++++++++++++++++
 .../exec/testdata/ls-fail-no-exit-code.yaml   |  4 ++
 3 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 plugin/exec/testdata/ls-fail-no-exit-code.yaml

diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go
index defebb0..84fb1f9 100644
--- a/plugin/exec/assertions.go
+++ b/plugin/exec/assertions.go
@@ -166,9 +166,13 @@ func newAssertions(
 	outPipe *bytes.Buffer,
 	errPipe *bytes.Buffer,
 ) api.Assertions {
+	expExitCode := 0
+	if e != nil {
+		expExitCode = e.ExitCode
+	}
 	a := &assertions{
 		failures:    []error{},
-		expExitCode: exitCode,
+		expExitCode: expExitCode,
 		exitCode:    exitCode,
 	}
 	if e != nil {
diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go
index 60f40e8..67f339d 100644
--- a/plugin/exec/eval_test.go
+++ b/plugin/exec/eval_test.go
@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"flag"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -71,6 +72,52 @@ func TestExitCode(t *testing.T) {
 	require.Nil(err)
 }
 
+func TestFailExecExitCodeNotSpecified(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "ls-fail-no-exit-code.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
+func TestExecFailExitCodeNotSpecified(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailExecExitCodeNotSpecified",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+	debugout := string(outerr)
+	ec := 2
+	// Yay, different exit codes for the same not found error...
+	if runtime.GOOS == "darwin" {
+		ec = 1
+	}
+	msg := fmt.Sprintf(
+		"assertion failed: not equal: expected 0 but got %d", ec,
+	)
+	require.Contains(debugout, msg)
+}
+
 func TestShellList(t *testing.T) {
 	require := require.New(t)
 
diff --git a/plugin/exec/testdata/ls-fail-no-exit-code.yaml b/plugin/exec/testdata/ls-fail-no-exit-code.yaml
new file mode 100644
index 0000000..35f16af
--- /dev/null
+++ b/plugin/exec/testdata/ls-fail-no-exit-code.yaml
@@ -0,0 +1,4 @@
+name: ls-fail-no-exit-code
+description: a scenario that runs the `ls` command with a non-0 exit code and no assertion on exit code
+tests:
+  - exec: ls /this/dir/does/not/exist

From df4c3d6785276c26edd9492529b85ef3d35d47ae Mon Sep 17 00:00:00 2001
From: Jay Pipes <jaypipes@gmail.com>
Date: Mon, 8 Jul 2024 13:37:42 -0400
Subject: [PATCH 31/31] update Github Action runners

Bring in latest step security and checkout action.

Signed-off-by: Jay Pipes <jaypipes@gmail.com>
---
 .github/workflows/fmtcheck.yml | 5 +++--
 .github/workflows/lint.yml     | 1 +
 .github/workflows/test.yml     | 5 +++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/fmtcheck.yml b/.github/workflows/fmtcheck.yml
index 242acc1..11cafcb 100644
--- a/.github/workflows/fmtcheck.yml
+++ b/.github/workflows/fmtcheck.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
      - name: harden runner
-       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          egress-policy: block
          disable-sudo: true
@@ -26,8 +26,9 @@ jobs:
            raw.githubusercontent.com:443
            objects.githubusercontent.com:443
            proxy.golang.org:443
+           blob.core.windows.net:443
      - name: checkout code
-       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 50c153f..1f5b2e6 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -27,6 +27,7 @@ jobs:
            raw.githubusercontent.com:443
            objects.githubusercontent.com:443
            proxy.golang.org:443
+           blob.core.windows.net:443
      - name: checkout code
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a727f30..cade935 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
      - name: harden runner
-       uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
+       uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          egress-policy: block
          disable-sudo: true
@@ -29,8 +29,9 @@ jobs:
            raw.githubusercontent.com:443
            objects.githubusercontent.com:443
            proxy.golang.org:443
+           blob.core.windows.net:443
      - name: checkout code
-       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: setup go
        uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
        with:
