Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ The canonical multi-cluster topology. Each compute plane is registered separatel
```sh
# 1. Bring up the control plane (one-time, on the control-plane cluster):
KUBECONFIG=cp.yaml nvcf-cli self-hosted install --control-plane | kubectl apply -f -
nvcf-cli self-hosted check --control-plane --wait 5m
KUBECONFIG=cp.yaml nvcf-cli self-hosted status

# 2. Register + install each compute plane:
for CTX in admin@gpu-east-1 admin@gpu-west-1 admin@gpu-eu-1; do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ Full subcommand list. Always pair with [flags.md](flags.md) for global flags and
| Command | Purpose | Output | Notes |
|---|---|---|---|
| `self-hosted check --pre [flags]` | Pre-flight validation (no admin creds) | streaming events | Per-role with `--control-plane-context` / `--compute-plane-context`; `--local-only` skips kubectl |
| `self-hosted check --control-plane [--wait DUR]` | Every CP release healthy | streaming events | Polls until pass or `--wait` elapses |
| `self-hosted check --compute-plane --cluster-name=X [--wait DUR]` | Per-cluster worker state | streaming events | NVCA + operator + JWKS-fingerprint match |
| `self-hosted check --all [--wait DUR]` | Fan-out over all of the above | streaming events | |
| `self-hosted check --control-plane` | Not implemented yet | streaming events | Reports a failed check and exits 2; use `self-hosted status` for component health |
| `self-hosted check --compute-plane` | Not implemented yet | streaming events | Reports a failed check and exits 2; use `cluster-agent validate` for compute-plane health |
| `self-hosted check --all [--wait DUR]` | Fan-out over the implemented checks (currently the `--pre` set) | streaming events | |
| `self-hosted install --control-plane` | Render control-plane manifests | YAML on stdout | Pipe to `kubectl apply -f -` |
| `self-hosted install --compute-plane --cluster-name=X` | Register cluster + render compute manifests | YAML on stdout | One invocation per GPU cluster |
| `self-hosted up --cluster-name=X` | One-shot first install (both planes) | (empty) stdout, progress on stderr | Always installs both planes; for compute-only use `add-compute-plane` |
Expand Down
86 changes: 81 additions & 5 deletions src/clis/nvcf-cli/cmd/self_hosted_check.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ var selfHostedCheckCmd = &cobra.Command{
func init() {
selfHostedCmd.AddCommand(selfHostedCheckCmd)
selfHostedCheckCmd.Flags().BoolVar(&checkPre, "pre", false, "Run pre-flight (local-host + cluster readiness)")
selfHostedCheckCmd.Flags().BoolVar(&checkControlPlane, "control-plane", false, "Run control-plane health checks")
selfHostedCheckCmd.Flags().BoolVar(&checkControlPlane, "control-plane", false, "Run control-plane health checks (not yet implemented: reports a failure and exits non-zero)")
selfHostedCheckCmd.Flags().BoolVar(&checkComputePlane, "compute-plane", false,
"Run compute-plane health checks. Requires --cluster-name.")
"Run compute-plane health checks (not yet implemented: reports a failure and exits non-zero)")
selfHostedCheckCmd.Flags().BoolVar(&checkAll, "all", false, "Run all check categories")
selfHostedCheckCmd.Flags().StringVar(&checkClusterName, "cluster-name", "", "Cluster name for compute-plane checks")
selfHostedCheckCmd.Flags().BoolVar(&checkLocalOnly, "local-only", false, "Run local-host checks only (no kubectl contact)")
Expand Down Expand Up @@ -194,8 +194,8 @@ func runSelfHostedCheck(c *cobra.Command, _ []string) error {
Message: "forced failure (test seam)",
}}, results...)
}
// control-plane / compute-plane wired in M3/M4 — placeholder no-op for M2.
return results
results = append(results, unimplementedCategoryResults(ctx, sink)...)
return ensureChecksRan(ctx, sink, results)
}

if selfHostedWait == "" {
Expand All @@ -204,7 +204,7 @@ func runSelfHostedCheck(c *cobra.Command, _ []string) error {
emitCheckFinal(ctx, sink, lastResults)
maybeShowClusterValidatorLogs(c.ErrOrStderr(), lastResults)
if anyFailed(lastResults) {
return &ExitCodeError{Code: 2, Msg: "pre-flight checks failed"}
return failedChecksExit(lastResults)
}
return nil
}
Expand All @@ -221,6 +221,12 @@ func runSelfHostedCheck(c *cobra.Command, _ []string) error {

for {
lastResults = runOnce()
if hasUnimplementedCheck(lastResults) {
// Polling cannot make an unimplemented check pass.
emitCheckFinal(ctx, sink, lastResults)
maybeShowClusterValidatorLogs(c.ErrOrStderr(), lastResults)
return failedChecksExit(lastResults)
}
if !anyFailed(lastResults) {
emitCheckFinal(ctx, sink, lastResults)
maybeShowClusterValidatorLogs(c.ErrOrStderr(), lastResults)
Expand All @@ -240,6 +246,76 @@ func runSelfHostedCheck(c *cobra.Command, _ []string) error {
}
}

const (
checkIDControlPlaneHealth = "control-plane-health"
checkIDComputePlaneHealth = "compute-plane-health"
checkIDNoChecksRun = "no-checks-run"
)

// emitFailedCheck emits one failed check as a complete category, so renderers
// and --json consumers see the same event sequence as for a real check.
func emitFailedCheck(ctx context.Context, sink progress.EventSink, category, id, message string) selfhosted.CheckResult {
res := selfhosted.CheckResult{
ID: id,
Category: category,
Severity: "error",
Passed: false,
Message: message,
}
_ = sink.Emit(ctx, progress.CheckStarted{Category: category, ID: id, Message: message})
_ = sink.Emit(ctx, progress.CheckCompleted{
Category: category,
ID: id,
Passed: false,
Severity: res.Severity,
Message: message,
})
_ = sink.Emit(ctx, progress.CategoryCompleted{Category: category, FailedCount: 1})
return res
}

// unimplementedCategoryResults reports --control-plane and --compute-plane as
// failed checks. Neither flag has probes behind it yet, and a silent no-op
// reads as a passing health check.
func unimplementedCategoryResults(ctx context.Context, sink progress.EventSink) []selfhosted.CheckResult {
var out []selfhosted.CheckResult
if checkControlPlane {
out = append(out, emitFailedCheck(ctx, sink, "control-plane", checkIDControlPlaneHealth,
"control-plane health checks are not implemented; nothing was verified"))
}
if checkComputePlane {
out = append(out, emitFailedCheck(ctx, sink, "compute-plane", checkIDComputePlaneHealth,
"compute-plane health checks are not implemented; nothing was verified"))
}
return out
}

// ensureChecksRan turns a run that checked nothing into a failure, so an empty
// result set can never read as a pass.
func ensureChecksRan(ctx context.Context, sink progress.EventSink, results []selfhosted.CheckResult) []selfhosted.CheckResult {
if len(results) > 0 {
return results
}
return append(results, emitFailedCheck(ctx, sink, "check", checkIDNoChecksRun,
"no checks were run; nothing was verified"))
}

func hasUnimplementedCheck(results []selfhosted.CheckResult) bool {
for _, r := range results {
if r.ID == checkIDControlPlaneHealth || r.ID == checkIDComputePlaneHealth {
return true
}
}
return false
}

func failedChecksExit(results []selfhosted.CheckResult) error {
if hasUnimplementedCheck(results) {
return &ExitCodeError{Code: 2, Msg: "requested health checks are not implemented"}
}
return &ExitCodeError{Code: 2, Msg: "pre-flight checks failed"}
}

// maybeShowClusterValidatorLogs prints the cleaned cluster-validator transcript
// to the given writer when --show-logs is set, framed by markers so operators
// can find it in mixed CLI output. Silent no-op when:
Expand Down
129 changes: 129 additions & 0 deletions src/clis/nvcf-cli/cmd/self_hosted_check_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"nvcf-cli/internal/selfhosted"
"nvcf-cli/internal/selfhosted/progress"
)

Expand Down Expand Up @@ -343,6 +344,134 @@ func TestCheck_SplitClusterMode(t *testing.T) {
assert.Contains(t, categories, "compute-plane-cluster", "expected compute-plane-cluster in split mode")
}

// resetCheckFlags clears check-flag globals before and after the test; cobra
// keeps flag values across Execute calls, so earlier tests leak into later ones.
func resetCheckFlags(t *testing.T) {
t.Helper()
reset := func() {
selfHostedJSON = false
selfHostedOutput = "text"
selfHostedWait = ""
checkLocalOnly = false
checkPre = false
checkControlPlane = false
checkComputePlane = false
checkAll = false
}
reset()
t.Cleanup(reset)
}

func finalEvent(t *testing.T, lines []map[string]any) map[string]any {
t.Helper()
for _, l := range lines {
if l["event"] == "final" {
return l
}
}
t.Fatal("no final event emitted")
return nil
}

// TestCheck_UnimplementedCategoryFlagsFail verifies --control-plane and
// --compute-plane report a failed check and exit 2 instead of a vacuous pass.
func TestCheck_UnimplementedCategoryFlagsFail(t *testing.T) {
tests := []struct {
flag, id string
}{
{"--control-plane", "control-plane-health"},
{"--compute-plane", "compute-plane-health"},
}
for _, tt := range tests {
t.Run(tt.flag, func(t *testing.T) {
resetCheckFlags(t)
var stderr bytes.Buffer
rootCmd.SetErr(&stderr)
rootCmd.SetOut(&bytes.Buffer{})

rootCmd.SetArgs([]string{"self-hosted", "check", tt.flag, "--json"})
err := rootCmd.Execute()

var exitErr *ExitCodeError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, 2, exitErr.Code)

lines := parseJSONLLines(t, stderr.String())
var found bool
for _, l := range lines {
if l["event"] == "check_completed" && l["id"] == tt.id {
found = true
assert.Equal(t, false, l["passed"])
assert.Equal(t, "error", l["severity"])
assert.Contains(t, l["message"], "not implemented")
}
}
assert.True(t, found, "expected a failed %s check_completed event", tt.id)

final := finalEvent(t, lines)
assert.Equal(t, false, final["success"])
assert.Equal(t, "failed", final["verdict"])
assert.EqualValues(t, 1, final["failedCount"])
assert.EqualValues(t, 0, final["passedCount"])
})
}
}

// TestCheck_UnimplementedCategoryWaitFailsFast verifies --wait does not poll
// for a check that cannot ever pass.
func TestCheck_UnimplementedCategoryWaitFailsFast(t *testing.T) {
resetCheckFlags(t)
var stderr bytes.Buffer
rootCmd.SetErr(&stderr)
rootCmd.SetOut(&bytes.Buffer{})

rootCmd.SetArgs([]string{"self-hosted", "check", "--control-plane", "--wait", "1m", "--json"})
start := time.Now()
err := rootCmd.Execute()

var exitErr *ExitCodeError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, 2, exitErr.Code)
assert.Less(t, time.Since(start), 4*time.Second, "must not wait out the poll interval")
assert.Equal(t, false, finalEvent(t, parseJSONLLines(t, stderr.String()))["success"])
}

// TestCheck_AllDoesNotReportUnimplemented verifies --all keeps meaning "every
// available check" and does not add the unimplemented-category failures.
func TestCheck_AllDoesNotReportUnimplemented(t *testing.T) {
resetCheckFlags(t)
var stderr bytes.Buffer
rootCmd.SetErr(&stderr)
rootCmd.SetOut(&bytes.Buffer{})

rootCmd.SetArgs([]string{"self-hosted", "check", "--all", "--local-only", "--json"})
_ = rootCmd.Execute()

for _, l := range parseJSONLLines(t, stderr.String()) {
assert.NotEqual(t, "control-plane-health", l["id"])
assert.NotEqual(t, "compute-plane-health", l["id"])
}
}

func TestEnsureChecksRan(t *testing.T) {
t.Run("empty results become a failure", func(t *testing.T) {
sink := &recordingSink{}
got := ensureChecksRan(context.Background(), sink, nil)
require.Len(t, got, 1)
assert.Equal(t, "no-checks-run", got[0].ID)
assert.False(t, got[0].Passed)
assert.True(t, anyFailed(got))
assert.NotEmpty(t, sink.events, "the failure must reach the renderer")
})
t.Run("existing results are untouched", func(t *testing.T) {
sink := &recordingSink{}
in := []selfhosted.CheckResult{{ID: "x", Passed: true, Severity: "info"}}
got := ensureChecksRan(context.Background(), sink, in)
assert.Equal(t, in, got)
assert.Empty(t, sink.events)
})
}

// parseJSONLLines splits s into non-empty lines, skips any non-JSON lines
// (e.g. cobra error messages written to stderr), and unmarshals each JSON line
// as an object. Returns them in order.
Expand Down
Loading
Loading