Create a Sweep
const url = 'https://example.com/apis/nodus.dev/v1/namespaces/example/sweeps';const options = { method: 'POST', headers: {'Content-Type': 'application/json'}, body: '{"apiVersion":"example","kind":"example","metadata":{"annotations":{"additionalProperty":"example"},"creationTimestamp":"example","deletionGracePeriodSeconds":1,"deletionTimestamp":"example","finalizers":["example"],"generateName":"example","generation":1,"labels":{"additionalProperty":"example"},"managedFields":[{"apiVersion":"example","fieldsType":"example","fieldsV1":{"additionalProperty":"example"},"manager":"example","operation":"example","subresource":"example","time":"example"}],"name":"example","namespace":"example","ownerReferences":[{"apiVersion":"example","blockOwnerDeletion":true,"controller":true,"kind":"example","name":"example","uid":"example"}],"resourceVersion":"example","selfLink":"example","uid":"example"},"spec":{"matrix":{"gpu":[{"count":1,"exact":true,"interconnect":"example","minMemory":"example","type":["example"]}],"params":{"additionalProperty":["example"]},"regions":["example"]},"maxCostUSD":"example","maxParallel":1,"repetitions":1,"state":"example","template":{"kind":"example","spec":{"activeDeadlineSeconds":1,"args":["example"],"backoffLimit":1,"command":["example"],"completeByTime":"example","completions":1,"connections":["example"],"data":{"columns":{"additionalProperty":"example"},"format":"example","path":"example","revision":1,"validationFraction":"example","volume":"example"},"distributed":{"gpusPerNode":1,"launcher":"example","maxAssemblyRetries":1,"network":"example","nodes":1,"startupTimeout":"example","totalGPUs":1,"transport":"example"},"env":[{"name":"example","value":"example","valueFrom":{"secretKeyRef":{"key":"example","name":"example"}}}],"environment":{"heldOutTasks":1,"name":"example","seed":1,"trainTasks":1},"evaluation":{"baseline":true,"final":true,"minComparisonTasks":1},"expectedDuration":"example","exportTo":{"volume":"example"},"grading":{"maxParallel":1},"image":"example","imagePullSecrets":["example"],"imageRef":"example","initCommand":{"command":["example"],"timeout":"example"},"inputs":[{"bucket":{"connection":"example","sha256":"example","uri":"example"},"fromStage":{"output":"example","stage":"example"},"fromWorkspace":{"name":"example","path":"example"},"name":"example","output":{"index":1,"job":"example","name":"example"},"url":{"sha256":"example","url":"example"},"volume":{"name":"example","revision":1}}],"maxCostUSD":"example","mode":"example","model":{"secret":"example","uri":"example","volume":{"name":"example","revision":1}},"network":{"egress":{"allow":["example"],"policy":"example","presets":["example"]}},"outputs":[{"fromRanks":"example","name":"example","path":"example","sink":{"connection":"example","mode":"example","table":"example"}}],"parallelism":1,"parameters":"example","placement":{"dedicated":true,"interruptible":"example","maxRateUSDPerHour":"example","offerings":["example"],"pool":"example","profile":"example","queueTimeout":"example","regions":["example"]},"recovery":{"checkpoint":{"format":"example","integration":"example","interval":"example","maxSize":"example","paths":["example"],"retainAfterFinish":"example"},"continuity":"example","maxAttempts":1,"minProgressDuration":"example","onInterruption":"example"},"resources":{"cpu":"example","disk":"example","gpu":{"count":1,"exact":true,"interconnect":"example","minMemory":"example","type":["example"]},"memory":"example","nodes":1},"runtime":"example","scaledownWindow":"example","secrets":[{"name":"example","version":1}],"serviceAccountName":"example","sidecars":[{"command":["example"],"env":[{"name":"example","value":"example","valueFrom":{"secretKeyRef":{"key":"example","name":"example"}}}],"image":"example","name":"example","startupTimeout":"example"}],"source":{"blob":"example","git":{"path":"example","ref":"example","repo":"example"}},"state":"example","task":"example","teacher":{"secret":"example","uri":"example","volume":{"name":"example","revision":1}},"timeout":"example","tracking":{"mlflow":{"experiment":"example","secret":"example","uri":"example"},"wandb":{"connection":"example"}},"ttlSecondsAfterFinished":1,"volumes":[{"mountPath":"example","readOnly":true,"revision":1,"volume":"example"}],"workingDir":"example"}}},"status":{"best":{"byCost":1,"byThroughput":1},"cells":[{"costUSD":"example","gpu":"example","index":1,"object":"example","params":{"additionalProperty":"example"},"phase":"example","region":"example","repetition":1,"unitsPerSecond":"example","wallSeconds":1}],"completionTime":"example","conditions":[{"lastTransitionTime":"example","message":"example","observedGeneration":1,"reason":"example","status":"example","type":"example"}],"cost":{"bySegment":{"bootCoveredSeconds":1,"bootSeconds":1,"bootUSD":"example","coveredByNodus":["example"],"restoreCoveredSeconds":1,"restoreSeconds":1,"restoreUSD":"example","runningCoveredSeconds":1,"runningSeconds":1,"runningUSD":"example","teardownCoveredSeconds":1,"teardownSeconds":1,"teardownUSD":"example"},"final":true,"fundedUntilTime":"example","heldUSD":"example","limitUSD":"example","rateUSDPerHour":"example","totalUSD":"example"},"estimate":{"assemblyBoundUSD":"example","blockingReasons":[{"code":"example","fix":"example","message":"example"}],"concurrentHoldUSD":"example","costUSD":{"p50":"example","p90":"example"},"gangHoldUSD":"example","holdUSD":"example","minimumChargeUSD":"example","overheadUSD":{"p50":"example","p90":"example"},"placementPreview":{"explain":"example","interruptible":true,"offering":"example","regionClass":"example","warmReuse":true},"pricebookVersion":"example","rateUSDPerHour":"example","startup":{"cold":{"p50":"example","p90":"example"},"warm":{"p50":"example","p90":"example"}},"topology":"example","validUntil":"example","warnings":[{"code":"example","message":"example"}]},"message":"example","observedGeneration":1,"phase":"example","reason":"example","startTime":"example","templateETags":["example"]}}'};
try { const response = await fetch(url, options); const data = await response.json(); console.log(data);} catch (error) { console.error(error);}curl --request POST \ --url https://example.com/apis/nodus.dev/v1/namespaces/example/sweeps \ --header 'Content-Type: application/json' \ --data '{ "apiVersion": "example", "kind": "example", "metadata": { "annotations": { "additionalProperty": "example" }, "creationTimestamp": "example", "deletionGracePeriodSeconds": 1, "deletionTimestamp": "example", "finalizers": [ "example" ], "generateName": "example", "generation": 1, "labels": { "additionalProperty": "example" }, "managedFields": [ { "apiVersion": "example", "fieldsType": "example", "fieldsV1": { "additionalProperty": "example" }, "manager": "example", "operation": "example", "subresource": "example", "time": "example" } ], "name": "example", "namespace": "example", "ownerReferences": [ { "apiVersion": "example", "blockOwnerDeletion": true, "controller": true, "kind": "example", "name": "example", "uid": "example" } ], "resourceVersion": "example", "selfLink": "example", "uid": "example" }, "spec": { "matrix": { "gpu": [ { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] } ], "params": { "additionalProperty": [ "example" ] }, "regions": [ "example" ] }, "maxCostUSD": "example", "maxParallel": 1, "repetitions": 1, "state": "example", "template": { "kind": "example", "spec": { "activeDeadlineSeconds": 1, "args": [ "example" ], "backoffLimit": 1, "command": [ "example" ], "completeByTime": "example", "completions": 1, "connections": [ "example" ], "data": { "columns": { "additionalProperty": "example" }, "format": "example", "path": "example", "revision": 1, "validationFraction": "example", "volume": "example" }, "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "environment": { "heldOutTasks": 1, "name": "example", "seed": 1, "trainTasks": 1 }, "evaluation": { "baseline": true, "final": true, "minComparisonTasks": 1 }, "expectedDuration": "example", "exportTo": { "volume": "example" }, "grading": { "maxParallel": 1 }, "image": "example", "imagePullSecrets": [ "example" ], "imageRef": "example", "initCommand": { "command": [ "example" ], "timeout": "example" }, "inputs": [ { "bucket": { "connection": "example", "sha256": "example", "uri": "example" }, "fromStage": { "output": "example", "stage": "example" }, "fromWorkspace": { "name": "example", "path": "example" }, "name": "example", "output": { "index": 1, "job": "example", "name": "example" }, "url": { "sha256": "example", "url": "example" }, "volume": { "name": "example", "revision": 1 } } ], "maxCostUSD": "example", "mode": "example", "model": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "network": { "egress": { "allow": [ "example" ], "policy": "example", "presets": [ "example" ] } }, "outputs": [ { "fromRanks": "example", "name": "example", "path": "example", "sink": { "connection": "example", "mode": "example", "table": "example" } } ], "parallelism": 1, "parameters": "example", "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "recovery": { "checkpoint": { "format": "example", "integration": "example", "interval": "example", "maxSize": "example", "paths": [ "example" ], "retainAfterFinish": "example" }, "continuity": "example", "maxAttempts": 1, "minProgressDuration": "example", "onInterruption": "example" }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "runtime": "example", "scaledownWindow": "example", "secrets": [ { "name": "example", "version": 1 } ], "serviceAccountName": "example", "sidecars": [ { "command": [ "example" ], "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "image": "example", "name": "example", "startupTimeout": "example" } ], "source": { "blob": "example", "git": { "path": "example", "ref": "example", "repo": "example" } }, "state": "example", "task": "example", "teacher": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "timeout": "example", "tracking": { "mlflow": { "experiment": "example", "secret": "example", "uri": "example" }, "wandb": { "connection": "example" } }, "ttlSecondsAfterFinished": 1, "volumes": [ { "mountPath": "example", "readOnly": true, "revision": 1, "volume": "example" } ], "workingDir": "example" } } }, "status": { "best": { "byCost": 1, "byThroughput": 1 }, "cells": [ { "costUSD": "example", "gpu": "example", "index": 1, "object": "example", "params": { "additionalProperty": "example" }, "phase": "example", "region": "example", "repetition": 1, "unitsPerSecond": "example", "wallSeconds": 1 } ], "completionTime": "example", "conditions": [ { "lastTransitionTime": "example", "message": "example", "observedGeneration": 1, "reason": "example", "status": "example", "type": "example" } ], "cost": { "bySegment": { "bootCoveredSeconds": 1, "bootSeconds": 1, "bootUSD": "example", "coveredByNodus": [ "example" ], "restoreCoveredSeconds": 1, "restoreSeconds": 1, "restoreUSD": "example", "runningCoveredSeconds": 1, "runningSeconds": 1, "runningUSD": "example", "teardownCoveredSeconds": 1, "teardownSeconds": 1, "teardownUSD": "example" }, "final": true, "fundedUntilTime": "example", "heldUSD": "example", "limitUSD": "example", "rateUSDPerHour": "example", "totalUSD": "example" }, "estimate": { "assemblyBoundUSD": "example", "blockingReasons": [ { "code": "example", "fix": "example", "message": "example" } ], "concurrentHoldUSD": "example", "costUSD": { "p50": "example", "p90": "example" }, "gangHoldUSD": "example", "holdUSD": "example", "minimumChargeUSD": "example", "overheadUSD": { "p50": "example", "p90": "example" }, "placementPreview": { "explain": "example", "interruptible": true, "offering": "example", "regionClass": "example", "warmReuse": true }, "pricebookVersion": "example", "rateUSDPerHour": "example", "startup": { "cold": { "p50": "example", "p90": "example" }, "warm": { "p50": "example", "p90": "example" } }, "topology": "example", "validUntil": "example", "warnings": [ { "code": "example", "message": "example" } ] }, "message": "example", "observedGeneration": 1, "phase": "example", "reason": "example", "startTime": "example", "templateETags": [ "example" ] } }'Parameters
Section titled “ Parameters ”Path Parameters
Section titled “ Path Parameters ”The project.
Query Parameters
Section titled “ Query Parameters ”Header Parameters
Section titled “ Header Parameters ”Request Bodyrequired
Section titled “ Request Bodyrequired ”Sweep fans a Job or TrainingJob template out over a matrix of GPUs, regions and parameters, and reports cost, wall time and throughput per cell. Parameters reach a Job as NODUS_PARAM_
object
object
object
object
object
object
object
SweepSpec describes a Sweep. Everything is immutable after create except maxParallel, maxCostUSD (raise only) and state.
object
SweepMatrix lists the values of each swept dimension. An empty dimension keeps the template’s value.
object
GPU overrides resources.gpu per cell, at most 16 values, in the same “H100:1” form.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Params maps Job environment names to strings, or TrainingJob parameter paths to JSON values encoded as strings. At most 16 names with 64 values each.
object
Regions narrows placement.regions to one region per cell; it may narrow, never widen, the template’s.
MaxCostUSD is one cap shared by every cell. Reaching it suspends every running cell with MaxCostReached; raising it resumes them.
MaxParallel is how many cells may run at once, 1 to 256; the default is 4.
Repetitions runs every combination this many times, 1 to 64; the default is 1.
State is the desired state: Running (the default), Suspended, or the cancel state. It propagates to the cells.
SweepTemplate is the object every cell creates.
object
Kind is the cell kind: Job or TrainingJob.
object
ActiveDeadlineSeconds is accepted as an alias of timeout on write and folded into it.
BackoffLimit is how many customer failures (non-zero exits, OOM) across all indexes are retried before the Job fails (0–100).
CompleteByTime is a finish-by hint for placement and checkpoint cadence. It never stops the Job.
Completions is how many indexes must succeed (1–10000). Each index gets NODUS_INDEX and JOB_COMPLETION_INDEX.
TrainingData is a dataset in a Volume.
object
Columns maps the runtime’s column roles (prompt, completion, chosen, rejected, text) to dataset columns.
object
Format is Auto (default), JSONL, JSON or CSV.
Path is a file or directory inside the Volume.
Revision pins a revision.
ValidationFraction is the share held out for validation, as a decimal string (default “0.05”).
Volume is the Volume holding the data.
Distributed runs the Job as a gang of nodes that start together and restart together (Beta).
object
GPUsPerNode is 1, 2, 4 or 8 and equals resources.gpu.count when both are set.
Launcher is Plain (env contract only), Torchrun, Ray, Verl, Accelerate or Deepspeed.
MaxAssemblyRetries is how many times a timed-out assembly is re-placed (0–5, default 2).
Network is Colocated (one provider and region), Regional (one region class) or Global.
Nodes is the gang size (2–16; the beta caps it at 8). Exactly one of nodes and totalGPUs.
StartupTimeout is the gang assembly budget (5m–60m, default 15m).
TotalGPUs is resolved into status.topology at admission (2–64).
Transport is Direct (private or direct paths only) or Auto (relayed paths allowed).
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
EnvironmentBinding selects an Environment and its task counts.
object
HeldOutTasks is the number of held-out tasks evaluated before and after training.
Name is nodus/
Seed selects the tasks deterministically (default 0).
TrainTasks is the number of training tasks.
Evaluation controls the baseline and final evaluation.
object
Baseline evaluates the model before training (default true).
Final evaluates the model after training (default true).
MinComparisonTasks is the smallest held-out set a comparison is reported on (default and minimum 16).
ExpectedDuration is the declared runtime the estimate uses first.
ExportTarget is a Volume the outputs are also written to.
object
Volume is the Volume.
GradingPolicy bounds grading.
object
MaxParallel is the most grading Sandboxes at once (default 4, at most 32).
InitCommand is a preflight run before the command.
object
Command is the argv.
Timeout is at most 30m.
Inputs are materialized read-only under /nodus/inputs/
JobInput is one input: exactly one of volume, output, bucket, url and fromWorkspace. It is mounted at /nodus/inputs/
object
BucketInput is an object in customer storage.
object
Connection is an S3 Connection that can read it.
SHA256 is checked at end of stream when set.
URI is s3://bucket/key.
StageInput names an output of a Pipeline stage.
object
Output is the output name.
Stage is the producing stage.
WorkspaceInput mounts a Workspace home.
object
Name is the Workspace.
Path is a subdirectory of the home.
Name matches ^[A-Za-z][A-Za-z0-9_]{0,63}$ and is unique among inputs.
OutputInput names a committed output of another Job.
object
Index selects an index of an Indexed Job (default 0).
Job is the producing Job.
Name is the output.
URLInput is a file downloaded over HTTPS.
object
SHA256 is the expected digest.
URL is https:// only.
VolumeInput names a Volume revision.
object
Name is the Volume.
Revision pins a revision.
MaxCostUSD caps what the Job may spend. Reaching it suspends the Job; it can only be raised.
Mode is Train (default) or Evaluate, and must be one of the runtime’s modes.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Network sets egress for the container.
object
Egress is the outbound network policy.
object
Allow lists DNS names or *.suffix patterns (AllowList only; at most 128).
Policy is Deny, AllowList or Open.
Presets add curated host lists such as python-packages or huggingface.
Outputs are files or directories collected when an index succeeds (at most 32). Everything under /nodus/outputs is also collected as the output named “outputs”.
JobOutput is a declared output.
object
FromRanks is Zero (rank 0 only) or All (every rank under rank-
Name matches ^[a-z0-9._-]{1,64}$; the nodus. prefix is reserved.
Path is an absolute file or directory path; directories are archived as tar.zst.
OutputSink loads an output into a customer database table.
object
Connection is a Postgres, Neon or Supabase Connection.
Mode is Append (default) or Replace.
Table is the destination table.
Parallelism is how many indexes run at once (1 to min(completions, 256)).
Parameters are validated against the runtime’s JSON Schema (at most 64 KiB) over the runtime defaults.
Placement constrains where work may run.
object
Dedicated is reserved; only true is accepted.
Interruptible is Never, Allow (the scheduler decides by expected cost) or Prefer.
MaxRateUSDPerHour filters out offerings above this customer rate.
Offerings restricts placement to these Offerings.
Pool is a BYOC Pool in the project.
Profile is Balanced, Cost or Speed.
QueueTimeout is how long the Job may wait for capacity before failing with CapacityUnavailable (default 24h; 0s waits forever).
Regions are region classes (us, eu, …); empty means any.
Recovery says how progress survives lost capacity.
object
CheckpointPolicy configures checkpoints.
object
Format is Snapshot, or Dcp for torch.distributed.checkpoint (the default for distributed Jobs).
Integration is Auto, None or HFTrainer.
Interval is auto or a duration from 1m to 6h.
MaxSize is checked before upload (default 1Ti, at most 2Ti).
Paths are saved together in one checkpoint (at most 64; default /nodus/state).
RetainAfterFinish keeps checkpoints this long after the Job finishes (default 168h).
Continuity is Checkpointed (restore the latest checkpoint), Restartable (progress cursor) or Ephemeral.
MaxAttempts is how many infrastructure replacements each index may use (1–32; gangs count restarts).
MinProgressDuration is the run time that counts as progress (default 2m).
OnInterruption is Resume, or Fail to fail instead of recovering or suspending.
Resources are capacity floors. For GPU work the offering’s host shape applies if larger.
object
CPU is the vCPU floor.
Disk is the ephemeral disk floor.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Memory is the host memory floor.
Nodes above 1 is shorthand for spec.distributed.nodes and is stored in that form.
Runtime is a Ready TrainingRuntime: nodus/
ScaledownWindow keeps the capacity warm for reuse this long after the Job ends (0s–20m).
SecretMount mounts one Secret, optionally pinned to a version. The bare name is accepted on write.
object
Name is the Secret.
Version pins a version; unset pins the latest at admission.
Sidecar is a helper container with native-sidecar semantics.
object
Command is the sidecar’s argv.
Env adds environment variables.
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
Image defaults to the Job’s image.
Name is unique within the Job.
StartupTimeout bounds how long the sidecar may take to become ready.
Source is the code for the container: exactly one of git and blob.
object
Blob is sha256:
GitSource is a repository checkout.
object
Path is the checkout path (default workingDir).
Ref is a branch, tag or commit.
Repo is owner/name or an https:// URL.
State is the desired state: Running (the default), Suspended, or the cancel state that stops the Job for good.
Task overrides the runtime’s task; Pretrain on sft trains on raw text with packing.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Timeout is the wall-clock limit counted from the first Provisioning, excluding time spent Suspended. The Job then fails with DeadlineExceeded.
Tracking names customer-owned trackers; they are never the source of truth for comparisons.
object
MLflowTracking is an MLflow server.
object
Experiment is the experiment name.
Secret holds MLFLOW_TRACKING_TOKEN or username and password.
URI is the tracking server (https://).
WandBTracking is a Weights & Biases project.
object
Connection is a WandB Connection.
TTLSecondsAfterFinished deletes the Job, its outputs and checkpoints this long after it finishes.
VolumeMount mounts a Volume.
object
MountPath is the absolute mount point; system paths are refused.
ReadOnly mounts it read-only.
Revision pins a revision; only with readOnly.
Volume names the Volume.
SweepStatus is what Nodus observed about a Sweep.
object
SweepBest points at the best succeeded cells.
object
ByCost is the index of the cheapest succeeded cell.
ByThroughput is the index of the succeeded cell with the most units per second.
Cells has one entry per cell, in index order.
SweepCell is one cell of the matrix and what it measured.
object
CostUSD is what the cell has cost.
GPU is the cell’s GPU request, when the matrix sweeps GPUs.
Index is the cell’s position; the cell Job is named
Object is the cell Job’s name.
Params are the cell’s parameter values.
object
Phase is the cell Job’s phase; Pending before the Job exists.
Region is the cell’s region, when the matrix sweeps regions.
Repetition counts from 0.
UnitsPerSecond is the throughput the program reported, as a decimal string.
WallSeconds is the cell’s wall time from start to completion.
CompletionTime is when the Sweep reached a terminal phase.
Conditions are the Sweep’s conditions.
object
Cost is status.cost on every kind that spends money. It is a display copy of the ledger: amounts are decimal USD strings, and nothing is charged from these fields.
object
CostSegments splits a charge by billing segment.
object
BootCoveredSeconds is the wall-clock time in Boot that Nodus paid for instead, counted like BootSeconds.
BootSeconds is the charged wall-clock time spent in Boot, with parallel allocations counted once; omitted when there was none.
BootUSD is the charge from the start of billing until the program started.
CoveredByNodus lists the segments (Boot, Running, Restore, Teardown) with time Nodus paid for instead of charging it, such as the start and teardown of a run that failed because of Nodus.
RestoreCoveredSeconds is the wall-clock time in Restore that Nodus paid for instead, counted like BootSeconds.
RestoreSeconds is the charged wall-clock time spent in Restore, counted like BootSeconds.
RestoreUSD is the charge for bringing a replacement up after capacity was lost.
RunningCoveredSeconds is the wall-clock time in Running that Nodus paid for instead, counted like BootSeconds.
RunningSeconds is the charged wall-clock time spent in Running, counted like BootSeconds.
RunningUSD is the charge while the program ran.
TeardownCoveredSeconds is the wall-clock time in Teardown that Nodus paid for instead, counted like BootSeconds.
TeardownSeconds is the charged wall-clock time from stop until deletion was confirmed, counted like BootSeconds; the billing increment adds to TeardownUSD and not to this.
TeardownUSD is the charge from stop until deletion was confirmed, with the billing increment.
Final is true once the final charge has posted.
FundedUntilTime is when the current holds stop paying for the object.
HeldUSD is what is currently held for it.
LimitUSD is the object’s maxCostUSD, if it has one.
RateUSDPerHour is the sum of the hourly rates of its capacity that is still billing.
TotalUSD is what has been charged for this object so far.
Estimate is returned in status.estimate by a dry-run (?dryRun=All) and on create for every compute kind (ADR-031). Amounts are decimal USD strings; the ETag of the dry-run response binds a later create to it.
object
AssemblyBoundUSD is the most a multi-node Job can cost if its members never assemble.
BlockingReasons say why a real create would fail now, with the amounts and a fix.
BlockingReason says why a create would fail now.
object
Code is the error code the create would return, for example InsufficientCredits.
Fix says what to do.
Message states the reason with the amounts involved.
ConcurrentHoldUSD is the total of the holds for every slot that starts at once.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
GangHoldUSD is the total of the member holds of a multi-node Job.
HoldUSD is the first hold that must fit your balance, every matching Budget and the object’s own cap.
MinimumChargeUSD is the minimum charge quoted by this estimate.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
PlacementPreview is the customer-safe description of the capacity a create would use.
object
Explain is the scheduler’s explanation, the same shape as Attempt status.placement.explain.
Interruptible reports whether the capacity can be reclaimed.
Offering is the offering id, for example h100-sxm-80g-x8-us.
RegionClass is the region class of the offering.
WarmReuse reports whether warm capacity would be reused.
PricebookVersion is the price book the estimate used.
RateUSDPerHour is the hourly rate of the chosen capacity.
StartupEstimate is the expected time to start.
object
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
Topology is the resolved multi-node topology of a gang Job.
ValidUntil is when the prices this estimate used stop being current. A create bound by If-Match is admitted after it too, and never placed above rateUSDPerHour plus 10 %.
Warnings are non-blocking notes, for example ColdStart or InterruptibleSelected.
EstimateWarning is a non-blocking note on an estimate.
object
Code is the CamelCase warning code.
Message explains the warning.
Message explains the phase in a sentence.
ObservedGeneration is the spec generation this status reflects.
Phase is the lifecycle phase, from the run-to-completion vocabulary.
Reason is the machine-readable reason for the phase, for example CellsFailed or MaxCostReached.
StartTime is when the first cell Job was created.
TemplateETags freeze the reviewed TrainingJob cells; child admission rejects changed dependencies.
Example generated
{ "apiVersion": "example", "kind": "example", "metadata": { "annotations": { "additionalProperty": "example" }, "creationTimestamp": "example", "deletionGracePeriodSeconds": 1, "deletionTimestamp": "example", "finalizers": [ "example" ], "generateName": "example", "generation": 1, "labels": { "additionalProperty": "example" }, "managedFields": [ { "apiVersion": "example", "fieldsType": "example", "fieldsV1": { "additionalProperty": "example" }, "manager": "example", "operation": "example", "subresource": "example", "time": "example" } ], "name": "example", "namespace": "example", "ownerReferences": [ { "apiVersion": "example", "blockOwnerDeletion": true, "controller": true, "kind": "example", "name": "example", "uid": "example" } ], "resourceVersion": "example", "selfLink": "example", "uid": "example" }, "spec": { "matrix": { "gpu": [ { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] } ], "params": { "additionalProperty": [ "example" ] }, "regions": [ "example" ] }, "maxCostUSD": "example", "maxParallel": 1, "repetitions": 1, "state": "example", "template": { "kind": "example", "spec": { "activeDeadlineSeconds": 1, "args": [ "example" ], "backoffLimit": 1, "command": [ "example" ], "completeByTime": "example", "completions": 1, "connections": [ "example" ], "data": { "columns": { "additionalProperty": "example" }, "format": "example", "path": "example", "revision": 1, "validationFraction": "example", "volume": "example" }, "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "environment": { "heldOutTasks": 1, "name": "example", "seed": 1, "trainTasks": 1 }, "evaluation": { "baseline": true, "final": true, "minComparisonTasks": 1 }, "expectedDuration": "example", "exportTo": { "volume": "example" }, "grading": { "maxParallel": 1 }, "image": "example", "imagePullSecrets": [ "example" ], "imageRef": "example", "initCommand": { "command": [ "example" ], "timeout": "example" }, "inputs": [ { "bucket": { "connection": "example", "sha256": "example", "uri": "example" }, "fromStage": { "output": "example", "stage": "example" }, "fromWorkspace": { "name": "example", "path": "example" }, "name": "example", "output": { "index": 1, "job": "example", "name": "example" }, "url": { "sha256": "example", "url": "example" }, "volume": { "name": "example", "revision": 1 } } ], "maxCostUSD": "example", "mode": "example", "model": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "network": { "egress": { "allow": [ "example" ], "policy": "example", "presets": [ "example" ] } }, "outputs": [ { "fromRanks": "example", "name": "example", "path": "example", "sink": { "connection": "example", "mode": "example", "table": "example" } } ], "parallelism": 1, "parameters": "example", "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "recovery": { "checkpoint": { "format": "example", "integration": "example", "interval": "example", "maxSize": "example", "paths": [ "example" ], "retainAfterFinish": "example" }, "continuity": "example", "maxAttempts": 1, "minProgressDuration": "example", "onInterruption": "example" }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "runtime": "example", "scaledownWindow": "example", "secrets": [ { "name": "example", "version": 1 } ], "serviceAccountName": "example", "sidecars": [ { "command": [ "example" ], "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "image": "example", "name": "example", "startupTimeout": "example" } ], "source": { "blob": "example", "git": { "path": "example", "ref": "example", "repo": "example" } }, "state": "example", "task": "example", "teacher": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "timeout": "example", "tracking": { "mlflow": { "experiment": "example", "secret": "example", "uri": "example" }, "wandb": { "connection": "example" } }, "ttlSecondsAfterFinished": 1, "volumes": [ { "mountPath": "example", "readOnly": true, "revision": 1, "volume": "example" } ], "workingDir": "example" } } }, "status": { "best": { "byCost": 1, "byThroughput": 1 }, "cells": [ { "costUSD": "example", "gpu": "example", "index": 1, "object": "example", "params": { "additionalProperty": "example" }, "phase": "example", "region": "example", "repetition": 1, "unitsPerSecond": "example", "wallSeconds": 1 } ], "completionTime": "example", "conditions": [ { "lastTransitionTime": "example", "message": "example", "observedGeneration": 1, "reason": "example", "status": "example", "type": "example" } ], "cost": { "bySegment": { "bootCoveredSeconds": 1, "bootSeconds": 1, "bootUSD": "example", "coveredByNodus": [ "example" ], "restoreCoveredSeconds": 1, "restoreSeconds": 1, "restoreUSD": "example", "runningCoveredSeconds": 1, "runningSeconds": 1, "runningUSD": "example", "teardownCoveredSeconds": 1, "teardownSeconds": 1, "teardownUSD": "example" }, "final": true, "fundedUntilTime": "example", "heldUSD": "example", "limitUSD": "example", "rateUSDPerHour": "example", "totalUSD": "example" }, "estimate": { "assemblyBoundUSD": "example", "blockingReasons": [ { "code": "example", "fix": "example", "message": "example" } ], "concurrentHoldUSD": "example", "costUSD": { "p50": "example", "p90": "example" }, "gangHoldUSD": "example", "holdUSD": "example", "minimumChargeUSD": "example", "overheadUSD": { "p50": "example", "p90": "example" }, "placementPreview": { "explain": "example", "interruptible": true, "offering": "example", "regionClass": "example", "warmReuse": true }, "pricebookVersion": "example", "rateUSDPerHour": "example", "startup": { "cold": { "p50": "example", "p90": "example" }, "warm": { "p50": "example", "p90": "example" } }, "topology": "example", "validUntil": "example", "warnings": [ { "code": "example", "message": "example" } ] }, "message": "example", "observedGeneration": 1, "phase": "example", "reason": "example", "startTime": "example", "templateETags": [ "example" ] }}Responses
Section titled “ Responses ”OK
Sweep fans a Job or TrainingJob template out over a matrix of GPUs, regions and parameters, and reports cost, wall time and throughput per cell. Parameters reach a Job as NODUS_PARAM_
object
object
object
object
object
object
object
SweepSpec describes a Sweep. Everything is immutable after create except maxParallel, maxCostUSD (raise only) and state.
object
SweepMatrix lists the values of each swept dimension. An empty dimension keeps the template’s value.
object
GPU overrides resources.gpu per cell, at most 16 values, in the same “H100:1” form.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Params maps Job environment names to strings, or TrainingJob parameter paths to JSON values encoded as strings. At most 16 names with 64 values each.
object
Regions narrows placement.regions to one region per cell; it may narrow, never widen, the template’s.
MaxCostUSD is one cap shared by every cell. Reaching it suspends every running cell with MaxCostReached; raising it resumes them.
MaxParallel is how many cells may run at once, 1 to 256; the default is 4.
Repetitions runs every combination this many times, 1 to 64; the default is 1.
State is the desired state: Running (the default), Suspended, or the cancel state. It propagates to the cells.
SweepTemplate is the object every cell creates.
object
Kind is the cell kind: Job or TrainingJob.
object
ActiveDeadlineSeconds is accepted as an alias of timeout on write and folded into it.
BackoffLimit is how many customer failures (non-zero exits, OOM) across all indexes are retried before the Job fails (0–100).
CompleteByTime is a finish-by hint for placement and checkpoint cadence. It never stops the Job.
Completions is how many indexes must succeed (1–10000). Each index gets NODUS_INDEX and JOB_COMPLETION_INDEX.
TrainingData is a dataset in a Volume.
object
Columns maps the runtime’s column roles (prompt, completion, chosen, rejected, text) to dataset columns.
object
Format is Auto (default), JSONL, JSON or CSV.
Path is a file or directory inside the Volume.
Revision pins a revision.
ValidationFraction is the share held out for validation, as a decimal string (default “0.05”).
Volume is the Volume holding the data.
Distributed runs the Job as a gang of nodes that start together and restart together (Beta).
object
GPUsPerNode is 1, 2, 4 or 8 and equals resources.gpu.count when both are set.
Launcher is Plain (env contract only), Torchrun, Ray, Verl, Accelerate or Deepspeed.
MaxAssemblyRetries is how many times a timed-out assembly is re-placed (0–5, default 2).
Network is Colocated (one provider and region), Regional (one region class) or Global.
Nodes is the gang size (2–16; the beta caps it at 8). Exactly one of nodes and totalGPUs.
StartupTimeout is the gang assembly budget (5m–60m, default 15m).
TotalGPUs is resolved into status.topology at admission (2–64).
Transport is Direct (private or direct paths only) or Auto (relayed paths allowed).
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
EnvironmentBinding selects an Environment and its task counts.
object
HeldOutTasks is the number of held-out tasks evaluated before and after training.
Name is nodus/
Seed selects the tasks deterministically (default 0).
TrainTasks is the number of training tasks.
Evaluation controls the baseline and final evaluation.
object
Baseline evaluates the model before training (default true).
Final evaluates the model after training (default true).
MinComparisonTasks is the smallest held-out set a comparison is reported on (default and minimum 16).
ExpectedDuration is the declared runtime the estimate uses first.
ExportTarget is a Volume the outputs are also written to.
object
Volume is the Volume.
GradingPolicy bounds grading.
object
MaxParallel is the most grading Sandboxes at once (default 4, at most 32).
InitCommand is a preflight run before the command.
object
Command is the argv.
Timeout is at most 30m.
Inputs are materialized read-only under /nodus/inputs/
JobInput is one input: exactly one of volume, output, bucket, url and fromWorkspace. It is mounted at /nodus/inputs/
object
BucketInput is an object in customer storage.
object
Connection is an S3 Connection that can read it.
SHA256 is checked at end of stream when set.
URI is s3://bucket/key.
StageInput names an output of a Pipeline stage.
object
Output is the output name.
Stage is the producing stage.
WorkspaceInput mounts a Workspace home.
object
Name is the Workspace.
Path is a subdirectory of the home.
Name matches ^[A-Za-z][A-Za-z0-9_]{0,63}$ and is unique among inputs.
OutputInput names a committed output of another Job.
object
Index selects an index of an Indexed Job (default 0).
Job is the producing Job.
Name is the output.
URLInput is a file downloaded over HTTPS.
object
SHA256 is the expected digest.
URL is https:// only.
VolumeInput names a Volume revision.
object
Name is the Volume.
Revision pins a revision.
MaxCostUSD caps what the Job may spend. Reaching it suspends the Job; it can only be raised.
Mode is Train (default) or Evaluate, and must be one of the runtime’s modes.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Network sets egress for the container.
object
Egress is the outbound network policy.
object
Allow lists DNS names or *.suffix patterns (AllowList only; at most 128).
Policy is Deny, AllowList or Open.
Presets add curated host lists such as python-packages or huggingface.
Outputs are files or directories collected when an index succeeds (at most 32). Everything under /nodus/outputs is also collected as the output named “outputs”.
JobOutput is a declared output.
object
FromRanks is Zero (rank 0 only) or All (every rank under rank-
Name matches ^[a-z0-9._-]{1,64}$; the nodus. prefix is reserved.
Path is an absolute file or directory path; directories are archived as tar.zst.
OutputSink loads an output into a customer database table.
object
Connection is a Postgres, Neon or Supabase Connection.
Mode is Append (default) or Replace.
Table is the destination table.
Parallelism is how many indexes run at once (1 to min(completions, 256)).
Parameters are validated against the runtime’s JSON Schema (at most 64 KiB) over the runtime defaults.
Placement constrains where work may run.
object
Dedicated is reserved; only true is accepted.
Interruptible is Never, Allow (the scheduler decides by expected cost) or Prefer.
MaxRateUSDPerHour filters out offerings above this customer rate.
Offerings restricts placement to these Offerings.
Pool is a BYOC Pool in the project.
Profile is Balanced, Cost or Speed.
QueueTimeout is how long the Job may wait for capacity before failing with CapacityUnavailable (default 24h; 0s waits forever).
Regions are region classes (us, eu, …); empty means any.
Recovery says how progress survives lost capacity.
object
CheckpointPolicy configures checkpoints.
object
Format is Snapshot, or Dcp for torch.distributed.checkpoint (the default for distributed Jobs).
Integration is Auto, None or HFTrainer.
Interval is auto or a duration from 1m to 6h.
MaxSize is checked before upload (default 1Ti, at most 2Ti).
Paths are saved together in one checkpoint (at most 64; default /nodus/state).
RetainAfterFinish keeps checkpoints this long after the Job finishes (default 168h).
Continuity is Checkpointed (restore the latest checkpoint), Restartable (progress cursor) or Ephemeral.
MaxAttempts is how many infrastructure replacements each index may use (1–32; gangs count restarts).
MinProgressDuration is the run time that counts as progress (default 2m).
OnInterruption is Resume, or Fail to fail instead of recovering or suspending.
Resources are capacity floors. For GPU work the offering’s host shape applies if larger.
object
CPU is the vCPU floor.
Disk is the ephemeral disk floor.
GPURequest asks for accelerators. A family (H100) matches any of its variants and never another family.
object
Count is the GPUs per node: 1, 2, 4 or 8.
Exact pins the exact variant instead of matching the family.
Interconnect is Any or NVLink.
MinMemory is the per-GPU memory floor.
Type lists 1–8 accelerator ids or families from the catalog.
Memory is the host memory floor.
Nodes above 1 is shorthand for spec.distributed.nodes and is stored in that form.
Runtime is a Ready TrainingRuntime: nodus/
ScaledownWindow keeps the capacity warm for reuse this long after the Job ends (0s–20m).
SecretMount mounts one Secret, optionally pinned to a version. The bare name is accepted on write.
object
Name is the Secret.
Version pins a version; unset pins the latest at admission.
Sidecar is a helper container with native-sidecar semantics.
object
Command is the sidecar’s argv.
Env adds environment variables.
EnvVar is one environment variable: a literal value or a Secret key.
object
Name matches ^[A-Za-z_][A-Za-z0-9_]*$.
Value is the literal value.
EnvVarSource names where an env value comes from.
object
SecretKeySelector selects one key of a Secret.
object
Key is the key within it.
Name is the Secret.
Image defaults to the Job’s image.
Name is unique within the Job.
StartupTimeout bounds how long the sidecar may take to become ready.
Source is the code for the container: exactly one of git and blob.
object
Blob is sha256:
GitSource is a repository checkout.
object
Path is the checkout path (default workingDir).
Ref is a branch, tag or commit.
Repo is owner/name or an https:// URL.
State is the desired state: Running (the default), Suspended, or the cancel state that stops the Job for good.
Task overrides the runtime’s task; Pretrain on sft trains on raw text with packing.
ModelSource is exactly one of uri and volume.
object
Secret names a Secret with HF_TOKEN for gated models.
URI is hf://
ModelVolume names a Volume revision.
object
Name is the Volume.
Revision pins a revision (latest Ready at admission when unset).
Timeout is the wall-clock limit counted from the first Provisioning, excluding time spent Suspended. The Job then fails with DeadlineExceeded.
Tracking names customer-owned trackers; they are never the source of truth for comparisons.
object
MLflowTracking is an MLflow server.
object
Experiment is the experiment name.
Secret holds MLFLOW_TRACKING_TOKEN or username and password.
URI is the tracking server (https://).
WandBTracking is a Weights & Biases project.
object
Connection is a WandB Connection.
TTLSecondsAfterFinished deletes the Job, its outputs and checkpoints this long after it finishes.
VolumeMount mounts a Volume.
object
MountPath is the absolute mount point; system paths are refused.
ReadOnly mounts it read-only.
Revision pins a revision; only with readOnly.
Volume names the Volume.
SweepStatus is what Nodus observed about a Sweep.
object
SweepBest points at the best succeeded cells.
object
ByCost is the index of the cheapest succeeded cell.
ByThroughput is the index of the succeeded cell with the most units per second.
Cells has one entry per cell, in index order.
SweepCell is one cell of the matrix and what it measured.
object
CostUSD is what the cell has cost.
GPU is the cell’s GPU request, when the matrix sweeps GPUs.
Index is the cell’s position; the cell Job is named
Object is the cell Job’s name.
Params are the cell’s parameter values.
object
Phase is the cell Job’s phase; Pending before the Job exists.
Region is the cell’s region, when the matrix sweeps regions.
Repetition counts from 0.
UnitsPerSecond is the throughput the program reported, as a decimal string.
WallSeconds is the cell’s wall time from start to completion.
CompletionTime is when the Sweep reached a terminal phase.
Conditions are the Sweep’s conditions.
object
Cost is status.cost on every kind that spends money. It is a display copy of the ledger: amounts are decimal USD strings, and nothing is charged from these fields.
object
CostSegments splits a charge by billing segment.
object
BootCoveredSeconds is the wall-clock time in Boot that Nodus paid for instead, counted like BootSeconds.
BootSeconds is the charged wall-clock time spent in Boot, with parallel allocations counted once; omitted when there was none.
BootUSD is the charge from the start of billing until the program started.
CoveredByNodus lists the segments (Boot, Running, Restore, Teardown) with time Nodus paid for instead of charging it, such as the start and teardown of a run that failed because of Nodus.
RestoreCoveredSeconds is the wall-clock time in Restore that Nodus paid for instead, counted like BootSeconds.
RestoreSeconds is the charged wall-clock time spent in Restore, counted like BootSeconds.
RestoreUSD is the charge for bringing a replacement up after capacity was lost.
RunningCoveredSeconds is the wall-clock time in Running that Nodus paid for instead, counted like BootSeconds.
RunningSeconds is the charged wall-clock time spent in Running, counted like BootSeconds.
RunningUSD is the charge while the program ran.
TeardownCoveredSeconds is the wall-clock time in Teardown that Nodus paid for instead, counted like BootSeconds.
TeardownSeconds is the charged wall-clock time from stop until deletion was confirmed, counted like BootSeconds; the billing increment adds to TeardownUSD and not to this.
TeardownUSD is the charge from stop until deletion was confirmed, with the billing increment.
Final is true once the final charge has posted.
FundedUntilTime is when the current holds stop paying for the object.
HeldUSD is what is currently held for it.
LimitUSD is the object’s maxCostUSD, if it has one.
RateUSDPerHour is the sum of the hourly rates of its capacity that is still billing.
TotalUSD is what has been charged for this object so far.
Estimate is returned in status.estimate by a dry-run (?dryRun=All) and on create for every compute kind (ADR-031). Amounts are decimal USD strings; the ETag of the dry-run response binds a later create to it.
object
AssemblyBoundUSD is the most a multi-node Job can cost if its members never assemble.
BlockingReasons say why a real create would fail now, with the amounts and a fix.
BlockingReason says why a create would fail now.
object
Code is the error code the create would return, for example InsufficientCredits.
Fix says what to do.
Message states the reason with the amounts involved.
ConcurrentHoldUSD is the total of the holds for every slot that starts at once.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
GangHoldUSD is the total of the member holds of a multi-node Job.
HoldUSD is the first hold that must fit your balance, every matching Budget and the object’s own cap.
MinimumChargeUSD is the minimum charge quoted by this estimate.
USDBand is a median and a 90th-percentile amount in USD.
object
P50 is the median amount.
P90 is the 90th-percentile amount.
PlacementPreview is the customer-safe description of the capacity a create would use.
object
Explain is the scheduler’s explanation, the same shape as Attempt status.placement.explain.
Interruptible reports whether the capacity can be reclaimed.
Offering is the offering id, for example h100-sxm-80g-x8-us.
RegionClass is the region class of the offering.
WarmReuse reports whether warm capacity would be reused.
PricebookVersion is the price book the estimate used.
RateUSDPerHour is the hourly rate of the chosen capacity.
StartupEstimate is the expected time to start.
object
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
DurationBand is a median and a 90th-percentile duration, as Go duration strings.
object
P50 is the median duration.
P90 is the 90th-percentile duration.
Topology is the resolved multi-node topology of a gang Job.
ValidUntil is when the prices this estimate used stop being current. A create bound by If-Match is admitted after it too, and never placed above rateUSDPerHour plus 10 %.
Warnings are non-blocking notes, for example ColdStart or InterruptibleSelected.
EstimateWarning is a non-blocking note on an estimate.
object
Code is the CamelCase warning code.
Message explains the warning.
Message explains the phase in a sentence.
ObservedGeneration is the spec generation this status reflects.
Phase is the lifecycle phase, from the run-to-completion vocabulary.
Reason is the machine-readable reason for the phase, for example CellsFailed or MaxCostReached.
StartTime is when the first cell Job was created.
TemplateETags freeze the reviewed TrainingJob cells; child admission rejects changed dependencies.
Example generated
{ "apiVersion": "example", "kind": "example", "metadata": { "annotations": { "additionalProperty": "example" }, "creationTimestamp": "example", "deletionGracePeriodSeconds": 1, "deletionTimestamp": "example", "finalizers": [ "example" ], "generateName": "example", "generation": 1, "labels": { "additionalProperty": "example" }, "managedFields": [ { "apiVersion": "example", "fieldsType": "example", "fieldsV1": { "additionalProperty": "example" }, "manager": "example", "operation": "example", "subresource": "example", "time": "example" } ], "name": "example", "namespace": "example", "ownerReferences": [ { "apiVersion": "example", "blockOwnerDeletion": true, "controller": true, "kind": "example", "name": "example", "uid": "example" } ], "resourceVersion": "example", "selfLink": "example", "uid": "example" }, "spec": { "matrix": { "gpu": [ { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] } ], "params": { "additionalProperty": [ "example" ] }, "regions": [ "example" ] }, "maxCostUSD": "example", "maxParallel": 1, "repetitions": 1, "state": "example", "template": { "kind": "example", "spec": { "activeDeadlineSeconds": 1, "args": [ "example" ], "backoffLimit": 1, "command": [ "example" ], "completeByTime": "example", "completions": 1, "connections": [ "example" ], "data": { "columns": { "additionalProperty": "example" }, "format": "example", "path": "example", "revision": 1, "validationFraction": "example", "volume": "example" }, "distributed": { "gpusPerNode": 1, "launcher": "example", "maxAssemblyRetries": 1, "network": "example", "nodes": 1, "startupTimeout": "example", "totalGPUs": 1, "transport": "example" }, "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "environment": { "heldOutTasks": 1, "name": "example", "seed": 1, "trainTasks": 1 }, "evaluation": { "baseline": true, "final": true, "minComparisonTasks": 1 }, "expectedDuration": "example", "exportTo": { "volume": "example" }, "grading": { "maxParallel": 1 }, "image": "example", "imagePullSecrets": [ "example" ], "imageRef": "example", "initCommand": { "command": [ "example" ], "timeout": "example" }, "inputs": [ { "bucket": { "connection": "example", "sha256": "example", "uri": "example" }, "fromStage": { "output": "example", "stage": "example" }, "fromWorkspace": { "name": "example", "path": "example" }, "name": "example", "output": { "index": 1, "job": "example", "name": "example" }, "url": { "sha256": "example", "url": "example" }, "volume": { "name": "example", "revision": 1 } } ], "maxCostUSD": "example", "mode": "example", "model": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "network": { "egress": { "allow": [ "example" ], "policy": "example", "presets": [ "example" ] } }, "outputs": [ { "fromRanks": "example", "name": "example", "path": "example", "sink": { "connection": "example", "mode": "example", "table": "example" } } ], "parallelism": 1, "parameters": "example", "placement": { "dedicated": true, "interruptible": "example", "maxRateUSDPerHour": "example", "offerings": [ "example" ], "pool": "example", "profile": "example", "queueTimeout": "example", "regions": [ "example" ] }, "recovery": { "checkpoint": { "format": "example", "integration": "example", "interval": "example", "maxSize": "example", "paths": [ "example" ], "retainAfterFinish": "example" }, "continuity": "example", "maxAttempts": 1, "minProgressDuration": "example", "onInterruption": "example" }, "resources": { "cpu": "example", "disk": "example", "gpu": { "count": 1, "exact": true, "interconnect": "example", "minMemory": "example", "type": [ "example" ] }, "memory": "example", "nodes": 1 }, "runtime": "example", "scaledownWindow": "example", "secrets": [ { "name": "example", "version": 1 } ], "serviceAccountName": "example", "sidecars": [ { "command": [ "example" ], "env": [ { "name": "example", "value": "example", "valueFrom": { "secretKeyRef": { "key": "example", "name": "example" } } } ], "image": "example", "name": "example", "startupTimeout": "example" } ], "source": { "blob": "example", "git": { "path": "example", "ref": "example", "repo": "example" } }, "state": "example", "task": "example", "teacher": { "secret": "example", "uri": "example", "volume": { "name": "example", "revision": 1 } }, "timeout": "example", "tracking": { "mlflow": { "experiment": "example", "secret": "example", "uri": "example" }, "wandb": { "connection": "example" } }, "ttlSecondsAfterFinished": 1, "volumes": [ { "mountPath": "example", "readOnly": true, "revision": 1, "volume": "example" } ], "workingDir": "example" } } }, "status": { "best": { "byCost": 1, "byThroughput": 1 }, "cells": [ { "costUSD": "example", "gpu": "example", "index": 1, "object": "example", "params": { "additionalProperty": "example" }, "phase": "example", "region": "example", "repetition": 1, "unitsPerSecond": "example", "wallSeconds": 1 } ], "completionTime": "example", "conditions": [ { "lastTransitionTime": "example", "message": "example", "observedGeneration": 1, "reason": "example", "status": "example", "type": "example" } ], "cost": { "bySegment": { "bootCoveredSeconds": 1, "bootSeconds": 1, "bootUSD": "example", "coveredByNodus": [ "example" ], "restoreCoveredSeconds": 1, "restoreSeconds": 1, "restoreUSD": "example", "runningCoveredSeconds": 1, "runningSeconds": 1, "runningUSD": "example", "teardownCoveredSeconds": 1, "teardownSeconds": 1, "teardownUSD": "example" }, "final": true, "fundedUntilTime": "example", "heldUSD": "example", "limitUSD": "example", "rateUSDPerHour": "example", "totalUSD": "example" }, "estimate": { "assemblyBoundUSD": "example", "blockingReasons": [ { "code": "example", "fix": "example", "message": "example" } ], "concurrentHoldUSD": "example", "costUSD": { "p50": "example", "p90": "example" }, "gangHoldUSD": "example", "holdUSD": "example", "minimumChargeUSD": "example", "overheadUSD": { "p50": "example", "p90": "example" }, "placementPreview": { "explain": "example", "interruptible": true, "offering": "example", "regionClass": "example", "warmReuse": true }, "pricebookVersion": "example", "rateUSDPerHour": "example", "startup": { "cold": { "p50": "example", "p90": "example" }, "warm": { "p50": "example", "p90": "example" } }, "topology": "example", "validUntil": "example", "warnings": [ { "code": "example", "message": "example" } ] }, "message": "example", "observedGeneration": 1, "phase": "example", "reason": "example", "startTime": "example", "templateETags": [ "example" ] }}default
Section titled “ default ”An error: a metav1.Status whose reason is a registered code.
Status is the error body of every API response: a Kubernetes metav1.Status (so kubectl and client-go understand it) plus three top-level extensions that those clients ignore (ADR-028).
object
object
object
Docs is the URL of the code’s docs page.
Fix says what to do next, for example a CLI command or the field to change.
object
object
RequestID identifies the request in logs and support tickets.
Example generated
{ "apiVersion": "example", "code": 1, "details": { "causes": [ { "field": "example", "message": "example", "reason": "example" } ], "group": "example", "kind": "example", "name": "example", "retryAfterSeconds": 1, "uid": "example" }, "docs": "example", "fix": "example", "kind": "example", "message": "example", "metadata": { "continue": "example", "remainingItemCount": 1, "resourceVersion": "example", "selfLink": "example", "shardInfo": { "selector": "example" } }, "reason": "example", "requestId": "example", "status": "example"}