--- updatedAt: 2026-08-06T09:44:04.000Z --- # Customizing a TensorRT Cloud Sweep Configuration Expand the `sweep_config` parameter below to explore the sweep configuration schema. # OpenAPI definition ```json { "openapi": "3.1.0", "info": { "title": "TensorRT Cloud API", "description": "TensorRT Cloud is a service for building and benchmarking TensorRT and TensorRT-LLM engines on the cloud. Users can submit builds (build and benchmark a single engine) as well as sweeps (build and benchmark multiple engines over a range of configurations to find the optimal build config).", "version": "1.0.0", "summary": "API for launching and managing TensorRT Cloud builds and sweeps." }, "servers": [ { "url": "https://api.ngc.nvidia.com" } ], "paths": { "/v1/trtc/orgs/{org}/sweeps": { "post": { "tags": [ "Sweep" ], "summary": "Customizing a TensorRT Cloud Sweep Configuration", "description": "Expand the `sweep_config` parameter below to explore the sweep configuration schema.", "operationId": "trigger_sweep_v1_trtc_orgs__org__sweeps_post", "parameters": [ { "description": "The NGC organization ID.", "required": true, "schema": { "type": "string", "title": "Org", "description": "The NGC organization ID." }, "name": "org", "in": "path" } ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SweepRequestBody" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": {} } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } } }, "components": { "schemas": { "BuildInput": { "properties": { "type": { "$ref": "#/components/schemas/FileType" }, "source": { "oneOf": [ { "$ref": "#/components/schemas/HFRepoInputSource" }, { "$ref": "#/components/schemas/URLInputSource" }, { "$ref": "#/components/schemas/NGCPrivateRegistryInputSource" } ], "title": "Source", "discriminator": { "propertyName": "source_type", "mapping": { "huggingface_repo": "#/components/schemas/HFRepoInputSource", "ngc_private_registry": "#/components/schemas/NGCPrivateRegistryInputSource", "url": "#/components/schemas/URLInputSource" } } } }, "additionalProperties": false, "type": "object", "required": [ "type", "source" ], "title": "BuildInput" }, "CustomRequestsProfile": { "properties": { "concurrency": { "anyOf": [ { "type": "integer", "exclusiveMinimum": 0 }, { "type": "string", "enum": [ "max_batch_size" ], "const": "max_batch_size" }, { "type": "null" } ], "title": "Concurrency" }, "request_rate": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Request Rate" }, "request_ratio": { "type": "integer", "title": "Request Ratio", "default": 3 }, "max_num_request": { "type": "integer", "title": "Max Num Request", "default": 2000 }, "input_tokens_mean": { "type": "integer", "minimum": 0, "title": "Input Tokens Mean", "default": 1024 }, "input_tokens_stdev": { "type": "integer", "minimum": 0, "title": "Input Tokens Stdev", "default": 0 }, "output_tokens_mean": { "type": "integer", "minimum": 0, "title": "Output Tokens Mean", "default": 512 }, "output_tokens_stdev": { "type": "integer", "minimum": 0, "title": "Output Tokens Stdev", "default": 0 }, "streaming": { "type": "boolean", "title": "Streaming", "default": true } }, "additionalProperties": false, "type": "object", "title": "CustomRequestsProfile", "description": "Same as `RequestsProfile` but overwrites some defaults,\naccording to historical values in planner" }, "DtypeEnum": { "type": "string", "enum": [ "float16", "bfloat16", "float32" ], "title": "DtypeEnum" }, "DynamicRuntimeConfig": { "properties": { "enable_chunked_context": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Enable Chunked Context" }, "scheduler_policy": { "anyOf": [ { "type": "string" }, { "items": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Scheduler Policy" }, "multi_block_mode": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Multi Block Mode" }, "enable_context_fmha_fp32_acc": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Enable Context Fmha Fp32 Acc" }, "kv_cache_free_gpu_mem_fraction": { "anyOf": [ { "type": "number" }, { "items": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Kv Cache Free Gpu Mem Fraction" }, "enable_kv_cache_reuse": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Enable Kv Cache Reuse" }, "cuda_graph_mode": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Cuda Graph Mode" }, "cuda_graph_cache_size": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Cuda Graph Cache Size" } }, "additionalProperties": false, "type": "object", "required": [ "enable_chunked_context", "scheduler_policy", "multi_block_mode", "enable_context_fmha_fp32_acc", "kv_cache_free_gpu_mem_fraction", "enable_kv_cache_reuse", "cuda_graph_mode", "cuda_graph_cache_size" ], "title": "DynamicRuntimeConfig", "description": "Defines part of the runtime configuration that we can do sweep over.\nDefaults for the fields are coming from `get_parent`" }, "FileType": { "type": "string", "enum": [ "onnx", "trt_engine", "trt_metrics", "timing_cache", "huggingface_checkpoint", "huggingface_checkpoint_weightless", "lora_huggingface_checkpoints", "tokenizer", "trtllm_checkpoint", "trtllm_checkpoint_config", "trtllm_engine", "trtllm_metrics", "system_info" ], "title": "FileType" }, "HFRepoInputSource": { "properties": { "source_type": { "type": "string", "enum": [ "huggingface_repo" ], "const": "huggingface_repo", "title": "Source Type", "default": "huggingface_repo" }, "id": { "type": "string", "title": "Id" }, "revision": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Revision" }, "token": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Token" }, "allowed_extensions": { "anyOf": [ { "items": { "type": "string", "pattern": "\\.[a-zA-Z0-9]+$" }, "type": "array" }, { "type": "null" } ], "title": "Allowed Extensions" } }, "additionalProperties": false, "type": "object", "required": [ "id", "source_type" ], "title": "HFRepoInputSource" }, "HTTPValidationError": { "properties": { "detail": { "items": { "$ref": "#/components/schemas/ValidationError" }, "type": "array", "title": "Detail" } }, "type": "object", "title": "HTTPValidationError" }, "HardwareConfig": { "properties": { "os": { "type": "string", "enum": [ "linux", "windows" ], "title": "Os", "default": "linux" }, "gpu": { "type": "string", "title": "Gpu", "known_gpus": [ "A10", "A100", "A100_PCIE", "A10G", "A2", "A30", "A40", "B200", "GTX1630", "GTX1650", "GTX1650SUPER", "GTX1650TI", "GTX1650TI_MAX_Q", "GTX1650_MAX_Q", "GTX1660", "GTX1660SUPER", "GTX1660TI", "GTX1660TI_MAX_Q", "H100", "H100_PCIE", "H200", "H200_PCIE", "H800", "L4", "L40", "L40S", "RTX2000_ADA", "RTX2050", "RTX2060", "RTX2060SUPER", "RTX2060_MAX_Q", "RTX2070", "RTX2070SUPER", "RTX2070SUPER_MAX_Q", "RTX2070_MAX_Q", "RTX2080", "RTX2080SUPER", "RTX2080SUPER_MAX_Q", "RTX2080TI", "RTX2080_MAX_Q", "RTX3050", "RTX3060", "RTX3060TI", "RTX3070", "RTX3070TI", "RTX3080", "RTX3080TI", "RTX3090", "RTX3090TI", "RTX4000_ADA", "RTX4000_SFF_ADA", "RTX4060", "RTX4060TI", "RTX4070", "RTX4070SUPER", "RTX4070TI", "RTX4070TISUPER", "RTX4080", "RTX4080SUPER", "RTX4090", "RTX4090_D", "RTX4500_ADA", "RTX5000_ADA", "RTX5070", "RTX5070TI", "RTX5080", "RTX5090", "RTX5880_ADA", "RTX6000_ADA", "RTXA1000", "RTXA2000", "RTXA400", "RTXA4000", "RTXA4500", "RTXA5000", "RTXA5500", "RTXA6000", "T4", "T600", "TITANRTX", "TITANV", "TITANV_CEOEDITION", "V100_PCIE_FHHL" ], "enum": [ "A10", "A100", "A100_PCIE", "A10G", "A2", "A30", "A40", "B200", "GTX1630", "GTX1650", "GTX1650SUPER", "GTX1650TI", "GTX1650TI_MAX_Q", "GTX1650_MAX_Q", "GTX1660", "GTX1660SUPER", "GTX1660TI", "GTX1660TI_MAX_Q", "H100", "H100_PCIE", "H200", "H200_PCIE", "H800", "L4", "L40", "L40S", "RTX2000_ADA", "RTX2050", "RTX2060", "RTX2060SUPER", "RTX2060_MAX_Q", "RTX2070", "RTX2070SUPER", "RTX2070SUPER_MAX_Q", "RTX2070_MAX_Q", "RTX2080", "RTX2080SUPER", "RTX2080SUPER_MAX_Q", "RTX2080TI", "RTX2080_MAX_Q", "RTX3050", "RTX3060", "RTX3060TI", "RTX3070", "RTX3070TI", "RTX3080", "RTX3080TI", "RTX3090", "RTX3090TI", "RTX4000_ADA", "RTX4000_SFF_ADA", "RTX4060", "RTX4060TI", "RTX4070", "RTX4070SUPER", "RTX4070TI", "RTX4070TISUPER", "RTX4080", "RTX4080SUPER", "RTX4090", "RTX4090_D", "RTX4500_ADA", "RTX5000_ADA", "RTX5070", "RTX5070TI", "RTX5080", "RTX5090", "RTX5880_ADA", "RTX6000_ADA", "RTXA1000", "RTXA2000", "RTXA400", "RTXA4000", "RTXA4500", "RTXA5000", "RTXA5500", "RTXA6000", "T4", "T600", "TITANRTX", "TITANV", "TITANV_CEOEDITION", "V100_PCIE_FHHL" ] } }, "additionalProperties": false, "type": "object", "required": [ "gpu" ], "title": "HardwareConfig", "description": "Configuration for the platform to run the sweep on,\nwhich includes the OS and the GPU." }, "MultiBenchmarkConfig": { "properties": { "run_summarize_py": { "type": "boolean", "title": "Run Summarize Py", "description": "Whether to run the summarize.py script after building the engine. This is useful to get a quick assessment of the engine's accuracy. It should not be used as a serious benchmark of the engine's accuracy, but rather as a quick sanity check.", "default": false }, "do_test_run": { "type": "boolean", "title": "Do Test Run", "description": "Whether to run a test completion on the built TRT LLM engine. This is useful as a smoke test to ensure that the engine is not producing garbage outputs.", "default": true }, "dynamic_runtime_config": { "$ref": "#/components/schemas/DynamicRuntimeConfig" }, "perf_configs": { "items": { "$ref": "#/components/schemas/PerfConfig" }, "type": "array", "title": "Perf Configs" } }, "additionalProperties": false, "type": "object", "required": [ "dynamic_runtime_config" ], "title": "MultiBenchmarkConfig", "description": "Configuration for running benchmarks on a TRT LLM engine.\nIt includes a sweepable runtime configuration as well as a list of\n(concurrency, ISL, OSL) profiles to optimize for.\nThe runtime configuration is sweepable, with the dynamic (sweepable) fields\ndefined in `dynamic_runtime_config`, and the static (non-sweepable) fields\ndefined inside each `perf_config` benchmark profile." }, "MultiDeviceConfig": { "properties": { "tp_size": { "type": "integer", "title": "Tp Size", "default": 1 }, "pp_size": { "type": "integer", "title": "Pp Size", "default": 1 }, "cp_size": { "type": "integer", "title": "Cp Size", "default": 1 } }, "additionalProperties": false, "type": "object", "title": "MultiDeviceConfig" }, "NGCArtifactType": { "type": "string", "enum": [ "model", "resource" ], "title": "NGCArtifactType" }, "NGCPrivateRegistryBuildOutput": { "properties": { "artifact_type": { "$ref": "#/components/schemas/NGCArtifactType", "default": "model" }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target" }, "file_patterns": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "File Patterns" }, "exclude_patterns": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Exclude Patterns" }, "nvapi_token": { "type": "string", "title": "Nvapi Token" }, "type": { "type": "string", "enum": [ "ngc_private_registry" ], "const": "ngc_private_registry", "title": "Type", "default": "ngc_private_registry" } }, "type": "object", "required": [ "nvapi_token", "type" ], "title": "NGCPrivateRegistryBuildOutput" }, "NGCPrivateRegistryInputSource": { "properties": { "artifact_type": { "$ref": "#/components/schemas/NGCArtifactType", "default": "model" }, "target": { "type": "string", "title": "Target" }, "file_patterns": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "File Patterns" }, "exclude_patterns": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Exclude Patterns" }, "nvapi_token": { "type": "string", "title": "Nvapi Token" }, "source_type": { "type": "string", "enum": [ "ngc_private_registry" ], "const": "ngc_private_registry", "title": "Source Type", "default": "ngc_private_registry" } }, "type": "object", "required": [ "target", "nvapi_token", "source_type" ], "title": "NGCPrivateRegistryInputSource" }, "PerfConfig": { "properties": { "static_runtime_config": { "$ref": "#/components/schemas/StaticRuntimeConfig", "default": "{\"benchmarking_tool\": \"gptManagerBenchmark\", \"batching_type\": null, \"beam_width\": null, \"warm_up\": null, \"eos_id\": null, \"pad_id\": null, \"max_tokens_in_paged_kvcache\": null, \"max_attention_window\": null, \"sink_token_len\": null, \"random_seed\": null, \"cross_kv_cache_fraction\": null, \"max_batch_size\": null, \"max_num_tokens\": null, \"enable_trt_overlap\": null, \"enable_exp_delays\": null, \"return_context_logits\": null, \"return_generation_logits\": null, \"first_batch_delay\": null, \"static_emulated_batch_size\": null, \"static_emulated_timeout\": null, \"log_level\": \"info\", \"log_iteration_data\": null, \"wait_sleep\": null, \"kv_host_cache_bytes\": null, \"kv_dont_onboard_blocks\": null, \"max_prompt_len\": null, \"gpu_weights_percent\": null, \"medusa_choices\": null, \"eagle_choices\": null, \"eagle_posterior_threshold\": null, \"temperature\": null, \"extra_gpt_manager_benchmark_args\": null}" }, "requests_config": { "$ref": "#/components/schemas/CustomRequestsProfile", "default": "{\"concurrency\": 1, \"request_rate\": null, \"request_ratio\": 3, \"max_num_request\": 2000, \"input_tokens_mean\": 1024, \"input_tokens_stdev\": 0, \"output_tokens_mean\": 512, \"output_tokens_stdev\": 0, \"streaming\": true}" } }, "type": "object", "title": "PerfConfig", "description": "Wraps `StaticRuntimeConfig` and `CustomRequestsProfile`.\nBenchmark config contains list of PerfConfig" }, "QuantizeFormat_FP8": { "properties": { "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "fp8" ], "const": "fp8", "title": "Qformat", "default": "fp8" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_FP8", "required": [ "qformat" ] }, "QuantizeFormat_FULL_PREC": { "properties": { "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "full_prec" ], "const": "full_prec", "title": "Qformat", "default": "full_prec" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_FULL_PREC", "required": [ "qformat" ] }, "QuantizeFormat_INT4_AWQ": { "properties": { "awq_block_size": { "type": "integer", "title": "Awq Block Size", "default": 128 }, "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "int4_awq" ], "const": "int4_awq", "title": "Qformat", "default": "int4_awq" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_INT4_AWQ", "required": [ "qformat" ] }, "QuantizeFormat_INT4_WO": { "properties": { "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "int4_wo" ], "const": "int4_wo", "title": "Qformat", "default": "int4_wo" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_INT4_WO", "required": [ "qformat" ] }, "QuantizeFormat_INT8_SQ": { "properties": { "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "int8_sq" ], "const": "int8_sq", "title": "Qformat", "default": "int8_sq" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_INT8_SQ", "required": [ "qformat" ] }, "QuantizeFormat_INT8_WO": { "properties": { "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "int8_wo" ], "const": "int8_wo", "title": "Qformat", "default": "int8_wo" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_INT8_WO", "required": [ "qformat" ] }, "QuantizeFormat_NVFP4": { "properties": { "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "nvfp4" ], "const": "nvfp4", "title": "Qformat", "default": "nvfp4" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_NVFP4", "required": [ "qformat" ] }, "QuantizeFormat_W4A8_AWQ": { "properties": { "awq_block_size": { "type": "integer", "title": "Awq Block Size", "default": 128 }, "kv_cache_dtype": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeKvCacheDtypeEnum" }, { "type": "null" } ] }, "qformat": { "type": "string", "enum": [ "w4a8_awq" ], "const": "w4a8_awq", "title": "Qformat", "default": "w4a8_awq" } }, "additionalProperties": false, "type": "object", "title": "QuantizeFormat_W4A8_AWQ", "required": [ "qformat" ] }, "QuantizeKvCacheDtypeEnum": { "type": "string", "enum": [ "int8", "fp8" ], "title": "QuantizeKvCacheDtypeEnum" }, "StaticRuntimeConfig": { "properties": { "benchmarking_tool": { "type": "string", "enum": [ "gptManagerBenchmark" ], "const": "gptManagerBenchmark", "title": "Benchmarking Tool", "default": "gptManagerBenchmark" }, "batching_type": { "enum": [ "IFB", "UIFB", "V1", null ], "title": "Batching Type" }, "beam_width": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Beam Width" }, "warm_up": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Warm Up" }, "eos_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Eos Id" }, "pad_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Pad Id" }, "max_tokens_in_paged_kvcache": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Tokens In Paged Kvcache" }, "max_attention_window": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Attention Window" }, "sink_token_len": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Sink Token Len" }, "random_seed": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Random Seed" }, "cross_kv_cache_fraction": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cross Kv Cache Fraction" }, "max_batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Batch Size" }, "max_num_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Num Tokens" }, "enable_trt_overlap": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Enable Trt Overlap" }, "enable_exp_delays": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Enable Exp Delays" }, "return_context_logits": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Return Context Logits" }, "return_generation_logits": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Return Generation Logits" }, "first_batch_delay": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "First Batch Delay" }, "static_emulated_batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Static Emulated Batch Size" }, "static_emulated_timeout": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Static Emulated Timeout" }, "log_level": { "type": "string", "enum": [ "verbose", "info", "warning", "error", "internal_error" ], "title": "Log Level", "default": "info" }, "log_iteration_data": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Log Iteration Data" }, "wait_sleep": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Wait Sleep" }, "kv_host_cache_bytes": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Kv Host Cache Bytes" }, "kv_dont_onboard_blocks": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Kv Dont Onboard Blocks" }, "max_prompt_len": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Prompt Len" }, "gpu_weights_percent": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Gpu Weights Percent" }, "medusa_choices": { "anyOf": [ { "items": { "items": { "type": "integer" }, "type": "array" }, "type": "array" }, { "type": "null" } ], "title": "Medusa Choices" }, "eagle_choices": { "anyOf": [ { "items": { "items": { "type": "integer" }, "type": "array" }, "type": "array" }, { "type": "null" } ], "title": "Eagle Choices" }, "eagle_posterior_threshold": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Eagle Posterior Threshold" }, "temperature": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Temperature" }, "extra_gpt_manager_benchmark_args": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Extra Gpt Manager Benchmark Args" } }, "additionalProperties": false, "type": "object", "title": "StaticRuntimeConfig", "description": "This is a non-tunable part of runtime configuration.\nCombined with `DynamicRuntimeConfig` it composes the\nentire GptManagerBenchmarkRuntimeConfig.\nFields that are included into `DynamicRuntimeConfig`,\nare excluded from `StaticRuntimeConfig`.", "required": [ "benchmarking_tool" ] }, "SweepRequestBody": { "properties": { "sweep_config": { "$ref": "#/components/schemas/TRTLLMSweepConfig", "description": "The TRTLLM sweep configuration." } }, "additionalProperties": false, "type": "object", "required": [ "sweep_config" ], "title": "SweepRequestBody", "description": "Request body schema for submitting a sweep." }, "TRTLLMBuildSearchSpaceV017": { "properties": { "trtllm_version": { "type": "string", "enum": [ "0.17.0" ], "const": "0.17.0", "title": "Trtllm Version", "default": "0.17.0" }, "trtllm_model_family": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trtllm Model Family", "examples": [ "gemma", "gpt", "llama", "nemotron_nas", "phi", "qwen" ] }, "recipe_name": { "type": "string", "enum": [ "trtllm_trtcloud" ], "const": "trtllm_trtcloud", "title": "Recipe Name", "default": "trtllm_trtcloud" }, "dtype": { "anyOf": [ { "$ref": "#/components/schemas/DtypeEnum" }, { "items": { "anyOf": [ { "$ref": "#/components/schemas/DtypeEnum" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Dtype" }, "quantization": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeFormat_FP8" }, { "$ref": "#/components/schemas/QuantizeFormat_NVFP4" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_W4A8_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_FULL_PREC" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_SQ" }, { "items": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeFormat_FP8" }, { "$ref": "#/components/schemas/QuantizeFormat_NVFP4" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_W4A8_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_FULL_PREC" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_SQ" } ] }, "type": "array" } ], "title": "Quantization", "default": "{\"kv_cache_dtype\": null, \"qformat\": \"full_prec\"}" }, "multidevice": { "anyOf": [ { "$ref": "#/components/schemas/MultiDeviceConfig" }, { "items": { "$ref": "#/components/schemas/MultiDeviceConfig" }, "type": "array" } ], "title": "Multidevice", "default": "{\"tp_size\": 1, \"pp_size\": 1, \"cp_size\": 1}" }, "bert_attention_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Bert Attention Plugin" }, "bert_context_fmha_fp32_acc": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Bert Context Fmha Fp32 Acc" }, "context_fmha": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Context Fmha" }, "enable_debug_output": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Enable Debug Output" }, "fast_build": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Fast Build" }, "fp8_rowwise_gemm_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Fp8 Rowwise Gemm Plugin" }, "gather_all_token_logits": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gather All Token Logits" }, "gather_context_logits": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gather Context Logits" }, "gather_generation_logits": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gather Generation Logits" }, "gemm_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "fp8", "nvfp4", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "fp8", "nvfp4", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gemm Plugin" }, "gemm_swiglu_plugin": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gemm Swiglu Plugin" }, "gpt_attention_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gpt Attention Plugin" }, "logits_dtype": { "anyOf": [ { "type": "string", "enum": [ "float16", "float32" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "float16", "float32" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Logits Dtype" }, "lora_ckpt_source": { "anyOf": [ { "type": "string", "enum": [ "hf", "nemo" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "hf", "nemo" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Lora Ckpt Source" }, "lora_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Lora Plugin" }, "lora_target_modules": { "anyOf": [ { "items": { "type": "string", "enum": [ "attn_qkv", "attn_q", "attn_k", "attn_v", "attn_dense", "mlp_h_to_4h", "mlp_4h_to_h", "mlp_gate", "cross_attn_qkv", "cross_attn_q", "cross_attn_k", "cross_attn_v", "cross_attn_dense", "moe_h_to_4h", "moe_4h_to_h", "moe_gate", "moe_router", "mlp_router" ] }, "type": "array" }, { "items": { "anyOf": [ { "items": { "type": "string", "enum": [ "attn_qkv", "attn_q", "attn_k", "attn_v", "attn_dense", "mlp_h_to_4h", "mlp_4h_to_h", "mlp_gate", "cross_attn_qkv", "cross_attn_q", "cross_attn_k", "cross_attn_v", "cross_attn_dense", "moe_h_to_4h", "moe_4h_to_h", "moe_gate", "moe_router", "mlp_router" ] }, "type": "array" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Lora Target Modules" }, "low_latency_gemm_plugin": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Low Latency Gemm Plugin" }, "low_latency_gemm_swiglu_plugin": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Low Latency Gemm Swiglu Plugin" }, "mamba_conv1d_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Mamba Conv1D Plugin" }, "max_batch_size": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Batch Size" }, "max_beam_width": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Beam Width" }, "max_draft_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Draft Len" }, "max_encoder_input_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Encoder Input Len" }, "max_input_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Input Len" }, "max_lora_rank": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Lora Rank" }, "max_num_tokens": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Num Tokens" }, "max_prompt_embedding_table_size": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Prompt Embedding Table Size" }, "max_seq_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Seq Len" }, "moe_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Moe Plugin" }, "monitor_memory": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Monitor Memory" }, "multiple_profiles": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Multiple Profiles" }, "nccl_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Nccl Plugin" }, "opt_num_tokens": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Opt Num Tokens" }, "paged_state": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Paged State" }, "pp_reduce_scatter": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Pp Reduce Scatter" }, "profiling_verbosity": { "anyOf": [ { "type": "string", "enum": [ "layer_names_only", "detailed", "none" ] }, { "type": "null" } ], "title": "Profiling Verbosity" }, "reduce_fusion": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Reduce Fusion" }, "remove_input_padding": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Remove Input Padding" }, "speculative_decoding_mode": { "anyOf": [ { "type": "string", "enum": [ "draft_tokens_external", "lookahead_decoding", "medusa", "explicit_draft_tokens", "eagle" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "draft_tokens_external", "lookahead_decoding", "medusa", "explicit_draft_tokens", "eagle" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Speculative Decoding Mode" }, "streamingllm": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Streamingllm" }, "strip_plan": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Strip Plan" }, "tokens_per_block": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Tokens Per Block" }, "use_fp8_context_fmha": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Use Fp8 Context Fmha" }, "use_fused_mlp": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Use Fused Mlp" }, "use_paged_context_fmha": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Use Paged Context Fmha" }, "user_buffer": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "User Buffer" }, "weight_sparsity": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Weight Sparsity" }, "weight_streaming": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Weight Streaming" } }, "additionalProperties": false, "type": "object", "title": "TRTLLMBuildSearchSpaceV017", "description": "Search space model for the TRTLLM 0.17.0 build recipe.", "required": [ "trtllm_version", "recipe_name" ] }, "TRTLLMBuildSearchSpaceV018": { "properties": { "trtllm_version": { "type": "string", "enum": [ "0.18.2" ], "const": "0.18.2", "title": "Trtllm Version", "default": "0.18.2" }, "trtllm_model_family": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trtllm Model Family", "examples": [ "gemma", "gpt", "llama", "nemotron_nas", "phi", "qwen" ] }, "recipe_name": { "type": "string", "enum": [ "trtllm_trtcloud" ], "const": "trtllm_trtcloud", "title": "Recipe Name", "default": "trtllm_trtcloud" }, "dtype": { "anyOf": [ { "$ref": "#/components/schemas/DtypeEnum" }, { "items": { "anyOf": [ { "$ref": "#/components/schemas/DtypeEnum" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Dtype" }, "quantization": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeFormat_FP8" }, { "$ref": "#/components/schemas/QuantizeFormat_NVFP4" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_W4A8_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_FULL_PREC" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_SQ" }, { "items": { "anyOf": [ { "$ref": "#/components/schemas/QuantizeFormat_FP8" }, { "$ref": "#/components/schemas/QuantizeFormat_NVFP4" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_W4A8_AWQ" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_INT4_WO" }, { "$ref": "#/components/schemas/QuantizeFormat_FULL_PREC" }, { "$ref": "#/components/schemas/QuantizeFormat_INT8_SQ" } ] }, "type": "array" } ], "title": "Quantization", "default": "{\"kv_cache_dtype\": null, \"qformat\": \"full_prec\"}" }, "multidevice": { "anyOf": [ { "$ref": "#/components/schemas/MultiDeviceConfig" }, { "items": { "$ref": "#/components/schemas/MultiDeviceConfig" }, "type": "array" } ], "title": "Multidevice", "default": "{\"tp_size\": 1, \"pp_size\": 1, \"cp_size\": 1}" }, "bert_attention_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Bert Attention Plugin" }, "bert_context_fmha_fp32_acc": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Bert Context Fmha Fp32 Acc" }, "context_fmha": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Context Fmha" }, "enable_debug_output": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Enable Debug Output" }, "fast_build": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Fast Build" }, "fp8_rowwise_gemm_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Fp8 Rowwise Gemm Plugin" }, "gather_all_token_logits": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gather All Token Logits" }, "gather_context_logits": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gather Context Logits" }, "gather_generation_logits": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gather Generation Logits" }, "gemm_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "fp8", "nvfp4", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "fp8", "nvfp4", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gemm Plugin" }, "gemm_swiglu_plugin": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gemm Swiglu Plugin" }, "gpt_attention_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Gpt Attention Plugin" }, "logits_dtype": { "anyOf": [ { "type": "string", "enum": [ "float16", "float32" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "float16", "float32" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Logits Dtype" }, "lora_ckpt_source": { "anyOf": [ { "type": "string", "enum": [ "hf", "nemo" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "hf", "nemo" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Lora Ckpt Source" }, "lora_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Lora Plugin" }, "lora_target_modules": { "anyOf": [ { "items": { "type": "string", "enum": [ "attn_qkv", "attn_q", "attn_k", "attn_v", "attn_dense", "mlp_h_to_4h", "mlp_4h_to_h", "mlp_gate", "cross_attn_qkv", "cross_attn_q", "cross_attn_k", "cross_attn_v", "cross_attn_dense", "moe_h_to_4h", "moe_4h_to_h", "moe_gate", "moe_router", "mlp_router" ] }, "type": "array" }, { "items": { "anyOf": [ { "items": { "type": "string", "enum": [ "attn_qkv", "attn_q", "attn_k", "attn_v", "attn_dense", "mlp_h_to_4h", "mlp_4h_to_h", "mlp_gate", "cross_attn_qkv", "cross_attn_q", "cross_attn_k", "cross_attn_v", "cross_attn_dense", "moe_h_to_4h", "moe_4h_to_h", "moe_gate", "moe_router", "mlp_router" ] }, "type": "array" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Lora Target Modules" }, "low_latency_gemm_plugin": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Low Latency Gemm Plugin" }, "low_latency_gemm_swiglu_plugin": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "fp8", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Low Latency Gemm Swiglu Plugin" }, "mamba_conv1d_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Mamba Conv1D Plugin" }, "max_batch_size": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Batch Size" }, "max_beam_width": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Beam Width" }, "max_draft_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Draft Len" }, "max_encoder_input_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Encoder Input Len" }, "max_input_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Input Len" }, "max_lora_rank": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Lora Rank" }, "max_num_tokens": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Num Tokens" }, "max_prompt_embedding_table_size": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Prompt Embedding Table Size" }, "max_seq_len": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Max Seq Len" }, "moe_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Moe Plugin" }, "monitor_memory": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Monitor Memory" }, "multiple_profiles": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Multiple Profiles" }, "nccl_plugin": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "auto", "float16", "float32", "bfloat16", "int32", "disable" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Nccl Plugin" }, "opt_num_tokens": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Opt Num Tokens" }, "paged_state": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Paged State" }, "pp_reduce_scatter": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Pp Reduce Scatter" }, "profiling_verbosity": { "anyOf": [ { "type": "string", "enum": [ "layer_names_only", "detailed", "none" ] }, { "type": "null" } ], "title": "Profiling Verbosity" }, "reduce_fusion": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Reduce Fusion" }, "remove_input_padding": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Remove Input Padding" }, "speculative_decoding_mode": { "anyOf": [ { "type": "string", "enum": [ "draft_tokens_external", "lookahead_decoding", "medusa", "explicit_draft_tokens", "eagle" ] }, { "items": { "anyOf": [ { "type": "string", "enum": [ "draft_tokens_external", "lookahead_decoding", "medusa", "explicit_draft_tokens", "eagle" ] }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Speculative Decoding Mode" }, "streamingllm": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Streamingllm" }, "strip_plan": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Strip Plan" }, "tokens_per_block": { "anyOf": [ { "type": "integer" }, { "items": { "anyOf": [ { "type": "integer" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Tokens Per Block" }, "use_fp8_context_fmha": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Use Fp8 Context Fmha" }, "use_fused_mlp": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Use Fused Mlp" }, "use_paged_context_fmha": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Use Paged Context Fmha" }, "user_buffer": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "User Buffer" }, "weight_sparsity": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Weight Sparsity" }, "weight_streaming": { "anyOf": [ { "type": "boolean" }, { "items": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ] }, "type": "array" }, { "type": "null" } ], "title": "Weight Streaming" } }, "additionalProperties": false, "type": "object", "title": "TRTLLMBuildSearchSpaceV018", "description": "Search space model for the TRTLLM 0.18.2 build recipe.", "required": [ "trtllm_version", "recipe_name" ] }, "TRTLLMSearchStrategyConfig": { "properties": { "name": { "type": "string", "enum": [ "grid", "random", "vizier", "heuristic" ], "title": "Name", "description": "\n The name of the search strategy to use.\n * `grid` - Iterate over all possible configurations one by one.\n * `random` - Randomly sample configurations.\n * `vizier` - Sample using Google's 'Vizier' black-box optimizer.\n * `heuristic` - Sample using TRTLLM-specific heuristics.\n ", "default": "random" }, "max_trials": { "type": "integer", "title": "Max Trials", "description": "The maximum number of trial builds to run in the sweep.", "default": 32 }, "batch_size": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Batch Size", "description": "Number of trials to run in parallel. If not provided, defaults to min(16, max_trials). When name='vizier', use batch_size <= (max_trials + 15 - 1) // 15)." }, "use_priority_trials": { "type": "boolean", "title": "Use Priority Trials", "description": "Whether to run priority trials. Priority trials use set trial configs which are not chosen by the sampler. The default TRTLLM build config is run as a priority trial.", "default": true }, "optimization_objective": { "type": "string", "enum": [ "throughput", "latency" ], "title": "Optimization Objective", "description": "\n The objective to optimize for during search. For each quantization:\n * `throughput` - selects a single engine with minimum tp * pp to maximize tokens/second\n * `latency` - selects a single engine with maximum tp * pp to minimize latency\n ", "default": "throughput" }, "model_architecture": { "anyOf": [ { "type": "string", "enum": [ "nvidia/mistral-nemo-12b", "meta-llama/llama-2-7b", "meta-llama/codellama-7b-hf", "meta-llama/codellama-7b-python-hf", "meta-llama/llama-2-13b", "meta-llama/codellama-13b-hf", "meta-llama/codellama-13b-python-hf", "meta-llama/llama-2-34b", "meta-llama/llama-2-70b", "meta-llama/codellama-70b-hf", "meta-llama/codellama-70b-instruct-hf", "meta-llama/meta-llama-3-8b", "meta-llama/meta-llama-3-70b", "meta-llama/llama-3.1-8b", "meta-llama/llama-3.1-70b", "meta-llama/llama-3.2-1b", "meta-llama/llama-3.2-3b", "meta-llama/llama-3.3-70b", "google/gemma-2b", "google/gemma-7b", "google/gemma-2-2b", "google/gemma-2-9b", "google/gemma-2-27b", "mistralai/mistral-7b-v0.1", "mistralai/mistral-7b-v0.2", "mistralai/mistral-7b-v0.3", "mistralai/mixtral-8x7b-v0.1", "mistralai/mixtral-8x22b-v0.1", "mistralai/mixtral-8x22b-instruct-v0.1", "microsoft/phi-2", "bigcode/starcoder2-7b", "qwen/qwen2.5-0.5b", "qwen/qwen2.5-1.5b", "qwen/qwen2.5-1.5b-instruct", "qwen/qwen2.5-3b", "qwen/qwen2.5-7b", "qwen/qwen2.5-7b-instruct", "qwen/qwen2.5-14b", "qwen/qwen2.5-14b-instruct", "qwen/qwen2.5-32b", "qwen/qwen2.5-32b-instruct", "qwen/qwen2.5-72b", "qwen/qwen2.5-72b-instruct" ] }, { "type": "null" } ], "title": "Model Architecture", "description": "The model architecture of the model to optimize. Only needs to be provided when using the `heuristic` sampler and the build input is not given as a Hugging Face repo id.", "known_architectures": [ "nvidia/mistral-nemo-12b", "meta-llama/llama-2-7b", "meta-llama/codellama-7b-hf", "meta-llama/codellama-7b-python-hf", "meta-llama/llama-2-13b", "meta-llama/codellama-13b-hf", "meta-llama/codellama-13b-python-hf", "meta-llama/llama-2-34b", "meta-llama/llama-2-70b", "meta-llama/codellama-70b-hf", "meta-llama/codellama-70b-instruct-hf", "meta-llama/meta-llama-3-8b", "meta-llama/meta-llama-3-70b", "meta-llama/llama-3.1-8b", "meta-llama/llama-3.1-70b", "meta-llama/llama-3.2-1b", "meta-llama/llama-3.2-3b", "meta-llama/llama-3.3-70b", "google/gemma-2b", "google/gemma-7b", "google/gemma-2-2b", "google/gemma-2-9b", "google/gemma-2-27b", "mistralai/mistral-7b-v0.1", "mistralai/mistral-7b-v0.2", "mistralai/mistral-7b-v0.3", "mistralai/mixtral-8x7b-v0.1", "mistralai/mixtral-8x22b-v0.1", "mistralai/mixtral-8x22b-instruct-v0.1", "microsoft/phi-2", "bigcode/starcoder2-7b", "qwen/qwen2.5-0.5b", "qwen/qwen2.5-1.5b", "qwen/qwen2.5-1.5b-instruct", "qwen/qwen2.5-3b", "qwen/qwen2.5-7b", "qwen/qwen2.5-7b-instruct", "qwen/qwen2.5-14b", "qwen/qwen2.5-14b-instruct", "qwen/qwen2.5-32b", "qwen/qwen2.5-32b-instruct", "qwen/qwen2.5-72b", "qwen/qwen2.5-72b-instruct" ] } }, "additionalProperties": false, "type": "object", "title": "TRTLLMSearchStrategyConfig" }, "TRTLLMSweepConfig": { "properties": { "search_strategy": { "$ref": "#/components/schemas/TRTLLMSearchStrategyConfig", "description": "The configuration for the search strategy to use for the sweep. This includes the sampler configuration, the optimization objective, and any additional information to provide to the sampler." }, "build_inputs": { "items": { "$ref": "#/components/schemas/BuildInput" }, "type": "array", "title": "Build Inputs", "description": "The input model to optimize. The simplest way to provide an input is to provide a `huggingface_checkpoint` sourced from a Hugging Face repo. The input model can also be sourced from the user's NGC Private Registry, or from a custom URL. For more advanced use cases, please see the TensorRT Cloud User Guide." }, "build_output": { "title": "Build Output", "description": "Location to upload the completed sweep results.", "$ref": "#/components/schemas/NGCPrivateRegistryBuildOutput" }, "benchmark": { "$ref": "#/components/schemas/MultiBenchmarkConfig" }, "hardware": { "$ref": "#/components/schemas/HardwareConfig" }, "trtllm_build": { "oneOf": [ { "$ref": "#/components/schemas/TRTLLMBuildSearchSpaceV017" }, { "$ref": "#/components/schemas/TRTLLMBuildSearchSpaceV018" } ], "title": "Trtllm Build", "description": "The configuration for the TRT LLM engine build to use for the sweep. This includes the TRT LLM version, quantization mode(s), multi-gpu settings, and trtllm-build parameters. A field in the `trtllm_build` config can be swept over by providing an array of values instead of a single value.", "discriminator": { "propertyName": "trtllm_version", "mapping": { "0.17.0": "#/components/schemas/TRTLLMBuildSearchSpaceV017", "0.18.2": "#/components/schemas/TRTLLMBuildSearchSpaceV018" } } } }, "additionalProperties": false, "type": "object", "required": [ "build_inputs", "benchmark", "hardware", "trtllm_build" ], "title": "TRTLLMSweepConfig" }, "URLInputSource": { "properties": { "source_type": { "type": "string", "enum": [ "url" ], "const": "url", "title": "Source Type", "default": "url" }, "url": { "type": "string", "title": "Url" } }, "additionalProperties": false, "type": "object", "required": [ "url", "source_type" ], "title": "URLInputSource" }, "ValidationError": { "properties": { "loc": { "items": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "type": "array", "title": "Location" }, "type": { "type": "string", "title": "Error Type" } }, "type": "object", "required": [ "loc", "msg", "type" ], "title": "ValidationError" } }, "securitySchemes": { "nvapi_token": { "type": "http", "description": "nvapi token", "scheme": "bearer" } } }, "security": [ { "nvapi_token": [] } ], "x-readme": { "explorer-enabled": true, "proxy-enabled": true } } ```