---
updatedAt: 2026-08-06T10:00:33.000Z
---

# Request response from the model

Invokes inference using the model chat parameters. If uploading large images, this POST should be used in conjunction with the NVCF API which allows for the upload of large assets. 
You can find details on how to use NVCF Asset APIs here: https://docs.api.nvidia.com/cloud-functions/reference/createasset

# OpenAPI definition

```json
{
  "openapi": "3.1.0",
  "info": {
    "title": "NVIDIA NIM API for nvidia/vila",
    "description": "The NVIDIA NIM REST API. Please see https://docs.api.nvidia.com/nim for more details.",
    "version": "1.0.0",
    "termsOfService": "https://nvidia.com/legal/terms-of-use",
    "contact": {
      "name": "NVIDIA Support",
      "url": "https://help.nvidia.com/"
    },
    "license": {
      "name": "NVIDIA AI Foundation Models Community License",
      "url": "https://docs.nvidia.com/ai-foundation-models-community-license.pdf"
    }
  },
  "servers": [
    {
      "url": "https://ai.api.nvidia.com/v1"
    }
  ],
  "tags": [
    {
      "name": "Multimodal API",
      "description": "This API performs inference using visual language understanding models"
    }
  ],
  "paths": {
    "/vlm/nvidia/vila": {
      "post": {
        "tags": [
          "Multimodal API"
        ],
        "summary": "Request response from the model",
        "description": "Invokes inference using the model chat parameters. If uploading large images, this POST should be used in conjunction with the NVCF API which allows for the upload of large assets. \nYou can find details on how to use NVCF Asset APIs here: https://docs.api.nvidia.com/cloud-functions/reference/createasset",
        "operationId": "invokeFunction",
        "parameters": [
          {
            "in": "header",
            "name": "NVCF-INPUT-ASSET-REFERENCES",
            "schema": {
              "type": "string",
              "maxLength": 370,
              "format": "uuid"
            },
            "required": false,
            "description": "String of asset IDs separated by commas. Data is uploaded to AWS S3 using NVCF Asset APIs and associated with these asset IDs.If the size of an image is more than 180KB, it needs to be uploaded to a presigned S3 URL bucket. The presigned URL allows for secure and temporary access to the S3 bucket for uploading the image. Once the asset is requested, an asset ID is generated for it. Please include this asset ID in this header and to use the uploaded image in a prompt, you need to refer to it using the following format: `<img src=\"data:image/png;asset_id,{asset_id}\" />`."
          }
        ],
        "requestBody": {
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/NIMLLMChatCompletionRequest"
              }
            }
          },
          "required": true
        },
        "responses": {
          "200": {
            "description": "Invocation is fulfilled",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/NIMLLMChatCompletionResponse"
                }
              },
              "text/event-stream": {
                "schema": {
                  "$ref": "#/components/schemas/NIMLLMChatCompletionStreamResponse"
                }
              }
            }
          },
          "202": {
            "description": "Result is pending. Client should poll using the requestId.\n",
            "content": {
              "application/json": {
                "example": {},
                "schema": {}
              }
            },
            "headers": {
              "NVCF-REQID": {
                "description": "requestId required for pooling",
                "schema": {
                  "type": "string",
                  "format": "uuid",
                  "maxLength": 36
                }
              },
              "NVCF-STATUS": {
                "description": "Invocation status",
                "schema": {
                  "type": "string",
                  "format": "^[a-zA-Z-]{1,64}$",
                  "maxLength": 64
                }
              }
            }
          },
          "422": {
            "description": "Validation failed, provided entity could not be processed.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Errors"
                },
                "example": {
                  "type": "urn:nvcf-worker-service:problem-details:unprocessable-entity",
                  "title": "Unprocessable Entity",
                  "status": 422,
                  "detail": "string",
                  "instance": "/v2/nvcf/pexec/functions/bc205f8e-1740-40df-8d32-c4321763498a",
                  "requestId": "3fa85f64-5717-4562-b3fc-2c963f66afa6"
                }
              }
            }
          },
          "500": {
            "description": "The invocation ended with an error.",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/Errors"
                },
                "example": {
                  "type": "urn:nvcf-worker-service:problem-details:internal-server-error",
                  "title": "Internal Server Error",
                  "status": 500,
                  "detail": "string",
                  "instance": "/v2/nvcf/pexec/functions/bc205f8e-1740-40df-8d32-c4321763498a",
                  "requestId": "3fa85f64-5717-4562-b3fc-2c963f66afa6"
                }
              }
            }
          }
        },
        "x-nvai-meta": {
          "name": "Create chat completion",
          "returns": "Returns a [chat completion](/docs/api-reference/chat/object) object.\n",
          "path": "create",
          "examples": [
            {
              "name": "warehouse_warning_tape.mp4",
              "input": {
                "text": "Elaborate what the worker is doing, why he would be taking this action and also provide information on what he is wearing.",
                "image": "https://assets.ngc.nvidia.com/products/api-catalog/vila/warehouse_warning_tape.jpeg",
                "video": "https://assets.ngc.nvidia.com/products/api-catalog/vila/warehouse_warning_tape.mp4"
              },
              "output": {
                "text": "The worker, who is wearing a neon vest, black pants, and a yellow hard hat, is pulling a yellow caution tape across the aisle. This action suggests that he is cordoning off the area, likely for safety or maintenance reasons. The caution tape is a common safety measure used to block off areas that are not safe to enter or that require special attention."
              },
              "requestJson": "{ \"max_tokens\": 1024, \"temperature\": 0.2, \"top_p\": 0.7, \"seed\": 50, \"model\": \"nvidia/vila\", \"messages\": [ { \"role\": \"user\", \"content\": \"<%- input.text %>:<video src=\\\"<%- input.video %>\\\"/>\" } ], \"stream\": false }",
              "responseJson": "{ \"id\": \"chat-02a6636a19554eb58258779539e9b980\", \"object\": \"chat.completion\", \"created\": 1729011479, \"model\": \"nvidia/vila\", \"choices\": [ { \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": \"<%- output.text %>\" }, \"finish_reason\": \"stop\" } ], \"usage\": { \"completion_tokens\": 231, \"prompt_tokens\": 211, \"total_tokens\": 442 } }"
            },
            {
              "name": "its.mp4",
              "input": {
                "text": "Describe the events on the road and report any anomalies.",
                "image": "https://assets.ngc.nvidia.com/products/api-catalog/real-time-visual-understanding-agent/its.jpg",
                "video": "https://assets.ngc.nvidia.com/products/api-catalog/vila/its_short.mp4"
              },
              "output": {
                "text": "The video begins with a clear view of a road intersection with white markings and a STOP sign visible. The scene is set during the daytime with shadows indicating the sun is low in the sky. A red car enters the intersection from the left, followed by a yellow car. As the yellow car attempts to pass the red car, it collides with the red car, causing both vehicles to spin out of control. A police car with flashing lights arrives on the scene, followed by an ambulance. The police car and ambulance are seen attending to the situation, with the police car positioning itself behind the yellow car and the ambulance approaching from the right side of the frame.\n"
              },
              "requestJson": "{ \"max_tokens\": 1024, \"temperature\": 0.2, \"top_p\": 0.7, \"seed\": 50, \"model\": \"nvidia/vila\", \"messages\": [ { \"role\": \"user\", \"content\": \"<%- input.text %>:<video src=\\\"<%- input.video %>\\\"/>\" } ], \"stream\": false }",
              "responseJson": "{ \"id\": \"chat-02a6636a19554eb58258779539e9b980\", \"object\": \"chat.completion\", \"created\": 1729011479, \"model\": \"nvidia/vila\", \"choices\": [ { \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": \"<%- output.text %>\" }, \"finish_reason\": \"stop\" } ], \"usage\": { \"completion_tokens\": 231, \"prompt_tokens\": 211, \"total_tokens\": 442 } }"
            },
            {
              "name": "forklifts",
              "input": {
                "text": "What is common across the 3 images? Which image has worker present?",
                "images": [
                  "https://assets.ngc.nvidia.com/products/api-catalog/vila/forklift_1.png",
                  "https://assets.ngc.nvidia.com/products/api-catalog/vila/forklift_2.png",
                  "https://assets.ngc.nvidia.com/products/api-catalog/vila/warehouse_forklift.png"
                ]
              },
              "output": {
                "text": "The common element across the 3 images is the forklift. The image on the right has a worker present, who is operating the forklift."
              },
              "requestJson": "{ \"max_tokens\": 1024, \"temperature\": 0.2, \"top_p\": 0.7, \"seed\": 50, \"model\": \"nvidia/vila\", \"messages\": [ { \"role\": \"user\", \"content\": \"<%- input.text %>:<video src=\\\"<%- input.video %>\\\"/>\" } ], \"stream\": false }",
              "responseJson": "{ \"id\": \"chat-02a6636a19554eb58258779539e9b980\", \"object\": \"chat.completion\", \"created\": 1729011479, \"model\": \"nvidia/vila\", \"choices\": [ { \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": \"<%- output.text %>\" }, \"finish_reason\": \"stop\" } ], \"usage\": { \"completion_tokens\": 32, \"prompt_tokens\": 617, \"total_tokens\": 649 } }"
            },
            {
              "name": "automotive",
              "input": {
                "text": "Describe what's happening in the scene?",
                "image": "https://assets.ngc.nvidia.com/products/api-catalog/vila/auto.jpg",
                "video": "https://assets.ngc.nvidia.com/products/api-catalog/vila/auto.mp4"
              },
              "output": {
                "text": "A person is crossing the road, walking from the right side of the frame towards the left. They're moving across the road, passing by the white and black stripes on the ground, which appear to be a pedestrian crossing. The person seems to be walking at a steady pace, and their movement is the main action happening in the scene."
              },
              "requestJson": "{ \"max_tokens\": 1024, \"temperature\": 0.2, \"top_p\": 0.7, \"seed\": 50, \"model\": \"nvidia/vila\", \"messages\": [ { \"role\": \"user\", \"content\": \"<%- input.text %>:<video src=\\\"<%- input.video %>\\\"/>\" } ], \"stream\": false }",
              "responseJson": "{ \"id\": \"chat-02a6636a19554eb58258779539e9b980\", \"object\": \"chat.completion\", \"created\": 1729011479, \"model\": \"nvidia/vila\", \"choices\": [ { \"index\": 0, \"message\": { \"role\": \"assistant\", \"content\": \"<%- output.text %>\" }, \"finish_reason\": \"stop\" } ], \"usage\": { \"completion_tokens\": 32, \"prompt_tokens\": 617, \"total_tokens\": 649 } }"
            }
          ],
          "templates": [
            {
              "title": "Default",
              "requestEjs": {
                "python": "<% let pyStream = request.stream.toString()[0].toUpperCase() + request.stream.toString().slice(1) %>\nimport requests\nimport os\nimport uuid\nimport sys\n\ninvoke_url = \"https://ai.api.nvidia.com/v1/vlm/nvidia/vila\"\nstream = <%- pyStream %>\nquery = \"Describe the scene\"\n\nkApiKey = os.getenv(\"TEST_NVCF_API_KEY\", \"\")\nassert kApiKey, \"Generate API_KEY and export TEST_NVCF_API_KEY=xxxx\"\n\nkNvcfAssetUrl = \"https://api.nvcf.nvidia.com/v2/nvcf/assets\"\n# ext: {mime, media}\nkSupportedList = {\n    \"png\": [\"image/png\", \"img\"],\n    \"jpg\": [\"image/jpg\", \"img\"],\n    \"jpeg\": [\"image/jpeg\", \"img\"],\n    \"mp4\": [\"video/mp4\", \"video\"],\n}\n\ndef get_extention(filename):\n    _, ext = os.path.splitext(filename)\n    ext = ext[1:].lower()\n    return ext\n\ndef mime_type(ext):\n    return kSupportedList[ext][0]\ndef media_type(ext):\n    return kSupportedList[ext][1]\n\ndef _upload_asset(media_file, description):\n    ext = get_extention(media_file)\n    assert ext in kSupportedList\n    data_input = open(media_file, \"rb\")\n    headers={\n        \"Authorization\": f\"Bearer {kApiKey}\",\n        \"Content-Type\": \"application/json\",\n        \"accept\": \"application/json\",\n    }\n    assert_url = kNvcfAssetUrl\n    authorize = requests.post(\n        assert_url,\n        headers = headers,\n        json={\"contentType\": f\"{mime_type(ext)}\", \"description\": description},\n        timeout=30,\n    )\n    authorize.raise_for_status()\n\n    authorize_res = authorize.json()\n    print(f\"uploadUrl: {authorize_res['uploadUrl']}\")\n    response = requests.put(\n        authorize_res[\"uploadUrl\"],\n        data=data_input,\n        headers={\n            \"x-amz-meta-nvcf-asset-description\": description,\n            \"content-type\": mime_type(ext),\n        },\n        timeout=300,\n    )\n\n    response.raise_for_status()\n    if response.status_code == 200:\n        print(f\"upload asset_id {authorize_res['assetId']} successfully!\")\n    else:\n        print(f\"upload asset_id {authorize_res['assetId']} failed.\")\n    return uuid.UUID(authorize_res[\"assetId\"])\n\ndef _delete_asset(asset_id):\n    headers = {\n        \"Authorization\": f\"Bearer {kApiKey}\",\n    }\n    assert_url = f\"{kNvcfAssetUrl}/{asset_id}\"\n    response = requests.delete(\n        assert_url, headers=headers, timeout=30\n    )\n    response.raise_for_status()\n\ndef chat_with_media_nvcf(infer_url, media_files, query: str, stream: bool = False):\n    asset_list = []\n    ext_list = []\n    media_content = \"\"\n    assert isinstance(media_files, list), f\"{media_files}\"\n    print(\"uploading {media_files} into s3\")\n    has_video = False\n    for media_file in media_files:\n        ext = get_extention(media_file)\n        assert ext in kSupportedList, f\"{media_file} format is not supported\"\n        if media_type(ext) == \"video\":\n            has_video = True\n        asset_id = _upload_asset(media_file, \"Reference media file\")\n        asset_list.append(f\"{asset_id}\")\n        ext_list.append(ext)\n        media_content += f'<{media_type(ext)} src=\"data:{mime_type(ext)};asset_id,{asset_id}\" />'\n    if has_video:\n        assert len(media_files) == 1, \"Only single video supported.\"\n    asset_seq = \",\".join(asset_list)\n    print(f\"received asset_id list: {asset_seq}\")\n    headers = {\n        \"Authorization\": f\"Bearer {kApiKey}\",\n        \"Content-Type\": \"application/json\",\n        \"NVCF-INPUT-ASSET-REFERENCES\": asset_seq,\n        \"NVCF-FUNCTION-ASSET-IDS\": asset_seq,\n        \"Accept\": \"application/json\",\n    }\n    if stream:\n        headers[\"Accept\"] = \"text/event-stream\"\n    response = None\n\n    messages = [\n        {\n            \"role\": \"user\",\n            \"content\": f\"{query} {media_content}\",\n        }\n    ]\n    payload = {\n        \"max_tokens\": 1024,\n        \"temperature\": 0.2,\n        \"top_p\": 0.7,\n        \"seed\": 50,\n        num_frames_per_inference: 8,\n        \"messages\": messages,\n        \"stream\": stream,\n        \"model\": \"nvidia/vila\",\n    }\n\n    response = requests.post(infer_url, headers=headers, json=payload, stream=stream)\n    if stream:\n        for line in response.iter_lines():\n            if line:\n                print(line.decode(\"utf-8\"))\n    else:\n        print(response.json())\n\n    print(f\"deleting assets: {asset_list}\")\n    for asset_id in asset_list:\n        _delete_asset(asset_id)\n\nif __name__ == \"__main__\":\n    \"\"\" export TEST_NVCF_API_KEY=xxx.\n        python test.py sample.mp4\n        python test.py sample1.png sample2.png\n    \"\"\"\n\n    if len(sys.argv) <= 1:\n        print(\"Usage: export TEST_NVCF_API_KEY=xxx\")\n        print(f\"       python {sys.argv[0]} sample1.png sample2.png ... sample16.png\")\n        print(f\"       python {sys.argv[0]} sample.mp4\")\n        sys.exit(1)\n    media_samples = list(sys.argv[1:])\n    chat_with_media_nvcf(invoke_url, media_samples, query, stream)\n",
                "node.js": "import fs from 'fs';\nimport axios from 'axios';\nimport path from 'path';\n\nconst invokeUrl = \"https://ai.api.nvidia.com/v1/vlm/nvidia/vila\";\nconst stream = <%- request.stream %>;\nconst query = 'Describe the scene';\n\nconst kNvcfAssetUrl = 'https://api.nvcf.nvidia.com/v2/nvcf/assets';\n\n// Retrieve the API Key from environment variables\nconst kApiKey = process.env.TEST_NVCF_API_KEY;\nif (!kApiKey) {\n  console.error(\"Generate API_KEY and export TEST_NVCF_API_KEY=xxxx\");\n  process.exit(1);\n}\n\nconst kSupportedList = {\n  \"png\": [\"image/png\", \"img\"],\n  \"jpg\": [\"image/jpg\", \"img\"],\n  \"jpeg\": [\"image/jpeg\", \"img\"],\n  \"mp4\": [\"video/mp4\", \"video\"]\n};\n\n// Get file extension\nfunction getExtension(filename) {\n  const ext = path.extname(filename).toLowerCase();\n  return ext.slice(1); // remove the leading dot\n}\n\n// Get MIME type\nfunction mimeType(ext) {\n  return kSupportedList[ext][0];\n}\n\n// Get media type\nfunction mediaType(ext) {\n  return kSupportedList[ext][1];\n}\n\n// Upload asset\nasync function uploadAsset(mediaFile, description) {\n  const ext = getExtension(mediaFile);\n  if (!(ext in kSupportedList)) {\n    throw new Error(`Unsupported file extension: ${ext}`);\n  }\n\n  const dataInput = fs.readFileSync(mediaFile); // Sync file read\n\n  const headers = {\n    \"Authorization\": `Bearer ${kApiKey}`,\n    \"Content-Type\": \"application/json\",\n    \"Accept\": \"application/json\"\n  };\n\n  const postData = {\n    contentType: mimeType(ext),\n    description: description\n  };\n\n  // First API call to authorize asset upload\n  const { data: authorizeRes } = await axios.post(kNvcfAssetUrl, postData, { headers });\n  console.log(`uploadUrl: ${authorizeRes.uploadUrl}`);\n\n  // Second API call to upload the file\n  const response = await axios.put(authorizeRes.uploadUrl, dataInput, {\n    headers: {\n      \"x-amz-meta-nvcf-asset-description\": description,\n      \"content-type\": mimeType(ext)\n    }\n  });\n\n  if (response.status === 200) {\n    console.log(`upload asset_id ${authorizeRes.assetId} successfully!`);\n    //return uuidParse(authorizeRes.assetId);\n    return authorizeRes.assetId.toString()\n  } else {\n    console.log(`upload asset_id ${authorizeRes.assetId} failed.`);\n    throw new Error(`Asset upload failed: ${authorizeRes.assetId}`);\n  }\n}\n\n// Delete asset\nasync function deleteAsset(assetId) {\n  const headers = {\n    \"Authorization\": `Bearer ${kApiKey}`\n  };\n  const url = `${kNvcfAssetUrl}/${assetId}`;\n  await axios.delete(url, { headers });\n}\n\n// Chat with media NVCF\nasync function chatWithMediaNvcf(inferUrl, mediaFiles, query, stream = false) {\n  const assetList = [];\n  const extList = [];\n  let mediaContent = \"\";\n  let hasVideo = false;\n\n  for (const mediaFile of mediaFiles) {\n    const ext = getExtension(mediaFile);\n    if (!(ext in kSupportedList)) {\n      throw new Error(`${mediaFile} format is not supported`);\n    }\n\n    if (mediaType(ext) === \"video\") {\n      hasVideo = true;\n    }\n\n    console.log(`uploading file: ${mediaFile}`);\n    const assetId = await uploadAsset(mediaFile, \"Reference media file\");\n    console.log(`assetId: ${assetId}`);\n    assetList.push(assetId);\n    extList.push(ext);\n    mediaContent += `<${mediaType(ext)} src=\"data:${mimeType(ext)};asset_id,${assetId}\" />`;\n  }\n\n  if (hasVideo && mediaFiles.length !== 1) {\n    throw new Error(\"Only a single video is supported.\");\n  }\n\n  const assetSeq = assetList.join(',');\n  console.log(`received asset_id list: ${assetSeq}`);\n\n  const headers = {\n    \"Authorization\": `Bearer ${kApiKey}`,\n    \"Content-Type\": \"application/json\",\n    \"NVCF-INPUT-ASSET-REFERENCES\": assetSeq,\n    \"NVCF-FUNCTION-ASSET-IDS\": assetSeq,\n    \"Accept\": \"application/json\"\n  };\n\n  if (stream) {\n    headers[\"Accept\"] = \"text/event-stream\";\n  }\n\n  const messages = [{\n    \"role\": \"user\",\n    \"content\": `${query} ${mediaContent}`\n  }];\n\n  const payload = {\n    max_tokens: 1024,\n    temperature: 0.2,\n    top_p: 0.7,\n    seed: 50,\n    num_frames_per_inference: 8,\n    messages: messages,\n    stream: stream,\n    model: \"nvidia/vila\"\n  };\n\n  // Post to the inference API\n  //let response = await axios.post(inferUrl, payload, { headers });\n  const response = await axios.post(inferUrl, payload, {\n    headers: headers,\n    responseType: stream ? 'stream' : 'json'\n  });\n\n  if (stream) {\n    response.data.on('data', (line) => {\n      console.log(line.toString());\n    });\n  } else {\n    console.log(JSON.stringify(response.data));\n  }\n\n  // Clean up uploaded assets\n  console.log(`deleting assets: ${assetList}`);\n  for (const assetId of assetList) {\n    await deleteAsset(assetId);\n  }\n}\n\n// Main function to run the script\nasync function main() {\n  const args = process.argv.slice(2);\n  if (args.length <= 0) {\n    console.log(\"Usage: export TEST_NVCF_API_KEY=xxx\");\n    console.log(`python ${process.argv[0]} sample1.png sample2.png ... sample16.png`);\n    console.log(`python ${process.argv[0]} sample.mp4`);\n    process.exit(1);\n  }\n\n  const mediaSamples = args;\n  await chatWithMediaNvcf(invokeUrl, mediaSamples, query, stream);\n}\n\nmain();\n",
                "curl": "#!/bin/bash\nstream=<%- request.stream %>\n# Check if TEST_NVCF_API_KEY is set\nif [ -z \"$TEST_NVCF_API_KEY\" ]; then\n  echo \"Generate API_KEY and export TEST_NVCF_API_KEY=xxxx\"\n  exit 1\nfi\n\ninvoke_url=\"https://ai.api.nvidia.com/v1/vlm/nvidia/vila\"\nkNvcfAssetUrl=\"https://api.nvcf.nvidia.com/v2/nvcf/assets\"\nquery=\"Describe the scene\"\n\n# supported table\nkSupportedList=(\"png:image/png,img\" \"jpg:image/jpg,img\" \"jpeg:image/jpeg,img\" \"mp4:video/mp4,video\")\n\n# get \"mime,media\"\nget_media_info() {\n  local ext=\"$1\"\n  for item in \"${kSupportedList[@]}\"; do\n    if [[ \"$item\" == \"$ext:\"* ]]; then\n      # Return \"mime,media\"\n      echo \"${item#*:}\"\n      return\n    fi\n  done\n  echo \"\"\n}\n\nget_extension() {\n  filename=\"$1\"\n  echo \"${filename##*.}\" | tr '[:upper:]' '[:lower:]'\n}\n\nupload_asset() {\n  media_file=\"$1\"\n  description=\"$2\"\n  ext=$(get_extension \"$media_file\")\n\n  media_info=$(get_media_info \"$ext\")\n  if [ -z \"$media_info\" ]; then\n    echo \"$media_file format is not supported\"\n    exit 1\n  fi\n\n  mime_type=\"${media_info%%,*}\"\n  media_type=\"${media_info#*,}\"\n\n  # Get upload URL\n  response=$(curl -s -X POST \"$kNvcfAssetUrl\" \\\n    -H \"Authorization: Bearer $TEST_NVCF_API_KEY\" \\\n    -H \"Content-Type: application/json\" \\\n    -H \"accept: application/json\" \\\n    -d \"{\\\"contentType\\\": \\\"$mime_type\\\", \\\"description\\\": \\\"$description\\\"}\")\n\n  upload_url=$(echo \"$response\" | jq -r '.uploadUrl')\n  asset_id=$(echo \"$response\" | jq -r '.assetId')\n\n  # Upload the asset file to the URL\n  curl -s -X PUT \"$upload_url\" \\\n    -H \"x-amz-meta-nvcf-asset-description: $description\" \\\n    -H \"content-type: $mime_type\" \\\n    --data-binary \"@$media_file\"\n\n  echo \"$asset_id\"\n}\n\ndelete_asset() {\n  asset_id=\"$1\"\n  curl -s -X DELETE \"$kNvcfAssetUrl/$asset_id\" \\\n    -H \"Authorization: Bearer $TEST_NVCF_API_KEY\"\n}\n\nchat_with_media_nvcf() {\n  infer_url=\"$1\"\n  query=\"$2\"\n  shift 2\n  media_files=(\"$@\")\n\n  asset_list=()\n  media_content=\"\"\n\n  has_video=false\n\n  for media_file in \"${media_files[@]}\"; do\n    ext=$(get_extension \"$media_file\")\n\n    media_info=$(get_media_info \"$ext\")\n    if [ -z \"$media_info\" ]; then\n      echo \"$media_file format is not supported\"\n      exit 1\n    fi\n    mime_type=\"${media_info%%,*}\"\n    media_type=\"${media_info#*,}\"\n\n    if [[ \"$mime_type\" == \"video\" ]]; then\n      has_video=true\n    fi\n\n    echo \"uploading media_file: $media_file\"\n    asset_id=$(upload_asset \"$media_file\" \"Reference media file\")\n    asset_list+=(\"$asset_id\")\n    media_content+=\"<$media_type src=\\\"data:$mime_type;asset_id,$asset_id\\\" />\"\n  done\n\n  if $has_video && [ \"${#media_files[@]}\" -gt 1 ]; then\n    echo \"Only single video supported.\"\n    exit 1\n  fi\n\n  asset_seq=$(IFS=,; echo \"${asset_list[*]}\")\n\n  headers=(\n    -H \"Authorization: Bearer $TEST_NVCF_API_KEY\"\n    -H \"Content-Type: application/json\"\n    -H \"NVCF-INPUT-ASSET-REFERENCES: $asset_seq\"\n    -H \"NVCF-FUNCTION-ASSET-IDS: $asset_seq\"\n  )\n  if [ \"$stream\" = true ]; then\n    headers+=(-H \"Accept: text/event-stream\")\n  else\n    headers+=(-H \"Accept: application/json\")\n  fi\n\n  payload=$(jq -n --arg query \"$query\" --arg media_content \"$media_content\" --argjson stream \"$stream\" '{\n    max_tokens: 1024,\n    temperature: 0.2,\n    top_p: 0.7,\n    seed: 50,\n    num_frames_per_inference: 8,\n    messages: [{\"role\": \"user\", \"content\": \"\\($query) \\($media_content)\"}],\n    stream: $stream,\n    model: \"nvidia/vila\"\n  }')\n\n  response=$(curl -s -X POST \"$infer_url\" \"${headers[@]}\" -d \"$payload\")\n\n  if [ \"$stream\" = true ]; then\n    echo \"$response\" | while IFS= read -r line; do\n      echo \"$line\"\n    done\n  else\n    echo \"$response\" | jq .\n  fi\n\n  # Cleanup uploaded assets\n  for asset_id in \"${asset_list[@]}\"; do\n    echo \"deleting asset $asset_id\"\n    delete_asset \"$asset_id\"\n  done\n}\n\nif [ \"$#\" -le 0 ]; then\n  echo \"Usage: export TEST_NVCF_API_KEY=xxx\"\n  echo \"       $0 sample1.png sample2.png ... sample16.png\"\n  echo \"       $0 sample.mp4\"\n  exit 1\nfi\n\nchat_with_media_nvcf \"$invoke_url\" \"$query\" \"$@\"\n"
              },
              "response": "{\n  \"choices\": [{\n    \"index\": 0,\n    \"message\": {\n      \"role\": \"assistant\",\n      \"content\": \"...\",\n    },\n    \"finish_reason\": \"stop\"\n  }],\n  \"usage\": {\n    \"prompt_tokens\": 9,\n    \"completion_tokens\": 12,\n    \"total_tokens\": 21\n  }\n}\n"
            }
          ]
        }
      }
    }
  },
  "security": [
    {
      "Token": []
    }
  ],
  "components": {
    "securitySchemes": {
      "Token": {
        "type": "http",
        "scheme": "bearer"
      }
    },
    "schemas": {
      "Errors": {
        "properties": {
          "type": {
            "type": "string",
            "format": "^.{1, 128}$",
            "maxLength": 128,
            "description": "Error type"
          },
          "title": {
            "type": "string",
            "format": "^.{1, 128}$",
            "maxLength": 128,
            "description": "Error title"
          },
          "status": {
            "type": "integer",
            "minimum": 100,
            "maximum": 999,
            "description": "Error status code"
          },
          "detail": {
            "type": "string",
            "format": "^.{1, 1024}$",
            "maxLength": 1024,
            "description": "Detailed information about the error"
          },
          "instance": {
            "type": "string",
            "format": "^.{1, 256}$",
            "maxLength": 256,
            "description": "Function instance used to invoke the request"
          },
          "requestId": {
            "type": "string",
            "format": "uuid",
            "maxLength": 36,
            "description": "UUID of the request"
          }
        },
        "type": "object",
        "required": [
          "type",
          "title",
          "status",
          "detail",
          "instance",
          "requestId"
        ],
        "title": "InvokeError"
      },
      "NIMLLMChatCompletionResponse": {
        "properties": {
          "id": {
            "type": "string",
            "title": "Id"
          },
          "object": {
            "type": "string",
            "enum": [
              "chat.completion"
            ],
            "const": "chat.completion",
            "title": "Object",
            "default": "chat.completion"
          },
          "created": {
            "type": "integer",
            "title": "Created"
          },
          "model": {
            "type": "string",
            "title": "Model"
          },
          "choices": {
            "description": "The list of completion choices the model generated for the input prompt.",
            "items": {
              "$ref": "#/components/schemas/ChatCompletionResponseChoice"
            },
            "maxItems": 1,
            "title": "Choices",
            "type": "array"
          },
          "usage": {
            "$ref": "#/components/schemas/UsageInfo"
          }
        },
        "required": [
          "id",
          "model",
          "choices",
          "usage"
        ],
        "title": "NIMLLMChatCompletionResponse",
        "type": "object"
      },
      "NIMLLMChatCompletionStreamResponse": {
        "properties": {
          "id": {
            "type": "string",
            "title": "Id"
          },
          "object": {
            "type": "string",
            "enum": [
              "chat.completion.chunk"
            ],
            "const": "chat.completion.chunk",
            "title": "Object",
            "default": "chat.completion.chunk"
          },
          "created": {
            "type": "integer",
            "title": "Created"
          },
          "model": {
            "type": "string",
            "title": "Model"
          },
          "choices": {
            "description": "The list of completion choices the model generated for the input prompt.",
            "items": {
              "$ref": "#/components/schemas/ChatCompletionResponseStreamChoice"
            },
            "maxItems": 1,
            "title": "Choices",
            "type": "array"
          },
          "usage": {
            "anyOf": [
              {
                "$ref": "#/components/schemas/UsageInfo"
              },
              {
                "type": "null"
              }
            ]
          }
        },
        "required": [
          "id",
          "model",
          "choices"
        ],
        "title": "NIMLLMChatCompletionStreamResponse",
        "type": "object"
      },
      "NIMLLMChatCompletionRequest": {
        "additionalProperties": false,
        "properties": {
          "messages": {
            "description": "A list of messages comprising the conversation so far. The roles of the messages must be alternating between `user` and `assistant`. The last input message should have role `user` or `assistant`. A message with the `system` role is optional, and must be the very first message if it is present.",
            "examples": [
              [
                {
                  "content": "Hi! What is in this image? <img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII==\" />",
                  "role": "user"
                }
              ],
              [
                {
                  "content": "Hi! What is in this image? <img src=\"data:image/png;asset_id,87b132b0-08f9-43ea-8fab-f107350a5d00\" />",
                  "role": "user"
                }
              ]
            ],
            "items": {
              "$ref": "#/components/schemas/NIMLLMChatCompletionMessage"
            },
            "maxItems": 1024,
            "minItems": 1,
            "title": "Messages",
            "type": "array"
          },
          "model": {
            "default": "nvidia/vila",
            "type": "string",
            "title": "Model",
            "description": "The model to use."
          },
          "temperature": {
            "default": 0.2,
            "description": "The sampling temperature to use for text generation. The higher the temperature value is, the less deterministic the output text will be. It is not recommended to modify both temperature and top_p in the same call.",
            "maximum": 1,
            "minimum": 0,
            "title": "Temperature",
            "type": "number"
          },
          "top_p": {
            "default": 0.7,
            "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with `top_p` probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. NVIDIA recommends that you alter this option or `temperature` but not both.",
            "maximum": 1,
            "exclusiveMinimum": 0,
            "title": "Top P",
            "type": "number"
          },
          "max_tokens": {
            "default": 1024,
            "description": "The maximum number of tokens to generate in any given call. Note that the model is not aware of this value, and generation will simply stop at the number of tokens specified.",
            "maximum": 2048,
            "minimum": 1,
            "title": "Max Tokens",
            "type": "integer"
          },
          "seed": {
            "anyOf": [
              {
                "maximum": 9223372036854776000,
                "minimum": -9223372036854776000,
                "type": "integer"
              },
              {
                "type": "null"
              }
            ],
            "default": 50,
            "description": "If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result.",
            "examples": [
              42
            ],
            "title": "Seed"
          },
          "stream": {
            "anyOf": [
              {
                "type": "boolean"
              },
              {
                "type": "null"
              }
            ],
            "default": false,
            "description": "If set, partial message deltas will be sent. Tokens will be sent as data-only server-sent events (SSE) as they become available (JSON responses are prefixed by `data: `), with the stream terminated by a `data: [DONE]` message.",
            "title": "Stream"
          },
          "num_frames_per_inference": {
            "anyOf": [
              {
                "type": "integer",
                "maximum": 16,
                "minimum": 1
              },
              {
                "type": "null"
              }
            ],
            "title": "Num Frames Per Inference",
            "description": "Number of frames to sample from the video or stream. They will be the input to model.",
            "default": 8,
            "examples": [
              8
            ]
          }
        },
        "required": [
          "messages",
          "model"
        ],
        "title": "NIMLLMChatCompletionRequest",
        "type": "object"
      },
      "ChatCompletionResponseChoice": {
        "properties": {
          "index": {
            "default": 0,
            "description": "The index of the choice in the list of choices (always 0).",
            "exclusiveMaximum": 1,
            "minimum": 0,
            "title": "Index",
            "type": "integer"
          },
          "message": {
            "allOf": [
              {
                "$ref": "#/components/schemas/NIMLLMChatCompletionMessage"
              }
            ],
            "description": "A chat completion message generated by the model.",
            "examples": [
              {
                "content": "The image features a gray and white checkered pattern on a white background.",
                "role": "assistant"
              }
            ]
          },
          "finish_reason": {
            "anyOf": [
              {
                "enum": [
                  "stop",
                  "length",
                  "tool_calls"
                ],
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": null,
            "description": "The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, or `length` if the maximum number of tokens specified in the request was reached.",
            "examples": [
              "stop"
            ],
            "title": "Finish Reason"
          },
          "stop_reason": {
            "anyOf": [
              {
                "type": "integer"
              },
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Stop Reason"
          }
        },
        "required": [
          "index",
          "message"
        ],
        "title": "ChatCompletionResponseChoice",
        "type": "object"
      },
      "ChatCompletionResponseStreamChoice": {
        "properties": {
          "index": {
            "default": 0,
            "description": "The index of the choice in the list of choices (always 0).",
            "exclusiveMaximum": 1,
            "minimum": 0,
            "title": "Index",
            "type": "integer"
          },
          "delta": {
            "allOf": [
              {
                "$ref": "#/components/schemas/NIMLLMChatCompletionMessage"
              }
            ],
            "description": "A chat completion delta generated by streamed model responses.",
            "examples": [
              {
                "content": "The",
                "role": "assistant"
              }
            ]
          },
          "finish_reason": {
            "anyOf": [
              {
                "enum": [
                  "stop",
                  "length",
                  "tool_calls"
                ],
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "default": null,
            "description": "The reason the model stopped generating tokens. This will be `stop` if the model hit a natural stop point or a provided stop sequence, or `length` if the maximum number of tokens specified in the request was reached. Will be `null` if the model has not finished generating.",
            "title": "Finish Reason"
          },
          "stop_reason": {
            "anyOf": [
              {
                "type": "integer"
              },
              {
                "type": "string"
              },
              {
                "type": "null"
              }
            ],
            "title": "Stop Reason"
          }
        },
        "required": [
          "index",
          "delta"
        ],
        "title": "ChatCompletionResponseStreamChoice",
        "type": "object"
      },
      "NIMLLMChatCompletionMessage": {
        "additionalProperties": false,
        "properties": {
          "role": {
            "description": "The role of the message author.",
            "enum": [
              "user",
              "assistant"
            ],
            "title": "Role",
            "type": "string"
          },
          "content": {
            "anyOf": [
              {
                "type": "string"
              },
              {
                "items": {
                  "anyOf": [
                    {
                      "$ref": "#/components/schemas/NIMVLMChatCompletionContentPartText"
                    },
                    {
                      "$ref": "#/components/schemas/NIMVLMChatCompletionContentPartImage"
                    },
                    {
                      "$ref": "#/components/schemas/NIMVLMChatCompletionContentPartVideo"
                    }
                  ]
                },
                "type": "array"
              },
              {
                "type": "null"
              }
            ],
            "default": null,
            "description": "The contents of the message.\n<br>Can only be `null` as part of a last request message with role=`assistant` (for \"completion mode\", i.e. providing the beginning of the assistant response).\n<br>To pass images (only with role=`user`):\n<br>    - When content is a string, images can be passed together with the text with `img` HTML tags with base64 data: `<img src=\"data:image/{format};base64,{base64encodedimage}\" />` .\n If the size of an image is more than 180KB, it needs to be uploaded to a presigned S3 bucket using NVCF Asset APIs. Once uploaded you can refer to it using the following format: `<img src=\"data:image/png;asset_id,{asset_id}\" />` .\n<br>    - When content is a list of objects, images can be passed with objects with type=`image_url`, and image_url containing the base64 image data: `data:image/{format};base64,{base64encodedimage}`. HTML `img` tags will not be parsed from objects with type=`text`.\n<br>    - In both cases, images can be PNG, JPG or JPEG.\n<br>For `system` and `assistant` roles, the object list format is not supported.\n",
            "title": "Content"
          }
        },
        "required": [
          "role",
          "content"
        ],
        "title": "NIMLLMChatCompletionMessage",
        "type": "object"
      },
      "UsageInfo": {
        "properties": {
          "prompt_tokens": {
            "type": "integer",
            "title": "Prompt Tokens",
            "default": 0
          },
          "total_tokens": {
            "type": "integer",
            "title": "Total Tokens",
            "default": 0
          },
          "completion_tokens": {
            "anyOf": [
              {
                "type": "integer"
              },
              {
                "type": "null"
              }
            ],
            "title": "Completion Tokens",
            "default": 0
          }
        },
        "additionalProperties": false,
        "type": "object",
        "title": "UsageInfo"
      },
      "NIMVLMChatCompletionContentPartImage": {
        "additionalProperties": false,
        "properties": {
          "type": {
            "type": "string",
            "enum": [
              "image_url"
            ],
            "const": "image_url",
            "description": "The type of the content part.",
            "title": "Type"
          },
          "image_url": {
            "allOf": [
              {
                "$ref": "#/components/schemas/ImageURL"
              }
            ],
            "description": "Image url"
          }
        },
        "required": [
          "type",
          "image_url"
        ],
        "title": "NIMVLMChatCompletionContentPartImage",
        "type": "object"
      },
      "ImageURL": {
        "additionalProperties": false,
        "properties": {
          "url": {
            "description": "Base64 encoded image data in form of `data:image/{format};base64,{base64encodedimage}`.\n        If the size of an image is more than 200KB, it needs to be uploaded to a presigned S3 bucket using NVCF Asset APIs.\n        Once uploaded you can refer to it using the following format: `data:image/png;asset_id,{asset_id}`.\n        Accepted formats are `jpg`, `png` and `jpeg`.",
            "maxLength": 204800,
            "pattern": "^[\\s\\S]+$",
            "title": "Url",
            "type": "string"
          }
        },
        "required": [
          "url"
        ],
        "title": "ImageURL",
        "type": "object"
      },
      "NIMVLMChatCompletionContentPartVideo": {
        "additionalProperties": false,
        "properties": {
          "video_url": {
            "description": "Video url",
            "allOf": [
              {
                "$ref": "#/components/schemas/VideoURL"
              }
            ]
          },
          "start_offset": {
            "description": "video offset position",
            "anyOf": [
              {
                "type": "number",
                "maximum": 9223372036854776000,
                "minimum": 0
              },
              {
                "type": "null"
              }
            ],
            "title": "Start Offset"
          },
          "duration": {
            "description": "video inference duration",
            "anyOf": [
              {
                "type": "number",
                "maximum": 9223372036854776000,
                "minimum": 0
              },
              {
                "type": "null"
              }
            ],
            "title": "Duration"
          },
          "type": {
            "type": "string",
            "description": "The type of the content part.",
            "enum": [
              "video_url"
            ],
            "const": "video_url",
            "title": "Type"
          }
        },
        "required": [
          "type",
          "video_url"
        ],
        "type": "object",
        "title": "NIMVLMChatCompletionContentPartVideo"
      },
      "VideoURL": {
        "additionalProperties": false,
        "properties": {
          "url": {
            "description": "Base64 H.264/H.265 mp4 video data in form of `data:video/mp4;base64,{base64encodedimage}`.\n        If the size of an image is more than 200KB, it needs to be uploaded to a presigned S3 bucket using NVCF Asset APIs.\n        Once uploaded you can refer to it using the following format: `data:video/mp4;asset_id,{asset_id}`.\n        Accepted formats are `mp4`.",
            "type": "string",
            "maxLength": 204800,
            "pattern": "^[\\s\\S]+$",
            "title": "Url"
          }
        },
        "type": "object",
        "required": [
          "url"
        ],
        "title": "VideoURL"
      },
      "NIMVLMChatCompletionContentPartText": {
        "additionalProperties": false,
        "properties": {
          "type": {
            "const": "text",
            "description": "The type of the content part.",
            "title": "Type"
          },
          "text": {
            "description": "The text content.",
            "title": "Text",
            "type": "string",
            "maxLength": 204800
          }
        },
        "required": [
          "type",
          "text"
        ],
        "title": "NIMVLMChatCompletionContentPartText",
        "type": "object"
      }
    }
  },
  "x-readme": {
    "explorer-enabled": true,
    "proxy-enabled": true
  }
}
```