{"catalog":"https://launchinyourcloud.com/api/v1/catalog.json","catalogRevision":"170de95bb7d832897d269051ad4500713acf96a62f3d01bc1d4e649679a56fde","category":{"description":"Model-serving runtimes, inference gateways and provider routing.","id":"ai-inference-gateways","name":"AI inference \u0026 gateways"},"cloud":{"boundary":"Subscription and resource group","classificationReview":{"reviewed":"2026-11-20","source":"https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal","summary":"Navigation group for the documented infrastructure offering; read each application path for its actual account boundary."},"defaultBoundary":"cloud-account","diligence":[{"reviewed":"2026-11-14","source":"https://learn.microsoft.com/en-us/marketplace/private-offers-overview","summary":"Microsoft Marketplace private offers negotiate product pricing and terms. Confirm the exact offer, billing scope and benefit eligibility rather than assuming all purchases count toward a commitment.","topic":"Private offers"},{"reviewed":"2026-11-14","source":"https://learn.microsoft.com/en-us/azure/private-link/private-link-overview","summary":"Azure Private Link uses private endpoints in a virtual network for supported Azure and partner services. Check application support, DNS and service availability.","topic":"Private connectivity"},{"reviewed":"2026-11-14","source":"https://learn.microsoft.com/en-us/azure/lighthouse/overview","summary":"Azure Lighthouse supports delegated management with customer-controlled scopes and actions. Agree the roles, audit requirements and revocation plan for the selected vendor.","topic":"Cross-tenant operations"},{"reviewed":"2026-11-14","source":"https://learn.microsoft.com/en-us/azure/azure-government/documentation-government-welcome","summary":"Azure Government uses isolated US datacenters and networks. Commercial Azure documentation alone does not establish a product path or compliance in Government.","topic":"Government environments"}],"id":"azure","identity":"Microsoft Entra identities and Azure RBAC","install":"Bicep/ARM templates or Terraform; review deployment scope and role assignments.","name":"Microsoft Azure","network":"Virtual Network (VNet)","providerType":"cloud-infrastructure","reviewed":"2026-11-10","short":"Azure","source":"https://learn.microsoft.com/en-us/azure/azure-resource-manager/bicep/overview"},"kind":"cloud-category","listings":[{"advisories":[{"action":"Upgrade the selected release line and recheck guardrail output and proxy-administrator permissions.","conditions":"Applicability depends on Bedrock guardrails and team-setting permissions.","id":"litellm-1-108-11-lts-security-fixes","listingIds":["litellm"],"reviewed":"2026-11-18","scope":"LiteLLM 1.108 LTS line; standard-line deployments should use their corresponding maintained release.","source":"https://github.com/BerriAI/litellm/releases/tag/v1.108.11","summary":"LiteLLM 1.108.11 LTS fixes Bedrock guarded_text handling and restricts os.environ team settings to proxy administrators.","title":"LiteLLM LTS proxy security fixes","type":"security-advisory"},{"action":"Upgrade the maintained LTS line to 1.108.12 or later and retest tool-policy enforcement.","conditions":"The request must use a blocked tool name longer than 64 characters.","disclosedOn":"2026-11-19","id":"litellm-1-108-12-tool-policy-bypass","listingIds":["litellm"],"reviewed":"2026-11-19","scope":"LiteLLM 1.108 LTS /v1/messages tool-policy enforcement.","source":"https://github.com/BerriAI/litellm/releases/tag/v1.108.12","summary":"LiteLLM 1.108.12 LTS prevents blocked tool names longer than 64 characters from bypassing tool_policy on /v1/messages.","title":"LiteLLM 1.108.12 LTS tool-policy fix","type":"security-advisory"}],"architecture":"Customer-operated gateway with optional PostgreSQL persistence and configured model endpoints","categories":["ai-inference-gateways","ai-tools"],"category":"AI inference \u0026 gateways","categoryFit":{"ai-inference-gateways":"Self-hosted model API gateway for routing, virtual keys, spend tracking and policy enforcement. It routes inference requests; it is not an inference engine for all supported models.","ai-tools":"Self-hosted AI API gateway providing model routing, access controls and usage tracking."},"cloudEvidence":[{"clouds":["aws","azure","gcp"],"notes":"Official production guidance covers EKS, AKS and GKE with Helm, plus Terraform on AWS and Google Cloud. The proxy, database and Redis deployment is separate from the model services it calls.","reviewed":"2026-11-12","sourceType":"vendor-documentation","title":"LiteLLM production deployment","url":"https://docs.litellm.ai/docs/proxy/deploy","vendorSupport":"not-assessed"},{"clouds":["railway","render"],"notes":"The current upstream deployment guide supplies Railway and Render deployment buttons in addition to its EKS/AKS/GKE instructions, followed by readiness and Admin UI verification. These are customer application deployments on managed platforms; configure persistent data, secrets and external model-provider endpoints separately.","reviewed":"2026-11-12","sourceType":"vendor-documentation","title":"LiteLLM Railway and Render deployment options","url":"https://docs.litellm.ai/docs/proxy/deploy","vendorSupport":"not-assessed"}],"cloudReviewed":"2026-11-12","clouds":["aws","azure","gcp","railway","render"],"deploymentPaths":[{"boundary":"cloud-account","cloud":"aws","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"cloud-account","cloud":"azure","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"cloud-account","cloud":"gcp","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"platform-workspace","cloud":"railway","controlPlane":"not-assessed","evidence":[1],"operator":"customer"},{"boundary":"platform-workspace","cloud":"render","controlPlane":"not-assessed","evidence":[1],"operator":"customer"}],"description":"Self-hosted AI API gateway providing model routing, access controls and usage tracking.","diligence":{"footprint":{"reviewed":"2026-11-20","scope":"Gateway per-worker load-test starting point; backing services and inference excluded","source":"https://docs.litellm.ai/docs/proxy/prod","status":"documented","summary":"The current production guide starts load testing at 1 vCPU and 4 GiB per worker, preferably one worker per Kubernetes pod. Memory depends on queued requests, payloads, stream duration and logging. PostgreSQL, Redis for multiple workers/instances, HA and model inference are additional."},"iac":{"reviewed":"2026-11-14","scope":"Upstream self-hosted deployment; not a provider support or production-readiness certification","source":"https://docs.litellm.ai/docs/proxy/deploy/helm","status":"documented","summary":"The official Helm guide documents Kubernetes installation and configurable database/secret settings. Select matching chart/image versions and review the separate scaling and production-readiness guidance."},"licensePricing":{"reviewed":"2026-11-20","scope":"Upstream core licence; commercial features, support and hosting priced separately","source":"https://raw.githubusercontent.com/BerriAI/litellm/main/LICENSE","status":"documented","summary":"Core code outside litellm/enterprise is MIT licensed; the enterprise directory has separate terms. Enterprise features and support need separate entitlement and pricing. Upstream model APIs, infrastructure and operating labour are additional."},"release":{"releasePublishedAt":"2026-11-19T05:05:03Z","releaseTag":"v1.110.1","reviewed":"2026-11-19","scope":"Upstream LiteLLM standard release; select and maintain either the standard or LTS line","source":"https://github.com/BerriAI/litellm/releases/tag/v1.110.1","status":"documented","summary":"LiteLLM v1.110.1 was published November 19, 2026. It refreshes the Wolfi base and changes the UI image to slim nginx, removing curl. The maintained v1.108.12 LTS line fixes a tool_policy bypass for blocked tool names longer than 64 characters.","supportingSources":[{"title":"Prior reviewed release context","url":"https://github.com/BerriAI/litellm/releases/tag/v1.110.0"},{"title":"LiteLLM v1.108.11 LTS","url":"https://github.com/BerriAI/litellm/releases/tag/v1.108.11"},{"title":"v1.108.12 LTS security fix","url":"https://github.com/BerriAI/litellm/releases/tag/v1.108.12"}]},"securityControls":{"reviewed":"2026-11-19","scope":"LiteLLM 1.108 LTS /v1/messages tool_policy; standard and other maintained lines require their own release review","source":"https://github.com/BerriAI/litellm/releases/tag/v1.108.12","status":"documented","summary":"Release images and Helm charts use keyless Cosign from v1.111.0-dev.3; verify the documented workflow identity and issuer. Earlier releases use historical keys. The October 21 v1.107.0 compromise also affected mutable image tags; compare digests and exposure windows because affected images had valid historical signatures. Follow the incident remediation guidance for executed artifacts. Current advisory review: LiteLLM 1.108.11 LTS fixes Bedrock guarded_text handling and restricts os.environ team settings to proxy administrators. Upgrade the selected release line and recheck guardrail output and proxy-administrator permissions. Current review: v1.108.12 LTS fixes tool_policy allowing a /v1/messages request through when a blocked tool name exceeded 64 characters. Upgrade the maintained LTS line and retest tool-policy enforcement.","supportingSources":[{"title":"Prior security-controls source","url":"https://github.com/BerriAI/litellm/releases/tag/v1.108.11"},{"title":"Canonical incident investigation","url":"https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain"},{"title":"Prior security-controls source","url":"https://docs.litellm.ai/blog/security-incident-oct-21-2026"}]},"sso":{"availability":"edition-dependent","reviewed":"2026-11-20","scope":"LiteLLM Admin UI OIDC; user-count/Enterprise entitlement applies","source":"https://docs.litellm.ai/docs/proxy/admin_ui_sso","status":"documented","summary":"The Admin UI documents OIDC SSO. From 1.76.0, it is free for up to five users; larger deployments require an Enterprise licence. Configure issuer, callbacks, roles and database/master-key prerequisites. This is Admin UI identity, distinct from model API virtual keys."}},"id":"litellm","infrastructure":["servers"],"infrastructureNotes":"The gateway runs as provisioned application containers or Kubernetes workloads; persistent key/spend storage can use PostgreSQL. Inference compute is a separate dependency rather than included in the gateway.","infrastructureReviewed":"2026-11-10","infrastructureSources":["https://docs.litellm.ai/docs/proxy/deploy"],"install":"Official container image, Docker Compose or Kubernetes deployment; production infrastructure examples are documented.","installMethods":["docker"],"kind":"product","license":"MIT outside the enterprise directory; enterprise code and features have separate license terms.","licenseModel":"open-source","licenseModelNotes":"The gateway has an MIT-licensed base and separately licensed enterprise features. The enterprise documentation describes a license key enabling restricted capabilities in the gateway, beyond paid hosting or support alone.","licenseModelSources":["https://github.com/BerriAI/litellm/blob/main/LICENSE","https://docs.litellm.ai/docs/enterprise"],"licenseReviewed":"2026-11-10","model":"Fully in your account","modelLabel":"Customer-operated","name":"LiteLLM Proxy Self-Hosted","notes":"The gateway forwards each request to its configured model provider. Self-hosting the proxy does not ensure prompts stay inside the account: external model APIs, guardrail services and logging integrations can receive data. Use appropriate self-hosted inference endpoints and explicitly review outbound integrations when required. For release integrity, release images and the Helm chart starting with v1.111.0-dev.3 use keyless Cosign signatures bound to LiteLLM's release workflow; earlier releases require the documented historical public keys. The October 21 incident affected PyPI v1.107.0 and Docker Hub/GHCR v1.107.0 plus mutable latest, main-latest and main-stable tags during the documented exposure windows. Check the incident's artifact digests and remediation instructions; the compromised images carried valid historical signatures, so signature verification alone does not establish safety. LiteLLM v1.110.0 changes authentication, guardrail, accounting, and S3 behavior. The 1.108.11 LTS patch also fixes Bedrock guarded_text handling and restricts os.environ team settings to proxy administrators; review both the selected release line and existing incident-response or image-signing guidance. LiteLLM v1.108.12 LTS fixes tool_policy allowing blocked /v1/messages tool names longer than 64 characters. The standard v1.110.1 UI image changes to slim nginx and removes curl while retaining wget and nc, so retest health probes and scripts that assume curl.","operations":"Customer manages gateway availability, credentials, database, access policies, logs and upgrades, including pinning versions and verifying release artifacts with the documented version-appropriate Cosign method; model providers remain separate services.","publisher":false,"reviewed":"2026-11-19","sources":[{"title":"Production deployment","url":"https://docs.litellm.ai/docs/proxy/deploy"},{"title":"Gateway request flow and functionality","url":"https://docs.litellm.ai/docs/simple_proxy"},{"title":"MIT core and enterprise exception","url":"https://github.com/BerriAI/litellm/blob/main/LICENSE"},{"title":"Licensing and edition terms","url":"https://docs.litellm.ai/docs/enterprise"},{"title":"Supply-chain incident and Cosign verification guidance","url":"https://docs.litellm.ai/blog/security-incident-oct-21-2026"},{"title":"Canonical LiteLLM incident investigation","url":"https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain"},{"title":"LiteLLM v1.110.0 release","url":"https://github.com/BerriAI/litellm/releases/tag/v1.110.0"},{"title":"LiteLLM v1.108.11 LTS release","url":"https://github.com/BerriAI/litellm/releases/tag/v1.108.11"},{"title":"LiteLLM v1.110.1","url":"https://github.com/BerriAI/litellm/releases/tag/v1.110.1"},{"title":"LiteLLM v1.108.12 LTS","url":"https://github.com/BerriAI/litellm/releases/tag/v1.108.12"}],"url":"https://launchinyourcloud.com/directory/litellm/","website":"https://www.litellm.ai/"},{"architecture":"Licensed NVIDIA AI containers and infrastructure software on customer GPU virtual machines, managed Kubernetes clusters or marketplace OpenShift","categories":["ai-tools","ai-inference-gateways"],"category":"AI tools \u0026 infrastructure","categoryFit":{"ai-inference-gateways":"Includes supported inference and model-serving components for deploying generative-AI and machine-learning workloads.","ai-tools":"Provides an enterprise-supported suite of AI frameworks, infrastructure software, operations components and development tools."},"cloudEvidence":[{"clouds":["aws","azure","gcp"],"notes":"NVIDIA documents standard instances, NVIDIA VM images, GPU-optimized VM images, managed Kubernetes and marketplace OpenShift paths across AWS, Azure and Google Cloud. License inclusion varies by image and offer; standard instances and managed Kubernetes generally require BYOL.","reviewed":"2026-11-16","sourceType":"vendor-documentation","title":"NVIDIA AI Enterprise on AWS, Azure and Google Cloud","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/overview.html","vendorSupport":"not-assessed"},{"clouds":["oci","alibaba"],"notes":"NVIDIA's current cloud matrix includes standard instances on Oracle Cloud Infrastructure and Alibaba Cloud, NVIDIA AI Enterprise VM images on OCI and GPU-optimized VM images on Alibaba Cloud. Customers must confirm the supported component set, certified instance and license path.","reviewed":"2026-11-16","sourceType":"vendor-documentation","title":"NVIDIA AI Enterprise on Oracle Cloud and Alibaba Cloud","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/overview.html","vendorSupport":"not-assessed"},{"applicableEdition":"NVIDIA AI Enterprise standard-instance BYOL","clouds":["tencent"],"notes":"NVIDIA documents installing its licensed stack on customer standard instances using a certified Ubuntu operating system, supported NVIDIA drivers, Docker and Container Toolkit, then pulling NGC containers. This is a BYOL installation rather than an included-license marketplace image. Select an instance from the supported-cloud-services matrix; the guide states one license per cloud instance, which must be reconciled with the general licensing guide and the customer agreement.","reviewed":"2026-11-16","sourceType":"vendor-documentation","title":"NVIDIA AI Enterprise on Tencent Cloud standard instances","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/tencent-standard-instance.html","vendorSupport":"not-assessed"},{"applicableEdition":"NVIDIA AI Enterprise standard-instance BYOL","clouds":["volcano"],"notes":"NVIDIA documents installing its licensed stack on customer standard instances using a certified Ubuntu operating system, supported NVIDIA drivers, Docker and Container Toolkit, then pulling NGC containers. This is a BYOL installation rather than an included-license marketplace image. Select an instance from the supported-cloud-services matrix; the guide states one license per cloud instance, which must be reconciled with the general licensing guide and the customer agreement.","reviewed":"2026-11-16","sourceType":"vendor-documentation","title":"NVIDIA AI Enterprise on ByteDance Volcano Engine standard instances","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/bytedance-volcano-engine-standard-instance.html","vendorSupport":"not-assessed"}],"cloudReviewed":"2026-11-16","clouds":["aws","azure","gcp","oci","alibaba","tencent","volcano"],"deploymentPaths":[{"boundary":"cloud-account","cloud":"aws","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"cloud-account","cloud":"azure","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"cloud-account","cloud":"gcp","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"cloud-account","cloud":"oci","controlPlane":"not-assessed","evidence":[1],"operator":"customer"},{"boundary":"cloud-account","cloud":"alibaba","controlPlane":"not-assessed","evidence":[1],"operator":"customer"},{"boundary":"cloud-account","cloud":"tencent","controlPlane":"not-assessed","evidence":[2],"operator":"customer"},{"boundary":"cloud-account","cloud":"volcano","controlPlane":"not-assessed","evidence":[3],"operator":"customer"}],"description":"Commercial enterprise AI software suite deployed on customer cloud GPU instances or managed Kubernetes clusters.","diligence":{"airGap":{"reviewed":"2026-11-16","scope":"NVIDIA License System DLS workflow; component licensing and application dependencies differ","source":"https://docs.nvidia.com/license-system/latest/nvidia-license-system-quick-start-guide/","status":"documented","summary":"NVIDIA documents registering a Delegated License Service appliance on an air-gapped network by manually transferring registration details through its Licensing Portal. This confirms an offline licensing workflow for applicable components, not disconnected compatibility of every AI Enterprise service or model.","supportingSources":[{"title":"When AI Enterprise requires the license system","url":"https://docs.nvidia.com/ai-enterprise/planning-resource/licensing-guide/latest/license-system.html"}]},"commitments":{"availability":"edition-dependent","reviewed":"2026-11-16","scope":"Vendor statement for AI Enterprise AWS Marketplace private offers only","source":"https://www.nvidia.com/en-us/data-center/lp/aws-marketplace-offer/","status":"documented","summary":"NVIDIA explicitly states that AI Enterprise AWS Marketplace private offers are eligible toward an AWS committed-spend agreement. Confirm the specific offer and buyer agreement; this statement does not establish Azure/GCP credit or eligibility for every image or BYOL purchase."},"compliance":{"reviewed":"2026-11-16","scope":"Selected software-artifact assurances versus a customer-deployment audit","source":"https://docs.nvidia.com/ai-enterprise/planning-resource/ai-enterprise-infrastructure-software/latest/enterprise-software-built-on-open-source.html","status":"inconclusive","summary":"Government Ready, STIG-hardening and FIPS-library statements describe selected NVIDIA software branches. The reviewed page does not establish a current independent attestation for the complete customer-account deployment, third-party integrations or every suite component."},"footprint":{"reviewed":"2026-11-20","scope":"Suite/component/model dependent; complete workload footprint unresolved","source":"https://docs.nvidia.com/ai-enterprise/latest/user-guide/index.html","status":"inconclusive","summary":"The suite includes multiple frameworks, drivers, operators and NIM services, each with its own support matrix. NIM directs buyers to model- and hardware-specific prerequisites. No suite-wide CPU/RAM/VRAM minimum is established; select components, model profiles, supported GPUs, concurrency and redundancy before sizing.","supportingSources":[{"title":"NIM component-specific prerequisite path","url":"https://docs.nvidia.com/nim/large-language-models/latest/get-started/index.html"}]},"iac":{"reviewed":"2026-11-16","scope":"Supported cloud matrix; component, operating system and instance compatibility must all match","source":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/overview.html","status":"documented","summary":"NVIDIA provides provider-specific image, standard-instance and Kubernetes procedures rather than one universal template. Tencent and Volcano standard-instance guides require a supported instance, certified Ubuntu, NVIDIA driver, Docker, Container Toolkit and NGC tooling, with BYOL.","supportingSources":[{"title":"Tencent standard instance","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/tencent-standard-instance.html"},{"title":"Volcano Engine standard instance","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/bytedance-volcano-engine-standard-instance.html"}]},"licensePricing":{"reviewed":"2026-11-20","scope":"Self-managed NVIDIA Partner Network list price; CSP marketplace terms differ","source":"https://docs.nvidia.com/ai-enterprise/planning-resource/licensing-guide/latest/pricing.html","status":"documented","summary":"The current licensing guide lists a one-year self-managed subscription at USD 4,500 per GPU including standard support. Multi-year, perpetual, educational and partner terms differ. Cloud-marketplace and bundled-hardware entitlements have separate rules; this software list price excludes GPU infrastructure and is not a workload quote. For Tencent/Volcano BYOL paths, the previously noted per-instance versus per-GPU unit still needs contract confirmation.","supportingSources":[{"title":"Cloud BYOL unit requires separate confirmation","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/tencent-standard-instance.html"},{"title":"Volcano Engine BYOL path","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/bytedance-volcano-engine-standard-instance.html"}]},"marketplace":{"availability":"edition-dependent","reviewed":"2026-11-16","scope":"Offer-specific licensing; no cloud-commitment eligibility established","source":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/overview.html","status":"documented","summary":"The deployment matrix distinguishes license-included AI Enterprise VM images/private offers from BYOL paths. A generic GPU-optimized image or Kubernetes installation does not include AI Enterprise licensing merely because it comes from a cloud provider."},"release":{"reviewed":"2026-11-18","scope":"AI Enterprise component and branch lifecycle; no single suite version covers every deployment","source":"https://docs.nvidia.com/ai-enterprise/lifecycle/latest/index.html","status":"documented","summary":"NVIDIA's November 2026 lifecycle material lists active 9.0/R620, 8.3/R595, and 7.9 LTSB/R580 branches with distinct support windows. NeMo Microservices support ended October 1, 2026, after 25.12 was designated the final GA release."},"securityControls":{"reviewed":"2026-11-16","scope":"AI Enterprise infrastructure software and application Production Branch; not an independent customer-environment attestation","source":"https://docs.nvidia.com/ai-enterprise/planning-resource/ai-enterprise-infrastructure-software/latest/enterprise-software-built-on-open-source.html","status":"documented","summary":"NVIDIA describes Government Ready infrastructure software and Production Branch application software using STIG-hardened containers, FIPS-compliant libraries and monthly security/maintenance patches. The statement is branch-specific; it does not certify every suite component, cloud image or customer deployment."},"sso":{"reviewed":"2026-11-20","scope":"Suite-wide runtime SSO cannot be established from distribution/account access","source":"https://docs.nvidia.com/ai-enterprise/latest/user-guide/index.html","status":"inconclusive","summary":"AI Enterprise is a suite with independently versioned application and infrastructure components. NGC/catalog account authentication does not establish SAML/OIDC for every deployed microservice or workload. Select the actual NIM/framework/management components and confirm their runtime identity integrations and entitlements."}},"id":"nvidia-ai-enterprise","infrastructure":["servers"],"infrastructureNotes":"NVIDIA AI Enterprise runs licensed AI containers and infrastructure components on persistent customer GPU or CPU instances and Kubernetes worker nodes.","infrastructureReviewed":"2026-11-16","infrastructureSources":["https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/overview.html","https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/index.html","https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/tencent-standard-instance.html","https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/bytedance-volcano-engine-standard-instance.html"],"install":"NVIDIA AI Enterprise or GPU-optimized cloud VM images, standard instances with BYOL, or managed Kubernetes on supported cloud providers","installMethods":["vm-image","helm","kubernetes"],"kind":"product","license":"Commercial NVIDIA AI Enterprise subscription, consumption or perpetual license; most customer-cloud paths require BYOL.","licenseModel":"commercial-only","licenseModelNotes":"NVIDIA AI Enterprise requires a commercial subscription, consumption or perpetual entitlement. Cloud instances are usually BYOL unless the selected on-demand VMI or private offer explicitly includes licensing. The Tencent and Volcano standard-instance guides state one license per cloud instance, while the general licensing guide describes per-GPU BYOL. Confirm the applicable unit for the selected instance, GPU count and agreement before purchase.","licenseModelSources":["https://docs.nvidia.com/ai-enterprise/planning-resource/licensing-guide/latest/licensing.html","https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/tencent-standard-instance.html","https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/bytedance-volcano-engine-standard-instance.html"],"licenseReviewed":"2026-11-16","model":"Fully in your account","modelLabel":"Customer-operated","name":"NVIDIA AI Enterprise","notes":"This entry covers customer-operated NVIDIA AI Enterprise software, not a managed AI service. A license is required and is not included with most standard, GPU-optimized VMI or managed-Kubernetes deployments; on-demand NVIDIA AI Enterprise VMIs and private offers use separate included-license processes. Not every component is available on every cloud or deployment model. GPU deployments are generally licensed per GPU, while CPU-only instances require an instance license. Tencent Cloud and ByteDance Volcano Engine have documented standard-instance BYOL paths. These guides require certified software and supported instance types; they do not establish that every GPU, component or provider image is eligible. Their per-instance wording differs from the general per-GPU licensing guidance, so confirm the applicable licensing unit. The November lifecycle matrix lists active 9.0/R620, 8.3/R595, and 7.9 LTSB/R580 branches with different end dates. NeMo Microservices support ended October 1, 2026, after 25.12 became the final GA release; select and patch each component branch independently.","operations":"Customer manages cloud GPU instances or Kubernetes, drivers, GPU Operator and container runtime, networking, storage, identities, NGC access, model services, monitoring, upgrades and license activation.","publisher":false,"reviewed":"2026-11-18","sources":[{"title":"NVIDIA AI Enterprise cloud deployment guide","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/index.html"},{"title":"NVIDIA AI Enterprise cloud deployment overview","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/overview.html"},{"title":"NVIDIA AI Enterprise on AWS","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/aws-overview.html"},{"title":"NVIDIA AI Enterprise on Azure","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/azure-overview.html"},{"title":"NVIDIA AI Enterprise on Google Cloud","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/google-overview.html"},{"title":"NVIDIA AI Enterprise licensing","url":"https://docs.nvidia.com/ai-enterprise/planning-resource/licensing-guide/latest/licensing.html"},{"title":"NVIDIA AI Enterprise on Tencent Cloud standard instances","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/tencent-standard-instance.html"},{"title":"NVIDIA AI Enterprise on ByteDance Volcano Engine standard instances","url":"https://docs.nvidia.com/ai-enterprise/deployment/cloud/latest/bytedance-volcano-engine-standard-instance.html"},{"title":"NVIDIA AI Enterprise documentation","url":"https://docs.nvidia.com/ai-enterprise/"},{"title":"NVIDIA AI Enterprise lifecycle","url":"https://docs.nvidia.com/ai-enterprise/lifecycle/latest/index.html"}],"url":"https://launchinyourcloud.com/directory/nvidia-ai-enterprise/","website":"https://www.nvidia.com/en-us/data-center/products/ai-enterprise/"},{"architecture":"Customer-operated model-serving process, model weights and inference hardware","categories":["ai-inference-gateways","ai-tools"],"category":"AI inference \u0026 gateways","categoryFit":{"ai-inference-gateways":"Model inference engine and OpenAI-compatible serving API for compatible model weights. Distinct from an API gateway that forwards calls to proprietary hosted models.","ai-tools":"Self-hosted model inference and serving engine with OpenAI-compatible APIs."},"cloudEvidence":[{"clouds":["aws"],"notes":"The project publishes Amazon EKS provisioning and vLLM deployment scripts in its versioned Production Stack guide. Verify current chart versions, model licenses, accelerator availability, GPU quotas and storage; this is customer-hosted inference, not a hosted model API.","reviewed":"2026-11-12","sourceType":"vendor-documentation","title":"vLLM Production Stack deployment on Amazon EKS","url":"https://docs.vllm.ai/projects/production-stack/en/vllm-stack-0.1.4/deployment/cloud-deployment/aws.html","vendorSupport":"not-assessed"},{"clouds":["azure"],"notes":"The project publishes Azure Kubernetes Service provisioning and vLLM deployment scripts in its versioned Production Stack guide. Verify current chart versions, model licenses, accelerator availability, GPU quotas and storage; this is customer-hosted inference, not a hosted model API.","reviewed":"2026-11-12","sourceType":"vendor-documentation","title":"vLLM Production Stack deployment on Azure Kubernetes Service","url":"https://docs.vllm.ai/projects/production-stack/en/vllm-stack-0.1.4/deployment/cloud-deployment/azure.html","vendorSupport":"not-assessed"},{"clouds":["gcp"],"notes":"The project publishes Google Kubernetes Engine provisioning and vLLM deployment scripts in its versioned Production Stack guide. Verify current chart versions, model licenses, accelerator availability, GPU quotas and storage; this is customer-hosted inference, not a hosted model API.","reviewed":"2026-11-12","sourceType":"vendor-documentation","title":"vLLM Production Stack deployment on Google Kubernetes Engine","url":"https://docs.vllm.ai/projects/production-stack/en/vllm-stack-0.1.4/deployment/cloud-deployment/gcp.html","vendorSupport":"not-assessed"}],"cloudReviewed":"2026-11-12","clouds":["aws","azure","gcp"],"deploymentPaths":[{"boundary":"cloud-account","cloud":"aws","controlPlane":"not-assessed","evidence":[0],"operator":"customer"},{"boundary":"cloud-account","cloud":"azure","controlPlane":"not-assessed","evidence":[1],"operator":"customer"},{"boundary":"cloud-account","cloud":"gcp","controlPlane":"not-assessed","evidence":[2],"operator":"customer"}],"description":"Self-hosted model inference and serving engine with OpenAI-compatible APIs.","diligence":{"footprint":{"reviewed":"2026-11-20","scope":"Model-specific inference sizing; no universal numeric footprint established","source":"https://docs.vllm.ai/en/stable/configuration/conserving_memory/","status":"inconclusive","summary":"The stable memory guide sizes serving around model weights, precision/quantization, tensor parallelism, context length, batch size and cache. A supported accelerator alone does not establish sufficient memory. Select the exact model and target throughput/latency, then benchmark VRAM/RAM and HA replica requirements."},"iac":{"artifactRef":"select a stable version and architecture tag; latest is moving","artifactUrl":"https://hub.docker.com/r/vllm/vllm-openai","reviewed":"2026-11-14","scope":"Stable upstream CUDA image path; not the latest developer-preview guide or a GPU sizing guarantee","source":"https://docs.vllm.ai/en/stable/deployment/docker/","status":"documented","summary":"The stable deployment guide links the official vllm/vllm-openai image for a CUDA-backed OpenAI-compatible server. Select a release/architecture tag and compatible GPU drivers; model memory, credentials and optional dependencies need separate review."},"licensePricing":{"reviewed":"2026-11-20","scope":"Upstream core licence; commercial features, support and hosting priced separately","source":"https://raw.githubusercontent.com/vllm-project/vllm/main/LICENSE","status":"documented","summary":"The reviewed upstream core is Apache-2.0 licensed. Optional services, bundled dependencies, commercial distributions and support can have separate terms. This records the source licence, not an infrastructure or operating-cost estimate. Model weights and external model APIs require their own review."},"release":{"releasePublishedAt":"2026-11-16T23:22:00Z","releaseTag":"v0.34.0","reviewed":"2026-11-17","scope":"Upstream vllm-project/vllm release record; commercial editions, LTS branches and provider packages can differ","source":"https://github.com/vllm-project/vllm/releases/tag/v0.34.0","status":"documented","summary":"The reviewed upstream release is v0.34.0, published 2026-11-16. This is a dated release snapshot, not a support designation, a future cadence promise or approval to replace a provider package."},"sso":{"reviewed":"2026-11-20","scope":"Upstream inference-server auth surface; federated user integration unresolved","source":"https://docs.vllm.ai/en/latest/usage/security/","status":"inconclusive","summary":"The security guide documents API-key protection for specified HTTP path prefixes and warns that other sensitive endpoints remain unprotected. It does not establish a native SAML/OIDC user sign-on workflow. Use a reviewed gateway/network design; an API bearer key alone is not corporate SSO or complete endpoint protection."}},"id":"vllm","infrastructure":["servers"],"infrastructureNotes":"vLLM is a running inference-server workload. Hardware support and capacity depend on the model and selected backend; the documented container paths include accelerator-specific images, and CPU support is separately documented. In v0.34.0, the KV-cache allocator sizes from free memory after warmup and defaults to gpu_memory_utilization=1.0; revalidate GPU headroom, and set 0.9 or disable extensible cache to retain prior sizing. ROCm 10.0 replaces 7.2 for published images and wheels.","infrastructureReviewed":"2026-11-17","infrastructureSources":["https://docs.vllm.ai/en/latest/deployment/docker/","https://github.com/vllm-project/vllm/releases/tag/v0.34.0"],"install":"Official vLLM Docker image or supported Python installation; choose a compatible hardware/backend configuration.","installMethods":["docker","source"],"kind":"product","license":"Apache 2.0 for vLLM; model weights and supporting dependencies have independent licenses and access conditions.","licenseModel":"open-source","licenseModelNotes":"The reviewed inference runtime is Apache-2.0 licensed. Its license does not grant rights to the model weights served by that runtime; check model and dependency terms separately.","licenseModelSources":["https://github.com/vllm-project/vllm/blob/main/LICENSE"],"licenseReviewed":"2026-11-10","model":"Fully in your account","modelLabel":"Customer-operated","name":"vLLM Self-Hosted","notes":"This entry covers the inference server, not a hosted model service. It can run supported model weights on customer infrastructure, but those weights may need external downloads or access approval and are not licensed by vLLM. Review default usage telemetry and optional outbound integrations; hosting inference does not automatically create an air-gapped system. vLLM v0.34.0 removes Model Runner V1; configurations setting VLLM_USE_V2_MODEL_RUNNER=0 now fail at startup. It changes the default block size from 16 to 64, enables extensible KV cache by default with gpu_memory_utilization defaulting to 1.0, drops ROCm 7.2 wheels and images in favor of ROCm 10.0, requires huggingface_hub 2.0.0 or later, and removes QuTLASS/fp_quant and Marlin W4A8 support. Review the upstream breaking-change table and test model, quantization, memory, and accelerator compatibility before rollout.","operations":"Customer manages model acquisition, hardware capacity, endpoint protection, availability, upgrades and model/runtime compatibility.","publisher":false,"reviewed":"2026-11-17","sources":[{"title":"Official Docker serving images","url":"https://docs.vllm.ai/en/latest/deployment/docker/"},{"title":"Online serving","url":"https://docs.vllm.ai/en/latest/serving/online_serving/"},{"title":"Usage statistics and opt-out","url":"https://docs.vllm.ai/en/latest/usage/usage_stats/"},{"title":"Apache 2.0 license","url":"https://github.com/vllm-project/vllm/blob/main/LICENSE"},{"title":"vLLM v0.34.0 breaking upgrade notes","url":"https://github.com/vllm-project/vllm/releases/tag/v0.34.0"}],"url":"https://launchinyourcloud.com/directory/vllm/","website":"https://vllm.ai/"}],"schemaVersion":1,"url":"https://launchinyourcloud.com/clouds/azure/ai-inference-gateways/"}