Current section
Files
Jump to
Current section
Files
lib/snakebridge_generated/vllm/__init__.ex
# Generated by SnakeBridge v0.15.1 - DO NOT EDIT MANUALLY
# Regenerate with: mix compile
# Library: vllm 0.14.0
# Python module: vllm
defmodule Vllm do
@moduledoc """
vLLM: a high-throughput and memory-efficient inference engine for LLMs
## Version
- Requested: 0.14.0
- Observed at generation: 0.14.0
## Runtime Options
All functions accept a `__runtime__` option for controlling execution behavior:
Vllm.some_function(args, __runtime__: [timeout: 120_000])
### Supported runtime options
- `:timeout` - Call timeout in milliseconds (default: 120,000ms / 2 minutes)
- `:timeout_profile` - Use a named profile (`:default`, `:ml_inference`, `:batch_job`, `:streaming`)
- `:stream_timeout` - Timeout for streaming operations (default: 1,800,000ms / 30 minutes)
- `:session_id` - Override the session ID for this call
- `:pool_name` - Target a specific Snakepit pool (multi-pool setups)
- `:affinity` - Override session affinity (`:hint`, `:strict_queue`, `:strict_fail_fast`)
### Timeout Profiles
- `:default` - 2 minute timeout for regular calls
- `:ml_inference` - 10 minute timeout for ML/LLM workloads
- `:batch_job` - Unlimited timeout for long-running jobs
- `:streaming` - 2 minute timeout, 30 minute stream_timeout
### Example with timeout override
# For a long-running ML inference call
Vllm.predict(data, __runtime__: [timeout_profile: :ml_inference])
# Or explicit timeout
Vllm.predict(data, __runtime__: [timeout: 600_000])
# Route to a pool and enforce strict affinity
Vllm.predict(data, __runtime__: [pool_name: :strict_pool, affinity: :strict_queue])
See `SnakeBridge.Defaults` for global timeout configuration.
"""
@doc false
def __snakebridge_python_name__, do: "vllm"
@doc false
def __snakebridge_library__, do: "vllm"
@doc false
def __functions__ do
[
{:modality_data, 0, __MODULE__, ""},
{:extract_trace_context, 1, __MODULE__, ""},
{:stateless_init_torch_distributed_process_group, 5, __MODULE__,
"A replacement for `torch.distributed.init_process_group` that does not"},
{:vllm_xla_check_recompilation, 0, __MODULE__, ""},
{:ld_library_path, 0, __MODULE__, ""},
{:vllm_use_standalone_compile, 0, __MODULE__, ""},
{:multi_modal_data_dict, 0, __MODULE__, ""},
{:_should_log_with_scope, 1, __MODULE__, "Decide whether to log based on scope"},
{:vllm_use_nccl_symm_mem, 0, __MODULE__, ""},
{:vllm_enable_inductor_coordinate_descent_tuning, 0, __MODULE__, ""},
{:get_conda_packages, 1, __MODULE__, ""},
{:vllm_marlin_use_atomic_add, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:vllm_media_connector, 0, __MODULE__, ""},
{:vllm_deep_gemm_warmup, 0, __MODULE__, ""},
{:logprobs_one_position, 0, __MODULE__, "dict() -> new empty dictionary"},
{:is_global_first_rank, 0, __MODULE__,
"Check if the current process is the first rank globally across all"},
{:vllm_kv_events_use_int_block_hashes, 0, __MODULE__, ""},
{:run_and_return_first_line, 2, __MODULE__,
"Run command using run_lambda and returns first line if output is not empty."},
{:multi_modal_data_dict, 0, __MODULE__,
"A Mapping is a generic container for associating key/value"},
{:vllm_video_loader_backend, 0, __MODULE__, ""},
{:_o, 0, __MODULE__, ""},
{:get_env_or_set_default, 0, __MODULE__, ""},
{:is_otel_available, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter_triton_gemm, 0, __MODULE__, ""},
{:_compute_chunked_local_num_tokens, 4, __MODULE__, ""},
{:multi_modal_placeholder_dict, 0, __MODULE__,
"A Mapping is a generic container for associating key/value"},
{:vllm_rocm_use_aiter_fp8_bmm, 0, __MODULE__, ""},
{:compile_factors, 0, __MODULE__, "Return env vars used for torch.compile cache keys."},
{:cpu_platform_plugin, 0, __MODULE__, ""},
{:vllm_logging_color, 0, __MODULE__, ""},
{:vllm_dp_rank_local, 0, __MODULE__, ""},
{:platform_plugins_group, 0, __MODULE__, ""},
{:ensure_divisibility, 2, __MODULE__,
"Ensure that numerator is divisible by the denominator."},
{:model_parallel_is_initialized, 0, __MODULE__,
"Check if tensor and pipeline parallel groups are initialized."},
{:otel_import_error_traceback, 0, __MODULE__, ""},
{:get_nvidia_smi, 0, __MODULE__, ""},
{:vllm_rocm_custom_paged_attn, 0, __MODULE__, ""},
{:vllm_tool_parse_regex_timeout_seconds, 0, __MODULE__, ""},
{:vllm_rocm_fp8_mfma_page_attn, 0, __MODULE__, ""},
{:get_cached_compilation_config, 0, __MODULE__,
"Cache config to avoid repeated calls to get_current_vllm_config()"},
{:format_model_inspection, 1, __MODULE__,
"Format a model into a transformers-style hierarchical string."},
{:vllm_api_key, 0, __MODULE__, ""},
{:current_platform, 0, __MODULE__, ""},
{:get_decode_context_model_parallel_rank, 0, __MODULE__,
"Return my rank for the decode context model parallel group."},
{:vllm_audio_fetch_timeout, 0, __MODULE__, ""},
{:vllm_image_fetch_timeout, 0, __MODULE__, ""},
{:vllm_dp_rank, 0, __MODULE__, ""},
{:default_logging_config, 0, __MODULE__, ""},
{:_compute_sp_num_tokens, 2, __MODULE__, ""},
{:__getattr__, 1, __MODULE__, ""},
{:_print_warning_once, 2, __MODULE__, "Logging configuration for vLLM."},
{:patched_fused_scaled_matmul_reduce_scatter, 9, __MODULE__, ""},
{:vllm_mxfp4_use_marlin, 0, __MODULE__, ""},
{:platform_plugins_group, 0, __MODULE__, ""},
{:vllm_logging_config_path, 0, __MODULE__, ""},
{:all_gather_fake, 4, __MODULE__, ""},
{:prompt_logprobs, 0, __MODULE__, ""},
{:memory_plan_reuse_patched, 1, __MODULE__, ""},
{:trace_headers, 0, __MODULE__, ""},
{:get_tensor_model_parallel_rank, 0, __MODULE__,
"Return my rank for the tensor model parallel group."},
{:multi_modal_uuid_dict, 0, __MODULE__,
"A Mapping is a generic container for associating key/value"},
{:logger, 0, __MODULE__, ""},
{:extract_trace_headers, 1, __MODULE__, ""},
{:ensure_model_parallel_initialized, 2, __MODULE__,
"Helper to initialize model parallel groups if they are not initialized,"},
{:vllm_all2_all_backend, 0, __MODULE__, ""},
{:vllm_model_redirect_path, 0, __MODULE__, ""},
{:vllm_profiler_delay_iters, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:vllm_use_ray_compiled_dag_channel_type, 0, __MODULE__, ""},
{:str_dtype_to_torch_dtype, 1, __MODULE__, ""},
{:__dir__, 0, __MODULE__, ""},
{:vllm_use_flashinfer_moe_fp16, 0, __MODULE__, ""},
{:get_dcp_group, 0, __MODULE__, ""},
{:_print_info_once, 2, __MODULE__, "Logging configuration for vLLM."},
{:vllm_debug_dump_path, 0, __MODULE__, ""},
{:to_enc_dec_tuple_list, 1, __MODULE__, ""},
{:env_info_fmt, 0, __MODULE__, ""},
{:vllm_rocm_sleep_mem_chunk_size, 0, __MODULE__, ""},
{:vllm_moriio_connector_read_mode, 0, __MODULE__, ""},
{:vllm_skip_p2_p_check, 0, __MODULE__, ""},
{:pretty_str, 1, __MODULE__, ""},
{:verbose, 0, __MODULE__, ""},
{:vllm_gpt_oss_harmony_system_instructions, 0, __MODULE__, ""},
{:get_cuda_module_loading_config, 0, __MODULE__, ""},
{:vllm_deepep_high_throughput_force_intra_node, 0, __MODULE__, ""},
{:singleton_prompt, 0, __MODULE__, ""},
{:vllm_config_root, 0, __MODULE__, ""},
{:__getattr__, 1, __MODULE__, "Gets environment variables lazily."},
{:vllm_v1_use_outlines_cache, 0, __MODULE__, ""},
{:vllm_debug_workspace, 0, __MODULE__, ""},
{:get_decode_context_model_parallel_world_size, 0, __MODULE__,
"Return world size for the decode context model parallel group."},
{:plugins_loaded, 0, __MODULE__, ""},
{:get_cachingallocator_config, 0, __MODULE__, ""},
{:disable_envs_cache, 0, __MODULE__,
"Resets the environment variables cache. It could be used to isolate environments"},
{:vllm_rocm_use_aiter_unified_attention, 0, __MODULE__, ""},
{:vllm_tuned_config_folder, 0, __MODULE__, ""},
{:vllm_sleep_when_idle, 0, __MODULE__, ""},
{:vllm_use_v2_model_runner, 0, __MODULE__, ""},
{:build_explicit_enc_dec_prompt, 2, __MODULE__, ""},
{:vllm_kv_cache_layout, 0, __MODULE__, ""},
{:vllm_nvfp4_gemm_backend, 0, __MODULE__, ""},
{:enable_trace_function_call, 1, __MODULE__,
"Enable tracing of every function call in code under `root_dir`."},
{:vllm_force_aot_load, 0, __MODULE__, ""},
{:vllm_flashinfer_workspace_buffer_size, 0, __MODULE__, ""},
{:is_forward_context_available, 0, __MODULE__, ""},
{:length_from_prompt_token_ids_or_embeds, 2, __MODULE__,
"Calculate the request length (in number of tokens) give either"},
{:get_ep_group, 0, __MODULE__, ""},
{:vllm_debug_log_api_server_response, 0, __MODULE__, ""},
{:_get_child_signature, 1, __MODULE__,
"Get a signature for a child module to detect duplicates."},
{:_current_platform, 0, __MODULE__, ""},
{:vllm_nixl_abort_request_timeout, 0, __MODULE__, ""},
{:stat_logger_plugins_group, 0, __MODULE__, ""},
{:vllm_moriio_post_batch_size, 0, __MODULE__, ""},
{:vllm_lora_resolver_cache_dir, 0, __MODULE__, ""},
{:load_general_plugins, 0, __MODULE__,
"WARNING: plugins can be loaded for multiple times in different"},
{:vllm_ringbuffer_warning_interval, 0, __MODULE__, ""},
{:split_tensor_along_last_dim, 2, __MODULE__, "Split a tensor along its last dimension."},
{:vllm_rocm_quick_reduce_max_size_bytes_mb, 0, __MODULE__, ""},
{:default_pip_patterns, 0, __MODULE__, ""},
{:supported_task, 0, __MODULE__, ""},
{:default_conda_patterns, 0, __MODULE__, ""},
{:vllm_docker_build_context, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:tpu_platform_plugin, 0, __MODULE__, ""},
{:vllm_execute_model_timeout_seconds, 0, __MODULE__, ""},
{:batchsize_logging_interval, 0, __MODULE__, ""},
{:vllm_use_ray_compiled_dag_overlap_comm, 0, __MODULE__, ""},
{:cuda_visible_devices, 0, __MODULE__, ""},
{:vllm_use_modelscope, 0, __MODULE__, ""},
{:vllm_enable_cudagraph_gc, 0, __MODULE__, ""},
{:get_mac_version, 1, __MODULE__, ""},
{:is_set, 1, __MODULE__, "Check if an environment variable is explicitly set."},
{:xpu_platform_plugin, 0, __MODULE__, ""},
{:run_and_read_all, 2, __MODULE__,
"Run command using run_lambda; reads and returns entire output if rc is 0."},
{:vllm_allow_chunked_local_attn_with_hybrid_kv_cache, 0, __MODULE__, ""},
{:patch_tensor_parallel_group, 1, __MODULE__,
"Patch the tp group temporarily until this function ends."},
{:run, 1, __MODULE__, "Return (return-code, stdout, stderr)."},
{:get_vllm_version, 0, __MODULE__, ""},
{:vllm_enable_responses_api_store, 0, __MODULE__, ""},
{:vllm_configure_logging, 0, __MODULE__, ""},
{:get_context_model_parallel_group, 0, __MODULE__, ""},
{:get_span_exporter, 1, __MODULE__, ""},
{:zip_enc_dec_prompts, 2, __MODULE__,
"Zip encoder and decoder prompts together into a list of"},
{:environment_variables, 0, __MODULE__, ""},
{:get_node_count, 0, __MODULE__,
"Return the total number of nodes in the distributed environment."},
{:get_attr_docs, 1, __MODULE__,
"Get any docstrings placed after attribute assignments in a class body."},
{:vllm_cudart_so_path, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:vllm_torch_cuda_profile, 0, __MODULE__, ""},
{:_format_index_ranges, 1, __MODULE__,
"Format indices into range notation (e.g., [0,1,2,4,5,6] -> '0-2, 4-6')."},
{:get_pip_packages, 1, __MODULE__,
"Return `pip list` output. Note: will also find conda-installed pytorch and numpy packages."},
{:vllm_blockscale_fp8_gemm_flashinfer, 0, __MODULE__, ""},
{:vllm_use_deep_gemm_tma_aligned_scales, 0, __MODULE__, ""},
{:get_vllm_port, 0, __MODULE__, "Get the port from VLLM_PORT environment variable."},
{:vllm_fused_moe_chunk_size, 0, __MODULE__, ""},
{:vllm_use_experimental_parser_context, 0, __MODULE__, ""},
{:s3_endpoint_url, 0, __MODULE__, ""},
{:vllm_tpu_using_pathways, 0, __MODULE__, ""},
{:vllm_disabled_kernels, 0, __MODULE__, ""},
{:broadcast_tensor_dict, 0, __MODULE__, ""},
{:vllm_http_timeout_keep_alive, 0, __MODULE__, ""},
{:get_libc_version, 0, __MODULE__, ""},
{:vllm_object_storage_shm_buffer_name, 0, __MODULE__, ""},
{:vllm_mla_disable, 0, __MODULE__, ""},
{:_date_format, 0, __MODULE__, ""},
{:get_pp_indices, 3, __MODULE__, "Try to evenly distribute layers across partitions."},
{:init_gloo_process_group, 4, __MODULE__,
"Stateless init ProcessGroup with gloo backend compatible with"},
{:create_forward_context, 2, __MODULE__, ""},
{:is_local_first_rank, 0, __MODULE__,
"Check if the current process is the first local rank (rank 0 on its node)."},
{:vllm_engine_iteration_timeout_s, 0, __MODULE__, ""},
{:_forward_context, 0, __MODULE__, ""},
{:vllm_moriio_num_workers, 0, __MODULE__, ""},
{:vllm_rocm_use_skinny_gemm, 0, __MODULE__, ""},
{:init_logger, 1, __MODULE__,
"The main purpose of this function is to ensure that loggers are"},
{:is_xnnpack_available, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter_paged_attn, 0, __MODULE__, ""},
{:init_world_group, 3, __MODULE__, ""},
{:vllm_enable_inductor_max_autotune, 0, __MODULE__, ""},
{:_is_otel_imported, 0, __MODULE__, ""},
{:init_distributed_environment, 0, __MODULE__, ""},
{:nested_tensors, 0, __MODULE__, ""},
{:get_env_vars, 0, __MODULE__, ""},
{:vllm_custom_scopes_for_profiling, 0, __MODULE__, ""},
{:has_triton, 0, __MODULE__, ""},
{:vllm_cpu_omp_threads_bind, 0, __MODULE__, ""},
{:vllm_allreduce_use_symm_mem, 0, __MODULE__, ""},
{:vllm_moe_dp_chunk_size, 0, __MODULE__, ""},
{:set_custom_all_reduce, 1, __MODULE__, ""},
{:_max_temp, 0, __MODULE__, ""},
{:get_windows_version, 1, __MODULE__, ""},
{:vllm_compute_nans_in_logits, 0, __MODULE__, ""},
{:prompt_type, 0, __MODULE__, ""},
{:get_default_cache_root, 0, __MODULE__, ""},
{:env_with_choices, 3, __MODULE__,
"Create a lambda that validates environment variable against allowed choices"},
{:get_platform, 0, __MODULE__, ""},
{:get_gpu_info, 1, __MODULE__, ""},
{:get_lsb_version, 1, __MODULE__, ""},
{:u_batch_slices, 0, __MODULE__, "Built-in mutable sequence."},
{:vllm_randomize_dp_dummy_inputs, 0, __MODULE__, ""},
{:vllm_nccl_so_path, 0, __MODULE__, ""},
{:batchsize_forward_time, 0, __MODULE__, ""},
{:nvcc_threads, 0, __MODULE__, ""},
{:vllm_gpt_oss_system_tool_mcp_labels, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:get_pcp_group, 0, __MODULE__, ""},
{:summarize_vllm_build_flags, 0, __MODULE__, ""},
{:vllm_nixl_side_channel_port, 0, __MODULE__, ""},
{:get_pretty_env_info, 0, __MODULE__, ""},
{:current_formatter_type, 0, __MODULE__, ""},
{:load_plugins_by_group, 1, __MODULE__, ""},
{:_use_color, 0, __MODULE__, "Logging configuration for vLLM."},
{:_trace_calls, 4, __MODULE__, "Logging configuration for vLLM."},
{:vllm_no_usage_stats, 0, __MODULE__, ""},
{:all_reduce_fake, 2, __MODULE__, ""},
{:config_type, 0, __MODULE__, "type(object) -> the object's type"},
{:vllm_pattern_match_debug, 0, __MODULE__, ""},
{:get_pp_group, 0, __MODULE__, ""},
{:vllm_torch_profiler_record_shapes, 0, __MODULE__, ""},
{:contains_trace_headers, 1, __MODULE__, ""},
{:iter_architecture_defaults, 0, __MODULE__, ""},
{:should_partition_patched, 2, __MODULE__,
"Return True if we should partition the inductor graph on this node"},
{:vllm_log_model_inspection, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:cached_tokenizer_from_config, 1, __MODULE__, ""},
{:create_sort_beams_key_function, 2, __MODULE__, ""},
{:vllm_rpc_base_path, 0, __MODULE__, ""},
{:k_scale_constant, 0, __MODULE__, ""},
{:get_env_info, 0, __MODULE__, ""},
{:get_current_vllm_config_or_none, 0, __MODULE__, ""},
{:torch_available, 0, __MODULE__, ""},
{:resolve_current_platform_cls_qualname, 0, __MODULE__, ""},
{:vllm_version_matches_substr, 1, __MODULE__,
"Check to see if the vLLM version matches a substring."},
{:s3_secret_access_key, 0, __MODULE__, ""},
{:vllm_media_loading_thread_count, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter_triton_rope, 0, __MODULE__, ""},
{:vllm_use_bytecode_hook, 0, __MODULE__, ""},
{:init_model_parallel_group, 3, __MODULE__, ""},
{:check_release_file, 1, __MODULE__, ""},
{:vllm_enable_fused_moe_activation_chunking, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:vllm_use_mega_aot_artifact, 0, __MODULE__, ""},
{:vllm_compile_cache_save_format, 0, __MODULE__, ""},
{:vllm_max_audio_clip_filesize_mb, 0, __MODULE__, ""},
{:vllm_nccl_include_path, 0, __MODULE__, ""},
{:get_cudnn_version, 1, __MODULE__,
"Return a list of libcudnn.so; it's hard to tell which one is being used."},
{:vllm_dp_size, 0, __MODULE__, ""},
{:vllm_torch_profiler_with_profile_memory, 0, __MODULE__, ""},
{:vllm_assets_cache, 0, __MODULE__, ""},
{:vllm_moe_use_deep_gemm, 0, __MODULE__, ""},
{:get_cmake_version, 1, __MODULE__, ""},
{:vllm_tool_json_error_automatic_retry, 0, __MODULE__, ""},
{:tensor_model_parallel_reduce_scatter, 1, __MODULE__,
"Reduce-Scatter the input tensor across model parallel group."},
{:get_nvidia_driver_version, 1, __MODULE__, ""},
{:init_logger, 1, __MODULE__,
"The main purpose of this function is to ensure that loggers are"},
{:vllm_torch_profiler_dir, 0, __MODULE__, ""},
{:load_plugins_by_group, 1, __MODULE__, ""},
{:update_config, 2, __MODULE__, ""},
{:maybe_convert_int, 1, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:vllm_nvtx_scopes_for_profiling, 0, __MODULE__, ""},
{:vllm_has_flashinfer_cubin, 0, __MODULE__, ""},
{:embeds_inputs, 1, __MODULE__,
"Construct [`EmbedsInputs`][vllm.inputs.data.EmbedsInputs] from optional"},
{:vllm_use_nvfp4_ct_emulations, 0, __MODULE__, ""},
{:get_world_group, 0, __MODULE__, ""},
{:use_aot_compile, 0, __MODULE__, ""},
{:multi_modal_data_dict, 0, __MODULE__,
"A Mapping is a generic container for associating key/value"},
{:generation_task, 0, __MODULE__, ""},
{:append_logprobs_for_next_position, 6, __MODULE__,
"Appends logprobs for the next position"},
{:create_prompt_logprobs, 1, __MODULE__,
"Creates a container to store prompt logprobs for a request"},
{:vllm_dp_master_port, 0, __MODULE__, ""},
{:tensor_model_parallel_all_reduce, 1, __MODULE__,
"All-reduce the input tensor across model parallel group."},
{:_reasoning_parsers_to_register, 0, __MODULE__, ""},
{:get_python_platform, 0, __MODULE__, ""},
{:vllm_gc_debug, 0, __MODULE__, ""},
{:global_http_connection, 0, __MODULE__, ""},
{:get_rocm_version, 1, __MODULE__,
"Returns the ROCm version if available, otherwise 'N/A'."},
{:vllm_rocm_use_aiter_mla, 0, __MODULE__, ""},
{:vllm_allow_insecure_serialization, 0, __MODULE__, ""},
{:vllm_engine_ready_timeout_s, 0, __MODULE__, ""},
{:vllm_use_deep_gemm, 0, __MODULE__, ""},
{:vllm_dbo_comm_sms, 0, __MODULE__, ""},
{:is_torch_equal_or_newer, 1, __MODULE__,
"Check if the installed torch version is >= the target version."},
{:get_tensor_model_parallel_world_size, 0, __MODULE__,
"Return world size for the tensor model parallel group."},
{:vllm_mooncake_abort_request_timeout, 0, __MODULE__, ""},
{:vllm_v1_output_proc_chunk_size, 0, __MODULE__, ""},
{:sched_yield, 0, __MODULE__, ""},
{:_print_debug_once, 2, __MODULE__, "Logging configuration for vLLM."},
{:vllm_use_flashinfer_moe_fp4, 0, __MODULE__, ""},
{:get_os, 1, __MODULE__, ""},
{:get_beam_search_score, 3, __MODULE__,
"Calculate the beam search score with length penalty."},
{:cached_get_tokenizer, 1, __MODULE__,
"Gets a tokenizer for the given model name via HuggingFace or ModelScope."},
{:vllm_cache_root, 0, __MODULE__, ""},
{:get_current_vllm_config, 0, __MODULE__, ""},
{:main, 0, __MODULE__, ""},
{:stream_finished, 0, __MODULE__, ""},
{:vllm_debug_mfu_metrics, 0, __MODULE__, ""},
{:vllm_use_fbgemm, 0, __MODULE__, ""},
{:_prev_minor_version, 0, __MODULE__,
"For the purpose of testing, return a previous minor version number."},
{:vllm_rocm_use_aiter_rmsnorm, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:vllm_port, 0, __MODULE__, ""},
{:_patch_get_raw_stream_if_needed, 0, __MODULE__,
"Workaround for TorchInductor autotune get_raw_stream() bug."},
{:vllm_assets_cache_model_clean, 0, __MODULE__, ""},
{:set_current_vllm_config, 1, __MODULE__, "Temporarily set the current vLLM config."},
{:vllm_xla_cache_path, 0, __MODULE__, ""},
{:default_plugins_group, 0, __MODULE__, ""},
{:_tool_parsers_to_register, 0, __MODULE__, ""},
{:all_reduce, 2, __MODULE__, ""},
{:vllm_pp_layer_partition, 0, __MODULE__, ""},
{:get_forward_context, 0, __MODULE__, "Get the current forward context."},
{:run_once, 1, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:vllm_moriio_qp_per_transfer, 0, __MODULE__, ""},
{:init_tracer, 2, __MODULE__, ""},
{:graph_capture, 1, __MODULE__,
"`graph_capture` is a context manager which should surround the code that"},
{:vllm_rocm_use_aiter_fp4_bmm, 0, __MODULE__, ""},
{:vllm_allow_long_max_model_len, 0, __MODULE__, ""},
{:register_lazy_tool_parsers, 0, __MODULE__, ""},
{:vllm_torch_profiler_with_stack, 0, __MODULE__, ""},
{:vllm_use_fused_moe_grouped_topk, 0, __MODULE__, ""},
{:init_logger, 1, __MODULE__,
"The main purpose of this function is to ensure that loggers are"},
{:vllm_video_fetch_timeout, 0, __MODULE__, ""},
{:vllm_use_flashinfer_sampler, 0, __MODULE__, ""},
{:reduce_scatter_fake, 4, __MODULE__, ""},
{:random_uuid, 0, __MODULE__, ""},
{:vllm_cpu_num_of_reserved_cpu, 0, __MODULE__, ""},
{:vllm_target_device, 0, __MODULE__, ""},
{:vllm_cpu_kvcache_space, 0, __MODULE__, ""},
{:vllm_ray_bundle_indices, 0, __MODULE__, ""},
{:init_logger, 1, __MODULE__,
"The main purpose of this function is to ensure that loggers are"},
{:suppress_stdout, 0, __MODULE__,
"Suppress stdout from C libraries at the file descriptor level."},
{:is_uv_venv, 0, __MODULE__, ""},
{:vllm_media_url_allow_redirects, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:token_inputs, 1, __MODULE__,
"Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional"},
{:vllm_keep_alive_on_engine_death, 0, __MODULE__, ""},
{:vllm_torch_profiler_use_gzip, 0, __MODULE__, ""},
{:vllm_rocm_fp8_padding, 0, __MODULE__, ""},
{:get_tcp_uri, 2, __MODULE__, ""},
{:generation_tasks, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:vllm_disable_compile_cache, 0, __MODULE__, ""},
{:pooling_task, 0, __MODULE__, ""},
{:get_graph_partition_signature_patched, 3, __MODULE__,
"Gets signature for each graph partition, including input nodes, output nodes, and"},
{:get_bad_words_logits_processors, 2, __MODULE__, ""},
{:direct_register_custom_op, 2, __MODULE__,
"`torch.library.custom_op` can have significant overhead because it"},
{:is_init_field, 2, __MODULE__, ""},
{:vllm_host_ip, 0, __MODULE__, ""},
{:vllm_loopback_ip, 0, __MODULE__, ""},
{:cuda_platform_plugin, 0, __MODULE__, ""},
{:vllm_msgpack_zero_copy_threshold, 0, __MODULE__, ""},
{:rocm_platform_plugin, 0, __MODULE__, ""},
{:use_sched_yield, 0, __MODULE__, ""},
{:vllm_mooncake_bootstrap_port, 0, __MODULE__, ""},
{:run_and_parse_first_match, 3, __MODULE__,
"Run command using run_lambda, returns the first regex match if it exists."},
{:vllm_log_batchsize_interval, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter_fusion_shared_experts, 0, __MODULE__, ""},
{:batched_tensor_inputs, 0, __MODULE__, "dict() -> new empty dictionary"},
{:vllm_use_deep_gemm_e8_m0, 0, __MODULE__, ""},
{:destroy_model_parallel, 0, __MODULE__, "Set the groups to none and destroy them."},
{:get_gcc_version, 1, __MODULE__, ""},
{:vllm_torch_profiler_dump_cuda_time_total, 0, __MODULE__, ""},
{:vllm_max_tokens_per_expert_fp4_moe, 0, __MODULE__, ""},
{:track_batchsize, 0, __MODULE__, ""},
{:cmake_build_type, 0, __MODULE__, ""},
{:_format_module_tree, 1, __MODULE__,
"Format a module tree with indentation, grouping identical layers."},
{:vllm_xla_use_spmd, 0, __MODULE__, ""},
{:_prev_minor_version_was, 1, __MODULE__,
"Check whether a given version matches the previous minor version."},
{:vllm_do_not_track, 0, __MODULE__, ""},
{:environment_variables, 0, __MODULE__, ""},
{:vllm_enable_v1_multiprocessing, 0, __MODULE__, ""},
{:no_color, 0, __MODULE__, ""},
{:resolve_obj_by_qualname, 1, __MODULE__,
"Resolve an object by its fully-qualified class name."},
{:vllm_disable_shared_experts_stream, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter_fp4_asm_gemm, 0, __MODULE__, ""},
{:get_cpu_info, 1, __MODULE__, ""},
{:tensor_model_parallel_gather, 1, __MODULE__,
"Gather the input tensor across model parallel group."},
{:vllm_enable_moe_dp_chunk, 0, __MODULE__, ""},
{:register_lazy_reasoning_parsers, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:cleanup_dist_env_and_memory, 0, __MODULE__, ""},
{:get_distributed_init_method, 2, __MODULE__, ""},
{:vllm_use_precompiled, 0, __MODULE__, ""},
{:vllm_disable_log_logo, 0, __MODULE__, ""},
{:get_clang_version, 1, __MODULE__, ""},
{:vllm_use_aot_compile, 0, __MODULE__, ""},
{:pooling_task, 0, __MODULE__, ""},
{:local_rank, 0, __MODULE__, ""},
{:vllm_logging_level, 0, __MODULE__, ""},
{:vllm_ray_per_worker_gpus, 0, __MODULE__, ""},
{:is_torch_equal, 1, __MODULE__,
"Check if the installed torch version is == the target version."},
{:vllm_rocm_use_aiter_mha, 0, __MODULE__, ""},
{:get_layers_from_vllm_config, 2, __MODULE__, "Get layers from the vLLM config."},
{:vllm_worker_multiproc_method, 0, __MODULE__, ""},
{:vllm_usage_stats_server, 0, __MODULE__, ""},
{:_sampling_eps, 0, __MODULE__, ""},
{:get_dp_group, 0, __MODULE__, ""},
{:stateless_destroy_torch_distributed_process_group, 1, __MODULE__,
"Destroy ProcessGroup returned by"},
{:vllm_profiler_max_iters, 0, __MODULE__, ""},
{:vllm_trace_function, 0, __MODULE__, ""},
{:current_platform, 0, __MODULE__, ""},
{:coordinate_batch_across_dp, 4, __MODULE__,
"Coordinates amongst all DP ranks to determine if and how the full batch"},
{:__all__, 0, __MODULE__, ""},
{:enable_envs_cache, 0, __MODULE__,
"Enables caching of environment variables. This is useful for performance"},
{:vllm_deepepll_nvfp4_dispatch, 0, __MODULE__, ""},
{:vllm_use_flashinfer_moe_mxfp4_bf16, 0, __MODULE__, ""},
{:decoder_only_inputs, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter_moe, 0, __MODULE__, ""},
{:vllm_flashinfer_moe_backend, 0, __MODULE__, ""},
{:vllm_use_flashinfer_moe_fp8, 0, __MODULE__, ""},
{:patched_fused_scaled_matmul_reduce_scatter_fake, 9, __MODULE__, ""},
{:vllm_rocm_use_aiter_linear, 0, __MODULE__, ""},
{:vllm_dp_master_ip, 0, __MODULE__, ""},
{:get_tokenizer, 1, __MODULE__,
"Gets a tokenizer for the given model name via HuggingFace or ModelScope."},
{:vllm_main_cuda_version, 0, __MODULE__, ""},
{:supports_xccl, 0, __MODULE__, ""},
{:get_default_config_root, 0, __MODULE__, ""},
{:vllm_mm_hasher_algorithm, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:vllm_rocm_use_aiter, 0, __MODULE__, ""},
{:vllm_plugins, 0, __MODULE__, ""},
{:get_running_cuda_version, 1, __MODULE__, ""},
{:_format, 0, __MODULE__, ""},
{:get_gpu_topo, 1, __MODULE__, ""},
{:vllm_use_triton_awq, 0, __MODULE__, ""},
{:_scalar_types_id_map, 0, __MODULE__, ""},
{:last_logging_time, 0, __MODULE__, ""},
{:vllm_skip_precompiled_version_suffix, 0, __MODULE__, ""},
{:vllm_ray_dp_pack_strategy, 0, __MODULE__, ""},
{:vllm_deepep_buffer_size_mb, 0, __MODULE__, ""},
{:tokenizer_registry, 0, __MODULE__, ""},
{:_init_trace, 0, __MODULE__, ""},
{:_is_envs_cache_enabled, 0, __MODULE__,
"Checked if __getattr__ is wrapped with functools.cache"},
{:vllm_log_stats_interval, 0, __MODULE__, ""},
{:sample_logprobs, 0, __MODULE__, ""},
{:env_list_with_choices, 3, __MODULE__,
"Create a lambda that validates environment variable"},
{:create_sample_logprobs, 1, __MODULE__,
"Creates a container to store decode logprobs for a request"},
{:destroy_distributed_environment, 0, __MODULE__, ""},
{:set_forward_context, 2, __MODULE__,
"A context manager that stores the current forward context,"},
{:_update_scheduler_patched, 1, __MODULE__,
"(Re)initializes the scheduler member. When initializing the scheduler, no CUBIN"},
{:vllm_server_dev_mode, 0, __MODULE__, ""},
{:__setattr__, 2, __MODULE__, ""},
{:prepare_communication_buffer_for_model, 1, __MODULE__,
"Prepare the communication buffer for the model."},
{:vllm_rocm_moe_padding, 0, __MODULE__, ""},
{:vllm_allow_runtime_lora_updating, 0, __MODULE__, ""},
{:q_scale_constant, 0, __MODULE__, ""},
{:builtin_platform_plugins, 0, __MODULE__, ""},
{:nested_tensors, 0, __MODULE__, ""},
{:vllm_cpu_sgl_kernel, 0, __MODULE__, ""},
{:vllm_mq_max_chunk_bytes_mb, 0, __MODULE__, ""},
{:vllm_marlin_input_dtype, 0, __MODULE__, ""},
{:multimodal_registry, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:logits_processor, 0, __MODULE__, ""},
{:try_match_architecture_defaults, 1, __MODULE__, ""},
{:forward_start_time, 0, __MODULE__, ""},
{:s3_access_key_id, 0, __MODULE__, ""},
{:all_gather, 4, __MODULE__, ""},
{:disable_compile_cache, 0, __MODULE__, ""},
{:vllm_deepep_low_latency_use_mnnvl, 0, __MODULE__, ""},
{:env_set_with_choices, 3, __MODULE__,
"Creates a lambda which that validates environment variable"},
{:vllm_xgrammar_cache_mb, 0, __MODULE__, ""},
{:initialize_model_parallel, 0, __MODULE__, "Initialize model parallel groups."},
{:vllm_shared_experts_stream_token_threshold, 0, __MODULE__, ""},
{:log_tracing_disabled_warning, 0, __MODULE__, ""},
{:get_tp_group, 0, __MODULE__, ""},
{:vllm_rocm_quick_reduce_cast_bf16_to_fp16, 0, __MODULE__, ""},
{:vllm_rpc_timeout, 0, __MODULE__, ""},
{:log_scope, 0, __MODULE__, "Logging configuration for vLLM."},
{:vllm_rocm_quick_reduce_quantization, 0, __MODULE__, ""},
{:singleton_inputs, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:resolve_obj_by_qualname, 1, __MODULE__,
"Resolve an object by its fully-qualified class name."},
{:logtime, 1, __MODULE__, "Logs the execution time of the decorated function."},
{:_get_module_info, 1, __MODULE__, "Get info string for a module."},
{:reduce_scatter, 4, __MODULE__, ""},
{:processor_inputs, 0, __MODULE__, ""},
{:vllm_use_flashinfer_moe_mxfp4_mxfp8, 0, __MODULE__, ""},
{:vllm_disable_pynccl, 0, __MODULE__, ""},
{:vllm_logging_stream, 0, __MODULE__, ""},
{:vllm_tpu_most_model_len, 0, __MODULE__, ""},
{:io_processor_plugins_group, 0, __MODULE__, ""},
{:vllm_use_ray_wrapped_pp_comm, 0, __MODULE__, ""},
{:divide, 2, __MODULE__,
"Ensure that numerator is divisible by the denominator and return"},
{:vllm_torch_profiler_with_flops, 0, __MODULE__, ""},
{:v_scale_constant, 0, __MODULE__, ""},
{:vllm_float32_matmul_precision, 0, __MODULE__, ""},
{:tensor_model_parallel_all_gather, 1, __MODULE__,
"All-gather the input tensor across model parallel group."},
{:config, 1, __MODULE__,
"A decorator that ensures all fields in a dataclass have default values"},
{:max_jobs, 0, __MODULE__, ""},
{:vllm_use_flashinfer_moe_mxfp4_mxfp8_cutlass, 0, __MODULE__, ""},
{:in_the_same_node_as, 1, __MODULE__,
"This is a collective operation that returns if each rank is in the same node"},
{:vllm_tpu_bucket_padding_gap, 0, __MODULE__, ""},
{:pooling_tasks, 0, __MODULE__, ""},
{:vllm_usage_source, 0, __MODULE__, ""},
{:_methods_to_patch, 0, __MODULE__, ""},
{:suppress_logging, 0, __MODULE__, "Logging configuration for vLLM."},
{:override_forward_context, 1, __MODULE__,
"A context manager that overrides the current forward context."},
{:maybe_convert_bool, 1, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:vllm_rocm_shuffle_kv_cache_layout, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""},
{:vllm_torch_profiler_disable_async_llm, 0, __MODULE__, ""},
{:vllm_logging_prefix, 0, __MODULE__, ""},
{:_configure_vllm_root_logger, 0, __MODULE__, "Logging configuration for vLLM."},
{:main, 0, __MODULE__, ""},
{:get_inner_dp_world_group, 0, __MODULE__, ""},
{:mask_64_bits, 0, __MODULE__, ""},
{:vllm_nixl_side_channel_host, 0, __MODULE__, ""},
{:logger, 0, __MODULE__, ""},
{:__all__, 0, __MODULE__, ""}
]
end
@doc false
def __classes__ do
[
{Vllm.Config.SpeechToTextConfig, "Configuration for speech-to-text models."},
{Vllm.Logger.ColoredFormatter, ""},
{Vllm.Config.ObservabilityConfig, "Configuration for observability - metrics and tracing."},
{Vllm.Outputs.RequestOutput,
"The output data of a completion request to the LLM.\n\nArgs:\n request_id: The unique ID of the request.\n prompt: The prompt string of the request.\n For encoder/decoder models, this is the\n decoder input prompt.\n prompt_token_ids: The token IDs of the prompt.\n For encoder/decoder models, this is the\n decoder input prompt token ids.\n prompt_logprobs: The log probabilities to return per prompt token.\n outputs: The output sequences of the request.\n finished: Whether the whole request is finished.\n metrics: Metrics associated with the request.\n lora_request: The LoRA request that was used to generate the output.\n encoder_prompt: The encoder prompt string of the request.\n None if decoder-only.\n encoder_prompt_token_ids: The token IDs of the encoder prompt.\n None if decoder-only.\n num_cached_tokens: The number of tokens with prefix cache hit.\n kv_transfer_params: The params for remote K/V transfer."},
{Vllm.Outputs.PoolingRequestOutput,
"The output data of a pooling request to the LLM.\n\nArgs:\n request_id (str): A unique identifier for the pooling request.\n outputs (PoolingOutput): The pooling results for the given input.\n prompt_token_ids (list[int]): A list of token IDs used in the prompt.\n num_cached_tokens: The number of tokens with prefix cache hit.\n finished (bool): A flag indicating whether the pooling is completed."},
{Vllm.SamplingParams.BeamSearchParams, "Beam search parameters for text generation."},
{Vllm.SamplingParams.PydanticMsgspecMixin, "Sampling parameters for text generation."},
{Vllm.Config.LoadConfig, "Configuration for loading the model weights."},
{Vllm.Platforms.Platform, ""},
{Vllm.PoolingParamsClass,
"API parameters for pooling models.\n\nAttributes:\n truncate_prompt_tokens: Controls prompt truncation.\n Set to -1 to use the model's default truncation size.\n Set to k to keep only the last k tokens (left truncation).\n Set to None to disable truncation.\n dimensions: Reduce the dimensions of embeddings\n if model support matryoshka representation.\n normalize: Deprecated, please use use_activation instead.\n softmax: Deprecated, please use use_activation instead.\n activation: Deprecated, please use use_activation instead.\n use_activation: Whether to apply activation function to\n the classification outputs."},
{Vllm.LoggingUtils.ColoredFormatter,
"Adds ANSI color codes to log levels for terminal output.\n\nThis formatter adds colors by injecting them into the format string for\nstatic elements (timestamp, filename, line number) and modifying the\nlevelname attribute for dynamic color selection."},
{Vllm.Config.ECTransferConfig, "Configuration for distributed EC cache transfer."},
{Vllm.ForwardContext.AttentionMetadata, ""},
{Vllm.Sequence.IntermediateTensors,
"For all pipeline stages except the last, we need to return the hidden\nstates and residuals to be sent to the next stage. This data structure\ncontains the hidden states and residuals for a request.\n\nEach stage also needs to handle its own kv_connector_output."},
{Vllm.Config.VllmConfig,
"Dataclass which contains all vllm-related configuration. This\nsimplifies passing around the distinct configurations in the codebase."},
{Vllm.Config.CacheConfig, "Configuration for the KV cache."},
{Vllm.ModelExecutor.PackedvLLMParameter,
"Parameter for model weights which are packed on disk.\nExample: GPTQ Marlin weights are int4 or int8, packed into int32.\nExtends the ModelWeightParameter to take in the\npacked factor, the packed dimension, and optionally, marlin\ntile size for marlin kernels. Adjusts the shard_size and\nshard_offset for fused linear layers model weight loading\nby accounting for packing and optionally, marlin tile size."},
{Vllm.Outputs.PoolingOutput,
"The output data of one pooling output of a request.\n\nArgs:\n data: The extracted hidden states."},
{Vllm.Config.EPLBConfig, "Configuration for Expert Parallel Load Balancing (EP)."},
{Vllm.Multimodal.Inputs.MultiModalKwargsItem,
"A collection of\n[`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]\ncorresponding to a data item in\n[`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]."},
{Vllm.Config.MultiModalConfig, "Controls the behavior of multimodal models."},
{Vllm.Inputs.TextPrompt, "Schema for a text prompt."},
{Vllm.LogitsProcess.NoBadWordsLogitsProcessor, ""},
{Vllm.Config.CompilationMode,
"The compilation approach used for torch.compile-based compilation of the\nmodel."},
{Vllm.Config.SupportsMetricsInfo, ""},
{Vllm.Inputs.TokenInputs, "Represents token-based inputs."},
{Vllm.Config.LoRAConfig, "Configuration for LoRA."},
{Vllm.Logger.NewLineFormatter, ""},
{Vllm.LLMEngine, "Legacy LLMEngine for backwards compatibility."},
{Vllm.Outputs.RequestStateStats, "Stats that need to be tracked across delta updates."},
{Vllm.Outputs.ClassificationRequestOutput,
"The output data of a pooling request to the LLM.\n\nArgs:\n request_id (str): A unique identifier for the pooling request.\n outputs (PoolingOutput): The pooling results for the given input.\n prompt_token_ids (list[int]): A list of token IDs used in the prompt.\n num_cached_tokens: The number of tokens with prefix cache hit.\n finished (bool): A flag indicating whether the pooling is completed."},
{Vllm.ScalarType,
"ScalarType can represent a wide range of floating point and integer\ntypes, in particular it can be used to represent sub-byte data types\n(something that torch.dtype currently does not support). It is also\ncapable of representing types with a bias, i.e.:\n `stored_value = value + bias`,\nthis is useful for quantized types (e.g. standard GPTQ 4bit uses a bias\nof 8). The implementation for this class can be found in\ncsrc/core/scalar_type.hpp, these type signatures should be kept in sync\nwith that file."},
{Vllm.Outputs.CompletionOutput,
"The output data of one completion output of a request.\n\nArgs:\n index: The index of the output in the request.\n text: The generated output text.\n token_ids: The token IDs of the generated output text.\n cumulative_logprob: The cumulative log probability of the generated\n output text.\n logprobs: The log probabilities of the top probability words at each\n position if the logprobs are requested.\n finish_reason: The reason why the sequence is finished.\n stop_reason: The stop string or token id that caused the completion\n to stop, None if the completion finished for some other reason\n including encountering the EOS token.\n lora_request: The LoRA request that was used to generate the output."},
{Vllm.ModelExecutor.BasevLLMParameter,
"Base parameter for vLLM linear layers. Extends the torch.nn.parameter\nby taking in a linear weight loader. Will copy the loaded weight\ninto the parameter when the provided weight loader is called."},
{Vllm.Multimodal.Inputs.MultiModalInputs,
"Represents the outputs of\n[`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor],\nready to be passed to vLLM internals."},
{Vllm.Sequence.KVConnectorOutput,
"Special type indicating an unconstrained type.\n\n- Any is compatible with every type.\n- Any assumed to have all methods.\n- All values assumed to be instances of Any.\n\nNote that all the above statements are true from the point of view of\nstatic type checkers. At runtime, Any should not be used with instance\nchecks."},
{Vllm.Tracing.BaseSpanAttributes, ""},
{Vllm.Config.PoolerConfig, "Controls the behavior of output pooling in pooling models."},
{Vllm.BeamSearch.LoRARequest,
"Request for a LoRA adapter.\n\nlora_int_id must be globally unique for a given adapter.\nThis is currently not enforced in vLLM."},
{Vllm.ScalarType.NanRepr,
"Enum members for `NanRepr`.\n\nMembers:\n- `NONE` = `0`: nans are not supported\n- `IEEE_754` = `1`: nans are: Exp all 1s, mantissa not all 0s\n- `EXTD_RANGE_MAX_MIN` = `2`: nans are: Exp all 1s, mantissa all 1s"},
{Vllm.LLM,
"An LLM for generating texts from given prompts and sampling parameters.\n\nThis class includes a tokenizer, a language model (possibly distributed\nacross multiple GPUs), and GPU memory space allocated for intermediate\nstates (aka KV cache). Given a batch of prompts and sampling parameters,\nthis class generates texts from the model, using an intelligent batching\nmechanism and efficient memory management.\n\nArgs:\n model: The name or path of a HuggingFace Transformers model.\n tokenizer: The name or path of a HuggingFace Transformers tokenizer.\n tokenizer_mode: The tokenizer mode. \"auto\" will use the fast tokenizer\n if available, and \"slow\" will always use the slow tokenizer.\n skip_tokenizer_init: If true, skip initialization of tokenizer and\n detokenizer. Expect valid prompt_token_ids and None for prompt\n from the input.\n trust_remote_code: Trust remote code (e.g., from HuggingFace) when\n downloading the model and tokenizer.\n allowed_local_media_path: Allowing API requests to read local images\n or videos from directories specified by the server file system.\n This is a security risk. Should only be enabled in trusted\n environments.\n allowed_media_domains: If set, only media URLs that belong to this\n domain can be used for multi-modal inputs.\n tensor_parallel_size: The number of GPUs to use for distributed\n execution with tensor parallelism.\n dtype: The data type for the model weights and activations. Currently,\n we support `float32`, `float16`, and `bfloat16`. If `auto`, we use\n the `dtype` attribute of the Transformers model's config. However,\n if the `dtype` in the config is `float32`, we will use `float16` instead.\n quantization: The method used to quantize the model weights. Currently,\n we support \"awq\", \"gptq\", and \"fp8\" (experimental).\n If None, we first check the `quantization_config` attribute in the\n model config file. If that is None, we assume the model weights are\n not quantized and use `dtype` to determine the data type of\n the weights.\n revision: The specific model version to use. It can be a branch name,\n a tag name, or a commit id.\n tokenizer_revision: The specific tokenizer version to use. It can be a\n branch name, a tag name, or a commit id.\n seed: The seed to initialize the random number generator for sampling.\n gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to\n reserve for the model weights, activations, and KV cache. Higher\n values will increase the KV cache size and thus improve the model's\n throughput. However, if the value is too high, it may cause out-of-\n memory (OOM) errors.\n kv_cache_memory_bytes: Size of KV Cache per GPU in bytes. By default,\n this is set to None and vllm can automatically infer the kv cache\n size based on gpu_memory_utilization. However, users may want to\n manually specify the kv cache memory size. kv_cache_memory_bytes\n allows more fine-grain control of how much memory gets used when\n compared with using gpu_memory_utilization. Note that\n kv_cache_memory_bytes (when not-None) ignores\n gpu_memory_utilization\n swap_space: The size (GiB) of CPU memory per GPU to use as swap space.\n This can be used for temporarily storing the states of the requests\n when their `best_of` sampling parameters are larger than 1. If all\n requests will have `best_of=1`, you can safely set this to 0.\n Noting that `best_of` is only supported in V0. Otherwise, too small\n values may cause out-of-memory (OOM) errors.\n cpu_offload_gb: The size (GiB) of CPU memory to use for offloading\n the model weights. This virtually increases the GPU memory space\n you can use to hold the model weights, at the cost of CPU-GPU data\n transfer for every forward pass.\n enforce_eager: Whether to enforce eager execution. If True, we will\n disable CUDA graph and always execute the model in eager mode.\n If False, we will use CUDA graph and eager execution in hybrid.\n enable_return_routed_experts: Whether to return routed experts.\n disable_custom_all_reduce: See\n [ParallelConfig][vllm.config.ParallelConfig].\n hf_token: The token to use as HTTP bearer authorization for remote files\n . If `True`, will use the token generated when running\n `huggingface-cli login` (stored in `~/.huggingface`).\n hf_overrides: If a dictionary, contains arguments to be forwarded to the\n HuggingFace config. If a callable, it is called to update the\n HuggingFace config.\n mm_processor_kwargs: Arguments to be forwarded to the model's processor\n for multi-modal data, e.g., image processor. Overrides for the\n multi-modal processor obtained from `AutoProcessor.from_pretrained`.\n The available overrides depend on the model that is being run.\n For example, for Phi-3-Vision: `{\"num_crops\": 4}`.\n pooler_config: Initialize non-default pooling config for the pooling\n model. e.g. `PoolerConfig(seq_pooling_type=\"MEAN\", normalize=False)`.\n compilation_config: Either an integer or a dictionary. If it is an\n integer, it is used as the mode of compilation optimization. If it\n is a dictionary, it can specify the full compilation configuration.\n attention_config: Configuration for attention mechanisms. Can be a\n dictionary or an AttentionConfig instance. If a dictionary, it will\n be converted to an AttentionConfig. Allows specifying the attention\n backend and other attention-related settings.\n **kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].\n\nNote:\n This class is intended to be used for offline inference. For online\n serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead."},
{Vllm.Inputs.EncoderDecoderInputs,
"The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they\nare passed to the model executor.\n\nThis specifies the required data for encoder-decoder models."},
{Vllm.Inputs.DataPrompt, "Represents generic inputs handled by IO processor plugins."},
{Vllm.Multimodal.MultiModalKwargsItems,
"A dictionary of\n[`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]s\nby modality."},
{Vllm.Config.KVEventsConfig, "Configuration for KV event publishing."},
{Vllm.Distributed.DeviceCommunicatorBase,
"Base class for device-specific communicator.\nIt can use the `cpu_group` to initialize the communicator.\nIf the device has PyTorch integration (PyTorch can recognize its\ncommunication backend), the `device_group` will also be given."},
{Vllm.Inputs.EmbedsPrompt, "Schema for a prompt provided via token embeddings."},
{Vllm.Config.StructuredOutputsConfig,
"Dataclass which contains structured outputs config for the engine."},
{Vllm.PoolingParams,
"API parameters for pooling models.\n\nAttributes:\n truncate_prompt_tokens: Controls prompt truncation.\n Set to -1 to use the model's default truncation size.\n Set to k to keep only the last k tokens (left truncation).\n Set to None to disable truncation.\n dimensions: Reduce the dimensions of embeddings\n if model support matryoshka representation.\n normalize: Deprecated, please use use_activation instead.\n softmax: Deprecated, please use use_activation instead.\n activation: Deprecated, please use use_activation instead.\n use_activation: Whether to apply activation function to\n the classification outputs."},
{Vllm.Inputs.EmbedsInputs, "Represents embeddings-based inputs."},
{Vllm.Logger.VllmLogger,
"Note:\n This class is just to provide type information.\n We actually patch the methods directly on the [`logging.Logger`][]\n instance to avoid conflicting with other libraries such as\n `intel_extension_for_pytorch.utils._logger`."},
{Vllm.ToolParsers.ToolParser,
"Abstract ToolParser class that should not be used directly. Provided\nproperties and methods should be used in\nderived classes."},
{Vllm.Outputs.EmbeddingRequestOutput,
"The output data of a pooling request to the LLM.\n\nArgs:\n request_id (str): A unique identifier for the pooling request.\n outputs (PoolingOutput): The pooling results for the given input.\n prompt_token_ids (list[int]): A list of token IDs used in the prompt.\n num_cached_tokens: The number of tokens with prefix cache hit.\n finished (bool): A flag indicating whether the pooling is completed."},
{Vllm.Multimodal.MultiModalDataBuiltins,
"Type annotations for modality types predefined by vLLM."},
{Vllm.Multimodal.Inputs.MultiModalFieldConfig,
"MultiModalFieldConfig(field: vllm.multimodal.inputs.BaseMultiModalField, modality: str)"},
{Vllm.Outputs.ScoringOutput,
"The output data of one scoring output of a request.\n\nArgs:\n score: The similarity score, which is a scalar value."},
{Vllm.AsyncLLMEngine, "Protocol class for Clients to Engine"},
{Vllm.TritonUtils.TritonLanguagePlaceholder, ""},
{Vllm.BeamSearch.BeamSearchInstance, ""},
{Vllm.PoolingParams.RequestOutputKind,
"Enum members for `RequestOutputKind`.\n\nMembers:\n- `CUMULATIVE` = `0`\n- `DELTA` = `1`\n- `FINAL_ONLY` = `2`"},
{Vllm.Exceptions.VLLMValidationError,
"vLLM-specific validation error for request validation failures.\n\nArgs:\n message: The error message describing the validation failure.\n parameter: Optional parameter name that failed validation.\n value: Optional value that was rejected during validation."},
{Vllm.SamplingParams.StructuredOutputsParams, "Sampling parameters for text generation."},
{Vllm.Outputs.EmbeddingOutput,
"The output data of one embedding output of a request.\n\nArgs:\n embedding: The embedding vector, which is a list of floats.\n Its length depends on the hidden dimension of the model."},
{Vllm.ToolParsers.ToolParserManager,
"Central registry for ToolParser implementations.\n\nSupports two modes:\n - Eager (immediate) registration via `register_module`\n - Lazy registration via `register_lazy_module`"},
{Vllm.Config.PassConfig,
"Configuration for custom Inductor passes.\n\nThis is separate from general `CompilationConfig` so that inductor passes\ndon't all have access to full configuration - that would create a cycle as\nthe `PassManager` is set as a property of config.\n\nYou must pass PassConfig to VLLMConfig constructor via the CompilationConfig\nconstructor. VLLMConfig's post_init does further initialization.\nIf used outside of the VLLMConfig, some fields may be left in an\nimproper state."},
{Vllm.BeamSearch.BeamSearchSequence,
"A sequence for beam search.\nIt keeps track of the tokens and the log probability of the sequence.\nThe text field is optional and will only be filled when the sequence is\nabout to be returned to the user."},
{Vllm.Connections.HTTPConnection, "Helper class to send HTTP requests."},
{Vllm.Config.ParallelConfig, "Configuration for the distributed execution."},
{Vllm.Multimodal.Inputs.PlaceholderRange,
"Placeholder location information for multi-modal data.\n\nExample:\n\nPrompt: `AAAA BBBB What is in these images?`\n\nImages A and B will have:\n\n```\nA: PlaceholderRange(offset=0, length=4)\nB: PlaceholderRange(offset=5, length=4)\n```"},
{Vllm.Platforms.PlatformEnum,
"Enum members for `PlatformEnum`.\n\nMembers:\n- `CUDA` = `1`\n- `ROCM` = `2`\n- `TPU` = `3`\n- `XPU` = `4`\n- `CPU` = `5`\n- `OOT` = `6`\n- `UNSPECIFIED` = `7`"},
{Vllm.TritonUtils.TritonPlaceholder, ""},
{Vllm.Multimodal.Inputs.MultiModalFieldElem,
"Represents a keyword argument inside a\n[`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]."},
{Vllm.BeamSearch.BeamSearchOutput,
"The output of beam search.\nIt contains the list of the best beam search sequences.\nThe length of the list is equal to the beam width."},
{Vllm.Config.AttentionConfig, "Configuration for attention mechanisms in vLLM."},
{Vllm.Logprobs.FlatLogprobs,
"Flat logprobs of a request into multiple primitive type lists.\n\nCompared to list[dict[int, Logprob]], this data structure reduced GC\noverhead significantly. As it flattened logprob information for\nall positions and ranks in to multiple primitive type lists (i.e.\nlogprobs, token_ids, ranks per token_ids, decoded_tokens).\nSo regardless of the sequence length and top_logprobs setup,\nFlatLogprobs would only introduce a constant amount of objects.\n\nAs each position might contains different amount of ranks,\nstart_indices_per_position would be used to access the logprob ranges\nfor different positions.\n\nNOTE: To reduce the migration overhead and improve backward compatibility,\nwe support the key Sequence APIs of list, so it could act as\nlist[LogprobsOnePosition]"},
{Vllm.SamplingParams.SamplingType,
"Enum members for `SamplingType`.\n\nMembers:\n- `GREEDY` = `0`\n- `RANDOM` = `1`\n- `RANDOM_SEED` = `2`"},
{Vllm.ForwardContext.BatchDescriptor,
"Batch descriptor for cudagraph dispatching. We should keep the num of\nitems as minimal as possible to properly and uniquely describe the padded\nbatch for cudagraph."},
{Vllm.Tracing.SpanAttributes, ""},
{Vllm.ForwardContext.DPMetadata,
"DPMetadata(max_tokens_across_dp_cpu: torch.Tensor, num_tokens_across_dp_cpu: torch.Tensor, local_sizes: list[int] | None = None)"},
{Vllm.Config.SpeculativeConfig, "Configuration for speculative decoding."},
{Vllm.Inputs.TokensPrompt, "Schema for a tokenized prompt."},
{Vllm.LoggingUtils.NewLineFormatter,
"Adds logging prefix to newlines to align multi-line messages."},
{Vllm.Platforms.CpuArchEnum,
"Enum members for `CpuArchEnum`.\n\nMembers:\n- `X86` = `1`\n- `ARM` = `2`\n- `POWERPC` = `3`\n- `S390X` = `4`\n- `RISCV` = `5`\n- `OTHER` = `6`\n- `UNKNOWN` = `7`"},
{Vllm.Reasoning.ReasoningParser,
"Abstract reasoning parser class that should not be used directly.\nProvided and methods should be used in derived classes.\n\nIt is used to extract reasoning content from the model output."},
{Vllm.Distributed.GroupCoordinator,
"PyTorch ProcessGroup wrapper for a group of processes.\nPyTorch ProcessGroup is bound to one specific communication backend,\n e.g. NCCL, Gloo, MPI, etc.\nGroupCoordinator takes charge of all the communication operations among\n the processes in the group. It manages both CPU and device\n communication."},
{Vllm.Distributed.GraphCaptureContext,
"GraphCaptureContext(stream: torch.cuda.streams.Stream)"},
{Vllm.Logprobs.Logprob,
"Infos for supporting OpenAI compatible logprobs and token ranks.\n\nAttributes:\n logprob: The logprob of chosen token\n rank: The vocab rank of chosen token (>=1)\n decoded_token: The decoded chosen token index"},
{Vllm.Multimodal.MultiModalHasher, ""},
{Vllm.Distributed.StatelessProcessGroup,
"A dataclass to hold a metadata store, and the rank, world_size of the\ngroup. Only use it to communicate metadata between processes.\nFor data-plane communication, create NCCL-related objects."},
{Vllm.Distributed.TensorMetadata, "TensorMetadata(device, dtype, size)"},
{Vllm.ScalarType.ScalarTypes, ""},
{Vllm.Reasoning.ReasoningParserManager,
"Central registry for ReasoningParser implementations.\n\nSupports two registration modes:\n - Eager registration via `register_module`\n - Lazy registration via `register_lazy_module`\n\nEach reasoning parser must inherit from `ReasoningParser`."},
{Vllm.ForwardContext,
"ForwardContext(no_compile_layers: dict[str, typing.Any], attn_metadata: dict[str, vllm.v1.attention.backend.AttentionMetadata] | list[dict[str, vllm.v1.attention.backend.AttentionMetadata]], virtual_engine: int, dp_metadata: vllm.forward_context.DPMetadata | None = None, cudagraph_runtime_mode: vllm.config.compilation.CUDAGraphMode = <CUDAGraphMode.NONE: 0>, batch_descriptor: vllm.forward_context.BatchDescriptor | None = None, ubatch_slices: list[vllm.v1.worker.ubatch_utils.UBatchSlice] | None = None, additional_kwargs: dict[str, typing.Any] = <factory>)"},
{Vllm.Outputs.ClassificationOutput,
"The output data of one classification output of a request.\n\nArgs:\n probs: The probability vector, which is a list of floats.\n Its length depends on the number of classes."},
{Vllm.Outputs.ScoringRequestOutput,
"The output data of a pooling request to the LLM.\n\nArgs:\n request_id (str): A unique identifier for the pooling request.\n outputs (PoolingOutput): The pooling results for the given input.\n prompt_token_ids (list[int]): A list of token IDs used in the prompt.\n num_cached_tokens: The number of tokens with prefix cache hit.\n finished (bool): A flag indicating whether the pooling is completed."},
{Vllm.Tokenizers.TokenizerLike, ""},
{Vllm.Config.CompilationConfig,
"Configuration for compilation.\n\nYou must pass CompilationConfig to VLLMConfig constructor.\nVLLMConfig's post_init does further initialization. If used outside of the\nVLLMConfig, some fields will be left in an improper state.\n\nIt has three parts:\n\n- Top-level Compilation control:\n - [`mode`][vllm.config.CompilationConfig.mode]\n - [`debug_dump_path`][vllm.config.CompilationConfig.debug_dump_path]\n - [`cache_dir`][vllm.config.CompilationConfig.cache_dir]\n - [`backend`][vllm.config.CompilationConfig.backend]\n - [`custom_ops`][vllm.config.CompilationConfig.custom_ops]\n - [`splitting_ops`][vllm.config.CompilationConfig.splitting_ops]\n - [`compile_mm_encoder`][vllm.config.CompilationConfig.compile_mm_encoder]\n- CudaGraph capture:\n - [`cudagraph_mode`][vllm.config.CompilationConfig.cudagraph_mode]\n - [`cudagraph_capture_sizes`]\n [vllm.config.CompilationConfig.cudagraph_capture_sizes]\n - [`max_cudagraph_capture_size`]\n [vllm.config.CompilationConfig.max_cudagraph_capture_size]\n - [`cudagraph_num_of_warmups`]\n [vllm.config.CompilationConfig.cudagraph_num_of_warmups]\n - [`cudagraph_copy_inputs`]\n [vllm.config.CompilationConfig.cudagraph_copy_inputs]\n- Inductor compilation:\n - [`compile_sizes`][vllm.config.CompilationConfig.compile_sizes]\n - [`compile_ranges_split_points`]\n [vllm.config.CompilationConfig.compile_ranges_split_points]\n - [`inductor_compile_config`]\n [vllm.config.CompilationConfig.inductor_compile_config]\n - [`inductor_passes`][vllm.config.CompilationConfig.inductor_passes]\n - custom inductor passes\n\nWhy we have different sizes for cudagraph and inductor:\n- cudagraph: a cudagraph captured for a specific size can only be used\n for the same size. We need to capture all the sizes we want to use.\n- inductor: a graph compiled by inductor for a general shape can be used\n for different sizes. Inductor can also compile for specific sizes,\n where it can have more information to optimize the graph with fully\n static shapes. However, we find the general shape compilation is\n sufficient for most cases. It might be beneficial to compile for\n certain small batchsizes, where inductor is good at optimizing."},
{Vllm.Config.DeviceConfig, "Configuration for the device to use for vLLM execution."},
{Vllm.SamplingParams,
"Sampling parameters for text generation.\n\nOverall, we follow the sampling parameters from the OpenAI text completion\nAPI (https://platform.openai.com/docs/api-reference/completions/create).\nIn addition, we support beam search, which is not supported by OpenAI."},
{Vllm.SamplingParamsClass,
"Sampling parameters for text generation.\n\nOverall, we follow the sampling parameters from the OpenAI text completion\nAPI (https://platform.openai.com/docs/api-reference/completions/create).\nIn addition, we support beam search, which is not supported by OpenAI."},
{Vllm.Config.CUDAGraphMode,
"Constants for the cudagraph mode in CompilationConfig.\nMeanwhile, the subset enum `NONE`, `PIECEWISE` and `FULL` are also\ntreated as concrete runtime mode for cudagraph runtime dispatching."},
{Vllm.Inputs.ExplicitEncoderDecoderPrompt,
"Represents an encoder/decoder model input prompt,\ncomprising an explicit encoder prompt and a decoder prompt.\n\nThe encoder and decoder prompts, respectively, may be formatted\naccording to any of the\n[`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] schemas,\nand are not required to have the same schema.\n\nOnly the encoder prompt may have multi-modal data. mm_processor_kwargs\nshould be at the top-level, and should not be set in the encoder/decoder\nprompts, since they are agnostic to the encoder/decoder.\n\nNote that an\n[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]\nmay not be used as an input to a decoder-only model,\nand that the `encoder_prompt` and `decoder_prompt`\nfields of this data structure themselves must be\n[`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] instances."},
{Vllm.Config.ProfilerConfig, "Dataclass which contains profiler config for the engine."},
{Vllm.Config.ModelConfig, "Configuration for the model."},
{Vllm.SamplingParams.RequestOutputKind,
"Enum members for `RequestOutputKind`.\n\nMembers:\n- `CUMULATIVE` = `0`\n- `DELTA` = `1`\n- `FINAL_ONLY` = `2`"},
{Vllm.BeamSearch.Logprob,
"Infos for supporting OpenAI compatible logprobs and token ranks.\n\nAttributes:\n logprob: The logprob of chosen token\n rank: The vocab rank of chosen token (>=1)\n decoded_token: The decoded chosen token index"},
{Vllm.CollectEnv.SystemEnv,
"SystemEnv(torch_version, is_debug_build, cuda_compiled_version, gcc_version, clang_version, cmake_version, os, libc_version, python_version, python_platform, is_cuda_available, cuda_runtime_version, cuda_module_loading, nvidia_driver_version, nvidia_gpu_models, cudnn_version, pip_version, pip_packages, conda_packages, hip_compiled_version, hip_runtime_version, miopen_runtime_version, caching_allocator_config, is_xnnpack_available, cpu_info, rocm_version, vllm_version, vllm_build_flags, gpu_topo, env_vars)"},
{Vllm.Multimodal.Inputs.MultiModalKwargsItems,
"A dictionary of\n[`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]s\nby modality."},
{Vllm.Multimodal.MultiModalRegistry,
"A registry that dispatches data processing according to the model."},
{Vllm.Config.SchedulerConfig, "Scheduler configuration."},
{Vllm.Config.KVTransferConfig, "Configuration for distributed KV cache transfer."},
{Vllm.SamplingParams.TokenizerLike, "Sampling parameters for text generation."},
{Vllm.LogitsProcess.TokenizerLike, ""}
]
end
@doc false
def __search__(query) do
SnakeBridge.Docs.search(__MODULE__, query)
end
@doc false
def doc(function) do
SnakeBridge.Docs.get(__MODULE__, function)
end
end