Packages

erllama

0.2.0
0.8.0 retired 0.7.0 retired 0.6.2 retired 0.6.1 retired 0.6.0 retired 0.5.1 retired 0.5.0 retired 0.4.0 retired 0.3.0 retired 0.2.0 retired 0.1.2 retired 0.1.1 retired 0.1.0 retired

Native Erlang/OTP wrapper around llama.cpp with token-exact tiered KV cache; cache more warm state than fits in RAM

Retired package: Renamed - Renamed to barrel_inference: https://github.com/barrel-platform/barrel_inference
HexDocs

Current section

Files

Jump to
Readme
13 Versions
0 Dependencies
Dependants
Files
Activity
erllama c_src llama.cpp ggml src ggml-cuda quantize.cuh
Raw

c_src/llama.cpp/ggml/src/ggml-cuda/quantize.cuh 

#pragma once



#include "common.cuh"

#include "mmq.cuh"



#include <cstdint>



#define CUDA_QUANTIZE_BLOCK_SIZE     256

#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128



static_assert(MATRIX_ROW_PADDING %    CUDA_QUANTIZE_BLOCK_SIZE      == 0, "Risk of out-of-bounds access.");

static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");



typedef void (*quantize_cuda_t)(

        const float * x, const int32_t * ids, void * vy,

        ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,

        int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);



void quantize_row_q8_1_cuda(

        const float * x, const int32_t * ids, void * vy,

        ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,

        int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);



void quantize_mmq_q8_1_cuda(

        const float * x, const int32_t * ids, void * vy,

        ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,

        int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);



void quantize_mmq_fp4_cuda(const float *   x,

                             const int32_t * ids,

                             void *          vy,

                             ggml_type       type_src0,

                             int64_t         ne00,

                             int64_t         s01,

                             int64_t         s02,

                             int64_t         s03,

                             int64_t         ne0,

                             int64_t         ne1,

                             int64_t         ne2,

                             int64_t         ne3,

                             cudaStream_t    stream);