#ifdef VEC
#define VEC_SIZE 4
#define SHMEM_TYPE vec4<f16>
#define DST_TYPE vec4<f32>
#define SRC0_TYPE vec4<SRC0_INNER_TYPE>
#define SRC1_TYPE vec4<SRC1_INNER_TYPE>

fn store_shmem(val: vec4<f16>, idx: u32) {
    shmem[idx] = val.x;
    shmem[idx + 1] = val.y;
    shmem[idx + 2] = val.z;
    shmem[idx + 3] = val.w;
}
#endif // VEC

#ifdef SCALAR
#define VEC_SIZE 1
#define SHMEM_TYPE f16
#define DST_TYPE f32
#define SRC0_TYPE SRC0_INNER_TYPE
#define SRC1_TYPE SRC1_INNER_TYPE

fn store_shmem(val: f16, idx: u32) {
    shmem[idx] = val;
}
#endif // SCALAR

#define QUANT_SHMEM shmem
#define QUANT_OUT_TYPE f16
#include "quant_inner_loops.tmpl"

#ifdef INIT_SRC0_SHMEM_FLOAT
fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
    for (var elem_idx = thread_id * VEC_SIZE; elem_idx < TILE_SRC0_SHMEM; elem_idx += TOTAL_WORKGROUP_SIZE * VEC_SIZE) {
        let tile_m = elem_idx / TILE_K;
        let tile_k = elem_idx % TILE_K;
        let global_m = offset_m + tile_m;
        let global_k = k_outer + tile_k;
        let src0_idx = batch_offset + global_m * params.stride_01 + global_k;
        let src0_val = select( // taking a slight performance hit to avoid oob
            SRC0_TYPE(0.0),
            src0[src0_idx/VEC_SIZE],
            global_m < params.m && global_k < params.k);
        store_shmem(SHMEM_TYPE(src0_val), elem_idx);
    }
}
#endif // INIT_SRC0_SHMEM_FLOAT

#ifndef MUL_MAT_ID
#ifdef INIT_SRC1_SHMEM_FLOAT
fn init_shmem_src1(thread_id: u32, batch_offset: u32, offset_n: u32, k_outer: u32) {
    for (var elem_idx = thread_id * VEC_SIZE; elem_idx < TILE_SRC1_SHMEM; elem_idx += TOTAL_WORKGROUP_SIZE * VEC_SIZE) {
        let tile_n = elem_idx / TILE_K;
        let tile_k = elem_idx % TILE_K;
        let global_n = offset_n + tile_n;
        let global_k = k_outer + tile_k;
        let src1_idx = batch_offset + global_n * params.stride_11 + global_k;
        let src1_val = select(
            SRC1_TYPE(0.0),
            src1[src1_idx/VEC_SIZE],
            global_n < params.n && global_k < params.k);
        store_shmem(SHMEM_TYPE(src1_val), TILE_SRC0_SHMEM + elem_idx);
    }
}
#endif // INIT_SRC1_SHMEM_FLOAT
#endif

#ifdef INIT_SRC0_SHMEM_Q1_0
const BLOCK_SIZE = 128u;
const BLOCK_SIZE_BYTES = 18u;
const NQ = 8u; // 8 weights (1 byte of qs) per thread per iteration

fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
    for (var i = thread_id * NQ; i < TILE_SRC0_SHMEM; i += TOTAL_WORKGROUP_SIZE * NQ) {
        let tile_m = i / TILE_K;
        let tile_k_start = i % TILE_K;
        let global_m = offset_m + tile_m;
        let global_k_start = k_outer + tile_k_start;

        if (global_m >= params.m) {
            break;
        }

        let block_k = global_k_start / BLOCK_SIZE;
        let byte_in_block = (global_k_start % BLOCK_SIZE) / 8u;
        let src0_idx = batch_offset + global_m * params.stride_01 + block_k;
        let block_byte_base = src0_idx * BLOCK_SIZE_BYTES;
        let d = load_f16_at_src0(block_byte_base);
        let q_byte = load_u32_at_src0(block_byte_base + 2u + byte_in_block) & 0xFFu;

        for (var bit = 0u; bit < NQ; bit++) {
            let global_k = global_k_start + bit;
            if (global_k < params.k) {
                shmem[i + bit] = select(-d, d, ((q_byte >> bit) & 1u) != 0u);
            }
        }
    }
}
#endif // INIT_SRC0_SHMEM_Q1_0

// legacy-quants
#if defined(INIT_SRC0_SHMEM_Q4_0) || defined(INIT_SRC0_SHMEM_Q4_1) || defined(INIT_SRC0_SHMEM_Q5_0) || defined(INIT_SRC0_SHMEM_Q5_1) || defined(INIT_SRC0_SHMEM_Q8_0) || defined(INIT_SRC0_SHMEM_Q8_1) || defined(INIT_SRC0_SHMEM_MXFP4)
const BLOCK_SIZE = 32u;
// the number of blocks per k-tile. Note that this currently only works if TILE_K is a multiple of BLOCK_SIZE, which may need to be rethought for larger quantized types.
override BLOCKS_K = TILE_K/BLOCK_SIZE;
const NQ = 16u;
#if defined(INIT_SRC0_SHMEM_Q8_0) || defined(INIT_SRC0_SHMEM_Q8_1)
const BYTES_PER_THREAD = 16u; // NQ(16) weights use 16 bytes of q
#else
const BYTES_PER_THREAD = 8u; // NQ(16) weights use 8 bytes of q
#endif
const BYTES_PER_INNER_LOOP = 4u; // == sizeof(q_packed)

fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
    for (var i = thread_id * NQ; i < TILE_SRC0_SHMEM; i += TOTAL_WORKGROUP_SIZE * NQ) {
        let block_idx = i / BLOCK_SIZE;
        let block_offset = (i % BLOCK_SIZE) / NQ;
        let shmem_idx = block_idx * BLOCK_SIZE + block_offset * BYTES_PER_THREAD;

        let tile_m = block_idx / BLOCKS_K;
        let global_m = offset_m + tile_m;
        let block_k = block_idx % BLOCKS_K;
        let global_block_k = k_outer / BLOCK_SIZE + block_k;

        if (global_m < params.m && global_block_k < params.k / BLOCK_SIZE) {
            let src0_idx = batch_offset + global_m * params.stride_01 + global_block_k;

#if defined(INIT_SRC0_SHMEM_Q4_0)
            let block_byte_base = src0_idx * 18u; // BLOCK_SIZE_BYTES = 18u;
            let d = load_f16_at_src0(block_byte_base);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 2u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0(q_byte_offset);
                dequant_q4_0_packed_to_shmem(q_packed, d, shmem_idx + j * BYTES_PER_INNER_LOOP);
            }
#endif // INIT_SRC0_SHMEM_Q4_0

#if defined(INIT_SRC0_SHMEM_Q4_1)
            let block_byte_base = src0_idx * 20u; // BLOCK_SIZE_BYTES = 20u;
            let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base));
            let d = f16(dm[0]);
            let m = f16(dm[1]);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 4u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0(q_byte_offset);

                for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) {
                    let q_byte = get_byte(q_packed, k);
                    let q_lo = f16(q_byte & 0xF) * d + m;
                    let q_hi = f16((q_byte >> 4) & 0xF) * d + m;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi;
                }
            }
#endif // INIT_SRC0_SHMEM_Q4_1

#if defined(INIT_SRC0_SHMEM_Q5_0)
            let block_byte_base = src0_idx * 22u; // BLOCK_SIZE_BYTES = 22u;

            let d  = load_f16_at_src0(block_byte_base);
            let qh_packed = load_u32_at_src0(block_byte_base + 2u);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 6u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0(q_byte_offset);

                for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) {
                    let q_byte = get_byte(q_packed, k);

                    let byte_idx = block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP + k;
                    let qh_hi = (qh_packed >> (byte_idx + 12u)) & 0x10;
                    let q_hi = (f16(((q_byte >> 4) & 0xF) | qh_hi) - 16.0) * d;
                    let qh_lo = ((qh_packed >> byte_idx) << 4) & 0x10;
                    let q_lo = (f16((q_byte & 0xF) | qh_lo) - 16.0) * d;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi;
                }
            }
#endif // INIT_SRC0_SHMEM_Q5_0

#if defined(INIT_SRC0_SHMEM_Q5_1)
            let block_byte_base = src0_idx * 24u; // BLOCK_SIZE_BYTES = 24u;

            let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base));
            let d  = f16(dm[0]);
            let m = f16(dm[1]);
            let qh_packed = load_u32_at_src0_aligned(block_byte_base + 4u);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 8u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0_aligned(q_byte_offset);

                for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) {
                    let q_byte = get_byte(q_packed, k);

                    let byte_idx = block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP + k;
                    let qh_hi = (qh_packed >> (byte_idx + 12u)) & 0x10;
                    let q_hi = f16(((q_byte >> 4) & 0xF) | qh_hi) * d + m;
                    let qh_lo = ((qh_packed >> byte_idx) << 4) & 0x10;
                    let q_lo = f16((q_byte & 0xF) | qh_lo) * d + m;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi;
                }
            }
#endif // INIT_SRC0_SHMEM_Q5_1

#if defined(INIT_SRC0_SHMEM_Q8_0)
            let block_byte_base = src0_idx * 34u; // BLOCK_SIZE_BYTES = 34u;
            let d = load_f16_at_src0(block_byte_base);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 2u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0(q_byte_offset);
                dequant_q8_0_packed_to_shmem(q_packed, d, shmem_idx + j * BYTES_PER_INNER_LOOP);
            }
#endif // INIT_SRC0_SHMEM_Q8_0

#if defined(INIT_SRC0_SHMEM_Q8_1)
            let block_byte_base = src0_idx * 36u; // BLOCK_SIZE_BYTES = 36u;
            let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base));
            let d = f16(dm[0]);
            let m = f16(dm[1]);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 4u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0(q_byte_offset);
                for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) {
                    let q_byte = get_byte_i32(q_packed, k);
                    let q_val = f16(q_byte) * d + m;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_val;
                }
            }
#endif // INIT_SRC0_SHMEM_Q8_1

#if defined(INIT_SRC0_SHMEM_MXFP4)
            let block_byte_base = src0_idx * 17u;
            let eu8 = get_byte(load_u32_at_src0_aligned(block_byte_base), block_byte_base & 3u);
            let e = ldexp(1.0, i32(eu8) - 128);

            // load NQ(16) weights
            for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
                let q_byte_offset = block_byte_base + 1u + block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP;
                let q_packed = load_u32_at_src0(q_byte_offset);
                for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) {
                    let q_byte = get_byte(q_packed, k);
                    let q_hi = f32(kvalues_mxfp4[(q_byte >> 4) & 0xF]) * e;
                    let q_lo = f32(kvalues_mxfp4[q_byte & 0xF]) * e;
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo);
                    shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi);
                }
            }
#endif // INIT_SRC0_SHMEM_MXFP4
        }
    }
}
#endif // legacy-quants

// k-quants
#if defined(INIT_SRC0_SHMEM_Q2_K) || defined(INIT_SRC0_SHMEM_Q3_K) || defined(INIT_SRC0_SHMEM_Q4_K) || defined(INIT_SRC0_SHMEM_Q5_K) || defined(INIT_SRC0_SHMEM_Q6_K)
const BLOCK_SIZE = 256u;
const NQ = 4u;

fn store_shmem_kquants(val: vec4<f16>, idx: u32) {
    shmem[idx] = val.x;
    shmem[idx + 1] = val.y;
    shmem[idx + 2] = val.z;
    shmem[idx + 3] = val.w;
}

fn load_byte_at_src0_aligned(byte_offset: u32) -> u32 {
    return get_byte(load_u32_at_src0_aligned(byte_offset), byte_offset % 4u);
}

fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
    for (var elem_idx = thread_id * NQ; elem_idx < TILE_SRC0_SHMEM; elem_idx += TOTAL_WORKGROUP_SIZE * NQ) {
        let tile_m = elem_idx / TILE_K;
        let tile_k = elem_idx % TILE_K;

        let global_m = offset_m + tile_m;
        let global_k = k_outer + tile_k;

        if (global_m >= params.m || global_k >= params.k) {
            store_shmem_kquants(vec4<f16>(f16(0.0), f16(0.0), f16(0.0), f16(0.0)), elem_idx);
            continue;
        }

        let block_k    = global_k / BLOCK_SIZE;
        let k_in_block = global_k % BLOCK_SIZE; // k_in_block % 4 == 0;

        let src0_idx = batch_offset + global_m * params.stride_01 + block_k;

#if defined(INIT_SRC0_SHMEM_Q2_K)
        let block_byte_base  = src0_idx * 84u; // BLOCK_SIZE_BYTES =  84u;
        let scales_byte_base = block_byte_base;
        let qs_byte_base     = block_byte_base + 16u;
        let dm_byte_base     = block_byte_base + 80u;

        let d_packed = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base));
        let d        = f16(d_packed[0]);
        let dmin     = f16(d_packed[1]);

        let chunk        = k_in_block / 128u;
        let pos_in_chunk = k_in_block % 32u;
        let sub_block    = k_in_block / 16u;
        let shift_phase  = (k_in_block % 128u) / 32u;

        // whole 2 bits (4 elems)
        let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk);
        let qs_vec4 = vec4<f16>(
            f16((qs_word >> (2u * shift_phase +  0u)) & 0x3u),
            f16((qs_word >> (2u * shift_phase +  8u)) & 0x3u),
            f16((qs_word >> (2u * shift_phase + 16u)) & 0x3u),
            f16((qs_word >> (2u * shift_phase + 24u)) & 0x3u),
        );

        let scale = load_byte_at_src0_aligned(scales_byte_base + sub_block);

        let dl = d * f16(scale & 0xFu);
        let ml = dmin * f16(scale >> 4u);

        store_shmem_kquants(qs_vec4 * dl - ml, elem_idx);
#endif // INIT_SRC0_SHMEM_Q2_K

#if defined(INIT_SRC0_SHMEM_Q3_K)
        let block_byte_base  = src0_idx * 110u; // BLOCK_SIZE_BYTES = 110u;
        let hmask_byte_base  = block_byte_base +  0u;
        let qs_byte_base     = block_byte_base + 32u;
        let scales_byte_base = block_byte_base + 96u;

        let d_all = load_f16_at_src0(block_byte_base + 108u);

        let chunk        = k_in_block / 128u;
        let pos_in_chunk = k_in_block % 32u;
        let sub_block    = k_in_block / 16u;
        let shift_phase  = (k_in_block % 128u) / 32u;

        let hmask_block       = pos_in_chunk;
        let hmask_shift_phase = k_in_block / 32u;

        // low 2 bits (4 elems)
        let q_lo2_word = load_u32_at_src0(qs_byte_base + 32u * chunk + 1u * hmask_block);
        let q_lo2_vec4 = vec4<f16>(
            f16((q_lo2_word >> (2u * shift_phase +  0u)) & 3u),
            f16((q_lo2_word >> (2u * shift_phase +  8u)) & 3u),
            f16((q_lo2_word >> (2u * shift_phase + 16u)) & 3u),
            f16((q_lo2_word >> (2u * shift_phase + 24u)) & 3u)
        );

        // high 1 bit (4 elems)
        let q_hi1_word = load_u32_at_src0(hmask_byte_base + pos_in_chunk);
        let q_hi1_vec4 = vec4<f16>(
            f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase +  0u)) & 1u) == 1u)),
            f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase +  8u)) & 1u) == 1u)),
            f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 16u)) & 1u) == 1u)),
            f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 24u)) & 1u) == 1u))
        );

        let q_vec4 = q_lo2_vec4 - q_hi1_vec4;

        let scale_low4 = (load_byte_at_src0_aligned(scales_byte_base + (sub_block % 8u)) >> (4u * (sub_block / 8u))) & 0xFu;
        let scale_hi2  = (load_byte_at_src0_aligned(scales_byte_base + 8u + (sub_block % 4u)) >> (2u * (sub_block / 4u))) & 3u;
        let dl         = d_all * (f16((scale_hi2 << 4u) | scale_low4) - 32.0);

        store_shmem_kquants(dl * q_vec4, elem_idx);
#endif // INIT_SRC0_SHMEM_Q3_K

#if defined(INIT_SRC0_SHMEM_Q4_K)
        let block_byte_base = src0_idx * 144u; // BLOCK_SIZE_BYTES = 144u;
        let dm_byte_base    = block_byte_base +  0u;
        let scale_byte_base = block_byte_base +  4u;
        let qs_byte_base    = block_byte_base + 16u;

        let dm   = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base));
        let d    = f16(dm[0]);
        let dmin = f16(dm[1]);

        let chunk        = k_in_block / 64u;
        let pos_in_chunk = (k_in_block % 64u) % 32u;
        let sub_block    = k_in_block / 32u;
        let shift_phase  = sub_block & 1u;

        // whole 4 bits (4 elems)
        let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk);
        let qs_vec4 = vec4<f16>(
            f16((qs_word >> (4u * shift_phase +  0u)) & 0xFu),
            f16((qs_word >> (4u * shift_phase +  8u)) & 0xFu),
            f16((qs_word >> (4u * shift_phase + 16u)) & 0xFu),
            f16((qs_word >> (4u * shift_phase + 24u)) & 0xFu)
        );

        var sc: u32;
        var mn: u32;

        if (sub_block < 4u) {
            let sc_byte  = get_byte(load_u32_at_src0_aligned(scale_byte_base), sub_block % 4u);
            let min_byte = get_byte(load_u32_at_src0_aligned(scale_byte_base + 4), sub_block % 4u);
            sc           = sc_byte & 63u;
            mn           = min_byte & 63u;
        } else {
            let sc_min_lo = get_byte(load_u32_at_src0_aligned(scale_byte_base + 8), (sub_block + 4u) % 4u);
            let sc_hi     = get_byte(load_u32_at_src0_aligned(scale_byte_base), (sub_block - 4u) % 4u);
            let min_hi    = get_byte(load_u32_at_src0_aligned(scale_byte_base + 4), sub_block % 4u);
            sc            = (sc_min_lo & 0xFu) | ((sc_hi >> 6u) << 4u);
            mn            = (sc_min_lo >> 4u) | ((min_hi >> 6u) << 4u);
        }

        let dl = d * f16(sc);
        let ml = dmin * f16(mn);

        store_shmem_kquants(dl * qs_vec4 - vec4(ml, ml, ml, ml), elem_idx);
#endif // INIT_SRC0_SHMEM_Q4_K

#if defined(INIT_SRC0_SHMEM_Q5_K)
        let block_byte_base = src0_idx * 176u; // BLOCK_SIZE_BYTES = 176u;
        let dm_byte_base    = block_byte_base +  0u;
        let scale_byte_base = block_byte_base +  4u;
        let qh_byte_base    = block_byte_base + 16u;
        let qs_byte_base    = block_byte_base + 48u;

        let dm   = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base));
        let d    = f16(dm[0]);
        let dmin = f16(dm[1]);

        let chunk        = k_in_block / 64u;
        let pos_in_chunk = (k_in_block % 64u) % 32u;
        let sub_block    = k_in_block / 32u;
        let shift_phase  = sub_block & 1u;

        let qh_block       = k_in_block % 32u;
        let qh_shift_phase = sub_block;

        // low 4 bits (4 elems)
        let qs_word     = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk);
        let qs_lo4_vec4 = vec4<f16>(
            f16((qs_word >> (4u * shift_phase +  0u)) & 0xFu),
            f16((qs_word >> (4u * shift_phase +  8u)) & 0xFu),
            f16((qs_word >> (4u * shift_phase + 16u)) & 0xFu),
            f16((qs_word >> (4u * shift_phase + 24u)) & 0xFu)
        );

        // high 1 bit (4 elems)
        let qh_word = load_u32_at_src0_aligned(qh_byte_base + qh_block);
        let qh_vec4 = vec4<f16>(
            f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase +  0u)) & 1u) == 1u)),
            f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase +  8u)) & 1u) == 1u)),
            f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 16u)) & 1u) == 1u)),
            f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 24u)) & 1u) == 1u))
        );

        var sc: u32;
        var mn: u32;

        if (sub_block < 4u) {
            let sc_byte  = get_byte(load_u32_at_src0_aligned(scale_byte_base), sub_block % 4u);
            let min_byte = get_byte(load_u32_at_src0_aligned(scale_byte_base + 4), sub_block % 4u);
            sc           = sc_byte & 63u;
            mn           = min_byte & 63u;
        } else {
            let sc_min_lo = get_byte(load_u32_at_src0_aligned(scale_byte_base + 8), (sub_block + 4u) % 4u);
            let sc_hi     = get_byte(load_u32_at_src0_aligned(scale_byte_base), (sub_block - 4u) % 4u);
            let min_hi    = get_byte(load_u32_at_src0_aligned(scale_byte_base + 4), sub_block % 4u);
            sc            = (sc_min_lo & 0xFu) | ((sc_hi >> 6u) << 4u);
            mn            = (sc_min_lo >> 4u) | ((min_hi >> 6u) << 4u);
        }

        let dl = d * f16(sc);
        let ml = dmin * f16(mn);

        store_shmem_kquants((qh_vec4 + qs_lo4_vec4) * dl - vec4<f16>(ml, ml, ml, ml), elem_idx);
#endif // INIT_SRC0_SHMEM_Q5_K

#if defined(INIT_SRC0_SHMEM_Q6_K)
        let block_byte_base  = src0_idx * 210u; // BLOCK_SIZE_BYTES = 210u;
        let ql_byte_base     = block_byte_base;
        let qh_byte_base     = block_byte_base + 128u;
        let scales_byte_base = block_byte_base + 192u;
        let d_byte_base      = block_byte_base + 208u;

        let d = load_f16_at_src0(d_byte_base);

        let chunk           = k_in_block / 128u;
        let ql_pos_in_chunk = (k_in_block % 128u) % 64u;
        let qh_pos_in_chunk = (k_in_block % 128u) % 32u;
        let sub_block       = k_in_block / 16u;
        let ql_shift_phase  = (k_in_block % 128u) / 64u;
        let qh_shift_phase  = (k_in_block % 128u) / 32u;

        // low 4 bits (4 elems)
        let ql_word     = load_u32_at_src0(ql_byte_base + 64u * chunk + 1u * ql_pos_in_chunk);
        let ql_lo4_vec4 = vec4<u32>(
            (ql_word >> (4u * ql_shift_phase +  0u)) & 0xFu,
            (ql_word >> (4u * ql_shift_phase +  8u)) & 0xFu,
            (ql_word >> (4u * ql_shift_phase + 16u)) & 0xFu,
            (ql_word >> (4u * ql_shift_phase + 24u)) & 0xFu
        );

        // hi 2 bits (4 elems)
        let qh_word     = load_u32_at_src0(qh_byte_base + 32u * chunk + 1u * qh_pos_in_chunk);
        let qh_hi2_vec4 = vec4<u32>(
            ((qh_word >> (2u * qh_shift_phase +  0u)) & 0x3u) << 4u,
            ((qh_word >> (2u * qh_shift_phase +  8u)) & 0x3u) << 4u,
            ((qh_word >> (2u * qh_shift_phase + 16u)) & 0x3u) << 4u,
            ((qh_word >> (2u * qh_shift_phase + 24u)) & 0x3u) << 4u,
        );

        let q_vec4 = vec4<f16>(qh_hi2_vec4 | ql_lo4_vec4) - vec4<f16>(32.0, 32.0, 32.0, 32.0);

        let scale_byte = scales_byte_base + 1u * sub_block;
        let scale_word = load_u32_at_src0_aligned(scale_byte);
        let scale      = get_byte_i32(scale_word, scale_byte & 3u);

        store_shmem_kquants(d * q_vec4 * f16(scale), elem_idx);
#endif // INIT_SRC0_SHMEM_Q6_K
    }
}
#endif // k-quants

#if defined(INIT_SRC0_SHMEM_IQ4_NL)
const BLOCK_SIZE = 32u;
const BLOCK_SIZE_BYTES = 18u;
const NQ = 4u;

fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
    for (var elem_idx = thread_id * NQ; elem_idx < TILE_SRC0_SHMEM; elem_idx += NQ * TOTAL_WORKGROUP_SIZE) {
        let tile_m = elem_idx / TILE_K;
        let tile_k = elem_idx % TILE_K;
        let global_m = offset_m + tile_m;
        let global_k = k_outer + tile_k;

        if (global_m >= params.m || global_k >= params.k) {
            shmem[elem_idx] = f16(0.0);
            continue;
        }

        let block_k    = global_k / BLOCK_SIZE;
        let k_in_block = global_k % BLOCK_SIZE; // k_in_block % 4 == 0;

        let src0_idx = batch_offset + global_m * params.stride_01 + block_k;

        let block_byte_base = src0_idx * BLOCK_SIZE_BYTES;
        let d_byte_base     = block_byte_base +  0u;
        let qs_byte_base    = block_byte_base +  2u;

        let d = load_f16_at_src0(d_byte_base);

        let id_qtr      = (k_in_block % 16u) / 4u;
        let shift_phase = k_in_block / 16u;

        let qs_u32    = load_u32_at_src0(qs_byte_base + 4u * id_qtr);

        shmem[elem_idx + 0u] = d * f16(kvalues_iq4nl[(qs_u32 >> ( 0u + 4u * shift_phase)) & 0xFu]);
        shmem[elem_idx + 1u] = d * f16(kvalues_iq4nl[(qs_u32 >> ( 8u + 4u * shift_phase)) & 0xFu]);
        shmem[elem_idx + 2u] = d * f16(kvalues_iq4nl[(qs_u32 >> (16u + 4u * shift_phase)) & 0xFu]);
        shmem[elem_idx + 3u] = d * f16(kvalues_iq4nl[(qs_u32 >> (24u + 4u * shift_phase)) & 0xFu]);
    }
}
#endif // INIT_SRC0_SHMEM_IQ4_NL

// i-quants (super block size: 256)
#if defined(INIT_SRC0_SHMEM_IQ4_XS) || defined(INIT_SRC0_SHMEM_IQ1_S) || defined(INIT_SRC0_SHMEM_IQ1_M) || defined(INIT_SRC0_SHMEM_IQ2_XXS) \
|| defined(INIT_SRC0_SHMEM_IQ2_XS) || defined(INIT_SRC0_SHMEM_IQ2_S) || defined(INIT_SRC0_SHMEM_IQ3_XXS) || defined(INIT_SRC0_SHMEM_IQ3_S)
const BLOCK_SIZE = 256u;
const NQ = 16u;

fn store_shmem_iquants(val: vec4<f16>, idx: u32) {
    shmem[idx] = val.x;
    shmem[idx + 1] = val.y;
    shmem[idx + 2] = val.z;
    shmem[idx + 3] = val.w;
}

fn load_byte_at_src0_aligned(byte_offset: u32) -> u32 {
    return get_byte(load_u32_at_src0_aligned(byte_offset), byte_offset % 4u);
}

#if defined(INIT_SRC0_SHMEM_IQ1_M) || defined(INIT_SRC0_SHMEM_IQ1_S)
fn create_iq_gw4(dl: f32, gw: u32, shift_base: u32, delta: f32) -> vec4<f16> {
    return vec4<f16>(
            f16(dl * (f32((bitcast<i32>(((gw >> (shift_base + 0u)) & 3u) << 30u) >> 30u)) + delta)),
            f16(dl * (f32((bitcast<i32>(((gw >> (shift_base + 2u)) & 3u) << 30u) >> 30u)) + delta)),
            f16(dl * (f32((bitcast<i32>(((gw >> (shift_base + 4u)) & 3u) << 30u) >> 30u)) + delta)),
            f16(dl * (f32((bitcast<i32>(((gw >> (shift_base + 6u)) & 3u) << 30u) >> 30u)) + delta)),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ4_XS)
fn create_iq_gw4(dl: f16, qs_u32: u32, shift_phase: u32) -> vec4<f16> {
    return vec4<f16>(
            dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase +  0u)) & 0xFu]),
            dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase +  8u)) & 0xFu]),
            dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 16u)) & 0xFu]),
            dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 24u)) & 0xFu]),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ2_XXS)
fn create_iq_gw4(ig: u32, grid_phase: u32) -> vec4<f32> {
    return vec4<f32>(
            f32(get_byte(iq2xxs_grid[(ig + grid_phase + 0u) / 4u], (ig + grid_phase + 0u) % 4u)),
            f32(get_byte(iq2xxs_grid[(ig + grid_phase + 1u) / 4u], (ig + grid_phase + 1u) % 4u)),
            f32(get_byte(iq2xxs_grid[(ig + grid_phase + 2u) / 4u], (ig + grid_phase + 2u) % 4u)),
            f32(get_byte(iq2xxs_grid[(ig + grid_phase + 3u) / 4u], (ig + grid_phase + 3u) % 4u)),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ2_XS)
fn create_iq_gw4(ig: u32, grid_phase: u32) -> vec4<f32> {
    return vec4<f32>(
            f32(get_byte(iq2xs_grid[(ig + grid_phase + 0u) / 4u], (ig + grid_phase + 0u) % 4u)),
            f32(get_byte(iq2xs_grid[(ig + grid_phase + 1u) / 4u], (ig + grid_phase + 1u) % 4u)),
            f32(get_byte(iq2xs_grid[(ig + grid_phase + 2u) / 4u], (ig + grid_phase + 2u) % 4u)),
            f32(get_byte(iq2xs_grid[(ig + grid_phase + 3u) / 4u], (ig + grid_phase + 3u) % 4u)),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ2_S)
fn create_iq_gw4(ig: u32, grid_phase: u32) -> vec4<f32> {
    return vec4<f32>(
            f32(get_byte(iq2s_grid[(ig + grid_phase + 0u) / 4u], (ig + grid_phase + 0u) % 4u)),
            f32(get_byte(iq2s_grid[(ig + grid_phase + 1u) / 4u], (ig + grid_phase + 1u) % 4u)),
            f32(get_byte(iq2s_grid[(ig + grid_phase + 2u) / 4u], (ig + grid_phase + 2u) % 4u)),
            f32(get_byte(iq2s_grid[(ig + grid_phase + 3u) / 4u], (ig + grid_phase + 3u) % 4u)),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ3_XXS)
fn create_iq_gw4(ig: u32) -> vec4<f32> {
    return vec4<f32>(
            f32(get_byte(iq3xxs_grid[ig], 0)),
            f32(get_byte(iq3xxs_grid[ig], 1)),
            f32(get_byte(iq3xxs_grid[ig], 2)),
            f32(get_byte(iq3xxs_grid[ig], 3)),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ3_S)
fn create_iq_gw4(ig: u32) -> vec4<f32> {
    return vec4<f32>(
            f32(get_byte(iq3s_grid[ig], 0)),
            f32(get_byte(iq3s_grid[ig], 1)),
            f32(get_byte(iq3s_grid[ig], 2)),
            f32(get_byte(iq3s_grid[ig], 3)),
        );
}
#endif

#if defined(INIT_SRC0_SHMEM_IQ2_XXS) || defined(INIT_SRC0_SHMEM_IQ2_XS) || defined(INIT_SRC0_SHMEM_IQ2_S) \
|| defined(INIT_SRC0_SHMEM_IQ3_XXS) || defined(INIT_SRC0_SHMEM_IQ3_S)
fn create_iq2_m4(signs: u32, mask_phase: u32) -> vec4<f32> {
    return vec4<f32>(
            select(1.0, -1.0, (get_byte(kmask_iq2xs[mask_phase], 0) & signs) != 0u),
            select(1.0, -1.0, (get_byte(kmask_iq2xs[mask_phase], 1) & signs) != 0u),
            select(1.0, -1.0, (get_byte(kmask_iq2xs[mask_phase], 2) & signs) != 0u),
            select(1.0, -1.0, (get_byte(kmask_iq2xs[mask_phase], 3) & signs) != 0u),
        );
}
#endif

fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u32) {
    for (var elem_idx = thread_id * NQ; elem_idx < TILE_SRC0_SHMEM; elem_idx += NQ * TOTAL_WORKGROUP_SIZE) {
        let tile_m = elem_idx / TILE_K;
        let tile_k = elem_idx % TILE_K;
        let global_m = offset_m + tile_m;
        let global_k = k_outer + tile_k;

        if (global_m >= params.m || global_k >= params.k) {
            let zero_vec4 = vec4<f16>(f16(0.0), f16(0.0), f16(0.0), f16(0.0));
            store_shmem_iquants(zero_vec4, elem_idx +  0u);
            store_shmem_iquants(zero_vec4, elem_idx +  4u);
            store_shmem_iquants(zero_vec4, elem_idx +  8u);
            store_shmem_iquants(zero_vec4, elem_idx + 12u);
            continue;
        }

        let block_k    = global_k / BLOCK_SIZE;
        let k_in_block = global_k % BLOCK_SIZE; // k_in_block % 16 == 0;

        let src0_idx = batch_offset + global_m * params.stride_01 + block_k;

#if defined(INIT_SRC0_SHMEM_IQ4_XS)
        let block_byte_base    = src0_idx * 136u; // BLOCK_SIZE_BYTES = 136u;
        let d_byte_base        = block_byte_base +  0u;
        let scales_l_byte_base = block_byte_base +  4u;
        let qs_byte_base       = block_byte_base +  8u;

        let d_scales_h = load_u32_at_src0_aligned(d_byte_base);
        let d          = bitcast<vec2<f16>>(d_scales_h).x;
        let scales_h   = d_scales_h >> 16u;

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let scales_l_u32 = load_u32_at_src0_aligned(scales_l_byte_base);
        let ls_lo        = (get_byte(scales_l_u32, sub_block / 2u) >> (4u * (sub_block % 2u))) & 0xFu;
        let ls_hi        = ((scales_h >> (2u * sub_block)) & 3u) << 4u;
        let dl           = d * f16(i32(ls_lo | ls_hi) - 32);

        let qs_0_3_u32   = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block +  0u);
        let qs_4_7_u32   = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block +  4u);
        let qs_8_11_u32  = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block +  8u);
        let qs_12_15_u32 = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block + 12u);

        store_shmem_iquants(create_iq_gw4(dl, qs_0_3_u32,   phase), elem_idx +  0u);
        store_shmem_iquants(create_iq_gw4(dl, qs_4_7_u32,   phase), elem_idx +  4u);
        store_shmem_iquants(create_iq_gw4(dl, qs_8_11_u32,  phase), elem_idx +  8u);
        store_shmem_iquants(create_iq_gw4(dl, qs_12_15_u32, phase), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ4_XS

#if defined(INIT_SRC0_SHMEM_IQ1_S)
        let block_byte_base = src0_idx * 50u; // BLOCK_SIZE_BYTES = 50u;
        let d_byte_base     = block_byte_base +  0u;
        let qs_byte_base    = block_byte_base +  2u;
        let qh_byte_base    = block_byte_base + 34u;

        let d = load_f16_as_f32_at_src0(d_byte_base);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let qh_u16 = load_u32_at_src0(qh_byte_base + sub_block * 2u) & 0xFFFFu;
        let qs_u16 = load_u32_at_src0(qs_byte_base + sub_block * 4u + phase * 2u) & 0xFFFFu;

        let dl    = d * (2.0 * f32((qh_u16 >> 12u) & 7u) + 1.0);
        let delta = select(IQ1_DELTA, -IQ1_DELTA, (qh_u16 & 0x8000u) != 0u);

        let gp0_grid_id = ((qs_u16 & 0xFFu) | (((qh_u16 >> (phase * 6u)) & 7u) << 8u)) * 8u;
        let gp1_grid_id = (((qs_u16 >> 8) & 0xFFu) | (((qh_u16 >> (phase * 6u + 3u)) & 7u) << 8u)) * 8u;

        let gp0_gw = iq1_grid[(gp0_grid_id) / 16u];
        let gp1_gw = iq1_grid[(gp1_grid_id) / 16u];

        let gp0_shift_base = (gp0_grid_id % 16u) * 2u;
        let gp1_shift_base = (gp1_grid_id % 16u) * 2u;

        store_shmem_iquants(create_iq_gw4(dl, gp0_gw, gp0_shift_base + 0u, delta), elem_idx +  0u);
        store_shmem_iquants(create_iq_gw4(dl, gp0_gw, gp0_shift_base + 8u, delta), elem_idx +  4u);
        store_shmem_iquants(create_iq_gw4(dl, gp1_gw, gp1_shift_base + 0u, delta), elem_idx +  8u);
        store_shmem_iquants(create_iq_gw4(dl, gp1_gw, gp1_shift_base + 8u, delta), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ1_S

#if defined(INIT_SRC0_SHMEM_IQ1_M)
        let block_byte_base  = src0_idx * 56u; // BLOCK_SIZE_BYTES = 56u;
        let qs_byte_base     = block_byte_base +  0u;
        let qh_byte_base     = block_byte_base + 32u;
        let scales_byte_base = block_byte_base + 48u;

        let scales0      = load_u32_at_src0_aligned(scales_byte_base);
        let scales1      = load_u32_at_src0_aligned(scales_byte_base + 4u);
        let scale_packed = ((scales0 >> 12u) & 0xFu) |
                           ((scales0 >> 24u) & 0x00F0u) |
                           ((scales1 >>  4u) & 0x0F00u) |
                           ((scales1 >> 16u) & 0xF000u);
        let d = f32(bitcast<vec2<f16>>(scale_packed).x);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let scale_u32 = select(scales0, scales1, sub_block >= 4u);
        let scale_u3  = (scale_u32 >> (16u * ((sub_block / 2u) % 2u) + 6u * (sub_block % 2u) + 3u * phase)) & 0x7u;
        let dl        = d * f32(2u * scale_u3 + 1u);

        let qh_u8  = (load_u32_at_src0_aligned(qh_byte_base + 4u * (sub_block / 2u)) >> (16u * (sub_block % 2u) + 8u * phase)) & 0xFFu;
        let qs_u16 = (load_u32_at_src0_aligned(qs_byte_base + 4u * sub_block) >> (16u * phase)) & 0xFFFFu;

        let gp0_grid_id = ((qs_u16 & 0xFFu) | ((qh_u8 & 7u) << 8u)) * 8u;
        let gp0_delta   = select(IQ1_DELTA, -IQ1_DELTA, (qh_u8 & 0x8u) != 0u);

        let gp1_grid_id = (((qs_u16 >> 8u) & 0xFFu) | (((qh_u8 >> 4u) & 7u) << 8u)) * 8u;
        let gp1_delta   = select(IQ1_DELTA, -IQ1_DELTA, (qh_u8 & 0x80u) != 0u);

        let gp0_gw = iq1_grid[(gp0_grid_id) / 16u];
        let gp1_gw = iq1_grid[(gp1_grid_id) / 16u];

        let gp0_shift_base = (gp0_grid_id % 16u) * 2u;
        let gp1_shift_base = (gp1_grid_id % 16u) * 2u;

        store_shmem_iquants(create_iq_gw4(dl, gp0_gw, gp0_shift_base + 0u, gp0_delta), elem_idx +  0u);
        store_shmem_iquants(create_iq_gw4(dl, gp0_gw, gp0_shift_base + 8u, gp0_delta), elem_idx +  4u);
        store_shmem_iquants(create_iq_gw4(dl, gp1_gw, gp1_shift_base + 0u, gp1_delta), elem_idx +  8u);
        store_shmem_iquants(create_iq_gw4(dl, gp1_gw, gp1_shift_base + 8u, gp1_delta), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ1_M

#if defined(INIT_SRC0_SHMEM_IQ2_XXS)
        let block_byte_base = src0_idx * 66u; // BLOCK_SIZE_BYTES = 66u;
        let d_byte_base     = block_byte_base +  0u;
        let qs_byte_base    = block_byte_base +  2u;

        let d = load_f16_as_f32_at_src0(d_byte_base);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let aux0 = load_u32_at_src0(qs_byte_base + 8u * sub_block +  0u);
        let aux1 = load_u32_at_src0(qs_byte_base + 8u * sub_block + 4u);
        let db   = d * (0.5 + f32(aux1 >> 28u)) * 0.25;

        let gp0_ig = get_byte(aux0, 2u * phase + 0u) * 8u;
        let gp1_ig = get_byte(aux0, 2u * phase + 1u) * 8u;

        let gp0_is = (aux1 >> (14u * phase + 0u)) & 127u;
        let gp1_is = (aux1 >> (14u * phase + 7u)) & 127u;

        let gp0_signs = get_byte(ksigns_iq2xs[gp0_is / 4u], gp0_is % 4u);
        let gp1_signs = get_byte(ksigns_iq2xs[gp1_is / 4u], gp1_is % 4u);

        let m_0_3_val4   = create_iq2_m4(gp0_signs, 0);
        let m_4_7_val4   = create_iq2_m4(gp0_signs, 1);
        let m_8_11_val4  = create_iq2_m4(gp1_signs, 0);
        let m_12_15_val4 = create_iq2_m4(gp1_signs, 1);

        let gw_0_3_val4   = create_iq_gw4(gp0_ig, 0);
        let gw_4_7_val4   = create_iq_gw4(gp0_ig, 4);
        let gw_8_11_val4  = create_iq_gw4(gp1_ig, 0);
        let gw_12_15_val4 = create_iq_gw4(gp1_ig, 4);

        store_shmem_iquants(vec4<f16>(db * m_0_3_val4 * gw_0_3_val4),     elem_idx +  0u);
        store_shmem_iquants(vec4<f16>(db * m_4_7_val4 * gw_4_7_val4),     elem_idx +  4u);
        store_shmem_iquants(vec4<f16>(db * m_8_11_val4 * gw_8_11_val4),   elem_idx +  8u);
        store_shmem_iquants(vec4<f16>(db * m_12_15_val4 * gw_12_15_val4), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ2_XXS

#if defined(INIT_SRC0_SHMEM_IQ2_XS)
        let block_byte_base  = src0_idx * 74u; // BLOCK_SIZE_BYTES = 74u;
        let d_byte_base      = block_byte_base +  0u;
        let qs_byte_base     = block_byte_base +  2u;
        let scales_byte_base = block_byte_base + 66u;

        let d = load_f16_as_f32_at_src0(d_byte_base);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let scale = (load_byte_at_src0_aligned(scales_byte_base + 1u * sub_block) >> (4u * phase)) & 0xFu;
        let db    = d * (0.5 + f32(scale)) * 0.25;

        let qs_u32 = load_u32_at_src0(qs_byte_base + 8u * sub_block + 4u * phase);

        let gp0_ig = (qs_u32 & 0x1FFu) * 8u;
        let gp1_ig = ((qs_u32 >> 16u) & 0x1FFu) * 8u;

        let gp0_is = (qs_u32 >>  9u) & 0x7Fu;
        let gp1_is = (qs_u32 >> 25u) & 0x7Fu;

        let gp0_signs = get_byte(ksigns_iq2xs[gp0_is / 4u], gp0_is % 4u);
        let gp1_signs = get_byte(ksigns_iq2xs[gp1_is / 4u], gp1_is % 4u);

        let m_0_3_val4   = create_iq2_m4(gp0_signs, 0);
        let m_4_7_val4   = create_iq2_m4(gp0_signs, 1);
        let m_8_11_val4  = create_iq2_m4(gp1_signs, 0);
        let m_12_15_val4 = create_iq2_m4(gp1_signs, 1);

        let gw_0_3_val4   = create_iq_gw4(gp0_ig, 0);
        let gw_4_7_val4   = create_iq_gw4(gp0_ig, 4);
        let gw_8_11_val4  = create_iq_gw4(gp1_ig, 0);
        let gw_12_15_val4 = create_iq_gw4(gp1_ig, 4);

        store_shmem_iquants(vec4<f16>(db * m_0_3_val4 * gw_0_3_val4),     elem_idx +  0u);
        store_shmem_iquants(vec4<f16>(db * m_4_7_val4 * gw_4_7_val4),     elem_idx +  4u);
        store_shmem_iquants(vec4<f16>(db * m_8_11_val4 * gw_8_11_val4),   elem_idx +  8u);
        store_shmem_iquants(vec4<f16>(db * m_12_15_val4 * gw_12_15_val4), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ2_XS

#if defined(INIT_SRC0_SHMEM_IQ2_S)
        let block_byte_base  = src0_idx * 82u; // BLOCK_SIZE_BYTES = 82u;
        let d_byte_base      = block_byte_base +  0u;
        let qs_byte_base     = block_byte_base +  2u;
        let qh_byte_base     = block_byte_base + 66u;
        let scales_byte_base = block_byte_base + 74u;

        let d = load_f16_as_f32_at_src0(d_byte_base);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let scale = (load_byte_at_src0_aligned(scales_byte_base + 1u * sub_block) >> (4u * phase)) & 0xFu;
        let db    = d * (0.5 + f32(scale)) * 0.25;

        let qs_u16    = load_u32_at_src0(qs_byte_base + 4u * sub_block + 2u * phase) & 0xFFFFu;
        let signs_u16 = load_u32_at_src0(qs_byte_base + 32u + 4u * sub_block + 2u * phase) & 0xFFFFu;
        let qh_u4     = (load_byte_at_src0_aligned(qh_byte_base + 1u * sub_block) >> (4u * phase)) & 0xFu;

        let gp0_ig = ((qs_u16 & 0xFFu) | ((qh_u4 & 0x3u) << 8u)) * 8u;
        let gp1_ig = (((qs_u16 >> 8u) & 0xFFu) | ((qh_u4 & 0xCu) << 6u)) * 8u;

        let gp0_signs = get_byte(signs_u16, 0);
        let gp1_signs = get_byte(signs_u16, 1);

        let m_0_3_val4   = create_iq2_m4(gp0_signs, 0);
        let m_4_7_val4   = create_iq2_m4(gp0_signs, 1);
        let m_8_11_val4  = create_iq2_m4(gp1_signs, 0);
        let m_12_15_val4 = create_iq2_m4(gp1_signs, 1);

        let gw_0_3_val4   = create_iq_gw4(gp0_ig, 0);
        let gw_4_7_val4   = create_iq_gw4(gp0_ig, 4);
        let gw_8_11_val4  = create_iq_gw4(gp1_ig, 0);
        let gw_12_15_val4 = create_iq_gw4(gp1_ig, 4);

        store_shmem_iquants(vec4<f16>(db * m_0_3_val4 * gw_0_3_val4),     elem_idx +  0u);
        store_shmem_iquants(vec4<f16>(db * m_4_7_val4 * gw_4_7_val4),     elem_idx +  4u);
        store_shmem_iquants(vec4<f16>(db * m_8_11_val4 * gw_8_11_val4),   elem_idx +  8u);
        store_shmem_iquants(vec4<f16>(db * m_12_15_val4 * gw_12_15_val4), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ2_S

#if defined(INIT_SRC0_SHMEM_IQ3_XXS)
        let block_byte_base = src0_idx * 98u; // BLOCK_SIZE_BYTES = 98u;
        let d_byte_base     = block_byte_base +  0u;
        let qs_byte_base    = block_byte_base +  2u;

        let d = load_f16_as_f32_at_src0(d_byte_base);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let qs_u32   = load_u32_at_src0(qs_byte_base + 8u * sub_block + 4u * phase);
        let sign_u32 = load_u32_at_src0(qs_byte_base + 64u + 4u * sub_block);
        let db       = d * (0.5 + f32(sign_u32 >> 28u)) * 0.5;

        let ig_0_3   = get_byte(qs_u32, 0);
        let ig_4_7   = get_byte(qs_u32, 1);
        let ig_8_11  = get_byte(qs_u32, 2);
        let ig_12_15 = get_byte(qs_u32, 3);

        let gp0_is = (sign_u32 >> (14u * phase + 0u)) & 0x7Fu;
        let gp1_is = (sign_u32 >> (14u * phase + 7u)) & 0x7Fu;

        let gp0_signs = get_byte(ksigns_iq2xs[gp0_is / 4u], gp0_is % 4u);
        let gp1_signs = get_byte(ksigns_iq2xs[gp1_is / 4u], gp1_is % 4u);

        let m_0_3_val4   = create_iq2_m4(gp0_signs, 0);
        let m_4_7_val4   = create_iq2_m4(gp0_signs, 1);
        let m_8_11_val4  = create_iq2_m4(gp1_signs, 0);
        let m_12_15_val4 = create_iq2_m4(gp1_signs, 1);

        let gw_0_3_val4   = create_iq_gw4(ig_0_3);
        let gw_4_7_val4   = create_iq_gw4(ig_4_7);
        let gw_8_11_val4  = create_iq_gw4(ig_8_11);
        let gw_12_15_val4 = create_iq_gw4(ig_12_15);

        store_shmem_iquants(vec4<f16>(db * m_0_3_val4 * gw_0_3_val4),     elem_idx +  0u);
        store_shmem_iquants(vec4<f16>(db * m_4_7_val4 * gw_4_7_val4),     elem_idx +  4u);
        store_shmem_iquants(vec4<f16>(db * m_8_11_val4 * gw_8_11_val4),   elem_idx +  8u);
        store_shmem_iquants(vec4<f16>(db * m_12_15_val4 * gw_12_15_val4), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ3_XXS

#if defined(INIT_SRC0_SHMEM_IQ3_S)
        let block_byte_base  = src0_idx * 110u; // BLOCK_SIZE_BYTES = 110u;
        let d_byte_base      = block_byte_base +   0u;
        let qs_byte_base     = block_byte_base +   2u;
        let qh_byte_base     = block_byte_base +  66u;
        let signs_byte_base  = block_byte_base +  74u;
        let scales_byte_base = block_byte_base + 106u;

        let d = load_f16_as_f32_at_src0(d_byte_base);

        let sub_block = k_in_block / 32u;
        let phase     = (k_in_block / NQ) % 2u;

        let scale = (load_byte_at_src0_aligned(scales_byte_base + 1u * (sub_block / 2u)) >> (4u * (sub_block % 2u))) & 0xFu;
        let db    = d * (1.0 + 2.0 * f32(scale));

        let qs_u32    = load_u32_at_src0(qs_byte_base + 8u * sub_block + 4u * phase);
        let qh_u4     = (load_byte_at_src0_aligned(qh_byte_base + 1u * sub_block) >> (4u * phase)) & 0xFu;
        let signs_u16 = (load_u32_at_src0(signs_byte_base + 4u * sub_block + 2u * phase)) & 0xFFFFu;

        let ig_0_3   = ((qs_u32 >>  0u) & 0xFFu) | ((qh_u4 & 0x1u) << 8u);
        let ig_4_7   = ((qs_u32 >>  8u) & 0xFFu) | ((qh_u4 & 0x2u) << 7u);
        let ig_8_11  = ((qs_u32 >> 16u) & 0xFFu) | ((qh_u4 & 0x4u) << 6u);
        let ig_12_15 = ((qs_u32 >> 24u) & 0xFFu) | ((qh_u4 & 0x8u) << 5u);

        let gp0_signs = get_byte(signs_u16, 0);
        let gp1_signs = get_byte(signs_u16, 1);

        let m_0_3_val4   = create_iq2_m4(gp0_signs, 0);
        let m_4_7_val4   = create_iq2_m4(gp0_signs, 1);
        let m_8_11_val4  = create_iq2_m4(gp1_signs, 0);
        let m_12_15_val4 = create_iq2_m4(gp1_signs, 1);

        let gw_0_3_val4   = create_iq_gw4(ig_0_3);
        let gw_4_7_val4   = create_iq_gw4(ig_4_7);
        let gw_8_11_val4  = create_iq_gw4(ig_8_11);
        let gw_12_15_val4 = create_iq_gw4(ig_12_15);

        store_shmem_iquants(vec4<f16>(db * m_0_3_val4 * gw_0_3_val4),     elem_idx +  0u);
        store_shmem_iquants(vec4<f16>(db * m_4_7_val4 * gw_4_7_val4),     elem_idx +  4u);
        store_shmem_iquants(vec4<f16>(db * m_8_11_val4 * gw_8_11_val4),   elem_idx +  8u);
        store_shmem_iquants(vec4<f16>(db * m_12_15_val4 * gw_12_15_val4), elem_idx + 12u);
#endif // INIT_SRC0_SHMEM_IQ3_S
    }
}
#endif // i-quants (super block size: 256)