diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl index 457df07ff..978f2012b 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl @@ -49,7 +49,7 @@ fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u let row_offset = k_row * HEAD_DIM_QK; let global_block_idx = k_head_offset + global_k_row * params.stride_k1 + block_k; let block_byte_base = global_block_idx * K_BLOCK_SIZE_BYTES; - let d = f16_from_u16(load_k_u16_at(block_byte_base)); + let d = unpack2x16float(load_k_u16_at(block_byte_base)).x; let thread_byte_offset = block_offset * K_BYTES_PER_THREAD; let shmem_idx = row_offset + block_k * BLOCK_SIZE + thread_byte_offset; for (var j = 0u; j < K_BYTES_PER_THREAD / K_BYTES_PER_INNER_LOOP; j += 1u) { @@ -98,7 +98,7 @@ fn load_v_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, v_head_offset: u let row_offset = v_row * HEAD_DIM_V; let global_block_idx = v_head_offset + global_v_row * params.stride_v1 + block_k; let block_byte_base = global_block_idx * V_BLOCK_SIZE_BYTES; - let d = f16_from_u16(load_v_u16_at(block_byte_base)); + let d = unpack2x16float(load_v_u16_at(block_byte_base)).x; let thread_byte_offset = block_offset * V_BYTES_PER_THREAD; let shmem_idx = row_offset + block_k * BLOCK_SIZE + thread_byte_offset; for (var j = 0u; j < V_BYTES_PER_THREAD / V_BYTES_PER_INNER_LOOP; j += 1u) { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl index 1af6286ac..3e331b761 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl @@ -98,13 +98,13 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let byte_in_block = (global_k_start % BLOCK_SIZE) / 8u; let src0_idx = batch_offset + global_m * params.stride_01 + block_k; let block_byte_base = src0_idx * BLOCK_SIZE_BYTES; - let d = load_f16_at_src0(block_byte_base); + let d = load_f16_as_f32_at_src0(block_byte_base); let q_byte = load_u32_at_src0(block_byte_base + 2u + byte_in_block) & 0xFFu; for (var bit = 0u; bit < NQ; bit++) { let global_k = global_k_start + bit; if (global_k < params.k) { - shmem[i + bit] = select(-d, d, ((q_byte >> bit) & 1u) != 0u); + shmem[i + bit] = f16(select(-d, d, ((q_byte >> bit) & 1u) != 0u)); } } } @@ -123,11 +123,11 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 if (global_m < params.m && global_k < params.k) { let block_idx = batch_offset + global_m * params.stride_01 + global_k / BLOCK_SIZE; let block_byte_base = block_idx * BLOCK_SIZE_BYTES; - let d = load_f16_at_src0(block_byte_base); + let d = load_f16_as_f32_at_src0(block_byte_base); let q_packed = load_u32_at_src0(block_byte_base + 2u + (global_k % BLOCK_SIZE) / 4u); for (var k = 0u; k < NQ; k++) { let q = (q_packed >> (2u * k)) & 3u; - shmem[i + k] = (f16(q) - 1.0) * d; + shmem[i + k] = f16((f32(q) - 1.0) * d); } } } @@ -163,7 +163,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 #if defined(INIT_SRC0_SHMEM_Q4_0) let block_byte_base = src0_idx * 18u; // BLOCK_SIZE_BYTES = 18u; - let d = load_f16_at_src0(block_byte_base); + let d = load_f16_as_f32_at_src0(block_byte_base); // load NQ(16) weights for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) { @@ -176,8 +176,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 #if defined(INIT_SRC0_SHMEM_Q4_1) let block_byte_base = src0_idx * 20u; // BLOCK_SIZE_BYTES = 20u; let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base)); - let d = f16(dm[0]); - let m = f16(dm[1]); + let d = dm[0]; + let m = dm[1]; // load NQ(16) weights for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) { @@ -186,10 +186,10 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) { let q_byte = get_byte(q_packed, k); - let q_lo = f16(q_byte & 0xF) * d + m; - let q_hi = f16((q_byte >> 4) & 0xF) * d + m; - shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo; - shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi; + let q_lo = f32(q_byte & 0xF) * d + m; + let q_hi = f32((q_byte >> 4) & 0xF) * d + m; + shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo); + shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi); } } #endif // INIT_SRC0_SHMEM_Q4_1 @@ -197,7 +197,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 #if defined(INIT_SRC0_SHMEM_Q5_0) let block_byte_base = src0_idx * 22u; // BLOCK_SIZE_BYTES = 22u; - let d = load_f16_at_src0(block_byte_base); + let d = load_f16_as_f32_at_src0(block_byte_base); let qh_packed = load_u32_at_src0(block_byte_base + 2u); // load NQ(16) weights @@ -210,11 +210,11 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let byte_idx = block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP + k; let qh_hi = (qh_packed >> (byte_idx + 12u)) & 0x10; - let q_hi = (f16(((q_byte >> 4) & 0xF) | qh_hi) - 16.0) * d; + let q_hi = (f32(((q_byte >> 4) & 0xF) | qh_hi) - 16.0) * d; let qh_lo = ((qh_packed >> byte_idx) << 4) & 0x10; - let q_lo = (f16((q_byte & 0xF) | qh_lo) - 16.0) * d; - shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo; - shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi; + let q_lo = (f32((q_byte & 0xF) | qh_lo) - 16.0) * d; + shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo); + shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi); } } #endif // INIT_SRC0_SHMEM_Q5_0 @@ -223,8 +223,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let block_byte_base = src0_idx * 24u; // BLOCK_SIZE_BYTES = 24u; let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base)); - let d = f16(dm[0]); - let m = f16(dm[1]); + let d = dm[0]; + let m = dm[1]; let qh_packed = load_u32_at_src0_aligned(block_byte_base + 4u); // load NQ(16) weights @@ -237,18 +237,18 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let byte_idx = block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP + k; let qh_hi = (qh_packed >> (byte_idx + 12u)) & 0x10; - let q_hi = f16(((q_byte >> 4) & 0xF) | qh_hi) * d + m; + let q_hi = f32(((q_byte >> 4) & 0xF) | qh_hi) * d + m; let qh_lo = ((qh_packed >> byte_idx) << 4) & 0x10; - let q_lo = f16((q_byte & 0xF) | qh_lo) * d + m; - shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo; - shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi; + let q_lo = f32((q_byte & 0xF) | qh_lo) * d + m; + shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo); + shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi); } } #endif // INIT_SRC0_SHMEM_Q5_1 #if defined(INIT_SRC0_SHMEM_Q8_0) let block_byte_base = src0_idx * 34u; // BLOCK_SIZE_BYTES = 34u; - let d = load_f16_at_src0(block_byte_base); + let d = load_f16_as_f32_at_src0(block_byte_base); // load NQ(16) weights for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) { @@ -381,8 +381,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let dm_byte_base = block_byte_base + 80u; let d_packed = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base)); - let d = f16(d_packed[0]); - let dmin = f16(d_packed[1]); + let d = d_packed[0]; + let dmin = d_packed[1]; let chunk = k_in_block / 128u; let pos_in_chunk = k_in_block % 32u; @@ -391,19 +391,19 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 // whole 2 bits (4 elems) let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk); - let qs_vec4 = vec4( - f16((qs_word >> (2u * shift_phase + 0u)) & 0x3u), - f16((qs_word >> (2u * shift_phase + 8u)) & 0x3u), - f16((qs_word >> (2u * shift_phase + 16u)) & 0x3u), - f16((qs_word >> (2u * shift_phase + 24u)) & 0x3u), + let qs_vec4 = vec4( + f32((qs_word >> (2u * shift_phase + 0u)) & 0x3u), + f32((qs_word >> (2u * shift_phase + 8u)) & 0x3u), + f32((qs_word >> (2u * shift_phase + 16u)) & 0x3u), + f32((qs_word >> (2u * shift_phase + 24u)) & 0x3u), ); let scale = load_byte_at_src0_aligned(scales_byte_base + sub_block); - let dl = d * f16(scale & 0xFu); - let ml = dmin * f16(scale >> 4u); + let dl = d * f32(scale & 0xFu); + let ml = dmin * f32(scale >> 4u); - store_shmem_kquants(qs_vec4 * dl - ml, elem_idx); + store_shmem_kquants(vec4(qs_vec4 * dl - ml), elem_idx); #endif // INIT_SRC0_SHMEM_Q2_K #if defined(INIT_SRC0_SHMEM_Q3_K) @@ -412,7 +412,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let qs_byte_base = block_byte_base + 32u; let scales_byte_base = block_byte_base + 96u; - let d_all = load_f16_at_src0(block_byte_base + 108u); + let d_all = load_f16_as_f32_at_src0(block_byte_base + 108u); let chunk = k_in_block / 128u; let pos_in_chunk = k_in_block % 32u; @@ -424,29 +424,29 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 // low 2 bits (4 elems) let q_lo2_word = load_u32_at_src0(qs_byte_base + 32u * chunk + 1u * hmask_block); - let q_lo2_vec4 = vec4( - f16((q_lo2_word >> (2u * shift_phase + 0u)) & 3u), - f16((q_lo2_word >> (2u * shift_phase + 8u)) & 3u), - f16((q_lo2_word >> (2u * shift_phase + 16u)) & 3u), - f16((q_lo2_word >> (2u * shift_phase + 24u)) & 3u) + let q_lo2_vec4 = vec4( + f32((q_lo2_word >> (2u * shift_phase + 0u)) & 3u), + f32((q_lo2_word >> (2u * shift_phase + 8u)) & 3u), + f32((q_lo2_word >> (2u * shift_phase + 16u)) & 3u), + f32((q_lo2_word >> (2u * shift_phase + 24u)) & 3u) ); // high 1 bit (4 elems) let q_hi1_word = load_u32_at_src0(hmask_byte_base + pos_in_chunk); - let q_hi1_vec4 = vec4( - f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 0u)) & 1u) == 1u)), - f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 8u)) & 1u) == 1u)), - f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 16u)) & 1u) == 1u)), - f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 24u)) & 1u) == 1u)) + let q_hi1_vec4 = vec4( + select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 0u)) & 1u) == 1u), + select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 8u)) & 1u) == 1u), + select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 16u)) & 1u) == 1u), + select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 24u)) & 1u) == 1u) ); let q_vec4 = q_lo2_vec4 - q_hi1_vec4; let scale_low4 = (load_byte_at_src0_aligned(scales_byte_base + (sub_block % 8u)) >> (4u * (sub_block / 8u))) & 0xFu; let scale_hi2 = (load_byte_at_src0_aligned(scales_byte_base + 8u + (sub_block % 4u)) >> (2u * (sub_block / 4u))) & 3u; - let dl = d_all * (f16((scale_hi2 << 4u) | scale_low4) - 32.0); + let dl = d_all * (f32((scale_hi2 << 4u) | scale_low4) - 32.0); - store_shmem_kquants(dl * q_vec4, elem_idx); + store_shmem_kquants(vec4(dl * q_vec4), elem_idx); #endif // INIT_SRC0_SHMEM_Q3_K #if defined(INIT_SRC0_SHMEM_Q4_K) @@ -456,8 +456,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let qs_byte_base = block_byte_base + 16u; let dm = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base)); - let d = f16(dm[0]); - let dmin = f16(dm[1]); + let d = dm[0]; + let dmin = dm[1]; let chunk = k_in_block / 64u; let pos_in_chunk = (k_in_block % 64u) % 32u; @@ -466,11 +466,11 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 // whole 4 bits (4 elems) let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk); - let qs_vec4 = vec4( - f16((qs_word >> (4u * shift_phase + 0u)) & 0xFu), - f16((qs_word >> (4u * shift_phase + 8u)) & 0xFu), - f16((qs_word >> (4u * shift_phase + 16u)) & 0xFu), - f16((qs_word >> (4u * shift_phase + 24u)) & 0xFu) + let qs_vec4 = vec4( + f32((qs_word >> (4u * shift_phase + 0u)) & 0xFu), + f32((qs_word >> (4u * shift_phase + 8u)) & 0xFu), + f32((qs_word >> (4u * shift_phase + 16u)) & 0xFu), + f32((qs_word >> (4u * shift_phase + 24u)) & 0xFu) ); var sc: u32; @@ -489,10 +489,10 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 mn = (sc_min_lo >> 4u) | ((min_hi >> 6u) << 4u); } - let dl = d * f16(sc); - let ml = dmin * f16(mn); + let dl = d * f32(sc); + let ml = dmin * f32(mn); - store_shmem_kquants(dl * qs_vec4 - vec4(ml, ml, ml, ml), elem_idx); + store_shmem_kquants(vec4(dl * qs_vec4 - vec4(ml, ml, ml, ml)), elem_idx); #endif // INIT_SRC0_SHMEM_Q4_K #if defined(INIT_SRC0_SHMEM_Q5_K) @@ -503,8 +503,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let qs_byte_base = block_byte_base + 48u; let dm = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base)); - let d = f16(dm[0]); - let dmin = f16(dm[1]); + let d = dm[0]; + let dmin = dm[1]; let chunk = k_in_block / 64u; let pos_in_chunk = (k_in_block % 64u) % 32u; @@ -516,20 +516,20 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 // low 4 bits (4 elems) let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk); - let qs_lo4_vec4 = vec4( - f16((qs_word >> (4u * shift_phase + 0u)) & 0xFu), - f16((qs_word >> (4u * shift_phase + 8u)) & 0xFu), - f16((qs_word >> (4u * shift_phase + 16u)) & 0xFu), - f16((qs_word >> (4u * shift_phase + 24u)) & 0xFu) + let qs_lo4_vec4 = vec4( + f32((qs_word >> (4u * shift_phase + 0u)) & 0xFu), + f32((qs_word >> (4u * shift_phase + 8u)) & 0xFu), + f32((qs_word >> (4u * shift_phase + 16u)) & 0xFu), + f32((qs_word >> (4u * shift_phase + 24u)) & 0xFu) ); // high 1 bit (4 elems) let qh_word = load_u32_at_src0_aligned(qh_byte_base + qh_block); - let qh_vec4 = vec4( - f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 0u)) & 1u) == 1u)), - f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 8u)) & 1u) == 1u)), - f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 16u)) & 1u) == 1u)), - f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 24u)) & 1u) == 1u)) + let qh_vec4 = vec4( + select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 0u)) & 1u) == 1u), + select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 8u)) & 1u) == 1u), + select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 16u)) & 1u) == 1u), + select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 24u)) & 1u) == 1u) ); var sc: u32; @@ -548,10 +548,10 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 mn = (sc_min_lo >> 4u) | ((min_hi >> 6u) << 4u); } - let dl = d * f16(sc); - let ml = dmin * f16(mn); + let dl = d * f32(sc); + let ml = dmin * f32(mn); - store_shmem_kquants((qh_vec4 + qs_lo4_vec4) * dl - vec4(ml, ml, ml, ml), elem_idx); + store_shmem_kquants(vec4((qh_vec4 + qs_lo4_vec4) * dl - vec4(ml, ml, ml, ml)), elem_idx); #endif // INIT_SRC0_SHMEM_Q5_K #if defined(INIT_SRC0_SHMEM_Q6_K) @@ -561,7 +561,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let scales_byte_base = block_byte_base + 192u; let d_byte_base = block_byte_base + 208u; - let d = load_f16_at_src0(d_byte_base); + let d = load_f16_as_f32_at_src0(d_byte_base); let chunk = k_in_block / 128u; let ql_pos_in_chunk = (k_in_block % 128u) % 64u; @@ -588,13 +588,13 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 ((qh_word >> (2u * qh_shift_phase + 24u)) & 0x3u) << 4u, ); - let q_vec4 = vec4(qh_hi2_vec4 | ql_lo4_vec4) - vec4(32.0, 32.0, 32.0, 32.0); + let q_vec4 = vec4(qh_hi2_vec4 | ql_lo4_vec4) - vec4(32.0, 32.0, 32.0, 32.0); let scale_byte = scales_byte_base + 1u * sub_block; let scale_word = load_u32_at_src0_aligned(scale_byte); let scale = get_byte_i32(scale_word, scale_byte & 3u); - store_shmem_kquants(d * q_vec4 * f16(scale), elem_idx); + store_shmem_kquants(vec4(d * q_vec4 * f32(scale)), elem_idx); #endif // INIT_SRC0_SHMEM_Q6_K } } @@ -626,17 +626,17 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let d_byte_base = block_byte_base + 0u; let qs_byte_base = block_byte_base + 2u; - let d = load_f16_at_src0(d_byte_base); + let d = load_f16_as_f32_at_src0(d_byte_base); let id_qtr = (k_in_block % 16u) / 4u; let shift_phase = k_in_block / 16u; let qs_u32 = load_u32_at_src0(qs_byte_base + 4u * id_qtr); - shmem[elem_idx + 0u] = d * f16(kvalues_iq4nl[(qs_u32 >> ( 0u + 4u * shift_phase)) & 0xFu]); - shmem[elem_idx + 1u] = d * f16(kvalues_iq4nl[(qs_u32 >> ( 8u + 4u * shift_phase)) & 0xFu]); - shmem[elem_idx + 2u] = d * f16(kvalues_iq4nl[(qs_u32 >> (16u + 4u * shift_phase)) & 0xFu]); - shmem[elem_idx + 3u] = d * f16(kvalues_iq4nl[(qs_u32 >> (24u + 4u * shift_phase)) & 0xFu]); + shmem[elem_idx + 0u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> ( 0u + 4u * shift_phase)) & 0xFu])); + shmem[elem_idx + 1u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> ( 8u + 4u * shift_phase)) & 0xFu])); + shmem[elem_idx + 2u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> (16u + 4u * shift_phase)) & 0xFu])); + shmem[elem_idx + 3u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> (24u + 4u * shift_phase)) & 0xFu])); } } #endif // INIT_SRC0_SHMEM_IQ4_NL @@ -670,12 +670,12 @@ fn create_iq_gw4(dl: f32, gw: u32, shift_base: u32, delta: f32) -> vec4 { #endif #if defined(INIT_SRC0_SHMEM_IQ4_XS) -fn create_iq_gw4(dl: f16, qs_u32: u32, shift_phase: u32) -> vec4 { +fn create_iq_gw4(dl: f32, qs_u32: u32, shift_phase: u32) -> vec4 { return vec4( - dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 0u)) & 0xFu]), - dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 8u)) & 0xFu]), - dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 16u)) & 0xFu]), - dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 24u)) & 0xFu]), + f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 0u)) & 0xFu])), + f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 8u)) & 0xFu])), + f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 16u)) & 0xFu])), + f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 24u)) & 0xFu])), ); } #endif @@ -775,7 +775,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let qs_byte_base = block_byte_base + 8u; let d_scales_h = load_u32_at_src0_aligned(d_byte_base); - let d = bitcast>(d_scales_h).x; + let d = unpack2x16float(d_scales_h).x; let scales_h = d_scales_h >> 16u; let sub_block = k_in_block / 32u; @@ -784,7 +784,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3 let scales_l_u32 = load_u32_at_src0_aligned(scales_l_byte_base); let ls_lo = (get_byte(scales_l_u32, sub_block / 2u) >> (4u * (sub_block % 2u))) & 0xFu; let ls_hi = ((scales_h >> (2u * sub_block)) & 3u) << 4u; - let dl = d * f16(i32(ls_lo | ls_hi) - 32); + let dl = d * f32(i32(ls_lo | ls_hi) - 32); let qs_0_3_u32 = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block + 0u); let qs_4_7_u32 = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block + 4u); diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl index d1da46084..008fa1f27 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl @@ -1,21 +1,19 @@ #ifdef U32_DEQUANT_HELPERS -fn dequant_q4_0_packed_to_shmem(q_packed: u32, d: f16, dst_idx: u32) { - let scale = QUANT_OUT_TYPE(d); +fn dequant_q4_0_packed_to_shmem(q_packed: u32, d: f32, dst_idx: u32) { for (var k = 0u; k < 4u; k++) { let q_byte = get_byte(q_packed, k); - let q_hi = (QUANT_OUT_TYPE((q_byte >> 4) & 0xFu) - QUANT_OUT_TYPE(8.0)) * scale; - let q_lo = (QUANT_OUT_TYPE(q_byte & 0xFu) - QUANT_OUT_TYPE(8.0)) * scale; - QUANT_SHMEM[dst_idx + k] = q_lo; - QUANT_SHMEM[dst_idx + k + 16u] = q_hi; + let q_hi = (f32((q_byte >> 4) & 0xFu) - 8.0) * d; + let q_lo = (f32(q_byte & 0xFu) - 8.0) * d; + QUANT_SHMEM[dst_idx + k] = QUANT_OUT_TYPE(q_lo); + QUANT_SHMEM[dst_idx + k + 16u] = QUANT_OUT_TYPE(q_hi); } } -fn dequant_q8_0_packed_to_shmem(q_packed: u32, d: f16, dst_idx: u32) { - let scale = QUANT_OUT_TYPE(d); +fn dequant_q8_0_packed_to_shmem(q_packed: u32, d: f32, dst_idx: u32) { for (var k = 0u; k < 4u; k++) { let q_byte = get_byte_i32(q_packed, k); - let q_val = QUANT_OUT_TYPE(q_byte) * scale; - QUANT_SHMEM[dst_idx + k] = q_val; + let q_val = f32(q_byte) * d; + QUANT_SHMEM[dst_idx + k] = QUANT_OUT_TYPE(q_val); } } #endif