| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -11,31 +11,103 @@
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				namespace Shader::Backend::GLSL {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				namespace {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				static constexpr std::string_view cas_loop{R"(uint {};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				static constexpr std::string_view cas_loop{R"({};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				for (;;){{
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    uint old_value={};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    {}=atomicCompSwap({},old_value,{}({},{}));
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    if ({}==old_value){{break;}}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}})"};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void CasFunction(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                 const IR::Value& offset, std::string_view value, std::string_view function) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret{ctx.reg_alloc.Define(inst)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void SharedCasFunction(EmitContext& ctx, IR::Inst& inst, std::string_view offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                       std::string_view value, std::string_view function) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret{ctx.reg_alloc.Define(inst, Type::U32)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string smem{fmt::format("smem[{}/4]", offset)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add(cas_loop.data(), ret, smem, ret, smem, function, smem, value, ret);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void SsboCasFunction(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                     const IR::Value& offset, std::string_view value, std::string_view function) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret{ctx.reg_alloc.Define(inst, Type::U32)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string ssbo{fmt::format("ssbo{}[{}]", binding.U32(), offset.U32())};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add(cas_loop.data(), ret, ssbo, ret, ssbo, function, ssbo, value, ret);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void CasFunctionF32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                    const IR::Value& offset, std::string_view value, std::string_view function) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void SsboCasFunctionF32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                        const IR::Value& offset, std::string_view value,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                        std::string_view function) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string ssbo{fmt::format("ssbo{}[{}]", binding.U32(), offset.U32())};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string u32_value{fmt::format("floatBitsToUint({})", value)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret{ctx.reg_alloc.Define(inst)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret_32{ret + "_u32"};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add(cas_loop.data(), ret_32, ssbo, ret_32, ssbo, function, ssbo, value, ret_32);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add("float {}=uintBitsToFloat({});", ret, ret_32);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret{ctx.reg_alloc.Define(inst, Type::U32)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add(cas_loop.data(), ret, ssbo, ret, ssbo, function, ssbo, value, ret);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddF32("{}=uintBitsToFloat({});", inst, ret);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				} // namespace
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicIAdd32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicAdd(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				} // namespace
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicSMin32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string u32_value{fmt::format("uint({})", value)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SharedCasFunction(ctx, inst, pointer_offset, u32_value, "CasMinS32");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicUMin32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicMin(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicSMax32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string u32_value{fmt::format("uint({})", value)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SharedCasFunction(ctx, inst, pointer_offset, u32_value, "CasMaxS32");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicUMax32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicMax(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicInc32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                           std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SharedCasFunction(ctx, inst, pointer_offset, value, "CasIncrement");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicDec32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                           std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SharedCasFunction(ctx, inst, pointer_offset, value, "CasDecrement");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicAnd32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                           std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicAnd(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicOr32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                          std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicOr(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicXor32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                           std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicXor(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicExchange32(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                                std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU32("{}=atomicExchange(smem[{}/4],{});", inst, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitSharedAtomicExchange64(EmitContext& ctx, IR::Inst& inst, std::string_view pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                                std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    // LOG_WARNING("Int64 Atomics not supported, fallback to non-atomic");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const auto ret{ctx.reg_alloc.Define(inst, Type::U64)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add("{}=packUint2x32(uvec2(smem[{}/4],smem[({}+4)/4]));", ret, pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				            pointer_offset);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add("smem[{}/4]=unpackUint2x32({}).x;smem[({}+4)/4]=unpackUint2x32({}).y;", pointer_offset,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				            value, pointer_offset, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicIAdd32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -45,7 +117,7 @@ void EmitStorageAtomicIAdd32(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicSMin32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string u32_value{fmt::format("uint({})", value)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, u32_value, "CasMinS32");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, u32_value, "CasMinS32");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicUMin32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -56,7 +128,7 @@ void EmitStorageAtomicUMin32(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicSMax32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    const std::string u32_value{fmt::format("uint({})", value)};
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, u32_value, "CasMaxS32");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, u32_value, "CasMaxS32");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicUMax32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -66,12 +138,12 @@ void EmitStorageAtomicUMax32(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicInc32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasIncrement");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasIncrement");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicDec32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasDecrement");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasDecrement");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicAnd32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -97,7 +169,7 @@ void EmitStorageAtomicExchange32(EmitContext& ctx, IR::Inst& inst, const IR::Val
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicIAdd64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    // LOG_WARNING(..., "Op falling to non-atomic");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=packUint2x32(uvec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               binding.U32(), offset.U32() + 1);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add("ssbo{}[{}]+=unpackUint2x32({}).x;ssbo{}[{}]+=unpackUint2x32({}).y;", binding.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				            offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -106,7 +178,7 @@ void EmitStorageAtomicIAdd64(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicSMin64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    // LOG_WARNING(..., "Op falling to non-atomic");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddS64("{}=int64_t(ivec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddS64("{}=packInt2x32(ivec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               binding.U32(), offset.U32() + 1);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add("for(int i=0;i<2;++i){{ "
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				            "ssbo{}[{}+i]=uint(min(int(ssbo{}[{}+i]),unpackInt2x32(int64_t({}))[i]));}}",
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -116,7 +188,7 @@ void EmitStorageAtomicSMin64(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicUMin64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    // LOG_WARNING(..., "Op falling to non-atomic");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=packUint2x32(uvec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               binding.U32(), offset.U32() + 1);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "for(int i=0;i<2;++i){{ ssbo{}[{}+i]=min(ssbo{}[{}+i],unpackUint2x32(uint64_t({}))[i]);}}",
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -126,7 +198,7 @@ void EmitStorageAtomicUMin64(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicSMax64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    // LOG_WARNING(..., "Op falling to non-atomic");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddS64("{}=int64_t(ivec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddS64("{}=packInt2x32(ivec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               binding.U32(), offset.U32() + 1);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add("for(int i=0;i<2;++i){{ "
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				            "ssbo{}[{}+i]=uint(max(int(ssbo{}[{}+i]),unpackInt2x32(int64_t({}))[i]));}}",
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -136,7 +208,7 @@ void EmitStorageAtomicSMax64(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicUMax64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    // LOG_WARNING(..., "Op falling to non-atomic");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=packUint2x32(uvec2(ssbo{}[{}],ssbo{}[{}]));", inst, binding.U32(), offset.U32(),
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               binding.U32(), offset.U32() + 1);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.Add(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "for(int i=0;i<2;++i){{ssbo{}[{}+i]=max(ssbo{}[{}+i],unpackUint2x32(uint64_t({}))[i]);}}",
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				@@ -145,65 +217,69 @@ void EmitStorageAtomicUMax64(EmitContext& ctx, IR::Inst& inst, const IR::Value&
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicAnd64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(atomicAnd(ssbo{}[{}],unpackUint2x32({}).x),atomicAnd(ssbo{}[{}],"
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               "unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "{}=packUint2x32(uvec2(atomicAnd(ssbo{}[{}],unpackUint2x32({}).x),atomicAnd(ssbo{}[{}],"
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicOr64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                           const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(atomicOr(ssbo{}[{}],unpackUint2x32({}).x),atomicOr(ssbo{}[{}],"
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               "unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "{}=packUint2x32(uvec2(atomicOr(ssbo{}[{}],unpackUint2x32({}).x),atomicOr(ssbo{}[{}],"
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicXor64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                            const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(atomicXor(ssbo{}[{}],unpackUint2x32({}).x),atomicXor(ssbo{}[{}],"
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               "unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "{}=packUint2x32(uvec2(atomicXor(ssbo{}[{}],unpackUint2x32({}).x),atomicXor(ssbo{}[{}],"
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicExchange64(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                                 const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64("{}=uint64_t(uvec2(atomicExchange(ssbo{}[{}],unpackUint2x32({}).x),atomicExchange("
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               "ssbo{}[{}],unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				               inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    ctx.AddU64(
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "{}=packUint2x32(uvec2(atomicExchange(ssbo{}[{}],unpackUint2x32({}).x),atomicExchange("
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        "ssbo{}[{}],unpackUint2x32({}).y)));",
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				        inst, binding.U32(), offset.U32(), value, binding.U32(), offset.U32() + 1, value);
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicAddF32(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                             const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunctionF32(ctx, inst, binding, offset, value, "CasFloatAdd");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunctionF32(ctx, inst, binding, offset, value, "CasFloatAdd");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicAddF16x2(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                               const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasFloatAdd16x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasFloatAdd16x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicAddF32x2(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                               const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasFloatAdd32x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasFloatAdd32x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicMinF16x2(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                               const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasFloatMin16x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasFloatMin16x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicMinF32x2(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                               const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasFloatMin32x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasFloatMin32x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicMaxF16x2(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                               const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasFloatMax16x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasFloatMax16x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitStorageAtomicMaxF32x2(EmitContext& ctx, IR::Inst& inst, const IR::Value& binding,
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				                               const IR::Value& offset, std::string_view value) {
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    CasFunction(ctx, inst, binding, offset, value, "CasFloatMax32x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				    SsboCasFunction(ctx, inst, binding, offset, value, "CasFloatMax32x2");
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				}
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				
 | 
			
		
		
	
		
			
				 | 
				 | 
			
			 | 
			 | 
			
				void EmitGlobalAtomicIAdd32(EmitContext&) {
 | 
			
		
		
	
	
		
			
				
					
					| 
						
					 | 
				
			
			 | 
			 | 
			
				 
 |