Merge pull request #479 from lioncash/rsqrts

A64: Handle half-precision variants of FRSQRTS

src/backend/x64/emit_x64_floating_point.cpp

@@ -946,6 +946,7 @@ template<size_t fsize>
 static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* inst) {
     using FPT = mp::unsigned_integer_of_size<fsize>;
 
+    if constexpr (fsize != 16) {
         if (code.DoesCpuSupport(Xbyak::util::Cpu::tFMA) && code.DoesCpuSupport(Xbyak::util::Cpu::tAVX)) {
             auto args = ctx.reg_alloc.GetArgumentInfo(inst);
 
@@ -993,6 +994,7 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
             ctx.reg_alloc.DefineValue(inst, result);
             return;
         }
+    }
 
     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
     ctx.reg_alloc.HostCall(inst, args[0], args[1]);
@@ -1001,6 +1003,10 @@ static void EmitFPRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst*
     code.CallFunction(&FP::FPRSqrtStepFused<FPT>);
 }
 
+void EmitX64::EmitFPRSqrtStepFused16(EmitContext& ctx, IR::Inst* inst) {
+    EmitFPRSqrtStepFused<16>(code, ctx, inst);
+}
+
 void EmitX64::EmitFPRSqrtStepFused32(EmitContext& ctx, IR::Inst* inst) {
     EmitFPRSqrtStepFused<32>(code, ctx, inst);
 }

src/backend/x64/emit_x64_vector_floating_point.cpp

@@ -1273,6 +1273,7 @@ static void EmitRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* in
         }
     };
 
+    if constexpr (fsize != 16) {
         if (code.DoesCpuSupport(Xbyak::util::Cpu::tFMA) && code.DoesCpuSupport(Xbyak::util::Cpu::tAVX)) {
             auto args = ctx.reg_alloc.GetArgumentInfo(inst);
 
@@ -1314,10 +1315,15 @@ static void EmitRSqrtStepFused(BlockOfCode& code, EmitContext& ctx, IR::Inst* in
             ctx.reg_alloc.DefineValue(inst, result);
             return;
         }
+    }
 
     EmitThreeOpFallback(code, ctx, inst, fallback_fn);
 }
 
+void EmitX64::EmitFPVectorRSqrtStepFused16(EmitContext& ctx, IR::Inst* inst) {
+    EmitRSqrtStepFused<16>(code, ctx, inst);
+}
+
 void EmitX64::EmitFPVectorRSqrtStepFused32(EmitContext& ctx, IR::Inst* inst) {
     EmitRSqrtStepFused<32>(code, ctx, inst);
 }

src/common/fp/op/FPRSqrtStepFused.cpp

@@ -19,8 +19,8 @@ template<typename FPT>
 FPT FPRSqrtStepFused(FPT op1, FPT op2, FPCR fpcr, FPSR& fpsr) {
     op1 = FPNeg(op1);
 
-    const auto [type1, sign1, value1] = FPUnpack<FPT>(op1, fpcr, fpsr);
+    const auto [type1, sign1, value1] = FPUnpack(op1, fpcr, fpsr);
-    const auto [type2, sign2, value2] = FPUnpack<FPT>(op2, fpcr, fpsr);
+    const auto [type2, sign2, value2] = FPUnpack(op2, fpcr, fpsr);
 
     if (const auto maybe_nan = FPProcessNaNs(type1, type2, op1, op2, fpcr, fpsr)) {
         return *maybe_nan;
@@ -37,7 +37,7 @@ FPT FPRSqrtStepFused(FPT op1, FPT op2, FPCR fpcr, FPSR& fpsr) {
     }
 
     if (inf1 || inf2) {
-        return FPInfo<FPT>::Infinity(sign1 != sign2);
+        return FPT(FPInfo<FPT>::Infinity(sign1 != sign2));
     }
 
     // result_value = (3.0 + (value1 * value2)) / 2.0
@@ -45,11 +45,12 @@ FPT FPRSqrtStepFused(FPT op1, FPT op2, FPCR fpcr, FPSR& fpsr) {
     result_value.exponent--;
 
     if (result_value.mantissa == 0) {
-        return FPInfo<FPT>::Zero(fpcr.RMode() == RoundingMode::TowardsMinusInfinity);
+        return FPT(FPInfo<FPT>::Zero(fpcr.RMode() == RoundingMode::TowardsMinusInfinity));
     }
     return FPRound<FPT>(result_value, fpcr, fpsr);
 }
 
+template u16 FPRSqrtStepFused<u16>(u16 op1, u16 op2, FPCR fpcr, FPSR& fpsr);
 template u32 FPRSqrtStepFused<u32>(u32 op1, u32 op2, FPCR fpcr, FPSR& fpsr);
 template u64 FPRSqrtStepFused<u64>(u64 op1, u64 op2, FPCR fpcr, FPSR& fpsr);
 

src/frontend/A64/decoder/a64.inc

@@ -386,7 +386,7 @@ INST(FMULX_vec_2,            "FMULX",                                     "01011
 INST(FCMEQ_reg_2,            "FCMEQ (register)",                          "010111100z1mmmmm111001nnnnnddddd")
 INST(FRECPS_1,               "FRECPS",                                    "01011110010mmmmm001111nnnnnddddd")
 INST(FRECPS_2,               "FRECPS",                                    "010111100z1mmmmm111111nnnnnddddd")
-//INST(FRSQRTS_1,              "FRSQRTS",                                   "01011110110mmmmm001111nnnnnddddd")
+INST(FRSQRTS_1,              "FRSQRTS",                                   "01011110110mmmmm001111nnnnnddddd")
 INST(FRSQRTS_2,              "FRSQRTS",                                   "010111101z1mmmmm111111nnnnnddddd")
 //INST(FCMGE_reg_1,            "FCMGE (register)",                          "01111110010mmmmm001001nnnnnddddd")
 INST(FCMGE_reg_2,            "FCMGE (register)",                          "011111100z1mmmmm111001nnnnnddddd")
@@ -576,7 +576,7 @@ INST(INS_elt,                "INS (element)",                             "01101
 //INST(FMULX_vec_3,            "FMULX",                                     "0Q001110010mmmmm000111nnnnnddddd")
 //INST(FCMEQ_reg_3,            "FCMEQ (register)",                          "0Q001110010mmmmm001001nnnnnddddd")
 INST(FRECPS_3,               "FRECPS",                                    "0Q001110010mmmmm001111nnnnnddddd")
-//INST(FRSQRTS_3,              "FRSQRTS",                                   "0Q001110110mmmmm001111nnnnnddddd")
+INST(FRSQRTS_3,              "FRSQRTS",                                   "0Q001110110mmmmm001111nnnnnddddd")
 //INST(FCMGE_reg_3,            "FCMGE (register)",                          "0Q101110010mmmmm001001nnnnnddddd")
 //INST(FACGE_3,                "FACGE",                                     "0Q101110010mmmmm001011nnnnnddddd")
 //INST(FABD_3,                 "FABD",                                      "0Q101110110mmmmm000101nnnnnddddd")

src/frontend/A64/translate/impl/simd_scalar_three_same.cpp

@@ -316,6 +316,17 @@ bool TranslatorVisitor::FRECPS_2(bool sz, Vec Vm, Vec Vn, Vec Vd) {
     return true;
 }
 
+bool TranslatorVisitor::FRSQRTS_1(Vec Vm, Vec Vn, Vec Vd) {
+    const size_t esize = 16;
+
+    const IR::U16 operand1 = V_scalar(esize, Vn);
+    const IR::U16 operand2 = V_scalar(esize, Vm);
+    const IR::U16 result = ir.FPRSqrtStepFused(operand1, operand2);
+
+    V_scalar(esize, Vd, result);
+    return true;
+}
+
 bool TranslatorVisitor::FRSQRTS_2(bool sz, Vec Vm, Vec Vn, Vec Vd) {
     const size_t esize = sz ? 64 : 32;
 

src/frontend/A64/translate/impl/simd_three_same.cpp

@@ -965,6 +965,18 @@ bool TranslatorVisitor::FRECPS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) {
     return true;
 }
 
+bool TranslatorVisitor::FRSQRTS_3(bool Q, Vec Vm, Vec Vn, Vec Vd) {
+    const size_t esize = 16;
+    const size_t datasize = Q ? 128 : 64;
+
+    const IR::U128 operand1 = V(datasize, Vn);
+    const IR::U128 operand2 = V(datasize, Vm);
+    const IR::U128 result = ir.FPVectorRSqrtStepFused(esize, operand1, operand2);
+
+    V(datasize, Vd, result);
+    return true;
+}
+
 bool TranslatorVisitor::FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) {
     if (sz && !Q) {
         return ReservedValue();

src/frontend/ir/ir_emitter.cpp

@@ -1997,11 +1997,20 @@ U16U32U64 IREmitter::FPRSqrtEstimate(const U16U32U64& a) {
     }
 }
 
-U32U64 IREmitter::FPRSqrtStepFused(const U32U64& a, const U32U64& b) {
+U16U32U64 IREmitter::FPRSqrtStepFused(const U16U32U64& a, const U16U32U64& b) {
-    if (a.GetType() == Type::U32) {
+    ASSERT(a.GetType() == b.GetType());
+
+    switch (a.GetType()) {
+    case Type::U16:
+        return Inst<U16>(Opcode::FPRSqrtStepFused16, a, b);
+    case Type::U32:
         return Inst<U32>(Opcode::FPRSqrtStepFused32, a, b);
-    }
+    case Type::U64:
         return Inst<U64>(Opcode::FPRSqrtStepFused64, a, b);
+    default:
+        UNREACHABLE();
+        return U16U32U64{};
+    }
 }
 
 U32U64 IREmitter::FPSqrt(const U32U64& a) {
@@ -2335,6 +2344,8 @@ U128 IREmitter::FPVectorRSqrtEstimate(size_t esize, const U128& a) {
 
 U128 IREmitter::FPVectorRSqrtStepFused(size_t esize, const U128& a, const U128& b) {
     switch (esize) {
+    case 16:
+        return Inst<U128>(Opcode::FPVectorRSqrtStepFused16, a, b);
     case 32:
         return Inst<U128>(Opcode::FPVectorRSqrtStepFused32, a, b);
     case 64:

src/frontend/ir/ir_emitter.h

@@ -310,7 +310,7 @@ public:
     U16U32U64 FPRecipStepFused(const U16U32U64& a, const U16U32U64& b);
     U16U32U64 FPRoundInt(const U16U32U64& a, FP::RoundingMode rounding, bool exact);
     U16U32U64 FPRSqrtEstimate(const U16U32U64& a);
-    U32U64 FPRSqrtStepFused(const U32U64& a, const U32U64& b);
+    U16U32U64 FPRSqrtStepFused(const U16U32U64& a, const U16U32U64& b);
     U32U64 FPSqrt(const U32U64& a);
     U32U64 FPSub(const U32U64& a, const U32U64& b, bool fpcr_controlled);
     U16 FPDoubleToHalf(const U64& a, FP::RoundingMode rounding);

src/frontend/ir/microinstruction.cpp

@@ -287,6 +287,7 @@ bool Inst::ReadsFromAndWritesToFPSRCumulativeExceptionBits() const {
     case Opcode::FPRSqrtEstimate16:
     case Opcode::FPRSqrtEstimate32:
     case Opcode::FPRSqrtEstimate64:
+    case Opcode::FPRSqrtStepFused16:
     case Opcode::FPRSqrtStepFused32:
     case Opcode::FPRSqrtStepFused64:
     case Opcode::FPSqrt32:
@@ -350,6 +351,7 @@ bool Inst::ReadsFromAndWritesToFPSRCumulativeExceptionBits() const {
     case Opcode::FPVectorRSqrtEstimate16:
     case Opcode::FPVectorRSqrtEstimate32:
     case Opcode::FPVectorRSqrtEstimate64:
+    case Opcode::FPVectorRSqrtStepFused16:
     case Opcode::FPVectorRSqrtStepFused32:
     case Opcode::FPVectorRSqrtStepFused64:
     case Opcode::FPVectorSqrt32:

src/frontend/ir/opcodes.inc

@@ -506,6 +506,7 @@ OPCODE(FPRoundInt64,                                        U64,            U64,
 OPCODE(FPRSqrtEstimate16,                                   U16,            U16                                                             )
 OPCODE(FPRSqrtEstimate32,                                   U32,            U32                                                             )
 OPCODE(FPRSqrtEstimate64,                                   U64,            U64                                                             )
+OPCODE(FPRSqrtStepFused16,                                  U16,            U16,            U16                                             )
 OPCODE(FPRSqrtStepFused32,                                  U32,            U32,            U32                                             )
 OPCODE(FPRSqrtStepFused64,                                  U64,            U64,            U64                                             )
 OPCODE(FPSqrt32,                                            U32,            U32                                                             )
@@ -585,6 +586,7 @@ OPCODE(FPVectorRoundInt64,                                  U128,           U128
 OPCODE(FPVectorRSqrtEstimate16,                             U128,           U128                                                            )
 OPCODE(FPVectorRSqrtEstimate32,                             U128,           U128                                                            )
 OPCODE(FPVectorRSqrtEstimate64,                             U128,           U128                                                            )
+OPCODE(FPVectorRSqrtStepFused16,                            U128,           U128,           U128                                            )
 OPCODE(FPVectorRSqrtStepFused32,                            U128,           U128,           U128                                            )
 OPCODE(FPVectorRSqrtStepFused64,                            U128,           U128,           U128                                            )
 OPCODE(FPVectorSqrt32,                                      U128,           U128                                                            )