Skip to content

Commit c3260c6

Browse files
MacDueMacDue
authored
[IR] Add llvm.sincos intrinsic (#109825)
This adds the `llvm.sincos` intrinsic, legalization, and lowering. The `llvm.sincos` intrinsic takes a floating-point value and returns both the sine and cosine (as a struct). ``` declare { float, float } @llvm.sincos.f32(float %Val) declare { double, double } @llvm.sincos.f64(double %Val) declare { x86_fp80, x86_fp80 } @llvm.sincos.f80(x86_fp80 %Val) declare { fp128, fp128 } @llvm.sincos.f128(fp128 %Val) declare { ppc_fp128, ppc_fp128 } @llvm.sincos.ppcf128(ppc_fp128 %Val) declare { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %Val) ``` The lowering is built on top of the existing FSINCOS ISD node, with additional type legalization to allow for f16, f128, and vector values.
1 parent 7395ef5 commit c3260c6

23 files changed

+1300
-15
lines changed

‎llvm/docs/GlobalISel/GenericOpcode.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -633,8 +633,8 @@ G_FCEIL, G_FSQRT, G_FFLOOR, G_FRINT, G_FNEARBYINT
633633

634634
These correspond to the standard C functions of the same name.
635635

636-
G_FCOS, G_FSIN, G_FTAN, G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH, G_FTANH
637-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
636+
G_FCOS, G_FSIN, G_FSINCOS, G_FTAN, G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH, G_FTANH
637+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
638638

639639
These correspond to the standard C trigonometry functions of the same name.
640640

‎llvm/docs/LangRef.rst

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15512,6 +15512,8 @@ Semantics:
1551215512
This function returns the first value raised to the second power with an
1551315513
unspecified sequence of rounding operations.
1551415514

15515+
.. _t_llvm_sin:
15516+
1551515517
'``llvm.sin.*``' Intrinsic
1551615518
^^^^^^^^^^^^^^^^^^^^^^^^^^
1551715519

@@ -15549,6 +15551,8 @@ trapping or setting ``errno``.
1554915551
When specified with the fast-math-flag 'afn', the result may be approximated
1555015552
using a less accurate calculation.
1555115553

15554+
.. _t_llvm_cos:
15555+
1555215556
'``llvm.cos.*``' Intrinsic
1555315557
^^^^^^^^^^^^^^^^^^^^^^^^^^
1555415558

@@ -15882,6 +15886,50 @@ trapping or setting ``errno``.
1588215886
When specified with the fast-math-flag 'afn', the result may be approximated
1588315887
using a less accurate calculation.
1588415888

15889+
15890+
'``llvm.sincos.*``' Intrinsic
15891+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
15892+
15893+
Syntax:
15894+
"""""""
15895+
15896+
This is an overloaded intrinsic. You can use ``llvm.sincos`` on any
15897+
floating-point or vector of floating-point type. Not all targets support
15898+
all types however.
15899+
15900+
::
15901+
15902+
declare { float, float } @llvm.sincos.f32(float %Val)
15903+
declare { double, double } @llvm.sincos.f64(double %Val)
15904+
declare { x86_fp80, x86_fp80 } @llvm.sincos.f80(x86_fp80 %Val)
15905+
declare { fp128, fp128 } @llvm.sincos.f128(fp128 %Val)
15906+
declare { ppc_fp128, ppc_fp128 } @llvm.sincos.ppcf128(ppc_fp128 %Val)
15907+
declare { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %Val)
15908+
15909+
Overview:
15910+
"""""""""
15911+
15912+
The '``llvm.sincos.*``' intrinsics returns the sine and cosine of the operand.
15913+
15914+
Arguments:
15915+
""""""""""
15916+
15917+
The argument is a :ref:`floating-point <t_floating>` value or
15918+
:ref:`vector <t_vector>` of floating-point values. Returns two values matching
15919+
the argument type in a struct.
15920+
15921+
Semantics:
15922+
""""""""""
15923+
15924+
This intrinsic is equivalent to a calling both :ref:`llvm.sin <t_llvm_sin>`
15925+
and :ref:`llvm.cos <t_llvm_cos>` on the argument.
15926+
15927+
The first result is the sine of the argument and the second result is the cosine
15928+
of the argument.
15929+
15930+
When specified with the fast-math-flag 'afn', the result may be approximated
15931+
using a less accurate calculation.
15932+
1588515933
'``llvm.pow.*``' Intrinsic
1588615934
^^^^^^^^^^^^^^^^^^^^^^^^^^
1588715935

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1986,6 +1986,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
19861986
case Intrinsic::cos:
19871987
ISD = ISD::FCOS;
19881988
break;
1989+
case Intrinsic::sincos:
1990+
ISD = ISD::FSINCOS;
1991+
break;
19891992
case Intrinsic::tan:
19901993
ISD = ISD::FTAN;
19911994
break;

‎llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2009,6 +2009,13 @@ class MachineIRBuilder {
20092009
return buildInstr(TargetOpcode::G_FFREXP, {Fract, Exp}, {Src}, Flags);
20102010
}
20112011

2012+
/// Build and insert \p Sin, \p Cos = G_FSINCOS \p Src
2013+
MachineInstrBuilder
2014+
buildFSincos(const DstOp &Sin, const DstOp &Cos, const SrcOp &Src,
2015+
std::optional<unsigned> Flags = std::nullopt) {
2016+
return buildInstr(TargetOpcode::G_FSINCOS, {Sin, Cos}, {Src}, Flags);
2017+
}
2018+
20122019
/// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1
20132020
MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0,
20142021
const SrcOp &Src1) {

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1050,6 +1050,8 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
10501050
def int_nearbyint : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
10511051
def int_round : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
10521052
def int_roundeven : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
1053+
def int_sincos : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
1054+
[llvm_anyfloat_ty]>;
10531055

10541056
// Truncate a floating point number with a specific rounding mode
10551057
def int_fptrunc_round : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],

‎llvm/include/llvm/Support/TargetOpcodes.def

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -809,6 +809,9 @@ HANDLE_TARGET_OPCODE(G_FCOS)
809809
/// Floating point sine.
810810
HANDLE_TARGET_OPCODE(G_FSIN)
811811

812+
/// Floating point combined sine and cosine.
813+
HANDLE_TARGET_OPCODE(G_FSINCOS)
814+
812815
/// Floating point tangent.
813816
HANDLE_TARGET_OPCODE(G_FTAN)
814817

‎llvm/include/llvm/Target/GenericOpcodes.td

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1020,6 +1020,13 @@ def G_FSIN : GenericInstruction {
10201020
let hasSideEffects = false;
10211021
}
10221022

1023+
// Floating point combined sine and cosine.
1024+
def G_FSINCOS : GenericInstruction {
1025+
let OutOperandList = (outs type0:$dst1, type0:$dst2);
1026+
let InOperandList = (ins type0:$src1);
1027+
let hasSideEffects = false;
1028+
}
1029+
10231030
// Floating point tangent of a value.
10241031
def G_FTAN : GenericInstruction {
10251032
let OutOperandList = (outs type0:$dst);

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2343,6 +2343,13 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
23432343
MachineInstr::copyFlagsFromInstruction(CI));
23442344
return true;
23452345
}
2346+
case Intrinsic::sincos: {
2347+
ArrayRef<Register> VRegs = getOrCreateVRegs(CI);
2348+
MIRBuilder.buildFSincos(VRegs[0], VRegs[1],
2349+
getOrCreateVReg(*CI.getArgOperand(0)),
2350+
MachineInstr::copyFlagsFromInstruction(CI));
2351+
return true;
2352+
}
23462353
case Intrinsic::fptosi_sat:
23472354
MIRBuilder.buildFPTOSI_SAT(getOrCreateVReg(CI),
23482355
getOrCreateVReg(*CI.getArgOperand(0)));

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3714,6 +3714,17 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
37143714
}
37153715
break;
37163716
}
3717+
case ISD::FSINCOS: {
3718+
if (isSinCosLibcallAvailable(Node, TLI))
3719+
break;
3720+
EVT VT = Node->getValueType(0);
3721+
SDValue Op = Node->getOperand(0);
3722+
SDNodeFlags Flags = Node->getFlags();
3723+
Tmp1 = DAG.getNode(ISD::FSIN, dl, VT, Op, Flags);
3724+
Tmp2 = DAG.getNode(ISD::FCOS, dl, VT, Op, Flags);
3725+
Results.append({Tmp1, Tmp2});
3726+
break;
3727+
}
37173728
case ISD::FMAD:
37183729
llvm_unreachable("Illegal fmad should never be formed");
37193730

@@ -5586,6 +5597,16 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
55865597
Results.push_back(Tmp2.getValue(1));
55875598
break;
55885599
}
5600+
case ISD::FSINCOS: {
5601+
Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5602+
Tmp2 = DAG.getNode(ISD::FSINCOS, dl, DAG.getVTList(NVT, NVT), Tmp1,
5603+
Node->getFlags());
5604+
Tmp3 = DAG.getIntPtrConstant(0, dl, /*isTarget=*/true);
5605+
for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5606+
Results.push_back(
5607+
DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2.getValue(ResNum), Tmp3));
5608+
break;
5609+
}
55895610
case ISD::FFLOOR:
55905611
case ISD::FCEIL:
55915612
case ISD::FRINT:

‎llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
129129
case ISD::FLDEXP:
130130
case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
131131
case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
132+
case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
132133
case ISD::STRICT_FREM:
133134
case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
134135
case ISD::STRICT_FRINT:
@@ -774,6 +775,45 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
774775
return ReturnVal;
775776
}
776777

778+
SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
779+
assert(!N->isStrictFPOpcode() && "strictfp not implemented for fsincos");
780+
EVT VT = N->getValueType(0);
781+
RTLIB::Libcall LC = RTLIB::getFSINCOS(VT);
782+
783+
if (!TLI.getLibcallName(LC))
784+
return SDValue();
785+
786+
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
787+
SDValue StackSlotSin = DAG.CreateStackTemporary(NVT);
788+
SDValue StackSlotCos = DAG.CreateStackTemporary(NVT);
789+
790+
SDLoc DL(N);
791+
792+
TargetLowering::MakeLibCallOptions CallOptions;
793+
std::array Ops{GetSoftenedFloat(N->getOperand(0)), StackSlotSin,
794+
StackSlotCos};
795+
std::array OpsVT{VT, StackSlotSin.getValueType(),
796+
StackSlotCos.getValueType()};
797+
798+
// TODO: setTypeListBeforeSoften can't properly express multiple return types,
799+
// but since both returns have the same type for sincos it should be okay.
800+
CallOptions.setTypeListBeforeSoften({OpsVT}, VT, true);
801+
802+
auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, DL,
803+
/*Chain=*/SDValue());
804+
805+
auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
806+
int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
807+
auto PtrInfo =
808+
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
809+
return DAG.getLoad(NVT, DL, Chain, StackSlot, PtrInfo);
810+
};
811+
SetSoftenedFloat(SDValue(N, 0), CreateStackLoad(StackSlotSin));
812+
SetSoftenedFloat(SDValue(N, 1), CreateStackLoad(StackSlotCos));
813+
814+
return SDValue();
815+
}
816+
777817
SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
778818
return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
779819
RTLIB::REM_F32,
@@ -2704,6 +2744,10 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
27042744
case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
27052745
case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
27062746

2747+
case ISD::FSINCOS:
2748+
R = PromoteFloatRes_FSINCOS(N);
2749+
break;
2750+
27072751
case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
27082752
case ISD::STRICT_FP_ROUND:
27092753
R = PromoteFloatRes_STRICT_FP_ROUND(N);
@@ -2899,6 +2943,20 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
28992943
return Res;
29002944
}
29012945

2946+
SDValue DAGTypeLegalizer::PromoteFloatRes_FSINCOS(SDNode *N) {
2947+
EVT VT = N->getValueType(0);
2948+
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2949+
SDValue Op = GetPromotedFloat(N->getOperand(0));
2950+
SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, NVT}, Op);
2951+
2952+
for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2953+
++ResNum) {
2954+
SetPromotedFloat(SDValue(N, ResNum), Res.getValue(ResNum));
2955+
}
2956+
2957+
return SDValue();
2958+
}
2959+
29022960
// Explicit operation to reduce precision. Reduce the value to half precision
29032961
// and promote it back to the legal type.
29042962
SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
@@ -3148,6 +3206,10 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
31483206

31493207
case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
31503208

3209+
case ISD::FSINCOS:
3210+
R = SoftPromoteHalfRes_FSINCOS(N);
3211+
break;
3212+
31513213
case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
31523214
case ISD::ATOMIC_LOAD:
31533215
R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
@@ -3304,6 +3366,27 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
33043366
return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
33053367
}
33063368

3369+
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FSINCOS(SDNode *N) {
3370+
EVT OVT = N->getValueType(0);
3371+
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3372+
SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3373+
SDLoc dl(N);
3374+
3375+
// Promote to the larger FP type.
3376+
Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3377+
SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, NVT), Op);
3378+
3379+
// Convert back to FP16 as an integer.
3380+
ISD::NodeType Truncate = GetPromotionOpcode(NVT, OVT);
3381+
for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3382+
++ResNum) {
3383+
SDValue Trunc = DAG.getNode(Truncate, dl, MVT::i16, Res.getValue(ResNum));
3384+
SetSoftPromotedHalf(SDValue(N, ResNum), Trunc);
3385+
}
3386+
3387+
return SDValue();
3388+
}
3389+
33073390
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
33083391
EVT RVT = N->getValueType(0);
33093392
bool IsStrict = N->isStrictFPOpcode();

0 commit comments

Comments
 (0)