Auto merge of #46093 - scottmcm:lower-128-mir, r=nagisa

Add a MIR pass to lower 128-bit operators to lang item calls

Runs only with `-Z lower_128bit_ops` since it's not hooked into targets yet.

This isn't really useful on its own, but the declarations for the lang items need to be in the compiler before compiler-builtins can be updated to define them, so this is part 1 of at least 3.

cc #45676 @est31 @nagisa

Author: bors

src/librustc/middle/lang_items.rs

@@ -310,6 +310,34 @@ language_item_table! {
     NonZeroItem,                     "non_zero",                non_zero;
 
     DebugTraitLangItem,              "debug_trait",             debug_trait;
+
+    // A lang item for each of the 128-bit operators we can optionally lower.
+    I128AddFnLangItem,               "i128_add",                i128_add_fn;
+    U128AddFnLangItem,               "u128_add",                u128_add_fn;
+    I128SubFnLangItem,               "i128_sub",                i128_sub_fn;
+    U128SubFnLangItem,               "u128_sub",                u128_sub_fn;
+    I128MulFnLangItem,               "i128_mul",                i128_mul_fn;
+    U128MulFnLangItem,               "u128_mul",                u128_mul_fn;
+    I128DivFnLangItem,               "i128_div",                i128_div_fn;
+    U128DivFnLangItem,               "u128_div",                u128_div_fn;
+    I128RemFnLangItem,               "i128_rem",                i128_rem_fn;
+    U128RemFnLangItem,               "u128_rem",                u128_rem_fn;
+    I128ShlFnLangItem,               "i128_shl",                i128_shl_fn;
+    U128ShlFnLangItem,               "u128_shl",                u128_shl_fn;
+    I128ShrFnLangItem,               "i128_shr",                i128_shr_fn;
+    U128ShrFnLangItem,               "u128_shr",                u128_shr_fn;
+    // And overflow versions for the operators that are checkable.
+    // While MIR calls these Checked*, they return (T,bool), not Option<T>.
+    I128AddoFnLangItem,              "i128_addo",               i128_addo_fn;
+    U128AddoFnLangItem,              "u128_addo",               u128_addo_fn;
+    I128SuboFnLangItem,              "i128_subo",               i128_subo_fn;
+    U128SuboFnLangItem,              "u128_subo",               u128_subo_fn;
+    I128MuloFnLangItem,              "i128_mulo",               i128_mulo_fn;
+    U128MuloFnLangItem,              "u128_mulo",               u128_mulo_fn;
+    I128ShloFnLangItem,              "i128_shlo",               i128_shlo_fn;
+    U128ShloFnLangItem,              "u128_shlo",               u128_shlo_fn;
+    I128ShroFnLangItem,              "i128_shro",               i128_shro_fn;
+    U128ShroFnLangItem,              "u128_shro",               u128_shro_fn;
 }
 
 impl<'a, 'tcx, 'gcx> TyCtxt<'a, 'tcx, 'gcx> {

src/librustc/mir/mod.rs

@@ -170,6 +170,15 @@ impl<'tcx> Mir<'tcx> {
         &mut self.basic_blocks
     }
 
+    #[inline]
+    pub fn basic_blocks_and_local_decls_mut(&mut self) -> (
+        &mut IndexVec<BasicBlock, BasicBlockData<'tcx>>,
+        &mut LocalDecls<'tcx>,
+    ) {
+        self.cache.invalidate();
+        (&mut self.basic_blocks, &mut self.local_decls)
+    }
+
     #[inline]
     pub fn predecessors(&self) -> Ref<IndexVec<BasicBlock, Vec<BasicBlock>>> {
         self.cache.predecessors(self)

src/librustc/session/config.rs

@@ -1144,6 +1144,9 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
     saturating_float_casts: bool = (false, parse_bool, [TRACKED],
         "make float->int casts UB-free: numbers outside the integer type's range are clipped to \
          the max/min integer respectively, and NaN is mapped to 0"),
+    lower_128bit_ops: bool = (false, parse_bool, [TRACKED],
+        "rewrite operators on i128 and u128 into lang item calls (typically provided \
+         by compiler-builtins) so translation doesn't need to support them"),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc_mir/transform/lower_128bit.rs

@@ -0,0 +1,240 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Replaces 128-bit operators with lang item calls
+
+use rustc::hir::def_id::DefId;
+use rustc::middle::lang_items::LangItem;
+use rustc::mir::*;
+use rustc::ty::{Slice, Ty, TyCtxt, TypeVariants};
+use rustc_data_structures::indexed_vec::{Idx};
+use transform::{MirPass, MirSource};
+use syntax;
+
+pub struct Lower128Bit;
+
+impl MirPass for Lower128Bit {
+    fn run_pass<'a, 'tcx>(&self,
+                          tcx: TyCtxt<'a, 'tcx, 'tcx>,
+                          _src: MirSource,
+                          mir: &mut Mir<'tcx>) {
+        if !tcx.sess.opts.debugging_opts.lower_128bit_ops {
+            return
+        }
+
+        self.lower_128bit_ops(tcx, mir);
+    }
+}
+
+impl Lower128Bit {
+    fn lower_128bit_ops<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>, mir: &mut Mir<'tcx>) {
+        let mut new_blocks = Vec::new();
+        let cur_len = mir.basic_blocks().len();
+
+        let (basic_blocks, local_decls) = mir.basic_blocks_and_local_decls_mut();
+        for block in basic_blocks.iter_mut() {
+            for i in (0..block.statements.len()).rev() {
+                let (lang_item, rhs_kind) =
+                    if let Some((lang_item, rhs_kind)) =
+                        lower_to(&block.statements[i], local_decls, tcx)
+                    {
+                        (lang_item, rhs_kind)
+                    } else {
+                        continue;
+                    };
+
+                let rhs_override_ty = rhs_kind.ty(tcx);
+                let cast_local =
+                    match rhs_override_ty {
+                        None => None,
+                        Some(ty) => {
+                            let local_decl = LocalDecl::new_internal(
+                                ty, block.statements[i].source_info.span);
+                            Some(local_decls.push(local_decl))
+                        },
+                    };
+
+                let storage_dead = cast_local.map(|local| {
+                    Statement {
+                        source_info: block.statements[i].source_info,
+                        kind: StatementKind::StorageDead(local),
+                    }
+                });
+                let after_call = BasicBlockData {
+                    statements: storage_dead.into_iter()
+                        .chain(block.statements.drain((i+1)..)).collect(),
+                    is_cleanup: block.is_cleanup,
+                    terminator: block.terminator.take(),
+                };
+
+                let bin_statement = block.statements.pop().unwrap();
+                let (source_info, lvalue, lhs, mut rhs) = match bin_statement {
+                    Statement {
+                        source_info,
+                        kind: StatementKind::Assign(
+                            lvalue,
+                            Rvalue::BinaryOp(_, lhs, rhs))
+                    } => (source_info, lvalue, lhs, rhs),
+                    Statement {
+                        source_info,
+                        kind: StatementKind::Assign(
+                            lvalue,
+                            Rvalue::CheckedBinaryOp(_, lhs, rhs))
+                    } => (source_info, lvalue, lhs, rhs),
+                    _ => bug!("Statement doesn't match pattern any more?"),
+                };
+
+                if let Some(local) = cast_local {
+                    block.statements.push(Statement {
+                        source_info: source_info,
+                        kind: StatementKind::StorageLive(local),
+                    });
+                    block.statements.push(Statement {
+                        source_info: source_info,
+                        kind: StatementKind::Assign(
+                            Lvalue::Local(local),
+                            Rvalue::Cast(
+                                CastKind::Misc,
+                                rhs,
+                                rhs_override_ty.unwrap())),
+                    });
+                    rhs = Operand::Consume(Lvalue::Local(local));
+                }
+
+                let call_did = check_lang_item_type(
+                    lang_item, &lvalue, &lhs, &rhs, local_decls, tcx);
+
+                let bb = BasicBlock::new(cur_len + new_blocks.len());
+                new_blocks.push(after_call);
+
+                block.terminator =
+                    Some(Terminator {
+                        source_info,
+                        kind: TerminatorKind::Call {
+                            func: Operand::function_handle(tcx, call_did,
+                                Slice::empty(), source_info.span),
+                            args: vec![lhs, rhs],
+                            destination: Some((lvalue, bb)),
+                            cleanup: None,
+                        },
+                    });
+            }
+        }
+
+        basic_blocks.extend(new_blocks);
+    }
+}
+
+fn check_lang_item_type<'a, 'tcx, D>(
+    lang_item: LangItem,
+    lvalue: &Lvalue<'tcx>,
+    lhs: &Operand<'tcx>,
+    rhs: &Operand<'tcx>,
+    local_decls: &D,
+    tcx: TyCtxt<'a, 'tcx, 'tcx>)
+-> DefId
+    where D: HasLocalDecls<'tcx>
+{
+    let did = tcx.require_lang_item(lang_item);
+    let poly_sig = tcx.fn_sig(did);
+    let sig = tcx.no_late_bound_regions(&poly_sig).unwrap();
+    let lhs_ty = lhs.ty(local_decls, tcx);
+    let rhs_ty = rhs.ty(local_decls, tcx);
+    let lvalue_ty = lvalue.ty(local_decls, tcx).to_ty(tcx);
+    let expected = [lhs_ty, rhs_ty, lvalue_ty];
+    assert_eq!(sig.inputs_and_output[..], expected,
+        "lang item {}", tcx.def_symbol_name(did));
+    did
+}
+
+fn lower_to<'a, 'tcx, D>(statement: &Statement<'tcx>, local_decls: &D, tcx: TyCtxt<'a, 'tcx, 'tcx>)
+    -> Option<(LangItem, RhsKind)>
+    where D: HasLocalDecls<'tcx>
+{
+    match statement.kind {
+        StatementKind::Assign(_, Rvalue::BinaryOp(bin_op, ref lhs, _)) => {
+            let ty = lhs.ty(local_decls, tcx);
+            if let Some(is_signed) = sign_of_128bit(ty) {
+                return item_for_op(bin_op, is_signed);
+            }
+        },
+        StatementKind::Assign(_, Rvalue::CheckedBinaryOp(bin_op, ref lhs, _)) => {
+            let ty = lhs.ty(local_decls, tcx);
+            if let Some(is_signed) = sign_of_128bit(ty) {
+                return item_for_checked_op(bin_op, is_signed);
+            }
+        },
+        _ => {},
+    }
+    None
+}
+
+#[derive(Copy, Clone)]
+enum RhsKind {
+    Unchanged,
+    ForceU128,
+    ForceU32,
+}
+
+impl RhsKind {
+    fn ty<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>) -> Option<Ty<'tcx>> {
+        match *self {
+            RhsKind::Unchanged => None,
+            RhsKind::ForceU128 => Some(tcx.types.u128),
+            RhsKind::ForceU32 => Some(tcx.types.u32),
+        }
+    }
+}
+
+fn sign_of_128bit(ty: Ty) -> Option<bool> {
+    match ty.sty {
+        TypeVariants::TyInt(syntax::ast::IntTy::I128) => Some(true),
+        TypeVariants::TyUint(syntax::ast::UintTy::U128) => Some(false),
+        _ => None,
+    }
+}
+
+fn item_for_op(bin_op: BinOp, is_signed: bool) -> Option<(LangItem, RhsKind)> {
+    let i = match (bin_op, is_signed) {
+        (BinOp::Add, true) => (LangItem::I128AddFnLangItem, RhsKind::Unchanged),
+        (BinOp::Add, false) => (LangItem::U128AddFnLangItem, RhsKind::Unchanged),
+        (BinOp::Sub, true) => (LangItem::I128SubFnLangItem, RhsKind::Unchanged),
+        (BinOp::Sub, false) => (LangItem::U128SubFnLangItem, RhsKind::Unchanged),
+        (BinOp::Mul, true) => (LangItem::I128MulFnLangItem, RhsKind::Unchanged),
+        (BinOp::Mul, false) => (LangItem::U128MulFnLangItem, RhsKind::Unchanged),
+        (BinOp::Div, true) => (LangItem::I128DivFnLangItem, RhsKind::Unchanged),
+        (BinOp::Div, false) => (LangItem::U128DivFnLangItem, RhsKind::Unchanged),
+        (BinOp::Rem, true) => (LangItem::I128RemFnLangItem, RhsKind::Unchanged),
+        (BinOp::Rem, false) => (LangItem::U128RemFnLangItem, RhsKind::Unchanged),
+        (BinOp::Shl, true) => (LangItem::I128ShlFnLangItem, RhsKind::ForceU32),
+        (BinOp::Shl, false) => (LangItem::U128ShlFnLangItem, RhsKind::ForceU32),
+        (BinOp::Shr, true) => (LangItem::I128ShrFnLangItem, RhsKind::ForceU32),
+        (BinOp::Shr, false) => (LangItem::U128ShrFnLangItem, RhsKind::ForceU32),
+        _ => return None,
+    };
+    Some(i)
+}
+
+fn item_for_checked_op(bin_op: BinOp, is_signed: bool) -> Option<(LangItem, RhsKind)> {
+    let i = match (bin_op, is_signed) {
+        (BinOp::Add, true) => (LangItem::I128AddoFnLangItem, RhsKind::Unchanged),
+        (BinOp::Add, false) => (LangItem::U128AddoFnLangItem, RhsKind::Unchanged),
+        (BinOp::Sub, true) => (LangItem::I128SuboFnLangItem, RhsKind::Unchanged),
+        (BinOp::Sub, false) => (LangItem::U128SuboFnLangItem, RhsKind::Unchanged),
+        (BinOp::Mul, true) => (LangItem::I128MuloFnLangItem, RhsKind::Unchanged),
+        (BinOp::Mul, false) => (LangItem::U128MuloFnLangItem, RhsKind::Unchanged),
+        (BinOp::Shl, true) => (LangItem::I128ShloFnLangItem, RhsKind::ForceU128),
+        (BinOp::Shl, false) => (LangItem::U128ShloFnLangItem, RhsKind::ForceU128),
+        (BinOp::Shr, true) => (LangItem::I128ShroFnLangItem, RhsKind::ForceU128),
+        (BinOp::Shr, false) => (LangItem::U128ShroFnLangItem, RhsKind::ForceU128),
+        _ => bug!("That should be all the checked ones?"),
+    };
+    Some(i)
+}

src/librustc_mir/transform/mod.rs

@@ -42,6 +42,7 @@ pub mod copy_prop;
 pub mod generator;
 pub mod inline;
 pub mod nll;
+pub mod lower_128bit;
 
 pub(crate) fn provide(providers: &mut Providers) {
     self::qualify_consts::provide(providers);
@@ -241,6 +242,8 @@ fn optimized_mir<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, def_id: DefId) -> &'tcx
         // From here on out, regions are gone.
         erase_regions::EraseRegions,
 
+        lower_128bit::Lower128Bit,
+
         // Optimizations begin.
         inline::Inline,
         instcombine::InstCombine,