diff options
Diffstat (limited to 'zjit/src')
| -rw-r--r-- | zjit/src/backend/arm64/mod.rs | 89 | ||||
| -rw-r--r-- | zjit/src/backend/lir.rs | 55 | ||||
| -rw-r--r-- | zjit/src/backend/x86_64/mod.rs | 12 | ||||
| -rw-r--r-- | zjit/src/codegen.rs | 563 | ||||
| -rw-r--r-- | zjit/src/cruby.rs | 1 | ||||
| -rw-r--r-- | zjit/src/cruby_bindings.inc.rs | 6 | ||||
| -rw-r--r-- | zjit/src/gc.rs | 33 | ||||
| -rw-r--r-- | zjit/src/hir.rs | 132 | ||||
| -rw-r--r-- | zjit/src/invariants.rs | 27 | ||||
| -rw-r--r-- | zjit/src/profile.rs | 70 |
10 files changed, 686 insertions, 302 deletions
diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 3b7742f16e..c60ec53285 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -2059,4 +2059,93 @@ mod tests { 0x4: adds x1, x0, #1 "}); } + + #[test] + fn test_reorder_c_args_no_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[0], // mov x0, x0 (optimized away) + C_ARG_OPNDS[1], // mov x1, x1 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm!(cb, "100080d200023fd6", {" + 0x0: mov x16, #0 + 0x4: blr x16 + "}); + } + + #[test] + fn test_reorder_c_args_single_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[2], // mov x2, x2 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm!(cb, "f00300aae00301aae10310aa100080d200023fd6", {" + 0x0: mov x16, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x16 + 0xc: mov x16, #0 + 0x10: blr x16 + "}); + } + + #[test] + fn test_reorder_c_args_two_cycles() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle, and x2 and rcx form another cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[3], // mov x2, rcx + C_ARG_OPNDS[2], // mov rcx, x2 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm!(cb, "f00302aae20303aae30310aaf00300aae00301aae10310aa100080d200023fd6", {" + 0x0: mov x16, x2 + 0x4: mov x2, x3 + 0x8: mov x3, x16 + 0xc: mov x16, x0 + 0x10: mov x0, x1 + 0x14: mov x1, x16 + 0x18: mov x16, #0 + 0x1c: blr x16 + "}); + } + + #[test] + fn test_reorder_c_args_large_cycle() { + crate::options::rb_zjit_prepare_options(); + let (mut asm, mut cb) = setup_asm(); + + // x0, x1, and x2 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[2], // mov x1, x2 + C_ARG_OPNDS[0], // mov x2, x0 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm!(cb, "f00300aae00301aae10302aae20310aa100080d200023fd6", {" + 0x0: mov x16, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x2 + 0xc: mov x2, x16 + 0x10: mov x16, #0 + 0x14: blr x16 + "}); + } + } diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 54bef9d925..1bb4cd024b 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -147,26 +147,15 @@ impl Opnd } } - /// Return Some(Opnd) with a given num_bits if self has num_bits. - /// None if self doesn't have a num_bits field. - pub fn try_num_bits(&self, num_bits: u8) -> Option<Opnd> { - assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); - match *self { - Opnd::Reg(reg) => Some(Opnd::Reg(reg.with_num_bits(num_bits))), - Opnd::Mem(Mem { base, disp, .. }) => Some(Opnd::Mem(Mem { base, disp, num_bits })), - Opnd::VReg { idx, .. } => Some(Opnd::VReg { idx, num_bits }), - _ => None, - } - } - - /// Return Opnd with a given num_bits if self has num_bits. - /// Panic otherwise. This should be used only when you know which Opnd self is. + /// Return Opnd with a given num_bits if self has num_bits. Panic otherwise. #[track_caller] pub fn with_num_bits(&self, num_bits: u8) -> Opnd { - if let Some(opnd) = self.try_num_bits(num_bits) { - opnd - } else { - unreachable!("with_num_bits should not be used on: {self:?}"); + assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); + match *self { + Opnd::Reg(reg) => Opnd::Reg(reg.with_num_bits(num_bits)), + Opnd::Mem(Mem { base, disp, .. }) => Opnd::Mem(Mem { base, disp, num_bits }), + Opnd::VReg { idx, .. } => Opnd::VReg { idx, num_bits }, + _ => unreachable!("with_num_bits should not be used for: {self:?}"), } } @@ -1213,7 +1202,7 @@ impl Assembler /// Append an instruction onto the current list of instructions and update /// the live ranges of any instructions whose outputs are being used as /// operands to this instruction. - pub fn push_insn(&mut self, mut insn: Insn) { + pub fn push_insn(&mut self, insn: Insn) { // Index of this instruction let insn_idx = self.insns.len(); @@ -1225,7 +1214,7 @@ impl Assembler } // If we find any VReg from previous instructions, extend the live range to insn_idx - let mut opnd_iter = insn.opnd_iter_mut(); + let mut opnd_iter = insn.opnd_iter(); while let Some(opnd) = opnd_iter.next() { match *opnd { Opnd::VReg { idx, .. } | @@ -1391,13 +1380,15 @@ impl Assembler } } - // If the output VReg of this instruction is used by another instruction, - // we need to allocate a register to it + // Allocate a register for the output operand if it exists let vreg_idx = match insn.out_opnd() { Some(Opnd::VReg { idx, .. }) => Some(*idx), _ => None, }; - if vreg_idx.is_some() && live_ranges[vreg_idx.unwrap()].end() != index { + if vreg_idx.is_some() { + if live_ranges[vreg_idx.unwrap()].end() == index { + debug!("Allocating a register for VReg({}) at instruction index {} even though it does not live past this index", vreg_idx.unwrap(), index); + } // This is going to be the output operand that we will set on the // instruction. CCall and LiveReg need to use a specific register. let mut out_reg = match insn { @@ -1477,6 +1468,18 @@ impl Assembler } } + // If we have an output that dies at its definition (it is unused), free up the + // register + if let Some(idx) = vreg_idx { + if live_ranges[idx].end() == index { + if let Some(reg) = reg_mapping[idx] { + pool.dealloc_reg(®); + } else { + unreachable!("no register allocated for insn {:?}", insn); + } + } + } + // Push instruction(s) let is_ccall = matches!(insn, Insn::CCall { .. }); match insn { @@ -1684,6 +1687,7 @@ impl Assembler { } pub fn cpop_into(&mut self, opnd: Opnd) { + assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); self.push_insn(Insn::CPopInto(opnd)); } @@ -1831,6 +1835,7 @@ impl Assembler { } pub fn lea_into(&mut self, out: Opnd, opnd: Opnd) { + assert!(matches!(out, Opnd::Reg(_)), "Destination of lea_into must be a register, got: {out:?}"); self.push_insn(Insn::Lea { opnd, out }); } @@ -1856,7 +1861,7 @@ impl Assembler { } pub fn load_into(&mut self, dest: Opnd, opnd: Opnd) { - assert!(matches!(dest, Opnd::Reg(_) | Opnd::VReg{..}), "Destination of load_into must be a register"); + assert!(matches!(dest, Opnd::Reg(_)), "Destination of load_into must be a register, got: {dest:?}"); match (dest, opnd) { (Opnd::Reg(dest), Opnd::Reg(opnd)) if dest == opnd => {}, // skip if noop _ => self.push_insn(Insn::LoadInto { dest, opnd }), @@ -1882,6 +1887,7 @@ impl Assembler { } pub fn mov(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of mov must not be Opnd::VReg, got: {dest:?}"); self.push_insn(Insn::Mov { dest, src }); } @@ -1919,6 +1925,7 @@ impl Assembler { } pub fn store(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of store must not be Opnd::VReg, got: {dest:?}"); self.push_insn(Insn::Store { dest, src }); } diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index f15b32f946..2a02e1b725 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1247,13 +1247,14 @@ mod tests { #[test] fn test_reorder_c_args_no_cycle() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); asm.ccall(0 as _, vec![ C_ARG_OPNDS[0], // mov rdi, rdi (optimized away) C_ARG_OPNDS[1], // mov rsi, rsi (optimized away) ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); assert_disasm!(cb, "b800000000ffd0", {" 0x0: mov eax, 0 @@ -1263,6 +1264,7 @@ mod tests { #[test] fn test_reorder_c_args_single_cycle() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); // rdi and rsi form a cycle @@ -1271,7 +1273,7 @@ mod tests { C_ARG_OPNDS[0], // mov rsi, rdi C_ARG_OPNDS[2], // mov rdx, rdx (optimized away) ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); assert_disasm!(cb, "4989f34889fe4c89dfb800000000ffd0", {" 0x0: mov r11, rsi @@ -1284,6 +1286,7 @@ mod tests { #[test] fn test_reorder_c_args_two_cycles() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); // rdi and rsi form a cycle, and rdx and rcx form another cycle @@ -1293,7 +1296,7 @@ mod tests { C_ARG_OPNDS[3], // mov rdx, rcx C_ARG_OPNDS[2], // mov rcx, rdx ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); assert_disasm!(cb, "4989f34889fe4c89df4989cb4889d14c89dab800000000ffd0", {" 0x0: mov r11, rsi @@ -1309,6 +1312,7 @@ mod tests { #[test] fn test_reorder_c_args_large_cycle() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); // rdi, rsi, and rdx form a cycle @@ -1317,7 +1321,7 @@ mod tests { C_ARG_OPNDS[2], // mov rsi, rdx C_ARG_OPNDS[0], // mov rdx, rdi ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); assert_disasm!(cb, "4989f34889d64889fa4c89dfb800000000ffd0", {" 0x0: mov r11, rsi diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index a58950ab9a..0549365666 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -1,6 +1,7 @@ -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use std::rc::Rc; -use std::ffi::{c_int, c_void}; +use std::ffi::{c_int, c_long, c_void}; +use std::slice; use crate::asm::Label; use crate::backend::current::{Reg, ALLOC_REGS}; @@ -27,7 +28,7 @@ struct JITState { labels: Vec<Option<Target>>, /// ISEQ calls that need to be compiled later - iseq_calls: Vec<Rc<IseqCall>>, + iseq_calls: Vec<Rc<RefCell<IseqCall>>>, /// The number of bytes allocated for basic block arguments spilled onto the C stack c_stack_slots: usize, @@ -46,12 +47,8 @@ impl JITState { } /// Retrieve the output of a given instruction that has been compiled - fn get_opnd(&self, insn_id: InsnId) -> Option<lir::Opnd> { - let opnd = self.opnds[insn_id.0]; - if opnd.is_none() { - debug!("Failed to get_opnd({insn_id})"); - } - opnd + fn get_opnd(&self, insn_id: InsnId) -> lir::Opnd { + self.opnds[insn_id.0].expect(&format!("Failed to get_opnd({insn_id})")) } /// Find or create a label for a given BlockId @@ -130,13 +127,14 @@ fn gen_iseq_entry_point_body(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<CodePt }; // Stub callee ISEQs for JIT-to-JIT calls - for iseq_call in jit.iseq_calls.into_iter() { + for iseq_call in jit.iseq_calls.iter() { gen_iseq_call(cb, iseq, iseq_call)?; } // Remember the block address to reuse it later let payload = get_or_create_iseq_payload(iseq); payload.status = IseqStatus::Compiled(start_ptr); + payload.iseq_calls.extend(jit.iseq_calls); append_gc_offsets(iseq, &gc_offsets); // Return a JIT code address @@ -144,19 +142,20 @@ fn gen_iseq_entry_point_body(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<CodePt } /// Stub a branch for a JIT-to-JIT call -fn gen_iseq_call(cb: &mut CodeBlock, caller_iseq: IseqPtr, iseq_call: Rc<IseqCall>) -> Option<()> { +fn gen_iseq_call(cb: &mut CodeBlock, caller_iseq: IseqPtr, iseq_call: &Rc<RefCell<IseqCall>>) -> Option<()> { // Compile a function stub let Some(stub_ptr) = gen_function_stub(cb, iseq_call.clone()) else { // Failed to compile the stub. Bail out of compiling the caller ISEQ. debug!("Failed to compile iseq: could not compile stub: {} -> {}", - iseq_get_location(caller_iseq, 0), iseq_get_location(iseq_call.iseq, 0)); + iseq_get_location(caller_iseq, 0), iseq_get_location(iseq_call.borrow().iseq, 0)); return None; }; // Update the JIT-to-JIT call to call the stub let stub_addr = stub_ptr.raw_ptr(cb); - iseq_call.regenerate(cb, |asm| { - asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq_call.iseq, 0)); + let iseq = iseq_call.borrow().iseq; + iseq_call.borrow_mut().regenerate(cb, |asm| { + asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq, 0)); asm.ccall(stub_addr, vec![]); }); Some(()) @@ -209,7 +208,7 @@ fn gen_entry(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function, function_pt } /// Compile an ISEQ into machine code -fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<(CodePtr, Vec<Rc<IseqCall>>)> { +fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<(CodePtr, Vec<Rc<RefCell<IseqCall>>>)> { // Return an existing pointer if it's already compiled let payload = get_or_create_iseq_payload(iseq); match payload.status { @@ -231,6 +230,7 @@ fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<(CodePtr, Vec<Rc<IseqCa let result = gen_function(cb, iseq, &function); if let Some((start_ptr, gc_offsets, jit)) = result { payload.status = IseqStatus::Compiled(start_ptr); + payload.iseq_calls.extend(jit.iseq_calls.clone()); append_gc_offsets(iseq, &gc_offsets); Some((start_ptr, jit.iseq_calls)) } else { @@ -313,18 +313,24 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio // Convert InsnId to lir::Opnd macro_rules! opnd { ($insn_id:ident) => { - jit.get_opnd($insn_id.clone())? + jit.get_opnd($insn_id.clone()) }; } macro_rules! opnds { ($insn_ids:ident) => { { - Option::from_iter($insn_ids.iter().map(|insn_id| jit.get_opnd(*insn_id)))? + $insn_ids.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect::<Vec<_>>() } }; } + macro_rules! no_output { + ($call:expr) => { + { let () = $call; return Some(()); } + }; + } + if !matches!(*insn, Insn::Snapshot { .. }) { asm_comment!(asm, "Insn: {insn_id} {insn}"); } @@ -332,55 +338,63 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio let out_opnd = match insn { Insn::Const { val: Const::Value(val) } => gen_const(*val), Insn::NewArray { elements, state } => gen_new_array(asm, opnds!(elements), &function.frame_state(*state)), - Insn::NewRange { low, high, flag, state } => gen_new_range(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, elements, &function.frame_state(*state)), + Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), - Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state))?, - Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state))?, + // concatstrings shouldn't have 0 strings + // If it happens we abort the compilation for now + Insn::StringConcat { strings, .. } if strings.is_empty() => return None, + Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), + Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), Insn::Param { idx } => unreachable!("block.insns should not have Insn::Param({idx})"), Insn::Snapshot { .. } => return Some(()), // we don't need to do anything for this instruction at the moment - Insn::Jump(branch) => return gen_jump(jit, asm, branch), - Insn::IfTrue { val, target } => return gen_if_true(jit, asm, opnd!(val), target), - Insn::IfFalse { val, target } => return gen_if_false(jit, asm, opnd!(val), target), - Insn::SendWithoutBlock { cd, state, self_val, args, .. } => gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), opnd!(self_val), opnds!(args))?, + Insn::Jump(branch) => no_output!(gen_jump(jit, asm, branch)), + Insn::IfTrue { val, target } => no_output!(gen_if_true(jit, asm, opnd!(val), target)), + Insn::IfFalse { val, target } => no_output!(gen_if_false(jit, asm, opnd!(val), target)), + Insn::SendWithoutBlock { cd, state, self_val, args, .. } => gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), opnd!(self_val), opnds!(args)), // Give up SendWithoutBlockDirect for 6+ args since asm.ccall() doesn't support it. Insn::SendWithoutBlockDirect { cd, state, self_val, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => // +1 for self - gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), opnd!(self_val), opnds!(args))?, - Insn::SendWithoutBlockDirect { cme, iseq, self_val, args, state, .. } => gen_send_without_block_direct(cb, jit, asm, *cme, *iseq, opnd!(self_val), opnds!(args), &function.frame_state(*state))?, - Insn::InvokeBuiltin { bf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, opnds!(args))?, - Insn::Return { val } => return Some(gen_return(asm, opnd!(val))?), - Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right))?, - Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right))?, - Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right))?, - Insn::FixnumLe { left, right } => gen_fixnum_le(asm, opnd!(left), opnd!(right))?, - Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right))?, - Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right))?, - Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right))?, - Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right))?, - Insn::IsNil { val } => gen_isnil(asm, opnd!(val))?, - Insn::Test { val } => gen_test(asm, opnd!(val))?, - Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state))?, - Insn::GuardBitEquals { val, expected, state } => gen_guard_bit_equals(jit, asm, opnd!(val), *expected, &function.frame_state(*state))?, - Insn::PatchPoint { invariant, state } => return gen_patch_point(jit, asm, invariant, &function.frame_state(*state)), - Insn::CCall { cfun, args, name: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfun, opnds!(args))?, + gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), opnd!(self_val), opnds!(args)), + Insn::SendWithoutBlockDirect { cme, iseq, self_val, args, state, .. } => gen_send_without_block_direct(cb, jit, asm, *cme, *iseq, opnd!(self_val), opnds!(args), &function.frame_state(*state)), + // Ensure we have enough room fit ec, self, and arguments + // TODO remove this check when we have stack args (we can use Time.new to test it) + Insn::InvokeBuiltin { bf, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return None, + Insn::InvokeBuiltin { bf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, opnds!(args)), + Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right)), + Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right)), + Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right)), + Insn::FixnumLe { left, right } => gen_fixnum_le(asm, opnd!(left), opnd!(right)), + Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right)), + Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right)), + Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), + Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), + Insn::IsNil { val } => gen_isnil(asm, opnd!(val)), + Insn::Test { val } => gen_test(asm, opnd!(val)), + Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), + Insn::GuardBitEquals { val, expected, state } => gen_guard_bit_equals(jit, asm, opnd!(val), *expected, &function.frame_state(*state)), + Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), + Insn::CCall { cfun, args, name: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfun, opnds!(args)), Insn::GetIvar { self_val, id, state: _ } => gen_getivar(asm, opnd!(self_val), *id), - Insn::SetGlobal { id, val, state } => return gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state)), + Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), Insn::GetGlobal { id, state: _ } => gen_getglobal(asm, *id), - &Insn::GetLocal { ep_offset, level } => gen_getlocal_with_ep(asm, ep_offset, level)?, - &Insn::SetLocal { val, ep_offset, level } => return gen_setlocal_with_ep(asm, jit, function, val, ep_offset, level), - Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state))?, - Insn::SetIvar { self_val, id, val, state: _ } => return gen_setivar(asm, opnd!(self_val), *id, opnd!(val)), - Insn::SideExit { state, reason } => return gen_side_exit(jit, asm, reason, &function.frame_state(*state)), + &Insn::GetLocal { ep_offset, level } => gen_getlocal_with_ep(asm, ep_offset, level), + &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal_with_ep(asm, opnd!(val), function.type_of(val), ep_offset, level)), + Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), + Insn::SetIvar { self_val, id, val, state: _ } => no_output!(gen_setivar(asm, opnd!(self_val), *id, opnd!(val))), + Insn::SideExit { state, reason } => no_output!(gen_side_exit(jit, asm, reason, &function.frame_state(*state))), Insn::PutSpecialObject { value_type } => gen_putspecialobject(asm, *value_type), - Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state))?, - Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state))?, + Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), + Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state)), Insn::GetSpecialSymbol { symbol_type, state: _ } => gen_getspecial_symbol(asm, *symbol_type), Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), - &Insn::IncrCounter(counter) => return Some(gen_incr_counter(asm, counter)), - Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state))?, + &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), + Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state)), Insn::ArrayExtend { .. } | Insn::ArrayMax { .. } | Insn::ArrayPush { .. } @@ -388,7 +402,6 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio | Insn::FixnumDiv { .. } | Insn::FixnumMod { .. } | Insn::HashDup { .. } - | Insn::NewHash { .. } | Insn::Send { .. } | Insn::Throw { .. } | Insn::ToArray { .. } @@ -446,8 +459,8 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { ep_opnd } -fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Option<Opnd> { - gen_prepare_non_leaf_call(jit, asm, state)?; +fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); let iseq_opnd = Opnd::Value(jit.iseq.into()); @@ -459,12 +472,12 @@ fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *cons // Need to replicate what CALL_SIMPLE_METHOD does asm_comment!(asm, "side-exit if rb_vm_objtostring returns Qundef"); asm.cmp(ret, Qundef.into()); - asm.je(side_exit(jit, state, ObjToStringFallback)?); + asm.je(side_exit(jit, state, ObjToStringFallback)); - Some(ret) + ret } -fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, state: &FrameState) -> Option<Opnd> { +fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, state: &FrameState) -> Opnd { match op_type as defined_type { DEFINED_YIELD => { // `yield` goes to the block handler stowed in the "local" iseq which is @@ -476,21 +489,21 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); let pushval = asm.load(pushval.into()); asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); - Some(asm.csel_e(Qnil.into(), pushval.into())) + asm.csel_e(Qnil.into(), pushval.into()) } else { - Some(Qnil.into()) + Qnil.into() } } _ => { // Save the PC and SP because the callee may allocate or call #respond_to? - gen_prepare_non_leaf_call(jit, asm, state)?; + gen_prepare_non_leaf_call(jit, asm, state); // TODO: Inline the cases for each op_type // Call vm_defined(ec, reg_cfp, op_type, obj, v) let def_result = asm_ccall!(asm, rb_vm_defined, EC, CFP, op_type.into(), obj.into(), tested_value); asm.cmp(def_result.with_num_bits(8), 0.into()); - Some(asm.csel_ne(pushval.into(), Qnil.into())) + asm.csel_ne(pushval.into(), Qnil.into()) } } } @@ -498,69 +511,64 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, /// Get a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. /// We generate this instruction with level=0 only when the local variable is on the heap, so we /// can't optimize the level=0 case using the SP register. -fn gen_getlocal_with_ep(asm: &mut Assembler, local_ep_offset: u32, level: u32) -> Option<lir::Opnd> { +fn gen_getlocal_with_ep(asm: &mut Assembler, local_ep_offset: u32, level: u32) -> lir::Opnd { let ep = gen_get_ep(asm, level); - let offset = -(SIZEOF_VALUE_I32 * i32::try_from(local_ep_offset).ok()?); - Some(asm.load(Opnd::mem(64, ep, offset))) + let offset = -(SIZEOF_VALUE_I32 * i32::try_from(local_ep_offset).expect(&format!("Could not convert local_ep_offset {local_ep_offset} to i32"))); + asm.load(Opnd::mem(64, ep, offset)) } /// Set a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. /// We generate this instruction with level=0 only when the local variable is on the heap, so we /// can't optimize the level=0 case using the SP register. -fn gen_setlocal_with_ep(asm: &mut Assembler, jit: &JITState, function: &Function, val: InsnId, local_ep_offset: u32, level: u32) -> Option<()> { +fn gen_setlocal_with_ep(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: u32, level: u32) { let ep = gen_get_ep(asm, level); // When we've proved that we're writing an immediate, // we can skip the write barrier. - if function.type_of(val).is_immediate() { - let offset = -(SIZEOF_VALUE_I32 * i32::try_from(local_ep_offset).ok()?); - asm.mov(Opnd::mem(64, ep, offset), jit.get_opnd(val)?); + if val_type.is_immediate() { + let offset = -(SIZEOF_VALUE_I32 * i32::try_from(local_ep_offset).expect(&format!("Could not convert local_ep_offset {local_ep_offset} to i32"))); + asm.mov(Opnd::mem(64, ep, offset), val); } else { // We're potentially writing a reference to an IMEMO/env object, // so take care of the write barrier with a function. - let local_index = c_int::try_from(local_ep_offset).ok().and_then(|idx| idx.checked_mul(-1))?; - asm_ccall!(asm, rb_vm_env_write, ep, local_index.into(), jit.get_opnd(val)?); + let local_index = c_int::try_from(local_ep_offset).ok().and_then(|idx| idx.checked_mul(-1)).expect(&format!("Could not turn {local_ep_offset} into a negative c_int")); + asm_ccall!(asm, rb_vm_env_write, ep, local_index.into(), val); } - Some(()) } -fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Option<Opnd> { +fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { unsafe extern "C" { fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; } // Anything could be called on const_missing - gen_prepare_non_leaf_call(jit, asm, state)?; + gen_prepare_non_leaf_call(jit, asm, state); - Some(asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic))) + asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) } -fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, args: Vec<Opnd>) -> Option<lir::Opnd> { - // Ensure we have enough room fit ec, self, and arguments - // TODO remove this check when we have stack args (we can use Time.new to test it) - if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) { - return None; - } - +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, args: Vec<Opnd>) -> lir::Opnd { + assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, + "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", + unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }, + bf.argc); // Anything can happen inside builtin functions - gen_prepare_non_leaf_call(jit, asm, state)?; + gen_prepare_non_leaf_call(jit, asm, state); let mut cargs = vec![EC]; cargs.extend(args); - let val = asm.ccall(bf.func_ptr as *const u8, cargs); - - Some(val) + asm.ccall(bf.func_ptr as *const u8, cargs) } /// Record a patch point that should be invalidated on a given invariant -fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) -> Option<()> { +fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { let payload_ptr = get_or_create_iseq_payload_ptr(jit.iseq); let label = asm.new_label("patch_point").unwrap_label(); let invariant = invariant.clone(); // Compile a side exit. Fill nop instructions if the last patch point is too close. - asm.patch_point(build_side_exit(jit, state, PatchPoint(invariant), Some(label))?); + asm.patch_point(build_side_exit(jit, state, PatchPoint(invariant), Some(label))); // Remember the current address as a patch point asm.pos_marker(move |code_ptr, cb| { @@ -583,13 +591,12 @@ fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invarian } } }); - Some(()) } /// Lowering for [`Insn::CCall`]. This is a low-level raw call that doesn't know /// anything about the callee, so handling for e.g. GC safety is dealt with elsewhere. -fn gen_ccall(asm: &mut Assembler, cfun: *const u8, args: Vec<Opnd>) -> Option<lir::Opnd> { - Some(asm.ccall(cfun, args)) +fn gen_ccall(asm: &mut Assembler, cfun: *const u8, args: Vec<Opnd>) -> lir::Opnd { + asm.ccall(cfun, args) } /// Emit an uncached instance variable lookup @@ -598,9 +605,8 @@ fn gen_getivar(asm: &mut Assembler, recv: Opnd, id: ID) -> Opnd { } /// Emit an uncached instance variable store -fn gen_setivar(asm: &mut Assembler, recv: Opnd, id: ID, val: Opnd) -> Option<()> { +fn gen_setivar(asm: &mut Assembler, recv: Opnd, id: ID, val: Opnd) { asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); - Some(()) } /// Look up global variables @@ -609,25 +615,23 @@ fn gen_getglobal(asm: &mut Assembler, id: ID) -> Opnd { } /// Intern a string -fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Option<Opnd> { +fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { gen_prepare_call_with_gc(asm, state); - Some(asm_ccall!(asm, rb_str_intern, val)) + asm_ccall!(asm, rb_str_intern, val) } /// Set global variables -fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) -> Option<()> { +fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) { // When trace_var is used, setting a global variable can cause exceptions - gen_prepare_non_leaf_call(jit, asm, state)?; + gen_prepare_non_leaf_call(jit, asm, state); asm_ccall!(asm, rb_gvar_set, id.0.into(), val); - Some(()) } /// Side-exit into the interpreter -fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, state: &FrameState) -> Option<()> { - asm.jmp(side_exit(jit, state, *reason)?); - Some(()) +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, state: &FrameState) { + asm.jmp(side_exit(jit, state, *reason)); } /// Emit a special object lookup @@ -734,25 +738,26 @@ fn gen_entry_params(asm: &mut Assembler, iseq: IseqPtr, entry_block: &Block) { } /// Set branch params to basic block arguments -fn gen_branch_params(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) -> Option<()> { - if !branch.args.is_empty() { - asm_comment!(asm, "set branch params: {}", branch.args.len()); - let mut moves: Vec<(Reg, Opnd)> = vec![]; - for (idx, &arg) in branch.args.iter().enumerate() { - match param_opnd(idx) { - Opnd::Reg(reg) => { - // If a parameter is a register, we need to parallel-move it - moves.push((reg, jit.get_opnd(arg)?)); - }, - param => { - // If a parameter is memory, we set it beforehand - asm.mov(param, jit.get_opnd(arg)?); - } +fn gen_branch_params(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { + if branch.args.is_empty() { + return; + } + + asm_comment!(asm, "set branch params: {}", branch.args.len()); + let mut moves: Vec<(Reg, Opnd)> = vec![]; + for (idx, &arg) in branch.args.iter().enumerate() { + match param_opnd(idx) { + Opnd::Reg(reg) => { + // If a parameter is a register, we need to parallel-move it + moves.push((reg, jit.get_opnd(arg))); + }, + param => { + // If a parameter is memory, we set it beforehand + asm.mov(param, jit.get_opnd(arg)); } } - asm.parallel_mov(moves); } - Some(()) + asm.parallel_mov(moves); } /// Get a method parameter on JIT entry. As of entry, whether EP is escaped or not solely @@ -795,18 +800,17 @@ fn gen_param(asm: &mut Assembler, idx: usize) -> lir::Opnd { } /// Compile a jump to a basic block -fn gen_jump(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) -> Option<()> { +fn gen_jump(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { // Set basic block arguments gen_branch_params(jit, asm, branch); // Jump to the basic block let target = jit.get_label(asm, branch.target); asm.jmp(target); - Some(()) } /// Compile a conditional branch to a basic block -fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) -> Option<()> { +fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { // If val is zero, move on to the next instruction. let if_false = asm.new_label("if_false"); asm.test(val, val); @@ -819,12 +823,10 @@ fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: asm.jmp(if_true); asm.write_label(if_false); - - Some(()) } /// Compile a conditional branch to a basic block -fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) -> Option<()> { +fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { // If val is not zero, move on to the next instruction. let if_true = asm.new_label("if_true"); asm.test(val, val); @@ -837,8 +839,6 @@ fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: asm.jmp(if_false); asm.write_label(if_true); - - Some(()) } /// Compile a dynamic dispatch without block @@ -849,8 +849,8 @@ fn gen_send_without_block( state: &FrameState, self_val: Opnd, args: Vec<Opnd>, -) -> Option<lir::Opnd> { - gen_spill_locals(jit, asm, state)?; +) -> lir::Opnd { + gen_spill_locals(jit, asm, state); // Spill the receiver and the arguments onto the stack. // They need to be on the interpreter stack to let the interpreter access them. // TODO: Avoid spilling operands that have been spilled before. @@ -879,7 +879,7 @@ fn gen_send_without_block( // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with // the frame's locals - Some(ret) + ret } /// Compile a direct jump to an ISEQ call without block @@ -892,13 +892,13 @@ fn gen_send_without_block_direct( recv: Opnd, args: Vec<Opnd>, state: &FrameState, -) -> Option<lir::Opnd> { +) -> lir::Opnd { // Save cfp->pc and cfp->sp for the caller frame gen_save_pc(asm, state); gen_save_sp(asm, state.stack().len() - args.len() - 1); // -1 for receiver - gen_spill_locals(jit, asm, state)?; - gen_spill_stack(jit, asm, state)?; + gen_spill_locals(jit, asm, state); + gen_spill_stack(jit, asm, state); // Set up the new frame // TODO: Lazily materialize caller frames on side exits or when needed @@ -944,7 +944,7 @@ fn gen_send_without_block_direct( let new_sp = asm.sub(SP, sp_offset.into()); asm.mov(SP, new_sp); - Some(ret) + ret } /// Compile a string resurrection @@ -974,7 +974,7 @@ fn gen_new_array( ) -> lir::Opnd { gen_prepare_call_with_gc(asm, state); - let length: ::std::os::raw::c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + let length: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); let new_array = asm_ccall!(asm, rb_ary_new_capa, length.into()); @@ -985,22 +985,55 @@ fn gen_new_array( new_array } +/// Compile a new hash instruction +fn gen_new_hash( + jit: &mut JITState, + asm: &mut Assembler, + elements: &Vec<(InsnId, InsnId)>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let cap: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + let new_hash = asm_ccall!(asm, rb_hash_new_with_size, lir::Opnd::Imm(cap)); + + if !elements.is_empty() { + let mut pairs = Vec::new(); + for (key_id, val_id) in elements.iter() { + let key = jit.get_opnd(*key_id); + let val = jit.get_opnd(*val_id); + pairs.push(key); + pairs.push(val); + } + + let argv = gen_push_opnds(jit, asm, &pairs); + let argc = (elements.len() * 2) as ::std::os::raw::c_long; + asm_ccall!(asm, rb_hash_bulk_insert, lir::Opnd::Imm(argc), argv, new_hash); + + gen_pop_opnds(asm, &pairs); + } + + new_hash +} + /// Compile a new range instruction fn gen_new_range( + jit: &JITState, asm: &mut Assembler, low: lir::Opnd, high: lir::Opnd, flag: RangeType, state: &FrameState, ) -> lir::Opnd { - gen_prepare_call_with_gc(asm, state); + // Sometimes calls `low.<=>(high)` + gen_prepare_non_leaf_call(jit, asm, state); // Call rb_range_new(low, high, flag) asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) } /// Compile code that exits from JIT code with a return value -fn gen_return(asm: &mut Assembler, val: lir::Opnd) -> Option<()> { +fn gen_return(asm: &mut Assembler, val: lir::Opnd) { // Pop the current frame (ec->cfp++) // Note: the return PC is already in the previous CFP asm_comment!(asm, "pop stack frame"); @@ -1015,31 +1048,28 @@ fn gen_return(asm: &mut Assembler, val: lir::Opnd) -> Option<()> { // Return from the function asm.frame_teardown(&[]); // matching the setup in :bb0-prologue: asm.cret(C_RET_OPND); - Some(()) } /// Compile Fixnum + Fixnum -fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Add left + right and test for overflow let left_untag = asm.sub(left, Opnd::Imm(1)); let out_val = asm.add(left_untag, right); - asm.jo(side_exit(jit, state, FixnumAddOverflow)?); + asm.jo(side_exit(jit, state, FixnumAddOverflow)); - Some(out_val) + out_val } /// Compile Fixnum - Fixnum -fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Subtract left - right and test for overflow let val_untag = asm.sub(left, right); - asm.jo(side_exit(jit, state, FixnumSubOverflow)?); - let out_val = asm.add(val_untag, Opnd::Imm(1)); - - Some(out_val) + asm.jo(side_exit(jit, state, FixnumSubOverflow)); + asm.add(val_untag, Opnd::Imm(1)) } /// Compile Fixnum * Fixnum -fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Do some bitwise gymnastics to handle tag bits // x * y is translated to (x >> 1) * (y - 1) + 1 let left_untag = asm.rshift(left, Opnd::UImm(1)); @@ -1047,107 +1077,105 @@ fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, rig let out_val = asm.mul(left_untag, right_untag); // Test for overflow - asm.jo_mul(side_exit(jit, state, FixnumMultOverflow)?); - let out_val = asm.add(out_val, Opnd::UImm(1)); - - Some(out_val) + asm.jo_mul(side_exit(jit, state, FixnumMultOverflow)); + asm.add(out_val, Opnd::UImm(1)) } /// Compile Fixnum == Fixnum -fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_e(Qtrue.into(), Qfalse.into())) + asm.csel_e(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum != Fixnum -fn gen_fixnum_neq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_neq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_ne(Qtrue.into(), Qfalse.into())) + asm.csel_ne(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum < Fixnum -fn gen_fixnum_lt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_lt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_l(Qtrue.into(), Qfalse.into())) + asm.csel_l(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum <= Fixnum -fn gen_fixnum_le(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_le(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_le(Qtrue.into(), Qfalse.into())) + asm.csel_le(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum > Fixnum -fn gen_fixnum_gt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_gt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_g(Qtrue.into(), Qfalse.into())) + asm.csel_g(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum >= Fixnum -fn gen_fixnum_ge(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_ge(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_ge(Qtrue.into(), Qfalse.into())) + asm.csel_ge(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum & Fixnum -fn gen_fixnum_and(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { - Some(asm.and(left, right)) +fn gen_fixnum_and(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.and(left, right) } /// Compile Fixnum | Fixnum -fn gen_fixnum_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { - Some(asm.or(left, right)) +fn gen_fixnum_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.or(left, right) } // Compile val == nil -fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> Option<lir::Opnd> { +fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { asm.cmp(val, Qnil.into()); // TODO: Implement and use setcc - Some(asm.csel_e(Opnd::Imm(1), Opnd::Imm(0))) + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) } -fn gen_anytostring(asm: &mut Assembler, val: lir::Opnd, str: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_anytostring(asm: &mut Assembler, val: lir::Opnd, str: lir::Opnd, state: &FrameState) -> lir::Opnd { gen_prepare_call_with_gc(asm, state); - Some(asm_ccall!(asm, rb_obj_as_string_result, str, val)) + asm_ccall!(asm, rb_obj_as_string_result, str, val) } /// Evaluate if a value is truthy /// Produces a CBool type (0 or 1) /// In Ruby, only nil and false are falsy /// Everything else evaluates to true -fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> Option<lir::Opnd> { +fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { // Test if any bit (outside of the Qnil bit) is on // See RB_TEST(), include/ruby/internal/special_consts.h asm.test(val, Opnd::Imm(!Qnil.as_i64())); - Some(asm.csel_e(0.into(), 1.into())) + asm.csel_e(0.into(), 1.into()) } /// Compile a type check with a side exit -fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> Option<lir::Opnd> { +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { if guard_type.is_subtype(types::Fixnum) { asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); - asm.jz(side_exit(jit, state, GuardType(guard_type))?); + asm.jz(side_exit(jit, state, GuardType(guard_type))); } else if guard_type.is_subtype(types::Flonum) { // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); - asm.jne(side_exit(jit, state, GuardType(guard_type))?); + asm.jne(side_exit(jit, state, GuardType(guard_type))); } else if guard_type.is_subtype(types::StaticSymbol) { // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG - // Use 8-bit comparison like YJIT does - debug_assert!(val.try_num_bits(8).is_some(), "GuardType should not be used for a known constant, but val was: {val:?}"); - asm.cmp(val.try_num_bits(8)?, Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); - asm.jne(side_exit(jit, state, GuardType(guard_type))?); + // Use 8-bit comparison like YJIT does. GuardType should not be used + // for a known VALUE, which with_num_bits() does not support. + asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.jne(side_exit(jit, state, GuardType(guard_type))); } else if guard_type.is_subtype(types::NilClass) { asm.cmp(val, Qnil.into()); - asm.jne(side_exit(jit, state, GuardType(guard_type))?); + asm.jne(side_exit(jit, state, GuardType(guard_type))); } else if guard_type.is_subtype(types::TrueClass) { asm.cmp(val, Qtrue.into()); - asm.jne(side_exit(jit, state, GuardType(guard_type))?); + asm.jne(side_exit(jit, state, GuardType(guard_type))); } else if guard_type.is_subtype(types::FalseClass) { asm.cmp(val, Qfalse.into()); - asm.jne(side_exit(jit, state, GuardType(guard_type))?); + asm.jne(side_exit(jit, state, GuardType(guard_type))); } else if guard_type.is_immediate() { // All immediate types' guard should have been handled above panic!("unexpected immediate guard type: {guard_type}"); @@ -1162,7 +1190,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard }; // Check if it's a special constant - let side_exit = side_exit(jit, state, GuardType(guard_type))?; + let side_exit = side_exit(jit, state, GuardType(guard_type)); asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); asm.jnz(side_exit.clone()); @@ -1178,18 +1206,18 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard } else { unimplemented!("unsupported type: {guard_type}"); } - Some(val) + val } /// Compile an identity check with a side exit -fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: VALUE, state: &FrameState) -> Option<lir::Opnd> { +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: VALUE, state: &FrameState) -> lir::Opnd { asm.cmp(val, Opnd::Value(expected)); - asm.jnz(side_exit(jit, state, GuardBitEquals(expected))?); - Some(val) + asm.jnz(side_exit(jit, state, GuardBitEquals(expected))); + val } /// Generate code that increments a counter in ZJIT stats -fn gen_incr_counter(asm: &mut Assembler, counter: Counter) -> () { +fn gen_incr_counter(asm: &mut Assembler, counter: Counter) { let ptr = counter_ptr(counter); let ptr_reg = asm.load(Opnd::const_ptr(ptr as *const u8)); let counter_opnd = Opnd::mem(64, ptr_reg, 0); @@ -1221,30 +1249,27 @@ fn gen_save_sp(asm: &mut Assembler, stack_size: usize) { } /// Spill locals onto the stack. -fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) -> Option<()> { +fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // TODO: Avoid spilling locals that have been spilled before and not changed. asm_comment!(asm, "spill locals"); for (idx, &insn_id) in state.locals().enumerate() { - asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(jit.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)?); + asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(jit.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); } - Some(()) } /// Spill the virtual stack onto the stack. -fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) -> Option<()> { +fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // This function does not call gen_save_sp() at the moment because // gen_send_without_block_direct() spills stack slots above SP for arguments. asm_comment!(asm, "spill stack"); for (idx, &insn_id) in state.stack().enumerate() { - asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)?); + asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); } - Some(()) } /// Prepare for calling a C function that may call an arbitrary method. /// Use gen_prepare_call_with_gc() if the method is leaf but allocates objects. -#[must_use] -fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) -> Option<()> { +fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // TODO: Lazily materialize caller frames when needed // Save PC for backtraces and allocation tracing gen_save_pc(asm, state); @@ -1252,11 +1277,10 @@ fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameS // Save SP and spill the virtual stack in case it raises an exception // and the interpreter uses the stack for handling the exception gen_save_sp(asm, state.stack().len()); - gen_spill_stack(jit, asm, state)?; + gen_spill_stack(jit, asm, state); // Spill locals in case the method looks at caller Bindings - gen_spill_locals(jit, asm, state)?; - Some(()) + gen_spill_locals(jit, asm, state); } /// Prepare for calling a C function that may allocate objects and trigger GC. @@ -1325,11 +1349,17 @@ fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { local_size_and_idx_to_ep_offset(local_size as usize, local_idx) } -/// Convert the number of locals and a local index to an offset in the EP +/// Convert the number of locals and a local index to an offset from the EP pub fn local_size_and_idx_to_ep_offset(local_size: usize, local_idx: usize) -> i32 { local_size as i32 - local_idx as i32 - 1 + VM_ENV_DATA_SIZE as i32 } +/// Convert the number of locals and a local index to an offset from the BP. +/// We don't move the SP register after entry, so we often use SP as BP. +pub fn local_size_and_idx_to_bp_offset(local_size: usize, local_idx: usize) -> i32 { + local_size_and_idx_to_ep_offset(local_size, local_idx) + 1 +} + /// Convert ISEQ into High-level IR fn compile_iseq(iseq: IseqPtr) -> Option<Function> { let mut function = match iseq_to_hir(iseq) { @@ -1353,20 +1383,20 @@ fn compile_iseq(iseq: IseqPtr) -> Option<Function> { } /// Build a Target::SideExit for non-PatchPoint instructions -fn side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReason) -> Option<Target> { +fn side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReason) -> Target { build_side_exit(jit, state, reason, None) } /// Build a Target::SideExit out of a FrameState -fn build_side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReason, label: Option<Label>) -> Option<Target> { +fn build_side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReason, label: Option<Label>) -> Target { let mut stack = Vec::new(); for &insn_id in state.stack() { - stack.push(jit.get_opnd(insn_id)?); + stack.push(jit.get_opnd(insn_id)); } let mut locals = Vec::new(); for &insn_id in state.locals() { - locals.push(jit.get_opnd(insn_id)?); + locals.push(jit.get_opnd(insn_id)); } let target = Target::SideExit { @@ -1376,7 +1406,7 @@ fn build_side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReaso reason, label, }; - Some(target) + target } /// Return true if a given ISEQ is known to escape EP to the heap on entry. @@ -1425,26 +1455,41 @@ c_callable! { /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. - fn function_stub_hit(iseq_call_ptr: *const c_void, ec: EcPtr, sp: *mut VALUE) -> *const u8 { + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { with_vm_lock(src_loc!(), || { - // gen_push_frame() doesn't set PC and SP, so we need to set them before exit. + // gen_push_frame() doesn't set PC, so we need to set them before exit. // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. - let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; - let cfp = unsafe { get_ec_cfp(ec) }; - let pc = unsafe { rb_iseq_pc_at_idx(iseq_call.iseq, 0) }; // TODO: handle opt_pc once supported + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const RefCell<IseqCall>) }; + let iseq = iseq_call.borrow().iseq; + let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; // TODO: handle opt_pc once supported unsafe { rb_set_cfp_pc(cfp, pc) }; - unsafe { rb_set_cfp_sp(cfp, sp) }; + + // JIT-to-JIT calls don't set SP or fill nils to uninitialized (non-argument) locals. + // We need to set them if we side-exit from function_stub_hit. + fn spill_stack(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE) { + unsafe { + // Set SP which gen_push_frame() doesn't set + rb_set_cfp_sp(cfp, sp); + + // Fill nils to uninitialized (non-argument) locals + let local_size = get_iseq_body_local_table_size(iseq) as usize; + let num_params = get_iseq_body_param_size(iseq) as usize; + let base = sp.offset(-local_size_and_idx_to_bp_offset(local_size, num_params) as isize); + slice::from_raw_parts_mut(base, local_size - num_params).fill(Qnil); + } + } // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. let cb = ZJITState::get_code_block(); - let payload = get_or_create_iseq_payload(iseq_call.iseq); + let payload = get_or_create_iseq_payload(iseq); if cb.has_dropped_bytes() || payload.status == IseqStatus::CantCompile { // We'll use this Rc again, so increment the ref count decremented by from_raw. - unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const RefCell<IseqCall>); } // Exit to the interpreter + spill_stack(iseq, cfp, sp); return ZJITState::get_exit_trampoline().raw_ptr(cb); } @@ -1454,6 +1499,7 @@ c_callable! { code_ptr } else { // Exit to the interpreter + spill_stack(iseq, cfp, sp); ZJITState::get_exit_trampoline() }; cb.mark_all_executable(); @@ -1463,22 +1509,23 @@ c_callable! { } /// Compile an ISEQ for a function stub -fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &Rc<IseqCall>) -> Option<CodePtr> { +fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &Rc<RefCell<IseqCall>>) -> Option<CodePtr> { // Compile the stubbed ISEQ - let Some((code_ptr, iseq_calls)) = gen_iseq(cb, iseq_call.iseq) else { - debug!("Failed to compile iseq: gen_iseq failed: {}", iseq_get_location(iseq_call.iseq, 0)); + let Some((code_ptr, iseq_calls)) = gen_iseq(cb, iseq_call.borrow().iseq) else { + debug!("Failed to compile iseq: gen_iseq failed: {}", iseq_get_location(iseq_call.borrow().iseq, 0)); return None; }; // Stub callee ISEQs for JIT-to-JIT calls - for callee_iseq_call in iseq_calls.into_iter() { - gen_iseq_call(cb, iseq_call.iseq, callee_iseq_call)?; + for callee_iseq_call in iseq_calls.iter() { + gen_iseq_call(cb, iseq_call.borrow().iseq, callee_iseq_call)?; } // Update the stub to call the code pointer let code_addr = code_ptr.raw_ptr(cb); - iseq_call.regenerate(cb, |asm| { - asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq_call.iseq, 0)); + let iseq = iseq_call.borrow().iseq; + iseq_call.borrow_mut().regenerate(cb, |asm| { + asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq, 0)); asm.ccall(code_addr, vec![]); }); @@ -1486,9 +1533,9 @@ fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &Rc<IseqCall>) -> Optio } /// Compile a stub for an ISEQ called by SendWithoutBlockDirect -fn gen_function_stub(cb: &mut CodeBlock, iseq_call: Rc<IseqCall>) -> Option<CodePtr> { +fn gen_function_stub(cb: &mut CodeBlock, iseq_call: Rc<RefCell<IseqCall>>) -> Option<CodePtr> { let mut asm = Assembler::new(); - asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.iseq, 0)); + asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.borrow().iseq, 0)); // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. // Use load_into instead of mov, which is split on arm64, to avoid clobbering ALLOC_REGS. @@ -1516,7 +1563,7 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Option<CodePtr> { const { assert!(ALLOC_REGS.len() % 2 == 0, "x86_64 would need to push one more if we push an odd number of regs"); } // Compile the stubbed ISEQ - let jump_addr = asm_ccall!(asm, function_stub_hit, SCRATCH_OPND, EC, SP); + let jump_addr = asm_ccall!(asm, function_stub_hit, SCRATCH_OPND, CFP, SP); asm.mov(SCRATCH_OPND, jump_addr); asm_comment!(asm, "restore argument registers"); @@ -1550,16 +1597,8 @@ pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Option<CodePtr> { }) } -fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>, state: &FrameState) -> Option<Opnd> { - let n = strings.len(); - - // concatstrings shouldn't have 0 strings - // If it happens we abort the compilation for now - if n == 0 { - return None; - } - - gen_prepare_non_leaf_call(jit, asm, state)?; +fn gen_push_opnds(jit: &mut JITState, asm: &mut Assembler, opnds: &[Opnd]) -> lir::Opnd { + let n = opnds.len(); // Calculate the compile-time NATIVE_STACK_PTR offset from NATIVE_BASE_PTR // At this point, frame_setup(&[], jit.c_stack_slots) has been called, @@ -1567,28 +1606,51 @@ fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd> let frame_size = aligned_stack_bytes(jit.c_stack_slots); let allocation_size = aligned_stack_bytes(n); - asm_comment!(asm, "allocate {} bytes on C stack for {} strings", allocation_size, n); + asm_comment!(asm, "allocate {} bytes on C stack for {} values", allocation_size, n); asm.sub_into(NATIVE_STACK_PTR, allocation_size.into()); // Calculate the total offset from NATIVE_BASE_PTR to our buffer let total_offset_from_base = (frame_size + allocation_size) as i32; - for (idx, &string_opnd) in strings.iter().enumerate() { + for (idx, &opnd) in opnds.iter().enumerate() { let slot_offset = -total_offset_from_base + (idx as i32 * SIZEOF_VALUE_I32); asm.mov( Opnd::mem(VALUE_BITS, NATIVE_BASE_PTR, slot_offset), - string_opnd + opnd ); } - let first_string_ptr = asm.lea(Opnd::mem(64, NATIVE_BASE_PTR, -total_offset_from_base)); - - let result = asm_ccall!(asm, rb_str_concat_literals, n.into(), first_string_ptr); + asm.lea(Opnd::mem(64, NATIVE_BASE_PTR, -total_offset_from_base)) +} +fn gen_pop_opnds(asm: &mut Assembler, opnds: &[Opnd]) { asm_comment!(asm, "restore C stack pointer"); + let allocation_size = aligned_stack_bytes(opnds.len()); asm.add_into(NATIVE_STACK_PTR, allocation_size.into()); +} + +fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); - Some(result) + let first_opnd_ptr = gen_push_opnds(jit, asm, &values); + + let tmp_ary = asm_ccall!(asm, rb_ary_tmp_new_from_values, Opnd::Imm(0), values.len().into(), first_opnd_ptr); + let result = asm_ccall!(asm, rb_reg_new_ary, tmp_ary, opt.into()); + asm_ccall!(asm, rb_ary_clear, tmp_ary); + + gen_pop_opnds(asm, &values); + + result +} + +fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_string_ptr = gen_push_opnds(jit, asm, &strings); + let result = asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr); + gen_pop_opnds(asm, &strings); + + result } /// Given the number of spill slots needed for a function, return the number of bytes @@ -1602,7 +1664,7 @@ fn aligned_stack_bytes(num_slots: usize) -> usize { impl Assembler { /// Make a C call while marking the start and end positions for IseqCall - fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &Rc<IseqCall>) -> Opnd { + fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &Rc<RefCell<IseqCall>>) -> Opnd { // We need to create our own branch rc objects so that we can move the closure below let start_iseq_call = iseq_call.clone(); let end_iseq_call = iseq_call.clone(); @@ -1611,10 +1673,10 @@ impl Assembler { fptr, opnds, move |code_ptr, _| { - start_iseq_call.start_addr.set(Some(code_ptr)); + start_iseq_call.borrow_mut().start_addr.set(Some(code_ptr)); }, move |code_ptr, _| { - end_iseq_call.end_addr.set(Some(code_ptr)); + end_iseq_call.borrow_mut().end_addr.set(Some(code_ptr)); }, ) } @@ -1622,9 +1684,9 @@ impl Assembler { /// Store info about a JIT-to-JIT call #[derive(Debug)] -struct IseqCall { +pub struct IseqCall { /// Callee ISEQ that start_addr jumps to - iseq: IseqPtr, + pub iseq: IseqPtr, /// Position where the call instruction starts start_addr: Cell<Option<CodePtr>>, @@ -1635,12 +1697,13 @@ struct IseqCall { impl IseqCall { /// Allocate a new IseqCall - fn new(iseq: IseqPtr) -> Rc<Self> { - Rc::new(IseqCall { + fn new(iseq: IseqPtr) -> Rc<RefCell<Self>> { + let iseq_call = IseqCall { iseq, start_addr: Cell::new(None), end_addr: Cell::new(None), - }) + }; + Rc::new(RefCell::new(iseq_call)) } /// Regenerate a IseqCall with a given callback diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index eff99daf9b..3ec2954c73 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -269,7 +269,6 @@ pub type IseqPtr = *const rb_iseq_t; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ShapeId(pub u32); -pub const SPECIAL_CONST_SHAPE_ID: ShapeId = ShapeId(RB_SPECIAL_CONST_SHAPE_ID); pub const INVALID_SHAPE_ID: ShapeId = ShapeId(RB_INVALID_SHAPE_ID); // Given an ISEQ pointer, convert PC to insn_idx diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 5c939fabe7..ee6d4d5e0e 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -30,6 +30,11 @@ impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> { fmt.write_str("__IncompleteArrayField") } } +pub const ONIG_OPTION_IGNORECASE: u32 = 1; +pub const ONIG_OPTION_EXTEND: u32 = 2; +pub const ONIG_OPTION_MULTILINE: u32 = 4; +pub const ARG_ENCODING_FIXED: u32 = 16; +pub const ARG_ENCODING_NONE: u32 = 32; pub const INTEGER_REDEFINED_OP_FLAG: u32 = 1; pub const FLOAT_REDEFINED_OP_FLAG: u32 = 2; pub const STRING_REDEFINED_OP_FLAG: u32 = 4; @@ -719,7 +724,6 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; -pub const RB_SPECIAL_CONST_SHAPE_ID: _bindgen_ty_38 = 33554432; pub const RB_INVALID_SHAPE_ID: _bindgen_ty_38 = 4294967295; pub type _bindgen_ty_38 = u32; pub type rb_iseq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword; diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index 52a036d49e..3462b80232 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -1,6 +1,9 @@ // This module is responsible for marking/moving objects on GC. +use std::cell::RefCell; +use std::rc::Rc; use std::{ffi::c_void, ops::Range}; +use crate::codegen::IseqCall; use crate::{cruby::*, profile::IseqProfile, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; use crate::stats::Counter::gc_time_ns; @@ -15,6 +18,9 @@ pub struct IseqPayload { /// GC offsets of the JIT code. These are the addresses of objects that need to be marked. pub gc_offsets: Vec<CodePtr>, + + /// JIT-to-JIT calls in the ISEQ. The IseqPayload's ISEQ is the caller of it. + pub iseq_calls: Vec<Rc<RefCell<IseqCall>>>, } impl IseqPayload { @@ -23,6 +29,7 @@ impl IseqPayload { status: IseqStatus::NotCompiled, profile: IseqProfile::new(iseq_size), gc_offsets: vec![], + iseq_calls: vec![], } } } @@ -112,6 +119,16 @@ pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { with_time_stat(gc_time_ns, || iseq_update_references(payload)); } +/// GC callback for updating object references after all object moves +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_update_references() { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.update_references(); +} + fn iseq_mark(payload: &IseqPayload) { // Mark objects retained by profiling instructions payload.profile.each_object(|object| { @@ -135,10 +152,22 @@ fn iseq_mark(payload: &IseqPayload) { /// This is a mirror of [iseq_mark]. fn iseq_update_references(payload: &mut IseqPayload) { // Move objects retained by profiling instructions - payload.profile.each_object_mut(|object| { - *object = unsafe { rb_gc_location(*object) }; + payload.profile.each_object_mut(|old_object| { + let new_object = unsafe { rb_gc_location(*old_object) }; + if *old_object != new_object { + *old_object = new_object; + } }); + // Move ISEQ references in IseqCall + for iseq_call in payload.iseq_calls.iter_mut() { + let old_iseq = iseq_call.borrow().iseq; + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.borrow_mut().iseq = new_iseq; + } + } + // Move objects baked in JIT code let cb = ZJITState::get_code_block(); for &offset in payload.gc_offsets.iter() { diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index c50c1ce985..e7aaf64f28 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -439,6 +439,7 @@ pub enum SideExitReason { CalleeSideExit, ObjToStringFallback, UnknownSpecialVariable(u64), + UnhandledDefinedType(usize), } impl std::fmt::Display for SideExitReason { @@ -472,6 +473,9 @@ pub enum Insn { StringIntern { val: InsnId, state: InsnId }, StringConcat { strings: Vec<InsnId>, state: InsnId }, + /// Combine count stack values into a regexp + ToRegexp { opt: usize, values: Vec<InsnId>, state: InsnId }, + /// Put special object (VMCORE, CBASE, etc.) based on value_type PutSpecialObject { value_type: SpecialObjectType }, @@ -634,7 +638,6 @@ impl Insn { // NewHash's operands may be hashed and compared for equality, which could have // side-effects. Insn::NewHash { elements, .. } => elements.len() > 0, - Insn::NewRange { .. } => false, Insn::ArrayDup { .. } => false, Insn::HashDup { .. } => false, Insn::Test { .. } => false, @@ -656,6 +659,9 @@ impl Insn { Insn::GetLocal { .. } => false, Insn::IsNil { .. } => false, Insn::CCall { elidable, .. } => !elidable, + // TODO: NewRange is effects free if we can prove the two ends to be Fixnum, + // but we don't have type information here in `impl Insn`. See rb_range_new(). + Insn::NewRange { .. } => true, _ => true, } } @@ -667,6 +673,14 @@ pub struct InsnPrinter<'a> { ptr_map: &'a PtrPrintMap, } +static REGEXP_FLAGS: &[(u32, &str)] = &[ + (ONIG_OPTION_MULTILINE, "MULTILINE"), + (ONIG_OPTION_IGNORECASE, "IGNORECASE"), + (ONIG_OPTION_EXTEND, "EXTENDED"), + (ARG_ENCODING_FIXED, "FIXEDENCODING"), + (ARG_ENCODING_NONE, "NOENCODING"), +]; + impl<'a> std::fmt::Display for InsnPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match &self.inner { @@ -715,6 +729,28 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Ok(()) } + Insn::ToRegexp { values, opt, .. } => { + write!(f, "ToRegexp")?; + let mut prefix = " "; + for value in values { + write!(f, "{prefix}{value}")?; + prefix = ", "; + } + + let opt = *opt as u32; + if opt != 0 { + write!(f, ", ")?; + let mut sep = ""; + for (flag, name) in REGEXP_FLAGS { + if opt & flag != 0 { + write!(f, "{sep}{name}")?; + sep = "|"; + } + } + } + + Ok(()) + } Insn::Test { val } => { write!(f, "Test {val}") } Insn::IsNil { val } => { write!(f, "IsNil {val}") } Insn::Jump(target) => { write!(f, "Jump {target}") } @@ -1178,6 +1214,7 @@ impl Function { &StringCopy { val, chilled, state } => StringCopy { val: find!(val), chilled, state }, &StringIntern { val, state } => StringIntern { val: find!(val), state: find!(state) }, &StringConcat { ref strings, state } => StringConcat { strings: find_vec!(strings), state: find!(state) }, + &ToRegexp { opt, ref values, state } => ToRegexp { opt, values: find_vec!(values), state }, &Test { val } => Test { val: find!(val) }, &IsNil { val } => IsNil { val: find!(val) }, &Jump(ref target) => Jump(find_branch_edge!(target)), @@ -1304,6 +1341,7 @@ impl Function { Insn::StringCopy { .. } => types::StringExact, Insn::StringIntern { .. } => types::Symbol, Insn::StringConcat { .. } => types::StringExact, + Insn::ToRegexp { .. } => types::RegexpExact, Insn::NewArray { .. } => types::ArrayExact, Insn::ArrayDup { .. } => types::ArrayExact, Insn::NewHash { .. } => types::HashExact, @@ -1330,7 +1368,7 @@ impl Function { Insn::SendWithoutBlockDirect { .. } => types::BasicObject, Insn::Send { .. } => types::BasicObject, Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), - Insn::Defined { .. } => types::BasicObject, + Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), Insn::DefinedIvar { .. } => types::BasicObject, Insn::GetConstantPath { .. } => types::BasicObject, Insn::ArrayMax { .. } => types::BasicObject, @@ -1938,14 +1976,18 @@ impl Function { worklist.extend(strings); worklist.push_back(state); } + &Insn::ToRegexp { ref values, state, .. } => { + worklist.extend(values); + worklist.push_back(state); + } | &Insn::Return { val } | &Insn::Throw { val, .. } - | &Insn::Defined { v: val, .. } | &Insn::Test { val } | &Insn::SetLocal { val, .. } | &Insn::IsNil { val } => worklist.push_back(val), &Insn::SetGlobal { val, state, .. } + | &Insn::Defined { v: val, state, .. } | &Insn::StringIntern { val, state } | &Insn::StringCopy { val, state, .. } | &Insn::GuardType { val, state, .. } @@ -2862,6 +2904,15 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let insn_id = fun.push_insn(block, Insn::StringConcat { strings, state: exit_id }); state.stack_push(insn_id); } + YARVINSN_toregexp => { + // First arg contains the options (multiline, extended, ignorecase) used to create the regexp + let opt = get_arg(pc, 0).as_usize(); + let count = get_arg(pc, 1).as_usize(); + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + let values = state.stack_pop_n(count)?; + let insn_id = fun.push_insn(block, Insn::ToRegexp { opt, values, state: exit_id }); + state.stack_push(insn_id); + } YARVINSN_newarray => { let count = get_arg(pc, 0).as_usize(); let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); @@ -2957,6 +3008,11 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let pushval = get_arg(pc, 2); let v = state.stack_pop()?; let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + if op_type == DEFINED_METHOD.try_into().unwrap() { + // TODO(Shopify/ruby#703): Fix codegen for defined?(method call expr) + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDefinedType(op_type)}); + break; // End the block + } state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v, state: exit_id })); } YARVINSN_definedivar => { @@ -4204,10 +4260,10 @@ mod tests { fn test@<compiled>:2: bb0(v0:BasicObject): v2:NilClass = Const Value(nil) - v4:BasicObject = Defined constant, v2 - v6:BasicObject = Defined func, v0 + v4:StringExact|NilClass = Defined constant, v2 + v6:StringExact|NilClass = Defined func, v0 v7:NilClass = Const Value(nil) - v9:BasicObject = Defined global-variable, v7 + v9:StringExact|NilClass = Defined global-variable, v7 v11:ArrayExact = NewArray v4, v6, v9 Return v11 "#]]); @@ -5325,6 +5381,47 @@ mod tests { } #[test] + fn test_toregexp() { + eval(r##" + def test = /#{1}#{2}#{3}/ + "##); + assert_method_hir_with_opcode("test", YARVINSN_toregexp, expect![[r#" + fn test@<compiled>:2: + bb0(v0:BasicObject): + v2:Fixnum[1] = Const Value(1) + v4:BasicObject = ObjToString v2 + v6:String = AnyToString v2, str: v4 + v7:Fixnum[2] = Const Value(2) + v9:BasicObject = ObjToString v7 + v11:String = AnyToString v7, str: v9 + v12:Fixnum[3] = Const Value(3) + v14:BasicObject = ObjToString v12 + v16:String = AnyToString v12, str: v14 + v18:RegexpExact = ToRegexp v6, v11, v16 + Return v18 + "#]]); + } + + #[test] + fn test_toregexp_with_options() { + eval(r##" + def test = /#{1}#{2}/mixn + "##); + assert_method_hir_with_opcode("test", YARVINSN_toregexp, expect![[r#" + fn test@<compiled>:2: + bb0(v0:BasicObject): + v2:Fixnum[1] = Const Value(1) + v4:BasicObject = ObjToString v2 + v6:String = AnyToString v2, str: v4 + v7:Fixnum[2] = Const Value(2) + v9:BasicObject = ObjToString v7 + v11:String = AnyToString v7, str: v9 + v13:RegexpExact = ToRegexp v6, v11, MULTILINE|IGNORECASE|EXTENDED|NOENCODING + Return v13 + "#]]); + } + + #[test] fn throw() { eval(" define_method(:throw_return) { return 1 } @@ -6104,6 +6201,29 @@ mod opt_tests { } #[test] + fn test_do_not_eliminate_new_range_non_fixnum() { + eval(" + def test() + _ = (-'a'..'b') + 0 + end + test; test + "); + assert_optimized_method_hir("test", expect![[r#" + fn test@<compiled>:3: + bb0(v0:BasicObject): + v1:NilClass = Const Value(nil) + v4:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v6:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v8:StringExact = StringCopy v6 + v10:RangeExact = NewRange v4 NewRangeInclusive v8 + v11:Fixnum[0] = Const Value(0) + Return v11 + "#]]); + } + + #[test] fn test_eliminate_new_array_with_elements() { eval(" def test(a) diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index 3f291415be..14fea76d1b 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -1,6 +1,6 @@ use std::{collections::{HashMap, HashSet}, mem}; -use crate::{backend::lir::{asm_comment, Assembler}, cruby::{rb_callable_method_entry_t, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID}, gc::IseqPayload, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr}; +use crate::{backend::lir::{asm_comment, Assembler}, cruby::{rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID, VALUE}, gc::IseqPayload, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr}; use crate::stats::with_time_stat; use crate::stats::Counter::invalidation_time_ns; use crate::gc::remove_gc_offsets; @@ -56,6 +56,31 @@ pub struct Invariants { single_ractor_patch_points: HashSet<PatchPoint>, } +impl Invariants { + /// Update object references in Invariants + pub fn update_references(&mut self) { + Self::update_iseq_references(&mut self.ep_escape_iseqs); + Self::update_iseq_references(&mut self.no_ep_escape_iseqs); + } + + /// Update ISEQ references in a given HashSet<IseqPtr> + fn update_iseq_references(iseqs: &mut HashSet<IseqPtr>) { + let mut moved: Vec<IseqPtr> = Vec::with_capacity(iseqs.len()); + + iseqs.retain(|&old_iseq| { + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + moved.push(new_iseq); + } + old_iseq == new_iseq + }); + + for new_iseq in moved { + iseqs.insert(new_iseq); + } + } +} + /// Called when a basic operator is redefined. Note that all the blocks assuming /// the stability of different operators are invalidated together and we don't /// do fine-grained tracking. diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 7ffaea29dc..771d90cb0e 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -98,19 +98,32 @@ fn profile_operands(profiler: &mut Profiler, profile: &mut IseqProfile, n: usize let obj = profiler.peek_at_stack((n - i - 1) as isize); // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or // drop them or something - let ty = ProfiledType::new(obj.class_of(), obj.shape_id_of()); + let ty = ProfiledType::new(obj); unsafe { rb_gc_writebarrier(profiler.iseq.into(), ty.class()) }; types[i].observe(ty); } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Flags(u32); + +impl Flags { + const NONE: u32 = 0; + const IS_IMMEDIATE: u32 = 1 << 0; + + pub fn none() -> Self { Self(Self::NONE) } + + pub fn immediate() -> Self { Self(Self::IS_IMMEDIATE) } + pub fn is_immediate(self) -> bool { (self.0 & Self::IS_IMMEDIATE) != 0 } +} + /// opt_send_without_block/opt_plus/... should store: /// * the class of the receiver, so we can do method lookup /// * the shape of the receiver, so we can optimize ivar lookup /// with those two, pieces of information, we can also determine when an object is an immediate: -/// * Integer + SPECIAL_CONST_SHAPE_ID == Fixnum -/// * Float + SPECIAL_CONST_SHAPE_ID == Flonum -/// * Symbol + SPECIAL_CONST_SHAPE_ID == StaticSymbol +/// * Integer + IS_IMMEDIATE == Fixnum +/// * Float + IS_IMMEDIATE == Flonum +/// * Symbol + IS_IMMEDIATE == StaticSymbol /// * NilClass == Nil /// * TrueClass == True /// * FalseClass == False @@ -118,6 +131,7 @@ fn profile_operands(profiler: &mut Profiler, profile: &mut IseqProfile, n: usize pub struct ProfiledType { class: VALUE, shape: ShapeId, + flags: Flags, } impl Default for ProfiledType { @@ -127,12 +141,42 @@ impl Default for ProfiledType { } impl ProfiledType { - fn new(class: VALUE, shape: ShapeId) -> Self { - Self { class, shape } + fn new(obj: VALUE) -> Self { + if obj == Qfalse { + return Self { class: unsafe { rb_cFalseClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qtrue { + return Self { class: unsafe { rb_cTrueClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qnil { + return Self { class: unsafe { rb_cNilClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.fixnum_p() { + return Self { class: unsafe { rb_cInteger }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.flonum_p() { + return Self { class: unsafe { rb_cFloat }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.static_sym_p() { + return Self { class: unsafe { rb_cSymbol }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + Self { class: obj.class_of(), shape: obj.shape_id_of(), flags: Flags::none() } } pub fn empty() -> Self { - Self { class: VALUE(0), shape: INVALID_SHAPE_ID } + Self { class: VALUE(0), shape: INVALID_SHAPE_ID, flags: Flags::none() } } pub fn is_empty(&self) -> bool { @@ -148,27 +192,27 @@ impl ProfiledType { } pub fn is_fixnum(&self) -> bool { - self.class == unsafe { rb_cInteger } && self.shape == SPECIAL_CONST_SHAPE_ID + self.class == unsafe { rb_cInteger } && self.flags.is_immediate() } pub fn is_flonum(&self) -> bool { - self.class == unsafe { rb_cFloat } && self.shape == SPECIAL_CONST_SHAPE_ID + self.class == unsafe { rb_cFloat } && self.flags.is_immediate() } pub fn is_static_symbol(&self) -> bool { - self.class == unsafe { rb_cSymbol } && self.shape == SPECIAL_CONST_SHAPE_ID + self.class == unsafe { rb_cSymbol } && self.flags.is_immediate() } pub fn is_nil(&self) -> bool { - self.class == unsafe { rb_cNilClass } && self.shape == SPECIAL_CONST_SHAPE_ID + self.class == unsafe { rb_cNilClass } && self.flags.is_immediate() } pub fn is_true(&self) -> bool { - self.class == unsafe { rb_cTrueClass } && self.shape == SPECIAL_CONST_SHAPE_ID + self.class == unsafe { rb_cTrueClass } && self.flags.is_immediate() } pub fn is_false(&self) -> bool { - self.class == unsafe { rb_cFalseClass } && self.shape == SPECIAL_CONST_SHAPE_ID + self.class == unsafe { rb_cFalseClass } && self.flags.is_immediate() } } |