|
| 1 | +use cranelift::codegen::ir::UserFuncName; |
| 2 | +use cranelift::prelude::*; |
| 3 | +use std::ffi::c_void; |
| 4 | + |
| 5 | +#[repr(C)] |
| 6 | +pub struct FastFFIFunction { |
| 7 | + pub code: *const c_void, |
| 8 | + pub handle: *mut c_void, |
| 9 | +} |
| 10 | + |
| 11 | +#[derive(Clone, Copy, Eq, PartialEq)] |
| 12 | +#[repr(u8)] |
| 13 | +enum FastFFIType { |
| 14 | + Void = 0, |
| 15 | + I8 = 1, |
| 16 | + U8 = 2, |
| 17 | + I16 = 3, |
| 18 | + U16 = 4, |
| 19 | + I32 = 5, |
| 20 | + U32 = 6, |
| 21 | + I64 = 7, |
| 22 | + U64 = 8, |
| 23 | + F32 = 9, |
| 24 | + F64 = 10, |
| 25 | + Pointer = 11, |
| 26 | + Buffer = 12, |
| 27 | +} |
| 28 | + |
| 29 | +impl FastFFIType { |
| 30 | + fn from_raw(raw: u8) -> Option<Self> { |
| 31 | + match raw { |
| 32 | + 0 => Some(Self::Void), |
| 33 | + 1 => Some(Self::I8), |
| 34 | + 2 => Some(Self::U8), |
| 35 | + 3 => Some(Self::I16), |
| 36 | + 4 => Some(Self::U16), |
| 37 | + 5 => Some(Self::I32), |
| 38 | + 6 => Some(Self::U32), |
| 39 | + 7 => Some(Self::I64), |
| 40 | + 8 => Some(Self::U64), |
| 41 | + 9 => Some(Self::F32), |
| 42 | + 10 => Some(Self::F64), |
| 43 | + 11 => Some(Self::Pointer), |
| 44 | + 12 => Some(Self::Buffer), |
| 45 | + _ => None, |
| 46 | + } |
| 47 | + } |
| 48 | +} |
| 49 | + |
| 50 | +fn convert(ty: FastFFIType, wrapper: bool, isize_ty: Type) -> AbiParam { |
| 51 | + match ty { |
| 52 | + FastFFIType::Void => AbiParam::new(types::INVALID), |
| 53 | + FastFFIType::I8 => { |
| 54 | + if wrapper { |
| 55 | + AbiParam::new(types::I32) |
| 56 | + } else { |
| 57 | + AbiParam::new(types::I8).sext() |
| 58 | + } |
| 59 | + } |
| 60 | + FastFFIType::U8 => { |
| 61 | + if wrapper { |
| 62 | + AbiParam::new(types::I32) |
| 63 | + } else { |
| 64 | + AbiParam::new(types::I8).uext() |
| 65 | + } |
| 66 | + } |
| 67 | + FastFFIType::I16 => { |
| 68 | + if wrapper { |
| 69 | + AbiParam::new(types::I32) |
| 70 | + } else { |
| 71 | + AbiParam::new(types::I16).sext() |
| 72 | + } |
| 73 | + } |
| 74 | + FastFFIType::U16 => { |
| 75 | + if wrapper { |
| 76 | + AbiParam::new(types::I32) |
| 77 | + } else { |
| 78 | + AbiParam::new(types::I16).uext() |
| 79 | + } |
| 80 | + } |
| 81 | + FastFFIType::I32 | FastFFIType::U32 => AbiParam::new(types::I32), |
| 82 | + FastFFIType::I64 | FastFFIType::U64 => AbiParam::new(types::I64), |
| 83 | + FastFFIType::F32 => AbiParam::new(types::F32), |
| 84 | + FastFFIType::F64 => AbiParam::new(types::F64), |
| 85 | + FastFFIType::Pointer => AbiParam::new(isize_ty), |
| 86 | + FastFFIType::Buffer => { |
| 87 | + if wrapper { |
| 88 | + AbiParam::new(isize_ty) |
| 89 | + } else { |
| 90 | + AbiParam::new(isize_ty) |
| 91 | + } |
| 92 | + } |
| 93 | + } |
| 94 | +} |
| 95 | + |
| 96 | +#[no_mangle] |
| 97 | +pub unsafe extern "C" fn node_fast_ffi_compile( |
| 98 | + target_slot: *const c_void, |
| 99 | + result_type: u8, |
| 100 | + parameter_types: *const u8, |
| 101 | + parameter_count: usize, |
| 102 | + buffer_data: *const c_void, |
| 103 | + raise_invalid_buffer: *const c_void, |
| 104 | + raise_library_closed: *const c_void, |
| 105 | + out: *mut FastFFIFunction, |
| 106 | +) -> bool { |
| 107 | + if target_slot.is_null() || out.is_null() || parameter_types.is_null() { |
| 108 | + return false; |
| 109 | + } |
| 110 | + |
| 111 | + let result_type = match FastFFIType::from_raw(result_type) { |
| 112 | + Some(value) => value, |
| 113 | + None => return false, |
| 114 | + }; |
| 115 | + |
| 116 | + let parameter_types = unsafe { std::slice::from_raw_parts(parameter_types, parameter_count) }; |
| 117 | + let mut params = Vec::with_capacity(parameter_count); |
| 118 | + for raw in parameter_types { |
| 119 | + let Some(param) = FastFFIType::from_raw(*raw) else { |
| 120 | + return false; |
| 121 | + }; |
| 122 | + params.push(param); |
| 123 | + } |
| 124 | + |
| 125 | + let Ok(compiled) = compile( |
| 126 | + target_slot, |
| 127 | + result_type, |
| 128 | + ¶ms, |
| 129 | + buffer_data, |
| 130 | + raise_invalid_buffer, |
| 131 | + raise_library_closed, |
| 132 | + ) else { |
| 133 | + return false; |
| 134 | + }; |
| 135 | + |
| 136 | + unsafe { |
| 137 | + *out = compiled; |
| 138 | + } |
| 139 | + true |
| 140 | +} |
| 141 | + |
| 142 | +#[no_mangle] |
| 143 | +pub unsafe extern "C" fn node_fast_ffi_free(handle: *mut c_void) { |
| 144 | + if handle.is_null() { |
| 145 | + return; |
| 146 | + } |
| 147 | + unsafe { |
| 148 | + drop(Box::from_raw(handle as *mut memmap2::Mmap)); |
| 149 | + } |
| 150 | +} |
| 151 | + |
| 152 | +fn compile( |
| 153 | + target_slot: *const c_void, |
| 154 | + result_type: FastFFIType, |
| 155 | + parameter_types: &[FastFFIType], |
| 156 | + buffer_data: *const c_void, |
| 157 | + raise_invalid_buffer: *const c_void, |
| 158 | + raise_library_closed: *const c_void, |
| 159 | +) -> Result<FastFFIFunction, Box<dyn std::error::Error>> { |
| 160 | + let mut flag_builder = settings::builder(); |
| 161 | + flag_builder.set("is_pic", "true")?; |
| 162 | + flag_builder.set("opt_level", "speed_and_size")?; |
| 163 | + let flags = settings::Flags::new(flag_builder); |
| 164 | + let isa = cranelift_native::builder() |
| 165 | + .map_err(|err| format!("Cranelift native ISA error: {err}"))? |
| 166 | + .finish(flags)?; |
| 167 | + |
| 168 | + #[cfg(target_pointer_width = "32")] |
| 169 | + let isize_ty = types::I32; |
| 170 | + #[cfg(target_pointer_width = "64")] |
| 171 | + let isize_ty = types::I64; |
| 172 | + |
| 173 | + let mut wrapper_sig = cranelift::codegen::ir::Signature::new(isa.default_call_conv()); |
| 174 | + let mut target_sig = cranelift::codegen::ir::Signature::new(isa.default_call_conv()); |
| 175 | + let mut buffer_sig = cranelift::codegen::ir::Signature::new(isa.default_call_conv()); |
| 176 | + let mut raise_sig = cranelift::codegen::ir::Signature::new(isa.default_call_conv()); |
| 177 | + let mut raise_closed_sig = cranelift::codegen::ir::Signature::new(isa.default_call_conv()); |
| 178 | + |
| 179 | + // V8 receiver. |
| 180 | + wrapper_sig.params.push(AbiParam::new(isize_ty)); |
| 181 | + for param in parameter_types { |
| 182 | + wrapper_sig.params.push(convert(*param, true, isize_ty)); |
| 183 | + target_sig.params.push(convert(*param, false, isize_ty)); |
| 184 | + } |
| 185 | + // FastApiCallbackOptions&. |
| 186 | + wrapper_sig.params.push(AbiParam::new(isize_ty)); |
| 187 | + |
| 188 | + if result_type != FastFFIType::Void { |
| 189 | + wrapper_sig |
| 190 | + .returns |
| 191 | + .push(convert(result_type, true, isize_ty)); |
| 192 | + target_sig |
| 193 | + .returns |
| 194 | + .push(convert(result_type, false, isize_ty)); |
| 195 | + } |
| 196 | + |
| 197 | + buffer_sig.params.push(AbiParam::new(isize_ty)); |
| 198 | + buffer_sig.returns.push(AbiParam::new(isize_ty)); |
| 199 | + |
| 200 | + raise_sig.params.push(AbiParam::new(isize_ty)); |
| 201 | + raise_sig.params.push(AbiParam::new(types::I32)); |
| 202 | + |
| 203 | + raise_closed_sig.params.push(AbiParam::new(isize_ty)); |
| 204 | + |
| 205 | + let mut ctx = cranelift::codegen::Context::new(); |
| 206 | + let mut fn_builder_ctx = FunctionBuilderContext::new(); |
| 207 | + ctx.func = cranelift::codegen::ir::Function::with_name_signature( |
| 208 | + UserFuncName::testcase("node_fast_ffi_wrapper".to_string()), |
| 209 | + wrapper_sig, |
| 210 | + ); |
| 211 | + |
| 212 | + let mut f = FunctionBuilder::new(&mut ctx.func, &mut fn_builder_ctx); |
| 213 | + let target_sig = f.import_signature(target_sig); |
| 214 | + let buffer_sig = f.import_signature(buffer_sig); |
| 215 | + let raise_sig = f.import_signature(raise_sig); |
| 216 | + let raise_closed_sig = f.import_signature(raise_closed_sig); |
| 217 | + |
| 218 | + let entry = f.create_block(); |
| 219 | + let error = f.create_block(); |
| 220 | + let closed_error = f.create_block(); |
| 221 | + f.append_block_param(error, types::I32); |
| 222 | + f.set_cold_block(error); |
| 223 | + f.set_cold_block(closed_error); |
| 224 | + f.append_block_params_for_function_params(entry); |
| 225 | + f.switch_to_block(entry); |
| 226 | + f.seal_block(entry); |
| 227 | + |
| 228 | + let wrapper_args = f.block_params(entry).to_owned(); |
| 229 | + let options = wrapper_args[parameter_types.len() + 1]; |
| 230 | + let mut native_args = Vec::with_capacity(parameter_types.len()); |
| 231 | + let mut next = f.create_block(); |
| 232 | + |
| 233 | + for (index, param) in parameter_types.iter().enumerate() { |
| 234 | + let arg = wrapper_args[index + 1]; |
| 235 | + match param { |
| 236 | + FastFFIType::I8 | FastFFIType::U8 => { |
| 237 | + native_args.push(f.ins().ireduce(types::I8, arg)); |
| 238 | + } |
| 239 | + FastFFIType::I16 | FastFFIType::U16 => { |
| 240 | + native_args.push(f.ins().ireduce(types::I16, arg)); |
| 241 | + } |
| 242 | + FastFFIType::Buffer => { |
| 243 | + if buffer_data.is_null() { |
| 244 | + return Err("missing buffer helper".into()); |
| 245 | + } |
| 246 | + let callee = f.ins().iconst(isize_ty, buffer_data as i64); |
| 247 | + let call = f.ins().call_indirect(buffer_sig, callee, &[arg]); |
| 248 | + let ptr = f.inst_results(call)[0]; |
| 249 | + native_args.push(ptr); |
| 250 | + |
| 251 | + let sentinel = f.ins().iconst(isize_ty, -1); |
| 252 | + let invalid = f.ins().icmp(IntCC::Equal, ptr, sentinel); |
| 253 | + let index = f.ins().iconst(types::I32, index as i64); |
| 254 | + f.ins().brif(invalid, error, &[index], next, &[]); |
| 255 | + f.switch_to_block(next); |
| 256 | + f.seal_block(next); |
| 257 | + next = f.create_block(); |
| 258 | + } |
| 259 | + _ => native_args.push(arg), |
| 260 | + } |
| 261 | + } |
| 262 | + |
| 263 | + let target_slot = f.ins().iconst(isize_ty, target_slot as i64); |
| 264 | + let callee = f.ins().load(isize_ty, MemFlags::trusted(), target_slot, 0); |
| 265 | + let null = f.ins().iconst(isize_ty, 0); |
| 266 | + let is_closed = f.ins().icmp(IntCC::Equal, callee, null); |
| 267 | + let call_target = f.create_block(); |
| 268 | + f.ins().brif(is_closed, closed_error, &[], call_target, &[]); |
| 269 | + f.switch_to_block(call_target); |
| 270 | + f.seal_block(call_target); |
| 271 | + |
| 272 | + let call = f.ins().call_indirect(target_sig, callee, &native_args); |
| 273 | + let mut results = f.inst_results(call).to_owned(); |
| 274 | + match result_type { |
| 275 | + FastFFIType::I8 | FastFFIType::I16 => { |
| 276 | + results[0] = f.ins().sextend(types::I32, results[0]); |
| 277 | + } |
| 278 | + FastFFIType::U8 | FastFFIType::U16 => { |
| 279 | + results[0] = f.ins().uextend(types::I32, results[0]); |
| 280 | + } |
| 281 | + _ => {} |
| 282 | + } |
| 283 | + f.ins().return_(&results); |
| 284 | + |
| 285 | + f.switch_to_block(closed_error); |
| 286 | + f.seal_block(closed_error); |
| 287 | + if !raise_library_closed.is_null() { |
| 288 | + let callee = f.ins().iconst(isize_ty, raise_library_closed as i64); |
| 289 | + f.ins().call_indirect(raise_closed_sig, callee, &[options]); |
| 290 | + } |
| 291 | + if result_type == FastFFIType::Void { |
| 292 | + f.ins().return_(&[]); |
| 293 | + } else { |
| 294 | + let ret = convert(result_type, true, isize_ty).value_type; |
| 295 | + let zero = if ret == types::F32 { |
| 296 | + f.ins().f32const(0.0) |
| 297 | + } else if ret == types::F64 { |
| 298 | + f.ins().f64const(0.0) |
| 299 | + } else { |
| 300 | + f.ins().iconst(ret, 0) |
| 301 | + }; |
| 302 | + f.ins().return_(&[zero]); |
| 303 | + } |
| 304 | + |
| 305 | + f.switch_to_block(error); |
| 306 | + f.seal_block(error); |
| 307 | + let error_index = f.block_params(error)[0]; |
| 308 | + if result_type == FastFFIType::Void { |
| 309 | + if !raise_invalid_buffer.is_null() { |
| 310 | + let callee = f.ins().iconst(isize_ty, raise_invalid_buffer as i64); |
| 311 | + f.ins() |
| 312 | + .call_indirect(raise_sig, callee, &[options, error_index]); |
| 313 | + } |
| 314 | + f.ins().return_(&[]); |
| 315 | + } else { |
| 316 | + if !raise_invalid_buffer.is_null() { |
| 317 | + let callee = f.ins().iconst(isize_ty, raise_invalid_buffer as i64); |
| 318 | + f.ins() |
| 319 | + .call_indirect(raise_sig, callee, &[options, error_index]); |
| 320 | + } |
| 321 | + let ret = convert(result_type, true, isize_ty).value_type; |
| 322 | + let zero = if ret == types::F32 { |
| 323 | + f.ins().f32const(0.0) |
| 324 | + } else if ret == types::F64 { |
| 325 | + f.ins().f64const(0.0) |
| 326 | + } else { |
| 327 | + f.ins().iconst(ret, 0) |
| 328 | + }; |
| 329 | + f.ins().return_(&[zero]); |
| 330 | + } |
| 331 | + |
| 332 | + f.finalize(); |
| 333 | + |
| 334 | + cranelift::codegen::verifier::verify_function(&ctx.func, isa.flags())?; |
| 335 | + let mut ctrl_plane = Default::default(); |
| 336 | + ctx.optimize(&*isa, &mut ctrl_plane)?; |
| 337 | + let code_info = ctx |
| 338 | + .compile(&*isa, &mut ctrl_plane) |
| 339 | + .map_err(|err| format!("Cranelift compile error: {err:?}"))?; |
| 340 | + |
| 341 | + let data = code_info.buffer.data(); |
| 342 | + let mut mutable = memmap2::MmapMut::map_anon(data.len())?; |
| 343 | + mutable.copy_from_slice(data); |
| 344 | + let executable = mutable.make_exec()?; |
| 345 | + let code = executable.as_ptr() as *const c_void; |
| 346 | + let handle = Box::into_raw(Box::new(executable)) as *mut c_void; |
| 347 | + |
| 348 | + Ok(FastFFIFunction { code, handle }) |
| 349 | +} |
0 commit comments