Ruby: rjit.c Source File

Ruby 3.4.4p34 (2025-05-14 revision a38531fd3f617bf734ef7d6c595325f69985ea1d)

rjit.c
1/**********************************************************************
2
3 rjit.c - Ruby JIT compiler functions
4
5 Copyright (C) 2023 Takashi Kokubun <k0kubun@ruby-lang.org>.
6
7**********************************************************************/
8
9#include "rjit.h" // defines USE_RJIT
10
11#if USE_RJIT
12
13#include "constant.h"
14#include "id_table.h"
15#include "internal.h"
16#include "internal/class.h"
17#include "internal/cmdlineopt.h"
18#include "internal/cont.h"
19#include "internal/file.h"
20#include "internal/hash.h"
21#include "internal/process.h"
22#include "internal/warnings.h"
23#include "vm_sync.h"
24#include "ractor_core.h"
25
26#ifdef __sun
27#define __EXTENSIONS__ 1
28#endif
29
30#include "vm_core.h"
31#include "vm_callinfo.h"
32#include "rjit_c.h"
33#include "ruby_assert.h"
34#include "ruby/debug.h"
35#include "ruby/thread.h"
36#include "ruby/version.h"
37#include "builtin.h"
38#include "insns.inc"
39#include "insns_info.inc"
40#include "internal/compile.h"
41#include "internal/gc.h"
42
43#include <sys/wait.h>
44#include <sys/time.h>
45#include <dlfcn.h>
46#include <errno.h>
47#ifdef HAVE_FCNTL_H
48#include <fcntl.h>
49#endif
50#ifdef HAVE_SYS_PARAM_H
51# include <sys/param.h>
52#endif
53#include "dln.h"
54
55// For mmapp(), sysconf()
56#ifndef _WIN32
57#include <unistd.h>
58#include <sys/mman.h>
59#endif
60
61#include "ruby/util.h"
62
63// A copy of RJIT portion of MRI options since RJIT initialization. We
64// need them as RJIT threads still can work when the most MRI data were
65// freed.
66struct rb_rjit_options rb_rjit_opts;
67
68// true if RJIT is enabled.
69bool rb_rjit_enabled = false;
70// true if --rjit-stats (used before rb_rjit_opts is set)
71bool rb_rjit_stats_enabled = false;
72// true if --rjit-trace-exits (used before rb_rjit_opts is set)
73bool rb_rjit_trace_exits_enabled = false;
74// true if JIT-ed code should be called. When `ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS`
75// and `rb_rjit_call_p == false`, any JIT-ed code execution is cancelled as soon as possible.
76bool rb_rjit_call_p = false;
77// A flag to communicate that rb_rjit_call_p should be disabled while it's temporarily false.
78static bool rjit_cancel_p = false;
79
80// `rb_ec_ractor_hooks(ec)->events` is moved to this variable during compilation.
81 rb_event_flag_t rb_rjit_global_events = 0;
82
83// Basically rb_rjit_opts.stats, but this becomes false during RJIT compilation.
84static bool rjit_stats_p = false;
85
86// RubyVM::RJIT
87static VALUE rb_mRJIT = 0;
88// RubyVM::RJIT::C
89static VALUE rb_mRJITC = 0;
90// RubyVM::RJIT::Compiler
91static VALUE rb_RJITCompiler = 0;
92// RubyVM::RJIT::CPointer::Struct_rb_iseq_t
93static VALUE rb_cRJITIseqPtr = 0;
94// RubyVM::RJIT::CPointer::Struct_rb_control_frame_t
95static VALUE rb_cRJITCfpPtr = 0;
96// RubyVM::RJIT::Hooks
97static VALUE rb_mRJITHooks = 0;
98
99// Frames for --rjit-trace-exits
100 VALUE rb_rjit_raw_samples = 0;
101// Line numbers for --rjit-trace-exits
102 VALUE rb_rjit_line_samples = 0;
103
104// Postponed job handle for triggering rjit_iseq_update_references
105static rb_postponed_job_handle_t rjit_iseq_update_references_pjob;
106
107// A default threshold used to add iseq to JIT.
108#define DEFAULT_CALL_THRESHOLD 10
109// Size of executable memory block in MiB.
110#define DEFAULT_EXEC_MEM_SIZE 64
111
112#define opt_match_noarg(s, l, name) \
113 opt_match(s, l, name) && (*(s) ? (rb_warn("argument to --rjit-" name " is ignored"), 1) : 1)
114#define opt_match_arg(s, l, name) \
115 opt_match(s, l, name) && (*(s) ? 1 : (rb_raise(rb_eRuntimeError, "--rjit-" name " needs an argument"), 0))
116
117void
118rb_rjit_setup_options(const char *s, struct rb_rjit_options *rjit_opt)
119{
120 const size_t l = strlen(s);
121 if (l == 0) {
122 return;
123 }
124 else if (opt_match_arg(s, l, "exec-mem-size")) {
125 rjit_opt->exec_mem_size = atoi(s + 1);
126 }
127 else if (opt_match_arg(s, l, "call-threshold")) {
128 rjit_opt->call_threshold = atoi(s + 1);
129 }
130 else if (opt_match_noarg(s, l, "stats")) {
131 rjit_opt->stats = true;
132 }
133 else if (opt_match_noarg(s, l, "disable")) {
134 rjit_opt->disable = true;
135 }
136 else if (opt_match_noarg(s, l, "trace")) {
137 rjit_opt->trace = true;
138 }
139 else if (opt_match_noarg(s, l, "trace-exits")) {
140 rjit_opt->trace_exits = true;
141 }
142 else if (opt_match_noarg(s, l, "dump-disasm")) {
143 rjit_opt->dump_disasm = true;
144 }
145 else if (opt_match_noarg(s, l, "verify-ctx")) {
146 rjit_opt->verify_ctx = true;
147 }
148 else {
149 rb_raise(rb_eRuntimeError,
150 "invalid RJIT option '%s' (--help will show valid RJIT options)", s);
151 }
152}
153
154#define M(shortopt, longopt, desc) RUBY_OPT_MESSAGE(shortopt, longopt, desc)
155const struct ruby_opt_message rb_rjit_option_messages[] = {
156 M("--rjit-exec-mem-size=num", "", "Size of executable memory block in MiB (default: " STRINGIZE(DEFAULT_EXEC_MEM_SIZE) ")."),
157 M("--rjit-call-threshold=num", "", "Number of calls to trigger JIT (default: " STRINGIZE(DEFAULT_CALL_THRESHOLD) ")."),
158 M("--rjit-stats", "", "Enable collecting RJIT statistics."),
159 M("--rjit-disable", "", "Disable RJIT for lazily enabling it with RubyVM::RJIT.enable."),
160 M("--rjit-trace", "", "Allow TracePoint during JIT compilation."),
161 M("--rjit-trace-exits", "", "Trace side exit locations."),
162#ifdef HAVE_LIBCAPSTONE
163 M("--rjit-dump-disasm", "", "Dump all JIT code"),
164#endif
165 {0}
166};
167#undef M
168
169struct rb_rjit_runtime_counters rb_rjit_counters = { 0 };
170
171extern VALUE rb_gc_enable(void);
172extern VALUE rb_gc_disable(void);
173extern uint64_t rb_vm_insns_count;
174
175// Disable GC, TracePoint, JIT, stats, and $!
176#define WITH_RJIT_ISOLATED_USING_PC(using_pc, stmt) do { \
177 VALUE was_disabled = rb_gc_disable(); \
178 \
179 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(GET_EC()); \
180 rb_rjit_global_events = global_hooks->events; \
181 \
182 const VALUE *pc = NULL; \
183 if (rb_rjit_opts.trace) { \
184 pc = GET_EC()->cfp->pc; \
185 if (!using_pc) GET_EC()->cfp->pc = 0; /* avoid crashing on calc_lineno */ \
186 } \
187 else global_hooks->events = 0; \
188 \
189 bool original_call_p = rb_rjit_call_p; \
190 rb_rjit_call_p = false; \
191 \
192 rjit_stats_p = false; \
193 uint64_t insns_count = rb_vm_insns_count; \
194 \
195 VALUE err = rb_errinfo(); \
196 \
197 stmt; \
198 \
199 rb_set_errinfo(err); \
200 \
201 rb_vm_insns_count = insns_count; \
202 rjit_stats_p = rb_rjit_opts.stats; \
203 \
204 rb_rjit_call_p = (rjit_cancel_p ? false : original_call_p); \
205 \
206 if (rb_rjit_opts.trace) GET_EC()->cfp->pc = pc; \
207 else global_hooks->events = rb_rjit_global_events; \
208 \
209 if (!was_disabled) rb_gc_enable(); \
210} while (0);
211#define WITH_RJIT_ISOLATED(stmt) WITH_RJIT_ISOLATED_USING_PC(false, stmt)
212
213void
214rb_rjit_cancel_all(const char *reason)
215{
216 if (!rb_rjit_enabled)
217 return;
218
219 rb_rjit_call_p = false;
220 rjit_cancel_p = true;
221}
222
223void
224rb_rjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop)
225{
226 if (!rb_rjit_call_p) return;
227 rb_rjit_call_p = false;
228}
229
230static void
231rjit_cme_invalidate(void *data)
232{
233 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
234 WITH_RJIT_ISOLATED({
235 rb_funcall(rb_mRJITHooks, rb_intern("on_cme_invalidate"), 1, SIZET2NUM((size_t)data));
236 });
237}
238
239extern int rb_workqueue_register(unsigned flags, rb_postponed_job_func_t func, void *data);
240
241void
242rb_rjit_cme_invalidate(rb_callable_method_entry_t *cme)
243{
244 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
245 // Asynchronously hook the Ruby code since running Ruby in the middle of cme invalidation is dangerous.
246 rb_workqueue_register(0, rjit_cme_invalidate, (void *)cme);
247}
248
249void
250rb_rjit_before_ractor_spawn(void)
251{
252 if (!rb_rjit_call_p) return;
253 rb_rjit_call_p = false;
254}
255
256static void
257rjit_constant_state_changed(void *data)
258{
259 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
260 RB_VM_LOCK_ENTER();
261 rb_vm_barrier();
262
263 WITH_RJIT_ISOLATED({
264 rb_funcall(rb_mRJITHooks, rb_intern("on_constant_state_changed"), 1, SIZET2NUM((size_t)data));
265 });
266
267 RB_VM_LOCK_LEAVE();
268}
269
270void
271rb_rjit_constant_state_changed(ID id)
272{
273 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
274 // Asynchronously hook the Ruby code since this is hooked during a "Ruby critical section".
275 rb_workqueue_register(0, rjit_constant_state_changed, (void *)id);
276}
277
278void
279rb_rjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx)
280{
281 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
282
283 RB_VM_LOCK_ENTER();
284 rb_vm_barrier();
285
286 WITH_RJIT_ISOLATED({
287 rb_funcall(rb_mRJITHooks, rb_intern("on_constant_ic_update"), 3,
288 SIZET2NUM((size_t)iseq), SIZET2NUM((size_t)ic), UINT2NUM(insn_idx));
289 });
290
291 RB_VM_LOCK_LEAVE();
292}
293
294void
295rb_rjit_tracing_invalidate_all(rb_event_flag_t new_iseq_events)
296{
297 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
298 WITH_RJIT_ISOLATED({
299 rb_funcall(rb_mRJITHooks, rb_intern("on_tracing_invalidate_all"), 1, UINT2NUM(new_iseq_events));
300 });
301}
302
303static void
304rjit_iseq_update_references(void *data)
305{
306 if (!rb_rjit_enabled || !rb_rjit_call_p || !rb_mRJITHooks) return;
307 WITH_RJIT_ISOLATED({
308 rb_funcall(rb_mRJITHooks, rb_intern("on_update_references"), 0);
309 });
310}
311
312void
313rb_rjit_iseq_update_references(struct rb_iseq_constant_body *const body)
314{
315 if (!rb_rjit_enabled) return;
316
317 if (body->rjit_blocks) {
318 body->rjit_blocks = rb_gc_location(body->rjit_blocks);
319 }
320
321 // Asynchronously hook the Ruby code to avoid allocation during GC.compact.
322 // Using _one because it's too slow to invalidate all for each ISEQ. Thus
323 // not giving an ISEQ pointer.
324 rb_postponed_job_trigger(rjit_iseq_update_references_pjob);
325}
326
327void
328rb_rjit_iseq_mark(VALUE rjit_blocks)
329{
330 if (!rb_rjit_enabled) return;
331
332 // Note: This wasn't enough for some reason.
333 // We actually rely on RubyVM::RJIT::GC_REFS to mark this.
334 if (rjit_blocks) {
335 rb_gc_mark_movable(rjit_blocks);
336 }
337}
338
339// Called by rb_vm_mark()
340void
341rb_rjit_mark(void)
342{
343 if (!rb_rjit_enabled)
344 return;
345 RUBY_MARK_ENTER("rjit");
346
347 // Pin object pointers used in this file
348 rb_gc_mark(rb_RJITCompiler);
349 rb_gc_mark(rb_cRJITIseqPtr);
350 rb_gc_mark(rb_cRJITCfpPtr);
351 rb_gc_mark(rb_mRJITHooks);
352 rb_gc_mark(rb_rjit_raw_samples);
353 rb_gc_mark(rb_rjit_line_samples);
354
355 RUBY_MARK_LEAVE("rjit");
356}
357
358void
359rb_rjit_free_iseq(const rb_iseq_t *iseq)
360{
361 // TODO: implement this. GC_REFS should remove this iseq's mjit_blocks
362}
363
364// TODO: Use this in more places
365 VALUE
366rb_rjit_iseq_new(rb_iseq_t *iseq)
367{
368 return rb_funcall(rb_cRJITIseqPtr, rb_intern("new"), 1, SIZET2NUM((size_t)iseq));
369}
370
371void
372rb_rjit_compile(const rb_iseq_t *iseq)
373{
374 RB_VM_LOCK_ENTER();
375 rb_vm_barrier();
376
377 WITH_RJIT_ISOLATED_USING_PC(true, {
378 VALUE iseq_ptr = rb_funcall(rb_cRJITIseqPtr, rb_intern("new"), 1, SIZET2NUM((size_t)iseq));
379 VALUE cfp_ptr = rb_funcall(rb_cRJITCfpPtr, rb_intern("new"), 1, SIZET2NUM((size_t)GET_EC()->cfp));
380 rb_funcall(rb_RJITCompiler, rb_intern("compile"), 2, iseq_ptr, cfp_ptr);
381 });
382
383 RB_VM_LOCK_LEAVE();
384}
385
386void *
387rb_rjit_entry_stub_hit(VALUE branch_stub)
388{
389 VALUE result;
390
391 RB_VM_LOCK_ENTER();
392 rb_vm_barrier();
393
394 rb_control_frame_t *cfp = GET_EC()->cfp;
395
396 WITH_RJIT_ISOLATED_USING_PC(true, {
397 VALUE cfp_ptr = rb_funcall(rb_cRJITCfpPtr, rb_intern("new"), 1, SIZET2NUM((size_t)cfp));
398 result = rb_funcall(rb_RJITCompiler, rb_intern("entry_stub_hit"), 2, branch_stub, cfp_ptr);
399 });
400
401 RB_VM_LOCK_LEAVE();
402
403 return (void *)NUM2SIZET(result);
404}
405
406void *
407rb_rjit_branch_stub_hit(VALUE branch_stub, int sp_offset, int target0_p)
408{
409 VALUE result;
410
411 RB_VM_LOCK_ENTER();
412 rb_vm_barrier();
413
414 rb_control_frame_t *cfp = GET_EC()->cfp;
415 cfp->sp += sp_offset; // preserve stack values, also using the actual sp_offset to make jit.peek_at_stack work
416
417 WITH_RJIT_ISOLATED({
418 VALUE cfp_ptr = rb_funcall(rb_cRJITCfpPtr, rb_intern("new"), 1, SIZET2NUM((size_t)cfp));
419 result = rb_funcall(rb_RJITCompiler, rb_intern("branch_stub_hit"), 3, branch_stub, cfp_ptr, RBOOL(target0_p));
420 });
421
422 cfp->sp -= sp_offset; // reset for consistency with the code without the stub
423
424 RB_VM_LOCK_LEAVE();
425
426 return (void *)NUM2SIZET(result);
427}
428
429void
430rb_rjit_init(const struct rb_rjit_options *opts)
431{
432 VM_ASSERT(rb_rjit_enabled);
433
434 // Normalize options
435 rb_rjit_opts = *opts;
436 if (rb_rjit_opts.exec_mem_size == 0)
437 rb_rjit_opts.exec_mem_size = DEFAULT_EXEC_MEM_SIZE;
438 if (rb_rjit_opts.call_threshold == 0)
439 rb_rjit_opts.call_threshold = DEFAULT_CALL_THRESHOLD;
440#ifndef HAVE_LIBCAPSTONE
441 if (rb_rjit_opts.dump_disasm)
442 rb_warn("libcapstone has not been linked. Ignoring --rjit-dump-disasm.");
443#endif
444
445 // RJIT doesn't support miniruby, but it might reach here by RJIT_FORCE_ENABLE.
446 rb_mRJIT = rb_const_get(rb_cRubyVM, rb_intern("RJIT"));
447 if (!rb_const_defined(rb_mRJIT, rb_intern("Compiler"))) {
448 rb_warn("Disabling RJIT because RubyVM::RJIT::Compiler is not defined");
449 rb_rjit_enabled = false;
450 return;
451 }
452 rjit_iseq_update_references_pjob = rb_postponed_job_preregister(0, rjit_iseq_update_references, NULL);
453 if (rjit_iseq_update_references_pjob == POSTPONED_JOB_HANDLE_INVALID) {
454 rb_bug("Could not preregister postponed job for RJIT");
455 }
456 rb_mRJITC = rb_const_get(rb_mRJIT, rb_intern("C"));
457 VALUE rb_cRJITCompiler = rb_const_get(rb_mRJIT, rb_intern("Compiler"));
458 rb_RJITCompiler = rb_funcall(rb_cRJITCompiler, rb_intern("new"), 0);
459 rb_cRJITIseqPtr = rb_funcall(rb_mRJITC, rb_intern("rb_iseq_t"), 0);
460 rb_cRJITCfpPtr = rb_funcall(rb_mRJITC, rb_intern("rb_control_frame_t"), 0);
461 rb_mRJITHooks = rb_const_get(rb_mRJIT, rb_intern("Hooks"));
462 if (rb_rjit_opts.trace_exits) {
463 rb_rjit_raw_samples = rb_ary_new();
464 rb_rjit_line_samples = rb_ary_new();
465 }
466
467 // Enable RJIT and stats from here
468 rb_rjit_call_p = !rb_rjit_opts.disable;
469 rjit_stats_p = rb_rjit_opts.stats;
470}
471
472//
473// Primitive for rjit.rb
474//
475
476// Same as `rb_rjit_opts.stats`, but this is used before rb_rjit_opts is set.
477static VALUE
478rjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self)
479{
480 return RBOOL(rb_rjit_stats_enabled);
481}
482
483// Same as `rb_rjit_opts.trace_exits`, but this is used before rb_rjit_opts is set.
484static VALUE
485rjit_trace_exits_enabled_p(rb_execution_context_t *ec, VALUE self)
486{
487 return RBOOL(rb_rjit_trace_exits_enabled);
488}
489
490// Disable anything that could impact stats. It ends up disabling JIT calls as well.
491static VALUE
492rjit_stop_stats(rb_execution_context_t *ec, VALUE self)
493{
494 rb_rjit_call_p = false;
495 rjit_stats_p = false;
496 return Qnil;
497}
498
499#include "rjit.rbinc"
500
501#endif // USE_RJIT
debug.h
rb_postponed_job_handle_t
unsigned int rb_postponed_job_handle_t
The type of a handle returned from rb_postponed_job_preregister and passed to rb_postponed_job_trigge...
Definition debug.h:665
rb_postponed_job_trigger
void rb_postponed_job_trigger(rb_postponed_job_handle_t h)
Triggers a pre-registered job registered with rb_postponed_job_preregister, scheduling it for executi...
Definition vm_trace.c:1786
rb_postponed_job_preregister
rb_postponed_job_handle_t rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, void *data)
Pre-registers a func in Ruby's postponed job preregistration table, returning an opaque handle which ...
Definition vm_trace.c:1752
rb_postponed_job_func_t
void(* rb_postponed_job_func_t)(void *arg)
Type of postponed jobs.
Definition debug.h:659
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
SIZET2NUM
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
UINT2NUM
#define UINT2NUM
Old name of RB_UINT2NUM.
Definition int.h:46
NUM2SIZET
#define NUM2SIZET
Old name of RB_NUM2SIZE.
Definition size_t.h:61
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1428
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:466
rb_funcall
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
Definition vm_eval.c:1099
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3243
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3565
thread.h
util.h
version.h
rb_iseq_constant_body
Definition vm_core.h:404
ruby_opt_message
Definition cmdlineopt.h:48
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40