Ruby: thread_pthread_mn.c Source File

Ruby 3.4.6p54 (2025-09-16 revision dbd83256b1cec76c69756ecb8758b9e1079833de)

thread_pthread_mn.c
1// included by "thread_pthread.c"
2
3#if USE_MN_THREADS
4
5static void timer_thread_unregister_waiting(rb_thread_t *th, int fd, enum thread_sched_waiting_flag flags);
6
7static bool
8timer_thread_cancel_waiting(rb_thread_t *th)
9{
10 bool canceled = false;
11
12 if (th->sched.waiting_reason.flags) {
13 rb_native_mutex_lock(&timer_th.waiting_lock);
14 {
15 if (th->sched.waiting_reason.flags) {
16 canceled = true;
17 ccan_list_del_init(&th->sched.waiting_reason.node);
18 if (th->sched.waiting_reason.flags & (thread_sched_waiting_io_read | thread_sched_waiting_io_write)) {
19 timer_thread_unregister_waiting(th, th->sched.waiting_reason.data.fd, th->sched.waiting_reason.flags);
20 }
21 th->sched.waiting_reason.flags = thread_sched_waiting_none;
22 }
23 }
24 rb_native_mutex_unlock(&timer_th.waiting_lock);
25 }
26
27 return canceled;
28}
29
30static void
31ubf_event_waiting(void *ptr)
32{
33 rb_thread_t *th = (rb_thread_t *)ptr;
34 struct rb_thread_sched *sched = TH_SCHED(th);
35
36 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
37
38 VM_ASSERT(th->nt == NULL || !th_has_dedicated_nt(th));
39
40 // only once. it is safe because th->interrupt_lock is already acquired.
41 th->unblock.func = NULL;
42 th->unblock.arg = NULL;
43
44 bool canceled = timer_thread_cancel_waiting(th);
45
46 thread_sched_lock(sched, th);
47 {
48 if (sched->running == th) {
49 RUBY_DEBUG_LOG("not waiting yet");
50 }
51 else if (canceled) {
52 thread_sched_to_ready_common(sched, th, true, false);
53 }
54 else {
55 RUBY_DEBUG_LOG("already not waiting");
56 }
57 }
58 thread_sched_unlock(sched, th);
59}
60
61static bool timer_thread_register_waiting(rb_thread_t *th, int fd, enum thread_sched_waiting_flag flags, rb_hrtime_t *rel);
62
63// return true if timed out
64static bool
65thread_sched_wait_events(struct rb_thread_sched *sched, rb_thread_t *th, int fd, enum thread_sched_waiting_flag events, rb_hrtime_t *rel)
66{
67 VM_ASSERT(!th_has_dedicated_nt(th)); // on SNT
68
69 volatile bool timedout = false, need_cancel = false;
70
71 if (timer_thread_register_waiting(th, fd, events, rel)) {
72 RUBY_DEBUG_LOG("wait fd:%d", fd);
73
74 RB_VM_SAVE_MACHINE_CONTEXT(th);
75 setup_ubf(th, ubf_event_waiting, (void *)th);
76
77 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
78
79 thread_sched_lock(sched, th);
80 {
81 if (th->sched.waiting_reason.flags == thread_sched_waiting_none) {
82 // already awaken
83 }
84 else if (RUBY_VM_INTERRUPTED(th->ec)) {
85 need_cancel = true;
86 }
87 else {
88 RUBY_DEBUG_LOG("sleep");
89
90 th->status = THREAD_STOPPED_FOREVER;
91 thread_sched_wakeup_next_thread(sched, th, true);
92 thread_sched_wait_running_turn(sched, th, true);
93
94 RUBY_DEBUG_LOG("wakeup");
95 }
96
97 timedout = th->sched.waiting_reason.data.result == 0;
98 }
99 thread_sched_unlock(sched, th);
100
101 if (need_cancel) {
102 timer_thread_cancel_waiting(th);
103 }
104
105 setup_ubf(th, NULL, NULL); // TODO: maybe it is already NULL?
106
107 th->status = THREAD_RUNNABLE;
108 }
109 else {
110 RUBY_DEBUG_LOG("can not wait fd:%d", fd);
111 return false;
112 }
113
114 VM_ASSERT(sched->running == th);
115
116 return timedout;
117}
118
120
121static int
122get_sysconf_page_size(void)
123{
124 static long page_size = 0;
125
126 if (UNLIKELY(page_size == 0)) {
127 page_size = sysconf(_SC_PAGESIZE);
128 VM_ASSERT(page_size < INT_MAX);
129 }
130 return (int)page_size;
131}
132
133#define MSTACK_CHUNK_SIZE (512 * 1024 * 1024) // 512MB
134#define MSTACK_PAGE_SIZE get_sysconf_page_size()
135#define MSTACK_CHUNK_PAGE_NUM (MSTACK_CHUNK_SIZE / MSTACK_PAGE_SIZE - 1) // 1 is start redzone
136
137// 512MB chunk
138// 131,072 pages (> 65,536)
139// 0th page is Redzone. Start from 1st page.
140
141/*
142 * <--> machine stack + vm stack
143 * ----------------------------------
144 * |HD...|RZ| ... |RZ| ... ... |RZ|
145 * <------------- 512MB ------------->
146 */
147
148static struct nt_stack_chunk_header {
149 struct nt_stack_chunk_header *prev_chunk;
150 struct nt_stack_chunk_header *prev_free_chunk;
151
152 uint16_t start_page;
153 uint16_t stack_count;
154 uint16_t uninitialized_stack_count;
155
156 uint16_t free_stack_pos;
157 uint16_t free_stack[];
158} *nt_stack_chunks = NULL,
159 *nt_free_stack_chunks = NULL;
160
161struct nt_machine_stack_footer {
162 struct nt_stack_chunk_header *ch;
163 size_t index;
164};
165
166static rb_nativethread_lock_t nt_machine_stack_lock = RB_NATIVETHREAD_LOCK_INIT;
167
168#include <sys/mman.h>
169
170// vm_stack_size + machine_stack_size + 1 * (guard page size)
171static inline size_t
172nt_thread_stack_size(void)
173{
174 static size_t msz;
175 if (LIKELY(msz > 0)) return msz;
176
177 rb_vm_t *vm = GET_VM();
178 int sz = (int)(vm->default_params.thread_vm_stack_size + vm->default_params.thread_machine_stack_size + MSTACK_PAGE_SIZE);
179 int page_num = roomof(sz, MSTACK_PAGE_SIZE);
180 msz = (size_t)page_num * MSTACK_PAGE_SIZE;
181 return msz;
182}
183
184static struct nt_stack_chunk_header *
185nt_alloc_thread_stack_chunk(void)
186{
187 int mmap_flags = MAP_ANONYMOUS | MAP_PRIVATE;
188#if defined(MAP_STACK) && !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__)
189 mmap_flags |= MAP_STACK;
190#endif
191
192 const char *m = (void *)mmap(NULL, MSTACK_CHUNK_SIZE, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
193 if (m == MAP_FAILED) {
194 return NULL;
195 }
196
197 ruby_annotate_mmap(m, MSTACK_CHUNK_SIZE, "Ruby:nt_alloc_thread_stack_chunk");
198
199 size_t msz = nt_thread_stack_size();
200 int header_page_cnt = 1;
201 int stack_count = ((MSTACK_CHUNK_PAGE_NUM - header_page_cnt) * MSTACK_PAGE_SIZE) / msz;
202 int ch_size = sizeof(struct nt_stack_chunk_header) + sizeof(uint16_t) * stack_count;
203
204 if (ch_size > MSTACK_PAGE_SIZE * header_page_cnt) {
205 header_page_cnt = (ch_size + MSTACK_PAGE_SIZE - 1) / MSTACK_PAGE_SIZE;
206 stack_count = ((MSTACK_CHUNK_PAGE_NUM - header_page_cnt) * MSTACK_PAGE_SIZE) / msz;
207 }
208
209 VM_ASSERT(stack_count <= UINT16_MAX);
210
211 struct nt_stack_chunk_header *ch = (struct nt_stack_chunk_header *)m;
212
213 ch->start_page = header_page_cnt;
214 ch->prev_chunk = nt_stack_chunks;
215 ch->prev_free_chunk = nt_free_stack_chunks;
216 ch->uninitialized_stack_count = ch->stack_count = (uint16_t)stack_count;
217 ch->free_stack_pos = 0;
218
219 RUBY_DEBUG_LOG("ch:%p start_page:%d stack_cnt:%d stack_size:%d", ch, (int)ch->start_page, (int)ch->stack_count, (int)msz);
220
221 return ch;
222}
223
224static void *
225nt_stack_chunk_get_stack_start(struct nt_stack_chunk_header *ch, size_t idx)
226{
227 const char *m = (char *)ch;
228 return (void *)(m + ch->start_page * MSTACK_PAGE_SIZE + idx * nt_thread_stack_size());
229}
230
231static struct nt_machine_stack_footer *
232nt_stack_chunk_get_msf(const rb_vm_t *vm, const char *mstack)
233{
234 // TODO: stack direction
235 const size_t msz = vm->default_params.thread_machine_stack_size;
236 return (struct nt_machine_stack_footer *)&mstack[msz - sizeof(struct nt_machine_stack_footer)];
237}
238
239static void *
240nt_stack_chunk_get_stack(const rb_vm_t *vm, struct nt_stack_chunk_header *ch, size_t idx, void **vm_stack, void **machine_stack)
241{
242 // TODO: only support stack going down
243 // [VM ... <GUARD> machine stack ...]
244
245 const char *vstack, *mstack;
246 const char *guard_page;
247 vstack = nt_stack_chunk_get_stack_start(ch, idx);
248 guard_page = vstack + vm->default_params.thread_vm_stack_size;
249 mstack = guard_page + MSTACK_PAGE_SIZE;
250
251 struct nt_machine_stack_footer *msf = nt_stack_chunk_get_msf(vm, mstack);
252 msf->ch = ch;
253 msf->index = idx;
254
255#if 0
256 RUBY_DEBUG_LOG("msf:%p vstack:%p-%p guard_page:%p-%p mstack:%p-%p", msf,
257 vstack, (void *)(guard_page-1),
258 guard_page, (void *)(mstack-1),
259 mstack, (void *)(msf));
260#endif
261
262 *vm_stack = (void *)vstack;
263 *machine_stack = (void *)mstack;
264
265 return (void *)guard_page;
266}
267
268 RBIMPL_ATTR_MAYBE_UNUSED()
269static void
270nt_stack_chunk_dump(void)
271{
272 struct nt_stack_chunk_header *ch;
273 int i;
274
275 fprintf(stderr, "** nt_stack_chunks\n");
276 ch = nt_stack_chunks;
277 for (i=0; ch; i++, ch = ch->prev_chunk) {
278 fprintf(stderr, "%d %p free_pos:%d\n", i, (void *)ch, (int)ch->free_stack_pos);
279 }
280
281 fprintf(stderr, "** nt_free_stack_chunks\n");
282 ch = nt_free_stack_chunks;
283 for (i=0; ch; i++, ch = ch->prev_free_chunk) {
284 fprintf(stderr, "%d %p free_pos:%d\n", i, (void *)ch, (int)ch->free_stack_pos);
285 }
286}
287
288static int
289nt_guard_page(const char *p, size_t len)
290{
291 if (mprotect((void *)p, len, PROT_NONE) != -1) {
292 return 0;
293 }
294 else {
295 return errno;
296 }
297}
298
299static int
300nt_alloc_stack(rb_vm_t *vm, void **vm_stack, void **machine_stack)
301{
302 int err = 0;
303
304 rb_native_mutex_lock(&nt_machine_stack_lock);
305 {
306 retry:
307 if (nt_free_stack_chunks) {
308 struct nt_stack_chunk_header *ch = nt_free_stack_chunks;
309 if (ch->free_stack_pos > 0) {
310 RUBY_DEBUG_LOG("free_stack_pos:%d", ch->free_stack_pos);
311 nt_stack_chunk_get_stack(vm, ch, ch->free_stack[--ch->free_stack_pos], vm_stack, machine_stack);
312 }
313 else if (ch->uninitialized_stack_count > 0) {
314 RUBY_DEBUG_LOG("uninitialized_stack_count:%d", ch->uninitialized_stack_count);
315
316 size_t idx = ch->stack_count - ch->uninitialized_stack_count--;
317 void *guard_page = nt_stack_chunk_get_stack(vm, ch, idx, vm_stack, machine_stack);
318 err = nt_guard_page(guard_page, MSTACK_PAGE_SIZE);
319 }
320 else {
321 nt_free_stack_chunks = ch->prev_free_chunk;
322 ch->prev_free_chunk = NULL;
323 goto retry;
324 }
325 }
326 else {
327 struct nt_stack_chunk_header *p = nt_alloc_thread_stack_chunk();
328 if (p == NULL) {
329 err = errno;
330 }
331 else {
332 nt_free_stack_chunks = nt_stack_chunks = p;
333 goto retry;
334 }
335 }
336 }
337 rb_native_mutex_unlock(&nt_machine_stack_lock);
338
339 return err;
340}
341
342static void
343nt_madvise_free_or_dontneed(void *addr, size_t len)
344{
345 /* There is no real way to perform error handling here. Both MADV_FREE
346 * and MADV_DONTNEED are both documented to pretty much only return EINVAL
347 * for a huge variety of errors. It's indistinguishable if madvise fails
348 * because the parameters were bad, or because the kernel we're running on
349 * does not support the given advice. This kind of free-but-don't-unmap
350 * is best-effort anyway, so don't sweat it.
351 *
352 * n.b. A very common case of "the kernel doesn't support MADV_FREE and
353 * returns EINVAL" is running under the `rr` debugger; it makes all
354 * MADV_FREE calls return EINVAL. */
355
356#if defined(MADV_FREE)
357 int r = madvise(addr, len, MADV_FREE);
358 // Return on success, or else try MADV_DONTNEED
359 if (r == 0) return;
360#endif
361#if defined(MADV_DONTNEED)
362 madvise(addr, len, MADV_DONTNEED);
363#endif
364}
365
366static void
367nt_free_stack(void *mstack)
368{
369 if (!mstack) return;
370
371 rb_native_mutex_lock(&nt_machine_stack_lock);
372 {
373 struct nt_machine_stack_footer *msf = nt_stack_chunk_get_msf(GET_VM(), mstack);
374 struct nt_stack_chunk_header *ch = msf->ch;
375 int idx = (int)msf->index;
376 void *stack = nt_stack_chunk_get_stack_start(ch, idx);
377
378 RUBY_DEBUG_LOG("stack:%p mstack:%p ch:%p index:%d", stack, mstack, ch, idx);
379
380 if (ch->prev_free_chunk == NULL) {
381 ch->prev_free_chunk = nt_free_stack_chunks;
382 nt_free_stack_chunks = ch;
383 }
384 ch->free_stack[ch->free_stack_pos++] = idx;
385
386 // clear the stack pages
387 nt_madvise_free_or_dontneed(stack, nt_thread_stack_size());
388 }
389 rb_native_mutex_unlock(&nt_machine_stack_lock);
390}
391
392
393static int
394native_thread_check_and_create_shared(rb_vm_t *vm)
395{
396 bool need_to_make = false;
397
398 rb_native_mutex_lock(&vm->ractor.sched.lock);
399 {
400 unsigned int snt_cnt = vm->ractor.sched.snt_cnt;
401 if (!vm->ractor.main_ractor->threads.sched.enable_mn_threads) snt_cnt++; // do not need snt for main ractor
402
403 if (((int)snt_cnt < MINIMUM_SNT) ||
404 (snt_cnt < vm->ractor.cnt &&
405 snt_cnt < vm->ractor.sched.max_cpu)) {
406
407 RUBY_DEBUG_LOG("added snt:%u dnt:%u ractor_cnt:%u grq_cnt:%u",
408 vm->ractor.sched.snt_cnt,
409 vm->ractor.sched.dnt_cnt,
410 vm->ractor.cnt,
411 vm->ractor.sched.grq_cnt);
412
413 vm->ractor.sched.snt_cnt++;
414 need_to_make = true;
415 }
416 else {
417 RUBY_DEBUG_LOG("snt:%d ractor_cnt:%d", (int)vm->ractor.sched.snt_cnt, (int)vm->ractor.cnt);
418 }
419 }
420 rb_native_mutex_unlock(&vm->ractor.sched.lock);
421
422 if (need_to_make) {
423 struct rb_native_thread *nt = native_thread_alloc();
424 nt->vm = vm;
425 return native_thread_create0(nt);
426 }
427 else {
428 return 0;
429 }
430}
431
432static COROUTINE
433co_start(struct coroutine_context *from, struct coroutine_context *self)
434{
435#ifdef RUBY_ASAN_ENABLED
436 __sanitizer_finish_switch_fiber(self->fake_stack,
437 (const void**)&from->stack_base, &from->stack_size);
438#endif
439
440 rb_thread_t *th = (rb_thread_t *)self->argument;
441 struct rb_thread_sched *sched = TH_SCHED(th);
442 VM_ASSERT(th->nt != NULL);
443 VM_ASSERT(th == sched->running);
444 VM_ASSERT(sched->lock_owner == NULL);
445
446 // RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
447
448 thread_sched_set_lock_owner(sched, th);
449 thread_sched_add_running_thread(TH_SCHED(th), th);
450 thread_sched_unlock(sched, th);
451 {
452 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th);
453 call_thread_start_func_2(th);
454 }
455 thread_sched_lock(sched, NULL);
456
457 RUBY_DEBUG_LOG("terminated th:%d", (int)th->serial);
458
459 // Thread is terminated
460
461 struct rb_native_thread *nt = th->nt;
462 bool is_dnt = th_has_dedicated_nt(th);
463 native_thread_assign(NULL, th);
464 rb_ractor_set_current_ec(th->ractor, NULL);
465
466 if (is_dnt) {
467 // SNT became DNT while running. Just return to the nt_context
468
469 th->sched.finished = true;
470 coroutine_transfer0(self, nt->nt_context, true);
471 }
472 else {
473 rb_vm_t *vm = th->vm;
474 bool has_ready_ractor = vm->ractor.sched.grq_cnt > 0; // at least this ractor is not queued
475 rb_thread_t *next_th = sched->running;
476
477 if (!has_ready_ractor && next_th && !next_th->nt) {
478 // switch to the next thread
479 thread_sched_set_lock_owner(sched, NULL);
480 th->sched.finished = true;
481 thread_sched_switch0(th->sched.context, next_th, nt, true);
482 }
483 else {
484 // switch to the next Ractor
485 th->sched.finished = true;
486 coroutine_transfer0(self, nt->nt_context, true);
487 }
488 }
489
490 rb_bug("unreachable");
491}
492
493static int
494native_thread_create_shared(rb_thread_t *th)
495{
496 // setup coroutine
497 rb_vm_t *vm = th->vm;
498 void *vm_stack = NULL, *machine_stack = NULL;
499 int err = nt_alloc_stack(vm, &vm_stack, &machine_stack);
500 if (err) return err;
501
502 VM_ASSERT(vm_stack < machine_stack);
503
504 // setup vm stack
505 size_t vm_stack_words = th->vm->default_params.thread_vm_stack_size/sizeof(VALUE);
506 rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_words);
507
508 // setup machine stack
509 size_t machine_stack_size = vm->default_params.thread_machine_stack_size - sizeof(struct nt_machine_stack_footer);
510 th->ec->machine.stack_start = (void *)((uintptr_t)machine_stack + machine_stack_size);
511 th->ec->machine.stack_maxsize = machine_stack_size; // TODO
512 th->sched.context_stack = machine_stack;
513
514 th->sched.context = ruby_xmalloc(sizeof(struct coroutine_context));
515 coroutine_initialize(th->sched.context, co_start, machine_stack, machine_stack_size);
516 th->sched.context->argument = th;
517
518 RUBY_DEBUG_LOG("th:%u vm_stack:%p machine_stack:%p", rb_th_serial(th), vm_stack, machine_stack);
519 thread_sched_to_ready(TH_SCHED(th), th);
520
521 // setup nt
522 return native_thread_check_and_create_shared(th->vm);
523}
524
525#else // USE_MN_THREADS
526
527static int
528native_thread_create_shared(rb_thread_t *th)
529{
530 rb_bug("unreachable");
531}
532
533static bool
534thread_sched_wait_events(struct rb_thread_sched *sched, rb_thread_t *th, int fd, enum thread_sched_waiting_flag events, rb_hrtime_t *rel)
535{
536 rb_bug("unreachable");
537}
538
539#endif // USE_MN_THREADS
540
542#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
543
544static bool
545fd_readable_nonblock(int fd)
546{
547 struct pollfd pfd = {
548 .fd = fd,
549 .events = POLLIN,
550 };
551 return poll(&pfd, 1, 0) != 0;
552}
553
554static bool
555fd_writable_nonblock(int fd)
556{
557 struct pollfd pfd = {
558 .fd = fd,
559 .events = POLLOUT,
560 };
561 return poll(&pfd, 1, 0) != 0;
562}
563
564static void
565verify_waiting_list(void)
566{
567#if VM_CHECK_MODE > 0
568 struct rb_thread_sched_waiting *w, *prev_w = NULL;
569
570 // waiting list's timeout order should be [1, 2, 3, ..., 0, 0, 0]
571
572 ccan_list_for_each(&timer_th.waiting, w, node) {
573 // fprintf(stderr, "verify_waiting_list th:%u abs:%lu\n", rb_th_serial(wth), (unsigned long)wth->sched.waiting_reason.data.timeout);
574 if (prev_w) {
575 rb_hrtime_t timeout = w->data.timeout;
576 rb_hrtime_t prev_timeout = w->data.timeout;
577 VM_ASSERT(timeout == 0 || prev_timeout <= timeout);
578 }
579 prev_w = w;
580 }
581#endif
582}
583
584#if HAVE_SYS_EVENT_H // kqueue helpers
585
586static enum thread_sched_waiting_flag
587kqueue_translate_filter_to_flags(int16_t filter)
588{
589 switch (filter) {
590 case EVFILT_READ:
591 return thread_sched_waiting_io_read;
592 case EVFILT_WRITE:
593 return thread_sched_waiting_io_write;
594 case EVFILT_TIMER:
595 return thread_sched_waiting_timeout;
596 default:
597 rb_bug("kevent filter:%d not supported", filter);
598 }
599}
600
601static int
602kqueue_wait(rb_vm_t *vm)
603{
604 struct timespec calculated_timeout;
605 struct timespec *timeout = NULL;
606 int timeout_ms = timer_thread_set_timeout(vm);
607
608 if (timeout_ms >= 0) {
609 calculated_timeout.tv_sec = timeout_ms / 1000;
610 calculated_timeout.tv_nsec = (timeout_ms % 1000) * 1000000;
611 timeout = &calculated_timeout;
612 }
613
614 return kevent(timer_th.event_fd, NULL, 0, timer_th.finished_events, KQUEUE_EVENTS_MAX, timeout);
615}
616
617static void
618kqueue_create(void)
619{
620 if ((timer_th.event_fd = kqueue()) == -1) rb_bug("kqueue creation failed (errno:%d)", errno);
621 int flags = fcntl(timer_th.event_fd, F_GETFD);
622 if (flags == -1) {
623 rb_bug("kqueue GETFD failed (errno:%d)", errno);
624 }
625
626 flags |= FD_CLOEXEC;
627 if (fcntl(timer_th.event_fd, F_SETFD, flags) == -1) {
628 rb_bug("kqueue SETFD failed (errno:%d)", errno);
629 }
630}
631
632static void
633kqueue_unregister_waiting(int fd, enum thread_sched_waiting_flag flags)
634{
635 if (flags) {
636 struct kevent ke[2];
637 int num_events = 0;
638
639 if (flags & thread_sched_waiting_io_read) {
640 EV_SET(&ke[num_events], fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
641 num_events++;
642 }
643 if (flags & thread_sched_waiting_io_write) {
644 EV_SET(&ke[num_events], fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL);
645 num_events++;
646 }
647 if (kevent(timer_th.event_fd, ke, num_events, NULL, 0, NULL) == -1) {
648 perror("kevent");
649 rb_bug("unregister/kevent fails. errno:%d", errno);
650 }
651 }
652}
653
654static bool
655kqueue_already_registered(int fd)
656{
657 struct rb_thread_sched_waiting *w, *found_w = NULL;
658
659 ccan_list_for_each(&timer_th.waiting, w, node) {
660 // Similar to EEXIST in epoll_ctl, but more strict because it checks fd rather than flags
661 // for simplicity
662 if (w->flags && w->data.fd == fd) {
663 found_w = w;
664 break;
665 }
666 }
667 return found_w != NULL;
668}
669
670#endif // HAVE_SYS_EVENT_H
671
672// return false if the fd is not waitable or not need to wait.
673static bool
674timer_thread_register_waiting(rb_thread_t *th, int fd, enum thread_sched_waiting_flag flags, rb_hrtime_t *rel)
675{
676 RUBY_DEBUG_LOG("th:%u fd:%d flag:%d rel:%lu", rb_th_serial(th), fd, flags, rel ? (unsigned long)*rel : 0);
677
678 VM_ASSERT(th == NULL || TH_SCHED(th)->running == th);
679 VM_ASSERT(flags != 0);
680
681 rb_hrtime_t abs = 0; // 0 means no timeout
682
683 if (rel) {
684 if (*rel > 0) {
685 flags |= thread_sched_waiting_timeout;
686 }
687 else {
688 return false;
689 }
690 }
691
692 if (rel && *rel > 0) {
693 flags |= thread_sched_waiting_timeout;
694 }
695
696#if HAVE_SYS_EVENT_H
697 struct kevent ke[2];
698 int num_events = 0;
699#else
700 uint32_t epoll_events = 0;
701#endif
702 if (flags & thread_sched_waiting_timeout) {
703 VM_ASSERT(rel != NULL);
704 abs = rb_hrtime_add(rb_hrtime_now(), *rel);
705 }
706
707 if (flags & thread_sched_waiting_io_read) {
708 if (!(flags & thread_sched_waiting_io_force) && fd_readable_nonblock(fd)) {
709 RUBY_DEBUG_LOG("fd_readable_nonblock");
710 return false;
711 }
712 else {
713 VM_ASSERT(fd >= 0);
714#if HAVE_SYS_EVENT_H
715 EV_SET(&ke[num_events], fd, EVFILT_READ, EV_ADD, 0, 0, (void *)th);
716 num_events++;
717#else
718 epoll_events |= EPOLLIN;
719#endif
720 }
721 }
722
723 if (flags & thread_sched_waiting_io_write) {
724 if (!(flags & thread_sched_waiting_io_force) && fd_writable_nonblock(fd)) {
725 RUBY_DEBUG_LOG("fd_writable_nonblock");
726 return false;
727 }
728 else {
729 VM_ASSERT(fd >= 0);
730#if HAVE_SYS_EVENT_H
731 EV_SET(&ke[num_events], fd, EVFILT_WRITE, EV_ADD, 0, 0, (void *)th);
732 num_events++;
733#else
734 epoll_events |= EPOLLOUT;
735#endif
736 }
737 }
738
739 rb_native_mutex_lock(&timer_th.waiting_lock);
740 {
741#if HAVE_SYS_EVENT_H
742 if (num_events > 0) {
743 if (kqueue_already_registered(fd)) {
744 rb_native_mutex_unlock(&timer_th.waiting_lock);
745 return false;
746 }
747
748 if (kevent(timer_th.event_fd, ke, num_events, NULL, 0, NULL) == -1) {
749 RUBY_DEBUG_LOG("failed (%d)", errno);
750
751 switch (errno) {
752 case EBADF:
753 // the fd is closed?
754 case EINTR:
755 // signal received? is there a sensible way to handle this?
756 default:
757 perror("kevent");
758 rb_bug("register/kevent failed(fd:%d, errno:%d)", fd, errno);
759 }
760 }
761 RUBY_DEBUG_LOG("kevent(add, fd:%d) success", fd);
762 }
763#else
764 if (epoll_events) {
765 struct epoll_event event = {
766 .events = epoll_events,
767 .data = {
768 .ptr = (void *)th,
769 },
770 };
771 if (epoll_ctl(timer_th.event_fd, EPOLL_CTL_ADD, fd, &event) == -1) {
772 RUBY_DEBUG_LOG("failed (%d)", errno);
773
774 switch (errno) {
775 case EBADF:
776 // the fd is closed?
777 case EPERM:
778 // the fd doesn't support epoll
779 case EEXIST:
780 // the fd is already registered by another thread
781 rb_native_mutex_unlock(&timer_th.waiting_lock);
782 return false;
783 default:
784 perror("epoll_ctl");
785 rb_bug("register/epoll_ctl failed(fd:%d, errno:%d)", fd, errno);
786 }
787 }
788 RUBY_DEBUG_LOG("epoll_ctl(add, fd:%d, events:%d) success", fd, epoll_events);
789 }
790#endif
791
792 if (th) {
793 VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none);
794
795 // setup waiting information
796 {
797 th->sched.waiting_reason.flags = flags;
798 th->sched.waiting_reason.data.timeout = abs;
799 th->sched.waiting_reason.data.fd = fd;
800 th->sched.waiting_reason.data.result = 0;
801 }
802
803 if (abs == 0) { // no timeout
804 VM_ASSERT(!(flags & thread_sched_waiting_timeout));
805 ccan_list_add_tail(&timer_th.waiting, &th->sched.waiting_reason.node);
806 }
807 else {
808 RUBY_DEBUG_LOG("abs:%lu", (unsigned long)abs);
809 VM_ASSERT(flags & thread_sched_waiting_timeout);
810
811 // insert th to sorted list (TODO: O(n))
812 struct rb_thread_sched_waiting *w, *prev_w = NULL;
813
814 ccan_list_for_each(&timer_th.waiting, w, node) {
815 if ((w->flags & thread_sched_waiting_timeout) &&
816 w->data.timeout < abs) {
817 prev_w = w;
818 }
819 else {
820 break;
821 }
822 }
823
824 if (prev_w) {
825 ccan_list_add_after(&timer_th.waiting, &prev_w->node, &th->sched.waiting_reason.node);
826 }
827 else {
828 ccan_list_add(&timer_th.waiting, &th->sched.waiting_reason.node);
829 }
830
831 verify_waiting_list();
832
833 // update timeout seconds
834 timer_thread_wakeup();
835 }
836 }
837 else {
838 VM_ASSERT(abs == 0);
839 }
840 }
841 rb_native_mutex_unlock(&timer_th.waiting_lock);
842
843 return true;
844}
845
846static void
847timer_thread_unregister_waiting(rb_thread_t *th, int fd, enum thread_sched_waiting_flag flags)
848{
849 RUBY_DEBUG_LOG("th:%u fd:%d", rb_th_serial(th), fd);
850#if HAVE_SYS_EVENT_H
851 kqueue_unregister_waiting(fd, flags);
852#else
853 // Linux 2.6.9 or later is needed to pass NULL as data.
854 if (epoll_ctl(timer_th.event_fd, EPOLL_CTL_DEL, fd, NULL) == -1) {
855 switch (errno) {
856 case EBADF:
857 // just ignore. maybe fd is closed.
858 break;
859 default:
860 perror("epoll_ctl");
861 rb_bug("unregister/epoll_ctl fails. errno:%d", errno);
862 }
863 }
864#endif
865}
866
867static void
868timer_thread_setup_mn(void)
869{
870#if HAVE_SYS_EVENT_H
871 kqueue_create();
872 RUBY_DEBUG_LOG("kqueue_fd:%d", timer_th.event_fd);
873#else
874 if ((timer_th.event_fd = epoll_create1(EPOLL_CLOEXEC)) == -1) rb_bug("epoll_create (errno:%d)", errno);
875 RUBY_DEBUG_LOG("epoll_fd:%d", timer_th.event_fd);
876#endif
877 RUBY_DEBUG_LOG("comm_fds:%d/%d", timer_th.comm_fds[0], timer_th.comm_fds[1]);
878
879 timer_thread_register_waiting(NULL, timer_th.comm_fds[0], thread_sched_waiting_io_read | thread_sched_waiting_io_force, NULL);
880}
881
882static int
883event_wait(rb_vm_t *vm)
884{
885#if HAVE_SYS_EVENT_H
886 int r = kqueue_wait(vm);
887#else
888 int r = epoll_wait(timer_th.event_fd, timer_th.finished_events, EPOLL_EVENTS_MAX, timer_thread_set_timeout(vm));
889#endif
890 return r;
891}
892
893/*
894 * The purpose of the timer thread:
895 *
896 * (1) Periodic checking
897 * (1-1) Provide time slice for active NTs
898 * (1-2) Check NT shortage
899 * (1-3) Periodic UBF (global)
900 * (1-4) Lazy GRQ deq start
901 * (2) Receive notification
902 * (2-1) async I/O termination
903 * (2-2) timeout
904 * (2-2-1) sleep(n)
905 * (2-2-2) timeout(n), I/O, ...
906 */
907static void
908timer_thread_polling(rb_vm_t *vm)
909{
910 int r = event_wait(vm);
911
912 RUBY_DEBUG_LOG("r:%d errno:%d", r, errno);
913
914 switch (r) {
915 case 0: // timeout
916 RUBY_DEBUG_LOG("timeout%s", "");
917
918 ractor_sched_lock(vm, NULL);
919 {
920 // (1-1) timeslice
921 timer_thread_check_timeslice(vm);
922
923 // (1-4) lazy grq deq
924 if (vm->ractor.sched.grq_cnt > 0) {
925 RUBY_DEBUG_LOG("GRQ cnt: %u", vm->ractor.sched.grq_cnt);
926 rb_native_cond_signal(&vm->ractor.sched.cond);
927 }
928 }
929 ractor_sched_unlock(vm, NULL);
930
931 // (1-2)
932 native_thread_check_and_create_shared(vm);
933
934 break;
935
936 case -1:
937 switch (errno) {
938 case EINTR:
939 // simply retry
940 break;
941 default:
942 perror("event_wait");
943 rb_bug("event_wait errno:%d", errno);
944 }
945 break;
946
947 default:
948 RUBY_DEBUG_LOG("%d event(s)", r);
949
950#if HAVE_SYS_EVENT_H
951 for (int i=0; i<r; i++) {
952 rb_thread_t *th = (rb_thread_t *)timer_th.finished_events[i].udata;
953 int fd = (int)timer_th.finished_events[i].ident;
954 int16_t filter = timer_th.finished_events[i].filter;
955
956 if (th == NULL) {
957 // wakeup timerthread
958 RUBY_DEBUG_LOG("comm from fd:%d", timer_th.comm_fds[1]);
959 consume_communication_pipe(timer_th.comm_fds[0]);
960 }
961 else {
962 // wakeup specific thread by IO
963 RUBY_DEBUG_LOG("io event. wakeup_th:%u event:%s%s",
964 rb_th_serial(th),
965 (filter == EVFILT_READ) ? "read/" : "",
966 (filter == EVFILT_WRITE) ? "write/" : "");
967
968 rb_native_mutex_lock(&timer_th.waiting_lock);
969 {
970 if (th->sched.waiting_reason.flags) {
971 // delete from chain
972 ccan_list_del_init(&th->sched.waiting_reason.node);
973 timer_thread_unregister_waiting(th, fd, kqueue_translate_filter_to_flags(filter));
974
975 th->sched.waiting_reason.flags = thread_sched_waiting_none;
976 th->sched.waiting_reason.data.fd = -1;
977 th->sched.waiting_reason.data.result = filter;
978
979 timer_thread_wakeup_thread(th);
980 }
981 else {
982 // already released
983 }
984 }
985 rb_native_mutex_unlock(&timer_th.waiting_lock);
986 }
987 }
988#else
989 for (int i=0; i<r; i++) {
990 rb_thread_t *th = (rb_thread_t *)timer_th.finished_events[i].data.ptr;
991
992 if (th == NULL) {
993 // wakeup timerthread
994 RUBY_DEBUG_LOG("comm from fd:%d", timer_th.comm_fds[1]);
995 consume_communication_pipe(timer_th.comm_fds[0]);
996 }
997 else {
998 // wakeup specific thread by IO
999 uint32_t events = timer_th.finished_events[i].events;
1000
1001 RUBY_DEBUG_LOG("io event. wakeup_th:%u event:%s%s%s%s%s%s",
1002 rb_th_serial(th),
1003 (events & EPOLLIN) ? "in/" : "",
1004 (events & EPOLLOUT) ? "out/" : "",
1005 (events & EPOLLRDHUP) ? "RDHUP/" : "",
1006 (events & EPOLLPRI) ? "pri/" : "",
1007 (events & EPOLLERR) ? "err/" : "",
1008 (events & EPOLLHUP) ? "hup/" : "");
1009
1010 rb_native_mutex_lock(&timer_th.waiting_lock);
1011 {
1012 if (th->sched.waiting_reason.flags) {
1013 // delete from chain
1014 ccan_list_del_init(&th->sched.waiting_reason.node);
1015 timer_thread_unregister_waiting(th, th->sched.waiting_reason.data.fd, th->sched.waiting_reason.flags);
1016
1017 th->sched.waiting_reason.flags = thread_sched_waiting_none;
1018 th->sched.waiting_reason.data.fd = -1;
1019 th->sched.waiting_reason.data.result = (int)events;
1020
1021 timer_thread_wakeup_thread(th);
1022 }
1023 else {
1024 // already released
1025 }
1026 }
1027 rb_native_mutex_unlock(&timer_th.waiting_lock);
1028 }
1029 }
1030#endif
1031 }
1032}
1033
1034#else // HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H
1035
1036static void
1037timer_thread_setup_mn(void)
1038{
1039 // do nothing
1040}
1041
1042static void
1043timer_thread_polling(rb_vm_t *vm)
1044{
1045 int timeout = timer_thread_set_timeout(vm);
1046
1047 struct pollfd pfd = {
1048 .fd = timer_th.comm_fds[0],
1049 .events = POLLIN,
1050 };
1051
1052 int r = poll(&pfd, 1, timeout);
1053
1054 switch (r) {
1055 case 0: // timeout
1056 rb_native_mutex_lock(&vm->ractor.sched.lock);
1057 {
1058 // (1-1) timeslice
1059 timer_thread_check_timeslice(vm);
1060 }
1061 rb_native_mutex_unlock(&vm->ractor.sched.lock);
1062 break;
1063
1064 case -1: // error
1065 switch (errno) {
1066 case EINTR:
1067 // simply retry
1068 break;
1069 default:
1070 perror("poll");
1071 rb_bug("poll errno:%d", errno);
1072 break;
1073 }
1074
1075 case 1:
1076 consume_communication_pipe(timer_th.comm_fds[0]);
1077 break;
1078
1079 default:
1080 rb_bug("unreachbale");
1081 }
1082}
1083
1084#endif // HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H
len
int len
Length of the buffer.
Definition io.h:8
RUBY_INTERNAL_THREAD_EVENT_RESUMED
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
Definition thread.h:238
RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
Definition thread.h:245
RBIMPL_ATTR_MAYBE_UNUSED
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]]
Definition maybe_unused.h:35
errno
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
coroutine_context
Definition Context.h:36
rb_native_thread
Definition thread_none.h:11
rb_thread_sched_waiting
Definition thread_pthread.h:21
rb_thread_sched
Definition thread_none.h:16
timespec
Definition missing.h:74
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_cond_signal
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
ruby_xmalloc
void * ruby_xmalloc(size_t size)
Allocates a storage instance.
Definition gc.c:4581