1/* Copyright (C) 2003-2021 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include "pthreadP.h"
20#include <futex-internal.h>
21#include <atomic.h>
22#include <libc-lockP.h>
23#include <shlib-compat.h>
24
25unsigned long int __fork_generation attribute_hidden;
26
27
28static void
29clear_once_control (void *arg)
30{
31 pthread_once_t *once_control = (pthread_once_t *) arg;
32
33 /* Reset to the uninitialized state here. We don't need a stronger memory
34 order because we do not need to make any other of our writes visible to
35 other threads that see this value: This function will be called if we
36 get interrupted (see __pthread_once), so all we need to relay to other
37 threads is the state being reset again. */
38 atomic_store_relaxed (once_control, 0);
39 futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
40}
41
42
43/* This is similar to a lock implementation, but we distinguish between three
44 states: not yet initialized (0), initialization in progress
45 (__fork_generation | __PTHREAD_ONCE_INPROGRESS), and initialization
46 finished (__PTHREAD_ONCE_DONE); __fork_generation does not use the bits
47 that are used for __PTHREAD_ONCE_INPROGRESS and __PTHREAD_ONCE_DONE (which
48 is what __PTHREAD_ONCE_FORK_GEN_INCR is used for). If in the first state,
49 threads will try to run the initialization by moving to the second state;
50 the first thread to do so via a CAS on once_control runs init_routine,
51 other threads block.
52 When forking the process, some threads can be interrupted during the second
53 state; they won't be present in the forked child, so we need to restart
54 initialization in the child. To distinguish an in-progress initialization
55 from an interrupted initialization (in which case we need to reclaim the
56 lock), we look at the fork generation that's part of the second state: We
57 can reclaim iff it differs from the current fork generation.
58 XXX: This algorithm has an ABA issue on the fork generation: If an
59 initialization is interrupted, we then fork 2^30 times (30 bits of
60 once_control are used for the fork generation), and try to initialize
61 again, we can deadlock because we can't distinguish the in-progress and
62 interrupted cases anymore.
63 XXX: We split out this slow path because current compilers do not generate
64 as efficient code when the fast path in __pthread_once below is not in a
65 separate function. */
66static int
67__attribute__ ((noinline))
68__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
69{
70 while (1)
71 {
72 int val, newval;
73
74 /* We need acquire memory order for this load because if the value
75 signals that initialization has finished, we need to see any
76 data modifications done during initialization. */
77 val = atomic_load_acquire (once_control);
78 do
79 {
80 /* Check if the initialization has already been done. */
81 if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
82 return 0;
83
84 /* We try to set the state to in-progress and having the current
85 fork generation. We don't need atomic accesses for the fork
86 generation because it's immutable in a particular process, and
87 forked child processes start with a single thread that modified
88 the generation. */
89 newval = __fork_generation | __PTHREAD_ONCE_INPROGRESS;
90 /* We need acquire memory order here for the same reason as for the
91 load from once_control above. */
92 }
93 while (__glibc_unlikely (!atomic_compare_exchange_weak_acquire (
94 once_control, &val, newval)));
95
96 /* Check if another thread already runs the initializer. */
97 if ((val & __PTHREAD_ONCE_INPROGRESS) != 0)
98 {
99 /* Check whether the initializer execution was interrupted by a
100 fork. We know that for both values, __PTHREAD_ONCE_INPROGRESS
101 is set and __PTHREAD_ONCE_DONE is not. */
102 if (val == newval)
103 {
104 /* Same generation, some other thread was faster. Wait and
105 retry. */
106 futex_wait_simple ((unsigned int *) once_control,
107 (unsigned int) newval, FUTEX_PRIVATE);
108 continue;
109 }
110 }
111
112 /* This thread is the first here. Do the initialization.
113 Register a cleanup handler so that in case the thread gets
114 interrupted the initialization can be restarted. */
115 pthread_cleanup_combined_push (clear_once_control, once_control);
116
117 init_routine ();
118
119 pthread_cleanup_combined_pop (0);
120
121
122 /* Mark *once_control as having finished the initialization. We need
123 release memory order here because we need to synchronize with other
124 threads that want to use the initialized data. */
125 atomic_store_release (once_control, __PTHREAD_ONCE_DONE);
126
127 /* Wake up all other threads. */
128 futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
129 break;
130 }
131
132 return 0;
133}
134
135int
136___pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
137{
138 /* Fast path. See __pthread_once_slow. */
139 int val;
140 val = atomic_load_acquire (once_control);
141 if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
142 return 0;
143 else
144 return __pthread_once_slow (once_control, init_routine);
145}
146libc_hidden_ver (___pthread_once, __pthread_once)
147#ifndef SHARED
148strong_alias (___pthread_once, __pthread_once)
149#endif
150
151versioned_symbol (libc, ___pthread_once, pthread_once, GLIBC_2_34);
152#if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
153compat_symbol (libpthread, ___pthread_once, __pthread_once, GLIBC_2_0);
154compat_symbol (libpthread, ___pthread_once, pthread_once, GLIBC_2_0);
155#endif
156