| 1 | /*
|
|---|
| 2 | * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers
|
|---|
| 3 | * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
|
|---|
| 4 | * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
|
|---|
| 5 | * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved.
|
|---|
| 6 | *
|
|---|
| 7 | *
|
|---|
| 8 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
|
|---|
| 9 | * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
|
|---|
| 10 | *
|
|---|
| 11 | * Permission is hereby granted to use or copy this program
|
|---|
| 12 | * for any purpose, provided the above notices are retained on all copies.
|
|---|
| 13 | * Permission to modify the code and to distribute modified code is granted,
|
|---|
| 14 | * provided the above notices are retained, and a notice that the code was
|
|---|
| 15 | * modified is included with the above copyright notice.
|
|---|
| 16 | */
|
|---|
| 17 |
|
|---|
| 18 | #ifndef GC_LOCKS_H
|
|---|
| 19 | #define GC_LOCKS_H
|
|---|
| 20 |
|
|---|
| 21 | /*
|
|---|
| 22 | * Mutual exclusion between allocator/collector routines.
|
|---|
| 23 | * Needed if there is more than one allocator thread.
|
|---|
| 24 | * FASTLOCK() is assumed to try to acquire the lock in a cheap and
|
|---|
| 25 | * dirty way that is acceptable for a few instructions, e.g. by
|
|---|
| 26 | * inhibiting preemption. This is assumed to have succeeded only
|
|---|
| 27 | * if a subsequent call to FASTLOCK_SUCCEEDED() returns TRUE.
|
|---|
| 28 | * FASTUNLOCK() is called whether or not FASTLOCK_SUCCEEDED().
|
|---|
| 29 | * If signals cannot be tolerated with the FASTLOCK held, then
|
|---|
| 30 | * FASTLOCK should disable signals. The code executed under
|
|---|
| 31 | * FASTLOCK is otherwise immune to interruption, provided it is
|
|---|
| 32 | * not restarted.
|
|---|
| 33 | * DCL_LOCK_STATE declares any local variables needed by LOCK and UNLOCK
|
|---|
| 34 | * and/or DISABLE_SIGNALS and ENABLE_SIGNALS and/or FASTLOCK.
|
|---|
| 35 | * (There is currently no equivalent for FASTLOCK.)
|
|---|
| 36 | *
|
|---|
| 37 | * In the PARALLEL_MARK case, we also need to define a number of
|
|---|
| 38 | * other inline finctions here:
|
|---|
| 39 | * GC_bool GC_compare_and_exchange( volatile GC_word *addr,
|
|---|
| 40 | * GC_word old, GC_word new )
|
|---|
| 41 | * GC_word GC_atomic_add( volatile GC_word *addr, GC_word how_much )
|
|---|
| 42 | * void GC_memory_barrier( )
|
|---|
| 43 | *
|
|---|
| 44 | */
|
|---|
| 45 | # ifdef THREADS
|
|---|
| 46 | void GC_noop1 GC_PROTO((word));
|
|---|
| 47 | # ifdef PCR_OBSOLETE /* Faster, but broken with multiple lwp's */
|
|---|
| 48 | # include "th/PCR_Th.h"
|
|---|
| 49 | # include "th/PCR_ThCrSec.h"
|
|---|
| 50 | extern struct PCR_Th_MLRep GC_allocate_ml;
|
|---|
| 51 | # define DCL_LOCK_STATE PCR_sigset_t GC_old_sig_mask
|
|---|
| 52 | # define LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml)
|
|---|
| 53 | # define UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml)
|
|---|
| 54 | # define UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml)
|
|---|
| 55 | # define FASTLOCK() PCR_ThCrSec_EnterSys()
|
|---|
| 56 | /* Here we cheat (a lot): */
|
|---|
| 57 | # define FASTLOCK_SUCCEEDED() (*(int *)(&GC_allocate_ml) == 0)
|
|---|
| 58 | /* TRUE if nobody currently holds the lock */
|
|---|
| 59 | # define FASTUNLOCK() PCR_ThCrSec_ExitSys()
|
|---|
| 60 | # endif
|
|---|
| 61 | # ifdef PCR
|
|---|
| 62 | # include <base/PCR_Base.h>
|
|---|
| 63 | # include <th/PCR_Th.h>
|
|---|
| 64 | extern PCR_Th_ML GC_allocate_ml;
|
|---|
| 65 | # define DCL_LOCK_STATE \
|
|---|
| 66 | PCR_ERes GC_fastLockRes; PCR_sigset_t GC_old_sig_mask
|
|---|
| 67 | # define LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml)
|
|---|
| 68 | # define UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml)
|
|---|
| 69 | # define FASTLOCK() (GC_fastLockRes = PCR_Th_ML_Try(&GC_allocate_ml))
|
|---|
| 70 | # define FASTLOCK_SUCCEEDED() (GC_fastLockRes == PCR_ERes_okay)
|
|---|
| 71 | # define FASTUNLOCK() {\
|
|---|
| 72 | if( FASTLOCK_SUCCEEDED() ) PCR_Th_ML_Release(&GC_allocate_ml); }
|
|---|
| 73 | # endif
|
|---|
| 74 | # ifdef SRC_M3
|
|---|
| 75 | extern GC_word RT0u__inCritical;
|
|---|
| 76 | # define LOCK() RT0u__inCritical++
|
|---|
| 77 | # define UNLOCK() RT0u__inCritical--
|
|---|
| 78 | # endif
|
|---|
| 79 | # ifdef GC_SOLARIS_THREADS
|
|---|
| 80 | # include <thread.h>
|
|---|
| 81 | # include <signal.h>
|
|---|
| 82 | extern mutex_t GC_allocate_ml;
|
|---|
| 83 | # define LOCK() mutex_lock(&GC_allocate_ml);
|
|---|
| 84 | # define UNLOCK() mutex_unlock(&GC_allocate_ml);
|
|---|
| 85 | # endif
|
|---|
| 86 |
|
|---|
| 87 | /* Try to define GC_TEST_AND_SET and a matching GC_CLEAR for spin lock */
|
|---|
| 88 | /* acquisition and release. We need this for correct operation of the */
|
|---|
| 89 | /* incremental GC. */
|
|---|
| 90 | # ifdef __GNUC__
|
|---|
| 91 | # if defined(I386)
|
|---|
| 92 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 93 | int oldval;
|
|---|
| 94 | /* Note: the "xchg" instruction does not need a "lock" prefix */
|
|---|
| 95 | __asm__ __volatile__("xchgl %0, %1"
|
|---|
| 96 | : "=r"(oldval), "=m"(*(addr))
|
|---|
| 97 | : "0"(1), "m"(*(addr)) : "memory");
|
|---|
| 98 | return oldval;
|
|---|
| 99 | }
|
|---|
| 100 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 101 | # endif
|
|---|
| 102 | # if defined(IA64)
|
|---|
| 103 | # if defined(__INTEL_COMPILER)
|
|---|
| 104 | # include <ia64intrin.h>
|
|---|
| 105 | # endif
|
|---|
| 106 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 107 | long oldval, n = 1;
|
|---|
| 108 | # ifndef __INTEL_COMPILER
|
|---|
| 109 | __asm__ __volatile__("xchg4 %0=%1,%2"
|
|---|
| 110 | : "=r"(oldval), "=m"(*addr)
|
|---|
| 111 | : "r"(n), "1"(*addr) : "memory");
|
|---|
| 112 | # else
|
|---|
| 113 | oldval = _InterlockedExchange(addr, n);
|
|---|
| 114 | # endif
|
|---|
| 115 | return oldval;
|
|---|
| 116 | }
|
|---|
| 117 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 118 | /* Should this handle post-increment addressing?? */
|
|---|
| 119 | inline static void GC_clear(volatile unsigned int *addr) {
|
|---|
| 120 | # ifndef __INTEL_COMPILER
|
|---|
| 121 | __asm__ __volatile__("st4.rel %0=r0" : "=m" (*addr) : : "memory");
|
|---|
| 122 | # else
|
|---|
| 123 | // there is no st4 but I can use xchg I hope
|
|---|
| 124 | _InterlockedExchange(addr, 0);
|
|---|
| 125 | # endif
|
|---|
| 126 | }
|
|---|
| 127 | # define GC_CLEAR_DEFINED
|
|---|
| 128 | # endif
|
|---|
| 129 | # ifdef SPARC
|
|---|
| 130 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 131 | int oldval;
|
|---|
| 132 |
|
|---|
| 133 | __asm__ __volatile__("ldstub %1,%0"
|
|---|
| 134 | : "=r"(oldval), "=m"(*addr)
|
|---|
| 135 | : "m"(*addr) : "memory");
|
|---|
| 136 | return oldval;
|
|---|
| 137 | }
|
|---|
| 138 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 139 | # endif
|
|---|
| 140 | # ifdef M68K
|
|---|
| 141 | /* Contributed by Tony Mantler. I'm not sure how well it was */
|
|---|
| 142 | /* tested. */
|
|---|
| 143 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 144 | char oldval; /* this must be no longer than 8 bits */
|
|---|
| 145 |
|
|---|
| 146 | /* The return value is semi-phony. */
|
|---|
| 147 | /* 'tas' sets bit 7 while the return */
|
|---|
| 148 | /* value pretends bit 0 was set */
|
|---|
| 149 | __asm__ __volatile__(
|
|---|
| 150 | "tas %1@; sne %0; negb %0"
|
|---|
| 151 | : "=d" (oldval)
|
|---|
| 152 | : "a" (addr) : "memory");
|
|---|
| 153 | return oldval;
|
|---|
| 154 | }
|
|---|
| 155 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 156 | # endif
|
|---|
| 157 | # if defined(POWERPC)
|
|---|
| 158 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 159 | int oldval;
|
|---|
| 160 | int temp = 1; /* locked value */
|
|---|
| 161 |
|
|---|
| 162 | __asm__ __volatile__(
|
|---|
| 163 | "1:\tlwarx %0,0,%3\n" /* load and reserve */
|
|---|
| 164 | "\tcmpwi %0, 0\n" /* if load is */
|
|---|
| 165 | "\tbne 2f\n" /* non-zero, return already set */
|
|---|
| 166 | "\tstwcx. %2,0,%1\n" /* else store conditional */
|
|---|
| 167 | "\tbne- 1b\n" /* retry if lost reservation */
|
|---|
| 168 | "\tsync\n" /* import barrier */
|
|---|
| 169 | "2:\t\n" /* oldval is zero if we set */
|
|---|
| 170 | : "=&r"(oldval), "=p"(addr)
|
|---|
| 171 | : "r"(temp), "1"(addr)
|
|---|
| 172 | : "cr0","memory");
|
|---|
| 173 | return oldval;
|
|---|
| 174 | }
|
|---|
| 175 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 176 | inline static void GC_clear(volatile unsigned int *addr) {
|
|---|
| 177 | __asm__ __volatile__("lwsync" : : : "memory");
|
|---|
| 178 | *(addr) = 0;
|
|---|
| 179 | }
|
|---|
| 180 | # define GC_CLEAR_DEFINED
|
|---|
| 181 | # endif
|
|---|
| 182 | # if defined(ALPHA)
|
|---|
| 183 | inline static int GC_test_and_set(volatile unsigned int * addr)
|
|---|
| 184 | {
|
|---|
| 185 | unsigned long oldvalue;
|
|---|
| 186 | unsigned long temp;
|
|---|
| 187 |
|
|---|
| 188 | __asm__ __volatile__(
|
|---|
| 189 | "1: ldl_l %0,%1\n"
|
|---|
| 190 | " and %0,%3,%2\n"
|
|---|
| 191 | " bne %2,2f\n"
|
|---|
| 192 | " xor %0,%3,%0\n"
|
|---|
| 193 | " stl_c %0,%1\n"
|
|---|
| 194 | # ifdef __ELF__
|
|---|
| 195 | " beq %0,3f\n"
|
|---|
| 196 | # else
|
|---|
| 197 | " beq %0,1b\n"
|
|---|
| 198 | # endif
|
|---|
| 199 | " mb\n"
|
|---|
| 200 | "2:\n"
|
|---|
| 201 | # ifdef __ELF__
|
|---|
| 202 | ".section .text2,\"ax\"\n"
|
|---|
| 203 | "3: br 1b\n"
|
|---|
| 204 | ".previous"
|
|---|
| 205 | # endif
|
|---|
| 206 | :"=&r" (temp), "=m" (*addr), "=&r" (oldvalue)
|
|---|
| 207 | :"Ir" (1), "m" (*addr)
|
|---|
| 208 | :"memory");
|
|---|
| 209 |
|
|---|
| 210 | return oldvalue;
|
|---|
| 211 | }
|
|---|
| 212 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 213 | inline static void GC_clear(volatile unsigned int *addr) {
|
|---|
| 214 | __asm__ __volatile__("mb" : : : "memory");
|
|---|
| 215 | *(addr) = 0;
|
|---|
| 216 | }
|
|---|
| 217 | # define GC_CLEAR_DEFINED
|
|---|
| 218 | # endif /* ALPHA */
|
|---|
| 219 | # ifdef ARM32
|
|---|
| 220 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 221 | int oldval;
|
|---|
| 222 | /* SWP on ARM is very similar to XCHG on x86. */
|
|---|
| 223 | /* The first operand is the result, the second the value */
|
|---|
| 224 | /* to be stored. Both registers must be different from addr. */
|
|---|
| 225 | /* Make the address operand an early clobber output so it */
|
|---|
| 226 | /* doesn't overlap with the other operands. The early clobber*/
|
|---|
| 227 | /* on oldval is neccessary to prevent the compiler allocating */
|
|---|
| 228 | /* them to the same register if they are both unused. */
|
|---|
| 229 | __asm__ __volatile__("swp %0, %2, [%3]"
|
|---|
| 230 | : "=&r"(oldval), "=&r"(addr)
|
|---|
| 231 | : "r"(1), "1"(addr)
|
|---|
| 232 | : "memory");
|
|---|
| 233 | return oldval;
|
|---|
| 234 | }
|
|---|
| 235 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 236 | # endif /* ARM32 */
|
|---|
| 237 | # ifdef CRIS
|
|---|
| 238 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 239 | /* Ripped from linuxthreads/sysdeps/cris/pt-machine.h. */
|
|---|
| 240 | /* Included with Hans-Peter Nilsson's permission. */
|
|---|
| 241 | register unsigned long int ret;
|
|---|
| 242 |
|
|---|
| 243 | /* Note the use of a dummy output of *addr to expose the write.
|
|---|
| 244 | * The memory barrier is to stop *other* writes being moved past
|
|---|
| 245 | * this code.
|
|---|
| 246 | */
|
|---|
| 247 | __asm__ __volatile__("clearf\n"
|
|---|
| 248 | "0:\n\t"
|
|---|
| 249 | "movu.b [%2],%0\n\t"
|
|---|
| 250 | "ax\n\t"
|
|---|
| 251 | "move.b %3,[%2]\n\t"
|
|---|
| 252 | "bwf 0b\n\t"
|
|---|
| 253 | "clearf"
|
|---|
| 254 | : "=&r" (ret), "=m" (*addr)
|
|---|
| 255 | : "r" (addr), "r" ((int) 1), "m" (*addr)
|
|---|
| 256 | : "memory");
|
|---|
| 257 | return ret;
|
|---|
| 258 | }
|
|---|
| 259 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 260 | # endif /* CRIS */
|
|---|
| 261 | # ifdef S390
|
|---|
| 262 | inline static int GC_test_and_set(volatile unsigned int *addr) {
|
|---|
| 263 | int ret;
|
|---|
| 264 | __asm__ __volatile__ (
|
|---|
| 265 | " l %0,0(%2)\n"
|
|---|
| 266 | "0: cs %0,%1,0(%2)\n"
|
|---|
| 267 | " jl 0b"
|
|---|
| 268 | : "=&d" (ret)
|
|---|
| 269 | : "d" (1), "a" (addr)
|
|---|
| 270 | : "cc", "memory");
|
|---|
| 271 | return ret;
|
|---|
| 272 | }
|
|---|
| 273 | # endif
|
|---|
| 274 | # endif /* __GNUC__ */
|
|---|
| 275 | # if (defined(ALPHA) && !defined(__GNUC__))
|
|---|
| 276 | # ifndef OSF1
|
|---|
| 277 | --> We currently assume that if gcc is not used, we are
|
|---|
| 278 | --> running under Tru64.
|
|---|
| 279 | # endif
|
|---|
| 280 | # include <machine/builtins.h>
|
|---|
| 281 | # include <c_asm.h>
|
|---|
| 282 | # define GC_test_and_set(addr) __ATOMIC_EXCH_LONG(addr, 1)
|
|---|
| 283 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 284 | # define GC_clear(addr) { asm("mb"); *(volatile unsigned *)addr = 0; }
|
|---|
| 285 | # define GC_CLEAR_DEFINED
|
|---|
| 286 | # endif
|
|---|
| 287 | # if defined(MSWIN32)
|
|---|
| 288 | # define GC_test_and_set(addr) InterlockedExchange((LPLONG)addr,1)
|
|---|
| 289 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 290 | # endif
|
|---|
| 291 | # ifdef MIPS
|
|---|
| 292 | # ifdef LINUX
|
|---|
| 293 | # include <sys/tas.h>
|
|---|
| 294 | # define GC_test_and_set(addr) _test_and_set((int *) addr,1)
|
|---|
| 295 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 296 | # elif __mips < 3 || !(defined (_ABIN32) || defined(_ABI64)) \
|
|---|
| 297 | || !defined(_COMPILER_VERSION) || _COMPILER_VERSION < 700
|
|---|
| 298 | # ifdef __GNUC__
|
|---|
| 299 | # define GC_test_and_set(addr) _test_and_set((void *)addr,1)
|
|---|
| 300 | # else
|
|---|
| 301 | # define GC_test_and_set(addr) test_and_set((void *)addr,1)
|
|---|
| 302 | # endif
|
|---|
| 303 | # else
|
|---|
| 304 | # include <sgidefs.h>
|
|---|
| 305 | # include <mutex.h>
|
|---|
| 306 | # define GC_test_and_set(addr) __test_and_set32((void *)addr,1)
|
|---|
| 307 | # define GC_clear(addr) __lock_release(addr);
|
|---|
| 308 | # define GC_CLEAR_DEFINED
|
|---|
| 309 | # endif
|
|---|
| 310 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 311 | # endif /* MIPS */
|
|---|
| 312 | # if defined(_AIX)
|
|---|
| 313 | # include <sys/atomic_op.h>
|
|---|
| 314 | # if (defined(_POWER) || defined(_POWERPC))
|
|---|
| 315 | # if defined(__GNUC__)
|
|---|
| 316 | inline static void GC_memsync() {
|
|---|
| 317 | __asm__ __volatile__ ("sync" : : : "memory");
|
|---|
| 318 | }
|
|---|
| 319 | # else
|
|---|
| 320 | # ifndef inline
|
|---|
| 321 | # define inline __inline
|
|---|
| 322 | # endif
|
|---|
| 323 | # pragma mc_func GC_memsync { \
|
|---|
| 324 | "7c0004ac" /* sync (same opcode used for dcs)*/ \
|
|---|
| 325 | }
|
|---|
| 326 | # endif
|
|---|
| 327 | # else
|
|---|
| 328 | # error dont know how to memsync
|
|---|
| 329 | # endif
|
|---|
| 330 | inline static int GC_test_and_set(volatile unsigned int * addr) {
|
|---|
| 331 | int oldvalue = 0;
|
|---|
| 332 | if (compare_and_swap((void *)addr, &oldvalue, 1)) {
|
|---|
| 333 | GC_memsync();
|
|---|
| 334 | return 0;
|
|---|
| 335 | } else return 1;
|
|---|
| 336 | }
|
|---|
| 337 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 338 | inline static void GC_clear(volatile unsigned int *addr) {
|
|---|
| 339 | GC_memsync();
|
|---|
| 340 | *(addr) = 0;
|
|---|
| 341 | }
|
|---|
| 342 | # define GC_CLEAR_DEFINED
|
|---|
| 343 |
|
|---|
| 344 | # endif
|
|---|
| 345 | # if 0 /* defined(HP_PA) */
|
|---|
| 346 | /* The official recommendation seems to be to not use ldcw from */
|
|---|
| 347 | /* user mode. Since multithreaded incremental collection doesn't */
|
|---|
| 348 | /* work anyway on HP_PA, this shouldn't be a major loss. */
|
|---|
| 349 |
|
|---|
| 350 | /* "set" means 0 and "clear" means 1 here. */
|
|---|
| 351 | # define GC_test_and_set(addr) !GC_test_and_clear(addr);
|
|---|
| 352 | # define GC_TEST_AND_SET_DEFINED
|
|---|
| 353 | # define GC_clear(addr) GC_noop1((word)(addr)); *(volatile unsigned int *)addr = 1;
|
|---|
| 354 | /* The above needs a memory barrier! */
|
|---|
| 355 | # define GC_CLEAR_DEFINED
|
|---|
| 356 | # endif
|
|---|
| 357 | # if defined(GC_TEST_AND_SET_DEFINED) && !defined(GC_CLEAR_DEFINED)
|
|---|
| 358 | # ifdef __GNUC__
|
|---|
| 359 | inline static void GC_clear(volatile unsigned int *addr) {
|
|---|
| 360 | /* Try to discourage gcc from moving anything past this. */
|
|---|
| 361 | __asm__ __volatile__(" " : : : "memory");
|
|---|
| 362 | *(addr) = 0;
|
|---|
| 363 | }
|
|---|
| 364 | # else
|
|---|
| 365 | /* The function call in the following should prevent the */
|
|---|
| 366 | /* compiler from moving assignments to below the UNLOCK. */
|
|---|
| 367 | # define GC_clear(addr) GC_noop1((word)(addr)); \
|
|---|
| 368 | *((volatile unsigned int *)(addr)) = 0;
|
|---|
| 369 | # endif
|
|---|
| 370 | # define GC_CLEAR_DEFINED
|
|---|
| 371 | # endif /* !GC_CLEAR_DEFINED */
|
|---|
| 372 |
|
|---|
| 373 | # if !defined(GC_TEST_AND_SET_DEFINED)
|
|---|
| 374 | # define USE_PTHREAD_LOCKS
|
|---|
| 375 | # endif
|
|---|
| 376 |
|
|---|
| 377 | # if defined(GC_PTHREADS) && !defined(GC_SOLARIS_THREADS) \
|
|---|
| 378 | && !defined(GC_WIN32_THREADS)
|
|---|
| 379 | # define NO_THREAD (pthread_t)(-1)
|
|---|
| 380 | # include <pthread.h>
|
|---|
| 381 | # if defined(PARALLEL_MARK)
|
|---|
| 382 | /* We need compare-and-swap to update mark bits, where it's */
|
|---|
| 383 | /* performance critical. If USE_MARK_BYTES is defined, it is */
|
|---|
| 384 | /* no longer needed for this purpose. However we use it in */
|
|---|
| 385 | /* either case to implement atomic fetch-and-add, though that's */
|
|---|
| 386 | /* less performance critical, and could perhaps be done with */
|
|---|
| 387 | /* a lock. */
|
|---|
| 388 | # if defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 389 | /* Probably not useful, except for debugging. */
|
|---|
| 390 | /* We do use GENERIC_COMPARE_AND_SWAP on PA_RISC, but we */
|
|---|
| 391 | /* minimize its use. */
|
|---|
| 392 | extern pthread_mutex_t GC_compare_and_swap_lock;
|
|---|
| 393 |
|
|---|
| 394 | /* Note that if GC_word updates are not atomic, a concurrent */
|
|---|
| 395 | /* reader should acquire GC_compare_and_swap_lock. On */
|
|---|
| 396 | /* currently supported platforms, such updates are atomic. */
|
|---|
| 397 | extern GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 398 | GC_word old, GC_word new_val);
|
|---|
| 399 | # endif /* GENERIC_COMPARE_AND_SWAP */
|
|---|
| 400 | # if defined(I386)
|
|---|
| 401 | # if !defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 402 | /* Returns TRUE if the comparison succeeded. */
|
|---|
| 403 | inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 404 | GC_word old,
|
|---|
| 405 | GC_word new_val)
|
|---|
| 406 | {
|
|---|
| 407 | char result;
|
|---|
| 408 | __asm__ __volatile__("lock; cmpxchgl %2, %0; setz %1"
|
|---|
| 409 | : "+m"(*(addr)), "=r"(result)
|
|---|
| 410 | : "r" (new_val), "a"(old) : "memory");
|
|---|
| 411 | return (GC_bool) result;
|
|---|
| 412 | }
|
|---|
| 413 | # endif /* !GENERIC_COMPARE_AND_SWAP */
|
|---|
| 414 | inline static void GC_memory_barrier()
|
|---|
| 415 | {
|
|---|
| 416 | /* We believe the processor ensures at least processor */
|
|---|
| 417 | /* consistent ordering. Thus a compiler barrier */
|
|---|
| 418 | /* should suffice. */
|
|---|
| 419 | __asm__ __volatile__("" : : : "memory");
|
|---|
| 420 | }
|
|---|
| 421 | # endif /* I386 */
|
|---|
| 422 |
|
|---|
| 423 | # if defined(POWERPC)
|
|---|
| 424 | # if !defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 425 | # if CPP_WORDSZ == 64
|
|---|
| 426 | /* Returns TRUE if the comparison succeeded. */
|
|---|
| 427 | inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 428 | GC_word old, GC_word new_val)
|
|---|
| 429 | {
|
|---|
| 430 | unsigned long result, dummy;
|
|---|
| 431 | __asm__ __volatile__(
|
|---|
| 432 | "1:\tldarx %0,0,%5\n"
|
|---|
| 433 | "\tcmpd %0,%4\n"
|
|---|
| 434 | "\tbne 2f\n"
|
|---|
| 435 | "\tstdcx. %3,0,%2\n"
|
|---|
| 436 | "\tbne- 1b\n"
|
|---|
| 437 | "\tsync\n"
|
|---|
| 438 | "\tli %1, 1\n"
|
|---|
| 439 | "\tb 3f\n"
|
|---|
| 440 | "2:\tli %1, 0\n"
|
|---|
| 441 | "3:\t\n"
|
|---|
| 442 | : "=&r" (dummy), "=r" (result), "=p" (addr)
|
|---|
| 443 | : "r" (new_val), "r" (old), "2"(addr)
|
|---|
| 444 | : "cr0","memory");
|
|---|
| 445 | return (GC_bool) result;
|
|---|
| 446 | }
|
|---|
| 447 | # else
|
|---|
| 448 | /* Returns TRUE if the comparison succeeded. */
|
|---|
| 449 | inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 450 | GC_word old, GC_word new_val)
|
|---|
| 451 | {
|
|---|
| 452 | int result, dummy;
|
|---|
| 453 | __asm__ __volatile__(
|
|---|
| 454 | "1:\tlwarx %0,0,%5\n"
|
|---|
| 455 | "\tcmpw %0,%4\n"
|
|---|
| 456 | "\tbne 2f\n"
|
|---|
| 457 | "\tstwcx. %3,0,%2\n"
|
|---|
| 458 | "\tbne- 1b\n"
|
|---|
| 459 | "\tsync\n"
|
|---|
| 460 | "\tli %1, 1\n"
|
|---|
| 461 | "\tb 3f\n"
|
|---|
| 462 | "2:\tli %1, 0\n"
|
|---|
| 463 | "3:\t\n"
|
|---|
| 464 | : "=&r" (dummy), "=r" (result), "=p" (addr)
|
|---|
| 465 | : "r" (new_val), "r" (old), "2"(addr)
|
|---|
| 466 | : "cr0","memory");
|
|---|
| 467 | return (GC_bool) result;
|
|---|
| 468 | }
|
|---|
| 469 | # endif
|
|---|
| 470 | # endif /* !GENERIC_COMPARE_AND_SWAP */
|
|---|
| 471 | inline static void GC_memory_barrier()
|
|---|
| 472 | {
|
|---|
| 473 | __asm__ __volatile__("sync" : : : "memory");
|
|---|
| 474 | }
|
|---|
| 475 | # endif /* POWERPC */
|
|---|
| 476 |
|
|---|
| 477 | # if defined(IA64)
|
|---|
| 478 | # if !defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 479 | inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 480 | GC_word old, GC_word new_val)
|
|---|
| 481 | {
|
|---|
| 482 | unsigned long oldval;
|
|---|
| 483 | # if CPP_WORDSZ == 32
|
|---|
| 484 | __asm__ __volatile__(
|
|---|
| 485 | "addp4 %0=0,%1\n"
|
|---|
| 486 | "mov ar.ccv=%3 ;; cmpxchg4.rel %0=[%0],%2,ar.ccv"
|
|---|
| 487 | : "=&r"(oldval)
|
|---|
| 488 | : "r"(addr), "r"(new_val), "r"(old) : "memory");
|
|---|
| 489 | # else
|
|---|
| 490 | __asm__ __volatile__(
|
|---|
| 491 | "mov ar.ccv=%3 ;; cmpxchg8.rel %0=[%1],%2,ar.ccv"
|
|---|
| 492 | : "=r"(oldval)
|
|---|
| 493 | : "r"(addr), "r"(new_val), "r"(old) : "memory");
|
|---|
| 494 | # endif
|
|---|
| 495 | return (oldval == old);
|
|---|
| 496 | }
|
|---|
| 497 | # endif /* !GENERIC_COMPARE_AND_SWAP */
|
|---|
| 498 | # if 0
|
|---|
| 499 | /* Shouldn't be needed; we use volatile stores instead. */
|
|---|
| 500 | inline static void GC_memory_barrier()
|
|---|
| 501 | {
|
|---|
| 502 | __asm__ __volatile__("mf" : : : "memory");
|
|---|
| 503 | }
|
|---|
| 504 | # endif /* 0 */
|
|---|
| 505 | # endif /* IA64 */
|
|---|
| 506 | # if defined(ALPHA)
|
|---|
| 507 | # if !defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 508 | # if defined(__GNUC__)
|
|---|
| 509 | inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 510 | GC_word old, GC_word new_val)
|
|---|
| 511 | {
|
|---|
| 512 | unsigned long was_equal;
|
|---|
| 513 | unsigned long temp;
|
|---|
| 514 |
|
|---|
| 515 | __asm__ __volatile__(
|
|---|
| 516 | "1: ldq_l %0,%1\n"
|
|---|
| 517 | " cmpeq %0,%4,%2\n"
|
|---|
| 518 | " mov %3,%0\n"
|
|---|
| 519 | " beq %2,2f\n"
|
|---|
| 520 | " stq_c %0,%1\n"
|
|---|
| 521 | " beq %0,1b\n"
|
|---|
| 522 | "2:\n"
|
|---|
| 523 | " mb\n"
|
|---|
| 524 | :"=&r" (temp), "=m" (*addr), "=&r" (was_equal)
|
|---|
| 525 | : "r" (new_val), "Ir" (old)
|
|---|
| 526 | :"memory");
|
|---|
| 527 | return was_equal;
|
|---|
| 528 | }
|
|---|
| 529 | # else /* !__GNUC__ */
|
|---|
| 530 | inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr,
|
|---|
| 531 | GC_word old, GC_word new_val)
|
|---|
| 532 | {
|
|---|
| 533 | return __CMP_STORE_QUAD(addr, old, new_val, addr);
|
|---|
| 534 | }
|
|---|
| 535 | # endif /* !__GNUC__ */
|
|---|
| 536 | # endif /* !GENERIC_COMPARE_AND_SWAP */
|
|---|
| 537 | # ifdef __GNUC__
|
|---|
| 538 | inline static void GC_memory_barrier()
|
|---|
| 539 | {
|
|---|
| 540 | __asm__ __volatile__("mb" : : : "memory");
|
|---|
| 541 | }
|
|---|
| 542 | # else
|
|---|
| 543 | # define GC_memory_barrier() asm("mb")
|
|---|
| 544 | # endif /* !__GNUC__ */
|
|---|
| 545 | # endif /* ALPHA */
|
|---|
| 546 | # if defined(S390)
|
|---|
| 547 | # if !defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 548 | inline static GC_bool GC_compare_and_exchange(volatile C_word *addr,
|
|---|
| 549 | GC_word old, GC_word new_val)
|
|---|
| 550 | {
|
|---|
| 551 | int retval;
|
|---|
| 552 | __asm__ __volatile__ (
|
|---|
| 553 | # ifndef __s390x__
|
|---|
| 554 | " cs %1,%2,0(%3)\n"
|
|---|
| 555 | # else
|
|---|
| 556 | " csg %1,%2,0(%3)\n"
|
|---|
| 557 | # endif
|
|---|
| 558 | " ipm %0\n"
|
|---|
| 559 | " srl %0,28\n"
|
|---|
| 560 | : "=&d" (retval), "+d" (old)
|
|---|
| 561 | : "d" (new_val), "a" (addr)
|
|---|
| 562 | : "cc", "memory");
|
|---|
| 563 | return retval == 0;
|
|---|
| 564 | }
|
|---|
| 565 | # endif
|
|---|
| 566 | # endif
|
|---|
| 567 | # if !defined(GENERIC_COMPARE_AND_SWAP)
|
|---|
| 568 | /* Returns the original value of *addr. */
|
|---|
| 569 | inline static GC_word GC_atomic_add(volatile GC_word *addr,
|
|---|
| 570 | GC_word how_much)
|
|---|
| 571 | {
|
|---|
| 572 | GC_word old;
|
|---|
| 573 | do {
|
|---|
| 574 | old = *addr;
|
|---|
| 575 | } while (!GC_compare_and_exchange(addr, old, old+how_much));
|
|---|
| 576 | return old;
|
|---|
| 577 | }
|
|---|
| 578 | # else /* GENERIC_COMPARE_AND_SWAP */
|
|---|
| 579 | /* So long as a GC_word can be atomically updated, it should */
|
|---|
| 580 | /* be OK to read *addr without a lock. */
|
|---|
| 581 | extern GC_word GC_atomic_add(volatile GC_word *addr, GC_word how_much);
|
|---|
| 582 | # endif /* GENERIC_COMPARE_AND_SWAP */
|
|---|
| 583 |
|
|---|
| 584 | # endif /* PARALLEL_MARK */
|
|---|
| 585 |
|
|---|
| 586 | # if !defined(THREAD_LOCAL_ALLOC) && !defined(USE_PTHREAD_LOCKS)
|
|---|
| 587 | /* In the THREAD_LOCAL_ALLOC case, the allocation lock tends to */
|
|---|
| 588 | /* be held for long periods, if it is held at all. Thus spinning */
|
|---|
| 589 | /* and sleeping for fixed periods are likely to result in */
|
|---|
| 590 | /* significant wasted time. We thus rely mostly on queued locks. */
|
|---|
| 591 | # define USE_SPIN_LOCK
|
|---|
| 592 | extern volatile unsigned int GC_allocate_lock;
|
|---|
| 593 | extern void GC_lock(void);
|
|---|
| 594 | /* Allocation lock holder. Only set if acquired by client through */
|
|---|
| 595 | /* GC_call_with_alloc_lock. */
|
|---|
| 596 | # ifdef GC_ASSERTIONS
|
|---|
| 597 | # define LOCK() \
|
|---|
| 598 | { if (GC_test_and_set(&GC_allocate_lock)) GC_lock(); \
|
|---|
| 599 | SET_LOCK_HOLDER(); }
|
|---|
| 600 | # define UNLOCK() \
|
|---|
| 601 | { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
|
|---|
| 602 | GC_clear(&GC_allocate_lock); }
|
|---|
| 603 | # else
|
|---|
| 604 | # define LOCK() \
|
|---|
| 605 | { if (GC_test_and_set(&GC_allocate_lock)) GC_lock(); }
|
|---|
| 606 | # define UNLOCK() \
|
|---|
| 607 | GC_clear(&GC_allocate_lock)
|
|---|
| 608 | # endif /* !GC_ASSERTIONS */
|
|---|
| 609 | # if 0
|
|---|
| 610 | /* Another alternative for OSF1 might be: */
|
|---|
| 611 | # include <sys/mman.h>
|
|---|
| 612 | extern msemaphore GC_allocate_semaphore;
|
|---|
| 613 | # define LOCK() { if (msem_lock(&GC_allocate_semaphore, MSEM_IF_NOWAIT) \
|
|---|
| 614 | != 0) GC_lock(); else GC_allocate_lock = 1; }
|
|---|
| 615 | /* The following is INCORRECT, since the memory model is too weak. */
|
|---|
| 616 | /* Is this true? Presumably msem_unlock has the right semantics? */
|
|---|
| 617 | /* - HB */
|
|---|
| 618 | # define UNLOCK() { GC_allocate_lock = 0; \
|
|---|
| 619 | msem_unlock(&GC_allocate_semaphore, 0); }
|
|---|
| 620 | # endif /* 0 */
|
|---|
| 621 | # else /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */
|
|---|
| 622 | # ifndef USE_PTHREAD_LOCKS
|
|---|
| 623 | # define USE_PTHREAD_LOCKS
|
|---|
| 624 | # endif
|
|---|
| 625 | # endif /* THREAD_LOCAL_ALLOC */
|
|---|
| 626 | # ifdef USE_PTHREAD_LOCKS
|
|---|
| 627 | # include <pthread.h>
|
|---|
| 628 | extern pthread_mutex_t GC_allocate_ml;
|
|---|
| 629 | # ifdef GC_ASSERTIONS
|
|---|
| 630 | # define LOCK() \
|
|---|
| 631 | { GC_lock(); \
|
|---|
| 632 | SET_LOCK_HOLDER(); }
|
|---|
| 633 | # define UNLOCK() \
|
|---|
| 634 | { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \
|
|---|
| 635 | pthread_mutex_unlock(&GC_allocate_ml); }
|
|---|
| 636 | # else /* !GC_ASSERTIONS */
|
|---|
| 637 | # if defined(NO_PTHREAD_TRYLOCK)
|
|---|
| 638 | # define LOCK() GC_lock();
|
|---|
| 639 | # else /* !defined(NO_PTHREAD_TRYLOCK) */
|
|---|
| 640 | # define LOCK() \
|
|---|
| 641 | { if (0 != pthread_mutex_trylock(&GC_allocate_ml)) GC_lock(); }
|
|---|
| 642 | # endif
|
|---|
| 643 | # define UNLOCK() pthread_mutex_unlock(&GC_allocate_ml)
|
|---|
| 644 | # endif /* !GC_ASSERTIONS */
|
|---|
| 645 | # endif /* USE_PTHREAD_LOCKS */
|
|---|
| 646 | # define SET_LOCK_HOLDER() GC_lock_holder = pthread_self()
|
|---|
| 647 | # define UNSET_LOCK_HOLDER() GC_lock_holder = NO_THREAD
|
|---|
| 648 | # define I_HOLD_LOCK() (pthread_equal(GC_lock_holder, pthread_self()))
|
|---|
| 649 | extern VOLATILE GC_bool GC_collecting;
|
|---|
| 650 | # define ENTER_GC() GC_collecting = 1;
|
|---|
| 651 | # define EXIT_GC() GC_collecting = 0;
|
|---|
| 652 | extern void GC_lock(void);
|
|---|
| 653 | extern pthread_t GC_lock_holder;
|
|---|
| 654 | # ifdef GC_ASSERTIONS
|
|---|
| 655 | extern pthread_t GC_mark_lock_holder;
|
|---|
| 656 | # endif
|
|---|
| 657 | # endif /* GC_PTHREADS with linux_threads.c implementation */
|
|---|
| 658 | # if defined(GC_WIN32_THREADS)
|
|---|
| 659 | # if defined(GC_PTHREADS)
|
|---|
| 660 | # include <pthread.h>
|
|---|
| 661 | extern pthread_mutex_t GC_allocate_ml;
|
|---|
| 662 | # define LOCK() pthread_mutex_lock(&GC_allocate_ml)
|
|---|
| 663 | # define UNLOCK() pthread_mutex_unlock(&GC_allocate_ml)
|
|---|
| 664 | # else
|
|---|
| 665 | # include <windows.h>
|
|---|
| 666 | GC_API CRITICAL_SECTION GC_allocate_ml;
|
|---|
| 667 | # define LOCK() EnterCriticalSection(&GC_allocate_ml);
|
|---|
| 668 | # define UNLOCK() LeaveCriticalSection(&GC_allocate_ml);
|
|---|
| 669 | # endif
|
|---|
| 670 | # endif
|
|---|
| 671 | # ifndef SET_LOCK_HOLDER
|
|---|
| 672 | # define SET_LOCK_HOLDER()
|
|---|
| 673 | # define UNSET_LOCK_HOLDER()
|
|---|
| 674 | # define I_HOLD_LOCK() FALSE
|
|---|
| 675 | /* Used on platforms were locks can be reacquired, */
|
|---|
| 676 | /* so it doesn't matter if we lie. */
|
|---|
| 677 | # endif
|
|---|
| 678 | # else /* !THREADS */
|
|---|
| 679 | # define LOCK()
|
|---|
| 680 | # define UNLOCK()
|
|---|
| 681 | # endif /* !THREADS */
|
|---|
| 682 | # ifndef SET_LOCK_HOLDER
|
|---|
| 683 | # define SET_LOCK_HOLDER()
|
|---|
| 684 | # define UNSET_LOCK_HOLDER()
|
|---|
| 685 | # define I_HOLD_LOCK() FALSE
|
|---|
| 686 | /* Used on platforms were locks can be reacquired, */
|
|---|
| 687 | /* so it doesn't matter if we lie. */
|
|---|
| 688 | # endif
|
|---|
| 689 | # ifndef ENTER_GC
|
|---|
| 690 | # define ENTER_GC()
|
|---|
| 691 | # define EXIT_GC()
|
|---|
| 692 | # endif
|
|---|
| 693 |
|
|---|
| 694 | # ifndef DCL_LOCK_STATE
|
|---|
| 695 | # define DCL_LOCK_STATE
|
|---|
| 696 | # endif
|
|---|
| 697 | # ifndef FASTLOCK
|
|---|
| 698 | # define FASTLOCK() LOCK()
|
|---|
| 699 | # define FASTLOCK_SUCCEEDED() TRUE
|
|---|
| 700 | # define FASTUNLOCK() UNLOCK()
|
|---|
| 701 | # endif
|
|---|
| 702 |
|
|---|
| 703 | #endif /* GC_LOCKS_H */
|
|---|