Source file
src/runtime/malloc_generated.go
1
2
3
4 package runtime
5
6 import (
7 "internal/goarch"
8 "internal/goexperiment"
9 "internal/runtime/sys"
10 "unsafe"
11 )
12
13 func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
14 if doubleCheckMalloc {
15 if gcphase == _GCmarktermination {
16 throw("mallocgc called with gcphase == _GCmarktermination")
17 }
18 }
19
20 var mp *m
21
22 mp = acquirem()
23
24 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
25
26 if forceSlowPath {
27 releasem(mp)
28
29 const spc = spanClass(1<<1) | spanClass(0)
30 const elemsize = uintptr(8)
31 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
32
33 }
34
35 lockRankMayQueueFinalizer()
36
37 const sizeclass = 1
38 const elemsize = 8
39
40 if doubleCheckMalloc {
41
42 doubleCheckSmallScanNoHeader(size, typ, mp)
43
44 }
45 mp.mallocing = 1
46
47 checkGCTrigger := false
48 c := getMCache(mp)
49 const spc = spanClass(sizeclass<<1) | spanClass(0)
50 span := c.alloc[spc]
51
52 var v gclinkptr
53 var x unsafe.Pointer
54
55 {
56
57 var nextFreeFastResult gclinkptr
58 if span.allocCache != 0 {
59 theBit := sys.TrailingZeros64(span.allocCache)
60 result := span.freeindex + uint16(theBit)
61 if result < span.nelems {
62 freeidx := result + 1
63 if !(freeidx%64 == 0 && freeidx != span.nelems) {
64 span.allocCache >>= uint(theBit + 1)
65 span.freeindex = freeidx
66 span.allocCount++
67 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
68 }
69 }
70 }
71 v = nextFreeFastResult
72 if v == 0 {
73 v, span, checkGCTrigger = c.nextFree(spc)
74 }
75 x = unsafe.Pointer(v)
76 }
77
78 if span.needzero != 0 {
79 memclrNoHeapPointers(x, elemsize)
80 }
81 if goarch.PtrSize == 8 && elemsize == 8 {
82
83 c.scanAlloc += 8
84 } else {
85 dataSize := size
86 x := uintptr(x)
87
88 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(8)) {
89 throw("tried to write heap bits, but no heap bits in span")
90 }
91
92 src0 := readUintptr(getGCMask(typ))
93
94 const elemsize = 8
95
96 var scanSize uintptr
97 src := src0
98 if typ.Size_ == goarch.PtrSize {
99 src = (1 << (dataSize / goarch.PtrSize)) - 1
100
101 scanSize = dataSize
102 } else {
103
104 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
105 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
106 }
107 scanSize = typ.PtrBytes
108 for i := typ.Size_; i < dataSize; i += typ.Size_ {
109 src |= src0 << (i / goarch.PtrSize)
110 scanSize += typ.Size_
111 }
112 }
113
114 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
115 dst := unsafe.Pointer(dstBase)
116 o := (x - span.base()) / goarch.PtrSize
117 i := o / ptrBits
118 j := o % ptrBits
119 var bits uintptr = elemsize / goarch.PtrSize
120
121 var bitsIsPowerOfTwo = bits&(bits-1) == 0
122 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
123
124 bits0 := ptrBits - j
125 bits1 := bits - bits0
126 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
127 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
128 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
129 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
130 } else {
131
132 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
133 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
134 }
135
136 const doubleCheck = false
137 if doubleCheck {
138 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
139 }
140 if doubleCheckHeapSetType {
141 doubleCheckHeapType(x, dataSize, typ, nil, span)
142 }
143 c.scanAlloc += scanSize
144 }
145
146 publicationBarrier()
147
148 span.freeIndexForScan = span.freeindex
149
150 c.nextSample -= int64(elemsize)
151 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
152 profilealloc(mp, x, elemsize)
153 }
154 mp.mallocing = 0
155 releasem(mp)
156
157 if checkGCTrigger {
158 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
159 gcStart(t)
160 }
161 }
162
163 return x
164 }
165
166 func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
167 if doubleCheckMalloc {
168 if gcphase == _GCmarktermination {
169 throw("mallocgc called with gcphase == _GCmarktermination")
170 }
171 }
172
173 var mp *m
174
175 mp = acquirem()
176
177 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
178
179 if forceSlowPath {
180 releasem(mp)
181
182 const spc = spanClass(2<<1) | spanClass(0)
183 const elemsize = uintptr(16)
184 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
185
186 }
187
188 lockRankMayQueueFinalizer()
189
190 const sizeclass = 2
191 const elemsize = 16
192
193 if doubleCheckMalloc {
194
195 doubleCheckSmallScanNoHeader(size, typ, mp)
196
197 }
198 mp.mallocing = 1
199
200 checkGCTrigger := false
201 c := getMCache(mp)
202 const spc = spanClass(sizeclass<<1) | spanClass(0)
203 span := c.alloc[spc]
204
205 var v gclinkptr
206 var x unsafe.Pointer
207
208 {
209
210 var nextFreeFastResult gclinkptr
211 if span.allocCache != 0 {
212 theBit := sys.TrailingZeros64(span.allocCache)
213 result := span.freeindex + uint16(theBit)
214 if result < span.nelems {
215 freeidx := result + 1
216 if !(freeidx%64 == 0 && freeidx != span.nelems) {
217 span.allocCache >>= uint(theBit + 1)
218 span.freeindex = freeidx
219 span.allocCount++
220 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
221 }
222 }
223 }
224 v = nextFreeFastResult
225 if v == 0 {
226 v, span, checkGCTrigger = c.nextFree(spc)
227 }
228 x = unsafe.Pointer(v)
229 }
230
231 if span.needzero != 0 {
232 memclrNoHeapPointers(x, elemsize)
233 }
234 if goarch.PtrSize == 8 && elemsize == 8 {
235
236 c.scanAlloc += 8
237 } else {
238 dataSize := size
239 x := uintptr(x)
240
241 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(16)) {
242 throw("tried to write heap bits, but no heap bits in span")
243 }
244
245 src0 := readUintptr(getGCMask(typ))
246
247 const elemsize = 16
248
249 var scanSize uintptr
250 src := src0
251 if typ.Size_ == goarch.PtrSize {
252 src = (1 << (dataSize / goarch.PtrSize)) - 1
253
254 scanSize = dataSize
255 } else {
256
257 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
258 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
259 }
260 scanSize = typ.PtrBytes
261 for i := typ.Size_; i < dataSize; i += typ.Size_ {
262 src |= src0 << (i / goarch.PtrSize)
263 scanSize += typ.Size_
264 }
265 }
266
267 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
268 dst := unsafe.Pointer(dstBase)
269 o := (x - span.base()) / goarch.PtrSize
270 i := o / ptrBits
271 j := o % ptrBits
272 var bits uintptr = elemsize / goarch.PtrSize
273
274 var bitsIsPowerOfTwo = bits&(bits-1) == 0
275 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
276
277 bits0 := ptrBits - j
278 bits1 := bits - bits0
279 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
280 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
281 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
282 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
283 } else {
284
285 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
286 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
287 }
288
289 const doubleCheck = false
290 if doubleCheck {
291 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
292 }
293 if doubleCheckHeapSetType {
294 doubleCheckHeapType(x, dataSize, typ, nil, span)
295 }
296 c.scanAlloc += scanSize
297 }
298
299 publicationBarrier()
300
301 span.freeIndexForScan = span.freeindex
302
303 c.nextSample -= int64(elemsize)
304 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
305 profilealloc(mp, x, elemsize)
306 }
307 mp.mallocing = 0
308 releasem(mp)
309
310 if checkGCTrigger {
311 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
312 gcStart(t)
313 }
314 }
315
316 return x
317 }
318
319 func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
320 if doubleCheckMalloc {
321 if gcphase == _GCmarktermination {
322 throw("mallocgc called with gcphase == _GCmarktermination")
323 }
324 }
325
326 var mp *m
327
328 mp = acquirem()
329
330 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
331
332 if forceSlowPath {
333 releasem(mp)
334
335 const spc = spanClass(3<<1) | spanClass(0)
336 const elemsize = uintptr(24)
337 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
338
339 }
340
341 lockRankMayQueueFinalizer()
342
343 const sizeclass = 3
344 const elemsize = 24
345
346 if doubleCheckMalloc {
347
348 doubleCheckSmallScanNoHeader(size, typ, mp)
349
350 }
351 mp.mallocing = 1
352
353 checkGCTrigger := false
354 c := getMCache(mp)
355 const spc = spanClass(sizeclass<<1) | spanClass(0)
356 span := c.alloc[spc]
357
358 var v gclinkptr
359 var x unsafe.Pointer
360
361 {
362
363 var nextFreeFastResult gclinkptr
364 if span.allocCache != 0 {
365 theBit := sys.TrailingZeros64(span.allocCache)
366 result := span.freeindex + uint16(theBit)
367 if result < span.nelems {
368 freeidx := result + 1
369 if !(freeidx%64 == 0 && freeidx != span.nelems) {
370 span.allocCache >>= uint(theBit + 1)
371 span.freeindex = freeidx
372 span.allocCount++
373 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
374 }
375 }
376 }
377 v = nextFreeFastResult
378 if v == 0 {
379 v, span, checkGCTrigger = c.nextFree(spc)
380 }
381 x = unsafe.Pointer(v)
382 }
383
384 if span.needzero != 0 {
385 memclrNoHeapPointers(x, elemsize)
386 }
387 if goarch.PtrSize == 8 && elemsize == 8 {
388
389 c.scanAlloc += 8
390 } else {
391 dataSize := size
392 x := uintptr(x)
393
394 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(24)) {
395 throw("tried to write heap bits, but no heap bits in span")
396 }
397
398 src0 := readUintptr(getGCMask(typ))
399
400 const elemsize = 24
401
402 var scanSize uintptr
403 src := src0
404 if typ.Size_ == goarch.PtrSize {
405 src = (1 << (dataSize / goarch.PtrSize)) - 1
406
407 scanSize = dataSize
408 } else {
409
410 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
411 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
412 }
413 scanSize = typ.PtrBytes
414 for i := typ.Size_; i < dataSize; i += typ.Size_ {
415 src |= src0 << (i / goarch.PtrSize)
416 scanSize += typ.Size_
417 }
418 }
419
420 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
421 dst := unsafe.Pointer(dstBase)
422 o := (x - span.base()) / goarch.PtrSize
423 i := o / ptrBits
424 j := o % ptrBits
425 var bits uintptr = elemsize / goarch.PtrSize
426
427 var bitsIsPowerOfTwo = bits&(bits-1) == 0
428 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
429
430 bits0 := ptrBits - j
431 bits1 := bits - bits0
432 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
433 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
434 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
435 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
436 } else {
437
438 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
439 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
440 }
441
442 const doubleCheck = false
443 if doubleCheck {
444 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
445 }
446 if doubleCheckHeapSetType {
447 doubleCheckHeapType(x, dataSize, typ, nil, span)
448 }
449 c.scanAlloc += scanSize
450 }
451
452 publicationBarrier()
453
454 span.freeIndexForScan = span.freeindex
455
456 c.nextSample -= int64(elemsize)
457 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
458 profilealloc(mp, x, elemsize)
459 }
460 mp.mallocing = 0
461 releasem(mp)
462
463 if checkGCTrigger {
464 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
465 gcStart(t)
466 }
467 }
468
469 return x
470 }
471
472 func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
473 if doubleCheckMalloc {
474 if gcphase == _GCmarktermination {
475 throw("mallocgc called with gcphase == _GCmarktermination")
476 }
477 }
478
479 var mp *m
480
481 mp = acquirem()
482
483 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
484
485 if forceSlowPath {
486 releasem(mp)
487
488 const spc = spanClass(4<<1) | spanClass(0)
489 const elemsize = uintptr(32)
490 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
491
492 }
493
494 lockRankMayQueueFinalizer()
495
496 const sizeclass = 4
497 const elemsize = 32
498
499 if doubleCheckMalloc {
500
501 doubleCheckSmallScanNoHeader(size, typ, mp)
502
503 }
504 mp.mallocing = 1
505
506 checkGCTrigger := false
507 c := getMCache(mp)
508 const spc = spanClass(sizeclass<<1) | spanClass(0)
509 span := c.alloc[spc]
510
511 var v gclinkptr
512 var x unsafe.Pointer
513
514 {
515
516 var nextFreeFastResult gclinkptr
517 if span.allocCache != 0 {
518 theBit := sys.TrailingZeros64(span.allocCache)
519 result := span.freeindex + uint16(theBit)
520 if result < span.nelems {
521 freeidx := result + 1
522 if !(freeidx%64 == 0 && freeidx != span.nelems) {
523 span.allocCache >>= uint(theBit + 1)
524 span.freeindex = freeidx
525 span.allocCount++
526 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
527 }
528 }
529 }
530 v = nextFreeFastResult
531 if v == 0 {
532 v, span, checkGCTrigger = c.nextFree(spc)
533 }
534 x = unsafe.Pointer(v)
535 }
536
537 if span.needzero != 0 {
538 memclrNoHeapPointers(x, elemsize)
539 }
540 if goarch.PtrSize == 8 && elemsize == 8 {
541
542 c.scanAlloc += 8
543 } else {
544 dataSize := size
545 x := uintptr(x)
546
547 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(32)) {
548 throw("tried to write heap bits, but no heap bits in span")
549 }
550
551 src0 := readUintptr(getGCMask(typ))
552
553 const elemsize = 32
554
555 var scanSize uintptr
556 src := src0
557 if typ.Size_ == goarch.PtrSize {
558 src = (1 << (dataSize / goarch.PtrSize)) - 1
559
560 scanSize = dataSize
561 } else {
562
563 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
564 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
565 }
566 scanSize = typ.PtrBytes
567 for i := typ.Size_; i < dataSize; i += typ.Size_ {
568 src |= src0 << (i / goarch.PtrSize)
569 scanSize += typ.Size_
570 }
571 }
572
573 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
574 dst := unsafe.Pointer(dstBase)
575 o := (x - span.base()) / goarch.PtrSize
576 i := o / ptrBits
577 j := o % ptrBits
578 var bits uintptr = elemsize / goarch.PtrSize
579
580 var bitsIsPowerOfTwo = bits&(bits-1) == 0
581 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
582
583 bits0 := ptrBits - j
584 bits1 := bits - bits0
585 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
586 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
587 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
588 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
589 } else {
590
591 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
592 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
593 }
594
595 const doubleCheck = false
596 if doubleCheck {
597 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
598 }
599 if doubleCheckHeapSetType {
600 doubleCheckHeapType(x, dataSize, typ, nil, span)
601 }
602 c.scanAlloc += scanSize
603 }
604
605 publicationBarrier()
606
607 span.freeIndexForScan = span.freeindex
608
609 c.nextSample -= int64(elemsize)
610 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
611 profilealloc(mp, x, elemsize)
612 }
613 mp.mallocing = 0
614 releasem(mp)
615
616 if checkGCTrigger {
617 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
618 gcStart(t)
619 }
620 }
621
622 return x
623 }
624
625 func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
626 if doubleCheckMalloc {
627 if gcphase == _GCmarktermination {
628 throw("mallocgc called with gcphase == _GCmarktermination")
629 }
630 }
631
632 var mp *m
633
634 mp = acquirem()
635
636 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
637
638 if forceSlowPath {
639 releasem(mp)
640
641 const spc = spanClass(5<<1) | spanClass(0)
642 const elemsize = uintptr(48)
643 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
644
645 }
646
647 lockRankMayQueueFinalizer()
648
649 const sizeclass = 5
650 const elemsize = 48
651
652 if doubleCheckMalloc {
653
654 doubleCheckSmallScanNoHeader(size, typ, mp)
655
656 }
657 mp.mallocing = 1
658
659 checkGCTrigger := false
660 c := getMCache(mp)
661 const spc = spanClass(sizeclass<<1) | spanClass(0)
662 span := c.alloc[spc]
663
664 var v gclinkptr
665 var x unsafe.Pointer
666
667 {
668
669 var nextFreeFastResult gclinkptr
670 if span.allocCache != 0 {
671 theBit := sys.TrailingZeros64(span.allocCache)
672 result := span.freeindex + uint16(theBit)
673 if result < span.nelems {
674 freeidx := result + 1
675 if !(freeidx%64 == 0 && freeidx != span.nelems) {
676 span.allocCache >>= uint(theBit + 1)
677 span.freeindex = freeidx
678 span.allocCount++
679 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
680 }
681 }
682 }
683 v = nextFreeFastResult
684 if v == 0 {
685 v, span, checkGCTrigger = c.nextFree(spc)
686 }
687 x = unsafe.Pointer(v)
688 }
689
690 if span.needzero != 0 {
691 memclrNoHeapPointers(x, elemsize)
692 }
693 if goarch.PtrSize == 8 && elemsize == 8 {
694
695 c.scanAlloc += 8
696 } else {
697 dataSize := size
698 x := uintptr(x)
699
700 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(48)) {
701 throw("tried to write heap bits, but no heap bits in span")
702 }
703
704 src0 := readUintptr(getGCMask(typ))
705
706 const elemsize = 48
707
708 var scanSize uintptr
709 src := src0
710 if typ.Size_ == goarch.PtrSize {
711 src = (1 << (dataSize / goarch.PtrSize)) - 1
712
713 scanSize = dataSize
714 } else {
715
716 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
717 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
718 }
719 scanSize = typ.PtrBytes
720 for i := typ.Size_; i < dataSize; i += typ.Size_ {
721 src |= src0 << (i / goarch.PtrSize)
722 scanSize += typ.Size_
723 }
724 }
725
726 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
727 dst := unsafe.Pointer(dstBase)
728 o := (x - span.base()) / goarch.PtrSize
729 i := o / ptrBits
730 j := o % ptrBits
731 var bits uintptr = elemsize / goarch.PtrSize
732
733 var bitsIsPowerOfTwo = bits&(bits-1) == 0
734 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
735
736 bits0 := ptrBits - j
737 bits1 := bits - bits0
738 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
739 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
740 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
741 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
742 } else {
743
744 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
745 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
746 }
747
748 const doubleCheck = false
749 if doubleCheck {
750 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
751 }
752 if doubleCheckHeapSetType {
753 doubleCheckHeapType(x, dataSize, typ, nil, span)
754 }
755 c.scanAlloc += scanSize
756 }
757
758 publicationBarrier()
759
760 span.freeIndexForScan = span.freeindex
761
762 c.nextSample -= int64(elemsize)
763 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
764 profilealloc(mp, x, elemsize)
765 }
766 mp.mallocing = 0
767 releasem(mp)
768
769 if checkGCTrigger {
770 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
771 gcStart(t)
772 }
773 }
774
775 return x
776 }
777
778 func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
779 if doubleCheckMalloc {
780 if gcphase == _GCmarktermination {
781 throw("mallocgc called with gcphase == _GCmarktermination")
782 }
783 }
784
785 var mp *m
786
787 mp = acquirem()
788
789 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
790
791 if forceSlowPath {
792 releasem(mp)
793
794 const spc = spanClass(6<<1) | spanClass(0)
795 const elemsize = uintptr(64)
796 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
797
798 }
799
800 lockRankMayQueueFinalizer()
801
802 const sizeclass = 6
803 const elemsize = 64
804
805 if doubleCheckMalloc {
806
807 doubleCheckSmallScanNoHeader(size, typ, mp)
808
809 }
810 mp.mallocing = 1
811
812 checkGCTrigger := false
813 c := getMCache(mp)
814 const spc = spanClass(sizeclass<<1) | spanClass(0)
815 span := c.alloc[spc]
816
817 var v gclinkptr
818 var x unsafe.Pointer
819
820 {
821
822 var nextFreeFastResult gclinkptr
823 if span.allocCache != 0 {
824 theBit := sys.TrailingZeros64(span.allocCache)
825 result := span.freeindex + uint16(theBit)
826 if result < span.nelems {
827 freeidx := result + 1
828 if !(freeidx%64 == 0 && freeidx != span.nelems) {
829 span.allocCache >>= uint(theBit + 1)
830 span.freeindex = freeidx
831 span.allocCount++
832 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
833 }
834 }
835 }
836 v = nextFreeFastResult
837 if v == 0 {
838 v, span, checkGCTrigger = c.nextFree(spc)
839 }
840 x = unsafe.Pointer(v)
841 }
842
843 if span.needzero != 0 {
844 memclrNoHeapPointers(x, elemsize)
845 }
846 if goarch.PtrSize == 8 && elemsize == 8 {
847
848 c.scanAlloc += 8
849 } else {
850 dataSize := size
851 x := uintptr(x)
852
853 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(64)) {
854 throw("tried to write heap bits, but no heap bits in span")
855 }
856
857 src0 := readUintptr(getGCMask(typ))
858
859 const elemsize = 64
860
861 var scanSize uintptr
862 src := src0
863 if typ.Size_ == goarch.PtrSize {
864 src = (1 << (dataSize / goarch.PtrSize)) - 1
865
866 scanSize = dataSize
867 } else {
868
869 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
870 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
871 }
872 scanSize = typ.PtrBytes
873 for i := typ.Size_; i < dataSize; i += typ.Size_ {
874 src |= src0 << (i / goarch.PtrSize)
875 scanSize += typ.Size_
876 }
877 }
878
879 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
880 dst := unsafe.Pointer(dstBase)
881 o := (x - span.base()) / goarch.PtrSize
882 i := o / ptrBits
883 j := o % ptrBits
884 var bits uintptr = elemsize / goarch.PtrSize
885
886 var bitsIsPowerOfTwo = bits&(bits-1) == 0
887 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
888
889 bits0 := ptrBits - j
890 bits1 := bits - bits0
891 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
892 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
893 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
894 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
895 } else {
896
897 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
898 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
899 }
900
901 const doubleCheck = false
902 if doubleCheck {
903 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
904 }
905 if doubleCheckHeapSetType {
906 doubleCheckHeapType(x, dataSize, typ, nil, span)
907 }
908 c.scanAlloc += scanSize
909 }
910
911 publicationBarrier()
912
913 span.freeIndexForScan = span.freeindex
914
915 c.nextSample -= int64(elemsize)
916 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
917 profilealloc(mp, x, elemsize)
918 }
919 mp.mallocing = 0
920 releasem(mp)
921
922 if checkGCTrigger {
923 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
924 gcStart(t)
925 }
926 }
927
928 return x
929 }
930
931 func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
932 if doubleCheckMalloc {
933 if gcphase == _GCmarktermination {
934 throw("mallocgc called with gcphase == _GCmarktermination")
935 }
936 }
937
938 var mp *m
939
940 mp = acquirem()
941
942 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
943
944 if forceSlowPath {
945 releasem(mp)
946
947 const spc = spanClass(7<<1) | spanClass(0)
948 const elemsize = uintptr(80)
949 return mallocgcSmallScanSlowPath(size, typ, needzero, spc, elemsize)
950
951 }
952
953 lockRankMayQueueFinalizer()
954
955 const sizeclass = 7
956 const elemsize = 80
957
958 if doubleCheckMalloc {
959
960 doubleCheckSmallScanNoHeader(size, typ, mp)
961
962 }
963 mp.mallocing = 1
964
965 checkGCTrigger := false
966 c := getMCache(mp)
967 const spc = spanClass(sizeclass<<1) | spanClass(0)
968 span := c.alloc[spc]
969
970 var v gclinkptr
971 var x unsafe.Pointer
972
973 {
974
975 var nextFreeFastResult gclinkptr
976 if span.allocCache != 0 {
977 theBit := sys.TrailingZeros64(span.allocCache)
978 result := span.freeindex + uint16(theBit)
979 if result < span.nelems {
980 freeidx := result + 1
981 if !(freeidx%64 == 0 && freeidx != span.nelems) {
982 span.allocCache >>= uint(theBit + 1)
983 span.freeindex = freeidx
984 span.allocCount++
985 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
986 }
987 }
988 }
989 v = nextFreeFastResult
990 if v == 0 {
991 v, span, checkGCTrigger = c.nextFree(spc)
992 }
993 x = unsafe.Pointer(v)
994 }
995
996 if span.needzero != 0 {
997 memclrNoHeapPointers(x, elemsize)
998 }
999 if goarch.PtrSize == 8 && elemsize == 8 {
1000
1001 c.scanAlloc += 8
1002 } else {
1003 dataSize := size
1004 x := uintptr(x)
1005
1006 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(80)) {
1007 throw("tried to write heap bits, but no heap bits in span")
1008 }
1009
1010 src0 := readUintptr(getGCMask(typ))
1011
1012 const elemsize = 80
1013
1014 var scanSize uintptr
1015 src := src0
1016 if typ.Size_ == goarch.PtrSize {
1017 src = (1 << (dataSize / goarch.PtrSize)) - 1
1018
1019 scanSize = dataSize
1020 } else {
1021
1022 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
1023 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
1024 }
1025 scanSize = typ.PtrBytes
1026 for i := typ.Size_; i < dataSize; i += typ.Size_ {
1027 src |= src0 << (i / goarch.PtrSize)
1028 scanSize += typ.Size_
1029 }
1030 }
1031
1032 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
1033 dst := unsafe.Pointer(dstBase)
1034 o := (x - span.base()) / goarch.PtrSize
1035 i := o / ptrBits
1036 j := o % ptrBits
1037 var bits uintptr = elemsize / goarch.PtrSize
1038
1039 var bitsIsPowerOfTwo = bits&(bits-1) == 0
1040 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
1041
1042 bits0 := ptrBits - j
1043 bits1 := bits - bits0
1044 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
1045 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
1046 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
1047 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
1048 } else {
1049
1050 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
1051 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
1052 }
1053
1054 const doubleCheck = false
1055 if doubleCheck {
1056 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
1057 }
1058 if doubleCheckHeapSetType {
1059 doubleCheckHeapType(x, dataSize, typ, nil, span)
1060 }
1061 c.scanAlloc += scanSize
1062 }
1063
1064 publicationBarrier()
1065
1066 span.freeIndexForScan = span.freeindex
1067
1068 c.nextSample -= int64(elemsize)
1069 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1070 profilealloc(mp, x, elemsize)
1071 }
1072 mp.mallocing = 0
1073 releasem(mp)
1074
1075 if checkGCTrigger {
1076 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1077 gcStart(t)
1078 }
1079 }
1080
1081 return x
1082 }
1083
1084 func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1085 if doubleCheckMalloc {
1086 if gcphase == _GCmarktermination {
1087 throw("mallocgc called with gcphase == _GCmarktermination")
1088 }
1089 }
1090
1091 var mp *m
1092
1093 mp = acquirem()
1094
1095 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1096
1097 if forceSlowPath {
1098 releasem(mp)
1099
1100 return mallocgcTinySlowPath(size, typ, needzero)
1101
1102 }
1103
1104 lockRankMayQueueFinalizer()
1105
1106 const elemsize = 16
1107
1108 if doubleCheckMalloc {
1109 doubleCheckTiny(size, typ, mp)
1110 }
1111 mp.mallocing = 1
1112
1113 c := getMCache(mp)
1114 off := c.tinyoffset
1115
1116 if size&7 == 0 {
1117 off = alignUp(off, 8)
1118 } else if goarch.PtrSize == 4 && size == 12 {
1119
1120 off = alignUp(off, 8)
1121 } else if size&3 == 0 {
1122 off = alignUp(off, 4)
1123 } else if size&1 == 0 {
1124 off = alignUp(off, 2)
1125 }
1126 if off+size <= maxTinySize && c.tiny != 0 {
1127
1128 x := unsafe.Pointer(c.tiny + off)
1129 c.tinyoffset = off + size
1130 c.tinyAllocs++
1131 mp.mallocing = 0
1132 releasem(mp)
1133 const elemsize = 0
1134
1135 return x
1136 }
1137
1138 checkGCTrigger := false
1139 span := c.alloc[tinySpanClass]
1140
1141 const nbytes = 8192
1142 const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16)
1143 var nextFreeFastResult gclinkptr
1144 if span.allocCache != 0 {
1145 theBit := sys.TrailingZeros64(span.allocCache)
1146 result := span.freeindex + uint16(theBit)
1147 if result < nelems {
1148 freeidx := result + 1
1149 if !(freeidx%64 == 0 && freeidx != nelems) {
1150 span.allocCache >>= uint(theBit + 1)
1151 span.freeindex = freeidx
1152 span.allocCount++
1153 nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base())
1154 }
1155 }
1156 }
1157 v := nextFreeFastResult
1158 if v == 0 {
1159 v, span, checkGCTrigger = c.nextFree(tinySpanClass)
1160 }
1161 x := unsafe.Pointer(v)
1162 (*[2]uint64)(x)[0] = 0
1163 (*[2]uint64)(x)[1] = 0
1164
1165 if !raceenabled && (size < c.tinyoffset || c.tiny == 0) {
1166
1167 c.tiny = uintptr(x)
1168 c.tinyoffset = size
1169 }
1170
1171 publicationBarrier()
1172
1173 span.freeIndexForScan = span.freeindex
1174
1175 c.nextSample -= int64(elemsize)
1176 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1177 profilealloc(mp, x, elemsize)
1178 }
1179 mp.mallocing = 0
1180 releasem(mp)
1181
1182 if checkGCTrigger {
1183 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1184 gcStart(t)
1185 }
1186 }
1187
1188 return x
1189 }
1190
1191 func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1192 if doubleCheckMalloc {
1193 if gcphase == _GCmarktermination {
1194 throw("mallocgc called with gcphase == _GCmarktermination")
1195 }
1196 }
1197
1198 var mp *m
1199
1200 mp = acquirem()
1201
1202 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1203
1204 if forceSlowPath {
1205 releasem(mp)
1206
1207 const spc = spanClass(2<<1) | spanClass(1)
1208 const elemsize = uintptr(16)
1209 return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)
1210
1211 }
1212
1213 lockRankMayQueueFinalizer()
1214
1215 const sizeclass = 2
1216 const elemsize = 16
1217
1218 if doubleCheckMalloc {
1219
1220 doubleCheckSmallNoScan(typ, mp)
1221
1222 }
1223 mp.mallocing = 1
1224
1225 checkGCTrigger := false
1226 c := getMCache(mp)
1227 const spc = spanClass(sizeclass<<1) | spanClass(1)
1228 span := c.alloc[spc]
1229
1230 var v gclinkptr
1231 var x unsafe.Pointer
1232
1233 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
1234
1235 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
1236 mp.mallocing = 0
1237 releasem(mp)
1238
1239 return x
1240
1241 }
1242
1243 {
1244
1245 var nextFreeFastResult gclinkptr
1246 if span.allocCache != 0 {
1247 theBit := sys.TrailingZeros64(span.allocCache)
1248 result := span.freeindex + uint16(theBit)
1249 if result < span.nelems {
1250 freeidx := result + 1
1251 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1252 span.allocCache >>= uint(theBit + 1)
1253 span.freeindex = freeidx
1254 span.allocCount++
1255 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1256 }
1257 }
1258 }
1259 v = nextFreeFastResult
1260 if v == 0 {
1261 v, span, checkGCTrigger = c.nextFree(spc)
1262 }
1263 x = unsafe.Pointer(v)
1264 }
1265
1266 if needzero && span.needzero != 0 {
1267 memclrNoHeapPointers(x, elemsize)
1268 }
1269
1270 publicationBarrier()
1271
1272 span.freeIndexForScan = span.freeindex
1273
1274 c.nextSample -= int64(elemsize)
1275 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1276 profilealloc(mp, x, elemsize)
1277 }
1278 mp.mallocing = 0
1279 releasem(mp)
1280
1281 if checkGCTrigger {
1282 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1283 gcStart(t)
1284 }
1285 }
1286
1287 return x
1288 }
1289
1290 func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1291 if doubleCheckMalloc {
1292 if gcphase == _GCmarktermination {
1293 throw("mallocgc called with gcphase == _GCmarktermination")
1294 }
1295 }
1296
1297 var mp *m
1298
1299 mp = acquirem()
1300
1301 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1302
1303 if forceSlowPath {
1304 releasem(mp)
1305
1306 const spc = spanClass(3<<1) | spanClass(1)
1307 const elemsize = uintptr(24)
1308 return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)
1309
1310 }
1311
1312 lockRankMayQueueFinalizer()
1313
1314 const sizeclass = 3
1315 const elemsize = 24
1316
1317 if doubleCheckMalloc {
1318
1319 doubleCheckSmallNoScan(typ, mp)
1320
1321 }
1322 mp.mallocing = 1
1323
1324 checkGCTrigger := false
1325 c := getMCache(mp)
1326 const spc = spanClass(sizeclass<<1) | spanClass(1)
1327 span := c.alloc[spc]
1328
1329 var v gclinkptr
1330 var x unsafe.Pointer
1331
1332 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
1333
1334 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
1335 mp.mallocing = 0
1336 releasem(mp)
1337
1338 return x
1339
1340 }
1341
1342 {
1343
1344 var nextFreeFastResult gclinkptr
1345 if span.allocCache != 0 {
1346 theBit := sys.TrailingZeros64(span.allocCache)
1347 result := span.freeindex + uint16(theBit)
1348 if result < span.nelems {
1349 freeidx := result + 1
1350 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1351 span.allocCache >>= uint(theBit + 1)
1352 span.freeindex = freeidx
1353 span.allocCount++
1354 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1355 }
1356 }
1357 }
1358 v = nextFreeFastResult
1359 if v == 0 {
1360 v, span, checkGCTrigger = c.nextFree(spc)
1361 }
1362 x = unsafe.Pointer(v)
1363 }
1364
1365 if needzero && span.needzero != 0 {
1366 memclrNoHeapPointers(x, elemsize)
1367 }
1368
1369 publicationBarrier()
1370
1371 span.freeIndexForScan = span.freeindex
1372
1373 c.nextSample -= int64(elemsize)
1374 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1375 profilealloc(mp, x, elemsize)
1376 }
1377 mp.mallocing = 0
1378 releasem(mp)
1379
1380 if checkGCTrigger {
1381 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1382 gcStart(t)
1383 }
1384 }
1385
1386 return x
1387 }
1388
1389 func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1390 if doubleCheckMalloc {
1391 if gcphase == _GCmarktermination {
1392 throw("mallocgc called with gcphase == _GCmarktermination")
1393 }
1394 }
1395
1396 var mp *m
1397
1398 mp = acquirem()
1399
1400 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1401
1402 if forceSlowPath {
1403 releasem(mp)
1404
1405 const spc = spanClass(4<<1) | spanClass(1)
1406 const elemsize = uintptr(32)
1407 return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)
1408
1409 }
1410
1411 lockRankMayQueueFinalizer()
1412
1413 const sizeclass = 4
1414 const elemsize = 32
1415
1416 if doubleCheckMalloc {
1417
1418 doubleCheckSmallNoScan(typ, mp)
1419
1420 }
1421 mp.mallocing = 1
1422
1423 checkGCTrigger := false
1424 c := getMCache(mp)
1425 const spc = spanClass(sizeclass<<1) | spanClass(1)
1426 span := c.alloc[spc]
1427
1428 var v gclinkptr
1429 var x unsafe.Pointer
1430
1431 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
1432
1433 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
1434 mp.mallocing = 0
1435 releasem(mp)
1436
1437 return x
1438
1439 }
1440
1441 {
1442
1443 var nextFreeFastResult gclinkptr
1444 if span.allocCache != 0 {
1445 theBit := sys.TrailingZeros64(span.allocCache)
1446 result := span.freeindex + uint16(theBit)
1447 if result < span.nelems {
1448 freeidx := result + 1
1449 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1450 span.allocCache >>= uint(theBit + 1)
1451 span.freeindex = freeidx
1452 span.allocCount++
1453 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1454 }
1455 }
1456 }
1457 v = nextFreeFastResult
1458 if v == 0 {
1459 v, span, checkGCTrigger = c.nextFree(spc)
1460 }
1461 x = unsafe.Pointer(v)
1462 }
1463
1464 if needzero && span.needzero != 0 {
1465 memclrNoHeapPointers(x, elemsize)
1466 }
1467
1468 publicationBarrier()
1469
1470 span.freeIndexForScan = span.freeindex
1471
1472 c.nextSample -= int64(elemsize)
1473 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1474 profilealloc(mp, x, elemsize)
1475 }
1476 mp.mallocing = 0
1477 releasem(mp)
1478
1479 if checkGCTrigger {
1480 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1481 gcStart(t)
1482 }
1483 }
1484
1485 return x
1486 }
1487
1488 func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1489 if doubleCheckMalloc {
1490 if gcphase == _GCmarktermination {
1491 throw("mallocgc called with gcphase == _GCmarktermination")
1492 }
1493 }
1494
1495 var mp *m
1496
1497 mp = acquirem()
1498
1499 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1500
1501 if forceSlowPath {
1502 releasem(mp)
1503
1504 const spc = spanClass(5<<1) | spanClass(1)
1505 const elemsize = uintptr(48)
1506 return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)
1507
1508 }
1509
1510 lockRankMayQueueFinalizer()
1511
1512 const sizeclass = 5
1513 const elemsize = 48
1514
1515 if doubleCheckMalloc {
1516
1517 doubleCheckSmallNoScan(typ, mp)
1518
1519 }
1520 mp.mallocing = 1
1521
1522 checkGCTrigger := false
1523 c := getMCache(mp)
1524 const spc = spanClass(sizeclass<<1) | spanClass(1)
1525 span := c.alloc[spc]
1526
1527 var v gclinkptr
1528 var x unsafe.Pointer
1529
1530 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
1531
1532 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
1533 mp.mallocing = 0
1534 releasem(mp)
1535
1536 return x
1537
1538 }
1539
1540 {
1541
1542 var nextFreeFastResult gclinkptr
1543 if span.allocCache != 0 {
1544 theBit := sys.TrailingZeros64(span.allocCache)
1545 result := span.freeindex + uint16(theBit)
1546 if result < span.nelems {
1547 freeidx := result + 1
1548 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1549 span.allocCache >>= uint(theBit + 1)
1550 span.freeindex = freeidx
1551 span.allocCount++
1552 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1553 }
1554 }
1555 }
1556 v = nextFreeFastResult
1557 if v == 0 {
1558 v, span, checkGCTrigger = c.nextFree(spc)
1559 }
1560 x = unsafe.Pointer(v)
1561 }
1562
1563 if needzero && span.needzero != 0 {
1564 memclrNoHeapPointers(x, elemsize)
1565 }
1566
1567 publicationBarrier()
1568
1569 span.freeIndexForScan = span.freeindex
1570
1571 c.nextSample -= int64(elemsize)
1572 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1573 profilealloc(mp, x, elemsize)
1574 }
1575 mp.mallocing = 0
1576 releasem(mp)
1577
1578 if checkGCTrigger {
1579 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1580 gcStart(t)
1581 }
1582 }
1583
1584 return x
1585 }
1586
1587 func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1588 if doubleCheckMalloc {
1589 if gcphase == _GCmarktermination {
1590 throw("mallocgc called with gcphase == _GCmarktermination")
1591 }
1592 }
1593
1594 var mp *m
1595
1596 mp = acquirem()
1597
1598 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1599
1600 if forceSlowPath {
1601 releasem(mp)
1602
1603 const spc = spanClass(6<<1) | spanClass(1)
1604 const elemsize = uintptr(64)
1605 return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)
1606
1607 }
1608
1609 lockRankMayQueueFinalizer()
1610
1611 const sizeclass = 6
1612 const elemsize = 64
1613
1614 if doubleCheckMalloc {
1615
1616 doubleCheckSmallNoScan(typ, mp)
1617
1618 }
1619 mp.mallocing = 1
1620
1621 checkGCTrigger := false
1622 c := getMCache(mp)
1623 const spc = spanClass(sizeclass<<1) | spanClass(1)
1624 span := c.alloc[spc]
1625
1626 var v gclinkptr
1627 var x unsafe.Pointer
1628
1629 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
1630
1631 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
1632 mp.mallocing = 0
1633 releasem(mp)
1634
1635 return x
1636
1637 }
1638
1639 {
1640
1641 var nextFreeFastResult gclinkptr
1642 if span.allocCache != 0 {
1643 theBit := sys.TrailingZeros64(span.allocCache)
1644 result := span.freeindex + uint16(theBit)
1645 if result < span.nelems {
1646 freeidx := result + 1
1647 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1648 span.allocCache >>= uint(theBit + 1)
1649 span.freeindex = freeidx
1650 span.allocCount++
1651 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1652 }
1653 }
1654 }
1655 v = nextFreeFastResult
1656 if v == 0 {
1657 v, span, checkGCTrigger = c.nextFree(spc)
1658 }
1659 x = unsafe.Pointer(v)
1660 }
1661
1662 if needzero && span.needzero != 0 {
1663 memclrNoHeapPointers(x, elemsize)
1664 }
1665
1666 publicationBarrier()
1667
1668 span.freeIndexForScan = span.freeindex
1669
1670 c.nextSample -= int64(elemsize)
1671 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1672 profilealloc(mp, x, elemsize)
1673 }
1674 mp.mallocing = 0
1675 releasem(mp)
1676
1677 if checkGCTrigger {
1678 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1679 gcStart(t)
1680 }
1681 }
1682
1683 return x
1684 }
1685
1686 func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1687 if doubleCheckMalloc {
1688 if gcphase == _GCmarktermination {
1689 throw("mallocgc called with gcphase == _GCmarktermination")
1690 }
1691 }
1692
1693 var mp *m
1694
1695 mp = acquirem()
1696
1697 forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
1698
1699 if forceSlowPath {
1700 releasem(mp)
1701
1702 const spc = spanClass(7<<1) | spanClass(1)
1703 const elemsize = uintptr(80)
1704 return mallocgcSmallNoScanSlowPath(size, typ, needzero, spc, elemsize)
1705
1706 }
1707
1708 lockRankMayQueueFinalizer()
1709
1710 const sizeclass = 7
1711 const elemsize = 80
1712
1713 if doubleCheckMalloc {
1714
1715 doubleCheckSmallNoScan(typ, mp)
1716
1717 }
1718 mp.mallocing = 1
1719
1720 checkGCTrigger := false
1721 c := getMCache(mp)
1722 const spc = spanClass(sizeclass<<1) | spanClass(1)
1723 span := c.alloc[spc]
1724
1725 var v gclinkptr
1726 var x unsafe.Pointer
1727
1728 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
1729
1730 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
1731 mp.mallocing = 0
1732 releasem(mp)
1733
1734 return x
1735
1736 }
1737
1738 {
1739
1740 var nextFreeFastResult gclinkptr
1741 if span.allocCache != 0 {
1742 theBit := sys.TrailingZeros64(span.allocCache)
1743 result := span.freeindex + uint16(theBit)
1744 if result < span.nelems {
1745 freeidx := result + 1
1746 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1747 span.allocCache >>= uint(theBit + 1)
1748 span.freeindex = freeidx
1749 span.allocCount++
1750 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1751 }
1752 }
1753 }
1754 v = nextFreeFastResult
1755 if v == 0 {
1756 v, span, checkGCTrigger = c.nextFree(spc)
1757 }
1758 x = unsafe.Pointer(v)
1759 }
1760
1761 if needzero && span.needzero != 0 {
1762 memclrNoHeapPointers(x, elemsize)
1763 }
1764
1765 publicationBarrier()
1766
1767 span.freeIndexForScan = span.freeindex
1768
1769 c.nextSample -= int64(elemsize)
1770 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1771 profilealloc(mp, x, elemsize)
1772 }
1773 mp.mallocing = 0
1774 releasem(mp)
1775
1776 if checkGCTrigger {
1777 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1778 gcStart(t)
1779 }
1780 }
1781
1782 return x
1783 }
1784
1785 func mallocgcTinySlowPath(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
1786 if doubleCheckMalloc {
1787 if gcphase == _GCmarktermination {
1788 throw("mallocgc called with gcphase == _GCmarktermination")
1789 }
1790 }
1791
1792 var mp *m
1793
1794 gp := getg()
1795 if goexperiment.RuntimeSecret && gp.secret > 0 {
1796 return mallocgcSmallNoScanSC2(size, typ, needzero)
1797 }
1798
1799 lockRankMayQueueFinalizer()
1800
1801 if debug.malloc {
1802 if x := preMallocgcDebug(size, typ); x != nil {
1803 return x
1804 }
1805 }
1806
1807 if gcBlackenEnabled != 0 {
1808
1809 assistG := getg()
1810 if assistG.m.curg != nil {
1811 assistG = assistG.m.curg
1812 }
1813 assistG.gcAssistBytes -= int64(size)
1814 if assistG.gcAssistBytes < 0 {
1815 gcAssistAlloc(assistG)
1816 }
1817 }
1818
1819 mp = acquirem()
1820
1821 const elemsize = 16
1822
1823 if doubleCheckMalloc {
1824 doubleCheckTiny(size, typ, mp)
1825 }
1826 mp.mallocing = 1
1827
1828 c := getMCache(mp)
1829 off := c.tinyoffset
1830
1831 if size&7 == 0 {
1832 off = alignUp(off, 8)
1833 } else if goarch.PtrSize == 4 && size == 12 {
1834
1835 off = alignUp(off, 8)
1836 } else if size&3 == 0 {
1837 off = alignUp(off, 4)
1838 } else if size&1 == 0 {
1839 off = alignUp(off, 2)
1840 }
1841 if off+size <= maxTinySize && c.tiny != 0 {
1842
1843 x := unsafe.Pointer(c.tiny + off)
1844 c.tinyoffset = off + size
1845 c.tinyAllocs++
1846 mp.mallocing = 0
1847 releasem(mp)
1848 const elemsize = 0
1849 if gcBlackenEnabled != 0 && elemsize != 0 {
1850 if assistG := getg().m.curg; assistG != nil {
1851 assistG.gcAssistBytes -= int64(elemsize - size)
1852 }
1853 }
1854
1855 if debug.malloc {
1856 postMallocgcDebug(x, elemsize, typ)
1857 }
1858
1859 return x
1860 }
1861
1862 checkGCTrigger := false
1863 span := c.alloc[tinySpanClass]
1864
1865 const nbytes = 8192
1866 const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / 16)
1867 var nextFreeFastResult gclinkptr
1868 if span.allocCache != 0 {
1869 theBit := sys.TrailingZeros64(span.allocCache)
1870 result := span.freeindex + uint16(theBit)
1871 if result < nelems {
1872 freeidx := result + 1
1873 if !(freeidx%64 == 0 && freeidx != nelems) {
1874 span.allocCache >>= uint(theBit + 1)
1875 span.freeindex = freeidx
1876 span.allocCount++
1877 nextFreeFastResult = gclinkptr(uintptr(result)*16 + span.base())
1878 }
1879 }
1880 }
1881 v := nextFreeFastResult
1882 if v == 0 {
1883 v, span, checkGCTrigger = c.nextFree(tinySpanClass)
1884 }
1885 x := unsafe.Pointer(v)
1886 (*[2]uint64)(x)[0] = 0
1887 (*[2]uint64)(x)[1] = 0
1888
1889 if !raceenabled && (size < c.tinyoffset || c.tiny == 0) {
1890
1891 c.tiny = uintptr(x)
1892 c.tinyoffset = size
1893 }
1894
1895 publicationBarrier()
1896
1897 if writeBarrier.enabled {
1898
1899 gcmarknewobject(span, uintptr(x))
1900 } else {
1901
1902 span.freeIndexForScan = span.freeindex
1903 }
1904
1905 c.nextSample -= int64(elemsize)
1906 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
1907 profilealloc(mp, x, elemsize)
1908 }
1909 mp.mallocing = 0
1910 releasem(mp)
1911
1912 if checkGCTrigger {
1913 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
1914 gcStart(t)
1915 }
1916 }
1917 if gcBlackenEnabled != 0 && elemsize != 0 {
1918 if assistG := getg().m.curg; assistG != nil {
1919 assistG.gcAssistBytes -= int64(elemsize - size)
1920 }
1921 }
1922
1923 if debug.malloc {
1924 postMallocgcDebug(x, elemsize, typ)
1925 }
1926
1927 return x
1928 }
1929
1930 func mallocgcSmallScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer {
1931
1932 if doubleCheckMalloc {
1933 if gcphase == _GCmarktermination {
1934 throw("mallocgc called with gcphase == _GCmarktermination")
1935 }
1936 }
1937
1938 var mp *m
1939
1940 lockRankMayQueueFinalizer()
1941
1942 if debug.malloc {
1943 if x := preMallocgcDebug(size, typ); x != nil {
1944 return x
1945 }
1946 }
1947
1948 if gcBlackenEnabled != 0 {
1949
1950 assistG := getg()
1951 if assistG.m.curg != nil {
1952 assistG = assistG.m.curg
1953 }
1954 assistG.gcAssistBytes -= int64(size)
1955 if assistG.gcAssistBytes < 0 {
1956 gcAssistAlloc(assistG)
1957 }
1958 }
1959
1960 mp = acquirem()
1961
1962 if doubleCheckMalloc {
1963
1964 doubleCheckSmallScanNoHeader(size, typ, mp)
1965
1966 }
1967 mp.mallocing = 1
1968
1969 checkGCTrigger := false
1970 c := getMCache(mp)
1971
1972 span := c.alloc[spc]
1973
1974 var v gclinkptr
1975 var x unsafe.Pointer
1976
1977 {
1978
1979 var nextFreeFastResult gclinkptr
1980 if span.allocCache != 0 {
1981 theBit := sys.TrailingZeros64(span.allocCache)
1982 result := span.freeindex + uint16(theBit)
1983 if result < span.nelems {
1984 freeidx := result + 1
1985 if !(freeidx%64 == 0 && freeidx != span.nelems) {
1986 span.allocCache >>= uint(theBit + 1)
1987 span.freeindex = freeidx
1988 span.allocCount++
1989 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
1990 }
1991 }
1992 }
1993 v = nextFreeFastResult
1994 if v == 0 {
1995 v, span, checkGCTrigger = c.nextFree(spc)
1996 }
1997 x = unsafe.Pointer(v)
1998 }
1999
2000 if span.needzero != 0 {
2001 memclrNoHeapPointers(x, elemsize)
2002 }
2003 if goarch.PtrSize == 8 && elemsize == 8 {
2004
2005 c.scanAlloc += 8
2006 } else {
2007 dataSize := size
2008 x := uintptr(x)
2009
2010 if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(elemsize_)) {
2011 throw("tried to write heap bits, but no heap bits in span")
2012 }
2013
2014 src0 := readUintptr(getGCMask(typ))
2015
2016 var scanSize uintptr
2017 src := src0
2018 if typ.Size_ == goarch.PtrSize {
2019 src = (1 << (dataSize / goarch.PtrSize)) - 1
2020
2021 scanSize = dataSize
2022 } else {
2023
2024 if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
2025 throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
2026 }
2027 scanSize = typ.PtrBytes
2028 for i := typ.Size_; i < dataSize; i += typ.Size_ {
2029 src |= src0 << (i / goarch.PtrSize)
2030 scanSize += typ.Size_
2031 }
2032 }
2033
2034 dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
2035 dst := unsafe.Pointer(dstBase)
2036 o := (x - span.base()) / goarch.PtrSize
2037 i := o / ptrBits
2038 j := o % ptrBits
2039 var bits uintptr = elemsize / goarch.PtrSize
2040
2041 var bitsIsPowerOfTwo = bits&(bits-1) == 0
2042 if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
2043
2044 bits0 := ptrBits - j
2045 bits1 := bits - bits0
2046 dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
2047 dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
2048 *dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
2049 *dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
2050 } else {
2051
2052 dst := (*uintptr)(add(dst, i*goarch.PtrSize))
2053 *dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j)
2054 }
2055
2056 const doubleCheck = false
2057 if doubleCheck {
2058 writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
2059 }
2060 if doubleCheckHeapSetType {
2061 doubleCheckHeapType(x, dataSize, typ, nil, span)
2062 }
2063 c.scanAlloc += scanSize
2064 }
2065
2066 publicationBarrier()
2067
2068 if writeBarrier.enabled {
2069
2070 gcmarknewobject(span, uintptr(x))
2071 } else {
2072
2073 span.freeIndexForScan = span.freeindex
2074 }
2075
2076 c.nextSample -= int64(elemsize)
2077 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
2078 profilealloc(mp, x, elemsize)
2079 }
2080 mp.mallocing = 0
2081 releasem(mp)
2082
2083 if checkGCTrigger {
2084 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
2085 gcStart(t)
2086 }
2087 }
2088 gp := getg()
2089 if goexperiment.RuntimeSecret && gp.secret > 0 {
2090
2091 addSecret(x, size)
2092 }
2093
2094 if gcBlackenEnabled != 0 && elemsize != 0 {
2095 if assistG := getg().m.curg; assistG != nil {
2096 assistG.gcAssistBytes -= int64(elemsize - size)
2097 }
2098 }
2099
2100 if debug.malloc {
2101 postMallocgcDebug(x, elemsize, typ)
2102 }
2103
2104 return x
2105 }
2106
2107 func mallocgcSmallNoScanSlowPath(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer {
2108
2109 if doubleCheckMalloc {
2110 if gcphase == _GCmarktermination {
2111 throw("mallocgc called with gcphase == _GCmarktermination")
2112 }
2113 }
2114
2115 var mp *m
2116
2117 lockRankMayQueueFinalizer()
2118
2119 if debug.malloc {
2120 if x := preMallocgcDebug(size, typ); x != nil {
2121 return x
2122 }
2123 }
2124
2125 if gcBlackenEnabled != 0 {
2126
2127 assistG := getg()
2128 if assistG.m.curg != nil {
2129 assistG = assistG.m.curg
2130 }
2131 assistG.gcAssistBytes -= int64(size)
2132 if assistG.gcAssistBytes < 0 {
2133 gcAssistAlloc(assistG)
2134 }
2135 }
2136
2137 mp = acquirem()
2138
2139 if doubleCheckMalloc {
2140
2141 doubleCheckSmallNoScan(typ, mp)
2142
2143 }
2144 mp.mallocing = 1
2145
2146 checkGCTrigger := false
2147 c := getMCache(mp)
2148
2149 span := c.alloc[spc]
2150
2151 var v gclinkptr
2152 var x unsafe.Pointer
2153
2154 if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
2155
2156 x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
2157 mp.mallocing = 0
2158 releasem(mp)
2159
2160 goto post
2161
2162 }
2163
2164 {
2165
2166 var nextFreeFastResult gclinkptr
2167 if span.allocCache != 0 {
2168 theBit := sys.TrailingZeros64(span.allocCache)
2169 result := span.freeindex + uint16(theBit)
2170 if result < span.nelems {
2171 freeidx := result + 1
2172 if !(freeidx%64 == 0 && freeidx != span.nelems) {
2173 span.allocCache >>= uint(theBit + 1)
2174 span.freeindex = freeidx
2175 span.allocCount++
2176 nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
2177 }
2178 }
2179 }
2180 v = nextFreeFastResult
2181 if v == 0 {
2182 v, span, checkGCTrigger = c.nextFree(spc)
2183 }
2184 x = unsafe.Pointer(v)
2185 }
2186
2187 if needzero && span.needzero != 0 {
2188 memclrNoHeapPointers(x, elemsize)
2189 }
2190
2191 publicationBarrier()
2192
2193 if writeBarrier.enabled {
2194
2195 gcmarknewobject(span, uintptr(x))
2196 } else {
2197
2198 span.freeIndexForScan = span.freeindex
2199 }
2200
2201 c.nextSample -= int64(elemsize)
2202 if c.nextSample < 0 || MemProfileRate != c.memProfRate {
2203 profilealloc(mp, x, elemsize)
2204 }
2205 mp.mallocing = 0
2206 releasem(mp)
2207
2208 if checkGCTrigger {
2209 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
2210 gcStart(t)
2211 }
2212 }
2213
2214 post:
2215 gp := getg()
2216 if goexperiment.RuntimeSecret && gp.secret > 0 {
2217
2218 addSecret(x, size)
2219 }
2220
2221 if gcBlackenEnabled != 0 && elemsize != 0 {
2222 if assistG := getg().m.curg; assistG != nil {
2223 assistG.gcAssistBytes -= int64(elemsize - size)
2224 }
2225 }
2226
2227 if debug.malloc {
2228 postMallocgcDebug(x, elemsize, typ)
2229 }
2230
2231 return x
2232 }
2233
View as plain text