Source file
src/runtime/os_windows.go
1
2
3
4
5 package runtime
6
7 import (
8 "internal/abi"
9 "internal/goarch"
10 "runtime/internal/atomic"
11 "unsafe"
12 )
13
14
15 const (
16 _NSIG = 65
17 )
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63 type stdFunction unsafe.Pointer
64
65 var (
66
67
68
69 _AddVectoredExceptionHandler,
70 _CloseHandle,
71 _CreateEventA,
72 _CreateFileA,
73 _CreateIoCompletionPort,
74 _CreateThread,
75 _CreateWaitableTimerA,
76 _CreateWaitableTimerExW,
77 _DuplicateHandle,
78 _ExitProcess,
79 _FreeEnvironmentStringsW,
80 _GetConsoleMode,
81 _GetEnvironmentStringsW,
82 _GetProcAddress,
83 _GetProcessAffinityMask,
84 _GetQueuedCompletionStatusEx,
85 _GetStdHandle,
86 _GetSystemDirectoryA,
87 _GetSystemInfo,
88 _GetSystemTimeAsFileTime,
89 _GetThreadContext,
90 _SetThreadContext,
91 _LoadLibraryW,
92 _LoadLibraryA,
93 _PostQueuedCompletionStatus,
94 _QueryPerformanceCounter,
95 _QueryPerformanceFrequency,
96 _ResumeThread,
97 _SetConsoleCtrlHandler,
98 _SetErrorMode,
99 _SetEvent,
100 _SetProcessPriorityBoost,
101 _SetThreadPriority,
102 _SetUnhandledExceptionFilter,
103 _SetWaitableTimer,
104 _Sleep,
105 _SuspendThread,
106 _SwitchToThread,
107 _TlsAlloc,
108 _VirtualAlloc,
109 _VirtualFree,
110 _VirtualQuery,
111 _WaitForSingleObject,
112 _WaitForMultipleObjects,
113 _WriteConsoleW,
114 _WriteFile,
115 _ stdFunction
116
117
118
119 _AddDllDirectory,
120 _AddVectoredContinueHandler,
121 _LoadLibraryExA,
122 _LoadLibraryExW,
123 _ stdFunction
124
125
126
127
128
129
130
131
132
133 _RtlGenRandom stdFunction
134
135
136
137
138 _NtWaitForSingleObject stdFunction
139 _RtlGetCurrentPeb stdFunction
140 _RtlGetNtVersionNumbers stdFunction
141
142
143 _timeBeginPeriod,
144 _timeEndPeriod,
145 _WSAGetOverlappedResult,
146 _ stdFunction
147 )
148
149
150
151 func tstart_stdcall(newm *m)
152
153
154 func wintls()
155
156 type mOS struct {
157 threadLock mutex
158 thread uintptr
159
160 waitsema uintptr
161 resumesema uintptr
162
163 highResTimer uintptr
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186 preemptExtLock uint32
187 }
188
189
190 func os_sigpipe() {
191 throw("too many writes on closed pipe")
192 }
193
194
195 func open(name *byte, mode, perm int32) int32 {
196 throw("unimplemented")
197 return -1
198 }
199 func closefd(fd int32) int32 {
200 throw("unimplemented")
201 return -1
202 }
203 func read(fd int32, p unsafe.Pointer, n int32) int32 {
204 throw("unimplemented")
205 return -1
206 }
207
208 type sigset struct{}
209
210
211
212 func asmstdcall(fn unsafe.Pointer)
213
214 var asmstdcallAddr unsafe.Pointer
215
216 func windowsFindfunc(lib uintptr, name []byte) stdFunction {
217 if name[len(name)-1] != 0 {
218 throw("usage")
219 }
220 f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0])))
221 return stdFunction(unsafe.Pointer(f))
222 }
223
224 const _MAX_PATH = 260
225 var sysDirectory [_MAX_PATH + 1]byte
226 var sysDirectoryLen uintptr
227
228 func windowsLoadSystemLib(name []byte) uintptr {
229 if sysDirectoryLen == 0 {
230 l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1))
231 if l == 0 || l > uintptr(len(sysDirectory)-1) {
232 throw("Unable to determine system directory")
233 }
234 sysDirectory[l] = '\\'
235 sysDirectoryLen = l + 1
236 }
237 if useLoadLibraryEx {
238 return stdcall3(_LoadLibraryExA, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32)
239 } else {
240 absName := append(sysDirectory[:sysDirectoryLen], name...)
241 return stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&absName[0])))
242 }
243 }
244
245 const haveCputicksAsm = GOARCH == "386" || GOARCH == "amd64"
246
247 func loadOptionalSyscalls() {
248 var kernel32dll = []byte("kernel32.dll\000")
249 k32 := stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&kernel32dll[0])))
250 if k32 == 0 {
251 throw("kernel32.dll not found")
252 }
253 _AddDllDirectory = windowsFindfunc(k32, []byte("AddDllDirectory\000"))
254 _AddVectoredContinueHandler = windowsFindfunc(k32, []byte("AddVectoredContinueHandler\000"))
255 _LoadLibraryExA = windowsFindfunc(k32, []byte("LoadLibraryExA\000"))
256 _LoadLibraryExW = windowsFindfunc(k32, []byte("LoadLibraryExW\000"))
257 useLoadLibraryEx = (_LoadLibraryExW != nil && _LoadLibraryExA != nil && _AddDllDirectory != nil)
258
259 var advapi32dll = []byte("advapi32.dll\000")
260 a32 := windowsLoadSystemLib(advapi32dll)
261 if a32 == 0 {
262 throw("advapi32.dll not found")
263 }
264 _RtlGenRandom = windowsFindfunc(a32, []byte("SystemFunction036\000"))
265
266 var ntdll = []byte("ntdll.dll\000")
267 n32 := windowsLoadSystemLib(ntdll)
268 if n32 == 0 {
269 throw("ntdll.dll not found")
270 }
271 _NtWaitForSingleObject = windowsFindfunc(n32, []byte("NtWaitForSingleObject\000"))
272 _RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000"))
273 _RtlGetNtVersionNumbers = windowsFindfunc(n32, []byte("RtlGetNtVersionNumbers\000"))
274
275 if !haveCputicksAsm {
276 _QueryPerformanceCounter = windowsFindfunc(k32, []byte("QueryPerformanceCounter\000"))
277 if _QueryPerformanceCounter == nil {
278 throw("could not find QPC syscalls")
279 }
280 }
281
282 var winmmdll = []byte("winmm.dll\000")
283 m32 := windowsLoadSystemLib(winmmdll)
284 if m32 == 0 {
285 throw("winmm.dll not found")
286 }
287 _timeBeginPeriod = windowsFindfunc(m32, []byte("timeBeginPeriod\000"))
288 _timeEndPeriod = windowsFindfunc(m32, []byte("timeEndPeriod\000"))
289 if _timeBeginPeriod == nil || _timeEndPeriod == nil {
290 throw("timeBegin/EndPeriod not found")
291 }
292
293 var ws232dll = []byte("ws2_32.dll\000")
294 ws232 := windowsLoadSystemLib(ws232dll)
295 if ws232 == 0 {
296 throw("ws2_32.dll not found")
297 }
298 _WSAGetOverlappedResult = windowsFindfunc(ws232, []byte("WSAGetOverlappedResult\000"))
299 if _WSAGetOverlappedResult == nil {
300 throw("WSAGetOverlappedResult not found")
301 }
302
303 if windowsFindfunc(n32, []byte("wine_get_version\000")) != nil {
304
305 initWine(k32)
306 }
307 }
308
309 func monitorSuspendResume() {
310 const (
311 _DEVICE_NOTIFY_CALLBACK = 2
312 )
313 type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct {
314 callback uintptr
315 context uintptr
316 }
317
318 powrprof := windowsLoadSystemLib([]byte("powrprof.dll\000"))
319 if powrprof == 0 {
320 return
321 }
322 powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000"))
323 if powerRegisterSuspendResumeNotification == nil {
324 return
325 }
326 var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr {
327 for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink {
328 if mp.resumesema != 0 {
329 stdcall1(_SetEvent, mp.resumesema)
330 }
331 }
332 return 0
333 }
334 params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{
335 callback: compileCallback(*efaceOf(&fn), true),
336 }
337 handle := uintptr(0)
338 stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK,
339 uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&handle)))
340 }
341
342
343 func getLoadLibrary() uintptr {
344 return uintptr(unsafe.Pointer(_LoadLibraryW))
345 }
346
347
348 func getLoadLibraryEx() uintptr {
349 return uintptr(unsafe.Pointer(_LoadLibraryExW))
350 }
351
352
353 func getGetProcAddress() uintptr {
354 return uintptr(unsafe.Pointer(_GetProcAddress))
355 }
356
357 func getproccount() int32 {
358 var mask, sysmask uintptr
359 ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask)))
360 if ret != 0 {
361 n := 0
362 maskbits := int(unsafe.Sizeof(mask) * 8)
363 for i := 0; i < maskbits; i++ {
364 if mask&(1<<uint(i)) != 0 {
365 n++
366 }
367 }
368 if n != 0 {
369 return int32(n)
370 }
371 }
372
373 var info systeminfo
374 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
375 return int32(info.dwnumberofprocessors)
376 }
377
378 func getPageSize() uintptr {
379 var info systeminfo
380 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
381 return uintptr(info.dwpagesize)
382 }
383
384 const (
385 currentProcess = ^uintptr(0)
386 currentThread = ^uintptr(1)
387 )
388
389
390 func getlasterror() uint32
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406 var useLoadLibraryEx bool
407
408 var timeBeginPeriodRetValue uint32
409
410
411
412
413
414 const osRelaxMinNS = 60 * 1e6
415
416
417
418
419
420
421
422
423
424
425
426 func osRelax(relax bool) uint32 {
427 if haveHighResTimer {
428
429
430
431 return 0
432 }
433
434 if relax {
435 return uint32(stdcall1(_timeEndPeriod, 1))
436 } else {
437 return uint32(stdcall1(_timeBeginPeriod, 1))
438 }
439 }
440
441
442
443 var haveHighResTimer = false
444
445
446
447
448
449 func createHighResTimer() uintptr {
450 const (
451
452
453 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002
454
455 _SYNCHRONIZE = 0x00100000
456 _TIMER_QUERY_STATE = 0x0001
457 _TIMER_MODIFY_STATE = 0x0002
458 )
459 return stdcall4(_CreateWaitableTimerExW, 0, 0,
460 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION,
461 _SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE)
462 }
463
464 const highResTimerSupported = GOARCH == "386" || GOARCH == "amd64"
465
466 func initHighResTimer() {
467 if !highResTimerSupported {
468
469 return
470 }
471 h := createHighResTimer()
472 if h != 0 {
473 haveHighResTimer = true
474 stdcall1(_CloseHandle, h)
475 }
476 }
477
478
479 var canUseLongPaths bool
480
481
482
483 var longFileName [(_MAX_PATH+1)*2 + 1]byte
484
485
486
487
488
489
490
491
492
493
494
495 func initLongPathSupport() {
496 const (
497 IsLongPathAwareProcess = 0x80
498 PebBitFieldOffset = 3
499 OPEN_EXISTING = 3
500 ERROR_PATH_NOT_FOUND = 3
501 )
502
503
504 var maj, min, build uint32
505 stdcall3(_RtlGetNtVersionNumbers, uintptr(unsafe.Pointer(&maj)), uintptr(unsafe.Pointer(&min)), uintptr(unsafe.Pointer(&build)))
506 if maj < 10 || (maj == 10 && min == 0 && build&0xffff < 15063) {
507 return
508 }
509
510
511 bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset))
512 originalBitField := *bitField
513 *bitField |= IsLongPathAwareProcess
514
515
516
517
518
519
520
521
522 getRandomData(longFileName[len(longFileName)-33 : len(longFileName)-1])
523 start := copy(longFileName[:], sysDirectory[:sysDirectoryLen])
524 const dig = "0123456789abcdef"
525 for i := 0; i < 32; i++ {
526 longFileName[start+i*2] = dig[longFileName[len(longFileName)-33+i]>>4]
527 longFileName[start+i*2+1] = dig[longFileName[len(longFileName)-33+i]&0xf]
528 }
529 start += 64
530 for i := start; i < len(longFileName)-1; i++ {
531 longFileName[i] = 'A'
532 }
533 stdcall7(_CreateFileA, uintptr(unsafe.Pointer(&longFileName[0])), 0, 0, 0, OPEN_EXISTING, 0, 0)
534
535
536
537 if getlasterror() == ERROR_PATH_NOT_FOUND {
538 *bitField = originalBitField
539 println("runtime: warning: IsLongPathAwareProcess failed to enable long paths; proceeding in fixup mode")
540 return
541 }
542
543 canUseLongPaths = true
544 }
545
546 func osinit() {
547 asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall))
548
549 setBadSignalMsg()
550
551 loadOptionalSyscalls()
552
553 disableWER()
554
555 initExceptionHandler()
556
557 initHighResTimer()
558 timeBeginPeriodRetValue = osRelax(false)
559
560 initLongPathSupport()
561
562 ncpu = getproccount()
563
564 physPageSize = getPageSize()
565
566
567
568
569
570 stdcall2(_SetProcessPriorityBoost, currentProcess, 1)
571 }
572
573
574
575 var useQPCTime uint8
576
577 var qpcStartCounter int64
578 var qpcMultiplier int64
579
580
581 func nanotimeQPC() int64 {
582 var counter int64 = 0
583 stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter)))
584
585
586 return (counter - qpcStartCounter) * qpcMultiplier
587 }
588
589
590 func nowQPC() (sec int64, nsec int32, mono int64) {
591 var ft int64
592 stdcall1(_GetSystemTimeAsFileTime, uintptr(unsafe.Pointer(&ft)))
593
594 t := (ft - 116444736000000000) * 100
595
596 sec = t / 1000000000
597 nsec = int32(t - sec*1000000000)
598
599 mono = nanotimeQPC()
600 return
601 }
602
603 func initWine(k32 uintptr) {
604 _GetSystemTimeAsFileTime = windowsFindfunc(k32, []byte("GetSystemTimeAsFileTime\000"))
605 if _GetSystemTimeAsFileTime == nil {
606 throw("could not find GetSystemTimeAsFileTime() syscall")
607 }
608
609 _QueryPerformanceCounter = windowsFindfunc(k32, []byte("QueryPerformanceCounter\000"))
610 _QueryPerformanceFrequency = windowsFindfunc(k32, []byte("QueryPerformanceFrequency\000"))
611 if _QueryPerformanceCounter == nil || _QueryPerformanceFrequency == nil {
612 throw("could not find QPC syscalls")
613 }
614
615
616
617
618
619 var tmp int64
620 stdcall1(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&tmp)))
621 if tmp == 0 {
622 throw("QueryPerformanceFrequency syscall returned zero, running on unsupported hardware")
623 }
624
625
626
627
628 if tmp > (1<<31 - 1) {
629 throw("QueryPerformanceFrequency overflow 32 bit divider, check nosplit discussion to proceed")
630 }
631 qpcFrequency := int32(tmp)
632 stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&qpcStartCounter)))
633
634
635
636
637
638
639 qpcMultiplier = int64(timediv(1000000000, qpcFrequency, nil))
640
641 useQPCTime = 1
642 }
643
644
645 func getRandomData(r []byte) {
646 n := 0
647 if stdcall2(_RtlGenRandom, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 {
648 n = len(r)
649 }
650 extendRandom(r, n)
651 }
652
653 func goenvs() {
654
655
656
657 strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW))
658 p := (*[1 << 24]uint16)(strings)[:]
659
660 n := 0
661 for from, i := 0, 0; true; i++ {
662 if p[i] == 0 {
663
664 if i == from {
665 break
666 }
667 from = i + 1
668 n++
669 }
670 }
671 envs = make([]string, n)
672
673 for i := range envs {
674 envs[i] = gostringw(&p[0])
675 for p[0] != 0 {
676 p = p[1:]
677 }
678 p = p[1:]
679 }
680
681 stdcall1(_FreeEnvironmentStringsW, uintptr(strings))
682
683
684
685 var fn any = ctrlHandler
686 ctrlHandlerPC := compileCallback(*efaceOf(&fn), true)
687 stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1)
688
689 monitorSuspendResume()
690 }
691
692
693 var exiting uint32
694
695
696 func exit(code int32) {
697
698
699
700
701 lock(&suspendLock)
702 atomic.Store(&exiting, 1)
703 stdcall1(_ExitProcess, uintptr(code))
704 }
705
706
707
708
709
710
711 func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 {
712 const (
713 _STD_OUTPUT_HANDLE = ^uintptr(10)
714 _STD_ERROR_HANDLE = ^uintptr(11)
715 )
716 var handle uintptr
717 switch fd {
718 case 1:
719 handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE)
720 case 2:
721 handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE)
722 default:
723
724 handle = fd
725 }
726 isASCII := true
727 b := (*[1 << 30]byte)(buf)[:n]
728 for _, x := range b {
729 if x >= 0x80 {
730 isASCII = false
731 break
732 }
733 }
734
735 if !isASCII {
736 var m uint32
737 isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0
738
739
740 if isConsole {
741 return int32(writeConsole(handle, buf, n))
742 }
743 }
744 var written uint32
745 stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0)
746 return int32(written)
747 }
748
749 var (
750 utf16ConsoleBack [1000]uint16
751 utf16ConsoleBackLock mutex
752 )
753
754
755
756 func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int {
757 const surr2 = (surrogateMin + surrogateMax + 1) / 2
758
759
760 lock(&utf16ConsoleBackLock)
761
762 b := (*[1 << 30]byte)(buf)[:bufLen]
763 s := *(*string)(unsafe.Pointer(&b))
764
765 utf16tmp := utf16ConsoleBack[:]
766
767 total := len(s)
768 w := 0
769 for _, r := range s {
770 if w >= len(utf16tmp)-2 {
771 writeConsoleUTF16(handle, utf16tmp[:w])
772 w = 0
773 }
774 if r < 0x10000 {
775 utf16tmp[w] = uint16(r)
776 w++
777 } else {
778 r -= 0x10000
779 utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff
780 utf16tmp[w+1] = surr2 + uint16(r)&0x3ff
781 w += 2
782 }
783 }
784 writeConsoleUTF16(handle, utf16tmp[:w])
785 unlock(&utf16ConsoleBackLock)
786 return total
787 }
788
789
790
791
792 func writeConsoleUTF16(handle uintptr, b []uint16) {
793 l := uint32(len(b))
794 if l == 0 {
795 return
796 }
797 var written uint32
798 stdcall5(_WriteConsoleW,
799 handle,
800 uintptr(unsafe.Pointer(&b[0])),
801 uintptr(l),
802 uintptr(unsafe.Pointer(&written)),
803 0,
804 )
805 return
806 }
807
808
809 func semasleep(ns int64) int32 {
810 const (
811 _WAIT_ABANDONED = 0x00000080
812 _WAIT_OBJECT_0 = 0x00000000
813 _WAIT_TIMEOUT = 0x00000102
814 _WAIT_FAILED = 0xFFFFFFFF
815 )
816
817 var result uintptr
818 if ns < 0 {
819 result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE))
820 } else {
821 start := nanotime()
822 elapsed := int64(0)
823 for {
824 ms := int64(timediv(ns-elapsed, 1000000, nil))
825 if ms == 0 {
826 ms = 1
827 }
828 result = stdcall4(_WaitForMultipleObjects, 2,
829 uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})),
830 0, uintptr(ms))
831 if result != _WAIT_OBJECT_0+1 {
832
833 break
834 }
835 elapsed = nanotime() - start
836 if elapsed >= ns {
837 return -1
838 }
839 }
840 }
841 switch result {
842 case _WAIT_OBJECT_0:
843 return 0
844
845 case _WAIT_TIMEOUT:
846 return -1
847
848 case _WAIT_ABANDONED:
849 systemstack(func() {
850 throw("runtime.semasleep wait_abandoned")
851 })
852
853 case _WAIT_FAILED:
854 systemstack(func() {
855 print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n")
856 throw("runtime.semasleep wait_failed")
857 })
858
859 default:
860 systemstack(func() {
861 print("runtime: waitforsingleobject unexpected; result=", result, "\n")
862 throw("runtime.semasleep unexpected")
863 })
864 }
865
866 return -1
867 }
868
869
870 func semawakeup(mp *m) {
871 if stdcall1(_SetEvent, mp.waitsema) == 0 {
872 systemstack(func() {
873 print("runtime: setevent failed; errno=", getlasterror(), "\n")
874 throw("runtime.semawakeup")
875 })
876 }
877 }
878
879
880 func semacreate(mp *m) {
881 if mp.waitsema != 0 {
882 return
883 }
884 mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0)
885 if mp.waitsema == 0 {
886 systemstack(func() {
887 print("runtime: createevent failed; errno=", getlasterror(), "\n")
888 throw("runtime.semacreate")
889 })
890 }
891 mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0)
892 if mp.resumesema == 0 {
893 systemstack(func() {
894 print("runtime: createevent failed; errno=", getlasterror(), "\n")
895 throw("runtime.semacreate")
896 })
897 stdcall1(_CloseHandle, mp.waitsema)
898 mp.waitsema = 0
899 }
900 }
901
902
903
904
905
906
907
908 func newosproc(mp *m) {
909
910 thandle := stdcall6(_CreateThread, 0, 0,
911 abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)),
912 0, 0)
913
914 if thandle == 0 {
915 if atomic.Load(&exiting) != 0 {
916
917
918
919
920 lock(&deadlock)
921 lock(&deadlock)
922 }
923 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n")
924 throw("runtime.newosproc")
925 }
926
927
928 stdcall1(_CloseHandle, thandle)
929 }
930
931
932
933
934
935
936
937 func newosproc0(mp *m, stk unsafe.Pointer) {
938
939
940
941 throw("bad newosproc0")
942 }
943
944 func exitThread(wait *atomic.Uint32) {
945
946
947 throw("exitThread")
948 }
949
950
951
952 func mpreinit(mp *m) {
953 }
954
955
956 func sigsave(p *sigset) {
957 }
958
959
960 func msigrestore(sigmask sigset) {
961 }
962
963
964
965 func clearSignalHandlers() {
966 }
967
968
969 func sigblock(exiting bool) {
970 }
971
972
973
974 func minit() {
975 var thandle uintptr
976 if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
977 print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n")
978 throw("runtime.minit: duplicatehandle failed")
979 }
980
981 mp := getg().m
982 lock(&mp.threadLock)
983 mp.thread = thandle
984
985
986 if mp.highResTimer == 0 && haveHighResTimer {
987 mp.highResTimer = createHighResTimer()
988 if mp.highResTimer == 0 {
989 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
990 throw("CreateWaitableTimerEx when creating timer failed")
991 }
992 }
993 unlock(&mp.threadLock)
994
995
996
997 var mbi memoryBasicInformation
998 res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi))
999 if res == 0 {
1000 print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n")
1001 throw("VirtualQuery for stack base failed")
1002 }
1003
1004
1005
1006
1007
1008
1009 base := mbi.allocationBase + 16<<10
1010
1011 g0 := getg()
1012 if base > g0.stack.hi || g0.stack.hi-base > 64<<20 {
1013 print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n")
1014 throw("bad g0 stack")
1015 }
1016 g0.stack.lo = base
1017 g0.stackguard0 = g0.stack.lo + _StackGuard
1018 g0.stackguard1 = g0.stackguard0
1019
1020 stackcheck()
1021 }
1022
1023
1024
1025
1026 func unminit() {
1027 mp := getg().m
1028 lock(&mp.threadLock)
1029 if mp.thread != 0 {
1030 stdcall1(_CloseHandle, mp.thread)
1031 mp.thread = 0
1032 }
1033 unlock(&mp.threadLock)
1034 }
1035
1036
1037
1038
1039
1040 func mdestroy(mp *m) {
1041 if mp.highResTimer != 0 {
1042 stdcall1(_CloseHandle, mp.highResTimer)
1043 mp.highResTimer = 0
1044 }
1045 if mp.waitsema != 0 {
1046 stdcall1(_CloseHandle, mp.waitsema)
1047 mp.waitsema = 0
1048 }
1049 if mp.resumesema != 0 {
1050 stdcall1(_CloseHandle, mp.resumesema)
1051 mp.resumesema = 0
1052 }
1053 }
1054
1055
1056
1057
1058
1059
1060 func stdcall(fn stdFunction) uintptr {
1061 gp := getg()
1062 mp := gp.m
1063 mp.libcall.fn = uintptr(unsafe.Pointer(fn))
1064 resetLibcall := false
1065 if mp.profilehz != 0 && mp.libcallsp == 0 {
1066
1067 mp.libcallg.set(gp)
1068 mp.libcallpc = getcallerpc()
1069
1070
1071 mp.libcallsp = getcallersp()
1072 resetLibcall = true
1073 }
1074 asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall))
1075 if resetLibcall {
1076 mp.libcallsp = 0
1077 }
1078 return mp.libcall.r1
1079 }
1080
1081
1082 func stdcall0(fn stdFunction) uintptr {
1083 mp := getg().m
1084 mp.libcall.n = 0
1085 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&fn)))
1086 return stdcall(fn)
1087 }
1088
1089
1090
1091 func stdcall1(fn stdFunction, a0 uintptr) uintptr {
1092 mp := getg().m
1093 mp.libcall.n = 1
1094 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1095 return stdcall(fn)
1096 }
1097
1098
1099
1100 func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr {
1101 mp := getg().m
1102 mp.libcall.n = 2
1103 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1104 return stdcall(fn)
1105 }
1106
1107
1108
1109 func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr {
1110 mp := getg().m
1111 mp.libcall.n = 3
1112 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1113 return stdcall(fn)
1114 }
1115
1116
1117
1118 func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr {
1119 mp := getg().m
1120 mp.libcall.n = 4
1121 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1122 return stdcall(fn)
1123 }
1124
1125
1126
1127 func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr {
1128 mp := getg().m
1129 mp.libcall.n = 5
1130 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1131 return stdcall(fn)
1132 }
1133
1134
1135
1136 func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
1137 mp := getg().m
1138 mp.libcall.n = 6
1139 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1140 return stdcall(fn)
1141 }
1142
1143
1144
1145 func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr {
1146 mp := getg().m
1147 mp.libcall.n = 7
1148 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1149 return stdcall(fn)
1150 }
1151
1152
1153 func usleep2(dt int32)
1154 func usleep2HighRes(dt int32)
1155 func switchtothread()
1156
1157
1158 func osyield_no_g() {
1159 switchtothread()
1160 }
1161
1162
1163 func osyield() {
1164 systemstack(switchtothread)
1165 }
1166
1167
1168 func usleep_no_g(us uint32) {
1169 dt := -10 * int32(us)
1170 usleep2(dt)
1171 }
1172
1173
1174 func usleep(us uint32) {
1175 systemstack(func() {
1176 dt := -10 * int32(us)
1177
1178
1179 if haveHighResTimer && getg().m.highResTimer != 0 {
1180 usleep2HighRes(dt)
1181 } else {
1182 usleep2(dt)
1183 }
1184 })
1185 }
1186
1187 func ctrlHandler(_type uint32) uintptr {
1188 var s uint32
1189
1190 switch _type {
1191 case _CTRL_C_EVENT, _CTRL_BREAK_EVENT:
1192 s = _SIGINT
1193 case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT:
1194 s = _SIGTERM
1195 default:
1196 return 0
1197 }
1198
1199 if sigsend(s) {
1200 if s == _SIGTERM {
1201
1202
1203
1204
1205 block()
1206 }
1207 return 1
1208 }
1209 return 0
1210 }
1211
1212
1213 func callbackasm1()
1214
1215 var profiletimer uintptr
1216
1217 func profilem(mp *m, thread uintptr) {
1218
1219 var c *context
1220 var cbuf [unsafe.Sizeof(*c) + 15]byte
1221 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
1222
1223 c.contextflags = _CONTEXT_CONTROL
1224 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1225
1226 gp := gFromSP(mp, c.sp())
1227
1228 sigprof(c.ip(), c.sp(), c.lr(), gp, mp)
1229 }
1230
1231 func gFromSP(mp *m, sp uintptr) *g {
1232 if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1233 return gp
1234 }
1235 if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1236 return gp
1237 }
1238 if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1239 return gp
1240 }
1241 return nil
1242 }
1243
1244 func profileLoop() {
1245 stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST)
1246
1247 for {
1248 stdcall2(_WaitForSingleObject, profiletimer, _INFINITE)
1249 first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
1250 for mp := first; mp != nil; mp = mp.alllink {
1251 if mp == getg().m {
1252
1253 continue
1254 }
1255
1256 lock(&mp.threadLock)
1257
1258
1259
1260 if mp.thread == 0 || mp.profilehz == 0 || mp.blocked {
1261 unlock(&mp.threadLock)
1262 continue
1263 }
1264
1265 var thread uintptr
1266 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
1267 print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n")
1268 throw("duplicatehandle failed")
1269 }
1270 unlock(&mp.threadLock)
1271
1272
1273
1274
1275
1276 if int32(stdcall1(_SuspendThread, thread)) == -1 {
1277
1278 stdcall1(_CloseHandle, thread)
1279 continue
1280 }
1281 if mp.profilehz != 0 && !mp.blocked {
1282
1283
1284 profilem(mp, thread)
1285 }
1286 stdcall1(_ResumeThread, thread)
1287 stdcall1(_CloseHandle, thread)
1288 }
1289 }
1290 }
1291
1292 func setProcessCPUProfiler(hz int32) {
1293 if profiletimer == 0 {
1294 timer := stdcall3(_CreateWaitableTimerA, 0, 0, 0)
1295 atomic.Storeuintptr(&profiletimer, timer)
1296 newm(profileLoop, nil, -1)
1297 }
1298 }
1299
1300 func setThreadCPUProfiler(hz int32) {
1301 ms := int32(0)
1302 due := ^int64(^uint64(1 << 63))
1303 if hz > 0 {
1304 ms = 1000 / hz
1305 if ms == 0 {
1306 ms = 1
1307 }
1308 due = int64(ms) * -10000
1309 }
1310 stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0)
1311 atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz))
1312 }
1313
1314 const preemptMSupported = true
1315
1316
1317
1318 var suspendLock mutex
1319
1320 func preemptM(mp *m) {
1321 if mp == getg().m {
1322 throw("self-preempt")
1323 }
1324
1325
1326 if !atomic.Cas(&mp.preemptExtLock, 0, 1) {
1327
1328
1329 atomic.Xadd(&mp.preemptGen, 1)
1330 return
1331 }
1332
1333
1334 lock(&mp.threadLock)
1335 if mp.thread == 0 {
1336
1337 unlock(&mp.threadLock)
1338 atomic.Store(&mp.preemptExtLock, 0)
1339 atomic.Xadd(&mp.preemptGen, 1)
1340 return
1341 }
1342 var thread uintptr
1343 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
1344 print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n")
1345 throw("runtime.preemptM: duplicatehandle failed")
1346 }
1347 unlock(&mp.threadLock)
1348
1349
1350 var c *context
1351 var cbuf [unsafe.Sizeof(*c) + 15]byte
1352 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
1353 c.contextflags = _CONTEXT_CONTROL
1354
1355
1356
1357
1358
1359
1360 lock(&suspendLock)
1361
1362
1363 if int32(stdcall1(_SuspendThread, thread)) == -1 {
1364 unlock(&suspendLock)
1365 stdcall1(_CloseHandle, thread)
1366 atomic.Store(&mp.preemptExtLock, 0)
1367
1368
1369 atomic.Xadd(&mp.preemptGen, 1)
1370 return
1371 }
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1383
1384 unlock(&suspendLock)
1385
1386
1387 gp := gFromSP(mp, c.sp())
1388 if gp != nil && wantAsyncPreempt(gp) {
1389 if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok {
1390
1391 targetPC := abi.FuncPCABI0(asyncPreempt)
1392 switch GOARCH {
1393 default:
1394 throw("unsupported architecture")
1395 case "386", "amd64":
1396
1397 sp := c.sp()
1398 sp -= goarch.PtrSize
1399 *(*uintptr)(unsafe.Pointer(sp)) = newpc
1400 c.set_sp(sp)
1401 c.set_ip(targetPC)
1402
1403 case "arm":
1404
1405
1406
1407
1408
1409 sp := c.sp()
1410 sp -= goarch.PtrSize
1411 c.set_sp(sp)
1412 *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr())
1413 c.set_lr(newpc - 1)
1414 c.set_ip(targetPC)
1415
1416 case "arm64":
1417
1418
1419
1420
1421 sp := c.sp() - 16
1422 c.set_sp(sp)
1423 *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr())
1424 c.set_lr(newpc)
1425 c.set_ip(targetPC)
1426 }
1427 stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1428 }
1429 }
1430
1431 atomic.Store(&mp.preemptExtLock, 0)
1432
1433
1434 atomic.Xadd(&mp.preemptGen, 1)
1435
1436 stdcall1(_ResumeThread, thread)
1437 stdcall1(_CloseHandle, thread)
1438 }
1439
1440
1441
1442
1443
1444
1445
1446
1447 func osPreemptExtEnter(mp *m) {
1448 for !atomic.Cas(&mp.preemptExtLock, 0, 1) {
1449
1450
1451
1452
1453
1454
1455
1456
1457 osyield()
1458 }
1459
1460 }
1461
1462
1463
1464
1465
1466
1467
1468 func osPreemptExtExit(mp *m) {
1469 atomic.Store(&mp.preemptExtLock, 0)
1470 }
1471
View as plain text