Leaked source code of windows server 2003
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

648 lines
16 KiB

  1. /*++
  2. Copyright (c) 2000 Microsoft Corporation
  3. Module Name:
  4. allproc.c
  5. Abstract:
  6. This module allocates and initializes kernel resources required to
  7. start a new processor, and passes a complete process state structure
  8. to the hal to obtain a new processor.
  9. Author:
  10. David N. Cutler (davec) 5-May-2000
  11. Environment:
  12. Kernel mode only.
  13. Revision History:
  14. --*/
  15. #include "ki.h"
  16. #include "pool.h"
  17. //
  18. // Define local macros.
  19. //
  20. #define ROUNDUP16(x) (((x) + 15) & ~15)
  21. //
  22. // Define prototypes for forward referenced functions.
  23. //
  24. #if !defined(NT_UP)
  25. VOID
  26. KiCopyDescriptorMemory (
  27. IN PKDESCRIPTOR Source,
  28. IN PKDESCRIPTOR Destination,
  29. IN PVOID Base
  30. );
  31. NTSTATUS
  32. KiNotNumaQueryProcessorNode (
  33. IN ULONG ProcessorNumber,
  34. OUT PUSHORT Identifier,
  35. OUT PUCHAR Node
  36. );
  37. VOID
  38. KiSetDescriptorBase (
  39. IN USHORT Selector,
  40. IN PKGDTENTRY64 GdtBase,
  41. IN PVOID Base
  42. );
  43. PHALNUMAQUERYPROCESSORNODE KiQueryProcessorNode = KiNotNumaQueryProcessorNode;
  44. //
  45. // Statically allocate enough KNODE structures to allow memory management
  46. // to allocate pages by node during system initialization. As processors
  47. // are brought online, real KNODE structures are allocated in the correct
  48. // memory for the node.
  49. //
  50. #pragma data_seg("INITDATA")
  51. KNODE KiNodeInit[MAXIMUM_CCNUMA_NODES];
  52. #pragma data_seg()
  53. #pragma alloc_text(INIT, KiAllProcessorsStarted)
  54. #pragma alloc_text(INIT, KiCopyDescriptorMemory)
  55. #pragma alloc_text(INIT, KiNotNumaQueryProcessorNode)
  56. #pragma alloc_text(INIT, KiSetDescriptorBase)
  57. #endif // !defined(NT_UP)
  58. #pragma alloc_text(INIT, KeStartAllProcessors)
  59. ULONG KiBarrierWait = 0;
  60. VOID
  61. KeStartAllProcessors (
  62. VOID
  63. )
  64. /*++
  65. Routine Description:
  66. This function is called during phase 1 initialization on the master boot
  67. processor to start all of the other registered processors.
  68. Arguments:
  69. None.
  70. Return Value:
  71. None.
  72. --*/
  73. {
  74. #if !defined(NT_UP)
  75. ULONG AllocationSize;
  76. PUCHAR Base;
  77. PKPCR CurrentPcr = KeGetPcr();
  78. PVOID DataBlock;
  79. PVOID DpcStack;
  80. PKGDTENTRY64 GdtBase;
  81. ULONG GdtOffset;
  82. ULONG IdtOffset;
  83. PVOID KernelStack;
  84. PKNODE Node;
  85. UCHAR NodeNumber;
  86. UCHAR Number;
  87. PKPCR PcrBase;
  88. USHORT ProcessorId;
  89. KPROCESSOR_STATE ProcessorState;
  90. NTSTATUS Status;
  91. PKTSS64 SysTssBase;
  92. PETHREAD Thread;
  93. //
  94. // Do not start additional processors if the RELOCATEPHYSICAL loader
  95. // switch has been specified.
  96. //
  97. if (KeLoaderBlock->LoadOptions != NULL) {
  98. if (strstr(KeLoaderBlock->LoadOptions, "RELOCATEPHYSICAL") != NULL) {
  99. return;
  100. }
  101. }
  102. //
  103. // If processor zero is not on node zero, then move it to the appropriate
  104. // node.
  105. //
  106. if (KeNumberNodes > 1) {
  107. Status = KiQueryProcessorNode(0, &ProcessorId, &NodeNumber);
  108. if (NT_SUCCESS(Status)) {
  109. //
  110. // Adjust the data structures to reflect that P0 is not on Node 0.
  111. //
  112. if (NodeNumber != 0) {
  113. ASSERT(KeNodeBlock[0] == &KiNode0);
  114. KeNodeBlock[0]->ProcessorMask &= ~1;
  115. KiNodeInit[0] = *KeNodeBlock[0];
  116. KeNodeBlock[0] = &KiNodeInit[0];
  117. KiNode0 = *KeNodeBlock[NodeNumber];
  118. KeNodeBlock[NodeNumber] = &KiNode0;
  119. KeNodeBlock[NodeNumber]->ProcessorMask |= 1;
  120. }
  121. }
  122. }
  123. //
  124. // Calculate the size of the per processor data structures.
  125. //
  126. // This includes:
  127. //
  128. // PCR (including the PRCB)
  129. // System TSS
  130. // Idle Thread Object
  131. // Double Fault/NMI Panic Stack
  132. // Machine Check Stack
  133. // GDT
  134. // IDT
  135. //
  136. // If this is a multinode system, the KNODE structure is also allocated.
  137. //
  138. // A DPC and Idle stack are also allocated, but they are done separately.
  139. //
  140. AllocationSize = ROUNDUP16(sizeof(KPCR)) +
  141. ROUNDUP16(sizeof(KTSS64)) +
  142. ROUNDUP16(sizeof(ETHREAD)) +
  143. ROUNDUP16(DOUBLE_FAULT_STACK_SIZE) +
  144. ROUNDUP16(KERNEL_MCA_EXCEPTION_STACK_SIZE);
  145. AllocationSize += ROUNDUP16(sizeof(KNODE));
  146. //
  147. // Save the offset of the GDT in the allocation structure and add in
  148. // the size of the GDT.
  149. //
  150. GdtOffset = AllocationSize;
  151. AllocationSize +=
  152. CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr.Limit + 1;
  153. //
  154. // Save the offset of the IDT in the allocation structure and add in
  155. // the size of the IDT.
  156. //
  157. IdtOffset = AllocationSize;
  158. AllocationSize +=
  159. CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr.Limit + 1;
  160. //
  161. // If the registered number of processors is greater than the maximum
  162. // number of processors supported, then only allow the maximum number
  163. // of supported processors.
  164. //
  165. if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) {
  166. KeRegisteredProcessors = MAXIMUM_PROCESSORS;
  167. }
  168. //
  169. // Set barrier that will prevent any other processor from entering the
  170. // idle loop until all processors have been started.
  171. //
  172. KiBarrierWait = 1;
  173. //
  174. // Initialize the fixed part of the processor state that will be used to
  175. // start processors. Each processor starts in the system initialization
  176. // code with address of the loader parameter block as an argument.
  177. //
  178. RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE));
  179. ProcessorState.ContextFrame.Rcx = (ULONG64)KeLoaderBlock;
  180. ProcessorState.ContextFrame.Rip = (ULONG64)KiSystemStartup;
  181. ProcessorState.ContextFrame.SegCs = KGDT64_R0_CODE;
  182. ProcessorState.ContextFrame.SegDs = KGDT64_R3_DATA | RPL_MASK;
  183. ProcessorState.ContextFrame.SegEs = KGDT64_R3_DATA | RPL_MASK;
  184. ProcessorState.ContextFrame.SegFs = KGDT64_R3_CMTEB | RPL_MASK;
  185. ProcessorState.ContextFrame.SegGs = KGDT64_R3_DATA | RPL_MASK;
  186. ProcessorState.ContextFrame.SegSs = KGDT64_NULL;
  187. //
  188. // Loop trying to start a new processors until a new processor can't be
  189. // started or an allocation failure occurs.
  190. //
  191. Number = 0;
  192. while ((ULONG)KeNumberProcessors < KeRegisteredProcessors) {
  193. Number += 1;
  194. Status = KiQueryProcessorNode(Number, &ProcessorId, &NodeNumber);
  195. if (!NT_SUCCESS(Status)) {
  196. //
  197. // No such processor, advance to next.
  198. //
  199. continue;
  200. }
  201. Node = KeNodeBlock[NodeNumber];
  202. //
  203. // Allocate memory for the new processor specific data. If the
  204. // allocation fails, then stop starting processors.
  205. //
  206. DataBlock = MmAllocateIndependentPages(AllocationSize, NodeNumber);
  207. if (DataBlock == NULL) {
  208. break;
  209. }
  210. //
  211. // Allocate a pool tag table for the new processor.
  212. //
  213. if (ExCreatePoolTagTable(Number, NodeNumber) == NULL) {
  214. MmFreeIndependentPages(DataBlock, AllocationSize);
  215. break;
  216. }
  217. //
  218. // Zero the allocated memory.
  219. //
  220. Base = (PUCHAR)DataBlock;
  221. RtlZeroMemory(DataBlock, AllocationSize);
  222. //
  223. // Copy and initialize the GDT for the next processor.
  224. //
  225. KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr,
  226. &ProcessorState.SpecialRegisters.Gdtr,
  227. Base + GdtOffset);
  228. GdtBase = (PKGDTENTRY64)ProcessorState.SpecialRegisters.Gdtr.Base;
  229. //
  230. // Copy and initialize the IDT for the next processor.
  231. //
  232. KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr,
  233. &ProcessorState.SpecialRegisters.Idtr,
  234. Base + IdtOffset);
  235. //
  236. // Set the PCR base address for the next processor and set the
  237. // processor number.
  238. //
  239. // N.B. The PCR address is passed to the next processor by computing
  240. // the containing address with respect to the PRCB.
  241. //
  242. PcrBase = (PKPCR)Base;
  243. PcrBase->Number = Number;
  244. PcrBase->Prcb.Number = Number;
  245. Base += ROUNDUP16(sizeof(KPCR));
  246. //
  247. // Set the system TSS descriptor base for the next processor.
  248. //
  249. SysTssBase = (PKTSS64)Base;
  250. KiSetDescriptorBase(KGDT64_SYS_TSS / 16, GdtBase, SysTssBase);
  251. Base += ROUNDUP16(sizeof(KTSS64));
  252. //
  253. // Initialize the panic stack address for double fault and NMI.
  254. //
  255. Base += DOUBLE_FAULT_STACK_SIZE;
  256. SysTssBase->Ist[TSS_IST_PANIC] = (ULONG64)Base;
  257. //
  258. // Initialize the machine check stack address.
  259. //
  260. Base += KERNEL_MCA_EXCEPTION_STACK_SIZE;
  261. SysTssBase->Ist[TSS_IST_MCA] = (ULONG64)Base;
  262. //
  263. // Idle Thread thread object.
  264. //
  265. Thread = (PETHREAD)Base;
  266. Base += ROUNDUP16(sizeof(ETHREAD));
  267. //
  268. // Set other special registers in the processor state.
  269. //
  270. ProcessorState.SpecialRegisters.Cr0 = ReadCR0();
  271. ProcessorState.SpecialRegisters.Cr3 = ReadCR3();
  272. ProcessorState.ContextFrame.EFlags = 0;
  273. ProcessorState.SpecialRegisters.Tr = KGDT64_SYS_TSS;
  274. GdtBase[KGDT64_SYS_TSS / 16].Bytes.Flags1 = 0x89;
  275. ProcessorState.SpecialRegisters.Cr4 = ReadCR4();
  276. //
  277. // Allocate a kernel stack and a DPC stack for the next processor.
  278. //
  279. KernelStack = MmCreateKernelStack(FALSE, NodeNumber);
  280. if (KernelStack == NULL) {
  281. MmFreeIndependentPages(DataBlock, AllocationSize);
  282. break;
  283. }
  284. DpcStack = MmCreateKernelStack(FALSE, NodeNumber);
  285. if (DpcStack == NULL) {
  286. MmDeleteKernelStack(KernelStack, FALSE);
  287. MmFreeIndependentPages(DataBlock, AllocationSize);
  288. break;
  289. }
  290. //
  291. // Initialize the kernel stack for the system TSS.
  292. //
  293. SysTssBase->Rsp0 = (ULONG64)KernelStack - sizeof(PVOID) * 4;
  294. ProcessorState.ContextFrame.Rsp = (ULONG64)KernelStack;
  295. //
  296. // If this is the first processor on this node, then use the space
  297. // allocated for KNODE as the KNODE.
  298. //
  299. if (KeNodeBlock[NodeNumber] == &KiNodeInit[NodeNumber]) {
  300. Node = (PKNODE)Base;
  301. *Node = KiNodeInit[NodeNumber];
  302. KeNodeBlock[NodeNumber] = Node;
  303. }
  304. Base += ROUNDUP16(sizeof(KNODE));
  305. PcrBase->Prcb.ParentNode = Node;
  306. //
  307. // Adjust the loader block so it has the next processor state. Ensure
  308. // that the KernelStack has space for home registers for up to four
  309. // parameters.
  310. //
  311. KeLoaderBlock->KernelStack = (ULONG64)DpcStack - (sizeof(PVOID) * 4);
  312. KeLoaderBlock->Thread = (ULONG64)Thread;
  313. KeLoaderBlock->Prcb = (ULONG64)(&PcrBase->Prcb);
  314. //
  315. // Attempt to start the next processor. If a processor cannot be
  316. // started, then deallocate memory and stop starting processors.
  317. //
  318. if (HalStartNextProcessor(KeLoaderBlock, &ProcessorState) == 0) {
  319. ExDeletePoolTagTable (Number);
  320. MmFreeIndependentPages(DataBlock, AllocationSize);
  321. MmDeleteKernelStack(KernelStack, FALSE);
  322. MmDeleteKernelStack(DpcStack, FALSE);
  323. break;
  324. }
  325. Node->ProcessorMask |= AFFINITY_MASK(Number);
  326. //
  327. // Wait for processor to initialize.
  328. //
  329. while (*((volatile ULONG64 *)&KeLoaderBlock->Prcb) != 0) {
  330. KeYieldProcessor();
  331. }
  332. }
  333. //
  334. // All processors have been stated.
  335. //
  336. KiAllProcessorsStarted();
  337. //
  338. // Reset and synchronize the performance counters of all processors, by
  339. // applying a null adjustment to the interrupt time
  340. //
  341. KeAdjustInterruptTime(0);
  342. //
  343. // Allow all processors that were started to enter the idle loop and
  344. // begin execution.
  345. //
  346. KiBarrierWait = 0;
  347. #endif // !defined(NT_UP)
  348. return;
  349. }
  350. #if !defined(NT_UP)
  351. VOID
  352. KiSetDescriptorBase (
  353. IN USHORT Selector,
  354. IN PKGDTENTRY64 GdtBase,
  355. IN PVOID Base
  356. )
  357. /*++
  358. Routine Description:
  359. This function sets the base address of a descriptor to the specified
  360. base address.
  361. Arguments:
  362. Selector - Supplies the selector for the descriptor.
  363. GdtBase - Supplies a pointer to the GDT.
  364. Base - Supplies a pointer to the base address.
  365. Return Value:
  366. None.
  367. --*/
  368. {
  369. GdtBase = &GdtBase[Selector];
  370. GdtBase->BaseLow = (USHORT)((ULONG64)Base);
  371. GdtBase->Bytes.BaseMiddle = (UCHAR)((ULONG64)Base >> 16);
  372. GdtBase->Bytes.BaseHigh = (UCHAR)((ULONG64)Base >> 24);
  373. GdtBase->BaseUpper = (ULONG)((ULONG64)Base >> 32);
  374. return;
  375. }
  376. VOID
  377. KiCopyDescriptorMemory (
  378. IN PKDESCRIPTOR Source,
  379. IN PKDESCRIPTOR Destination,
  380. IN PVOID Base
  381. )
  382. /*++
  383. Routine Description:
  384. This function copies the specified descriptor memory to the new memory
  385. and initializes a descriptor for the new memory.
  386. Arguments:
  387. Source - Supplies a pointer to the source descriptor that describes
  388. the memory to copy.
  389. Destination - Supplies a pointer to the destination descriptor to be
  390. initialized.
  391. Base - Supplies a pointer to the new memory.
  392. Return Value:
  393. None.
  394. --*/
  395. {
  396. Destination->Limit = Source->Limit;
  397. Destination->Base = Base;
  398. RtlCopyMemory(Base, Source->Base, Source->Limit + 1);
  399. return;
  400. }
  401. VOID
  402. KiAllProcessorsStarted (
  403. VOID
  404. )
  405. /*++
  406. Routine Description:
  407. This routine is called once all processors in the system have been started.
  408. Arguments:
  409. None.
  410. Return Value:
  411. None.
  412. --*/
  413. {
  414. ULONG i;
  415. //
  416. // Make sure there are no references to the temporary nodes used during
  417. // initialization.
  418. //
  419. for (i = 0; i < KeNumberNodes; i += 1) {
  420. if (KeNodeBlock[i] == &KiNodeInit[i]) {
  421. //
  422. // No processor started on this node so no new node structure has
  423. // been allocated. This is possible if the node contains memory
  424. // only or IO busses. At this time we need to allocate a permanent
  425. // node structure for the node.
  426. //
  427. KeNodeBlock[i] = ExAllocatePoolWithTag(NonPagedPool,
  428. sizeof(KNODE),
  429. ' eK');
  430. if (KeNodeBlock[i]) {
  431. *KeNodeBlock[i] = KiNodeInit[i];
  432. }
  433. }
  434. //
  435. // Set the node number.
  436. //
  437. KeNodeBlock[i]->NodeNumber = (UCHAR)i;
  438. }
  439. for (i = KeNumberNodes; i < MAXIMUM_CCNUMA_NODES; i += 1) {
  440. KeNodeBlock[i] = NULL;
  441. }
  442. if (KeNumberNodes == 1) {
  443. //
  444. // For Non NUMA machines, Node 0 gets all processors.
  445. //
  446. KeNodeBlock[0]->ProcessorMask = KeActiveProcessors;
  447. }
  448. return;
  449. }
  450. NTSTATUS
  451. KiNotNumaQueryProcessorNode (
  452. IN ULONG ProcessorNumber,
  453. OUT PUSHORT Identifier,
  454. OUT PUCHAR Node
  455. )
  456. /*++
  457. Routine Description:
  458. This routine is a stub used on non NUMA systems to provide a
  459. consistent method of determining the NUMA configuration rather
  460. than checking for the presense of multiple nodes inline.
  461. Arguments:
  462. ProcessorNumber supplies the system logical processor number.
  463. Identifier supplies the address of a variable to receive
  464. the unique identifier for this processor.
  465. NodeNumber supplies the address of a variable to receive
  466. the number of the node this processor resides on.
  467. Return Value:
  468. Returns success.
  469. --*/
  470. {
  471. *Identifier = (USHORT)ProcessorNumber;
  472. *Node = 0;
  473. return STATUS_SUCCESS;
  474. }
  475. #endif // !defined(NT_UP)