Add x2APIC support

src/Demo/Lectures/PCIInterrupts.ZC

@@ -1,6 +1,7 @@
 //See $LK,"PCI",A="HI:PCI"$.
 
-#if mp_count > 1
+//I/O APIC destinations are 8-bit and 0xFF is broadcast
+#if mp_count > 1 && dev.mp_apic_ids[1] < XAPIC_BROADCAST_ID
 #define DEST_CPU				1
 #else
 #define DEST_CPU				0
@@ -11,25 +12,25 @@
 interrupt U0 IntMy40()
 {
 	lock {progress1++;}
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+	LAPICEoi;
 }
 
 interrupt U0 IntMy41()
 {
 	lock {progress2++;}
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+	LAPICEoi;
 }
 
 interrupt U0 IntMy42()
 {
 	lock {progress3++;}
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+	LAPICEoi;
 }
 
 interrupt U0 IntMy43()
 {
 	lock {progress4++;}
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+	LAPICEoi;
 }
 
 I64 vect[4] = {&IntMy40, &IntMy41, &IntMy42, &IntMy43};

src/Demo/MultiCore/Interrupts.ZC

@@ -1,12 +1,12 @@
 interrupt U0 IntBeepOn()
 {
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0; //signal end of interrupt
+	LAPICEoi; //signal end of interrupt
 	Sound(34);
 }
 
 interrupt U0 IntBeepOff()
 {
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+	LAPICEoi;
 	Sound;
 }
 

src/Home/Net/Drivers/E1000.ZC

@@ -421,7 +421,7 @@ interrupt U0 E1000IRQ()
 
 	E1000MMIORead(E1000_REG_ICR); // clear pending interrupts?
 
-	*(dev.uncached_alias + LAPIC_EOI)(U32*) = 0;
+	LAPICEoi;
 }
 
 U0 PCIInterruptsReroute(I64 base)

src/Home/Net/Drivers/PCNet.ZC

@@ -697,7 +697,7 @@ interrupt U0 PCNetIRQ()
 			   "$$BG$$$$FG$$");
 	}
 
-	*(dev.uncached_alias + LAPIC_EOI)(U32*) = 0;
+	LAPICEoi;
 }
 
 U0 PCIInterruptsReroute(I64 base)

src/Home/Net/Drivers/RTL8139.ZC

@@ -388,7 +388,7 @@ interrupt U0 RTL8139IRQ()
 		}
 	}
 
-	*(dev.uncached_alias + LAPIC_EOI)(U32*) = 0;
+	LAPICEoi;
 }
 
 U0 PCIInterruptsReroute(I64 base)

src/Home/Net/Utilities/NetHandler.ZC

@@ -97,7 +97,7 @@ interrupt U0 NetHandler()
 		   "==================="
 		   "$$BG$$$$FG$$");
 
-	*(dev.uncached_alias + LAPIC_EOI)(U32*) = 0;
+	LAPICEoi;
 }
 
 U0 NetHandlerInit()

src/Kernel/ACPI.ZC

@@ -0,0 +1,70 @@
+//ACPI table lookup.
+//Based on ACPI specification 6.6
+
+CACPIRSDP *ACPIRSDPFind()
+{//Find Root System Description Pointer. Return NULL if not found.
+	I64		 i, ebda;
+	U8 noreg checksum, *mem;
+
+	if (sys_acpi_rsdp) //passed by bootloader on UEFI systems, ACPI 6.6 section 5.2.5.2
+		return sys_acpi_rsdp;
+
+	//Search first KB of EBDA, then 0xE0000 - 0x100000, ACPI 6.6 section 5.2.5.1
+	ebda = *0x40E(U16 *) << 4;
+	if (ebda < 0x80000 || ebda >= 0xA0000)
+		ebda = 0x80000;
+
+	for (mem = ebda; mem < 0x100000; mem += 16) //anchor sits on 16-byte boundary
+	{
+		if (mem == ebda + 1024)
+			mem = 0xE0000;
+		if (!MemCompare(mem, "RSD PTR ", 8))
+		{
+			checksum = 0;
+			for (i = 0; i < 20; i++)
+				checksum += mem[i];
+
+			if (checksum == 0) //valid checksums sum to zero
+				return mem;
+		}
+	}
+
+	return NULL;
+}
+
+CACPISDTHeader *ACPITableFind(U8 *signature, I64 index=0)
+{//Find index'th table with signature. Return NULL if not found.
+//Only RSDP checksum is checked, firmware often gets table checksums wrong.
+	CACPIRSDP		*rsdp = ACPIRSDPFind;
+	CACPISDTHeader	*root, *table;
+	I64				 i, entry_size;
+
+	if (!rsdp)
+		return NULL;
+
+	if (rsdp->revision >= 2 && rsdp->xsdt_address) //XSDT must be used if present, ACPI 6.6 section 5.2.8
+	{
+		root = rsdp->xsdt_address;
+		entry_size = sizeof(U64);
+	}
+	else
+	{
+		root = rsdp->rsdt_address;
+		entry_size = sizeof(U32);
+	}
+	if (!root || root->length < sizeof(CACPISDTHeader))
+		return NULL;
+
+	for (i = sizeof(CACPISDTHeader); i + entry_size <= root->length; i += entry_size)
+	{
+		if (entry_size == sizeof(U64))
+			table = *(root(U8 *) + i)(U64 *);
+		else
+			table = *(root(U8 *) + i)(U32 *);
+
+		if (table && !MemCompare(table->signature, signature, 4) && !index--)
+			return table;
+	}
+
+	return NULL;
+}

src/Kernel/KInterrupts.ZC

@@ -5,10 +5,20 @@ INT_MP_CRASH_ADDR:: //Forward reference to work around compiler
 INT_WAKE::
 		PUSH	RDX
 		PUSH	RAX
-		MOV 	EAX, &dev
+		PUSH	RCX
+		CMP 	U64 [SYS_X2APIC], 0
+		JE		@@05
+		MOV 	ECX, IA32_X2APIC_EOI
+		XOR 	EAX, EAX
+		XOR 	EDX, EDX
+		MFENCE	//See $LK,"X2APICWrite",A="MN:X2APICWrite"$().
+		LFENCE
+		WRMSR
+		JMP 	@@10
+@@05: 	MOV 	RAX, U64 [&dev.uncached_alias]
 		MOV 	EDX, U32 LAPIC_EOI
-		MOV 	RAX, U64 CDevGlobals.uncached_alias[RAX]
 		MOV 	U32 [RAX+RDX], 0
+@@10: 	POP 	RCX
 		POP 	RAX
 		POP 	RDX
 		IRET
@@ -46,9 +56,17 @@ IRQ_TIMER::  //I_TIMER
 		TEST	RAX, RAX
 		JZ		@@20
 
-		MOV 	EAX, &dev
+		CMP 	U64 [SYS_X2APIC], 0
+		JE		@@17
+		MOV 	ECX, IA32_X2APIC_EOI
+		XOR 	EAX, EAX
+		XOR 	EDX, EDX
+		MFENCE	//See $LK,"X2APICWrite",A="MN:X2APICWrite"$().
+		LFENCE
+		WRMSR
+		JMP 	@@25
+@@17: 	MOV 	RAX, U64 [&dev.uncached_alias]
 		MOV 	EDX, U32 LAPIC_EOI
-		MOV 	RAX, U64 CDevGlobals.uncached_alias[RAX]
 		MOV 	U32 [RAX + RDX], 0
 		JMP 	@@25
 
@@ -161,11 +179,19 @@ U0 IntPICInit()
 	OutU8(PIC_2_DATA, 0xFF);
 }
 
+public U0 LAPICEoi()
+{//Signal end of interrupt to the local APIC.
+	if (sys_x2apic)
+		X2APICWrite(IA32_X2APIC_EOI, 0);
+	else
+		*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+}
+
 interrupt U0 IntNop()
 {//Make unplanned IRQs stop by all means!
 	OutU8(PIC_2, PIC_EOI);
 	OutU8(PIC_1, PIC_EOI);
-	*(dev.uncached_alias + LAPIC_EOI)(U32 *) = 0;
+	LAPICEoi;
 }
 
 I64 IntEntryFree(I64 irq)

src/Kernel/KStart16.ZC

@@ -68,6 +68,8 @@ SYS_BOOTLOADER_ID::		DU8	BL_ZEAL;
 
 SYS_FRAMEBUFFER_LIST::	DU8	sizeof(CVideoInfo) * VBE_MODES_NUM	DUP(0);
 
+SYS_ACPI_RSDP::			DU64	0;
+
 #assert $$ - SYS_KERNEL == sizeof(CKernel) - sizeof(CZXE)
 
 REQUESTED_SCREEN_WIDTH:		DU16	1024;

src/Kernel/KStart32.ZC

@@ -69,6 +69,7 @@ SYS_CPU_STRUCTS:: 		DU64		0;
 SYS_MP_COUNT::			DU64		1;
 SYS_MP_COUNT_INITIAL::	DU64		1;
 SYS_MP_COUNT_LOCK:: 	DU64		1;
+SYS_X2APIC::			DU64		0;
 
 		ALIGN 	DEFAULT_CACHE_LINE_WIDTH, OC_NOP
 SYS_CACHE_LINE_WIDTH::	DU64	DEFAULT_CACHE_LINE_WIDTH;

src/Kernel/KUtils.ZC

@@ -565,8 +565,7 @@ _MSR_READ::
 		MOV		RCX, SF_ARG1[RBP]
 		RDMSR
 		SHL		RDX, 32
-		MOV		EDX, EAX
-		MOV		RAX, RDX
+		OR		RAX, RDX
 		POP		RBP
 		RET1	8
 
@@ -582,6 +581,21 @@ _MSR_WRITE::
 		POP 	RBP
 		RET1	16
 
+/***********************************/
+_X2APIC_WRITE::
+		PUSH	RBP
+		MOV 	RBP, RSP
+		MOV 	RAX, SF_ARG2[RBP]
+		MOV 	RDX, RAX
+		SHR 	RDX, 32
+		MOV 	RCX, SF_ARG1[RBP]
+//x2APIC WRMSR isn't serializing, Intel SDM 325462-092 Vol. 3A section 13.12.3, AMD APM 40332 rev 4.10 Vol. 2 section 16.11.2
+		MFENCE
+		LFENCE
+		WRMSR
+		POP 	RBP
+		RET1	16
+
 /***********************************/
 _SYS_HLT::
 		CLI

src/Kernel/Kernel.PRJ

@@ -52,6 +52,7 @@
 #include "Job"
 #include "PCIBIOS"
 #include "PCI"
+#include "ACPI"
 #include "MultiProc"
 #include "EdLite"
 #include "BlkDev/MakeBlkDev"

src/Kernel/KernelA.HH

@@ -576,6 +576,7 @@ class CKernel
 	Bool			sys_is_uefi_booted;
 	U8				sys_bootloader_id;
 	CVideoInfo		sys_framebuffer_list[VBE_MODES_NUM];
+	U64				sys_acpi_rsdp;
 };
 
 // Bootloader IDs
@@ -637,6 +638,33 @@ class CKernel
 #define IA32_FS_BASE	0xC0000100
 #define IA32_GS_BASE	0xC0000101
 
+#define IA32_APIC_BASEF_X2APIC	(1 << 10)
+#define IA32_APIC_BASEF_ENABLED	(1 << 11)
+#define CPUID_1_ECX_X2APIC		(1 << 21)
+
+#define IA32_X2APIC_APICID		0x802
+#define IA32_X2APIC_VERSION		0x803
+#define IA32_X2APIC_TPR			0x808
+#define IA32_X2APIC_PPR			0x80A
+#define IA32_X2APIC_EOI			0x80B
+#define IA32_X2APIC_LDR			0x80D
+#define IA32_X2APIC_SVR			0x80F
+#define IA32_X2APIC_ISR0		0x810
+#define IA32_X2APIC_TMR0		0x818
+#define IA32_X2APIC_IRR0		0x820
+#define IA32_X2APIC_ESR			0x828
+#define IA32_X2APIC_ICR			0x830
+#define IA32_X2APIC_LVT_TIMER	0x832
+#define IA32_X2APIC_LVT_THERMAL	0x833
+#define IA32_X2APIC_LVT_PERF	0x834
+#define IA32_X2APIC_LVT_LINT0	0x835
+#define IA32_X2APIC_LVT_LINT1	0x836
+#define IA32_X2APIC_LVT_ERR		0x837
+
+//Physical destination broadcast IDs, Intel SDM 325462-092 Vol. 3A sections 13.6.2.1 and 13.12.9
+#define XAPIC_BROADCAST_ID	0xFF
+#define X2APIC_BROADCAST_ID	0xFFFFFFFF
+
 //Programmable Interrupt Controller
 #define PIC_1		0x20
 #define PIC_1_DATA 	0x21
@@ -665,6 +693,7 @@ class CKernel
 #define LAPIC_IRR 					(LAPIC_BASE + 0x200)
 #define LAPIC_ICR_LOW 				(LAPIC_BASE + 0x300)
 #define LAPIC_ICR_HIGH				(LAPIC_BASE + 0x310)
+#define LAPICF_ICR_SEND_PENDING		0x1000
 
 #define LAPIC_LVT_TIMER 			(LAPIC_BASE + 0x320)
 #define LAPIC_LVT_THERMAL 			(LAPIC_BASE + 0x330)
@@ -2390,6 +2419,44 @@ class CUAsmGlobals
 			mem_arg_mask;
 };
 
+#help_index "Devices/ACPI"
+//ACPI table parsing based on ACPI specification 6.6
+class CACPIRSDP
+{//ACPI 6.6 section 5.2.5.3
+	U8	signature[8], // 'RSD PTR '
+		checksum,
+		oem_id[6],
+		revision;
+	U32	rsdt_address,
+		length;
+	U64	xsdt_address;
+	U8	extended_checksum,
+		reserved[3];
+};
+
+class CACPISDTHeader
+{//ACPI 6.6 section 5.2.6
+	U8	signature[4];
+	U32	length;
+	U8	revision,
+		checksum,
+		oem_id[6],
+		oem_table_id[8];
+	U32	oem_revision,
+		creator_id,
+		creator_revision;
+};
+
+class CACPIDMAR
+{//Intel VT-d specification 5.20 section 8.1
+	CACPISDTHeader	header;
+	U8				host_address_width,
+					flags,
+					reserved[10];
+};
+#define DMARF_INTR_REMAP		1
+#define DMARF_X2APIC_OPT_OUT	2 //only valid with DMARF_INTR_REMAP
+
 #help_index "Devices/SMBIOS"
 //SMBIOS parsing based on SMBIOS specification 3.3.0 (document "DSP0134")
 #define SMBIOSt_BIOS				0
@@ -3615,7 +3682,7 @@ public class CDevGlobals
 	CIDTEntry	*idt;
 	U8			*mem64_ptr;
 	U8			*uncached_alias; //Alias of lowest 4Gig.
-	U8			 mp_apic_ids[MP_PROCESSORS_NUM];
+	U32			 mp_apic_ids[MP_PROCESSORS_NUM];
 	CMemRange	 mem32_head;
 	CPCIDev		 pci_head;
 	U64			 user_int_bitmap;