use alloc::vec::Vec;
use tracing::{error, warn};
use crate::{
acpi::tables::{self, BiosTables, InterruptControllerStruct, InterruptSourceOverride},
cpu::{self, idt::InterruptStackFrame64, Cpu, CPUS, MAX_CPUS},
memory_management::virtual_space::VirtualSpace,
sync::{once::OnceLock, spin::mutex::Mutex},
utils::vcell::RW,
};
use super::{
allocate_basic_user_interrupt, allocate_user_interrupt, allocate_user_interrupt_all_saved,
InterruptHandler,
};
const APIC_BAR_ENABLED: u64 = 1 << 11;
const APIC_BASE_MASK: u64 = 0xFFFF_FFFF_FFFF_F000;
static APIC: OnceLock<Mutex<Apic>> = OnceLock::new();
pub fn init(bios_tables: &BiosTables) {
if APIC.try_get().is_some() {
panic!("APIC already initialized");
}
disable_pic();
APIC.get_or_init(|| Mutex::new(Apic::new(bios_tables)));
}
fn disable_pic() {
unsafe {
cpu::io_out::<u8>(0x21, 0xFF);
cpu::io_out::<u8>(0xA1, 0xFF);
}
}
pub fn return_from_interrupt() {
APIC.get().lock().return_from_interrupt();
}
pub fn is_irq_assigned(irq_num: u8) -> bool {
APIC.get().lock().is_irq_assigned(irq_num)
}
pub fn assign_io_irq<H: InterruptHandler>(handler: H, interrupt_num: u8, cpu: &Cpu) {
APIC.get().lock().assign_io_irq(handler, interrupt_num, cpu)
}
#[allow(dead_code)]
pub fn assign_io_irq_custom<H: InterruptHandler, F>(
handler: H,
interrupt_num: u8,
cpu: &Cpu,
modify_entry: F,
) where
F: FnOnce(IoApicRedirectionBuilder) -> IoApicRedirectionBuilder,
{
APIC.get()
.lock()
.assign_io_irq_custom(handler, interrupt_num, cpu, modify_entry)
}
#[repr(C, align(4))]
struct ApicReg {
reg: RW<u32>,
pad: [u32; 3],
}
impl ApicReg {
fn write(&mut self, value: u32) {
unsafe { self.reg.write(value) };
}
fn read(&self) -> u32 {
self.reg.read()
}
}
#[repr(C)]
struct LocalVectorRegister {
reg: ApicReg,
}
#[allow(dead_code)]
impl LocalVectorRegister {
fn read(&self) -> LocalVectorRegisterBuilder {
LocalVectorRegisterBuilder {
reg: self.reg.read(),
}
}
fn write(&mut self, builder: LocalVectorRegisterBuilder) {
self.reg.write(builder.reg)
}
}
const LVT_VECTOR_MASK: u32 = 0xFF;
const LVT_MESSAGE_TYPE_MASK: u32 = 0x7 << 8;
const LVT_TRIGGER_MODE_MASK: u32 = 1 << 15;
const LVT_MASK_MASK: u32 = 1 << 16;
const LVT_TIMER_MODE_MASK: u32 = 1 << 17;
const SPURIOUS_ENABLE: u32 = 1 << 8;
#[derive(Default, Clone, Copy)]
struct LocalVectorRegisterBuilder {
reg: u32,
}
#[allow(dead_code)]
impl LocalVectorRegisterBuilder {
fn with_vector(mut self, vector: u8) -> Self {
self.reg = (self.reg & !LVT_VECTOR_MASK) | vector as u32;
self
}
fn with_message_type(mut self, message_type: u8) -> Self {
self.reg = (self.reg & !LVT_MESSAGE_TYPE_MASK) | ((message_type & 0x7) as u32) << 8;
self
}
fn with_trigger_mode(mut self, trigger_mode: bool) -> Self {
self.reg = (self.reg & !LVT_TRIGGER_MODE_MASK) | (trigger_mode as u32) << 15;
self
}
fn with_mask(mut self, mask: bool) -> Self {
self.reg = (self.reg & !LVT_MASK_MASK) | (mask as u32) << 16;
self
}
fn with_periodic_timer(mut self, timer_mode: bool) -> Self {
self.reg = (self.reg & !LVT_TIMER_MODE_MASK) | (timer_mode as u32) << 17;
self
}
}
#[repr(C, align(16))]
struct ApicMmio {
_pad1: [ApicReg; 2],
id: ApicReg,
version: ApicReg,
_pad2: [ApicReg; 4],
task_priority: ApicReg,
arbitration_priority: ApicReg,
processor_priority: ApicReg,
end_of_interrupt: ApicReg,
remote_read: ApicReg,
logical_destination: ApicReg,
destination_format: ApicReg,
spurious_interrupt_vector: ApicReg,
in_service: [ApicReg; 8],
trigger_mode: [ApicReg; 8],
interrupt_request: [ApicReg; 8],
error_status: ApicReg,
_pad3: [ApicReg; 7],
interrupt_command_low: ApicReg,
interrupt_command_high: ApicReg,
timer_local_vector_table: LocalVectorRegister,
thermal_local_vector_table: LocalVectorRegister,
performance_local_vector_table: LocalVectorRegister,
lint0_local_vector_table: LocalVectorRegister,
lint1_local_vector_table: LocalVectorRegister,
error_local_vector_table: LocalVectorRegister,
timer_initial_count: ApicReg,
timer_current_count: ApicReg,
_pad4: [ApicReg; 4],
timer_divide_configuration: ApicReg,
_pad5: ApicReg,
extended_apic_features: ApicReg,
extended_apic_control: ApicReg,
specific_end_of_interrupt: ApicReg,
_pad6: [ApicReg; 5],
interrupt_enable: [ApicReg; 8],
extended_interrupt_local_vector_tables: [ApicReg; 4],
}
#[allow(dead_code)]
mod io_apic {
pub const IO_APIC_ID: u32 = 0;
pub const IO_APIC_VERSION: u32 = 1;
pub const IO_APIC_ARBITRATION_ID: u32 = 2;
pub const IO_APIC_REDIRECTION_TABLE: u32 = 0x10;
pub const RDR_VECTOR_MASK: u64 = 0xFF;
pub const RDR_DELIVERY_MODE_MASK: u64 = 0x7 << 8;
pub const RDR_DESTINATION_MODE_MASK: u64 = 1 << 11;
pub const RDR_DELIVERY_STATUS_MASK: u64 = 1 << 12;
pub const RDR_PIN_POLARITY_MASK: u64 = 1 << 13;
pub const RDR_REMOTE_IRR_MASK: u64 = 1 << 14;
pub const RDR_TRIGGER_MODE_MASK: u64 = 1 << 15;
pub const RDR_MASK_MASK: u64 = 1 << 16;
pub const RDR_DESTINATION_PHYSICAL_MASK: u64 = 0x1F << 59;
pub const RDR_DESTINATION_LOGICAL_MASK: u64 = 0xFF << 56;
}
#[allow(dead_code)]
enum DestinationType {
Physical(u8),
Logical(u8),
}
#[derive(Default, Clone, Copy)]
pub struct IoApicRedirectionBuilder {
reg: u64,
}
#[allow(dead_code)]
impl IoApicRedirectionBuilder {
fn with_vector(mut self, vector: u8) -> Self {
self.reg = (self.reg & !io_apic::RDR_VECTOR_MASK) | vector as u64;
self
}
fn with_delivery_mode(mut self, delivery_mode: u8) -> Self {
self.reg =
(self.reg & !io_apic::RDR_DELIVERY_MODE_MASK) | ((delivery_mode & 0x7) as u64) << 8;
self
}
pub fn with_interrupt_polarity_low(mut self, polarity: bool) -> Self {
self.reg = (self.reg & !io_apic::RDR_PIN_POLARITY_MASK) | (polarity as u64) << 13;
self
}
pub fn with_trigger_mode_level(mut self, trigger_mode: bool) -> Self {
self.reg = (self.reg & !io_apic::RDR_TRIGGER_MODE_MASK) | (trigger_mode as u64) << 15;
self
}
pub fn with_mask(mut self, mask: bool) -> Self {
self.reg = (self.reg & !io_apic::RDR_MASK_MASK) | (mask as u64) << 16;
self
}
fn with_destination(mut self, destination: DestinationType) -> Self {
match destination {
DestinationType::Physical(d) => {
assert!(d < 32, "physical destination is out of range");
self.reg &= !io_apic::RDR_DESTINATION_MODE_MASK;
self.reg = (self.reg & !io_apic::RDR_DESTINATION_PHYSICAL_MASK)
| (((d & 0x1F) as u64) << 59);
}
DestinationType::Logical(d) => {
self.reg |= io_apic::RDR_DESTINATION_MODE_MASK;
self.reg = (self.reg & !io_apic::RDR_DESTINATION_LOGICAL_MASK) | ((d as u64) << 56);
}
}
self
}
}
#[repr(C, align(16))]
struct IoApicMmio {
register_select: ApicReg,
data: ApicReg,
}
impl IoApicMmio {
pub fn read_register(&mut self, register: u32) -> u32 {
self.register_select.write(register);
self.data.read()
}
pub fn write_register(&mut self, register: u32, value: u32) {
self.register_select.write(register);
self.data.write(value);
}
}
#[allow(dead_code)]
struct IoApic {
id: u8,
global_irq_base: u32,
n_entries: u8,
mmio: VirtualSpace<IoApicMmio>,
}
impl IoApic {
fn reset_all_interrupts(&mut self) {
for i in 0..self.n_entries {
let b = IoApicRedirectionBuilder::default()
.with_vector(0)
.with_mask(true);
self.write_redirect_entry(i, b);
}
}
fn read_register(&mut self, register: u32) -> u32 {
self.mmio.read_register(register)
}
fn write_register(&mut self, register: u32, value: u32) {
self.mmio.write_register(register, value)
}
fn write_redirect_entry(&mut self, entry: u8, builder: IoApicRedirectionBuilder) {
let lo = builder.reg as u32;
let hi = (builder.reg >> 32) as u32;
self.write_register(io_apic::IO_APIC_REDIRECTION_TABLE + entry as u32 * 2, lo);
self.write_register(
io_apic::IO_APIC_REDIRECTION_TABLE + entry as u32 * 2 + 1,
hi,
);
}
fn is_entry_taken(&mut self, entry: u8) -> bool {
let lo = self.read_register(io_apic::IO_APIC_REDIRECTION_TABLE + entry as u32 * 2);
lo as u64 & io_apic::RDR_VECTOR_MASK != 0
}
}
impl From<tables::IoApic> for IoApic {
fn from(table: tables::IoApic) -> Self {
assert!(table.io_apic_address != 0, "IO APIC address is 0");
assert_eq!(
table.io_apic_address & 0xF,
0,
"IO APIC address is not aligned"
);
let mut s = Self {
id: table.io_apic_id,
global_irq_base: table.global_system_interrupt_base,
n_entries: 0, mmio: unsafe { VirtualSpace::new(table.io_apic_address as u64).unwrap() },
};
s.n_entries = (s.read_register(io_apic::IO_APIC_VERSION) >> 16) as u8 + 1;
s
}
}
struct Apic {
mmio: VirtualSpace<ApicMmio>,
n_cpus: usize,
io_apics: Vec<IoApic>,
source_overrides: Vec<InterruptSourceOverride>,
}
impl Apic {
fn new(bios_tables: &BiosTables) -> Self {
let cpuid = unsafe { cpu::cpuid::cpuid!(cpu::cpuid::FN_FEAT) };
if cpuid.edx & cpu::cpuid::FEAT_EDX_APIC == 0 {
panic!("APIC is not supported");
}
let apic_bar = unsafe { cpu::msr::read(cpu::msr::APIC_BASE) };
if apic_bar & APIC_BAR_ENABLED == 0 {
unsafe {
cpu::msr::write(cpu::msr::APIC_BASE, apic_bar | APIC_BAR_ENABLED);
}
let apic_bar = unsafe { cpu::msr::read(cpu::msr::APIC_BASE) };
if apic_bar & APIC_BAR_ENABLED == 0 {
panic!("APIC is not enabled");
}
}
let mut apic_address = apic_bar & APIC_BASE_MASK;
let madt_table = bios_tables
.rsdt
.get_table::<tables::Apic>()
.expect("MADT table not found");
if madt_table.local_apic_address as u64 != apic_address {
warn!(
"MADT table has a different APIC address (CPU:{:X}, MADT:{:X}), using MADT...",
apic_address, madt_table.local_apic_address
);
apic_address = madt_table.local_apic_address as u64;
}
let mut n_cpus = 0;
let mut io_apics = Vec::new();
let mut source_overrides = Vec::new();
for strct in &madt_table.interrupt_controller_structs {
match strct {
InterruptControllerStruct::ProcessorLocalApic(s) => {
if s.flags & 1 == 0 {
continue;
}
if n_cpus >= MAX_CPUS {
warn!("too many CPUs, have {MAX_CPUS} already, ignoring the rest");
} else {
unsafe {
CPUS[n_cpus].init(n_cpus, s.apic_id);
}
n_cpus += 1;
}
}
InterruptControllerStruct::IoApic(s) => {
io_apics.push(s.clone().into());
}
InterruptControllerStruct::InterruptSourceOverride(s) => {
source_overrides.push(s.clone());
}
InterruptControllerStruct::NonMaskableInterrupt(_) => todo!(),
InterruptControllerStruct::LocalApicNmi(_s) => {
}
InterruptControllerStruct::LocalApicAddressOverride(s) => {
apic_address = s.local_apic_address;
}
InterruptControllerStruct::Unknown { struct_type, bytes } => {
warn!(
"unknown interrupt controller struct type {:#X} with {:#X?} bytes",
struct_type, bytes
);
}
}
}
assert!(
n_cpus > 0,
"no CPUs found in the MADT table, cannot continue"
);
assert!(
!io_apics.is_empty(),
"no IO APICs found in the MADT table, cannot continue"
);
assert_ne!(apic_address, 0, "APIC address is 0, cannot continue");
assert_eq!(apic_address & 0xF, 0, "APIC address is not aligned");
io_apics.iter_mut().for_each(|io_apic: &mut IoApic| {
io_apic.reset_all_interrupts();
});
let mut s = Self {
mmio: unsafe {
VirtualSpace::new(apic_address).expect("Could not map APIC virtual space")
},
n_cpus,
io_apics,
source_overrides,
};
s.initialize_spurious_interrupt();
s.disable_local_interrupts();
s.initialize_timer();
s.setup_error_interrupt();
s.return_from_interrupt();
s
}
fn return_from_interrupt(&mut self) {
self.mmio.end_of_interrupt.write(0);
}
fn initialize_spurious_interrupt(&mut self) {
let interrupt_num = allocate_basic_user_interrupt(spurious_handler);
self.mmio
.spurious_interrupt_vector
.write(SPURIOUS_ENABLE | interrupt_num as u32);
}
fn disable_local_interrupts(&mut self) {
let vector_table = LocalVectorRegisterBuilder::default().with_mask(true);
self.mmio.lint0_local_vector_table.write(vector_table);
self.mmio.lint1_local_vector_table.write(vector_table);
}
fn initialize_timer(&mut self) {
let interrupt_num = allocate_user_interrupt_all_saved(super::handlers::apic_timer_handler);
self.mmio.timer_divide_configuration.write(0b1011);
self.mmio.timer_initial_count.write(0x1000000);
let vector_table = LocalVectorRegisterBuilder::default()
.with_periodic_timer(true)
.with_mask(false)
.with_vector(interrupt_num);
self.mmio.timer_local_vector_table.write(vector_table);
}
fn setup_error_interrupt(&mut self) {
self.mmio.error_status.write(0);
self.mmio.error_status.write(0);
let interrupt_num = allocate_basic_user_interrupt(error_interrupt_handler);
let vector_table = LocalVectorRegisterBuilder::default()
.with_mask(false)
.with_vector(interrupt_num);
self.mmio.error_local_vector_table.write(vector_table);
}
fn get_irq_ioapic_entry(
&mut self,
irq_num: u8,
) -> Option<(u8, &mut IoApic, Option<&InterruptSourceOverride>)> {
let int_override = self
.source_overrides
.iter()
.find(|int_override| int_override.source == irq_num);
let mut interrupt_num = irq_num as u32;
if let Some(int_override) = int_override {
interrupt_num = int_override.global_system_interrupt;
}
self.io_apics
.iter_mut()
.find(|io_apic| {
io_apic.global_irq_base <= interrupt_num
&& interrupt_num < io_apic.global_irq_base + io_apic.n_entries as u32
})
.map(|io_apic| {
let entry_in_ioapic = interrupt_num - io_apic.global_irq_base;
(entry_in_ioapic as u8, io_apic, int_override)
})
}
fn is_irq_assigned(&mut self, irq_num: u8) -> bool {
let (entry_in_ioapic, io_apic, _) = self
.get_irq_ioapic_entry(irq_num)
.expect("Could not find IO APIC for the interrupt");
io_apic.is_entry_taken(entry_in_ioapic)
}
fn assign_io_irq<H: InterruptHandler>(&mut self, handler: H, irq_num: u8, cpu: &Cpu) {
self.assign_io_irq_custom(handler, irq_num, cpu, |b| b)
}
fn assign_io_irq_custom<H: InterruptHandler, F>(
&mut self,
handler: H,
irq_num: u8,
cpu: &Cpu,
modify_entry: F,
) where
F: FnOnce(IoApicRedirectionBuilder) -> IoApicRedirectionBuilder,
{
assert!(cpu.id < self.n_cpus, "CPU ID is out of range");
assert!(irq_num < 24, "interrupt number is out of range");
let (entry_in_ioapic, io_apic, int_override) = self
.get_irq_ioapic_entry(irq_num)
.expect("Could not find IO APIC for the interrupt");
assert!(
!io_apic.is_entry_taken(entry_in_ioapic),
"entry is already taken"
);
let trigger_mode_level = if let Some(int_override) = int_override {
match (int_override.flags >> 2) & 0b11 {
0b00 => {
false
}
0b01 => {
false
}
0b10 => {
panic!("Reserved value 0b10 used for trigger mode in interrupt override");
}
0b11 => {
true
}
_ => unreachable!(),
}
} else {
false
};
let polarity_low = if let Some(int_override) = int_override {
match int_override.flags & 0b11 {
0b00 => {
true
}
0b01 => {
false
}
0b10 => {
panic!("Reserved value 0b10 used for polarity in interrupt override");
}
0b11 => {
true
}
_ => unreachable!(),
}
} else {
false
};
let vector_num = allocate_user_interrupt(handler);
let b = IoApicRedirectionBuilder::default()
.with_vector(vector_num)
.with_delivery_mode(0) .with_interrupt_polarity_low(polarity_low) .with_trigger_mode_level(trigger_mode_level) .with_mask(false) .with_destination(DestinationType::Physical(cpu.apic_id));
let b = modify_entry(b);
io_apic.write_redirect_entry(entry_in_ioapic, b);
}
}
extern "x86-interrupt" fn spurious_handler(_frame: InterruptStackFrame64) {
warn!("Spurious interrupt");
return_from_interrupt();
}
extern "x86-interrupt" fn error_interrupt_handler(_frame: InterruptStackFrame64) {
let error_status = APIC.get().lock().mmio.error_status.read();
error!("APIC error: {:#X}", error_status);
APIC.get().lock().mmio.error_status.write(0);
return_from_interrupt();
}