/*++ Copyright (2) 2002 Microsoft Corporation Module Name: gart.c Abstract: This module contains the Graphics Aperture Redirection Table support routines for the MS AGP v3 Filter Driver Author: Eric F. Nelson (enelson) June 6, 2002 Revision History: --*/ #include "agp.h" #include "uagp35.h" NTSTATUS Uagp35CreateGart( IN PUAGP35_EXTENSION AgpContext, IN ULONG MinimumPages ); VOID Uagp35EnableGTLB( IN PUAGP35_EXTENSION AgpContext, IN BOOLEAN Enable ); NTSTATUS Uagp35SetRate( IN PUAGP35_EXTENSION AgpContext, IN ULONG AgpRate ); PGART_PTE32 Uagp35FindRangeInGart32( IN PVOID StartPte, IN PVOID EndPte, IN ULONG Length, IN BOOLEAN SearchBackward, IN ULONG SearchState ); #ifdef ALLOC_PRAGMA #pragma alloc_text(PAGE, AgpDisableAperture) #pragma alloc_text(PAGE, AgpQueryAperture) #pragma alloc_text(PAGE, AgpReserveMemory) #pragma alloc_text(PAGE, AgpReleaseMemory) #pragma alloc_text(PAGE, Uagp35CreateGart) #pragma alloc_text(PAGE, AgpMapMemory) #pragma alloc_text(PAGE, AgpUnMapMemory) #pragma alloc_text(PAGE, Uagp35FindRangeInGart32) #pragma alloc_text(PAGE, AgpFindFreeRun) #pragma alloc_text(PAGE, AgpGetMappedPages) #endif #define Uagp35EnableTB(_x_) Uagp35EnableGTLB((_x_), TRUE) #define Uagp35DisableTB(_x_) Uagp35EnableGTLB((_x_), FALSE) NTSTATUS AgpQueryAperture( IN PUAGP35_EXTENSION AgpContext, OUT PHYSICAL_ADDRESS *CurrentBase, OUT ULONG *CurrentSizeInPages, OUT OPTIONAL PIO_RESOURCE_LIST *pApertureRequirements ) /*++ Routine Description: Queries the current size of the GART aperture, and optionally returns the possible GART settings Arguments: AgpContext - Supplies our AGP3 context CurrentBase - Returns the current physical address of the GART CurrentSizeInPages - Returns the current GART size ApertureRequirements - If present, returns the possible GART settings Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { PHYSICAL_ADDRESS ApertureBase; PIO_RESOURCE_LIST Requirements; ULONG Index; ULONG Length; USHORT ApSize; ULONG AltCount; PAGED_CODE(); // // Get the current aperture base and size settings // if (!AgpContext->FourGBEnable) { AgpLibReadAgpTargetConfig(AgpContext, &ApertureBase.LowPart, APERTURE_BASE, sizeof(ApertureBase.LowPart)); } else { AgpLibReadAgpTargetConfig(AgpContext, &ApertureBase, APERTURE_BASE, sizeof(ApertureBase)); } AgpLibGetExtendedTargetCapability(AgpContext, ApertureSize, &ApSize); ASSERT(ApertureBase.QuadPart != 0); CurrentBase->QuadPart = ApertureBase.QuadPart & PCI_ADDRESS_MEMORY_ADDRESS_MASK; // // Convert APSIZE into the actual size of the aperture // switch (ApSize) { case AP_SIZE_4MB: AltCount = 1; *CurrentSizeInPages = 4 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_8MB: AltCount = 2; *CurrentSizeInPages = 8 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_16MB: AltCount = 3; *CurrentSizeInPages = 16 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_32MB: AltCount = 4; *CurrentSizeInPages = 32 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_64MB: AltCount = 5; *CurrentSizeInPages = 64 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_128MB: AltCount = 6; *CurrentSizeInPages = 128 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_256MB: AltCount = 7; *CurrentSizeInPages = 256 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_512MB: AltCount = 8; *CurrentSizeInPages = 512 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_1024MB: AltCount = 9; *CurrentSizeInPages = 1024 * (1024 * 1024 / PAGE_SIZE); break; case AP_SIZE_2048MB: AltCount = 10; *CurrentSizeInPages = 2048 * (1024 * 1024 / PAGE_SIZE); break; // // IO Resource descriptor's size is only 32-bits! // // case AP_SIZE_4096MB: // AltCount = 11; // *CurrentSizeInPages = 4096 * (1024 * 1024 / PAGE_SIZE); // break; default: AGPLOG(AGP_CRITICAL, ("AgpQueryAperture - Unexpected value %x for ApSize!\n", ApSize)); ASSERT(FALSE); AgpContext->ApertureStart.QuadPart = 0; AgpContext->ApertureLength = 0; return STATUS_UNSUCCESSFUL; } // // Remember the current aperture settings // AgpContext->ApertureStart.QuadPart = CurrentBase->QuadPart; AgpContext->ApertureLength = *CurrentSizeInPages * PAGE_SIZE; if (pApertureRequirements != NULL) { // // Since the BIOS probably describes the desired size, we // will only attempt alternates of the same size, or less // Requirements = ExAllocatePoolWithTag(PagedPool, sizeof(IO_RESOURCE_LIST) + (AltCount - 1) * sizeof(IO_RESOURCE_DESCRIPTOR), 'RpgA'); if (Requirements == NULL) { return STATUS_INSUFFICIENT_RESOURCES; } Requirements->Version = Requirements->Revision = 1; Requirements->Count = AltCount; Length = AgpContext->ApertureLength; for (Index = 0; Index < Requirements->Count; Index++) { Requirements->Descriptors[Index].Option = IO_RESOURCE_ALTERNATIVE; Requirements->Descriptors[Index].Type = CmResourceTypeMemory; Requirements->Descriptors[Index].ShareDisposition = CmResourceShareDeviceExclusive; Requirements->Descriptors[Index].Flags = CM_RESOURCE_MEMORY_READ_WRITE | CM_RESOURCE_MEMORY_PREFETCHABLE; Requirements->Descriptors[Index].u.Memory.Length = Length; Requirements->Descriptors[Index].u.Memory.Alignment = Length; Requirements->Descriptors[Index].u.Memory.MinimumAddress.QuadPart = 0; Requirements->Descriptors[Index].u.Memory.MaximumAddress.QuadPart = (ULONG)-1; Length /= 2; } *pApertureRequirements = Requirements; } return STATUS_SUCCESS; } NTSTATUS AgpSetAperture( IN PUAGP35_EXTENSION AgpContext, IN PHYSICAL_ADDRESS NewBase, IN ULONG NewSizeInPages ) /*++ Routine Description: Sets the GART aperture to the supplied settings Arguments: AgpContext - Supplies our AGP3 context NewBase - Supplies the new physical memory base for the GART NewSizeInPages - Supplies the new size for the GART Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { PCI_AGP_CONTROL AgpCtrl; USHORT ApSize; PHYSICAL_ADDRESS ApBase; // // Figure out the new APSIZE setting, make sure it is valid. // switch (NewSizeInPages) { case 4 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_4MB; break; case 8 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_8MB; break; case 16 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_16MB; break; case 32 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_32MB; break; case 64 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_64MB; break; case 128 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_128MB; break; case 256 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_256MB; break; case 512 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_512MB; break; case 1024 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_1024MB; break; case 2048 * (1024 * 1024 / PAGE_SIZE): ApSize = AP_SIZE_2048MB; break; // // IO Resource descriptor's size is only 32-bits! // // case AP_SIZE_4096MB: // AltCount = 11; // *CurrentSizeInPages = 4096 * (1024 * 1024 / PAGE_SIZE); // break; // case 4096 * (1024 * 1024 / PAGE_SIZE): // ApSize = AP_SIZE_4096MB; // break; default: AGPLOG(AGP_CRITICAL, ("AgpSetAperture - invalid GART size of %lx pages " "specified, aperture at %I64X.\n", NewSizeInPages, NewBase.QuadPart)); ASSERT(FALSE); return STATUS_INVALID_PARAMETER; } // // Make sure the supplied size is aligned on the appropriate boundary // ASSERT((NewBase.QuadPart & ((NewSizeInPages * PAGE_SIZE) - 1)) == 0); if ((NewBase.QuadPart & ((NewSizeInPages * PAGE_SIZE) - 1)) != 0 ) { AGPLOG(AGP_CRITICAL, ("AgpSetAperture - invalid base %I64X specified for GART " "aperture of %lx pages\n", NewBase.QuadPart, NewSizeInPages)); return STATUS_INVALID_PARAMETER; } // // Reprogram Special Target settings when the chip // is powered off, but ignore rate changes as those were already // applied during MasterInit // if (AgpContext->SpecialTarget & ~AGP_FLAG_SPECIAL_RESERVE) { AgpSpecialTarget(AgpContext, AgpContext->SpecialTarget & ~AGP_FLAG_SPECIAL_RESERVE); } // // Need to reset the hardware to match the supplied settings // // If the aperture is enabled, disable it, write the new settings, // then reenable the aperture // // // Disable the aperture // AgpLibGetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); if (AgpCtrl.AP_Enable == ON) { AgpCtrl.AP_Enable = OFF; if (!(AgpContext->SpecialTarget & AGP_FLAG_SPECIAL_NO_APERTURE_DISABLE)) { AgpLibSetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); } else { AGPLOG(AGP_WARNING, ("UAGP35MasterInit: AGP_FLAG_SPECIAL_NO_APERTURE_DISABLE\n") ); } } // // Write APSIZE first, as this will enable the correct bits in APBASE // that need to be written next // AgpLibSetExtendedTargetCapability(AgpContext, ApertureSize, &ApSize); // // Now we can update APBASE // ApBase.QuadPart = NewBase.LowPart & PCI_ADDRESS_MEMORY_ADDRESS_MASK; if (AgpContext->CapabilityId != PCI_CAPABILITY_ID_AGP_TARGET) { if (!AgpContext->FourGBEnable) { ASSERT(ApBase.HighPart == 0); AgpLibWriteAgpTargetConfig(AgpContext, &ApBase.LowPart, APERTURE_BASE, sizeof(ApBase.LowPart)); } else { AgpLibWriteAgpTargetConfig(AgpContext, &ApBase, APERTURE_BASE, sizeof(ApBase)); } } #if DBG // // Read back what we wrote, make sure it worked // { PHYSICAL_ADDRESS DbgBase; USHORT DbgSize; ULONGLONG ApBaseMask; ApBaseMask = AP_BASE_MASK(ApSize); AgpLibGetExtendedTargetCapability(AgpContext, ApertureSize, &DbgSize); if (!AgpContext->FourGBEnable) { DbgBase.HighPart = 0; AgpLibReadAgpTargetConfig(AgpContext, &DbgBase.LowPart, APERTURE_BASE, sizeof(DbgBase.LowPart)); } else { AgpLibReadAgpTargetConfig(AgpContext, &DbgBase, APERTURE_BASE, sizeof(DbgBase)); } ASSERT(DbgSize == ApSize); // // If this fires, it means our BAR has moved, might be good to track // this during AGP3 debug // ASSERT((DbgBase.QuadPart & ApBaseMask) == ApBase.QuadPart); } #endif // // Now enable the aperture if it was enabled before // if (AgpContext->GlobalEnable) { AgpLibGetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); ASSERT((AgpCtrl.AP_Enable == OFF) || (AgpContext->SpecialTarget & AGP_FLAG_SPECIAL_NO_APERTURE_DISABLE)); AgpCtrl.AP_Enable = ON; AgpLibSetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); } // // Update our extension to reflect the new GART setting // AgpContext->ApertureStart = NewBase; AgpContext->ApertureLength = NewSizeInPages * PAGE_SIZE; // // Enable the TB in case we are resuming from S3 or S4 // Uagp35EnableTB(AgpContext); // // If the GART has been allocated, rewrite the ATTBASE // if (AgpContext->Gart != NULL) { AgpLibSetExtendedTargetCapability(AgpContext, GartLow, &AgpContext->GartPhysical.LowPart); #ifdef _WIN64 AgpLibSetExtendedTargetCapability(AgpContext, GartHigh, &AgpContext->GartPhysical.HighPart); #endif } return STATUS_SUCCESS; } VOID AgpDisableAperture( IN PUAGP35_EXTENSION AgpContext ) /*++ Routine Description: Disables the GART aperture so that this resource is available for other devices Arguments: AgpContext - Supplies our AGP3 context Return Value: None --*/ { PCI_AGP_CONTROL AgpCtrl; // // Disable the aperture // AgpLibGetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); if (AgpCtrl.AP_Enable == ON) { AgpCtrl.AP_Enable = OFF; AgpLibSetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); } AgpContext->GlobalEnable = FALSE; // // Nuke the Gart! (It's meaningless now...) // if (AgpContext->Gart != NULL) { MmFreeContiguousMemory(AgpContext->Gart); AgpContext->Gart = NULL; AgpContext->GartLength = 0; } } NTSTATUS AgpReserveMemory( IN PUAGP35_EXTENSION AgpContext, IN OUT AGP_RANGE *Range ) /*++ Routine Description: Reserves a range of memory in the GART Arguments: AgpContext - Supplies our AGP3 Context Range - Supplies the AGP_RANGE structure (AGPLIB will have filled in NumberOfPages and Type, and this routine will fill in MemoryBase and Context) Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { ULONG Index; ULONG NewState; NTSTATUS Status; PGART_PTE32 FoundRange; BOOLEAN Backwards; PAGED_CODE(); ASSERT((Range->Type == MmNonCached) || (Range->Type == MmWriteCombined)); if (Range->NumberOfPages > (AgpContext->ApertureLength / PAGE_SIZE)) { return STATUS_INSUFFICIENT_RESOURCES; } // // If we have not allocated our GART yet, now is the time to do so // if (AgpContext->Gart == NULL) { ASSERT(AgpContext->GartLength == 0); Status = Uagp35CreateGart(AgpContext, Range->NumberOfPages); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_CRITICAL, ("UAGP35CreateGart failed %08lx to create GART of size " "%lx\n", Status, AgpContext->ApertureLength)); return Status; } } ASSERT(AgpContext->GartLength != 0); // // Now that we have a GART, try and find enough contiguous entries to // satisfy the request, requests for uncached memory will scan from high // addresses to low addresses, requests for write-combined memory will // scan from low addresses to high addresses, we will use a first-fit // algorithm to try and keep the allocations // packed and contiguous // Backwards = (Range->Type == MmNonCached) ? TRUE : FALSE; FoundRange = Uagp35FindRangeInGart32(&AgpContext->Gart[0], &AgpContext->Gart[(AgpContext->GartLength / sizeof(GART_PTE32)) - 1], Range->NumberOfPages, Backwards, GART_ENTRY_FREE); if (FoundRange == NULL) { // // A big enough chunk was not found. // AGPLOG(AGP_CRITICAL, ("AgpReserveMemory: Could not find %d contiguous free pages " "of type %d in GART at %08lx\n", Range->NumberOfPages, Range->Type, AgpContext->Gart)); // // This is where we could try and grow the GART // return STATUS_INSUFFICIENT_RESOURCES; } AGPLOG(AGP_NOISE, ("AgpReserveMemory: reserved %d pages at GART PTE %08lx\n", Range->NumberOfPages, FoundRange)); // // Set these pages to reserved // switch (Range->Type) { case MmNonCached: NewState = GART_ENTRY_RESERVED_UC; break; case MmWriteCombined: NewState = GART_ENTRY_RESERVED_WC; break; default: // // Unknown cache type, default to HW coherent cached // AGPLOG(AGP_NOISE, ("AgpReserveMemory: Unknown cache type %d\n", Range->Type)); case MmHardwareCoherentCached: NewState = GART_ENTRY_RESERVED_CC; } for (Index = 0; Index < Range->NumberOfPages; Index++) { ASSERT(FoundRange[Index].Soft.State == GART_ENTRY_FREE); FoundRange[Index].AsULONG = 0; FoundRange[Index].Soft.State = NewState; } Range->MemoryBase.QuadPart = AgpContext->ApertureStart.QuadPart + (FoundRange - &AgpContext->Gart[0]) * PAGE_SIZE; Range->Context = FoundRange; ASSERT(Range->MemoryBase.HighPart == 0); AGPLOG(AGP_NOISE, ("AgpReserveMemory: reserved memory handle %lx at PA %08lx\n", FoundRange, Range->MemoryBase.LowPart)); return STATUS_SUCCESS; } NTSTATUS AgpReleaseMemory( IN PUAGP35_EXTENSION AgpContext, IN PAGP_RANGE Range ) /*++ Routine Description: Releases memory previously reserved with AgpReserveMemory Arguments: AgpContext - Supplies our AGP3 context AgpRange - Supplies the range to be released Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { PGART_PTE32 Pte; ULONG Start; PAGED_CODE(); // // Go through and free all the PTEs, none of these should still // be valid at this point // for (Pte = Range->Context; Pte < ((PGART_PTE32)Range->Context + Range->NumberOfPages); Pte++) { switch (Range->Type) { case MmNonCached: ASSERT(Pte->Soft.State == GART_ENTRY_RESERVED_UC); break; case MmWriteCombined: ASSERT(Pte->Soft.State == GART_ENTRY_RESERVED_WC); break; default: // // Unknown cache type, default to HW coherent cached // AGPLOG(AGP_NOISE, ("AgpReleaseMemory: Unknown cache type %d\n", Range->Type)); case MmHardwareCoherentCached: ASSERT(Pte->Soft.State == GART_ENTRY_RESERVED_CC); } Pte->Soft.State = GART_ENTRY_FREE; } Range->MemoryBase.QuadPart = 0; return STATUS_SUCCESS; } NTSTATUS Uagp35CreateGart( IN PUAGP35_EXTENSION AgpContext, IN ULONG MinimumPages ) /*++ Routine Description: Allocates and initializes an empty GART, the current implementation attempts to allocate the entire GART on the first reserve Arguments: AgpContext - Supplies our AGP3 context MinimumPages - Supplies the minimum size (in pages) of the GART to be created Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { PGART_PTE32 Gart; ULONG GartLength; PHYSICAL_ADDRESS HighestAcceptable; PHYSICAL_ADDRESS LowestAcceptable; PHYSICAL_ADDRESS BoundaryMultiple; PHYSICAL_ADDRESS GartPhysical; ULONG Index; PAGED_CODE(); // // Try and get a chunk of contiguous memory big enough to map the // entire aperture // LowestAcceptable.QuadPart = 0; BoundaryMultiple.QuadPart = 0; #ifdef _WIN64 HighestAcceptable.QuadPart = (ULONGLONG)-1; #else HighestAcceptable.QuadPart = (ULONG)-1; #endif GartLength = BYTES_TO_PAGES(AgpContext->ApertureLength) * sizeof(GART_PTE32); Gart = MmAllocateContiguousMemorySpecifyCache(GartLength, LowestAcceptable, HighestAcceptable, BoundaryMultiple, MmNonCached); if (Gart == NULL) { AGPLOG(AGP_CRITICAL, ("UAGP35CreateGart: MmAllocateContiguousMemorySpecifyCache %lx " "failed\n", GartLength)); return STATUS_INSUFFICIENT_RESOURCES; } // // We successfully allocated a contiguous chunk of memory. // It should be page aligned already. // ASSERT(((ULONG_PTR)Gart & (PAGE_SIZE - 1)) == 0); // // Get the physical address // GartPhysical = MmGetPhysicalAddress(Gart); AGPLOG(AGP_NOISE, ("UAGP35CreateGart: GART of length %lx created at VA %08lx, " "PA %08lx\n", GartLength, Gart, GartPhysical.LowPart)); ASSERT(GartPhysical.HighPart == 0); ASSERT((GartPhysical.LowPart & (PAGE_SIZE - 1)) == 0); // // Initialize all the PTEs to free // for (Index = 0; Index < (GartLength / sizeof(GART_PTE32)); Index++) { Gart[Index].Soft.State = GART_ENTRY_FREE; } AgpLibSetExtendedTargetCapability(AgpContext, GartLow, &GartPhysical.LowPart); #ifdef _WIN64 AgpLibSetExtendedTargetCapability(AgpContext, GartHigh, &GartPhysical.HighPart); #endif // // Update our extension to reflect the current state. // AgpContext->Gart = Gart; AgpContext->GartLength = GartLength; AgpContext->GartPhysical = GartPhysical; return STATUS_SUCCESS; } NTSTATUS AgpMapMemory( IN PUAGP35_EXTENSION AgpContext, IN PAGP_RANGE Range, IN PMDL Mdl, IN ULONG OffsetInPages, OUT PHYSICAL_ADDRESS *MemoryBase ) /*++ Routine Description: Maps physical memory into the GART somewhere in the specified range Arguments: AgpContext - Supplies our AGP3 context Range - Supplies the AGP range that the memory should be mapped into Mdl - Supplies the MDL describing the physical pages to be mapped OffsetInPages - Supplies the offset into the reserved range where the mapping should begin MemoryBase - Returns the physical memory in the aperture where the pages were mapped Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { ULONG PageCount; PGART_PTE32 Pte; PGART_PTE32 StartPte; ULONG Index; ULONG TargetState; PPFN_NUMBER Page; BOOLEAN Backwards; GART_PTE32 NewPte; PCI_AGP_CONTROL AgpCtrl; PAGED_CODE(); ASSERT(Mdl->Next == NULL); NewPte.AsULONG = 0; StartPte = Range->Context; PageCount = BYTES_TO_PAGES(Mdl->ByteCount); ASSERT(PageCount <= Range->NumberOfPages); ASSERT(OffsetInPages <= Range->NumberOfPages); ASSERT(PageCount + OffsetInPages <= Range->NumberOfPages); ASSERT(PageCount > 0); switch (Range->Type) { case MmNonCached: TargetState = GART_ENTRY_RESERVED_UC; NewPte.Soft.State = GART_ENTRY_VALID_UC; break; case MmWriteCombined: TargetState = GART_ENTRY_RESERVED_WC; NewPte.Soft.State = GART_ENTRY_VALID_WC; break; default: // // Unknown cache type, default to HW coherent cached // AGPLOG(AGP_NOISE, ("AgpMapMemory: Unknown cache type %d\n", Range->Type)); case MmHardwareCoherentCached: TargetState = GART_ENTRY_RESERVED_CC; NewPte.Soft.State = GART_ENTRY_VALID_CC; } Pte = StartPte + OffsetInPages; // // We have a suitable range, now fill it in with the supplied MDL // ASSERT(Pte >= StartPte); ASSERT((Pte + PageCount) <= (StartPte + Range->NumberOfPages)); Page = (PPFN_NUMBER)(Mdl + 1); // // Disable the TB as per the (440?) spec, this is probably unnecessary // as there should be no valid entries in this range, and there should // be no invalid entries still in the TB, so flushing the TB seems // a little gratuitous but that's what the old spec said to do // Uagp35DisableTB(AgpContext); for (Index = 0; Index < PageCount; Index++) { ASSERT(Pte[Index].Soft.State == TargetState); #ifndef _WIN64 NewPte.Hard.PageLow = *Page++; #else NewPte.Hard.PageLow = (ULONG)*Page; NewPte.Hard.PageHigh = (ULONG)(*Page++ >> PAGE_HIGH_SHIFT); #endif Pte[Index].AsULONG = NewPte.AsULONG; ASSERT(Pte[Index].Hard.Valid == ON); } // // We have filled in all the PTEs, read back the last one we wrote // in order to flush the write buffers // NewPte.AsULONG = *(volatile ULONG *)&Pte[PageCount - 1].AsULONG; // // Re-enable the TB // Uagp35EnableTB(AgpContext); // // If we have not yet gotten around to enabling the GART aperture, // do it now // if (!AgpContext->GlobalEnable) { AGPLOG(AGP_NOISE, ("AgpMapMemory: Enabling global aperture access\n")); AgpLibGetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); AgpCtrl.AP_Enable = ON; AgpLibSetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); AgpContext->GlobalEnable = TRUE; } MemoryBase->QuadPart = Range->MemoryBase.QuadPart + ((Pte - StartPte) * PAGE_SIZE); return STATUS_SUCCESS; } NTSTATUS AgpUnMapMemory( IN PUAGP35_EXTENSION AgpContext, IN PAGP_RANGE AgpRange, IN ULONG NumberOfPages, IN ULONG OffsetInPages ) /*++ Routine Description: Unmaps previously mapped memory in the GART Arguments: AgpContext - Supplies our AGP3 context AgpRange - Supplies the AGP range that the memory should be freed from NumberOfPages - Supplies the number of pages in the range to be freed OffsetInPages - Supplies the offset into the range where the freeing should begin Return Value: STATUS_SUCCESS, or an appropriate error status --*/ { ULONG Index; PGART_PTE32 Pte; PGART_PTE32 LastChanged = NULL; PGART_PTE32 StartPte; ULONG NewState; PAGED_CODE(); ASSERT(OffsetInPages + NumberOfPages <= AgpRange->NumberOfPages); StartPte = AgpRange->Context; Pte = &StartPte[OffsetInPages]; if (AgpRange->Type == MmNonCached) { NewState = GART_ENTRY_RESERVED_UC; } else if (AgpRange->Type == MmWriteCombined) { NewState = GART_ENTRY_RESERVED_WC; } else { NewState = GART_ENTRY_RESERVED_CC; } // // Disable the TB to flush it // Uagp35DisableTB(AgpContext); for (Index = 0; Index < NumberOfPages; Index++) { if (Pte[Index].Hard.Valid) { Pte[Index].Soft.State = NewState; LastChanged = &Pte[Index]; } else { // // This page is not mapped, just skip it // AGPLOG(AGP_NOISE, ("AgpUnMapMemory: PTE %08lx (%08lx) at offset %d not " "mapped\n", &Pte[Index], Pte[Index].AsULONG, Index)); ASSERT(Pte[Index].Soft.State == NewState); } } // // We have invalidated all the PTEs, read back the last one we wrote // in order to flush the write buffers // if (LastChanged != NULL) { ULONG FlushWB; FlushWB = *(volatile ULONG *)&LastChanged->AsULONG; } // // Reenable the TB // Uagp35EnableTB(AgpContext); return STATUS_SUCCESS; } PGART_PTE32 Uagp35FindRangeInGart32( IN PGART_PTE32 StartPte, IN PGART_PTE32 EndPte, IN ULONG Length, IN BOOLEAN SearchBackward, IN ULONG SearchState ) /*++ Routine Description: Finds a contiguous range in a 32-bit PTE format GART, this routine can search either from the beginning of the GART forwards or the end of the GART backwards Arguments: StartIndex - Supplies the first GART PTE to search EndPte - Supplies the last GART PTE to search (inclusive) Length - Supplies the number of contiguous free entries to search for SearchBackward - TRUE indicates that the search should begin at EndPte and search backwards, FALSE indicates that the search should begin at StartPte and search forwards SearchState - Supplies the PTE state to look for Return Value: Pointer to the first PTE in the GART if a suitable range is found, and NULL if no suitable range exists --*/ { PGART_PTE32 Current; PGART_PTE32 Last; LONG Delta; ULONG Found; PGART_PTE32 Candidate; PAGED_CODE(); ASSERT(EndPte >= StartPte); ASSERT(Length <= (ULONG)(EndPte - StartPte + 1)); ASSERT(Length != 0); if (SearchBackward) { Current = EndPte; Last = StartPte - 1; Delta = -1; } else { Current = StartPte; Last = EndPte + 1; Delta = 1; } Found = 0; while (Current != Last) { if (Current->Soft.State == SearchState) { if (++Found == Length) { // // A suitable range was found, return it // if (SearchBackward) { return Current; } else { return(Current - Length + 1); } } } else { Found = 0; } Current += Delta; } // // A suitable range was not found // return NULL; } VOID Uagp35EnableGTLB( IN PUAGP35_EXTENSION AgpContext, IN BOOLEAN Enable ) /*++ Routine Description: Enables or disables the GTLB by setting or clearing the GTLB_Enable bit in the AGPCTRL register Arguments: AgpContext - Supplies our AGP3 context Enable - TRUE, GTLB_Enable is set to ON FALSE, GTLB_Enable is set to OFF Return Value: None --*/ { PCI_AGP_CONTROL AgpCtrl; AgpLibGetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); AgpCtrl.GTLB_Enable = (Enable) ? ON: OFF; AgpLibSetExtendedTargetCapability(AgpContext, AgpControl, &AgpCtrl); } VOID AgpFindFreeRun( IN PVOID AgpContext, IN PAGP_RANGE AgpRange, IN ULONG NumberOfPages, IN ULONG OffsetInPages, OUT ULONG *FreePages, OUT ULONG *FreeOffset ) /*++ Routine Description: Finds the first contiguous run of free pages in the specified part of the reserved range Arguments: AgpContext - Supplies our AGP3 context AgpRange - Supplies the AGP range NumberOfPages - Supplies the size of the region to be searched for free pages OffsetInPages - Supplies the start of the region to be searched for free pages FreePages - Returns the length of the first contiguous run of free pages FreeOffset - Returns the start of the first contiguous run of free pages Return Value: None --*/ { PGART_PTE32 Pte; ULONG Index; Pte = (PGART_PTE32)(AgpRange->Context) + OffsetInPages; // // Find the first free PTE // for (Index = 0; Index < NumberOfPages; Index++) { if (Pte[Index].Hard.Valid == OFF) { // // Found a free PTE, count the contiguous ones. // *FreeOffset = Index + OffsetInPages; *FreePages = 0; while ((Index < NumberOfPages) && (Pte[Index].Hard.Valid == OFF)) { *FreePages += 1; ++Index; } return; } } // // No free PTEs in the specified range // *FreePages = 0; } VOID AgpGetMappedPages( IN PVOID AgpContext, IN PAGP_RANGE AgpRange, IN ULONG NumberOfPages, IN ULONG OffsetInPages, OUT PMDL Mdl ) /*++ Routine Description: Returns the list of physical pages mapped into the specified range in the GART Arguments: AgpContext - Supplies our AGP3 context AgpRange - Supplies the AGP range NumberOfPages - Supplies the number of pages to be returned OffsetInPages - Supplies the start of the region Mdl - Returns the list of physical pages mapped in the specified range Return Value: None --*/ { PGART_PTE32 Pte; ULONG Index; PULONG Pages; ASSERT((NumberOfPages * PAGE_SIZE) == Mdl->ByteCount); Pages = (PULONG)(Mdl + 1); Pte = (PGART_PTE32)(AgpRange->Context) + OffsetInPages; for (Index = 0; Index < NumberOfPages; Index++) { ASSERT(Pte[Index].Hard.Valid == ON); Pages[Index] = Pte[Index].Hard.PageLow; } } NTSTATUS AgpSpecialTarget( IN PUAGP35_EXTENSION AgpContext, IN ULONGLONG DeviceFlags ) /*++ Routine Description: This routine makes "special" tweaks to the AGP chipset Arguments: AgpContext - Supplies the AGP context DeviceFlags - Flags indicating what tweaks to perform Return Value: STATUS_SUCCESS, or error --*/ { NTSTATUS Status; Status = STATUS_SUCCESS; // // Should we change the AGP rate? // if (DeviceFlags & AGP_FLAG_SPECIAL_RESERVE) { Status = Uagp35SetRate(AgpContext, (ULONG)((DeviceFlags & AGP_FLAG_SPECIAL_RESERVE) >> AGP_FLAG_SET_RATE_SHIFT)); if (!NT_SUCCESS(Status)) { return Status; } } // // Add more tweaks here... // AgpContext->SpecialTarget |= DeviceFlags; return Status; } NTSTATUS Uagp35SetRate( IN PUAGP35_EXTENSION AgpContext, IN ULONG AgpRate ) /*++ Routine Description: This routine sets the AGP rate Arguments: AgpContext - Supplies our AGP3 context AgpRate - Rate to set Return Value: STATUS_SUCCESS, or error status --*/ { NTSTATUS Status; ULONG TargetEnable; ULONG MasterEnable; PCI_AGP_CAPABILITY TargetCap; PCI_AGP_CAPABILITY MasterCap; BOOLEAN ReverseInit; // // Read capabilities // Status = AgpLibGetTargetCapability(AgpContext, &TargetCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibGetTargetCapability " "failed %08lx\n", Status)); return Status; } Status = AgpLibGetMasterCapability(AgpContext, &MasterCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibGetMasterCapability " "failed %08lx\n", Status)); return Status; } // // Map AGP3 mode rates (4X/8X) into AGP2 rate bits, checking one should // be good enough, so we'll just ASSERT for busted video cards // if (TargetCap.AGPStatus.Agp3Mode == ON) { ASSERT(MasterCap.AGPStatus.Agp3Mode == ON); if ((AgpRate != PCI_AGP_RATE_4X) && (AgpRate != 8)) { return STATUS_INVALID_PARAMETER; } AgpRate >>= 2; } // // Verify the requested rate is supported by both master and target // if (!(AgpRate & TargetCap.AGPStatus.Rate & MasterCap.AGPStatus.Rate)) { return STATUS_INVALID_PARAMETER; } // // Disable AGP while the pull the rug out from underneath // TargetEnable = TargetCap.AGPCommand.AGPEnable; TargetCap.AGPCommand.AGPEnable = 0; Status = AgpLibSetTargetCapability(AgpContext, &TargetCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibSetTargetCapability %08lx for " "Target failed %08lx\n", &TargetCap, Status)); return Status; } MasterEnable = MasterCap.AGPCommand.AGPEnable; MasterCap.AGPCommand.AGPEnable = 0; Status = AgpLibSetMasterCapability(AgpContext, &MasterCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibSetMasterCapability %08lx failed " "%08lx\n", &MasterCap, Status)); return Status; } MasterCap.AGPCommand.Rate = AgpRate; TargetCap.AGPCommand.Rate = AgpRate; MasterCap.AGPCommand.AGPEnable = MasterEnable; TargetCap.AGPCommand.AGPEnable = TargetEnable; // // Patch rate for early rev VIA 8X silicon errata // if ((AgpContext->SpecialTarget & AGP_FLAG_SPECIAL_VIA_AGP2_RATE_PATCH) && (TargetCap.AGPStatus.Agp3Mode == OFF)) { switch (AgpRate) { case PCI_AGP_RATE_1X: case PCI_AGP_RATE_2X: MasterCap.AGPCommand.Rate = PCI_AGP_RATE_1X; TargetCap.AGPCommand.Rate = PCI_AGP_RATE_4X; break; case PCI_AGP_RATE_4X: MasterCap.AGPCommand.Rate = PCI_AGP_RATE_4X; TargetCap.AGPCommand.Rate = PCI_AGP_RATE_1X; } AGPLOG(AGP_WARNING, ("UAGP35SetRate: AGP_FLAG_SPECIAL_VIA_AGP2_RATE_PATCH\n")); } // // Fire up AGP with new rate // ReverseInit = (AgpContext->SpecialTarget & AGP_FLAG_REVERSE_INITIALIZATION) == AGP_FLAG_REVERSE_INITIALIZATION; if (ReverseInit) { Status = AgpLibSetMasterCapability(AgpContext, &MasterCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibSetMasterCapability %08lx failed " "%08lx\n", &MasterCap, Status)); return Status; } } Status = AgpLibSetTargetCapability(AgpContext, &TargetCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibSetTargetCapability %08lx for " "Target failed %08lx\n", &TargetCap, Status)); return Status; } if (!ReverseInit) { Status = AgpLibSetMasterCapability(AgpContext, &MasterCap); if (!NT_SUCCESS(Status)) { AGPLOG(AGP_WARNING, ("UAGP35SetRate: AgpLibSetMasterCapability %08lx failed " "%08lx\n", &MasterCap, Status)); } } return Status; }