/*
* Copyright 2008 Advanced Micro Devices, Inc.
* Copyright 2008 Red Hat Inc.
* Copyright 2009 Jerome Glisse.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
* Authors: Dave Airlie
* Alex Deucher
* Jerome Glisse
*/
#include <linux/power_supply.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/console.h>
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/pci.h>
#include <linux/devcoredump.h>
#include <generated/utsrelease.h>
#include <linux/pci-p2pdma.h>
#include <linux/apple-gmux.h>
#include <drm/drm_aperture.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_crtc_helper.h>
#include <drm/drm_fb_helper.h>
#include <drm/drm_probe_helper.h>
#include <drm/amdgpu_drm.h>
#include <linux/vgaarb.h>
#include <linux/vga_switcheroo.h>
#include <linux/efi.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"
#include "amdgpu_i2c.h"
#include "atom.h"
#include "amdgpu_atombios.h"
#include "amdgpu_atomfirmware.h"
#include "amd_pcie.h"
#ifdef CONFIG_DRM_AMDGPU_SI
#include "si.h"
#endif
#ifdef CONFIG_DRM_AMDGPU_CIK
#include "cik.h"
#endif
#include "vi.h"
#include "soc15.h"
#include "nv.h"
#include "bif/bif_4_1_d.h"
#include <linux/firmware.h>
#include "amdgpu_vf_error.h"
#include "amdgpu_amdkfd.h"
#include "amdgpu_pm.h"
#include "amdgpu_xgmi.h"
#include "amdgpu_ras.h"
#include "amdgpu_pmu.h"
#include "amdgpu_fru_eeprom.h"
#include "amdgpu_reset.h"
#include <linux/suspend.h>
#include <drm/task_barrier.h>
#include <linux/pm_runtime.h>
#include <drm/drm_drv.h>
#if IS_ENABLED(CONFIG_X86)
#include <asm/intel-family.h>
#endif
MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
#define AMDGPU_RESUME_MS 2000
#define AMDGPU_MAX_RETRY_LIMIT 2
#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
static const struct drm_driver amdgpu_kms_driver;
const char *amdgpu_asic_name[] = {
"TAHITI",
"PITCAIRN",
"VERDE",
"OLAND",
"HAINAN",
"BONAIRE",
"KAVERI",
"KABINI",
"HAWAII",
"MULLINS",
"TOPAZ",
"TONGA",
"FIJI",
"CARRIZO",
"STONEY",
"POLARIS10",
"POLARIS11",
"POLARIS12",
"VEGAM",
"VEGA10",
"VEGA12",
"VEGA20",
"RAVEN",
"ARCTURUS",
"RENOIR",
"ALDEBARAN",
"NAVI10",
"CYAN_SKILLFISH",
"NAVI14",
"NAVI12",
"SIENNA_CICHLID",
"NAVY_FLOUNDER",
"VANGOGH",
"DIMGREY_CAVEFISH",
"BEIGE_GOBY",
"YELLOW_CARP",
"IP DISCOVERY",
"LAST",
};
/**
* DOC: pcie_replay_count
*
* The amdgpu driver provides a sysfs API for reporting the total number
* of PCIe replays (NAKs)
* The file pcie_replay_count is used for this and returns the total
* number of replays as a sum of the NAKs generated and NAKs received
*/
static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct drm_device *ddev = dev_get_drvdata(dev);
struct amdgpu_device *adev = drm_to_adev(ddev);
uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
return sysfs_emit(buf, "%llu\n", cnt);
}
static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
amdgpu_device_get_pcie_replay_count, NULL);
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
/**
* DOC: product_name
*
* The amdgpu driver provides a sysfs API for reporting the product name
* for the device
* The file product_name is used for this and returns the product name
* as returned from the FRU.
* NOTE: This is only available for certain server cards
*/
static ssize_t amdgpu_device_get_product_name(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct drm_device *ddev = dev_get_drvdata(dev);
struct amdgpu_device *adev = drm_to_adev(ddev);
return sysfs_emit(buf, "%s\n", adev->product_name);
}
static DEVICE_ATTR(product_name, S_IRUGO,
amdgpu_device_get_product_name, NULL);
/**
* DOC: product_number
*
* The amdgpu driver provides a sysfs API for reporting the part number
* for the device
* The file product_number is used for this and returns the part number
* as returned from the FRU.
* NOTE: This is only available for certain server cards
*/
static ssize_t amdgpu_device_get_product_number(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct drm_device *ddev = dev_get_drvdata(dev);
struct amdgpu_device *adev = drm_to_adev(ddev);
return sysfs_emit(buf, "%s\n", adev->product_number);
}
static DEVICE_ATTR(product_number, S_IRUGO,
amdgpu_device_get_product_number, NULL);
/**
* DOC: serial_number
*
* The amdgpu driver provides a sysfs API for reporting the serial number
* for the device
* The file serial_number is used for this and returns the serial number
* as returned from the FRU.
* NOTE: This is only available for certain server cards
*/
static ssize_t amdgpu_device_get_serial_number(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct drm_device *ddev = dev_get_drvdata(dev);
struct amdgpu_device *adev = drm_to_adev(ddev);
return sysfs_emit(buf, "%s\n", adev->serial);
}
static DEVICE_ATTR(serial_number, S_IRUGO,
amdgpu_device_get_serial_number, NULL);
/**
* amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
*
* @dev: drm_device pointer
*
* Returns true if the device is a dGPU with ATPX power control,
* otherwise return false.
*/
bool amdgpu_device_supports_px(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
return true;
return false;
}
/**
* amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
*
* @dev: drm_device pointer
*
* Returns true if the device is a dGPU with ACPI power control,
* otherwise return false.
*/
bool amdgpu_device_supports_boco(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
if (adev->has_pr3 ||
((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
return true;
return false;
}
/**
* amdgpu_device_supports_baco - Does the device support BACO
*
* @dev: drm_device pointer
*
* Returns true if the device supporte BACO,
* otherwise return false.
*/
bool amdgpu_device_supports_baco(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
return amdgpu_asic_supports_baco(adev);
}
/**
* amdgpu_device_supports_smart_shift - Is the device dGPU with
* smart shift support
*
* @dev: drm_device pointer
*
* Returns true if the device is a dGPU with Smart Shift support,
* otherwise returns false.
*/
bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
{
return (amdgpu_device_supports_boco(dev) &&
amdgpu_acpi_is_power_shift_control_supported());
}
/*
* VRAM access helper functions
*/
/**
* amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
*
* @adev: amdgpu_device pointer
* @pos: offset of the buffer in vram
* @buf: virtual address of the buffer in system memory
* @size: read/write size, sizeof(@buf) must > @size
* @write: true - write to vram, otherwise - read from vram
*/
void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
void *buf, size_t size, bool write)
{
unsigned long flags;
uint32_t hi = ~0, tmp = 0;
uint32_t *data = buf;
uint64_t last;
int idx;
if (!drm_dev_enter(adev_to_drm(adev), &idx))
return;
BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
spin_lock_irqsave(&adev->mmio_idx_lock, flags);
for (last = pos + size; pos < last; pos += 4) {
tmp = pos >> 31;
WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
if (tmp != hi) {
WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
hi = tmp;
}
if (write)
WREG32_NO_KIQ(mmMM_DATA, *data++);
else
*data++ = RREG32_NO_KIQ(mmMM_DATA);
}
spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
drm_dev_exit(idx);
}
/**
* amdgpu_device_aper_access - access vram by vram aperature
*
* @adev: amdgpu_device pointer
* @pos: offset of the buffer in vram
* @buf: virtual address of the buffer in system memory
* @size: read/write size, sizeof(@buf) must > @size
* @write: true - write to vram, otherwise - read from vram
*
* The return value means how many bytes have been transferred.
*/
size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
void *buf, size_t size, bool write)
{
#ifdef CONFIG_64BIT
void __iomem *addr;
size_t count = 0;
uint64_t last;
if (!adev->mman.aper_base_kaddr)
return 0;
last = min(pos + size, adev->gmc.visible_vram_size);
if (last > pos) {
addr = adev->mman.aper_base_kaddr + pos;
count = last - pos;
if (write) {
memcpy_toio(addr, buf, count);
mb();
amdgpu_device_flush_hdp(adev, NULL);
} else {
amdgpu_device_invalidate_hdp(adev, NULL);
mb();
memcpy_fromio(buf, addr, count);
}
}
return count;
#else
return 0;
#endif
}
/**
* amdgpu_device_vram_access - read/write a buffer in vram
*
* @adev: amdgpu_device pointer
* @pos: offset of the buffer in vram
* @buf: virtual address of the buffer in system memory
* @size: read/write size, sizeof(@buf) must > @size
* @write: true - write to vram, otherwise - read from vram
*/
void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
void *buf, size_t size, bool write)
{
size_t count;
/* try to using vram apreature to access vram first */
count = amdgpu_device_aper_access(adev, pos, buf, size, write);
size -= count;
if (size) {
/* using MM to access rest vram */
pos += count;
buf += count;
amdgpu_device_mm_access(adev, pos, buf, size, write);
}
}
/*
* register access helper functions.
*/
/* Check if hw access should be skipped because of hotplug or device error */
bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
{
if (adev->no_hw_access)
return true;
#ifdef CONFIG_LOCKDEP
/*
* This is a bit complicated to understand, so worth a comment. What we assert
* here is that the GPU reset is not running on another thread in parallel.
*
* For this we trylock the read side of the reset semaphore, if that succeeds
* we know that the reset is not running in paralell.
*
* If the trylock fails we assert that we are either already holding the read
* side of the lock or are the reset thread itself and hold the write side of
* the lock.
*/
if (in_task()) {
if (down_read_trylock(&adev->reset_domain->sem))
up_read(&adev->reset_domain->sem);
else
lockdep_assert_held(&adev->reset_domain->sem);
}
#endif
return false;
}
/**
* amdgpu_device_rreg - read a memory mapped IO or indirect register
*
* @adev: amdgpu_device pointer
* @reg: dword aligned register offset
* @acc_flags: access flags which require special behavior
*
* Returns the 32 bit value from the offset specified.
*/
uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
uint32_t reg, uint32_t acc_flags)
{
uint32_t ret;
if (amdgpu_device_skip_hw_access(adev))
return 0;
if ((reg * 4) < adev->rmmio_size) {
if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
amdgpu_sriov_runtime(adev) &&
down_read_trylock(&adev->reset_domain->sem)) {
ret = amdgpu_kiq_rreg(adev, reg);
up_read(&adev->reset_domain->sem);
} else {
ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
}
} else {
ret = adev->pcie_rreg(adev, reg * 4);
}
trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
return ret;
}
/*
* MMIO register read with bytes helper functions
* @offset:bytes offset from MMIO start
*
*/
/**
* amdgpu_mm_rreg8 - read a memory mapped IO register
*
* @adev: amdgpu_device pointer
* @offset: byte aligned register offset
*
* Returns the 8 bit value from the offset specified.
*/
uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
{
if (amdgpu_device_skip_hw_access(adev))
return 0;
if (offset < adev->rmmio_size)
return (readb(adev->rmmio + offset));
BUG();
}
/*
* MMIO register write with bytes helper functions
* @offset:bytes offset from MMIO start
* @value: the value want to be written to the register
*
*/
/**
* amdgpu_mm_wreg8 - read a memory mapped IO register
*
* @adev: amdgpu_device pointer
* @offset: byte aligned register offset
* @value: 8 bit value to write
*
* Writes the value specified to the offset specified.
*/
void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
{
if (amdgpu_device_skip_hw_access(adev))
return;
if (offset < adev->rmmio_size)
writeb(value, adev->rmmio + offset);
else
BUG();
}
/**
* amdgpu_device_wreg - write to a memory mapped IO or indirect register
*
* @adev: amdgpu_device pointer
* @reg: dword aligned register offset
* @v: 32 bit value to write to the register
* @acc_flags: access flags which require special behavior
*
* Writes the value specified to the offset specified.
*/
void amdgpu_device_wreg(struct amdgpu_device *adev,
uint32_t reg, uint32_t v,
uint32_t acc_flags)
{
if (amdgpu_device_skip_hw_access(adev))
return;
if ((reg * 4) < adev->rmmio_size) {
if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
amdgpu_sriov_runtime(adev) &&
down_read_trylock(&adev->reset_domain->sem)) {
amdgpu_kiq_wreg(adev, reg, v);
up_read(&adev->reset_domain->sem);
} else {
writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
}
} else {
adev->pcie_wreg(adev, reg * 4, v);
}
trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
}
/**
* amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range
*
* @adev: amdgpu_device pointer
* @reg: mmio/rlc register
* @v: value to write
*
* this function is invoked only for the debugfs register access
*/
void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
uint32_t reg, uint32_t v)
{
if (amdgpu_device_skip_hw_access(adev))
return;
if (amdgpu_sriov_fullaccess(adev) &&
adev->gfx.rlc.funcs &&
adev->gfx.rlc.funcs->is_rlcg_access_range) {
if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
} else if ((reg * 4) >= adev->rmmio_size) {
adev->pcie_wreg(adev, reg * 4, v);
} else {
writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
}
}
/**
* amdgpu_mm_rdoorbell - read a doorbell dword
*
* @adev: amdgpu_device pointer
* @index: doorbell index
*
* Returns the value in the doorbell aperture at the
* requested doorbell index (CIK).
*/
u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
{
if (amdgpu_device_skip_hw_access(adev))
return 0;
if (index < adev->doorbell.num_doorbells) {
return readl(adev->doorbell.ptr + index);
} else {
DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
return 0;
}
}
/**
* amdgpu_mm_wdoorbell - write a doorbell dword
*
* @adev: amdgpu_device pointer
* @index: doorbell index
* @v: value to write
*
* Writes @v to the doorbell aperture at the
* requested doorbell index (CIK).
*/
void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
{
if (amdgpu_device_skip_hw_access(adev))
return;
if (index < adev->doorbell.num_doorbells) {
writel(v, adev->doorbell.ptr + index);
} else {
DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
}
}
/**
* amdgpu_mm_rdoorbell64 - read a doorbell Qword
*
* @adev: amdgpu_device pointer
* @index: doorbell index
*
* Returns the value in the doorbell aperture at the
* requested doorbell index (VEGA10+).
*/
u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
{
if (amdgpu_device_skip_hw_access(adev))
return 0;
if (index < adev->doorbell.num_doorbells) {
return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
} else {
DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
return 0;
}
}
/**
* amdgpu_mm_wdoorbell64 - write a doorbell Qword
*
* @adev: amdgpu_device pointer
* @index: doorbell index
* @v: value to write
*
* Writes @v to the doorbell aperture at the
* requested doorbell index (VEGA10+).
*/
void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
{
if (amdgpu_device_skip_hw_access(adev))
return;
if (index < adev->doorbell.num_doorbells) {
atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
} else {
DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
}
}
/**
* amdgpu_device_indirect_rreg - read an indirect register
*
* @adev: amdgpu_device pointer
* @pcie_index: mmio register offset
* @pcie_data: mmio register offset
* @reg_addr: indirect register address to read from
*
* Returns the value of indirect register @reg_addr
*/
u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
u32 pcie_index, u32 pcie_data,
u32 reg_addr)
{
unsigned long flags;
u32 r;
void __iomem *pcie_index_offset;
void __iomem *pcie_data_offset;
spin_lock_irqsave(&adev->pcie_idx_lock, flags);
pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
writel(reg_addr, pcie_index_offset);
readl(pcie_index_offset);
r = readl(pcie_data_offset);
spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
return r;
}
/**
* amdgpu_device_indirect_rreg64 - read a 64bits indirect register
*
* @adev: amdgpu_device pointer
* @pcie_index: mmio register offset
* @pcie_data: mmio register offset
* @reg_addr: indirect register address to read from
*
* Returns the value of indirect register @reg_addr
*/
u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
u32 pcie_index, u32 pcie_data,
u32 reg_addr)
{
unsigned long flags;
u64 r;
void __iomem *pcie_index_offset;
void __iomem *pcie_data_offset;
spin_lock_irqsave(&adev->pcie_idx_lock, flags);
pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
/* read low 32 bits */
writel(reg_addr, pcie_index_offset);
readl(pcie_index_offset);
r = readl(pcie_data_offset);
/* read high 32 bits */
writel(reg_addr + 4, pcie_index_offset);
readl(pcie_index_offset);
r |= ((u64)readl(pcie_data_offset) << 32);
spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
return r;
}
/**
* amdgpu_device_indirect_wreg - write an indirect register address
*
* @adev: amdgpu_device pointer
* @pcie_index: mmio register offset
* @pcie_data: mmio register offset
* @reg_addr: indirect register offset
* @reg_data: indirect register data
*
*/
void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
u32 pcie_index, u32 pcie_data,
u32 reg_addr, u32 reg_data)
{
unsigned long flags;
void __iomem *pcie_index_offset;
void __iomem *pcie_data_offset;
spin_lock_irqsave(&adev->pcie_idx_lock, flags);
pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
writel(reg_addr, pcie_index_offset);
readl(pcie_index_offset);
writel(reg_data, pcie_data_offset);
readl(pcie_data_offset);
spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
}
/**
* amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
*
* @adev: amdgpu_device pointer
* @pcie_index: mmio register offset
* @pcie_data: mmio register offset
* @reg_addr: indirect register offset
* @reg_data: indirect register data
*
*/
void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
u32 pcie_index, u32 pcie_data,
u32 reg_addr, u64 reg_data)
{
unsigned long flags;
void __iomem *pcie_index_offset;
void __iomem *pcie_data_offset;
spin_lock_irqsave(&adev->pcie_idx_lock, flags);
pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
/* write low 32 bits */
writel(reg_addr, pcie_index_offset);
readl(pcie_index_offset);
writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
readl(pcie_data_offset);
/* write high 32 bits */
writel(reg_addr + 4, pcie_index_offset);
readl(pcie_index_offset);
writel((u32)(reg_data >> 32), pcie_data_offset);
readl(pcie_data_offset);
spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
}
/**
* amdgpu_invalid_rreg - dummy reg read function
*
* @adev: amdgpu_device pointer
* @reg: offset of register
*
* Dummy register read function. Used for register blocks
* that certain asics don't have (all asics).
* Returns the value in the register.
*/
static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
{
DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
BUG();
return 0;
}
/**
* amdgpu_invalid_wreg - dummy reg write function
*
* @adev: amdgpu_device pointer
* @reg: offset of register
* @v: value to write to the register
*
* Dummy register read function. Used for register blocks
* that certain asics don't have (all asics).
*/
static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
{
DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
reg, v);
BUG();
}
/**
* amdgpu_invalid_rreg64 - dummy 64 bit reg read function
*
* @adev: amdgpu_device pointer
* @reg: offset of register
*
* Dummy register read function. Used for register blocks
* that certain asics don't have (all asics).
* Returns the value in the register.
*/
static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
{
DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
BUG();
return 0;
}
/**
* amdgpu_invalid_wreg64 - dummy reg write function
*
* @adev: amdgpu_device pointer
* @reg: offset of register
* @v: value to write to the register
*
* Dummy register read function. Used for register blocks
* that certain asics don't have (all asics).
*/
static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
{
DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
reg, v);
BUG();
}
/**
* amdgpu_block_invalid_rreg - dummy reg read function
*
* @adev: amdgpu_device pointer
* @block: offset of instance
* @reg: offset of register
*
* Dummy register read function. Used for register blocks
* that certain asics don't have (all asics).
* Returns the value in the register.
*/
static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
uint32_t block, uint32_t reg)
{
DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
reg, block);
BUG();
return 0;
}
/**
* amdgpu_block_invalid_wreg - dummy reg write function
*
* @adev: amdgpu_device pointer
* @block: offset of instance
* @reg: offset of register
* @v: value to write to the register
*
* Dummy register read function. Used for register blocks
* that certain asics don't have (all asics).
*/
static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
uint32_t block,
uint32_t reg, uint32_t v)
{
DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
reg, block, v);
BUG();
}
/**
* amdgpu_device_asic_init - Wrapper for atom asic_init
*
* @adev: amdgpu_device pointer
*
* Does any asic specific work and then calls atom asic init.
*/
static int amdgpu_device_asic_init(struct amdgpu_device *adev)
{
amdgpu_asic_pre_asic_init(adev);
if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
return amdgpu_atomfirmware_asic_init(adev, true);
else
return amdgpu_atom_asic_init(adev->mode_info.atom_context);
}
/**
* amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
*
* @adev: amdgpu_device pointer
*
* Allocates a scratch page of VRAM for use by various things in the
* driver.
*/
static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
{
return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_VRAM |
AMDGPU_GEM_DOMAIN_GTT,
&adev->mem_scratch.robj,
&adev->mem_scratch.gpu_addr,
(void **)&adev->mem_scratch.ptr);
}
/**
* amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
*
* @adev: amdgpu_device pointer
*
* Frees the VRAM scratch page.
*/
static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
{
amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
}
/**
* amdgpu_device_program_register_sequence - program an array of registers.
*
* @adev: amdgpu_device pointer
* @registers: pointer to the register array
* @array_size: size of the register array
*
* Programs an array or registers with and and or masks.
* This is a helper for setting golden registers.
*/
void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
const u32 *registers,
const u32 array_size)
{
u32 tmp, reg, and_mask, or_mask;
int i;
if (array_size % 3)
return;
for (i = 0; i < array_size; i +=3) {
reg = registers[i + 0];
and_mask = registers[i + 1];
or_mask = registers[i + 2];
if (and_mask == 0xffffffff) {
tmp = or_mask;
} else {
tmp = RREG32(reg);
tmp &= ~and_mask;
if (adev->family >= AMDGPU_FAMILY_AI)
tmp |= (or_mask & and_mask);
else
tmp |= or_mask;
}
WREG32(reg, tmp);
}
}
/**
* amdgpu_device_pci_config_reset - reset the GPU
*
* @adev: amdgpu_device pointer
*
* Resets the GPU using the pci config reset sequence.
* Only applicable to asics prior to vega10.
*/
void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
{
pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
}
/**
* amdgpu_device_pci_reset - reset the GPU using generic PCI means
*
* @adev: amdgpu_device pointer
*
* Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
*/
int amdgpu_device_pci_reset(struct amdgpu_device *adev)
{
return pci_reset_function(adev->pdev);
}
/*
* GPU doorbell aperture helpers function.
*/
/**
* amdgpu_device_doorbell_init - Init doorbell driver information.
*
* @adev: amdgpu_device pointer
*
* Init doorbell driver information (CIK)
* Returns 0 on success, error on failure.
*/
static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
{
/* No doorbell on SI hardware generation */
if (adev->asic_type < CHIP_BONAIRE) {
adev->doorbell.base = 0;
adev->doorbell.size = 0;
adev->doorbell.num_doorbells = 0;
adev->doorbell.ptr = NULL;
return 0;
}
if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
return -EINVAL;
amdgpu_asic_init_doorbell_index(adev);
/* doorbell bar mapping */
adev->doorbell.base = pci_resource_start(adev->pdev, 2);
adev->doorbell.size = pci_resource_len(adev->pdev, 2);
if (adev->enable_mes) {
adev->doorbell.num_doorbells =
adev->doorbell.size / sizeof(u32);
} else {
adev->doorbell.num_doorbells =
min_t(u32, adev->doorbell.size / sizeof(u32),
adev->doorbell_index.max_assignment+1);
if (adev->doorbell.num_doorbells == 0)
return -EINVAL;
/* For Vega, reserve and map two pages on doorbell BAR since SDMA
* paging queue doorbell use the second page. The
* AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
* doorbells are in the first page. So with paging queue enabled,
* the max num_doorbells should + 1 page (0x400 in dword)
*/
if (adev->asic_type >= CHIP_VEGA10)
adev->doorbell.num_doorbells += 0x400;
}
adev->doorbell.ptr = ioremap(adev->doorbell.base,
adev->doorbell.num_doorbells *
sizeof(u32));
if (adev->doorbell.ptr == NULL)
return -ENOMEM;
return 0;
}
/**
* amdgpu_device_doorbell_fini - Tear down doorbell driver information.
*
* @adev: amdgpu_device pointer
*
* Tear down doorbell driver information (CIK)
*/
static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
{
iounmap(adev->doorbell.ptr);
adev->doorbell.ptr = NULL;
}
/*
* amdgpu_device_wb_*()
* Writeback is the method by which the GPU updates special pages in memory
* with the status of certain GPU events (fences, ring pointers,etc.).
*/
/**
* amdgpu_device_wb_fini - Disable Writeback and free memory
*
* @adev: amdgpu_device pointer
*
* Disables Writeback and frees the Writeback memory (all asics).
* Used at driver shutdown.
*/
static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
{
if (adev->wb.wb_obj) {
amdgpu_bo_free_kernel(&adev->wb.wb_obj,
&adev->wb.gpu_addr,
(void **)&adev->wb.wb);
adev->wb.wb_obj = NULL;
}
}
/**
* amdgpu_device_wb_init - Init Writeback driver info and allocate memory
*
* @adev: amdgpu_device pointer
*
* Initializes writeback and allocates writeback memory (all asics).
* Used at driver startup.
* Returns 0 on success or an -error on failure.
*/
static int amdgpu_device_wb_init(struct amdgpu_device *adev)
{
int r;
if (adev->wb.wb_obj == NULL) {
/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
&adev->wb.wb_obj, &adev->wb.gpu_addr,
(void **)&adev->wb.wb);
if (r) {
dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
return r;
}
adev->wb.num_wb = AMDGPU_MAX_WB;
memset(&adev->wb.used, 0, sizeof(adev->wb.used));
/* clear wb memory */
memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
}
return 0;
}
/**
* amdgpu_device_wb_get - Allocate a wb entry
*
* @adev: amdgpu_device pointer
* @wb: wb index
*
* Allocate a wb slot for use by the driver (all asics).
* Returns 0 on success or -EINVAL on failure.
*/
int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
{
unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
if (offset < adev->wb.num_wb) {
__set_bit(offset, adev->wb.used);
*wb = offset << 3; /* convert to dw offset */
return 0;
} else {
return -EINVAL;
}
}
/**
* amdgpu_device_wb_free - Free a wb entry
*
* @adev: amdgpu_device pointer
* @wb: wb index
*
* Free a wb slot allocated for use by the driver (all asics)
*/
void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
{
wb >>= 3;
if (wb < adev->wb.num_wb)
__clear_bit(wb, adev->wb.used);
}
/**
* amdgpu_device_resize_fb_bar - try to resize FB BAR
*
* @adev: amdgpu_device pointer
*
* Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
* to fail, but if any of the BARs is not accessible after the size we abort
* driver loading by returning -ENODEV.
*/
int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
{
int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
struct pci_bus *root;
struct resource *res;
unsigned i;
u16 cmd;
int r;
/* Bypass for VF */
if (amdgpu_sriov_vf(adev))
return 0;
/* skip if the bios has already enabled large BAR */
if (adev->gmc.real_vram_size &&
(pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
return 0;
/* Check if the root BUS has 64bit memory resources */
root = adev->pdev->bus;
while (root->parent)
root = root->parent;
pci_bus_for_each_resource(root, res, i) {
if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
res->start > 0x100000000ull)
break;
}
/* Trying to resize is pointless without a root hub window above 4GB */
if (!res)
return 0;
/* Limit the BAR size to what is available */
rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
rbar_size);
/* Disable memory decoding while we change the BAR addresses and size */
pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
pci_write_config_word(adev->pdev, PCI_COMMAND,
cmd & ~PCI_COMMAND_MEMORY);
/* Free the VRAM and doorbell BAR, we most likely need to move both. */
amdgpu_device_doorbell_fini(adev);
if (adev->asic_type >= CHIP_BONAIRE)
pci_release_resource(adev->pdev, 2);
pci_release_resource(adev->pdev, 0);
r = pci_resize_resource(adev->pdev, 0, rbar_size);
if (r == -ENOSPC)
DRM_INFO("Not enough PCI address space for a large BAR.");
else if (r && r != -ENOTSUPP)
DRM_ERROR("Problem resizing BAR0 (%d).", r);
pci_assign_unassigned_bus_resources(adev->pdev->bus);
/* When the doorbell or fb BAR isn't available we have no chance of
* using the device.
*/
r = amdgpu_device_doorbell_init(adev);
if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
return -ENODEV;
pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
return 0;
}
/*
* GPU helpers function.
*/
/**
* amdgpu_device_need_post - check if the hw need post or not
*
* @adev: amdgpu_device pointer
*
* Check if the asic has been initialized (all asics) at driver startup
* or post is needed if hw reset is performed.
* Returns true if need or false if not.
*/
bool amdgpu_device_need_post(struct amdgpu_device *adev)
{
uint32_t reg;
if (amdgpu_sriov_vf(adev))
return false;
if (amdgpu_passthrough(adev)) {
/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
* some old smc fw still need driver do vPost otherwise gpu hang, while
* those smc fw version above 22.15 doesn't have this flaw, so we force
* vpost executed for smc version below 22.15
*/
if (adev->asic_type == CHIP_FIJI) {
int err;
uint32_t fw_ver;
err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
/* force vPost if error occured */
if (err)
return true;
fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
if (fw_ver < 0x00160e00)
return true;
}
}
/* Don't post if we need to reset whole hive on init */
if (adev->gmc.xgmi.pending_reset)
return false;
if (adev->has_hw_reset) {
adev->has_hw_reset = false;
return true;
}
/* bios scratch used on CIK+ */
if (adev->asic_type >= CHIP_BONAIRE)
return amdgpu_atombios_scratch_need_asic_init(adev);
/* check MEM_SIZE for older asics */
reg = amdgpu_asic_get_config_memsize(adev);
if ((reg != 0) && (reg != 0xffffffff))
return false;
return true;
}
/**
* amdgpu_device_should_use_aspm - check if the device should program ASPM
*
* @adev: amdgpu_device pointer
*
* Confirm whether the module parameter and pcie bridge agree that ASPM should
* be set for this device.
*
* Returns true if it should be used or false if not.
*/
bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
{
switch (amdgpu_aspm) {
case -1:
break;
case 0:
return false;
case 1:
return true;
default:
return false;
}
return pcie_aspm_enabled(adev->pdev);
}
bool amdgpu_device_aspm_support_quirk(void)
{
#if IS_ENABLED(CONFIG_X86)
struct cpuinfo_x86 *c = &cpu_data(0);
return !(c->x86 == 6 && c->x86_model == INTEL_FAM6_ALDERLAKE);
#else
return t