Linux kernel IOMMU代码分析笔记(5)——Interrupt Remapping初始化相关部分

irq_remap_ops定义在drivers\iommu\irq_remapping.h中:

struct irq_remap_ops {
    /* Check whether Interrupt Remapping is supported */
    int (*supported)(void);

    /* Initializes hardware and makes it ready for remapping interrupts */
    int  (*prepare)(void);

    /* Enables the remapping hardware */
    int  (*enable)(void);

    /* Disables the remapping hardware */
    void (*disable)(void);

    /* Reenables the remapping hardware */
    int  (*reenable)(int);

    /* Enable fault handling */
    int  (*enable_faulting)(void);

    /* IO-APIC setup routine */
    int (*setup_ioapic_entry)(int irq, struct IO_APIC_route_entry *,
                  unsigned int, int,
                  struct io_apic_irq_attr *);

    /* Set the CPU affinity of a remapped interrupt */
    int (*set_affinity)(struct irq_data *data, const struct cpumask *mask,
                bool force);

    /* Free an IRQ */
    int (*free_irq)(int);

    /* Create MSI msg to use for interrupt remapping */
    void (*compose_msi_msg)(struct pci_dev *,
                unsigned int, unsigned int,
                struct msi_msg *, u8);

    /* Allocate remapping resources for MSI */
    int (*msi_alloc_irq)(struct pci_dev *, int, int);

    /* Setup the remapped MSI irq */
    int (*msi_setup_irq)(struct pci_dev *, unsigned int, int, int);

    /* Setup interrupt remapping for an HPET MSI */
    int (*setup_hpet_msi)(unsigned int, unsigned int);
};

extern struct irq_remap_ops intel_irq_remap_ops;
extern struct irq_remap_ops amd_iommu_irq_ops;

可以看到,在结构体中定义了一系列的函数指针,每个函数的作用都写得很清楚。

针对intel处理器的结构体定义在drivers\iommu\intel_irq_remapping.c

struct irq_remap_ops intel_irq_remap_ops = {
    .supported      = intel_irq_remapping_supported,
    .prepare        = dmar_table_init,
    .enable         = intel_enable_irq_remapping,
    .disable        = disable_irq_remapping,
    .reenable       = reenable_irq_remapping,
    .enable_faulting    = enable_drhd_fault_handling,
    .setup_ioapic_entry = intel_setup_ioapic_entry,
    .set_affinity       = intel_ioapic_set_affinity,
    .free_irq       = free_irte,
    .compose_msi_msg    = intel_compose_msi_msg,
    .msi_alloc_irq      = intel_msi_alloc_irq,
    .msi_setup_irq      = intel_msi_setup_irq,
    .setup_hpet_msi     = intel_setup_hpet_msi,
};

AMD相关的暂不讨论。

enable_IR_x2apic函数(arch\x86\kernel\apic)中,包含着irq_remap_ops中初始化相关函数的调用:

void __init enable_IR_x2apic(void)
{
    ......
    /* Make sure irq_remap_ops are initialized */
    setup_irq_remapping_ops();

    hardware_init_ret = irq_remapping_prepare();
    if (hardware_init_ret && !x2apic_supported())
        return;

    ......   
    if (hardware_init_ret)
        ret = -1;
    else
        ret = enable_IR();

    ......
}

(1)

void __init setup_irq_remapping_ops(void)
{
    remap_ops = &intel_irq_remap_ops;

#ifdef CONFIG_AMD_IOMMU
    if (amd_iommu_irq_ops.prepare() == 0)
        remap_ops = &amd_iommu_irq_ops;
#endif
}

这个函数会让全局变量remap_ops这个指针指向intel_irq_remap_opsamd_iommu_irq_ops

(2)

int __init irq_remapping_prepare(void)
{
    if (!remap_ops || !remap_ops->prepare)
        return -ENODEV;

    return remap_ops->prepare();
}

irq_remapping_prepare会调用prepare函数(Intel平台即dmar_table_init)。

(3)

int __init enable_IR(void)
{
#ifdef CONFIG_IRQ_REMAP
    if (!irq_remapping_supported()) {
        pr_debug("intr-remapping not supported\n");
        return -1;
    }

    if (!x2apic_preenabled && skip_ioapic_setup) {
        pr_info("Skipped enabling intr-remap because of skipping "
            "io-apic setup\n");
        return -1;
    }

    return irq_remapping_enable();
#endif
    return -1;
}

enable_IR中会调用irq_remapping_supportedirq_remapping_enable

int irq_remapping_supported(void)
{
    if (disable_irq_remap)
        return 0;

    if (!remap_ops || !remap_ops->supported)
        return 0;

    return remap_ops->supported();
}

int __init irq_remapping_enable(void)
{
    int ret;

    if (!remap_ops || !remap_ops->enable)
        return -ENODEV;

    ret = remap_ops->enable();

    if (irq_remapping_enabled)
        irq_remapping_modify_x86_ops();

    return ret;
}

会先后调用supportedIntel平台即为intel_irq_remapping_supported)和enable函数(Intel平台即为intel_enable_irq_remapping)。

irq_remapping_modify_x86_ops会把其它函数赋值给相应的全局函数指针:

static void __init irq_remapping_modify_x86_ops(void)
{
    x86_io_apic_ops.disable     = irq_remapping_disable_io_apic;
    x86_io_apic_ops.set_affinity    = set_remapped_irq_affinity;
    x86_io_apic_ops.setup_entry = setup_ioapic_remapped_entry;
    x86_io_apic_ops.eoi_ioapic_pin  = eoi_ioapic_pin_remapped;
    x86_msi.setup_msi_irqs      = irq_remapping_setup_msi_irqs;
    x86_msi.setup_hpet_msi      = setup_hpet_msi_remapped;
    x86_msi.compose_msi_msg     = compose_remapped_msi_msg;
}

Shark代码分析笔记(2)——main函数

看一下main函数:

int main(int argc, char **argv)
{
    int ret = EXIT_FAILURE;
    int flags = 0, script;
    int base;

    if (argv[0] && argv[0][0]) progname = argv[0];

    lua_State *ls = lua_open();
    if (!ls) {
        l_message(progname, "cannot create state: not enough memory");
        return ret;
    }

    g_ls = ls;

    script = collectargs(argv, &flags);
    if (script <= 0) {  /* invalid args? */
        print_usage();
        return 0;
    }

    if (flags & FLAGS_VERSION)
        print_version();

    luaL_openlibs(ls);  /* open libraries */

    // Get package.preload so we can store builtins in it.
    lua_getglobal(ls, "package");
    lua_getfield(ls, -1, "preload");
    lua_remove(ls, -2); // Remove package

    // Store uv module definition at preload.uv
    lua_pushcfunction(ls, luaopen_luv);
    lua_setfield(ls, -2, "uv");

    luaL_openlib(ls, "shark", ll_shark, 0);

    lua_getglobal(ls, "shark");
    lua_pushboolean(ls, shark_verbose);
    lua_setfield(ls, -2, "verbose");
    lua_pop(ls, 1);

    int narg = getargs(ls, argv, script);  /* collect arguments */
    lua_setglobal(ls, "arg");

#include "shark_init.h"
    luaL_loadbuffer(ls, luaJIT_BC_shark_init, luaJIT_BC_shark_init_SIZE,
            NULL);
    if((ret = lua_pcall(ls, 0, 0, 0))) {
        ret = lua_report(ls, ret);
        goto out;
    }

    g_event_loop = luv_loop(ls);

    if((ret = luaL_loadfile(ls, argv[script]))) {
        ret = lua_report(ls, ret);
        goto out;
    }

    base = lua_gettop(ls) - 1;
    lua_pushcfunction(ls, lua_traceback);
    lua_insert(ls, base);

    if (lua_pcall(ls, 0, 0, base)) {
        fprintf(stderr, "%s\n", lua_tostring(ls, -1));
        exit(EXIT_FAILURE);
    }

    lua_pop(ls, 1);

    //TODO: move to lua init code
    uv_run(g_event_loop, UV_RUN_DEFAULT);

    ret = 0;
 out:
    lua_close(ls);
    return ret;
}  

(1)

if (argv[0] && argv[0][0]) progname = argv[0];

progname存的是运行程序名字:/path/to/shark

(2)

    lua_State *ls = lua_open();
    if (!ls) {
        l_message(progname, "cannot create state: not enough memory");
        return ret;
    }

    g_ls = ls;

创建一个新的Lua运行环境,为了后续使用。

(3)

script = collectargs(argv, &flags);
if (script <= 0) {  /* invalid args? */
    print_usage();
    return 0;
}

if (flags & FLAGS_VERSION)
    print_version();

先看一下print_version

static void print_version(void)
{
    fputs(SHARK_VERSION " -- " SHARK_COPYRIGHT ". " SHARK_URL "\n", stdout);
    exit(0);
}

比较简单,打印出版本就退出了。

再看一下print_usage

static void print_usage(void)
{
    fprintf(stderr,
    "usage: shark [options]... [script [args]...].\n"
    "Available options are:\n"
    "  -v        Show version information.\n");
    fflush(stderr);
}

可以看到shark的使用方法:

shark [options]... [script [args]...]

shark程序后面跟着可选参数,还有执行脚本。

(4)

luaL_openlibs(ls);  /* open libraries */

加载Luajit提供的函数库。

(5)

// Get package.preload so we can store builtins in it.
lua_getglobal(ls, "package");
lua_getfield(ls, -1, "preload");
lua_remove(ls, -2); // Remove package

lua_getglobal(ls, "package");用来把package这个table压入堆栈:

 ___________________________
|____ package(类型:table)___|  (-1)

lua_getfield(ls, -1, "preload");用来从index-1处取出keypreload值,也就是:package.preload这个table,并压入堆栈:

 ___________________________
|package.preload(类型:table)|  (-1)
|____ package(类型:table)___|  (-2)

lua_remove(ls, -2);package从堆栈中移除:

 ___________________________
|package.preload(类型:table)|  (-1)

(6)

// Store uv module definition at preload.uv
lua_pushcfunction(ls, luaopen_luv);
lua_setfield(ls, -2, "uv");

lua_pushcfunction(ls, luaopen_luv);luaopen_luv函数压入堆栈:

 ___________________________
|__luaopen_luv(类型:函数)___|  (-1)
|package.preload(类型:table)|  (-2)

lua_setfield(ls, -2, "uv");执行效果是:package.preload.uv = luaopen_luv,然后把luaopen_luv弹出堆栈:

 ____________________________________________
|package.preload(类型:table,uv = luaopen_luv)|  (-1)

(7)

luaL_openlib(ls, "shark", ll_shark, 0);

ll_shark数组定义如下:

static const struct luaL_reg ll_shark[] = {
        {"debuginfo_set", &shark_api_debuginfo_set},
        {"lua_ref", &shark_api_lua_ref},
        {"get_ref", &shark_api_get_ref},
        {"stats", &shark_api_stats},
        {"set_binary", &shark_api_set_binary},
        {"exec", &shark_api_exec},
//TODO: move to sock library
#ifndef BPF_DISABLE
        {"open_raw_sock", &shark_api_open_raw_sock},
        {"sock_attach_bpf", &shark_api_sock_attach_bpf},
        {"iptos", &shark_api_iptos},
#endif
    {NULL, NULL}
};

luaL_openlib(ls, "shark", ll_shark, 0);函数的作用是创建一个table然后将这个table赋给package.loaded[shark]和全局变量shark,并将ll_shark数组中的函数注册到这个table,最后把table压入堆栈:

 ————————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-1)
|package.preload(类型:table,uv = luaopen_luv)|  (-2)

(8)

    lua_getglobal(ls, "shark");
    lua_pushboolean(ls, shark_verbose);
    lua_setfield(ls, -2, "verbose");
    lua_pop(ls, 1);

lua_getglobal(ls, "shark");shark table压入堆栈:

 ____________________________________________
|       shark(类型:table)                    |  (-1)
 ————————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-2)
|package.preload(类型:table,uv = luaopen_luv)|  (-3)

lua_pushboolean(ls, shark_verbose);shark_verbose这个布尔值压入堆栈:

_____________________________________________
| ______shark_verbose(类型:bool)_____________|  (-1)
|       shark(类型:table)                    |  (-2)
 ————————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-3)
|package.preload(类型:table,uv = luaopen_luv)|  (-4)

lua_setfield(ls, -2, "verbose");执行效果是:shark.verbose = shark_verbose,然后把shark_verbose弹出堆栈:

 ____________________________________________
| shark(类型:table, verbose = shark_verbose) |  (-1)
 ————————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-2)
|package.preload(类型:table,uv = luaopen_luv)|  (-3)

lua_pop(ls, 1);把栈顶元素弹出堆栈:

 ————————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-1)
|package.preload(类型:table,uv = luaopen_luv)|  (-2)

(9)

int narg = getargs(ls, argv, script);  /* collect arguments */
lua_setglobal(ls, "arg");

getargs函数实现如下:

static int getargs(lua_State *ls, char **argv, int n)
{
    int narg;
    int i;
    int argc = 0;

    while (argv[argc])
        argc++;  /* count total number of arguments */

    narg = argc - (n + 1);  /* number of arguments to the script */
    luaL_checkstack(ls, narg + 3, "too many arguments to script");

    for (i = n + 1; i < argc; i++)
        lua_pushstring(ls, argv[i]);

    lua_createtable(ls, narg, n + 1);

    for (i = 0; i < argc; i++) {
        lua_pushstring(ls, argv[i]);
        lua_rawseti(ls, -2, i - n);
    }

    return narg;
}

getargs函数中,传入参数nLua脚本在命令行参数的索引,而narg则是脚本的参数。举个例子:

/root/shark/shark trace.lua 1 2

argc4n1narg2

luaL_checkstack(ls, narg + 3, "too many arguments to script");检查堆栈是否有足够的空间。

    for (i = n + 1; i < argc; i++)
        lua_pushstring(ls, argv[i]);

把参数12压入堆栈:

 ___________________________________________
|__________参数: 2__________________________|  (-1)
|          参数: 1                          |  (-2)
|———————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-3)
|package.preload(类型:table,uv = luaopen_luv)|  (-4)  

lua_createtable(ls, narg, n + 1);创建一个table(包含narg数组元素,n + 1非数组元素)并压入堆栈:

 ———————————————————————————————————————————
|______table________________________________|  (-1)
|__________参数: 2__________________________|  (-2)
|          参数: 1                          |  (-3)
|———————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-4)
|package.preload(类型:table,uv = luaopen_luv)|  (-5)  

看最后一个循环:

    for (i = 0; i < argc; i++) {
        lua_pushstring(ls, argv[i]);
        lua_rawseti(ls, -2, i - n);
    }

lua_pushstring(ls, argv[i]);依次把参数压入堆栈,lua_rawseti(ls, -2, i - n);则是把参数传入table

 —————————————————————————————————————————————————
|_table(index:value->1:shark;0:trace.lua;1:1;2:2_|  (-1)
|__________参数: 2_______________________________|  (-2)
|          参数: 1                               |  (-3)
|————————————————————————————————————————|
|___一个注册ll_shark函数数组的table________________|  (-4)
|package.preload(类型:table,uv = luaopen_luv)     |  (-5)  

lua_setglobal(ls, "arg");作用是把栈顶table弹出,并赋值给arg。所以arg就指向了这个tableindex:value->1:shark;0:trace.lua;1:1;2:2)。堆栈变为:

____________________________________________
|__________参数: 2__________________________|  (-1)
|          参数: 1                          |  (-2)
|———————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-3)
|package.preload(类型:table,uv = luaopen_luv)|  (-4) 

(10)

#include "shark_init.h"
luaL_loadbuffer(ls, luaJIT_BC_shark_init, luaJIT_BC_shark_init_SIZE,
        NULL);

shark_init.h是由shark_init.lua生成的(以后再详细介绍shark_init.lua),luaJIT_BC_shark_initluaJIT_BC_shark_init_SIZE也定义在shark_init.h文件中。

luaL_loadbuffer(ls, luaJIT_BC_shark_init, luaJIT_BC_shark_init_SIZE, NULL);luaJIT_BC_shark_init这个chunk压入堆栈:

____________________________________________
|_luaJIT_BC_shark_init chunk(类型:函数)______| (-1)
|__________参数: 2__________________________|  (-2)
|          参数: 1                          |  (-3)
|———————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-4)
|package.preload(类型:table,uv = luaopen_luv)|  (-5)  

(11)

if((ret = lua_pcall(ls, 0, 0, 0))) {
        ret = lua_report(ls, ret);
        goto out;
}

接下来lua_pcall(ls, 0, 0, 0)会运行luaJIT_BC_shark_init这个chunk。运行完后,把chunk弹出堆栈:

————————————————————————————————————————————
|__________参数: 2__________________________|  (-1)
|          参数: 1                          |  (-2)
|———————————————————————————————————————————
|___一个注册ll_shark函数数组的table___________|  (-3)
|package.preload(类型:table,uv = luaopen_luv)|  (-4)  

lua_report比较简单,就是如果出错的话,就从栈顶取出错误信息,打印完以后再弹栈:

int lua_report(lua_State *ls, int status)
{
    if (status && !lua_isnil(ls, -1)) {
        const char *msg = lua_tostring(ls, -1);
        if (msg == NULL)
            msg = "(error object is not a string)";
        l_message(progname, msg);
        lua_pop(ls, 1);
    }
    return status;
}  

(12)

    g_event_loop = luv_loop(ls);

    if((ret = luaL_loadfile(ls, argv[script]))) {
        ret = lua_report(ls, ret);
        goto out;
    }

    base = lua_gettop(ls) - 1;
    lua_pushcfunction(ls, lua_traceback);
    lua_insert(ls, base);

    if (lua_pcall(ls, 0, 0, base)) {
        fprintf(stderr, "%s\n", lua_tostring(ls, -1));
        exit(EXIT_FAILURE);
    }

    lua_pop(ls, 1);

    //TODO: move to lua init code
    uv_run(g_event_loop, UV_RUN_DEFAULT);

    ret = 0;
 out:
    lua_close(ls);
    return ret;

剩下这段代码就是运行脚本,其中lua_traceback是脚本出错时的处理函数。个人觉得细节上还有些问题,需要和作者沟通一下,这块代码暂时留个小尾巴。

Linux kernel 笔记 (11)——pr_fmt

kernel代码中,很多.c文件开头都会有pr_fmt的定义,例如drivers\iommu\dmar.c

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */

这个pr_fmt是针对这个特定的模块定义的。看一下include\linux\printk.h

#ifndef pr_fmt
#define pr_fmt(fmt) fmt
#endif

可以看到,如果这个模块没有定义自己的pr_fmt,那么就会使用默认的fmt。自己定义pr_fmt的好处是可以更清晰地打印这个模块的相关log,便于以后调试。

Linux kernel IOMMU代码分析笔记(4)——fault处理函数

dmar_fault是发生DMA remapping faultinterrupt remapping fault时的处理函数。代码如下:

irqreturn_t dmar_fault(int irq, void *dev_id)
{
    struct intel_iommu *iommu = dev_id;
    int reg, fault_index;
    u32 fault_status;
    unsigned long flag;

    raw_spin_lock_irqsave(&iommu->register_lock, flag);
    fault_status = readl(iommu->reg + DMAR_FSTS_REG);
    if (fault_status)
        pr_err("DRHD: handling fault status reg %x\n", fault_status);

    /* TBD: ignore advanced fault log currently */
    if (!(fault_status & DMA_FSTS_PPF))
        goto unlock_exit;

    fault_index = dma_fsts_fault_record_index(fault_status);
    reg = cap_fault_reg_offset(iommu->cap);
    while (1) {
        u8 fault_reason;
        u16 source_id;
        u64 guest_addr;
        int type;
        u32 data;

        /* highest 32 bits */
        data = readl(iommu->reg + reg +
                fault_index * PRIMARY_FAULT_REG_LEN + 12);
        if (!(data & DMA_FRCD_F))
            break;

        fault_reason = dma_frcd_fault_reason(data);
        type = dma_frcd_type(data);

        data = readl(iommu->reg + reg +
                fault_index * PRIMARY_FAULT_REG_LEN + 8);
        source_id = dma_frcd_source_id(data);

        guest_addr = dmar_readq(iommu->reg + reg +
                fault_index * PRIMARY_FAULT_REG_LEN);
        guest_addr = dma_frcd_page_addr(guest_addr);
        /* clear the fault */
        writel(DMA_FRCD_F, iommu->reg + reg +
            fault_index * PRIMARY_FAULT_REG_LEN + 12);

        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);

        dmar_fault_do_one(iommu, type, fault_reason,
                source_id, guest_addr);

        fault_index++;
        if (fault_index >= cap_num_fault_regs(iommu->cap))
            fault_index = 0;
        raw_spin_lock_irqsave(&iommu->register_lock, flag);
    }

    writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);

unlock_exit:
    raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
    return IRQ_HANDLED;
}  

分析一下这段代码:

(1)

fault_status = readl(iommu->reg + DMAR_FSTS_REG);
if (fault_status)
    pr_err("DRHD: handling fault status reg %x\n", fault_status);

/* TBD: ignore advanced fault log currently */
if (!(fault_status & DMA_FSTS_PPF))
    goto unlock_exit;

fault_index = dma_fsts_fault_record_index(fault_status);

DMAR_FSTS_REGFault Status Register,其中PPFPrimary Pending Fault)是否置位表明fault recording registers是否还有fault信息。而fault_index则是第一个含有fault信息的fault recording registers的索引。

(2)

reg = cap_fault_reg_offset(iommu->cap);

计算fault recording registers的地址偏移量。

(3)接下来的while循环会读取包含fault信息的fault recording registers

(4)

    data = readl(iommu->reg + reg +
            fault_index * PRIMARY_FAULT_REG_LEN + 12);
    if (!(data & DMA_FRCD_F))
        break;

    fault_reason = dma_frcd_fault_reason(data);
    type = dma_frcd_type(data);

    data = readl(iommu->reg + reg +
            fault_index * PRIMARY_FAULT_REG_LEN + 8);
    source_id = dma_frcd_source_id(data);

    guest_addr = dmar_readq(iommu->reg + reg +
            fault_index * PRIMARY_FAULT_REG_LEN);
    guest_addr = dma_frcd_page_addr(guest_addr);
    /* clear the fault */
    writel(DMA_FRCD_F, iommu->reg + reg +
        fault_index * PRIMARY_FAULT_REG_LEN + 12);

fault recording registers读取fault信息。需要注意的是,由于X86平台是小端模式,所以寄存器的高位内容会位于内存的高地址空间。另外,每读取完一个fault recording register信息,要把DMA_FRCD_F写回寄存器,用来表明软件已经读完了。

(5)dmar_fault_do_one是格式化打印的fault信息,其代码如下:

static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
        u8 fault_reason, u16 source_id, unsigned long long addr)
{
    const char *reason;
    int fault_type;

    reason = dmar_get_fault_reason(fault_reason, &fault_type);

    if (fault_type == INTR_REMAP)
        pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
               "fault index %llx\n"
            "INTR-REMAP:[fault reason %02d] %s\n",
            (source_id >> 8), PCI_SLOT(source_id & 0xFF),
            PCI_FUNC(source_id & 0xFF), addr >> 48,
            fault_reason, reason);
    else
        pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
               "fault addr %llx \n"
               "DMAR:[fault reason %02d] %s\n",
               (type ? "DMA Read" : "DMA Write"),
               (source_id >> 8), PCI_SLOT(source_id & 0xFF),
               PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
    return 0;
}

(6)

writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);

最后表明软件已处理完所有的fault信息。

Linux kernel IOMMU代码分析笔记(3)

看一下dmar_table_init这个函数:

int __init dmar_table_init(void)
{
    static int dmar_table_initialized;
    int ret;

    if (dmar_table_initialized == 0) {
        ret = parse_dmar_table();
        if (ret < 0) {
            if (ret != -ENODEV)
                pr_info("parse DMAR table failure.\n");
        } else  if (list_empty(&dmar_drhd_units)) {
            pr_info("No DMAR devices found\n");
            ret = -ENODEV;
        }

        if (ret < 0)
            dmar_table_initialized = ret;
        else
            dmar_table_initialized = 1;
    }

    return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
}

dmar_table_init其实就是调用了parse_dmar_table这个函数来解析 DMAR ACPI tableparse_dmar_table代码如下:

static int __init
parse_dmar_table(void)
{
    struct acpi_table_dmar *dmar;
    struct acpi_dmar_header *entry_header;
    int ret = 0;
    int drhd_count = 0;

    /*
     * Do it again, earlier dmar_tbl mapping could be mapped with
     * fixed map.
     */
    dmar_table_detect();

    /*
     * ACPI tables may not be DMA protected by tboot, so use DMAR copy
     * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
     */
    dmar_tbl = tboot_get_dmar_table(dmar_tbl);

    dmar = (struct acpi_table_dmar *)dmar_tbl;
    if (!dmar)
        return -ENODEV;

    if (dmar->width < PAGE_SHIFT - 1) {
        pr_warn("Invalid DMAR haw\n");
        return -EINVAL;
    }

    pr_info("Host address width %d\n", dmar->width + 1);

    entry_header = (struct acpi_dmar_header *)(dmar + 1);
    while (((unsigned long)entry_header) <
            (((unsigned long)dmar) + dmar_tbl->length)) {
        /* Avoid looping forever on bad ACPI tables */
        if (entry_header->length == 0) {
            pr_warn("Invalid 0-length structure\n");
            ret = -EINVAL;
            break;
        }

        dmar_table_print_dmar_entry(entry_header);

        switch (entry_header->type) {
        case ACPI_DMAR_TYPE_HARDWARE_UNIT:
            drhd_count++;
            ret = dmar_parse_one_drhd(entry_header);
            break;
        case ACPI_DMAR_TYPE_RESERVED_MEMORY:
            ret = dmar_parse_one_rmrr(entry_header);
            break;
        case ACPI_DMAR_TYPE_ATSR:
            ret = dmar_parse_one_atsr(entry_header);
            break;
        case ACPI_DMAR_HARDWARE_AFFINITY:
#ifdef CONFIG_ACPI_NUMA
            ret = dmar_parse_one_rhsa(entry_header);
#endif
            break;
        case ACPI_DMAR_TYPE_ANDD:
            ret = dmar_parse_one_andd(entry_header);
            break;
        default:
            pr_warn("Unknown DMAR structure type %d\n",
                entry_header->type);
            ret = 0; /* for forward compatibility */
            break;
        }
        if (ret)
            break;

        entry_header = ((void *)entry_header + entry_header->length);
    }
    if (drhd_count == 0)
        pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
    return ret;
}

parse_dmar_table函数中重要的工作有以下两点:
(1)检查host机器地址总线宽度:

if (dmar->width < PAGE_SHIFT - 1) {
        pr_warn("Invalid DMAR haw\n");
        return -EINVAL;
}

(2)遍历并解析DMAR table中的每一项,并加到相应的列表中:

while (((unsigned long)entry_header) <
            (((unsigned long)dmar) + dmar_tbl->length)) {
            ......
}

DMA Remapping Hardware单元为例:

static int __init
dmar_parse_one_drhd(struct acpi_dmar_header *header)
{
    struct acpi_dmar_hardware_unit *drhd;
    struct dmar_drhd_unit *dmaru;
    int ret = 0;

    drhd = (struct acpi_dmar_hardware_unit *)header;
    dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
    if (!dmaru)
        return -ENOMEM;

    dmaru->hdr = header;
    dmaru->reg_base_addr = drhd->address;
    dmaru->segment = drhd->segment;
    dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
    dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
                          ((void *)drhd) + drhd->header.length,
                          &dmaru->devices_cnt);
    if (dmaru->devices_cnt && dmaru->devices == NULL) {
        kfree(dmaru);
        return -ENOMEM;
    }

    ret = alloc_iommu(dmaru);
    if (ret) {
        dmar_free_dev_scope(&dmaru->devices,
                    &dmaru->devices_cnt);
        kfree(dmaru);
        return ret;
    }
    dmar_register_drhd_unit(dmaru);
    return 0;
}

最后dmar_register_drhd_unit这个函数会把解析好的DMA Remapping Hardware单元加到dmar_drhd_units这个列表中。

Linux kernel IOMMU代码分析笔记(2)

系统BIOS会检查平台的remapping硬件功能,并且会在主机的系统地址空间内定位memory-mapped remapping硬件寄存器。BIOS通过DMA Remapping Reporting (DMAR) ACPI table向系统软件报告remapping硬件单元信息。

除了RSDPFACS,所有的ACPI table定义都包含一个共同的header定义:

struct acpi_table_header {
    char signature[ACPI_NAME_SIZE]; /* ASCII table signature */
    u32 length;     /* Length of table in bytes, including this header */
    u8 revision;        /* ACPI Specification minor version number */
    u8 checksum;        /* To make sum of entire table == 0 */
    char oem_id[ACPI_OEM_ID_SIZE];  /* ASCII OEM identification */
    char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];  /* ASCII OEM table identification */
    u32 oem_revision;   /* OEM revision number */
    char asl_compiler_id[ACPI_NAME_SIZE];   /* ASCII ASL compiler vendor ID */
    u32 asl_compiler_revision;  /* ASL compiler version */
};

DMA Remapping table定义如下(可以看到包含有acpi_table_header):

struct acpi_table_dmar {
    struct acpi_table_header header;    /* Common ACPI table header */
    u8 width;       /* Host Address Width */
    u8 flags;
    u8 reserved[10];
};

对所有的DMA Remapping结构体,都会包含一个type和一个length

/* DMAR subtable header */

struct acpi_dmar_header {
    u16 type;
    u16 length;
};

DMA Remapping Hardware单元(类型为0)为例:

/* 0: Hardware Unit Definition */

struct acpi_dmar_hardware_unit {
    struct acpi_dmar_header header;
    u8 flags;
    u8 reserved;
    u16 segment;
    u64 address;        /* Register Base Address */
};

其它的还有acpi_dmar_reserved_memoryacpi_dmar_atsr等等定义。

Linux kernel IOMMU代码分析笔记(1)

Linux kernel代码版本是3.10

intel-iommu.h头文件定义了root-entry table address寄存器:

#define DMAR_RTADDR_REG 0x20    /* Root entry table */

DMAR_RTADDR_REG只在iommu_set_root_entry这个函数中使用(这个函数作用是更新root-entry table的地址):

static void iommu_set_root_entry(struct intel_iommu *iommu)
{
    void *addr;
    u32 sts;
    unsigned long flag;

    addr = iommu->root_entry;

    raw_spin_lock_irqsave(&iommu->register_lock, flag);
    dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));

    writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);

    /* Make sure hardware complete it */
    IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
              readl, (sts & DMA_GSTS_RTPS), sts);

    raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
}

DMAR_RTADDR_REG存储的是root-entry table的物理地址(virt_to_phys()函数把virtual address转成physical address)。整个root-entry table4KiB大小,并且要求起始地址是“页面对齐的(page-alignedpage长度是4KiB)”,所以DMAR_RTADDR_REG12 bits0。更新DMAR_RTADDR_REG代码如下:

dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));

更新完DMAR_RTADDR_REG寄存器,还要把global command寄存器的SRTPSet Root Table Pointer)位置1

writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);

最后读取Global Status寄存器的RTPSRoot Table Pointer Status)位,确认更新成功:

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
                  readl, (sts & DMA_GSTS_RTPS), sts);

printk函数简介

printk函数是Linux kernel开发中用来打印log的常用函数,同使用在用户空间程序的printf函数很类似。实现代码如下:

asmlinkage int printk(const char *fmt, ...)
{
    va_list args;
    int r;

#ifdef CONFIG_KGDB_KDB
    if (unlikely(kdb_trap_printk)) {
        va_start(args, fmt);
        r = vkdb_printf(fmt, args);
        va_end(args);
        return r;
    }
#endif
    va_start(args, fmt);
    r = vprintk_emit(0, -1, NULL, 0, fmt, args);
    va_end(args);

    return r;
}

举个例子(注意KERN_ALERT"DEBUG ..."之间没有逗号):

printk(KERN_ALERT "DEBUG: Passed %s %d \n",__FUNCTION__,__LINE__);

KERN_ALERT代表log的级别,参考kern_levels.h

#define KERN_EMERG  KERN_SOH "0"    /* system is unusable */
#define KERN_ALERT  KERN_SOH "1"    /* action must be taken immediately */
#define KERN_CRIT   KERN_SOH "2"    /* critical conditions */
#define KERN_ERR    KERN_SOH "3"    /* error conditions */
#define KERN_WARNING    KERN_SOH "4"    /* warning conditions */
#define KERN_NOTICE KERN_SOH "5"    /* normal but significant condition */
#define KERN_INFO   KERN_SOH "6"    /* informational */
#define KERN_DEBUG  KERN_SOH "7"    /* debug-level messages */ 

数字越大,代表优先级越低。

另外关于printk格式化字符串形式,参考printk-formats.txt

/proc/sys/kernel/printk这个文件显示了和printk相关的log级别设置:

[root@localhost kernel]# cat /proc/sys/kernel/printk
7       4       1       7

4个值的含义依次如下:
console_loglevel:当前consolelog级别,只有更高优先级的log才被允许打印到console
default_message_loglevel:当不指定log级别时,printk默认使用的log级别;
minimum_console_loglevelconsole能设定的最高log级别;
default_console_loglevel:默认的consolelog级别。

请看下面这个例子:

[root@localhost kernel]# echo 8 > /proc/sys/kernel/printk
[root@localhost kernel]# cat /proc/sys/kernel/printk
8       4       1       7

可以看到,console_loglevel的级别修改成了8

参考资料:
Debugging by printing

解决编译错误“fatal error: ‘libelf.h’ file not found”

原文发布于hellogcc网站。

最近在编译一个开源项目时,遇到这个编译错误:

fatal error: 'libelf.h' file not found
#include <libelf.h>
     ^
1 error generated.

解决方法是安装elfutils-libelf-devel这个软件包:

yum install elfutils-libelf-devel

或:

dnf install elfutils-libelf-devel

ODBC中的三种变量类型

无论是使用ODBC的新手或老兵,看到SQLCHARSOL_C_CHARSQL_CHAR,可能都会有些糊涂。它们之间的区别到底是什么呢?

(1)第一种是ODBC定义的数据类型(像SQLCHAR):SQL开头,后面跟着一系列大写字符,但是没有下划线。这种数据类型定义在sqltypes.h中,比如:

typedef signed short int   SQLSMALLINT;
typedef SQLSMALLINT     SQLRETURN;

#if (ODBCVER >= 0x0300)
typedef void *                  SQLHANDLE; 
......
#endif

这种数据类型可以用来定义变量,ODBC API的变量类型都是这种类型:

#if (ODBCVER >= 0x0300)
    SQLRETURN  SQL_API SQLAllocHandle(SQLSMALLINT HandleType,
                                      SQLHANDLE InputHandle, SQLHANDLE *OutputHandle);
#endif

这样做的好处是ODBC提供了一套自己的变量类型,相当于封装了一层,使用者不用关心底层具体的变量类型实现细节。即使底层的变量类型做了修改,使用者的代码也不用做任何改动。
这种数据类型和C语言数据类型的映射如下:

ODBC 类型 C 类型
SQLCHAR char
SQLSCHAR signed char
SQLINTEGER long int

(2)第二种是C语言的数据类型编码(SQL_C_CHAR)。他们并不是真正的数据类型,而是针对上面提到的的ODBC定义的数据类型的编码,所以不能用来定义变量。他们可以用作ODBC API的参数。这种数据类型定义在sqlext.h中,比如:

#define SQL_C_CHAR    SQL_CHAR             /* CHAR, VARCHAR, DECIMAL, NUMERIC */
#define SQL_C_LONG    SQL_INTEGER          /* INTEGER                      */
#define SQL_C_SHORT   SQL_SMALLINT         /* SMALLINT                     */
#define SQL_C_FLOAT   SQL_REAL             /* REAL                         */
#define SQL_C_DOUBLE  SQL_DOUBLE           /* FLOAT, DOUBLE                */

这种数据类型和ODBC数据类型的映射如下:

整数编码 ODBC 类型
SQL_C_CHAR SQLCHAR
SQL_S_STINYINT SQLSCHAR
SQL_C_SLONG SQLINTEGER

(3)第三种是SQL数据类型编码。同第二种一样,他们也不是真正的数据类型,而是提供SQL数据类型和编程语言数据类型的关联,因为也是数字编码,所以也不能用来定义变量。他们可以用作ODBC API的参数。这种数据类型定义在sqlext.h中,比如:

#define SQL_C_CHAR    SQL_CHAR             /* CHAR, VARCHAR, DECIMAL, NUMERIC */
#define SQL_C_LONG    SQL_INTEGER          /* INTEGER                      */
#define SQL_C_SHORT   SQL_SMALLINT         /* SMALLINT                     */
#define SQL_C_FLOAT   SQL_REAL             /* REAL                         */
#define SQL_C_DOUBLE  SQL_DOUBLE           /* FLOAT, DOUBLE                */

这种数据类型和SQL数据类型的映射如下:

整数编码 SQL类型
SQL_CHAR Char(n)
SQL_VARCHAR Varchar(n)
SQL_SMALLINT Smallint