[BACK]Return to patch-target_i386_nvmm_nvmm-all.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / pkgsrc / emulators / qemu / patches

Annotation of pkgsrc/emulators/qemu/patches/patch-target_i386_nvmm_nvmm-all.c, Revision 1.1

1.1     ! ryoon       1: $NetBSD$
        !             2:
        !             3: --- target/i386/nvmm/nvmm-all.c.orig   2021-05-06 05:09:24.911125954 +0000
        !             4: +++ target/i386/nvmm/nvmm-all.c
        !             5: @@ -0,0 +1,1226 @@
        !             6: +/*
        !             7: + * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
        !             8: + *
        !             9: + * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
        !            10: + *
        !            11: + * This work is licensed under the terms of the GNU GPL, version 2 or later.
        !            12: + * See the COPYING file in the top-level directory.
        !            13: + */
        !            14: +
        !            15: +#include "qemu/osdep.h"
        !            16: +#include "cpu.h"
        !            17: +#include "exec/address-spaces.h"
        !            18: +#include "exec/ioport.h"
        !            19: +#include "qemu-common.h"
        !            20: +#include "qemu/accel.h"
        !            21: +#include "sysemu/nvmm.h"
        !            22: +#include "sysemu/cpus.h"
        !            23: +#include "sysemu/runstate.h"
        !            24: +#include "qemu/main-loop.h"
        !            25: +#include "qemu/error-report.h"
        !            26: +#include "qapi/error.h"
        !            27: +#include "qemu/queue.h"
        !            28: +#include "migration/blocker.h"
        !            29: +#include "strings.h"
        !            30: +
        !            31: +#include "nvmm-accel-ops.h"
        !            32: +
        !            33: +#include <nvmm.h>
        !            34: +
        !            35: +struct qemu_vcpu {
        !            36: +    struct nvmm_vcpu vcpu;
        !            37: +    uint8_t tpr;
        !            38: +    bool stop;
        !            39: +
        !            40: +    /* Window-exiting for INTs/NMIs. */
        !            41: +    bool int_window_exit;
        !            42: +    bool nmi_window_exit;
        !            43: +
        !            44: +    /* The guest is in an interrupt shadow (POP SS, etc). */
        !            45: +    bool int_shadow;
        !            46: +};
        !            47: +
        !            48: +struct qemu_machine {
        !            49: +    struct nvmm_capability cap;
        !            50: +    struct nvmm_machine mach;
        !            51: +};
        !            52: +
        !            53: +/* -------------------------------------------------------------------------- */
        !            54: +
        !            55: +static bool nvmm_allowed;
        !            56: +static struct qemu_machine qemu_mach;
        !            57: +
        !            58: +static struct qemu_vcpu *
        !            59: +get_qemu_vcpu(CPUState *cpu)
        !            60: +{
        !            61: +    return (struct qemu_vcpu *)cpu->hax_vcpu;
        !            62: +}
        !            63: +
        !            64: +static struct nvmm_machine *
        !            65: +get_nvmm_mach(void)
        !            66: +{
        !            67: +    return &qemu_mach.mach;
        !            68: +}
        !            69: +
        !            70: +/* -------------------------------------------------------------------------- */
        !            71: +
        !            72: +static void
        !            73: +nvmm_set_segment(struct nvmm_x64_state_seg *nseg, const SegmentCache *qseg)
        !            74: +{
        !            75: +    uint32_t attrib = qseg->flags;
        !            76: +
        !            77: +    nseg->selector = qseg->selector;
        !            78: +    nseg->limit = qseg->limit;
        !            79: +    nseg->base = qseg->base;
        !            80: +    nseg->attrib.type = __SHIFTOUT(attrib, DESC_TYPE_MASK);
        !            81: +    nseg->attrib.s = __SHIFTOUT(attrib, DESC_S_MASK);
        !            82: +    nseg->attrib.dpl = __SHIFTOUT(attrib, DESC_DPL_MASK);
        !            83: +    nseg->attrib.p = __SHIFTOUT(attrib, DESC_P_MASK);
        !            84: +    nseg->attrib.avl = __SHIFTOUT(attrib, DESC_AVL_MASK);
        !            85: +    nseg->attrib.l = __SHIFTOUT(attrib, DESC_L_MASK);
        !            86: +    nseg->attrib.def = __SHIFTOUT(attrib, DESC_B_MASK);
        !            87: +    nseg->attrib.g = __SHIFTOUT(attrib, DESC_G_MASK);
        !            88: +}
        !            89: +
        !            90: +static void
        !            91: +nvmm_set_registers(CPUState *cpu)
        !            92: +{
        !            93: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !            94: +    struct nvmm_machine *mach = get_nvmm_mach();
        !            95: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !            96: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !            97: +    struct nvmm_x64_state *state = vcpu->state;
        !            98: +    uint64_t bitmap;
        !            99: +    size_t i;
        !           100: +    int ret;
        !           101: +
        !           102: +    assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
        !           103: +
        !           104: +    /* GPRs. */
        !           105: +    state->gprs[NVMM_X64_GPR_RAX] = env->regs[R_EAX];
        !           106: +    state->gprs[NVMM_X64_GPR_RCX] = env->regs[R_ECX];
        !           107: +    state->gprs[NVMM_X64_GPR_RDX] = env->regs[R_EDX];
        !           108: +    state->gprs[NVMM_X64_GPR_RBX] = env->regs[R_EBX];
        !           109: +    state->gprs[NVMM_X64_GPR_RSP] = env->regs[R_ESP];
        !           110: +    state->gprs[NVMM_X64_GPR_RBP] = env->regs[R_EBP];
        !           111: +    state->gprs[NVMM_X64_GPR_RSI] = env->regs[R_ESI];
        !           112: +    state->gprs[NVMM_X64_GPR_RDI] = env->regs[R_EDI];
        !           113: +#ifdef TARGET_X86_64
        !           114: +    state->gprs[NVMM_X64_GPR_R8]  = env->regs[R_R8];
        !           115: +    state->gprs[NVMM_X64_GPR_R9]  = env->regs[R_R9];
        !           116: +    state->gprs[NVMM_X64_GPR_R10] = env->regs[R_R10];
        !           117: +    state->gprs[NVMM_X64_GPR_R11] = env->regs[R_R11];
        !           118: +    state->gprs[NVMM_X64_GPR_R12] = env->regs[R_R12];
        !           119: +    state->gprs[NVMM_X64_GPR_R13] = env->regs[R_R13];
        !           120: +    state->gprs[NVMM_X64_GPR_R14] = env->regs[R_R14];
        !           121: +    state->gprs[NVMM_X64_GPR_R15] = env->regs[R_R15];
        !           122: +#endif
        !           123: +
        !           124: +    /* RIP and RFLAGS. */
        !           125: +    state->gprs[NVMM_X64_GPR_RIP] = env->eip;
        !           126: +    state->gprs[NVMM_X64_GPR_RFLAGS] = env->eflags;
        !           127: +
        !           128: +    /* Segments. */
        !           129: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_CS], &env->segs[R_CS]);
        !           130: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_DS], &env->segs[R_DS]);
        !           131: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_ES], &env->segs[R_ES]);
        !           132: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_FS], &env->segs[R_FS]);
        !           133: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_GS], &env->segs[R_GS]);
        !           134: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_SS], &env->segs[R_SS]);
        !           135: +
        !           136: +    /* Special segments. */
        !           137: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_GDT], &env->gdt);
        !           138: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_LDT], &env->ldt);
        !           139: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_TR], &env->tr);
        !           140: +    nvmm_set_segment(&state->segs[NVMM_X64_SEG_IDT], &env->idt);
        !           141: +
        !           142: +    /* Control registers. */
        !           143: +    state->crs[NVMM_X64_CR_CR0] = env->cr[0];
        !           144: +    state->crs[NVMM_X64_CR_CR2] = env->cr[2];
        !           145: +    state->crs[NVMM_X64_CR_CR3] = env->cr[3];
        !           146: +    state->crs[NVMM_X64_CR_CR4] = env->cr[4];
        !           147: +    state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
        !           148: +    state->crs[NVMM_X64_CR_XCR0] = env->xcr0;
        !           149: +
        !           150: +    /* Debug registers. */
        !           151: +    state->drs[NVMM_X64_DR_DR0] = env->dr[0];
        !           152: +    state->drs[NVMM_X64_DR_DR1] = env->dr[1];
        !           153: +    state->drs[NVMM_X64_DR_DR2] = env->dr[2];
        !           154: +    state->drs[NVMM_X64_DR_DR3] = env->dr[3];
        !           155: +    state->drs[NVMM_X64_DR_DR6] = env->dr[6];
        !           156: +    state->drs[NVMM_X64_DR_DR7] = env->dr[7];
        !           157: +
        !           158: +    /* FPU. */
        !           159: +    state->fpu.fx_cw = env->fpuc;
        !           160: +    state->fpu.fx_sw = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11);
        !           161: +    state->fpu.fx_tw = 0;
        !           162: +    for (i = 0; i < 8; i++) {
        !           163: +        state->fpu.fx_tw |= (!env->fptags[i]) << i;
        !           164: +    }
        !           165: +    state->fpu.fx_opcode = env->fpop;
        !           166: +    state->fpu.fx_ip.fa_64 = env->fpip;
        !           167: +    state->fpu.fx_dp.fa_64 = env->fpdp;
        !           168: +    state->fpu.fx_mxcsr = env->mxcsr;
        !           169: +    state->fpu.fx_mxcsr_mask = 0x0000FFFF;
        !           170: +    assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
        !           171: +    memcpy(state->fpu.fx_87_ac, env->fpregs, sizeof(env->fpregs));
        !           172: +    for (i = 0; i < CPU_NB_REGS; i++) {
        !           173: +        memcpy(&state->fpu.fx_xmm[i].xmm_bytes[0],
        !           174: +            &env->xmm_regs[i].ZMM_Q(0), 8);
        !           175: +        memcpy(&state->fpu.fx_xmm[i].xmm_bytes[8],
        !           176: +            &env->xmm_regs[i].ZMM_Q(1), 8);
        !           177: +    }
        !           178: +
        !           179: +    /* MSRs. */
        !           180: +    state->msrs[NVMM_X64_MSR_EFER] = env->efer;
        !           181: +    state->msrs[NVMM_X64_MSR_STAR] = env->star;
        !           182: +#ifdef TARGET_X86_64
        !           183: +    state->msrs[NVMM_X64_MSR_LSTAR] = env->lstar;
        !           184: +    state->msrs[NVMM_X64_MSR_CSTAR] = env->cstar;
        !           185: +    state->msrs[NVMM_X64_MSR_SFMASK] = env->fmask;
        !           186: +    state->msrs[NVMM_X64_MSR_KERNELGSBASE] = env->kernelgsbase;
        !           187: +#endif
        !           188: +    state->msrs[NVMM_X64_MSR_SYSENTER_CS]  = env->sysenter_cs;
        !           189: +    state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = env->sysenter_esp;
        !           190: +    state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = env->sysenter_eip;
        !           191: +    state->msrs[NVMM_X64_MSR_PAT] = env->pat;
        !           192: +    state->msrs[NVMM_X64_MSR_TSC] = env->tsc;
        !           193: +
        !           194: +    bitmap =
        !           195: +        NVMM_X64_STATE_SEGS |
        !           196: +        NVMM_X64_STATE_GPRS |
        !           197: +        NVMM_X64_STATE_CRS  |
        !           198: +        NVMM_X64_STATE_DRS  |
        !           199: +        NVMM_X64_STATE_MSRS |
        !           200: +        NVMM_X64_STATE_FPU;
        !           201: +
        !           202: +    ret = nvmm_vcpu_setstate(mach, vcpu, bitmap);
        !           203: +    if (ret == -1) {
        !           204: +        error_report("NVMM: Failed to set virtual processor context,"
        !           205: +            " error=%d", errno);
        !           206: +    }
        !           207: +}
        !           208: +
        !           209: +static void
        !           210: +nvmm_get_segment(SegmentCache *qseg, const struct nvmm_x64_state_seg *nseg)
        !           211: +{
        !           212: +    qseg->selector = nseg->selector;
        !           213: +    qseg->limit = nseg->limit;
        !           214: +    qseg->base = nseg->base;
        !           215: +
        !           216: +    qseg->flags =
        !           217: +        __SHIFTIN((uint32_t)nseg->attrib.type, DESC_TYPE_MASK) |
        !           218: +        __SHIFTIN((uint32_t)nseg->attrib.s, DESC_S_MASK) |
        !           219: +        __SHIFTIN((uint32_t)nseg->attrib.dpl, DESC_DPL_MASK) |
        !           220: +        __SHIFTIN((uint32_t)nseg->attrib.p, DESC_P_MASK) |
        !           221: +        __SHIFTIN((uint32_t)nseg->attrib.avl, DESC_AVL_MASK) |
        !           222: +        __SHIFTIN((uint32_t)nseg->attrib.l, DESC_L_MASK) |
        !           223: +        __SHIFTIN((uint32_t)nseg->attrib.def, DESC_B_MASK) |
        !           224: +        __SHIFTIN((uint32_t)nseg->attrib.g, DESC_G_MASK);
        !           225: +}
        !           226: +
        !           227: +static void
        !           228: +nvmm_get_registers(CPUState *cpu)
        !           229: +{
        !           230: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !           231: +    struct nvmm_machine *mach = get_nvmm_mach();
        !           232: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           233: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           234: +    X86CPU *x86_cpu = X86_CPU(cpu);
        !           235: +    struct nvmm_x64_state *state = vcpu->state;
        !           236: +    uint64_t bitmap, tpr;
        !           237: +    size_t i;
        !           238: +    int ret;
        !           239: +
        !           240: +    assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
        !           241: +
        !           242: +    bitmap =
        !           243: +        NVMM_X64_STATE_SEGS |
        !           244: +        NVMM_X64_STATE_GPRS |
        !           245: +        NVMM_X64_STATE_CRS  |
        !           246: +        NVMM_X64_STATE_DRS  |
        !           247: +        NVMM_X64_STATE_MSRS |
        !           248: +        NVMM_X64_STATE_FPU;
        !           249: +
        !           250: +    ret = nvmm_vcpu_getstate(mach, vcpu, bitmap);
        !           251: +    if (ret == -1) {
        !           252: +        error_report("NVMM: Failed to get virtual processor context,"
        !           253: +            " error=%d", errno);
        !           254: +    }
        !           255: +
        !           256: +    /* GPRs. */
        !           257: +    env->regs[R_EAX] = state->gprs[NVMM_X64_GPR_RAX];
        !           258: +    env->regs[R_ECX] = state->gprs[NVMM_X64_GPR_RCX];
        !           259: +    env->regs[R_EDX] = state->gprs[NVMM_X64_GPR_RDX];
        !           260: +    env->regs[R_EBX] = state->gprs[NVMM_X64_GPR_RBX];
        !           261: +    env->regs[R_ESP] = state->gprs[NVMM_X64_GPR_RSP];
        !           262: +    env->regs[R_EBP] = state->gprs[NVMM_X64_GPR_RBP];
        !           263: +    env->regs[R_ESI] = state->gprs[NVMM_X64_GPR_RSI];
        !           264: +    env->regs[R_EDI] = state->gprs[NVMM_X64_GPR_RDI];
        !           265: +#ifdef TARGET_X86_64
        !           266: +    env->regs[R_R8]  = state->gprs[NVMM_X64_GPR_R8];
        !           267: +    env->regs[R_R9]  = state->gprs[NVMM_X64_GPR_R9];
        !           268: +    env->regs[R_R10] = state->gprs[NVMM_X64_GPR_R10];
        !           269: +    env->regs[R_R11] = state->gprs[NVMM_X64_GPR_R11];
        !           270: +    env->regs[R_R12] = state->gprs[NVMM_X64_GPR_R12];
        !           271: +    env->regs[R_R13] = state->gprs[NVMM_X64_GPR_R13];
        !           272: +    env->regs[R_R14] = state->gprs[NVMM_X64_GPR_R14];
        !           273: +    env->regs[R_R15] = state->gprs[NVMM_X64_GPR_R15];
        !           274: +#endif
        !           275: +
        !           276: +    /* RIP and RFLAGS. */
        !           277: +    env->eip = state->gprs[NVMM_X64_GPR_RIP];
        !           278: +    env->eflags = state->gprs[NVMM_X64_GPR_RFLAGS];
        !           279: +
        !           280: +    /* Segments. */
        !           281: +    nvmm_get_segment(&env->segs[R_ES], &state->segs[NVMM_X64_SEG_ES]);
        !           282: +    nvmm_get_segment(&env->segs[R_CS], &state->segs[NVMM_X64_SEG_CS]);
        !           283: +    nvmm_get_segment(&env->segs[R_SS], &state->segs[NVMM_X64_SEG_SS]);
        !           284: +    nvmm_get_segment(&env->segs[R_DS], &state->segs[NVMM_X64_SEG_DS]);
        !           285: +    nvmm_get_segment(&env->segs[R_FS], &state->segs[NVMM_X64_SEG_FS]);
        !           286: +    nvmm_get_segment(&env->segs[R_GS], &state->segs[NVMM_X64_SEG_GS]);
        !           287: +
        !           288: +    /* Special segments. */
        !           289: +    nvmm_get_segment(&env->gdt, &state->segs[NVMM_X64_SEG_GDT]);
        !           290: +    nvmm_get_segment(&env->ldt, &state->segs[NVMM_X64_SEG_LDT]);
        !           291: +    nvmm_get_segment(&env->tr, &state->segs[NVMM_X64_SEG_TR]);
        !           292: +    nvmm_get_segment(&env->idt, &state->segs[NVMM_X64_SEG_IDT]);
        !           293: +
        !           294: +    /* Control registers. */
        !           295: +    env->cr[0] = state->crs[NVMM_X64_CR_CR0];
        !           296: +    env->cr[2] = state->crs[NVMM_X64_CR_CR2];
        !           297: +    env->cr[3] = state->crs[NVMM_X64_CR_CR3];
        !           298: +    env->cr[4] = state->crs[NVMM_X64_CR_CR4];
        !           299: +    tpr = state->crs[NVMM_X64_CR_CR8];
        !           300: +    if (tpr != qcpu->tpr) {
        !           301: +        qcpu->tpr = tpr;
        !           302: +        cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
        !           303: +    }
        !           304: +    env->xcr0 = state->crs[NVMM_X64_CR_XCR0];
        !           305: +
        !           306: +    /* Debug registers. */
        !           307: +    env->dr[0] = state->drs[NVMM_X64_DR_DR0];
        !           308: +    env->dr[1] = state->drs[NVMM_X64_DR_DR1];
        !           309: +    env->dr[2] = state->drs[NVMM_X64_DR_DR2];
        !           310: +    env->dr[3] = state->drs[NVMM_X64_DR_DR3];
        !           311: +    env->dr[6] = state->drs[NVMM_X64_DR_DR6];
        !           312: +    env->dr[7] = state->drs[NVMM_X64_DR_DR7];
        !           313: +
        !           314: +    /* FPU. */
        !           315: +    env->fpuc = state->fpu.fx_cw;
        !           316: +    env->fpstt = (state->fpu.fx_sw >> 11) & 0x7;
        !           317: +    env->fpus = state->fpu.fx_sw & ~0x3800;
        !           318: +    for (i = 0; i < 8; i++) {
        !           319: +        env->fptags[i] = !((state->fpu.fx_tw >> i) & 1);
        !           320: +    }
        !           321: +    env->fpop = state->fpu.fx_opcode;
        !           322: +    env->fpip = state->fpu.fx_ip.fa_64;
        !           323: +    env->fpdp = state->fpu.fx_dp.fa_64;
        !           324: +    env->mxcsr = state->fpu.fx_mxcsr;
        !           325: +    assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
        !           326: +    memcpy(env->fpregs, state->fpu.fx_87_ac, sizeof(env->fpregs));
        !           327: +    for (i = 0; i < CPU_NB_REGS; i++) {
        !           328: +        memcpy(&env->xmm_regs[i].ZMM_Q(0),
        !           329: +            &state->fpu.fx_xmm[i].xmm_bytes[0], 8);
        !           330: +        memcpy(&env->xmm_regs[i].ZMM_Q(1),
        !           331: +            &state->fpu.fx_xmm[i].xmm_bytes[8], 8);
        !           332: +    }
        !           333: +
        !           334: +    /* MSRs. */
        !           335: +    env->efer = state->msrs[NVMM_X64_MSR_EFER];
        !           336: +    env->star = state->msrs[NVMM_X64_MSR_STAR];
        !           337: +#ifdef TARGET_X86_64
        !           338: +    env->lstar = state->msrs[NVMM_X64_MSR_LSTAR];
        !           339: +    env->cstar = state->msrs[NVMM_X64_MSR_CSTAR];
        !           340: +    env->fmask = state->msrs[NVMM_X64_MSR_SFMASK];
        !           341: +    env->kernelgsbase = state->msrs[NVMM_X64_MSR_KERNELGSBASE];
        !           342: +#endif
        !           343: +    env->sysenter_cs  = state->msrs[NVMM_X64_MSR_SYSENTER_CS];
        !           344: +    env->sysenter_esp = state->msrs[NVMM_X64_MSR_SYSENTER_ESP];
        !           345: +    env->sysenter_eip = state->msrs[NVMM_X64_MSR_SYSENTER_EIP];
        !           346: +    env->pat = state->msrs[NVMM_X64_MSR_PAT];
        !           347: +    env->tsc = state->msrs[NVMM_X64_MSR_TSC];
        !           348: +
        !           349: +    x86_update_hflags(env);
        !           350: +}
        !           351: +
        !           352: +static bool
        !           353: +nvmm_can_take_int(CPUState *cpu)
        !           354: +{
        !           355: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !           356: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           357: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           358: +    struct nvmm_machine *mach = get_nvmm_mach();
        !           359: +
        !           360: +    if (qcpu->int_window_exit) {
        !           361: +        return false;
        !           362: +    }
        !           363: +
        !           364: +    if (qcpu->int_shadow || !(env->eflags & IF_MASK)) {
        !           365: +        struct nvmm_x64_state *state = vcpu->state;
        !           366: +
        !           367: +        /* Exit on interrupt window. */
        !           368: +        nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_INTR);
        !           369: +        state->intr.int_window_exiting = 1;
        !           370: +        nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_INTR);
        !           371: +
        !           372: +        return false;
        !           373: +    }
        !           374: +
        !           375: +    return true;
        !           376: +}
        !           377: +
        !           378: +static bool
        !           379: +nvmm_can_take_nmi(CPUState *cpu)
        !           380: +{
        !           381: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           382: +
        !           383: +    /*
        !           384: +     * Contrary to INTs, NMIs always schedule an exit when they are
        !           385: +     * completed. Therefore, if window-exiting is enabled, it means
        !           386: +     * NMIs are blocked.
        !           387: +     */
        !           388: +    if (qcpu->nmi_window_exit) {
        !           389: +        return false;
        !           390: +    }
        !           391: +
        !           392: +    return true;
        !           393: +}
        !           394: +
        !           395: +/*
        !           396: + * Called before the VCPU is run. We inject events generated by the I/O
        !           397: + * thread, and synchronize the guest TPR.
        !           398: + */
        !           399: +static void
        !           400: +nvmm_vcpu_pre_run(CPUState *cpu)
        !           401: +{
        !           402: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !           403: +    struct nvmm_machine *mach = get_nvmm_mach();
        !           404: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           405: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           406: +    X86CPU *x86_cpu = X86_CPU(cpu);
        !           407: +    struct nvmm_x64_state *state = vcpu->state;
        !           408: +    struct nvmm_vcpu_event *event = vcpu->event;
        !           409: +    bool has_event = false;
        !           410: +    bool sync_tpr = false;
        !           411: +    uint8_t tpr;
        !           412: +    int ret;
        !           413: +
        !           414: +    qemu_mutex_lock_iothread();
        !           415: +
        !           416: +    tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
        !           417: +    if (tpr != qcpu->tpr) {
        !           418: +        qcpu->tpr = tpr;
        !           419: +        sync_tpr = true;
        !           420: +    }
        !           421: +
        !           422: +    /*
        !           423: +     * Force the VCPU out of its inner loop to process any INIT requests
        !           424: +     * or commit pending TPR access.
        !           425: +     */
        !           426: +    if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
        !           427: +        cpu->exit_request = 1;
        !           428: +    }
        !           429: +
        !           430: +    if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
        !           431: +        if (nvmm_can_take_nmi(cpu)) {
        !           432: +            cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
        !           433: +            event->type = NVMM_VCPU_EVENT_INTR;
        !           434: +            event->vector = 2;
        !           435: +            has_event = true;
        !           436: +        }
        !           437: +    }
        !           438: +
        !           439: +    if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
        !           440: +        if (nvmm_can_take_int(cpu)) {
        !           441: +            cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
        !           442: +            event->type = NVMM_VCPU_EVENT_INTR;
        !           443: +            event->vector = cpu_get_pic_interrupt(env);
        !           444: +            has_event = true;
        !           445: +        }
        !           446: +    }
        !           447: +
        !           448: +    /* Don't want SMIs. */
        !           449: +    if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
        !           450: +        cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
        !           451: +    }
        !           452: +
        !           453: +    if (sync_tpr) {
        !           454: +        ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_CRS);
        !           455: +        if (ret == -1) {
        !           456: +            error_report("NVMM: Failed to get CPU state,"
        !           457: +                " error=%d", errno);
        !           458: +        }
        !           459: +
        !           460: +        state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
        !           461: +
        !           462: +        ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_CRS);
        !           463: +        if (ret == -1) {
        !           464: +            error_report("NVMM: Failed to set CPU state,"
        !           465: +                " error=%d", errno);
        !           466: +        }
        !           467: +    }
        !           468: +
        !           469: +    if (has_event) {
        !           470: +        ret = nvmm_vcpu_inject(mach, vcpu);
        !           471: +        if (ret == -1) {
        !           472: +            error_report("NVMM: Failed to inject event,"
        !           473: +                " error=%d", errno);
        !           474: +        }
        !           475: +    }
        !           476: +
        !           477: +    qemu_mutex_unlock_iothread();
        !           478: +}
        !           479: +
        !           480: +/*
        !           481: + * Called after the VCPU ran. We synchronize the host view of the TPR and
        !           482: + * RFLAGS.
        !           483: + */
        !           484: +static void
        !           485: +nvmm_vcpu_post_run(CPUState *cpu, struct nvmm_vcpu_exit *exit)
        !           486: +{
        !           487: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           488: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !           489: +    X86CPU *x86_cpu = X86_CPU(cpu);
        !           490: +    uint64_t tpr;
        !           491: +
        !           492: +    env->eflags = exit->exitstate.rflags;
        !           493: +    qcpu->int_shadow = exit->exitstate.int_shadow;
        !           494: +    qcpu->int_window_exit = exit->exitstate.int_window_exiting;
        !           495: +    qcpu->nmi_window_exit = exit->exitstate.nmi_window_exiting;
        !           496: +
        !           497: +    tpr = exit->exitstate.cr8;
        !           498: +    if (qcpu->tpr != tpr) {
        !           499: +        qcpu->tpr = tpr;
        !           500: +        qemu_mutex_lock_iothread();
        !           501: +        cpu_set_apic_tpr(x86_cpu->apic_state, qcpu->tpr);
        !           502: +        qemu_mutex_unlock_iothread();
        !           503: +    }
        !           504: +}
        !           505: +
        !           506: +/* -------------------------------------------------------------------------- */
        !           507: +
        !           508: +static void
        !           509: +nvmm_io_callback(struct nvmm_io *io)
        !           510: +{
        !           511: +    MemTxAttrs attrs = { 0 };
        !           512: +    int ret;
        !           513: +
        !           514: +    ret = address_space_rw(&address_space_io, io->port, attrs, io->data,
        !           515: +        io->size, !io->in);
        !           516: +    if (ret != MEMTX_OK) {
        !           517: +        error_report("NVMM: I/O Transaction Failed "
        !           518: +            "[%s, port=%u, size=%zu]", (io->in ? "in" : "out"),
        !           519: +            io->port, io->size);
        !           520: +    }
        !           521: +
        !           522: +    /* Needed, otherwise infinite loop. */
        !           523: +    current_cpu->vcpu_dirty = false;
        !           524: +}
        !           525: +
        !           526: +static void
        !           527: +nvmm_mem_callback(struct nvmm_mem *mem)
        !           528: +{
        !           529: +    cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write);
        !           530: +
        !           531: +    /* Needed, otherwise infinite loop. */
        !           532: +    current_cpu->vcpu_dirty = false;
        !           533: +}
        !           534: +
        !           535: +static struct nvmm_assist_callbacks nvmm_callbacks = {
        !           536: +    .io = nvmm_io_callback,
        !           537: +    .mem = nvmm_mem_callback
        !           538: +};
        !           539: +
        !           540: +/* -------------------------------------------------------------------------- */
        !           541: +
        !           542: +static int
        !           543: +nvmm_handle_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
        !           544: +{
        !           545: +    int ret;
        !           546: +
        !           547: +    ret = nvmm_assist_mem(mach, vcpu);
        !           548: +    if (ret == -1) {
        !           549: +        error_report("NVMM: Mem Assist Failed [gpa=%p]",
        !           550: +            (void *)vcpu->exit->u.mem.gpa);
        !           551: +    }
        !           552: +
        !           553: +    return ret;
        !           554: +}
        !           555: +
        !           556: +static int
        !           557: +nvmm_handle_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
        !           558: +{
        !           559: +    int ret;
        !           560: +
        !           561: +    ret = nvmm_assist_io(mach, vcpu);
        !           562: +    if (ret == -1) {
        !           563: +        error_report("NVMM: I/O Assist Failed [port=%d]",
        !           564: +            (int)vcpu->exit->u.io.port);
        !           565: +    }
        !           566: +
        !           567: +    return ret;
        !           568: +}
        !           569: +
        !           570: +static int
        !           571: +nvmm_handle_rdmsr(struct nvmm_machine *mach, CPUState *cpu,
        !           572: +    struct nvmm_vcpu_exit *exit)
        !           573: +{
        !           574: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           575: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           576: +    X86CPU *x86_cpu = X86_CPU(cpu);
        !           577: +    struct nvmm_x64_state *state = vcpu->state;
        !           578: +    uint64_t val;
        !           579: +    int ret;
        !           580: +
        !           581: +    switch (exit->u.rdmsr.msr) {
        !           582: +    case MSR_IA32_APICBASE:
        !           583: +        val = cpu_get_apic_base(x86_cpu->apic_state);
        !           584: +        break;
        !           585: +    case MSR_MTRRcap:
        !           586: +    case MSR_MTRRdefType:
        !           587: +    case MSR_MCG_CAP:
        !           588: +    case MSR_MCG_STATUS:
        !           589: +        val = 0;
        !           590: +        break;
        !           591: +    default: /* More MSRs to add? */
        !           592: +        val = 0;
        !           593: +        error_report("NVMM: Unexpected RDMSR 0x%x, ignored",
        !           594: +            exit->u.rdmsr.msr);
        !           595: +        break;
        !           596: +    }
        !           597: +
        !           598: +    ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
        !           599: +    if (ret == -1) {
        !           600: +        return -1;
        !           601: +    }
        !           602: +
        !           603: +    state->gprs[NVMM_X64_GPR_RAX] = (val & 0xFFFFFFFF);
        !           604: +    state->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
        !           605: +    state->gprs[NVMM_X64_GPR_RIP] = exit->u.rdmsr.npc;
        !           606: +
        !           607: +    ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
        !           608: +    if (ret == -1) {
        !           609: +        return -1;
        !           610: +    }
        !           611: +
        !           612: +    return 0;
        !           613: +}
        !           614: +
        !           615: +static int
        !           616: +nvmm_handle_wrmsr(struct nvmm_machine *mach, CPUState *cpu,
        !           617: +    struct nvmm_vcpu_exit *exit)
        !           618: +{
        !           619: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           620: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           621: +    X86CPU *x86_cpu = X86_CPU(cpu);
        !           622: +    struct nvmm_x64_state *state = vcpu->state;
        !           623: +    uint64_t val;
        !           624: +    int ret;
        !           625: +
        !           626: +    val = exit->u.wrmsr.val;
        !           627: +
        !           628: +    switch (exit->u.wrmsr.msr) {
        !           629: +    case MSR_IA32_APICBASE:
        !           630: +        cpu_set_apic_base(x86_cpu->apic_state, val);
        !           631: +        break;
        !           632: +    case MSR_MTRRdefType:
        !           633: +    case MSR_MCG_STATUS:
        !           634: +        break;
        !           635: +    default: /* More MSRs to add? */
        !           636: +        error_report("NVMM: Unexpected WRMSR 0x%x [val=0x%lx], ignored",
        !           637: +            exit->u.wrmsr.msr, val);
        !           638: +        break;
        !           639: +    }
        !           640: +
        !           641: +    ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
        !           642: +    if (ret == -1) {
        !           643: +        return -1;
        !           644: +    }
        !           645: +
        !           646: +    state->gprs[NVMM_X64_GPR_RIP] = exit->u.wrmsr.npc;
        !           647: +
        !           648: +    ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
        !           649: +    if (ret == -1) {
        !           650: +        return -1;
        !           651: +    }
        !           652: +
        !           653: +    return 0;
        !           654: +}
        !           655: +
        !           656: +static int
        !           657: +nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu,
        !           658: +    struct nvmm_vcpu_exit *exit)
        !           659: +{
        !           660: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !           661: +    int ret = 0;
        !           662: +
        !           663: +    qemu_mutex_lock_iothread();
        !           664: +
        !           665: +    if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
        !           666: +          (env->eflags & IF_MASK)) &&
        !           667: +        !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
        !           668: +        cpu->exception_index = EXCP_HLT;
        !           669: +        cpu->halted = true;
        !           670: +        ret = 1;
        !           671: +    }
        !           672: +
        !           673: +    qemu_mutex_unlock_iothread();
        !           674: +
        !           675: +    return ret;
        !           676: +}
        !           677: +
        !           678: +static int
        !           679: +nvmm_inject_ud(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
        !           680: +{
        !           681: +    struct nvmm_vcpu_event *event = vcpu->event;
        !           682: +
        !           683: +    event->type = NVMM_VCPU_EVENT_EXCP;
        !           684: +    event->vector = 6;
        !           685: +    event->u.excp.error = 0;
        !           686: +
        !           687: +    return nvmm_vcpu_inject(mach, vcpu);
        !           688: +}
        !           689: +
        !           690: +static int
        !           691: +nvmm_vcpu_loop(CPUState *cpu)
        !           692: +{
        !           693: +    struct CPUX86State *env = (CPUArchState *)cpu->env_ptr;
        !           694: +    struct nvmm_machine *mach = get_nvmm_mach();
        !           695: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !           696: +    struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           697: +    X86CPU *x86_cpu = X86_CPU(cpu);
        !           698: +    struct nvmm_vcpu_exit *exit = vcpu->exit;
        !           699: +    int ret;
        !           700: +
        !           701: +    /*
        !           702: +     * Some asynchronous events must be handled outside of the inner
        !           703: +     * VCPU loop. They are handled here.
        !           704: +     */
        !           705: +    if (cpu->interrupt_request & CPU_INTERRUPT_INIT) {
        !           706: +        nvmm_cpu_synchronize_state(cpu);
        !           707: +        do_cpu_init(x86_cpu);
        !           708: +        /* set int/nmi windows back to the reset state */
        !           709: +    }
        !           710: +    if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
        !           711: +        cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
        !           712: +        apic_poll_irq(x86_cpu->apic_state);
        !           713: +    }
        !           714: +    if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
        !           715: +         (env->eflags & IF_MASK)) ||
        !           716: +        (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
        !           717: +        cpu->halted = false;
        !           718: +    }
        !           719: +    if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
        !           720: +        nvmm_cpu_synchronize_state(cpu);
        !           721: +        do_cpu_sipi(x86_cpu);
        !           722: +    }
        !           723: +    if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
        !           724: +        cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
        !           725: +        nvmm_cpu_synchronize_state(cpu);
        !           726: +        apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
        !           727: +            env->tpr_access_type);
        !           728: +    }
        !           729: +
        !           730: +    if (cpu->halted) {
        !           731: +        cpu->exception_index = EXCP_HLT;
        !           732: +        qatomic_set(&cpu->exit_request, false);
        !           733: +        return 0;
        !           734: +    }
        !           735: +
        !           736: +    qemu_mutex_unlock_iothread();
        !           737: +    cpu_exec_start(cpu);
        !           738: +
        !           739: +    /*
        !           740: +     * Inner VCPU loop.
        !           741: +     */
        !           742: +    do {
        !           743: +        if (cpu->vcpu_dirty) {
        !           744: +            nvmm_set_registers(cpu);
        !           745: +            cpu->vcpu_dirty = false;
        !           746: +        }
        !           747: +
        !           748: +        if (qcpu->stop) {
        !           749: +            cpu->exception_index = EXCP_INTERRUPT;
        !           750: +            qcpu->stop = false;
        !           751: +            ret = 1;
        !           752: +            break;
        !           753: +        }
        !           754: +
        !           755: +        nvmm_vcpu_pre_run(cpu);
        !           756: +
        !           757: +        if (qatomic_read(&cpu->exit_request)) {
        !           758: +            nvmm_vcpu_stop(vcpu);
        !           759: +        }
        !           760: +
        !           761: +        /* Read exit_request before the kernel reads the immediate exit flag */
        !           762: +        smp_rmb();
        !           763: +        ret = nvmm_vcpu_run(mach, vcpu);
        !           764: +        if (ret == -1) {
        !           765: +            error_report("NVMM: Failed to exec a virtual processor,"
        !           766: +                " error=%d", errno);
        !           767: +            break;
        !           768: +        }
        !           769: +
        !           770: +        nvmm_vcpu_post_run(cpu, exit);
        !           771: +
        !           772: +        switch (exit->reason) {
        !           773: +        case NVMM_VCPU_EXIT_NONE:
        !           774: +            break;
        !           775: +        case NVMM_VCPU_EXIT_STOPPED:
        !           776: +            /*
        !           777: +             * The kernel cleared the immediate exit flag; cpu->exit_request
        !           778: +             * must be cleared after
        !           779: +             */
        !           780: +            smp_wmb();
        !           781: +            qcpu->stop = true;
        !           782: +            break;
        !           783: +        case NVMM_VCPU_EXIT_MEMORY:
        !           784: +            ret = nvmm_handle_mem(mach, vcpu);
        !           785: +            break;
        !           786: +        case NVMM_VCPU_EXIT_IO:
        !           787: +            ret = nvmm_handle_io(mach, vcpu);
        !           788: +            break;
        !           789: +        case NVMM_VCPU_EXIT_INT_READY:
        !           790: +        case NVMM_VCPU_EXIT_NMI_READY:
        !           791: +        case NVMM_VCPU_EXIT_TPR_CHANGED:
        !           792: +            break;
        !           793: +        case NVMM_VCPU_EXIT_HALTED:
        !           794: +            ret = nvmm_handle_halted(mach, cpu, exit);
        !           795: +            break;
        !           796: +        case NVMM_VCPU_EXIT_SHUTDOWN:
        !           797: +            qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
        !           798: +            cpu->exception_index = EXCP_INTERRUPT;
        !           799: +            ret = 1;
        !           800: +            break;
        !           801: +        case NVMM_VCPU_EXIT_RDMSR:
        !           802: +            ret = nvmm_handle_rdmsr(mach, cpu, exit);
        !           803: +            break;
        !           804: +        case NVMM_VCPU_EXIT_WRMSR:
        !           805: +            ret = nvmm_handle_wrmsr(mach, cpu, exit);
        !           806: +            break;
        !           807: +        case NVMM_VCPU_EXIT_MONITOR:
        !           808: +        case NVMM_VCPU_EXIT_MWAIT:
        !           809: +            ret = nvmm_inject_ud(mach, vcpu);
        !           810: +            break;
        !           811: +        default:
        !           812: +            error_report("NVMM: Unexpected VM exit code 0x%lx [hw=0x%lx]",
        !           813: +                exit->reason, exit->u.inv.hwcode);
        !           814: +            nvmm_get_registers(cpu);
        !           815: +            qemu_mutex_lock_iothread();
        !           816: +            qemu_system_guest_panicked(cpu_get_crash_info(cpu));
        !           817: +            qemu_mutex_unlock_iothread();
        !           818: +            ret = -1;
        !           819: +            break;
        !           820: +        }
        !           821: +    } while (ret == 0);
        !           822: +
        !           823: +    cpu_exec_end(cpu);
        !           824: +    qemu_mutex_lock_iothread();
        !           825: +
        !           826: +    qatomic_set(&cpu->exit_request, false);
        !           827: +
        !           828: +    return ret < 0;
        !           829: +}
        !           830: +
        !           831: +/* -------------------------------------------------------------------------- */
        !           832: +
        !           833: +static void
        !           834: +do_nvmm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
        !           835: +{
        !           836: +    nvmm_get_registers(cpu);
        !           837: +    cpu->vcpu_dirty = true;
        !           838: +}
        !           839: +
        !           840: +static void
        !           841: +do_nvmm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
        !           842: +{
        !           843: +    nvmm_set_registers(cpu);
        !           844: +    cpu->vcpu_dirty = false;
        !           845: +}
        !           846: +
        !           847: +static void
        !           848: +do_nvmm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
        !           849: +{
        !           850: +    nvmm_set_registers(cpu);
        !           851: +    cpu->vcpu_dirty = false;
        !           852: +}
        !           853: +
        !           854: +static void
        !           855: +do_nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
        !           856: +{
        !           857: +    cpu->vcpu_dirty = true;
        !           858: +}
        !           859: +
        !           860: +void nvmm_cpu_synchronize_state(CPUState *cpu)
        !           861: +{
        !           862: +    if (!cpu->vcpu_dirty) {
        !           863: +        run_on_cpu(cpu, do_nvmm_cpu_synchronize_state, RUN_ON_CPU_NULL);
        !           864: +    }
        !           865: +}
        !           866: +
        !           867: +void nvmm_cpu_synchronize_post_reset(CPUState *cpu)
        !           868: +{
        !           869: +    run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
        !           870: +}
        !           871: +
        !           872: +void nvmm_cpu_synchronize_post_init(CPUState *cpu)
        !           873: +{
        !           874: +    run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
        !           875: +}
        !           876: +
        !           877: +void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu)
        !           878: +{
        !           879: +    run_on_cpu(cpu, do_nvmm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
        !           880: +}
        !           881: +
        !           882: +/* -------------------------------------------------------------------------- */
        !           883: +
        !           884: +static Error *nvmm_migration_blocker;
        !           885: +
        !           886: +/*
        !           887: + * The nvmm_vcpu_stop() mechanism breaks races between entering the VMM
        !           888: + * and another thread signaling the vCPU thread to exit.
        !           889: + */
        !           890: +
        !           891: +static void
        !           892: +nvmm_ipi_signal(int sigcpu)
        !           893: +{
        !           894: +    if (current_cpu) {
        !           895: +        struct qemu_vcpu *qcpu = get_qemu_vcpu(current_cpu);
        !           896: +        struct nvmm_vcpu *vcpu = &qcpu->vcpu;
        !           897: +        nvmm_vcpu_stop(vcpu);
        !           898: +    }
        !           899: +}
        !           900: +
        !           901: +static void
        !           902: +nvmm_init_cpu_signals(void)
        !           903: +{
        !           904: +    struct sigaction sigact;
        !           905: +    sigset_t set;
        !           906: +
        !           907: +    /* Install the IPI handler. */
        !           908: +    memset(&sigact, 0, sizeof(sigact));
        !           909: +    sigact.sa_handler = nvmm_ipi_signal;
        !           910: +    sigaction(SIG_IPI, &sigact, NULL);
        !           911: +
        !           912: +    /* Allow IPIs on the current thread. */
        !           913: +    sigprocmask(SIG_BLOCK, NULL, &set);
        !           914: +    sigdelset(&set, SIG_IPI);
        !           915: +    pthread_sigmask(SIG_SETMASK, &set, NULL);
        !           916: +}
        !           917: +
        !           918: +int
        !           919: +nvmm_init_vcpu(CPUState *cpu)
        !           920: +{
        !           921: +    struct nvmm_machine *mach = get_nvmm_mach();
        !           922: +    struct nvmm_vcpu_conf_cpuid cpuid;
        !           923: +    struct nvmm_vcpu_conf_tpr tpr;
        !           924: +    Error *local_error = NULL;
        !           925: +    struct qemu_vcpu *qcpu;
        !           926: +    int ret, err;
        !           927: +
        !           928: +    nvmm_init_cpu_signals();
        !           929: +
        !           930: +    if (nvmm_migration_blocker == NULL) {
        !           931: +        error_setg(&nvmm_migration_blocker,
        !           932: +            "NVMM: Migration not supported");
        !           933: +
        !           934: +        (void)migrate_add_blocker(nvmm_migration_blocker, &local_error);
        !           935: +        if (local_error) {
        !           936: +            error_report_err(local_error);
        !           937: +            migrate_del_blocker(nvmm_migration_blocker);
        !           938: +            error_free(nvmm_migration_blocker);
        !           939: +            return -EINVAL;
        !           940: +        }
        !           941: +    }
        !           942: +
        !           943: +    qcpu = g_malloc0(sizeof(*qcpu));
        !           944: +    if (qcpu == NULL) {
        !           945: +        error_report("NVMM: Failed to allocate VCPU context.");
        !           946: +        return -ENOMEM;
        !           947: +    }
        !           948: +
        !           949: +    ret = nvmm_vcpu_create(mach, cpu->cpu_index, &qcpu->vcpu);
        !           950: +    if (ret == -1) {
        !           951: +        err = errno;
        !           952: +        error_report("NVMM: Failed to create a virtual processor,"
        !           953: +            " error=%d", err);
        !           954: +        g_free(qcpu);
        !           955: +        return -err;
        !           956: +    }
        !           957: +
        !           958: +    memset(&cpuid, 0, sizeof(cpuid));
        !           959: +    cpuid.mask = 1;
        !           960: +    cpuid.leaf = 0x00000001;
        !           961: +    cpuid.u.mask.set.edx = CPUID_MCE | CPUID_MCA | CPUID_MTRR;
        !           962: +    ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CPUID,
        !           963: +        &cpuid);
        !           964: +    if (ret == -1) {
        !           965: +        err = errno;
        !           966: +        error_report("NVMM: Failed to configure a virtual processor,"
        !           967: +            " error=%d", err);
        !           968: +        g_free(qcpu);
        !           969: +        return -err;
        !           970: +    }
        !           971: +
        !           972: +    ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CALLBACKS,
        !           973: +        &nvmm_callbacks);
        !           974: +    if (ret == -1) {
        !           975: +        err = errno;
        !           976: +        error_report("NVMM: Failed to configure a virtual processor,"
        !           977: +            " error=%d", err);
        !           978: +        g_free(qcpu);
        !           979: +        return -err;
        !           980: +    }
        !           981: +
        !           982: +    if (qemu_mach.cap.arch.vcpu_conf_support & NVMM_CAP_ARCH_VCPU_CONF_TPR) {
        !           983: +        memset(&tpr, 0, sizeof(tpr));
        !           984: +        tpr.exit_changed = 1;
        !           985: +        ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_TPR, &tpr);
        !           986: +        if (ret == -1) {
        !           987: +            err = errno;
        !           988: +            error_report("NVMM: Failed to configure a virtual processor,"
        !           989: +                " error=%d", err);
        !           990: +            g_free(qcpu);
        !           991: +            return -err;
        !           992: +        }
        !           993: +    }
        !           994: +
        !           995: +    cpu->vcpu_dirty = true;
        !           996: +    cpu->hax_vcpu = (struct hax_vcpu_state *)qcpu;
        !           997: +
        !           998: +    return 0;
        !           999: +}
        !          1000: +
        !          1001: +int
        !          1002: +nvmm_vcpu_exec(CPUState *cpu)
        !          1003: +{
        !          1004: +    int ret, fatal;
        !          1005: +
        !          1006: +    while (1) {
        !          1007: +        if (cpu->exception_index >= EXCP_INTERRUPT) {
        !          1008: +            ret = cpu->exception_index;
        !          1009: +            cpu->exception_index = -1;
        !          1010: +            break;
        !          1011: +        }
        !          1012: +
        !          1013: +        fatal = nvmm_vcpu_loop(cpu);
        !          1014: +
        !          1015: +        if (fatal) {
        !          1016: +            error_report("NVMM: Failed to execute a VCPU.");
        !          1017: +            abort();
        !          1018: +        }
        !          1019: +    }
        !          1020: +
        !          1021: +    return ret;
        !          1022: +}
        !          1023: +
        !          1024: +void
        !          1025: +nvmm_destroy_vcpu(CPUState *cpu)
        !          1026: +{
        !          1027: +    struct nvmm_machine *mach = get_nvmm_mach();
        !          1028: +    struct qemu_vcpu *qcpu = get_qemu_vcpu(cpu);
        !          1029: +
        !          1030: +    nvmm_vcpu_destroy(mach, &qcpu->vcpu);
        !          1031: +    g_free(cpu->hax_vcpu);
        !          1032: +}
        !          1033: +
        !          1034: +/* -------------------------------------------------------------------------- */
        !          1035: +
        !          1036: +static void
        !          1037: +nvmm_update_mapping(hwaddr start_pa, ram_addr_t size, uintptr_t hva,
        !          1038: +    bool add, bool rom, const char *name)
        !          1039: +{
        !          1040: +    struct nvmm_machine *mach = get_nvmm_mach();
        !          1041: +    int ret, prot;
        !          1042: +
        !          1043: +    if (add) {
        !          1044: +        prot = PROT_READ | PROT_EXEC;
        !          1045: +        if (!rom) {
        !          1046: +            prot |= PROT_WRITE;
        !          1047: +        }
        !          1048: +        ret = nvmm_gpa_map(mach, hva, start_pa, size, prot);
        !          1049: +    } else {
        !          1050: +        ret = nvmm_gpa_unmap(mach, hva, start_pa, size);
        !          1051: +    }
        !          1052: +
        !          1053: +    if (ret == -1) {
        !          1054: +        error_report("NVMM: Failed to %s GPA range '%s' PA:%p, "
        !          1055: +            "Size:%p bytes, HostVA:%p, error=%d",
        !          1056: +            (add ? "map" : "unmap"), name, (void *)(uintptr_t)start_pa,
        !          1057: +            (void *)size, (void *)hva, errno);
        !          1058: +    }
        !          1059: +}
        !          1060: +
        !          1061: +static void
        !          1062: +nvmm_process_section(MemoryRegionSection *section, int add)
        !          1063: +{
        !          1064: +    MemoryRegion *mr = section->mr;
        !          1065: +    hwaddr start_pa = section->offset_within_address_space;
        !          1066: +    ram_addr_t size = int128_get64(section->size);
        !          1067: +    unsigned int delta;
        !          1068: +    uintptr_t hva;
        !          1069: +
        !          1070: +    if (!memory_region_is_ram(mr)) {
        !          1071: +        return;
        !          1072: +    }
        !          1073: +
        !          1074: +    /* Adjust start_pa and size so that they are page-aligned. */
        !          1075: +    delta = qemu_real_host_page_size - (start_pa & ~qemu_real_host_page_mask);
        !          1076: +    delta &= ~qemu_real_host_page_mask;
        !          1077: +    if (delta > size) {
        !          1078: +        return;
        !          1079: +    }
        !          1080: +    start_pa += delta;
        !          1081: +    size -= delta;
        !          1082: +    size &= qemu_real_host_page_mask;
        !          1083: +    if (!size || (start_pa & ~qemu_real_host_page_mask)) {
        !          1084: +        return;
        !          1085: +    }
        !          1086: +
        !          1087: +    hva = (uintptr_t)memory_region_get_ram_ptr(mr) +
        !          1088: +        section->offset_within_region + delta;
        !          1089: +
        !          1090: +    nvmm_update_mapping(start_pa, size, hva, add,
        !          1091: +        memory_region_is_rom(mr), mr->name);
        !          1092: +}
        !          1093: +
        !          1094: +static void
        !          1095: +nvmm_region_add(MemoryListener *listener, MemoryRegionSection *section)
        !          1096: +{
        !          1097: +    memory_region_ref(section->mr);
        !          1098: +    nvmm_process_section(section, 1);
        !          1099: +}
        !          1100: +
        !          1101: +static void
        !          1102: +nvmm_region_del(MemoryListener *listener, MemoryRegionSection *section)
        !          1103: +{
        !          1104: +    nvmm_process_section(section, 0);
        !          1105: +    memory_region_unref(section->mr);
        !          1106: +}
        !          1107: +
        !          1108: +static void
        !          1109: +nvmm_transaction_begin(MemoryListener *listener)
        !          1110: +{
        !          1111: +    /* nothing */
        !          1112: +}
        !          1113: +
        !          1114: +static void
        !          1115: +nvmm_transaction_commit(MemoryListener *listener)
        !          1116: +{
        !          1117: +    /* nothing */
        !          1118: +}
        !          1119: +
        !          1120: +static void
        !          1121: +nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section)
        !          1122: +{
        !          1123: +    MemoryRegion *mr = section->mr;
        !          1124: +
        !          1125: +    if (!memory_region_is_ram(mr)) {
        !          1126: +        return;
        !          1127: +    }
        !          1128: +
        !          1129: +    memory_region_set_dirty(mr, 0, int128_get64(section->size));
        !          1130: +}
        !          1131: +
        !          1132: +static MemoryListener nvmm_memory_listener = {
        !          1133: +    .begin = nvmm_transaction_begin,
        !          1134: +    .commit = nvmm_transaction_commit,
        !          1135: +    .region_add = nvmm_region_add,
        !          1136: +    .region_del = nvmm_region_del,
        !          1137: +    .log_sync = nvmm_log_sync,
        !          1138: +    .priority = 10,
        !          1139: +};
        !          1140: +
        !          1141: +static void
        !          1142: +nvmm_ram_block_added(RAMBlockNotifier *n, void *host, size_t size)
        !          1143: +{
        !          1144: +    struct nvmm_machine *mach = get_nvmm_mach();
        !          1145: +    uintptr_t hva = (uintptr_t)host;
        !          1146: +    int ret;
        !          1147: +
        !          1148: +    ret = nvmm_hva_map(mach, hva, size);
        !          1149: +
        !          1150: +    if (ret == -1) {
        !          1151: +        error_report("NVMM: Failed to map HVA, HostVA:%p "
        !          1152: +            "Size:%p bytes, error=%d",
        !          1153: +            (void *)hva, (void *)size, errno);
        !          1154: +    }
        !          1155: +}
        !          1156: +
        !          1157: +static struct RAMBlockNotifier nvmm_ram_notifier = {
        !          1158: +    .ram_block_added = nvmm_ram_block_added
        !          1159: +};
        !          1160: +
        !          1161: +/* -------------------------------------------------------------------------- */
        !          1162: +
        !          1163: +static int
        !          1164: +nvmm_accel_init(MachineState *ms)
        !          1165: +{
        !          1166: +    int ret, err;
        !          1167: +
        !          1168: +    ret = nvmm_init();
        !          1169: +    if (ret == -1) {
        !          1170: +        err = errno;
        !          1171: +        error_report("NVMM: Initialization failed, error=%d", errno);
        !          1172: +        return -err;
        !          1173: +    }
        !          1174: +
        !          1175: +    ret = nvmm_capability(&qemu_mach.cap);
        !          1176: +    if (ret == -1) {
        !          1177: +        err = errno;
        !          1178: +        error_report("NVMM: Unable to fetch capability, error=%d", errno);
        !          1179: +        return -err;
        !          1180: +    }
        !          1181: +    if (qemu_mach.cap.version < NVMM_KERN_VERSION) {
        !          1182: +        error_report("NVMM: Unsupported version %u", qemu_mach.cap.version);
        !          1183: +        return -EPROGMISMATCH;
        !          1184: +    }
        !          1185: +    if (qemu_mach.cap.state_size != sizeof(struct nvmm_x64_state)) {
        !          1186: +        error_report("NVMM: Wrong state size %u", qemu_mach.cap.state_size);
        !          1187: +        return -EPROGMISMATCH;
        !          1188: +    }
        !          1189: +
        !          1190: +    ret = nvmm_machine_create(&qemu_mach.mach);
        !          1191: +    if (ret == -1) {
        !          1192: +        err = errno;
        !          1193: +        error_report("NVMM: Machine creation failed, error=%d", errno);
        !          1194: +        return -err;
        !          1195: +    }
        !          1196: +
        !          1197: +    memory_listener_register(&nvmm_memory_listener, &address_space_memory);
        !          1198: +    ram_block_notifier_add(&nvmm_ram_notifier);
        !          1199: +
        !          1200: +    printf("NetBSD Virtual Machine Monitor accelerator is operational\n");
        !          1201: +    return 0;
        !          1202: +}
        !          1203: +
        !          1204: +int
        !          1205: +nvmm_enabled(void)
        !          1206: +{
        !          1207: +    return nvmm_allowed;
        !          1208: +}
        !          1209: +
        !          1210: +static void
        !          1211: +nvmm_accel_class_init(ObjectClass *oc, void *data)
        !          1212: +{
        !          1213: +    AccelClass *ac = ACCEL_CLASS(oc);
        !          1214: +    ac->name = "NVMM";
        !          1215: +    ac->init_machine = nvmm_accel_init;
        !          1216: +    ac->allowed = &nvmm_allowed;
        !          1217: +}
        !          1218: +
        !          1219: +static const TypeInfo nvmm_accel_type = {
        !          1220: +    .name = ACCEL_CLASS_NAME("nvmm"),
        !          1221: +    .parent = TYPE_ACCEL,
        !          1222: +    .class_init = nvmm_accel_class_init,
        !          1223: +};
        !          1224: +
        !          1225: +static void
        !          1226: +nvmm_type_init(void)
        !          1227: +{
        !          1228: +    type_register_static(&nvmm_accel_type);
        !          1229: +}
        !          1230: +
        !          1231: +type_init(nvmm_type_init);

CVSweb <webmaster@jp.NetBSD.org>