emulate.c 123 KB
Newer Older
Avi Kivity's avatar
Avi Kivity committed
1
/******************************************************************************
2
 * emulate.c
Avi Kivity's avatar
Avi Kivity committed
3
4
5
6
7
8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
Avi Kivity's avatar
Avi Kivity committed
10
11
 *
 * Copyright (C) 2006 Qumranet
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity's avatar
Avi Kivity committed
13
14
15
16
17
18
19
20
21
22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
Avi Kivity's avatar
Avi Kivity committed
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
Avi Kivity's avatar
Avi Kivity committed
28

29
#include "x86.h"
30
#include "tss.h"
31

32
33
34
/*
 * Operand types
 */
35
36
37
38
39
40
41
42
43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44
45
46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48
49
50
51
52
53
54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55
56
57
58
59
60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
Paolo Bonzini's avatar
Paolo Bonzini committed
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64
65
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
66
67

#define OpBits             5  /* Width of operand field */
68
#define OpMask             ((1ull << OpBits) - 1)
69

Avi Kivity's avatar
Avi Kivity committed
70
71
72
73
74
75
76
77
78
79
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
80
#define ByteOp      (1<<0)	/* 8-bit operands. */
Avi Kivity's avatar
Avi Kivity committed
81
/* Destination operand type. */
82
83
84
85
86
87
88
89
90
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
91
#define DstAccLo    (OpAccLo << DstShift)
92
#define DstMask     (OpMask << DstShift)
Avi Kivity's avatar
Avi Kivity committed
93
/* Source operand type. */
94
95
96
97
98
99
100
101
102
103
104
105
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
Paolo Bonzini's avatar
Paolo Bonzini committed
106
#define SrcXLat     (OpXLat << SrcShift)
107
108
109
110
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
111
#define SrcImm64    (OpImm64 << SrcShift)
112
#define SrcDX       (OpDX << SrcShift)
113
#define SrcMem8     (OpMem8 << SrcShift)
114
#define SrcAccHi    (OpAccHi << SrcShift)
115
#define SrcMask     (OpMask << SrcShift)
116
117
118
119
120
121
122
123
124
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
125
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
126
#define Sse         (1<<18)     /* SSE Vector instruction */
127
128
129
130
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
131
/* Misc flags */
132
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
133
#define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136
#define Undefined   (1<<25) /* No Such Instruction */
137
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
138
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
139
#define No64	    (1<<28)
140
#define PageTable   (1 << 29)   /* instruction used to write page table */
141
#define NotImpl     (1 << 30)   /* instruction is not implemented */
142
/* Source 2 operand type */
143
#define Src2Shift   (31)
144
#define Src2None    (OpNone << Src2Shift)
145
#define Src2Mem     (OpMem << Src2Shift)
146
147
148
149
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
150
151
152
153
154
155
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
156
#define Src2Mask    (OpMask << Src2Shift)
Avi Kivity's avatar
Avi Kivity committed
157
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
158
159
160
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
161
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
162
#define NoWrite     ((u64)1 << 45)  /* No writeback */
163
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
Avi Kivity's avatar
Avi Kivity committed
164

165
#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
Avi Kivity's avatar
Avi Kivity committed
166

167
168
169
170
171
172
173
174
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
175

176
177
178
179
180
181
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
182
183
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
184
185
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
186
 * ex:     rsi        (in:fastop pointer, out:zero if exception)
187
188
189
190
191
192
193
194
195
196
197
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

198
struct opcode {
199
200
	u64 flags : 56;
	u64 intercept : 8;
201
	union {
202
		int (*execute)(struct x86_emulate_ctxt *ctxt);
203
204
205
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
206
		const struct escape *esc;
207
		void (*fastop)(struct fastop *fake);
208
	} u;
209
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
210
211
212
213
214
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
215
216
};

217
218
219
220
221
222
223
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

224
225
226
227
228
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

Avi Kivity's avatar
Avi Kivity committed
229
/* EFLAGS bit definitions. */
230
231
232
233
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
234
235
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
236
237
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
Avi Kivity's avatar
Avi Kivity committed
238
239
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
240
#define EFLG_IF (1<<9)
241
#define EFLG_TF (1<<8)
Avi Kivity's avatar
Avi Kivity committed
242
243
244
245
246
247
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

248
249
250
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

Avi Kivity's avatar
Avi Kivity committed
287
288
289
290
291
292
/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

293
294
295
296
297
298
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

299
300
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

301
302
303
304
305
306
307
308
309
310
311
312
313
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

314
315
#define FOPNOP() FOP_ALIGN FOP_RET

316
#define FOP1E(op,  dst) \
317
318
319
320
	FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET

#define FOP1EEX(op,  dst) \
	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
321
322
323
324
325
326
327
328
329

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

330
331
332
333
334
335
336
337
338
/* 1-operand, using src2 (for MUL/DIV r/m) */
#define FASTOP1SRC2(op, name) \
	FOP_START(name) \
	FOP1E(op, cl) \
	FOP1E(op, cx) \
	FOP1E(op, ecx) \
	ON64(FOP1E(op, rcx)) \
	FOP_END

339
340
341
342
343
344
345
346
347
/* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
#define FASTOP1SRC2EX(op, name) \
	FOP_START(name) \
	FOP1EEX(op, cl) \
	FOP1EEX(op, cx) \
	FOP1EEX(op, ecx) \
	ON64(FOP1EEX(op, rcx)) \
	FOP_END

348
349
350
351
352
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
353
354
355
356
	FOP2E(op##b, al, dl) \
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
357
358
	FOP_END

359
360
361
362
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
363
364
365
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
366
367
	FOP_END

368
369
370
371
372
373
374
375
376
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

377
378
379
380
381
382
383
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
384
385
386
	FOP3E(op##w, ax, dx, cl) \
	FOP3E(op##l, eax, edx, cl) \
	ON64(FOP3E(op##q, rax, rdx, cl)) \
387
388
	FOP_END

389
390
391
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

392
393
394
asm(".global kvm_fastop_exception \n"
    "kvm_fastop_exception: xor %esi, %esi; ret");

395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

Paolo Bonzini's avatar
Paolo Bonzini committed
414
415
416
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

417
418
419
420
421
422
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
423
424
425
426
427
428
429
430
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
431
432
433
		.next_rip   = ctxt->eip,
	};

434
	return ctxt->ops->intercept(ctxt, &info, stage);
435
436
}

Avi Kivity's avatar
Avi Kivity committed
437
438
439
440
441
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

442
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
443
{
444
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
445
446
}

Avi Kivity's avatar
Avi Kivity committed
447
448
449
450
451
452
453
454
455
456
457
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

458
459
460
461
462
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

Avi Kivity's avatar
Avi Kivity committed
463
/* Access/update address held in a register, based on addressing mode. */
464
static inline unsigned long
465
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
466
{
467
	if (ctxt->ad_bytes == sizeof(unsigned long))
468
469
		return reg;
	else
470
		return reg & ad_mask(ctxt);
471
472
473
}

static inline unsigned long
474
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
475
{
476
	return address_mask(ctxt, reg);
477
478
}

479
480
481
482
483
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

484
static inline void
485
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
486
{
487
488
	ulong mask;

489
	if (ctxt->ad_bytes == sizeof(unsigned long))
490
		mask = ~0UL;
491
	else
492
493
494
495
496
497
		mask = ad_mask(ctxt);
	masked_increment(reg, mask, inc);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
498
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
499
}
Avi Kivity's avatar
Avi Kivity committed
500

501
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
502
{
503
	register_address_increment(ctxt, &ctxt->_eip, rel);
504
}
505

506
507
508
509
510
511
512
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

513
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
514
{
515
516
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
517
518
}

519
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
520
521
522
523
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

524
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
525
526
}

527
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
528
{
529
	if (!ctxt->has_seg_override)
530
531
		return 0;

532
	return ctxt->seg_override;
533
534
}

535
536
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
537
{
538
539
540
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
541
	return X86EMUL_PROPAGATE_FAULT;
542
543
}

544
545
546
547
548
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

549
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
550
{
551
	return emulate_exception(ctxt, GP_VECTOR, err, true);
552
553
}

554
555
556
557
558
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

559
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
560
{
561
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
562
563
}

564
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
565
{
566
	return emulate_exception(ctxt, TS_VECTOR, err, true);
567
568
}

569
570
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
571
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
572
573
}

Avi Kivity's avatar
Avi Kivity committed
574
575
576
577
578
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

622
static int __linearize(struct x86_emulate_ctxt *ctxt,
623
		     struct segmented_address addr,
624
		     unsigned size, bool write, bool fetch,
625
626
		     ulong *linear)
{
627
628
	struct desc_struct desc;
	bool usable;
629
	ulong la;
630
	u32 lim;
631
	u16 sel;
632
	unsigned cpl;
633

634
	la = seg_base(ctxt, addr.seg) + addr.ea;
635
636
637
638
639
640
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
641
642
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
643
644
		if (!usable)
			goto bad;
645
646
647
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
648
649
			goto bad;
		/* unreadable code segment */
650
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
651
652
653
654
655
656
657
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
Guo Chao's avatar
Guo Chao committed
658
			/* expand-down segment */
659
660
661
662
663
664
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
665
		cpl = ctxt->ops->cpl(ctxt);
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
681
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
682
		la &= (u32)-1;
683
684
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
685
686
	*linear = la;
	return X86EMUL_CONTINUE;
687
688
bad:
	if (addr.seg == VCPU_SREG_SS)
689
		return emulate_ss(ctxt, sel);
690
	else
691
		return emulate_gp(ctxt, sel);
692
693
}

694
695
696
697
698
699
700
701
702
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


703
704
705
706
707
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
708
709
710
	int rc;
	ulong linear;

711
	rc = linearize(ctxt, addr, size, false, &linear);
712
713
	if (rc != X86EMUL_CONTINUE)
		return rc;
714
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
715
716
}

717
718
719
720
721
722
723
724
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
725
{
726
	struct fetch_cache *fc = &ctxt->fetch;
727
	int rc;
728
	int size, cur_size;
729

730
	if (ctxt->_eip == fc->end) {
731
		unsigned long linear;
732
733
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
734
		cur_size = fc->end - fc->start;
735
736
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
737
		rc = __linearize(ctxt, addr, size, false, true, &linear);
738
		if (unlikely(rc != X86EMUL_CONTINUE))
739
			return rc;
740
741
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
742
		if (unlikely(rc != X86EMUL_CONTINUE))
743
			return rc;
744
		fc->end += size;
745
	}
746
747
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
748
	return X86EMUL_CONTINUE;
749
750
751
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
752
			 void *dest, unsigned size)
753
{
754
	int rc;
755

756
	/* x86 instructions are limited to 15 bytes. */
757
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
758
		return X86EMUL_UNHANDLEABLE;
759
	while (size--) {
760
		rc = do_insn_fetch_byte(ctxt, dest++);
761
		if (rc != X86EMUL_CONTINUE)
762
763
			return rc;
	}
764
	return X86EMUL_CONTINUE;
765
766
}

767
/* Fetch next part of the instruction being emulated. */
768
#define insn_fetch(_type, _ctxt)					\
769
({	unsigned long _x;						\
770
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
771
772
773
774
775
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

776
777
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
778
779
780
781
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

782
783
784
785
786
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
787
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
788
			     int highbyte_regs)
Avi Kivity's avatar
Avi Kivity committed
789
790
791
792
{
	void *p;

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
793
794
795
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
Avi Kivity's avatar
Avi Kivity committed
796
797
798
799
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
800
			   struct segmented_address addr,
Avi Kivity's avatar
Avi Kivity committed
801
802
803
804
805
806
807
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
808
	rc = segmented_read_std(ctxt, addr, size, 2);
809
	if (rc != X86EMUL_CONTINUE)
Avi Kivity's avatar
Avi Kivity committed
810
		return rc;
811
	addr.ea += 2;
812
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
Avi Kivity's avatar
Avi Kivity committed
813
814
815
	return rc;
}

816
817
818
819
820
821
822
823
824
825
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

826
827
FASTOP1SRC2(mul, mul_ex);
FASTOP1SRC2(imul, imul_ex);
828
829
FASTOP1SRC2EX(div, div_ex);
FASTOP1SRC2EX(idiv, idiv_ex);
830

831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);
FASTOP2CL(rcr);
FASTOP2CL(shl);
FASTOP2CL(shr);
FASTOP2CL(sar);

FASTOP2W(bsf);
FASTOP2W(bsr);
FASTOP2W(bt);
FASTOP2W(bts);
FASTOP2W(btr);
FASTOP2W(btc);

856
857
FASTOP2(xadd);

858
static u8 test_cc(unsigned int condition, unsigned long flags)
859
{
860
861
	u8 rc;
	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
862

863
	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
864
	asm("push %[flags]; popf; call *%[fastop]"
865
866
	    : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
	return rc;
867
868
}

869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
static void fetch_register_operand(struct operand *op)
{
	switch (op->bytes) {
	case 1:
		op->val = *(u8 *)op->addr.reg;
		break;
	case 2:
		op->val = *(u16 *)op->addr.reg;
		break;
	case 4:
		op->val = *(u32 *)op->addr.reg;
		break;
	case 8:
		op->val = *(u64 *)op->addr.reg;
		break;
	}
}

Avi Kivity's avatar
Avi Kivity committed
887
888
889
890
static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
891
892
893
894
895
896
897
898
	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
Avi Kivity's avatar
Avi Kivity committed
899
#ifdef CONFIG_X86_64
900
901
902
903
904
905
906
907
	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
Avi Kivity's avatar
Avi Kivity committed
908
909
910
911
912
913
914
915
916
917
918
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
			  int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
919
920
921
922
923
924
925
926
	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
Avi Kivity's avatar
Avi Kivity committed
927
#ifdef CONFIG_X86_64
928
929
930
931
932
933
934
935
	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
Avi Kivity's avatar
Avi Kivity committed
936
937
938
939
940
941
#endif
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

Avi Kivity's avatar
Avi Kivity committed
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
{
	ctxt->ops->get_fpu(ctxt);
	switch (reg) {
	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
	default: BUG();
	}
	ctxt->ops->put_fpu(ctxt);
}

976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
static int em_fninit(struct x86_emulate_ctxt *ctxt)
{
	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fninit");
	ctxt->ops->put_fpu(ctxt);
	return X86EMUL_CONTINUE;
}

static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
{
	u16 fcw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstcw %0": "+m"(fcw));
	ctxt->ops->put_fpu(ctxt);

	/* force 2 byte destination */
	ctxt->dst.bytes = 2;
	ctxt->dst.val = fcw;

	return X86EMUL_CONTINUE;
}

static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
{
	u16 fsw;

	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
		return emulate_nm(ctxt);

	ctxt->ops->get_fpu(ctxt);
	asm volatile("fnstsw %0": "+m"(fsw));
	ctxt->ops->put_fpu(ctxt);

	/* force 2 byte destination */
	ctxt->dst.bytes = 2;
	ctxt->dst.val = fsw;

	return X86EMUL_CONTINUE;
}

Avi Kivity's avatar
Avi Kivity committed
1023
static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1024
				    struct operand *op)
1025
{
1026
1027
	unsigned reg = ctxt->modrm_reg;
	int highbyte_regs = ctxt->rex_prefix == 0;
1028

1029
1030
	if (!(ctxt->d & ModRM))
		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
Avi Kivity's avatar
Avi Kivity committed
1031

1032
	if (ctxt->d & Sse) {
Avi Kivity's avatar
Avi Kivity committed
1033
1034
1035
1036
1037
1038
		op->type = OP_XMM;
		op->bytes = 16;
		op->addr.xmm = reg;
		read_sse_reg(ctxt, &op->vec_val, reg);
		return;
	}
Avi Kivity's avatar
Avi Kivity committed
1039
1040
1041
1042
1043
1044
1045
	if (ctxt->d & Mmx) {
		reg &= 7;
		op->type = OP_MM;
		op->bytes = 8;
		op->addr.mm = reg;
		return;
	}
Avi Kivity's avatar
Avi Kivity committed
1046

1047
	op->type = OP_REG;
1048
	if (ctxt->d & ByteOp) {
1049
		op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1050
1051
		op->bytes = 1;
	} else {
1052
		op->addr.reg = decode_register(ctxt, reg, 0);
1053
		op->bytes = ctxt->op_bytes;
1054
	}
1055
	fetch_register_operand(op);
1056
1057
1058
	op->orig_val = op->val;
}

1059
1060
1061
1062
1063
1064
static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
{
	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
		ctxt->modrm_seg = VCPU_SREG_SS;
}

1065
static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1066
			struct operand *op)
1067
1068
{
	u8 sib;
1069
	int index_reg = 0, base_reg = 0, scale;
1070
	int rc = X86EMUL_CONTINUE;
1071
	ulong modrm_ea = 0;
1072

1073
1074
1075
1076
	if (ctxt->rex_prefix) {
		ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1;	/* REX.R */
		index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
		ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1077
1078
	}

1079
1080
1081
1082
	ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
	ctxt->modrm_rm |= (ctxt->modrm & 0x07);
	ctxt->modrm_seg = VCPU_SREG_DS;
1083

1084
	if (ctxt->modrm_mod == 3) {
1085
1086
		int highbyte_regs = ctxt->rex_prefix == 0;

1087
		op->type = OP_REG;
1088
		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1089
1090
		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
					       highbyte_regs && (ctxt->d & ByteOp));
1091
		if (ctxt->d & Sse) {
Avi Kivity's avatar
Avi Kivity committed
1092
1093
			op->type = OP_XMM;
			op->bytes = 16;
1094
1095
			op->addr.xmm = ctxt->modrm_rm;
			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
Avi Kivity's avatar
Avi Kivity committed
1096
1097
			return rc;
		}
Avi Kivity's avatar
Avi Kivity committed
1098
1099
1100
1101
1102
1103
		if (ctxt->d & Mmx) {
			op->type = OP_MM;
			op->bytes = 8;
			op->addr.xmm = ctxt->modrm_rm & 7;
			return rc;
		}
1104
		fetch_register_operand(op);
1105
1106
1107
		return rc;
	}

1108
1109
	op->type = OP_MEM;

1110
	if (ctxt->ad_bytes == 2) {
1111
1112
1113
1114
		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1115
1116

		/* 16-bit ModR/M decode. */
1117
		switch (ctxt->modrm_mod) {
1118
		case 0:
1119
			if (ctxt->modrm_rm == 6)
1120
				modrm_ea += insn_fetch(u16, ctxt);
1121
1122
			break;
		case 1:
1123
			modrm_ea += insn_fetch(s8, ctxt);
1124
1125
			break;
		case 2:
1126
			modrm_ea += insn_fetch(u16, ctxt);
1127
1128
			break;
		}
1129
		switch (ctxt->modrm_rm) {
1130
		case 0:
1131
			modrm_ea += bx + si;
1132
1133
			break;
		case 1:
1134
			modrm_ea += bx + di;
1135
1136
			break;
		case 2:
1137
			modrm_ea += bp + si;
1138
1139
			break;
		case 3: