emulate.c 127 KB
Newer Older
Avi Kivity's avatar
Avi Kivity committed
1
/******************************************************************************
2
 * emulate.c
Avi Kivity's avatar
Avi Kivity committed
3
4
5
6
7
8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
Avi Kivity's avatar
Avi Kivity committed
10
11
 *
 * Copyright (C) 2006 Qumranet
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity's avatar
Avi Kivity committed
13
14
15
16
17
18
19
20
21
22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
Avi Kivity's avatar
Avi Kivity committed
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
Avi Kivity's avatar
Avi Kivity committed
28

29
#include "x86.h"
30
#include "tss.h"
31

32
33
34
/*
 * Operand types
 */
35
36
37
38
39
40
41
42
43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44
45
46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48
49
50
51
52
53
54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55
56
57
58
59
60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
Paolo Bonzini's avatar
Paolo Bonzini committed
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64
65
#define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
#define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
66
67

#define OpBits             5  /* Width of operand field */
68
#define OpMask             ((1ull << OpBits) - 1)
69

Avi Kivity's avatar
Avi Kivity committed
70
71
72
73
74
75
76
77
78
79
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
80
#define ByteOp      (1<<0)	/* 8-bit operands. */
Avi Kivity's avatar
Avi Kivity committed
81
/* Destination operand type. */
82
83
84
85
86
87
88
89
90
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
91
#define DstAccLo    (OpAccLo << DstShift)
92
#define DstMask     (OpMask << DstShift)
Avi Kivity's avatar
Avi Kivity committed
93
/* Source operand type. */
94
95
96
97
98
99
100
101
102
103
104
105
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
Paolo Bonzini's avatar
Paolo Bonzini committed
106
#define SrcXLat     (OpXLat << SrcShift)
107
108
109
110
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
111
#define SrcImm64    (OpImm64 << SrcShift)
112
#define SrcDX       (OpDX << SrcShift)
113
#define SrcMem8     (OpMem8 << SrcShift)
114
#define SrcAccHi    (OpAccHi << SrcShift)
115
#define SrcMask     (OpMask << SrcShift)
116
117
118
119
120
121
122
123
124
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
125
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
126
#define Sse         (1<<18)     /* SSE Vector instruction */
127
128
129
130
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
131
/* Misc flags */
132
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
133
#define VendorSpecific (1<<22) /* Vendor specific instruction */
134
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136
#define Undefined   (1<<25) /* No Such Instruction */
137
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
138
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
139
#define No64	    (1<<28)
140
#define PageTable   (1 << 29)   /* instruction used to write page table */
141
#define NotImpl     (1 << 30)   /* instruction is not implemented */
142
/* Source 2 operand type */
143
#define Src2Shift   (31)
144
#define Src2None    (OpNone << Src2Shift)
145
#define Src2Mem     (OpMem << Src2Shift)
146
147
148
149
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
150
151
152
153
154
155
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
156
#define Src2Mask    (OpMask << Src2Shift)
Avi Kivity's avatar
Avi Kivity committed
157
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
158
159
160
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
161
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
162
#define NoWrite     ((u64)1 << 45)  /* No writeback */
163
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
Avi Kivity's avatar
Avi Kivity committed
164

165
166
#define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)

167
168
169
170
171
172
173
174
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
175

176
177
178
179
180
181
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
182
183
 * dst:    rax        (in/out)
 * src:    rdx        (in/out)
184
185
186
187
188
189
190
191
192
193
194
195
196
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

197
struct opcode {
198
199
	u64 flags : 56;
	u64 intercept : 8;
200
	union {
201
		int (*execute)(struct x86_emulate_ctxt *ctxt);
202
203
204
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
205
		const struct escape *esc;
206
		void (*fastop)(struct fastop *fake);
207
	} u;
208
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
209
210
211
212
213
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
214
215
};

216
217
218
219
220
221
222
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

223
224
225
226
227
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

Avi Kivity's avatar
Avi Kivity committed
228
/* EFLAGS bit definitions. */
229
230
231
232
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
233
234
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
235
236
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
Avi Kivity's avatar
Avi Kivity committed
237
238
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
239
#define EFLG_IF (1<<9)
240
#define EFLG_TF (1<<8)
Avi Kivity's avatar
Avi Kivity committed
241
242
243
244
245
246
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

247
248
249
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

Avi Kivity's avatar
Avi Kivity committed
286
287
288
289
290
291
292
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

293
#if defined(CONFIG_X86_64)
Avi Kivity's avatar
Avi Kivity committed
294
295
296
297
298
299
300
301
302
303
304
305
306
307
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
Avi Kivity's avatar
Avi Kivity committed
323
324
325
326
327
328
329
330
331

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

332
333
334
335
336
337
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

338
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
339
340
341
342
343
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
344
345
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
346
			  "=&r" (_tmp)					\
347
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
348
	} while (0)
349
350


Avi Kivity's avatar
Avi Kivity committed
351
/* Raw emulation: instruction has two explicit operands. */
352
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
353
354
355
	do {								\
		unsigned long _tmp;					\
									\
356
		switch ((ctxt)->dst.bytes) {				\
357
		case 2:							\
358
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
359
360
			break;						\
		case 4:							\
361
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
362
363
			break;						\
		case 8:							\
364
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
365
366
			break;						\
		}							\
Avi Kivity's avatar
Avi Kivity committed
367
368
	} while (0)

369
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
Avi Kivity's avatar
Avi Kivity committed
370
	do {								     \
371
		unsigned long _tmp;					     \
372
		switch ((ctxt)->dst.bytes) {				     \
Avi Kivity's avatar
Avi Kivity committed
373
		case 1:							     \
374
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
Avi Kivity's avatar
Avi Kivity committed
375
376
			break;						     \
		default:						     \
377
			__emulate_2op_nobyte(ctxt, _op,			     \
Avi Kivity's avatar
Avi Kivity committed
378
379
380
381
382
383
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
384
385
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
Avi Kivity's avatar
Avi Kivity committed
386
387

/* Source operand is byte, word, long or quad sized. */
388
389
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
390
391

/* Source operand is word, long or quad sized. */
392
393
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
394

395
/* Instruction has three operands and one operand is stored in ECX register */
396
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
397
398
	do {								\
		unsigned long _tmp;					\
399
400
401
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
402
403
404
405
406
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
407
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
408
409
410
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
411
412
413
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
414
415
	} while (0)

416
#define emulate_2op_cl(ctxt, _op)					\
417
	do {								\
418
		switch ((ctxt)->dst.bytes) {				\
419
		case 2:							\
420
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
421
422
			break;						\
		case 4:							\
423
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
424
425
			break;						\
		case 8:							\
426
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
427
428
			break;						\
		}							\
429
430
	} while (0)

431
#define __emulate_1op(ctxt, _op, _suffix)				\
Avi Kivity's avatar
Avi Kivity committed
432
433
434
	do {								\
		unsigned long _tmp;					\
									\
435
436
437
438
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
439
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
440
441
442
443
444
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
445
#define emulate_1op(ctxt, _op)						\
446
	do {								\
447
448
449
450
451
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
Avi Kivity's avatar
Avi Kivity committed
452
453
454
		}							\
	} while (0)

455
456
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

457
458
459
460
461
462
463
464
465
466
467
468
469
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

470
471
#define FOPNOP() FOP_ALIGN FOP_RET

472
473
474
475
476
477
478
479
480
481
482
#define FOP1E(op,  dst) \
	FOP_ALIGN #op " %" #dst " \n\t" FOP_RET

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

483
484
485
486
487
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
488
489
490
491
	FOP2E(op##b, al, dl) \
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
492
493
	FOP_END

494
495
496
497
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
498
499
500
	FOP2E(op##w, ax, dx) \
	FOP2E(op##l, eax, edx) \
	ON64(FOP2E(op##q, rax, rdx)) \
501
502
	FOP_END

503
504
505
506
507
508
509
510
511
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

512
513
514
515
516
517
518
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
519
520
521
	FOP3E(op##w, ax, dx, cl) \
	FOP3E(op##l, eax, edx, cl) \
	ON64(FOP3E(op##q, rax, rdx, cl)) \
522
523
	FOP_END

524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

Paolo Bonzini's avatar
Paolo Bonzini committed
546
547
548
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

549
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
550
551
	do {								\
		unsigned long _tmp;					\
552
553
		ulong *rax = &ctxt->dst.val;				\
		ulong *rdx = &ctxt->src.val;				\
554
555
556
557
558
559
560
561
562
563
564
565
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
566
567
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
568
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src2.val));	\
569
570
	} while (0)

571
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
572
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
573
	do {								\
574
		switch((ctxt)->src.bytes) {				\
575
		case 1:							\
576
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
577
578
			break;						\
		case 2:							\
579
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
580
581
			break;						\
		case 4:							\
582
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
583
584
			break;						\
		case 8: ON64(						\
585
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
586
587
588
589
			break;						\
		}							\
	} while (0)

590
591
592
593
594
595
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
596
597
598
599
600
601
602
603
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
604
605
606
		.next_rip   = ctxt->eip,
	};

607
	return ctxt->ops->intercept(ctxt, &info, stage);
608
609
}

Avi Kivity's avatar
Avi Kivity committed
610
611
612
613
614
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

615
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
616
{
617
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
618
619
}

Avi Kivity's avatar
Avi Kivity committed
620
621
622
623
624
625
626
627
628
629
630
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

631
632
633
634
635
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

Avi Kivity's avatar
Avi Kivity committed
636
/* Access/update address held in a register, based on addressing mode. */
637
static inline unsigned long
638
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
639
{
640
	if (ctxt->ad_bytes == sizeof(unsigned long))
641
642
		return reg;
	else
643
		return reg & ad_mask(ctxt);
644
645
646
}

static inline unsigned long
647
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
648
{
649
	return address_mask(ctxt, reg);
650
651
}

652
653
654
655
656
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

657
static inline void
658
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
659
{
660
661
	ulong mask;

662
	if (ctxt->ad_bytes == sizeof(unsigned long))
663
		mask = ~0UL;
664
	else
665
666
667
668
669
670
		mask = ad_mask(ctxt);
	masked_increment(reg, mask, inc);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
671
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
672
}
Avi Kivity's avatar
Avi Kivity committed
673

674
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
675
{
676
	register_address_increment(ctxt, &ctxt->_eip, rel);
677
}
678

679
680
681
682
683
684
685
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

686
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
687
{
688
689
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
690
691
}

692
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
693
694
695
696
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

697
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
698
699
}

700
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
701
{
702
	if (!ctxt->has_seg_override)
703
704
		return 0;

705
	return ctxt->seg_override;
706
707
}

708
709
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
710
{
711
712
713
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
714
	return X86EMUL_PROPAGATE_FAULT;
715
716
}

717
718
719
720
721
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

722
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
723
{
724
	return emulate_exception(ctxt, GP_VECTOR, err, true);
725
726
}

727
728
729
730
731
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

732
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
733
{
734
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
735
736
}

737
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
738
{
739
	return emulate_exception(ctxt, TS_VECTOR, err, true);
740
741
}

742
743
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
744
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
745
746
}

Avi Kivity's avatar
Avi Kivity committed
747
748
749
750
751
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

795
static int __linearize(struct x86_emulate_ctxt *ctxt,
796
		     struct segmented_address addr,
797
		     unsigned size, bool write, bool fetch,
798
799
		     ulong *linear)
{
800
801
	struct desc_struct desc;
	bool usable;
802
	ulong la;
803
	u32 lim;
804
	u16 sel;
805
	unsigned cpl;
806

807
	la = seg_base(ctxt, addr.seg) + addr.ea;
808
809
810
811
812
813
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
814
815
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
816
817
		if (!usable)
			goto bad;
818
819
820
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
821
822
			goto bad;
		/* unreadable code segment */
823
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
824
825
826
827
828
829
830
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
Guo Chao's avatar
Guo Chao committed
831
			/* expand-down segment */
832
833
834
835
836
837
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
838
		cpl = ctxt->ops->cpl(ctxt);
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
854
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
855
		la &= (u32)-1;
856
857
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
858
859
	*linear = la;
	return X86EMUL_CONTINUE;
860
861
bad:
	if (addr.seg == VCPU_SREG_SS)
862
		return emulate_ss(ctxt, sel);
863
	else
864
		return emulate_gp(ctxt, sel);
865
866
}

867
868
869
870
871
872
873
874
875
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


876
877
878
879
880
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
881
882
883
	int rc;
	ulong linear;

884
	rc = linearize(ctxt, addr, size, false, &linear);
885
886
	if (rc != X86EMUL_CONTINUE)
		return rc;
887
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
888
889
}

890
891
892
893
894
895
896
897
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
898
{
899
	struct fetch_cache *fc = &ctxt->fetch;
900
	int rc;
901
	int size, cur_size;
902

903
	if (ctxt->_eip == fc->end) {
904
		unsigned long linear;
905
906
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
907
		cur_size = fc->end - fc->start;
908
909
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
910
		rc = __linearize(ctxt, addr, size, false, true, &linear);
911
		if (unlikely(rc != X86EMUL_CONTINUE))
912
			return rc;
913
914
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
915
		if (unlikely(rc != X86EMUL_CONTINUE))
916
			return rc;
917
		fc->end += size;
918
	}
919
920
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
921
	return X86EMUL_CONTINUE;
922
923
924
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
925
			 void *dest, unsigned size)
926
{
927
	int rc;
928

929
	/* x86 instructions are limited to 15 bytes. */
930
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
931
		return X86EMUL_UNHANDLEABLE;
932
	while (size--) {
933
		rc = do_insn_fetch_byte(ctxt, dest++);
934
		if (rc != X86EMUL_CONTINUE)
935
936
			return rc;
	}
937
	return X86EMUL_CONTINUE;
938
939
}

940
/* Fetch next part of the instruction being emulated. */
941
#define insn_fetch(_type, _ctxt)					\
942
({	unsigned long _x;						\
943
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
944
945
946
947
948
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

949
950
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
951
952
953
954
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

955
956
957
958
959
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
960
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
961
			     int highbyte_regs)
Avi Kivity's avatar
Avi Kivity committed
962
963
964
965
{
	void *p;

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
966
967
968
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
Avi Kivity's avatar
Avi Kivity committed
969
970
971
972
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
973
			   struct segmented_address addr,
Avi Kivity's avatar
Avi Kivity committed
974
975
976
977
978
979
980
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
981
	rc = segmented_read_std(ctxt, addr, size, 2);
982
	if (rc != X86EMUL_CONTINUE)
Avi Kivity's avatar
Avi Kivity committed
983
		return rc;
984
	addr.ea += 2;
985
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
Avi Kivity's avatar
Avi Kivity committed
986
987
988
	return rc;
}

989
990
991
992
993
994
995
996
997
998
999
1000
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

FASTOP3WCL(shld);
FASTOP3WCL(shrd);