emulate.c 126 KB
Newer Older
Avi Kivity's avatar
Avi Kivity committed
1
/******************************************************************************
2
 * emulate.c
Avi Kivity's avatar
Avi Kivity committed
3
4
5
6
7
8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
Avi Kivity's avatar
Avi Kivity committed
10
11
 *
 * Copyright (C) 2006 Qumranet
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity's avatar
Avi Kivity committed
13
14
15
16
17
18
19
20
21
22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
Avi Kivity's avatar
Avi Kivity committed
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
Avi Kivity's avatar
Avi Kivity committed
28

29
#include "x86.h"
30
#include "tss.h"
31

32
33
34
/*
 * Operand types
 */
35
36
37
38
39
40
41
42
43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44
45
46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48
49
50
51
52
53
54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55
56
57
58
59
60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
Paolo Bonzini's avatar
Paolo Bonzini committed
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64
65

#define OpBits             5  /* Width of operand field */
66
#define OpMask             ((1ull << OpBits) - 1)
67

Avi Kivity's avatar
Avi Kivity committed
68
69
70
71
72
73
74
75
76
77
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
78
#define ByteOp      (1<<0)	/* 8-bit operands. */
Avi Kivity's avatar
Avi Kivity committed
79
/* Destination operand type. */
80
81
82
83
84
85
86
87
88
89
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
Avi Kivity's avatar
Avi Kivity committed
90
/* Source operand type. */
91
92
93
94
95
96
97
98
99
100
101
102
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
Paolo Bonzini's avatar
Paolo Bonzini committed
103
#define SrcXLat     (OpXLat << SrcShift)
104
105
106
107
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
108
#define SrcImm64    (OpImm64 << SrcShift)
109
#define SrcDX       (OpDX << SrcShift)
110
#define SrcMem8     (OpMem8 << SrcShift)
111
#define SrcMask     (OpMask << SrcShift)
112
113
114
115
116
117
118
119
120
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
121
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
122
#define Sse         (1<<18)     /* SSE Vector instruction */
123
124
125
126
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
127
/* Misc flags */
128
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
129
#define VendorSpecific (1<<22) /* Vendor specific instruction */
130
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
131
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
132
#define Undefined   (1<<25) /* No Such Instruction */
133
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
134
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
135
#define No64	    (1<<28)
136
#define PageTable   (1 << 29)   /* instruction used to write page table */
137
#define NotImpl     (1 << 30)   /* instruction is not implemented */
138
/* Source 2 operand type */
139
#define Src2Shift   (31)
140
141
142
143
144
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
145
146
147
148
149
150
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
151
#define Src2Mask    (OpMask << Src2Shift)
Avi Kivity's avatar
Avi Kivity committed
152
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
153
154
155
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
156
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
157
#define NoWrite     ((u64)1 << 45)  /* No writeback */
158
#define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
Avi Kivity's avatar
Avi Kivity committed
159

160
161
162
163
164
165
166
167
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
168

169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
 * dst:    [rdx]:rax  (in/out)
 * src:    rbx        (in/out)
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

190
struct opcode {
191
192
	u64 flags : 56;
	u64 intercept : 8;
193
	union {
194
		int (*execute)(struct x86_emulate_ctxt *ctxt);
195
196
197
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
198
		const struct escape *esc;
199
		void (*fastop)(struct fastop *fake);
200
	} u;
201
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
202
203
204
205
206
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
207
208
};

209
210
211
212
213
214
215
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

216
217
218
219
220
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

Avi Kivity's avatar
Avi Kivity committed
221
/* EFLAGS bit definitions. */
222
223
224
225
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
226
227
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
228
229
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
Avi Kivity's avatar
Avi Kivity committed
230
231
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
232
#define EFLG_IF (1<<9)
233
#define EFLG_TF (1<<8)
Avi Kivity's avatar
Avi Kivity committed
234
235
236
237
238
239
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

240
241
242
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

Avi Kivity's avatar
Avi Kivity committed
279
280
281
282
283
284
285
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

286
#if defined(CONFIG_X86_64)
Avi Kivity's avatar
Avi Kivity committed
287
288
289
290
291
292
293
294
295
296
297
298
299
300
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
Avi Kivity's avatar
Avi Kivity committed
316
317
318
319
320
321
322
323
324

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

325
326
327
328
329
330
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

331
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
332
333
334
335
336
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
337
338
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
339
			  "=&r" (_tmp)					\
340
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
341
	} while (0)
342
343


Avi Kivity's avatar
Avi Kivity committed
344
/* Raw emulation: instruction has two explicit operands. */
345
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
346
347
348
	do {								\
		unsigned long _tmp;					\
									\
349
		switch ((ctxt)->dst.bytes) {				\
350
		case 2:							\
351
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
352
353
			break;						\
		case 4:							\
354
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
355
356
			break;						\
		case 8:							\
357
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
358
359
			break;						\
		}							\
Avi Kivity's avatar
Avi Kivity committed
360
361
	} while (0)

362
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
Avi Kivity's avatar
Avi Kivity committed
363
	do {								     \
364
		unsigned long _tmp;					     \
365
		switch ((ctxt)->dst.bytes) {				     \
Avi Kivity's avatar
Avi Kivity committed
366
		case 1:							     \
367
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
Avi Kivity's avatar
Avi Kivity committed
368
369
			break;						     \
		default:						     \
370
			__emulate_2op_nobyte(ctxt, _op,			     \
Avi Kivity's avatar
Avi Kivity committed
371
372
373
374
375
376
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
377
378
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
Avi Kivity's avatar
Avi Kivity committed
379
380

/* Source operand is byte, word, long or quad sized. */
381
382
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
383
384

/* Source operand is word, long or quad sized. */
385
386
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
387

388
/* Instruction has three operands and one operand is stored in ECX register */
389
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
390
391
	do {								\
		unsigned long _tmp;					\
392
393
394
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
395
396
397
398
399
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
400
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
401
402
403
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
404
405
406
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
407
408
	} while (0)

409
#define emulate_2op_cl(ctxt, _op)					\
410
	do {								\
411
		switch ((ctxt)->dst.bytes) {				\
412
		case 2:							\
413
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
414
415
			break;						\
		case 4:							\
416
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
417
418
			break;						\
		case 8:							\
419
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
420
421
			break;						\
		}							\
422
423
	} while (0)

424
#define __emulate_1op(ctxt, _op, _suffix)				\
Avi Kivity's avatar
Avi Kivity committed
425
426
427
	do {								\
		unsigned long _tmp;					\
									\
428
429
430
431
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
432
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
433
434
435
436
437
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
438
#define emulate_1op(ctxt, _op)						\
439
	do {								\
440
441
442
443
444
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
Avi Kivity's avatar
Avi Kivity committed
445
446
447
		}							\
	} while (0)

448
449
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

450
451
452
453
454
455
456
457
458
459
460
461
462
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

463
464
#define FOPNOP() FOP_ALIGN FOP_RET

465
466
467
468
469
470
471
472
473
474
475
#define FOP1E(op,  dst) \
	FOP_ALIGN #op " %" #dst " \n\t" FOP_RET

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

476
477
478
479
480
481
482
483
484
485
486
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
	FOP2E(op##b, al, bl) \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

487
488
489
490
491
492
493
494
495
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

496
497
498
499
500
501
502
503
504
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

505
506
507
508
509
510
511
512
513
514
515
516
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP3E(op##w, ax, bx, cl) \
	FOP3E(op##l, eax, ebx, cl) \
	ON64(FOP3E(op##q, rax, rbx, cl)) \
	FOP_END

517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

Paolo Bonzini's avatar
Paolo Bonzini committed
539
540
541
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

542
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
543
544
	do {								\
		unsigned long _tmp;					\
545
546
		ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX);		\
		ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX);		\
547
548
549
550
551
552
553
554
555
556
557
558
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
559
560
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
561
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val));	\
562
563
	} while (0)

564
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
565
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
566
	do {								\
567
		switch((ctxt)->src.bytes) {				\
568
		case 1:							\
569
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
570
571
			break;						\
		case 2:							\
572
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
573
574
			break;						\
		case 4:							\
575
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
576
577
			break;						\
		case 8: ON64(						\
578
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
579
580
581
582
			break;						\
		}							\
	} while (0)

583
584
585
586
587
588
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
589
590
591
592
593
594
595
596
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
597
598
599
		.next_rip   = ctxt->eip,
	};

600
	return ctxt->ops->intercept(ctxt, &info, stage);
601
602
}

Avi Kivity's avatar
Avi Kivity committed
603
604
605
606
607
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

608
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
609
{
610
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
611
612
}

Avi Kivity's avatar
Avi Kivity committed
613
614
615
616
617
618
619
620
621
622
623
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

624
625
626
627
628
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

Avi Kivity's avatar
Avi Kivity committed
629
/* Access/update address held in a register, based on addressing mode. */
630
static inline unsigned long
631
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
632
{
633
	if (ctxt->ad_bytes == sizeof(unsigned long))
634
635
		return reg;
	else
636
		return reg & ad_mask(ctxt);
637
638
639
}

static inline unsigned long
640
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
641
{
642
	return address_mask(ctxt, reg);
643
644
}

645
646
647
648
649
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

650
static inline void
651
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
652
{
653
654
	ulong mask;

655
	if (ctxt->ad_bytes == sizeof(unsigned long))
656
		mask = ~0UL;
657
	else
658
659
660
661
662
663
		mask = ad_mask(ctxt);
	masked_increment(reg, mask, inc);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
664
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
665
}
Avi Kivity's avatar
Avi Kivity committed
666

667
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
668
{
669
	register_address_increment(ctxt, &ctxt->_eip, rel);
670
}
671

672
673
674
675
676
677
678
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

679
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
680
{
681
682
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
683
684
}

685
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
686
687
688
689
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

690
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
691
692
}

693
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
694
{
695
	if (!ctxt->has_seg_override)
696
697
		return 0;

698
	return ctxt->seg_override;
699
700
}

701
702
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
703
{
704
705
706
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
707
	return X86EMUL_PROPAGATE_FAULT;
708
709
}

710
711
712
713
714
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

715
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
716
{
717
	return emulate_exception(ctxt, GP_VECTOR, err, true);
718
719
}

720
721
722
723
724
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

725
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
726
{
727
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
728
729
}

730
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
731
{
732
	return emulate_exception(ctxt, TS_VECTOR, err, true);
733
734
}

735
736
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
737
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
738
739
}

Avi Kivity's avatar
Avi Kivity committed
740
741
742
743
744
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

788
static int __linearize(struct x86_emulate_ctxt *ctxt,
789
		     struct segmented_address addr,
790
		     unsigned size, bool write, bool fetch,
791
792
		     ulong *linear)
{
793
794
	struct desc_struct desc;
	bool usable;
795
	ulong la;
796
	u32 lim;
797
	u16 sel;
798
	unsigned cpl;
799

800
	la = seg_base(ctxt, addr.seg) + addr.ea;
801
802
803
804
805
806
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
807
808
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
809
810
		if (!usable)
			goto bad;
811
812
813
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
814
815
			goto bad;
		/* unreadable code segment */
816
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
817
818
819
820
821
822
823
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
Guo Chao's avatar
Guo Chao committed
824
			/* expand-down segment */
825
826
827
828
829
830
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
831
		cpl = ctxt->ops->cpl(ctxt);
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
847
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
848
		la &= (u32)-1;
849
850
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
851
852
	*linear = la;
	return X86EMUL_CONTINUE;
853
854
bad:
	if (addr.seg == VCPU_SREG_SS)
855
		return emulate_ss(ctxt, sel);
856
	else
857
		return emulate_gp(ctxt, sel);
858
859
}

860
861
862
863
864
865
866
867
868
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


869
870
871
872
873
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
874
875
876
	int rc;
	ulong linear;

877
	rc = linearize(ctxt, addr, size, false, &linear);
878
879
	if (rc != X86EMUL_CONTINUE)
		return rc;
880
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
881
882
}

883
884
885
886
887
888
889
890
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
891
{
892
	struct fetch_cache *fc = &ctxt->fetch;
893
	int rc;
894
	int size, cur_size;
895

896
	if (ctxt->_eip == fc->end) {
897
		unsigned long linear;
898
899
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
900
		cur_size = fc->end - fc->start;
901
902
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
903
		rc = __linearize(ctxt, addr, size, false, true, &linear);
904
		if (unlikely(rc != X86EMUL_CONTINUE))
905
			return rc;
906
907
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
908
		if (unlikely(rc != X86EMUL_CONTINUE))
909
			return rc;
910
		fc->end += size;
911
	}
912
913
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
914
	return X86EMUL_CONTINUE;
915
916
917
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
918
			 void *dest, unsigned size)
919
{
920
	int rc;
921

922
	/* x86 instructions are limited to 15 bytes. */
923
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
924
		return X86EMUL_UNHANDLEABLE;
925
	while (size--) {
926
		rc = do_insn_fetch_byte(ctxt, dest++);
927
		if (rc != X86EMUL_CONTINUE)
928
929
			return rc;
	}
930
	return X86EMUL_CONTINUE;
931
932
}

933
/* Fetch next part of the instruction being emulated. */
934
#define insn_fetch(_type, _ctxt)					\
935
({	unsigned long _x;						\
936
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
937
938
939
940
941
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

942
943
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
944
945
946
947
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

948
949
950
951
952
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
953
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
954
			     int highbyte_regs)
Avi Kivity's avatar
Avi Kivity committed
955
956
957
958
{
	void *p;

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
959
960
961
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
Avi Kivity's avatar
Avi Kivity committed
962
963
964
965
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
966
			   struct segmented_address addr,
Avi Kivity's avatar
Avi Kivity committed
967
968
969
970
971
972
973
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
974
	rc = segmented_read_std(ctxt, addr, size, 2);
975
	if (rc != X86EMUL_CONTINUE)
Avi Kivity's avatar
Avi Kivity committed
976
		return rc;
977
	addr.ea += 2;
978
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
Avi Kivity's avatar
Avi Kivity committed
979
980
981
	return rc;
}

982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);