emulate.c 126 KB
Newer Older
Avi Kivity's avatar
Avi Kivity committed
1
/******************************************************************************
2
 * emulate.c
Avi Kivity's avatar
Avi Kivity committed
3
4
5
6
7
8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
Avi Kivity's avatar
Avi Kivity committed
10
11
 *
 * Copyright (C) 2006 Qumranet
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity's avatar
Avi Kivity committed
13
14
15
16
17
18
19
20
21
22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
Avi Kivity's avatar
Avi Kivity committed
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
Avi Kivity's avatar
Avi Kivity committed
28

29
#include "x86.h"
30
#include "tss.h"
31

32
33
34
/*
 * Operand types
 */
35
36
37
38
39
40
41
42
43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44
45
46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48
49
50
51
52
53
54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55
56
57
58
59
60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
Paolo Bonzini's avatar
Paolo Bonzini committed
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64
65

#define OpBits             5  /* Width of operand field */
66
#define OpMask             ((1ull << OpBits) - 1)
67

Avi Kivity's avatar
Avi Kivity committed
68
69
70
71
72
73
74
75
76
77
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
78
#define ByteOp      (1<<0)	/* 8-bit operands. */
Avi Kivity's avatar
Avi Kivity committed
79
/* Destination operand type. */
80
81
82
83
84
85
86
87
88
89
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
Avi Kivity's avatar
Avi Kivity committed
90
/* Source operand type. */
91
92
93
94
95
96
97
98
99
100
101
102
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
Paolo Bonzini's avatar
Paolo Bonzini committed
103
#define SrcXLat     (OpXLat << SrcShift)
104
105
106
107
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
108
#define SrcImm64    (OpImm64 << SrcShift)
109
#define SrcDX       (OpDX << SrcShift)
110
#define SrcMem8     (OpMem8 << SrcShift)
111
#define SrcMask     (OpMask << SrcShift)
112
113
114
115
116
117
118
119
120
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
121
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
122
#define Sse         (1<<18)     /* SSE Vector instruction */
123
124
125
126
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
127
/* Misc flags */
128
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
129
#define VendorSpecific (1<<22) /* Vendor specific instruction */
130
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
131
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
132
#define Undefined   (1<<25) /* No Such Instruction */
133
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
134
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
135
#define No64	    (1<<28)
136
#define PageTable   (1 << 29)   /* instruction used to write page table */
137
#define NotImpl     (1 << 30)   /* instruction is not implemented */
138
/* Source 2 operand type */
139
#define Src2Shift   (31)
140
141
142
143
144
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
145
146
147
148
149
150
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
151
#define Src2Mask    (OpMask << Src2Shift)
Avi Kivity's avatar
Avi Kivity committed
152
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
153
154
155
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
156
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
157
#define NoWrite     ((u64)1 << 45)  /* No writeback */
Avi Kivity's avatar
Avi Kivity committed
158

159
160
161
162
163
164
165
166
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
167

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
 * dst:    [rdx]:rax  (in/out)
 * src:    rbx        (in/out)
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

189
struct opcode {
190
191
	u64 flags : 56;
	u64 intercept : 8;
192
	union {
193
		int (*execute)(struct x86_emulate_ctxt *ctxt);
194
195
196
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
197
		const struct escape *esc;
198
		void (*fastop)(struct fastop *fake);
199
	} u;
200
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201
202
203
204
205
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
206
207
};

208
209
210
211
212
213
214
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

215
216
217
218
219
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

Avi Kivity's avatar
Avi Kivity committed
220
/* EFLAGS bit definitions. */
221
222
223
224
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
225
226
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
227
228
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
Avi Kivity's avatar
Avi Kivity committed
229
230
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
231
#define EFLG_IF (1<<9)
232
#define EFLG_TF (1<<8)
Avi Kivity's avatar
Avi Kivity committed
233
234
235
236
237
238
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

239
240
241
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

Avi Kivity's avatar
Avi Kivity committed
278
279
280
281
282
283
284
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

285
#if defined(CONFIG_X86_64)
Avi Kivity's avatar
Avi Kivity committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
Avi Kivity's avatar
Avi Kivity committed
315
316
317
318
319
320
321
322
323

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

324
325
326
327
328
329
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

330
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
331
332
333
334
335
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
336
337
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
338
			  "=&r" (_tmp)					\
339
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
340
	} while (0)
341
342


Avi Kivity's avatar
Avi Kivity committed
343
/* Raw emulation: instruction has two explicit operands. */
344
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
345
346
347
	do {								\
		unsigned long _tmp;					\
									\
348
		switch ((ctxt)->dst.bytes) {				\
349
		case 2:							\
350
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
351
352
			break;						\
		case 4:							\
353
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
354
355
			break;						\
		case 8:							\
356
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
357
358
			break;						\
		}							\
Avi Kivity's avatar
Avi Kivity committed
359
360
	} while (0)

361
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
Avi Kivity's avatar
Avi Kivity committed
362
	do {								     \
363
		unsigned long _tmp;					     \
364
		switch ((ctxt)->dst.bytes) {				     \
Avi Kivity's avatar
Avi Kivity committed
365
		case 1:							     \
366
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
Avi Kivity's avatar
Avi Kivity committed
367
368
			break;						     \
		default:						     \
369
			__emulate_2op_nobyte(ctxt, _op,			     \
Avi Kivity's avatar
Avi Kivity committed
370
371
372
373
374
375
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
376
377
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
Avi Kivity's avatar
Avi Kivity committed
378
379

/* Source operand is byte, word, long or quad sized. */
380
381
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
382
383

/* Source operand is word, long or quad sized. */
384
385
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
386

387
/* Instruction has three operands and one operand is stored in ECX register */
388
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
389
390
	do {								\
		unsigned long _tmp;					\
391
392
393
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
394
395
396
397
398
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
399
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
400
401
402
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
403
404
405
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
406
407
	} while (0)

408
#define emulate_2op_cl(ctxt, _op)					\
409
	do {								\
410
		switch ((ctxt)->dst.bytes) {				\
411
		case 2:							\
412
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
413
414
			break;						\
		case 4:							\
415
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
416
417
			break;						\
		case 8:							\
418
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
419
420
			break;						\
		}							\
421
422
	} while (0)

423
#define __emulate_1op(ctxt, _op, _suffix)				\
Avi Kivity's avatar
Avi Kivity committed
424
425
426
	do {								\
		unsigned long _tmp;					\
									\
427
428
429
430
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
431
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
432
433
434
435
436
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
437
#define emulate_1op(ctxt, _op)						\
438
	do {								\
439
440
441
442
443
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
Avi Kivity's avatar
Avi Kivity committed
444
445
446
		}							\
	} while (0)

447
448
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

449
450
451
452
453
454
455
456
457
458
459
460
461
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

462
463
#define FOPNOP() FOP_ALIGN FOP_RET

464
465
466
467
468
469
470
471
472
473
474
#define FOP1E(op,  dst) \
	FOP_ALIGN #op " %" #dst " \n\t" FOP_RET

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

475
476
477
478
479
480
481
482
483
484
485
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
	FOP2E(op##b, al, bl) \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

486
487
488
489
490
491
492
493
494
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

495
496
497
498
499
500
501
502
503
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

504
505
506
507
508
509
510
511
512
513
514
515
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP3E(op##w, ax, bx, cl) \
	FOP3E(op##l, eax, ebx, cl) \
	ON64(FOP3E(op##q, rax, rbx, cl)) \
	FOP_END

516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

Paolo Bonzini's avatar
Paolo Bonzini committed
538
539
540
FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
FOP_END;

541
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
542
543
	do {								\
		unsigned long _tmp;					\
544
545
		ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX);		\
		ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX);		\
546
547
548
549
550
551
552
553
554
555
556
557
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
558
559
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
560
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val));	\
561
562
	} while (0)

563
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
564
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
565
	do {								\
566
		switch((ctxt)->src.bytes) {				\
567
		case 1:							\
568
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
569
570
			break;						\
		case 2:							\
571
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
572
573
			break;						\
		case 4:							\
574
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
575
576
			break;						\
		case 8: ON64(						\
577
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
578
579
580
581
			break;						\
		}							\
	} while (0)

582
583
584
585
586
587
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
588
589
590
591
592
593
594
595
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
596
597
598
		.next_rip   = ctxt->eip,
	};

599
	return ctxt->ops->intercept(ctxt, &info, stage);
600
601
}

Avi Kivity's avatar
Avi Kivity committed
602
603
604
605
606
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

607
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
608
{
609
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
610
611
}

Avi Kivity's avatar
Avi Kivity committed
612
613
614
615
616
617
618
619
620
621
622
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

623
624
625
626
627
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

Avi Kivity's avatar
Avi Kivity committed
628
/* Access/update address held in a register, based on addressing mode. */
629
static inline unsigned long
630
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
631
{
632
	if (ctxt->ad_bytes == sizeof(unsigned long))
633
634
		return reg;
	else
635
		return reg & ad_mask(ctxt);
636
637
638
}

static inline unsigned long
639
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
640
{
641
	return address_mask(ctxt, reg);
642
643
}

644
645
646
647
648
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

649
static inline void
650
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
651
{
652
653
	ulong mask;

654
	if (ctxt->ad_bytes == sizeof(unsigned long))
655
		mask = ~0UL;
656
	else
657
658
659
660
661
662
		mask = ad_mask(ctxt);
	masked_increment(reg, mask, inc);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
663
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
664
}
Avi Kivity's avatar
Avi Kivity committed
665

666
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
667
{
668
	register_address_increment(ctxt, &ctxt->_eip, rel);
669
}
670

671
672
673
674
675
676
677
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

678
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
679
{
680
681
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
682
683
}

684
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
685
686
687
688
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

689
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
690
691
}

692
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
693
{
694
	if (!ctxt->has_seg_override)
695
696
		return 0;

697
	return ctxt->seg_override;
698
699
}

700
701
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
702
{
703
704
705
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
706
	return X86EMUL_PROPAGATE_FAULT;
707
708
}

709
710
711
712
713
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

714
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
715
{
716
	return emulate_exception(ctxt, GP_VECTOR, err, true);
717
718
}

719
720
721
722
723
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

724
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
725
{
726
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
727
728
}

729
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
730
{
731
	return emulate_exception(ctxt, TS_VECTOR, err, true);
732
733
}

734
735
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
736
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
737
738
}

Avi Kivity's avatar
Avi Kivity committed
739
740
741
742
743
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

787
static int __linearize(struct x86_emulate_ctxt *ctxt,
788
		     struct segmented_address addr,
789
		     unsigned size, bool write, bool fetch,
790
791
		     ulong *linear)
{
792
793
	struct desc_struct desc;
	bool usable;
794
	ulong la;
795
	u32 lim;
796
	u16 sel;
797
	unsigned cpl;
798

799
	la = seg_base(ctxt, addr.seg) + addr.ea;
800
801
802
803
804
805
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
806
807
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
808
809
		if (!usable)
			goto bad;
810
811
812
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
813
814
			goto bad;
		/* unreadable code segment */
815
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
816
817
818
819
820
821
822
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
Guo Chao's avatar
Guo Chao committed
823
			/* expand-down segment */
824
825
826
827
828
829
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
830
		cpl = ctxt->ops->cpl(ctxt);
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
846
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
847
		la &= (u32)-1;
848
849
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
850
851
	*linear = la;
	return X86EMUL_CONTINUE;
852
853
bad:
	if (addr.seg == VCPU_SREG_SS)
854
		return emulate_ss(ctxt, sel);
855
	else
856
		return emulate_gp(ctxt, sel);
857
858
}

859
860
861
862
863
864
865
866
867
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


868
869
870
871
872
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
873
874
875
	int rc;
	ulong linear;

876
	rc = linearize(ctxt, addr, size, false, &linear);
877
878
	if (rc != X86EMUL_CONTINUE)
		return rc;
879
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
880
881
}

882
883
884
885
886
887
888
889
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
890
{
891
	struct fetch_cache *fc = &ctxt->fetch;
892
	int rc;
893
	int size, cur_size;
894

895
	if (ctxt->_eip == fc->end) {
896
		unsigned long linear;
897
898
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
899
		cur_size = fc->end - fc->start;
900
901
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
902
		rc = __linearize(ctxt, addr, size, false, true, &linear);
903
		if (unlikely(rc != X86EMUL_CONTINUE))
904
			return rc;
905
906
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
907
		if (unlikely(rc != X86EMUL_CONTINUE))
908
			return rc;
909
		fc->end += size;
910
	}
911
912
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
913
	return X86EMUL_CONTINUE;
914
915
916
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
917
			 void *dest, unsigned size)
918
{
919
	int rc;
920

921
	/* x86 instructions are limited to 15 bytes. */
922
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
923
		return X86EMUL_UNHANDLEABLE;
924
	while (size--) {
925
		rc = do_insn_fetch_byte(ctxt, dest++);
926
		if (rc != X86EMUL_CONTINUE)
927
928
			return rc;
	}
929
	return X86EMUL_CONTINUE;
930
931
}

932
/* Fetch next part of the instruction being emulated. */
933
#define insn_fetch(_type, _ctxt)					\
934
({	unsigned long _x;						\
935
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
936
937
938
939
940
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

941
942
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
943
944
945
946
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

947
948
949
950
951
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
952
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
953
			     int highbyte_regs)
Avi Kivity's avatar
Avi Kivity committed
954
955
956
957
{
	void *p;

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
958
959
960
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
Avi Kivity's avatar
Avi Kivity committed
961
962
963
964
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
965
			   struct segmented_address addr,
Avi Kivity's avatar
Avi Kivity committed
966
967
968
969
970
971
972
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
973
	rc = segmented_read_std(ctxt, addr, size, 2);
974
	if (rc != X86EMUL_CONTINUE)
Avi Kivity's avatar
Avi Kivity committed
975
		return rc;
976
	addr.ea += 2;
977
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
Avi Kivity's avatar
Avi Kivity committed
978
979
980
	return rc;
}

981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);