emulate.c 126 KB
Newer Older
Avi Kivity's avatar
Avi Kivity committed
1
/******************************************************************************
2
 * emulate.c
Avi Kivity's avatar
Avi Kivity committed
3
4
5
6
7
8
 *
 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
 *
 * Copyright (c) 2005 Keir Fraser
 *
 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9
 * privileged instructions:
Avi Kivity's avatar
Avi Kivity committed
10
11
 *
 * Copyright (C) 2006 Qumranet
12
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity's avatar
Avi Kivity committed
13
14
15
16
17
18
19
20
21
22
 *
 *   Avi Kivity <avi@qumranet.com>
 *   Yaniv Kamay <yaniv@qumranet.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
 */

23
#include <linux/kvm_host.h>
24
#include "kvm_cache_regs.h"
Avi Kivity's avatar
Avi Kivity committed
25
#include <linux/module.h>
26
#include <asm/kvm_emulate.h>
27
#include <linux/stringify.h>
Avi Kivity's avatar
Avi Kivity committed
28

29
#include "x86.h"
30
#include "tss.h"
31

32
33
34
/*
 * Operand types
 */
35
36
37
38
39
40
41
42
43
#define OpNone             0ull
#define OpImplicit         1ull  /* No generic decode */
#define OpReg              2ull  /* Register */
#define OpMem              3ull  /* Memory */
#define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
#define OpDI               5ull  /* ES:DI/EDI/RDI */
#define OpMem64            6ull  /* Memory, 64-bit */
#define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
#define OpDX               8ull  /* DX register */
44
45
46
#define OpCL               9ull  /* CL register (for shifts) */
#define OpImmByte         10ull  /* 8-bit sign extended immediate */
#define OpOne             11ull  /* Implied 1 */
47
#define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48
49
50
51
52
53
54
#define OpMem16           13ull  /* Memory operand (16-bit). */
#define OpMem32           14ull  /* Memory operand (32-bit). */
#define OpImmU            15ull  /* Immediate operand, zero extended */
#define OpSI              16ull  /* SI/ESI/RSI */
#define OpImmFAddr        17ull  /* Immediate far address */
#define OpMemFAddr        18ull  /* Far address in memory */
#define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55
56
57
58
59
60
#define OpES              20ull  /* ES */
#define OpCS              21ull  /* CS */
#define OpSS              22ull  /* SS */
#define OpDS              23ull  /* DS */
#define OpFS              24ull  /* FS */
#define OpGS              25ull  /* GS */
61
#define OpMem8            26ull  /* 8-bit zero extended memory operand */
62
#define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
Paolo Bonzini's avatar
Paolo Bonzini committed
63
#define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64
65

#define OpBits             5  /* Width of operand field */
66
#define OpMask             ((1ull << OpBits) - 1)
67

Avi Kivity's avatar
Avi Kivity committed
68
69
70
71
72
73
74
75
76
77
/*
 * Opcode effective-address decode tables.
 * Note that we only emulate instructions that have at least one memory
 * operand (excluding implicit stack references). We assume that stack
 * references and instruction fetches will never occur in special memory
 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
 * not be handled.
 */

/* Operand sizes: 8-bit operands or specified/overridden size. */
78
#define ByteOp      (1<<0)	/* 8-bit operands. */
Avi Kivity's avatar
Avi Kivity committed
79
/* Destination operand type. */
80
81
82
83
84
85
86
87
88
89
#define DstShift    1
#define ImplicitOps (OpImplicit << DstShift)
#define DstReg      (OpReg << DstShift)
#define DstMem      (OpMem << DstShift)
#define DstAcc      (OpAcc << DstShift)
#define DstDI       (OpDI << DstShift)
#define DstMem64    (OpMem64 << DstShift)
#define DstImmUByte (OpImmUByte << DstShift)
#define DstDX       (OpDX << DstShift)
#define DstMask     (OpMask << DstShift)
Avi Kivity's avatar
Avi Kivity committed
90
/* Source operand type. */
91
92
93
94
95
96
97
98
99
100
101
102
#define SrcShift    6
#define SrcNone     (OpNone << SrcShift)
#define SrcReg      (OpReg << SrcShift)
#define SrcMem      (OpMem << SrcShift)
#define SrcMem16    (OpMem16 << SrcShift)
#define SrcMem32    (OpMem32 << SrcShift)
#define SrcImm      (OpImm << SrcShift)
#define SrcImmByte  (OpImmByte << SrcShift)
#define SrcOne      (OpOne << SrcShift)
#define SrcImmUByte (OpImmUByte << SrcShift)
#define SrcImmU     (OpImmU << SrcShift)
#define SrcSI       (OpSI << SrcShift)
Paolo Bonzini's avatar
Paolo Bonzini committed
103
#define SrcXLat     (OpXLat << SrcShift)
104
105
106
107
#define SrcImmFAddr (OpImmFAddr << SrcShift)
#define SrcMemFAddr (OpMemFAddr << SrcShift)
#define SrcAcc      (OpAcc << SrcShift)
#define SrcImmU16   (OpImmU16 << SrcShift)
108
#define SrcImm64    (OpImm64 << SrcShift)
109
#define SrcDX       (OpDX << SrcShift)
110
#define SrcMem8     (OpMem8 << SrcShift)
111
#define SrcMask     (OpMask << SrcShift)
112
113
114
115
116
117
118
119
120
#define BitOp       (1<<11)
#define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
#define String      (1<<13)     /* String instruction (rep capable) */
#define Stack       (1<<14)     /* Stack instruction (push/pop) */
#define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
#define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
#define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
#define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
#define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
121
#define Escape      (5<<15)     /* Escape to coprocessor instruction */
122
#define Sse         (1<<18)     /* SSE Vector instruction */
123
124
125
126
/* Generic ModRM decode. */
#define ModRM       (1<<19)
/* Destination is only written; never read. */
#define Mov         (1<<20)
127
/* Misc flags */
128
#define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
129
#define VendorSpecific (1<<22) /* Vendor specific instruction */
130
#define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
131
#define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
132
#define Undefined   (1<<25) /* No Such Instruction */
133
#define Lock        (1<<26) /* lock prefix is allowed for the instruction */
134
#define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
135
#define No64	    (1<<28)
136
#define PageTable   (1 << 29)   /* instruction used to write page table */
137
#define NotImpl     (1 << 30)   /* instruction is not implemented */
138
/* Source 2 operand type */
139
#define Src2Shift   (31)
140
141
142
143
144
#define Src2None    (OpNone << Src2Shift)
#define Src2CL      (OpCL << Src2Shift)
#define Src2ImmByte (OpImmByte << Src2Shift)
#define Src2One     (OpOne << Src2Shift)
#define Src2Imm     (OpImm << Src2Shift)
145
146
147
148
149
150
#define Src2ES      (OpES << Src2Shift)
#define Src2CS      (OpCS << Src2Shift)
#define Src2SS      (OpSS << Src2Shift)
#define Src2DS      (OpDS << Src2Shift)
#define Src2FS      (OpFS << Src2Shift)
#define Src2GS      (OpGS << Src2Shift)
151
#define Src2Mask    (OpMask << Src2Shift)
Avi Kivity's avatar
Avi Kivity committed
152
#define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
153
154
155
#define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
#define Unaligned   ((u64)1 << 42)  /* Explicitly unaligned (e.g. MOVDQU) */
#define Avx         ((u64)1 << 43)  /* Advanced Vector Extensions */
156
#define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
157
#define NoWrite     ((u64)1 << 45)  /* No writeback */
Avi Kivity's avatar
Avi Kivity committed
158

159
160
161
162
163
164
165
166
#define X2(x...) x, x
#define X3(x...) X2(x), x
#define X4(x...) X2(x), X2(x)
#define X5(x...) X4(x), x
#define X6(x...) X4(x), X2(x)
#define X7(x...) X4(x), X3(x)
#define X8(x...) X4(x), X4(x)
#define X16(x...) X8(x), X8(x)
167

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
#define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
#define FASTOP_SIZE 8

/*
 * fastop functions have a special calling convention:
 *
 * dst:    [rdx]:rax  (in/out)
 * src:    rbx        (in/out)
 * src2:   rcx        (in)
 * flags:  rflags     (in/out)
 *
 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
 * different operand sizes can be reached by calculation, rather than a jump
 * table (which would be bigger than the code).
 *
 * fastop functions are declared as taking a never-defined fastop parameter,
 * so they can't be called from C directly.
 */

struct fastop;

189
struct opcode {
190
191
	u64 flags : 56;
	u64 intercept : 8;
192
	union {
193
		int (*execute)(struct x86_emulate_ctxt *ctxt);
194
195
196
		const struct opcode *group;
		const struct group_dual *gdual;
		const struct gprefix *gprefix;
197
		const struct escape *esc;
198
		void (*fastop)(struct fastop *fake);
199
	} u;
200
	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201
202
203
204
205
};

struct group_dual {
	struct opcode mod012[8];
	struct opcode mod3[8];
206
207
};

208
209
210
211
212
213
214
struct gprefix {
	struct opcode pfx_no;
	struct opcode pfx_66;
	struct opcode pfx_f2;
	struct opcode pfx_f3;
};

215
216
217
218
219
struct escape {
	struct opcode op[8];
	struct opcode high[64];
};

Avi Kivity's avatar
Avi Kivity committed
220
/* EFLAGS bit definitions. */
221
222
223
224
#define EFLG_ID (1<<21)
#define EFLG_VIP (1<<20)
#define EFLG_VIF (1<<19)
#define EFLG_AC (1<<18)
225
226
#define EFLG_VM (1<<17)
#define EFLG_RF (1<<16)
227
228
#define EFLG_IOPL (3<<12)
#define EFLG_NT (1<<14)
Avi Kivity's avatar
Avi Kivity committed
229
230
#define EFLG_OF (1<<11)
#define EFLG_DF (1<<10)
231
#define EFLG_IF (1<<9)
232
#define EFLG_TF (1<<8)
Avi Kivity's avatar
Avi Kivity committed
233
234
235
236
237
238
#define EFLG_SF (1<<7)
#define EFLG_ZF (1<<6)
#define EFLG_AF (1<<4)
#define EFLG_PF (1<<2)
#define EFLG_CF (1<<0)

239
240
241
#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
#define EFLG_RESERVED_ONE_MASK 2

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	if (!(ctxt->regs_valid & (1 << nr))) {
		ctxt->regs_valid |= 1 << nr;
		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
	}
	return ctxt->_regs[nr];
}

static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	ctxt->regs_valid |= 1 << nr;
	ctxt->regs_dirty |= 1 << nr;
	return &ctxt->_regs[nr];
}

static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
{
	reg_read(ctxt, nr);
	return reg_write(ctxt, nr);
}

static void writeback_registers(struct x86_emulate_ctxt *ctxt)
{
	unsigned reg;

	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
}

static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
{
	ctxt->regs_dirty = 0;
	ctxt->regs_valid = 0;
}

Avi Kivity's avatar
Avi Kivity committed
278
279
280
281
282
283
284
/*
 * Instruction emulation:
 * Most instructions are emulated directly via a fragment of inline assembly
 * code. This allows us to save/restore EFLAGS and thus very easily pick up
 * any modified flags.
 */

285
#if defined(CONFIG_X86_64)
Avi Kivity's avatar
Avi Kivity committed
286
287
288
289
290
291
292
293
294
295
296
297
298
299
#define _LO32 "k"		/* force 32-bit operand */
#define _STK  "%%rsp"		/* stack pointer */
#elif defined(__i386__)
#define _LO32 ""		/* force 32-bit operand */
#define _STK  "%%esp"		/* stack pointer */
#endif

/*
 * These EFLAGS bits are restored from saved value during emulation, and
 * any changes are written back to the saved value after emulation.
 */
#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)

/* Before executing instruction: restore necessary bits in EFLAGS. */
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
#define _PRE_EFLAGS(_sav, _msk, _tmp)					\
	/* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
	"movl %"_sav",%"_LO32 _tmp"; "                                  \
	"push %"_tmp"; "                                                \
	"push %"_tmp"; "                                                \
	"movl %"_msk",%"_LO32 _tmp"; "                                  \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"pushf; "                                                       \
	"notl %"_LO32 _tmp"; "                                          \
	"andl %"_LO32 _tmp",("_STK"); "                                 \
	"andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); "	\
	"pop  %"_tmp"; "                                                \
	"orl  %"_LO32 _tmp",("_STK"); "                                 \
	"popf; "                                                        \
	"pop  %"_sav"; "
Avi Kivity's avatar
Avi Kivity committed
315
316
317
318
319
320
321
322
323

/* After executing instruction: write-back necessary bits in EFLAGS. */
#define _POST_EFLAGS(_sav, _msk, _tmp) \
	/* _sav |= EFLAGS & _msk; */		\
	"pushf; "				\
	"pop  %"_tmp"; "			\
	"andl %"_msk",%"_LO32 _tmp"; "		\
	"orl  %"_LO32 _tmp",%"_sav"; "

324
325
326
327
328
329
#ifdef CONFIG_X86_64
#define ON64(x) x
#else
#define ON64(x)
#endif

330
#define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype)	\
331
332
333
334
335
	do {								\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "4", "2")			\
			_op _suffix " %"_x"3,%1; "			\
			_POST_EFLAGS("0", "4", "2")			\
336
337
			: "=m" ((ctxt)->eflags),			\
			  "+q" (*(_dsttype*)&(ctxt)->dst.val),		\
338
			  "=&r" (_tmp)					\
339
			: _y ((ctxt)->src.val), "i" (EFLAGS_MASK));	\
340
	} while (0)
341
342


Avi Kivity's avatar
Avi Kivity committed
343
/* Raw emulation: instruction has two explicit operands. */
344
#define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy)		\
345
346
347
	do {								\
		unsigned long _tmp;					\
									\
348
		switch ((ctxt)->dst.bytes) {				\
349
		case 2:							\
350
			____emulate_2op(ctxt,_op,_wx,_wy,"w",u16);	\
351
352
			break;						\
		case 4:							\
353
			____emulate_2op(ctxt,_op,_lx,_ly,"l",u32);	\
354
355
			break;						\
		case 8:							\
356
			ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
357
358
			break;						\
		}							\
Avi Kivity's avatar
Avi Kivity committed
359
360
	} while (0)

361
#define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy)		     \
Avi Kivity's avatar
Avi Kivity committed
362
	do {								     \
363
		unsigned long _tmp;					     \
364
		switch ((ctxt)->dst.bytes) {				     \
Avi Kivity's avatar
Avi Kivity committed
365
		case 1:							     \
366
			____emulate_2op(ctxt,_op,_bx,_by,"b",u8);	     \
Avi Kivity's avatar
Avi Kivity committed
367
368
			break;						     \
		default:						     \
369
			__emulate_2op_nobyte(ctxt, _op,			     \
Avi Kivity's avatar
Avi Kivity committed
370
371
372
373
374
375
					     _wx, _wy, _lx, _ly, _qx, _qy);  \
			break;						     \
		}							     \
	} while (0)

/* Source operand is byte-sized and may be restricted to just %cl. */
376
377
#define emulate_2op_SrcB(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
Avi Kivity's avatar
Avi Kivity committed
378
379

/* Source operand is byte, word, long or quad sized. */
380
381
#define emulate_2op_SrcV(ctxt, _op)					\
	__emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
382
383

/* Source operand is word, long or quad sized. */
384
385
#define emulate_2op_SrcV_nobyte(ctxt, _op)				\
	__emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
Avi Kivity's avatar
Avi Kivity committed
386

387
/* Instruction has three operands and one operand is stored in ECX register */
388
#define __emulate_2op_cl(ctxt, _op, _suffix, _type)		\
389
390
	do {								\
		unsigned long _tmp;					\
391
392
393
		_type _clv  = (ctxt)->src2.val;				\
		_type _srcv = (ctxt)->src.val;				\
		_type _dstv = (ctxt)->dst.val;				\
394
395
396
397
398
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "2")			\
			_op _suffix " %4,%1 \n"				\
			_POST_EFLAGS("0", "5", "2")			\
399
			: "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
400
401
402
			: "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK)	\
			);						\
									\
403
404
405
		(ctxt)->src2.val  = (unsigned long) _clv;		\
		(ctxt)->src2.val = (unsigned long) _srcv;		\
		(ctxt)->dst.val = (unsigned long) _dstv;		\
406
407
	} while (0)

408
#define emulate_2op_cl(ctxt, _op)					\
409
	do {								\
410
		switch ((ctxt)->dst.bytes) {				\
411
		case 2:							\
412
			__emulate_2op_cl(ctxt, _op, "w", u16);		\
413
414
			break;						\
		case 4:							\
415
			__emulate_2op_cl(ctxt, _op, "l", u32);		\
416
417
			break;						\
		case 8:							\
418
			ON64(__emulate_2op_cl(ctxt, _op, "q", ulong));	\
419
420
			break;						\
		}							\
421
422
	} while (0)

423
#define __emulate_1op(ctxt, _op, _suffix)				\
Avi Kivity's avatar
Avi Kivity committed
424
425
426
	do {								\
		unsigned long _tmp;					\
									\
427
428
429
430
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "3", "2")			\
			_op _suffix " %1; "				\
			_POST_EFLAGS("0", "3", "2")			\
431
			: "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
432
433
434
435
436
			  "=&r" (_tmp)					\
			: "i" (EFLAGS_MASK));				\
	} while (0)

/* Instruction has only one explicit operand (no source operand). */
437
#define emulate_1op(ctxt, _op)						\
438
	do {								\
439
440
441
442
443
		switch ((ctxt)->dst.bytes) {				\
		case 1:	__emulate_1op(ctxt, _op, "b"); break;		\
		case 2:	__emulate_1op(ctxt, _op, "w"); break;		\
		case 4:	__emulate_1op(ctxt, _op, "l"); break;		\
		case 8:	ON64(__emulate_1op(ctxt, _op, "q")); break;	\
Avi Kivity's avatar
Avi Kivity committed
444
445
446
		}							\
	} while (0)

447
448
static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));

449
450
451
452
453
454
455
456
457
458
459
460
461
#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
#define FOP_RET   "ret \n\t"

#define FOP_START(op) \
	extern void em_##op(struct fastop *fake); \
	asm(".pushsection .text, \"ax\" \n\t" \
	    ".global em_" #op " \n\t" \
            FOP_ALIGN \
	    "em_" #op ": \n\t"

#define FOP_END \
	    ".popsection")

462
463
#define FOPNOP() FOP_ALIGN FOP_RET

464
465
466
467
468
469
470
471
472
473
474
#define FOP1E(op,  dst) \
	FOP_ALIGN #op " %" #dst " \n\t" FOP_RET

#define FASTOP1(op) \
	FOP_START(op) \
	FOP1E(op##b, al) \
	FOP1E(op##w, ax) \
	FOP1E(op##l, eax) \
	ON64(FOP1E(op##q, rax))	\
	FOP_END

475
476
477
478
479
480
481
482
483
484
485
#define FOP2E(op,  dst, src)	   \
	FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET

#define FASTOP2(op) \
	FOP_START(op) \
	FOP2E(op##b, al, bl) \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

486
487
488
489
490
491
492
493
494
/* 2 operand, word only */
#define FASTOP2W(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP2E(op##w, ax, bx) \
	FOP2E(op##l, eax, ebx) \
	ON64(FOP2E(op##q, rax, rbx)) \
	FOP_END

495
496
497
498
499
500
501
502
503
/* 2 operand, src is CL */
#define FASTOP2CL(op) \
	FOP_START(op) \
	FOP2E(op##b, al, cl) \
	FOP2E(op##w, ax, cl) \
	FOP2E(op##l, eax, cl) \
	ON64(FOP2E(op##q, rax, cl)) \
	FOP_END

504
505
506
507
508
509
510
511
512
513
514
515
#define FOP3E(op,  dst, src, src2) \
	FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET

/* 3-operand, word-only, src2=cl */
#define FASTOP3WCL(op) \
	FOP_START(op) \
	FOPNOP() \
	FOP3E(op##w, ax, bx, cl) \
	FOP3E(op##l, eax, ebx, cl) \
	ON64(FOP3E(op##q, rax, rbx, cl)) \
	FOP_END

516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
/* Special case for SETcc - 1 instruction per cc */
#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"

FOP_START(setcc)
FOP_SETCC(seto)
FOP_SETCC(setno)
FOP_SETCC(setc)
FOP_SETCC(setnc)
FOP_SETCC(setz)
FOP_SETCC(setnz)
FOP_SETCC(setbe)
FOP_SETCC(setnbe)
FOP_SETCC(sets)
FOP_SETCC(setns)
FOP_SETCC(setp)
FOP_SETCC(setnp)
FOP_SETCC(setl)
FOP_SETCC(setnl)
FOP_SETCC(setle)
FOP_SETCC(setnle)
FOP_END;

538
#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
539
540
	do {								\
		unsigned long _tmp;					\
541
542
		ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX);		\
		ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX);		\
543
544
545
546
547
548
549
550
551
552
553
554
									\
		__asm__ __volatile__ (					\
			_PRE_EFLAGS("0", "5", "1")			\
			"1: \n\t"					\
			_op _suffix " %6; "				\
			"2: \n\t"					\
			_POST_EFLAGS("0", "5", "1")			\
			".pushsection .fixup,\"ax\" \n\t"		\
			"3: movb $1, %4 \n\t"				\
			"jmp 2b \n\t"					\
			".popsection \n\t"				\
			_ASM_EXTABLE(1b, 3b)				\
555
556
			: "=m" ((ctxt)->eflags), "=&r" (_tmp),		\
			  "+a" (*rax), "+d" (*rdx), "+qm"(_ex)		\
557
			: "i" (EFLAGS_MASK), "m" ((ctxt)->src.val));	\
558
559
	} while (0)

560
/* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
561
#define emulate_1op_rax_rdx(ctxt, _op, _ex)	\
562
	do {								\
563
		switch((ctxt)->src.bytes) {				\
564
		case 1:							\
565
			__emulate_1op_rax_rdx(ctxt, _op, "b", _ex);	\
566
567
			break;						\
		case 2:							\
568
			__emulate_1op_rax_rdx(ctxt, _op, "w", _ex);	\
569
570
			break;						\
		case 4:							\
571
			__emulate_1op_rax_rdx(ctxt, _op, "l", _ex);	\
572
573
			break;						\
		case 8: ON64(						\
574
			__emulate_1op_rax_rdx(ctxt, _op, "q", _ex));	\
575
576
577
578
			break;						\
		}							\
	} while (0)

579
580
581
582
583
584
static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
				    enum x86_intercept intercept,
				    enum x86_intercept_stage stage)
{
	struct x86_instruction_info info = {
		.intercept  = intercept,
585
586
587
588
589
590
591
592
		.rep_prefix = ctxt->rep_prefix,
		.modrm_mod  = ctxt->modrm_mod,
		.modrm_reg  = ctxt->modrm_reg,
		.modrm_rm   = ctxt->modrm_rm,
		.src_val    = ctxt->src.val64,
		.src_bytes  = ctxt->src.bytes,
		.dst_bytes  = ctxt->dst.bytes,
		.ad_bytes   = ctxt->ad_bytes,
593
594
595
		.next_rip   = ctxt->eip,
	};

596
	return ctxt->ops->intercept(ctxt, &info, stage);
597
598
}

Avi Kivity's avatar
Avi Kivity committed
599
600
601
602
603
static void assign_masked(ulong *dest, ulong src, ulong mask)
{
	*dest = (*dest & ~mask) | (src & mask);
}

604
static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
605
{
606
	return (1UL << (ctxt->ad_bytes << 3)) - 1;
607
608
}

Avi Kivity's avatar
Avi Kivity committed
609
610
611
612
613
614
615
616
617
618
619
static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
{
	u16 sel;
	struct desc_struct ss;

	if (ctxt->mode == X86EMUL_MODE_PROT64)
		return ~0UL;
	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
}

620
621
622
623
624
static int stack_size(struct x86_emulate_ctxt *ctxt)
{
	return (__fls(stack_mask(ctxt)) + 1) >> 3;
}

Avi Kivity's avatar
Avi Kivity committed
625
/* Access/update address held in a register, based on addressing mode. */
626
static inline unsigned long
627
address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
628
{
629
	if (ctxt->ad_bytes == sizeof(unsigned long))
630
631
		return reg;
	else
632
		return reg & ad_mask(ctxt);
633
634
635
}

static inline unsigned long
636
register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
637
{
638
	return address_mask(ctxt, reg);
639
640
}

641
642
643
644
645
static void masked_increment(ulong *reg, ulong mask, int inc)
{
	assign_masked(reg, *reg + inc, mask);
}

646
static inline void
647
register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
648
{
649
650
	ulong mask;

651
	if (ctxt->ad_bytes == sizeof(unsigned long))
652
		mask = ~0UL;
653
	else
654
655
656
657
658
659
		mask = ad_mask(ctxt);
	masked_increment(reg, mask, inc);
}

static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
{
660
	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
661
}
Avi Kivity's avatar
Avi Kivity committed
662

663
static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
664
{
665
	register_address_increment(ctxt, &ctxt->_eip, rel);
666
}
667

668
669
670
671
672
673
674
static u32 desc_limit_scaled(struct desc_struct *desc)
{
	u32 limit = get_desc_limit(desc);

	return desc->g ? (limit << 12) | 0xfff : limit;
}

675
static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
676
{
677
678
	ctxt->has_seg_override = true;
	ctxt->seg_override = seg;
679
680
}

681
static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
682
683
684
685
{
	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
		return 0;

686
	return ctxt->ops->get_cached_segment_base(ctxt, seg);
687
688
}

689
static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
690
{
691
	if (!ctxt->has_seg_override)
692
693
		return 0;

694
	return ctxt->seg_override;
695
696
}

697
698
static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
			     u32 error, bool valid)
699
{
700
701
702
	ctxt->exception.vector = vec;
	ctxt->exception.error_code = error;
	ctxt->exception.error_code_valid = valid;
703
	return X86EMUL_PROPAGATE_FAULT;
704
705
}

706
707
708
709
710
static int emulate_db(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, DB_VECTOR, 0, false);
}

711
static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
712
{
713
	return emulate_exception(ctxt, GP_VECTOR, err, true);
714
715
}

716
717
718
719
720
static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
{
	return emulate_exception(ctxt, SS_VECTOR, err, true);
}

721
static int emulate_ud(struct x86_emulate_ctxt *ctxt)
722
{
723
	return emulate_exception(ctxt, UD_VECTOR, 0, false);
724
725
}

726
static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
727
{
728
	return emulate_exception(ctxt, TS_VECTOR, err, true);
729
730
}

731
732
static int emulate_de(struct x86_emulate_ctxt *ctxt)
{
733
	return emulate_exception(ctxt, DE_VECTOR, 0, false);
734
735
}

Avi Kivity's avatar
Avi Kivity committed
736
737
738
739
740
static int emulate_nm(struct x86_emulate_ctxt *ctxt)
{
	return emulate_exception(ctxt, NM_VECTOR, 0, false);
}

741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
{
	u16 selector;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
	return selector;
}

static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
				 unsigned seg)
{
	u16 dummy;
	u32 base3;
	struct desc_struct desc;

	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
}

761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
/*
 * x86 defines three classes of vector instructions: explicitly
 * aligned, explicitly unaligned, and the rest, which change behaviour
 * depending on whether they're AVX encoded or not.
 *
 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
 * subject to the same check.
 */
static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
{
	if (likely(size < 16))
		return false;

	if (ctxt->d & Aligned)
		return true;
	else if (ctxt->d & Unaligned)
		return false;
	else if (ctxt->d & Avx)
		return false;
	else
		return true;
}

784
static int __linearize(struct x86_emulate_ctxt *ctxt,
785
		     struct segmented_address addr,
786
		     unsigned size, bool write, bool fetch,
787
788
		     ulong *linear)
{
789
790
	struct desc_struct desc;
	bool usable;
791
	ulong la;
792
	u32 lim;
793
	u16 sel;
794
	unsigned cpl;
795

796
	la = seg_base(ctxt, addr.seg) + addr.ea;
797
798
799
800
801
802
	switch (ctxt->mode) {
	case X86EMUL_MODE_PROT64:
		if (((signed long)la << 16) >> 16 != la)
			return emulate_gp(ctxt, 0);
		break;
	default:
803
804
		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
						addr.seg);
805
806
		if (!usable)
			goto bad;
807
808
809
		/* code segment in protected mode or read-only data segment */
		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
					|| !(desc.type & 2)) && write)
810
811
			goto bad;
		/* unreadable code segment */
812
		if (!fetch && (desc.type & 8) && !(desc.type & 2))
813
814
815
816
817
818
819
			goto bad;
		lim = desc_limit_scaled(&desc);
		if ((desc.type & 8) || !(desc.type & 4)) {
			/* expand-up segment */
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		} else {
Guo Chao's avatar
Guo Chao committed
820
			/* expand-down segment */
821
822
823
824
825
826
			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
				goto bad;
			lim = desc.d ? 0xffffffff : 0xffff;
			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
				goto bad;
		}
827
		cpl = ctxt->ops->cpl(ctxt);
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
		if (!(desc.type & 8)) {
			/* data segment */
			if (cpl > desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && !(desc.type & 4)) {
			/* nonconforming code segment */
			if (cpl != desc.dpl)
				goto bad;
		} else if ((desc.type & 8) && (desc.type & 4)) {
			/* conforming code segment */
			if (cpl < desc.dpl)
				goto bad;
		}
		break;
	}
843
	if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
844
		la &= (u32)-1;
845
846
	if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
		return emulate_gp(ctxt, 0);
847
848
	*linear = la;
	return X86EMUL_CONTINUE;
849
850
bad:
	if (addr.seg == VCPU_SREG_SS)
851
		return emulate_ss(ctxt, sel);
852
	else
853
		return emulate_gp(ctxt, sel);
854
855
}

856
857
858
859
860
861
862
863
864
static int linearize(struct x86_emulate_ctxt *ctxt,
		     struct segmented_address addr,
		     unsigned size, bool write,
		     ulong *linear)
{
	return __linearize(ctxt, addr, size, write, false, linear);
}


865
866
867
868
869
static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
			      struct segmented_address addr,
			      void *data,
			      unsigned size)
{
870
871
872
	int rc;
	ulong linear;

873
	rc = linearize(ctxt, addr, size, false, &linear);
874
875
	if (rc != X86EMUL_CONTINUE)
		return rc;
876
	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
877
878
}

879
880
881
882
883
884
885
886
/*
 * Fetch the next byte of the instruction being emulated which is pointed to
 * by ctxt->_eip, then increment ctxt->_eip.
 *
 * Also prefetch the remaining bytes of the instruction without crossing page
 * boundary if they are not in fetch_cache yet.
 */
static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
887
{
888
	struct fetch_cache *fc = &ctxt->fetch;
889
	int rc;
890
	int size, cur_size;
891

892
	if (ctxt->_eip == fc->end) {
893
		unsigned long linear;
894
895
		struct segmented_address addr = { .seg = VCPU_SREG_CS,
						  .ea  = ctxt->_eip };
896
		cur_size = fc->end - fc->start;
897
898
		size = min(15UL - cur_size,
			   PAGE_SIZE - offset_in_page(ctxt->_eip));
899
		rc = __linearize(ctxt, addr, size, false, true, &linear);
900
		if (unlikely(rc != X86EMUL_CONTINUE))
901
			return rc;
902
903
		rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
				      size, &ctxt->exception);
904
		if (unlikely(rc != X86EMUL_CONTINUE))
905
			return rc;
906
		fc->end += size;
907
	}
908
909
	*dest = fc->data[ctxt->_eip - fc->start];
	ctxt->_eip++;
910
	return X86EMUL_CONTINUE;
911
912
913
}

static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
914
			 void *dest, unsigned size)
915
{
916
	int rc;
917

918
	/* x86 instructions are limited to 15 bytes. */
919
	if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
920
		return X86EMUL_UNHANDLEABLE;
921
	while (size--) {
922
		rc = do_insn_fetch_byte(ctxt, dest++);
923
		if (rc != X86EMUL_CONTINUE)
924
925
			return rc;
	}
926
	return X86EMUL_CONTINUE;
927
928
}

929
/* Fetch next part of the instruction being emulated. */
930
#define insn_fetch(_type, _ctxt)					\
931
({	unsigned long _x;						\
932
	rc = do_insn_fetch(_ctxt, &_x, sizeof(_type));			\
933
934
935
936
937
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
	(_type)_x;							\
})

938
939
#define insn_fetch_arr(_arr, _size, _ctxt)				\
({	rc = do_insn_fetch(_ctxt, _arr, (_size));			\
940
941
942
943
	if (rc != X86EMUL_CONTINUE)					\
		goto done;						\
})

944
945
946
947
948
/*
 * Given the 'reg' portion of a ModRM byte, and a register block, return a
 * pointer into the block that addresses the relevant register.
 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
 */
949
static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
950
			     int highbyte_regs)
Avi Kivity's avatar
Avi Kivity committed
951
952
953
954
{
	void *p;

	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
955
956
957
		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
	else
		p = reg_rmw(ctxt, modrm_reg);
Avi Kivity's avatar
Avi Kivity committed
958
959
960
961
	return p;
}

static int read_descriptor(struct x86_emulate_ctxt *ctxt,
962
			   struct segmented_address addr,
Avi Kivity's avatar
Avi Kivity committed
963
964
965
966
967
968
969
			   u16 *size, unsigned long *address, int op_bytes)
{
	int rc;

	if (op_bytes == 2)
		op_bytes = 3;
	*address = 0;
970
	rc = segmented_read_std(ctxt, addr, size, 2);
971
	if (rc != X86EMUL_CONTINUE)
Avi Kivity's avatar
Avi Kivity committed
972
		return rc;
973
	addr.ea += 2;
974
	rc = segmented_read_std(ctxt, addr, address, op_bytes);
Avi Kivity's avatar
Avi Kivity committed
975
976
977
	return rc;
}

978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
FASTOP2(add);
FASTOP2(or);
FASTOP2(adc);
FASTOP2(sbb);
FASTOP2(and);
FASTOP2(sub);
FASTOP2(xor);
FASTOP2(cmp);
FASTOP2(test);

FASTOP3WCL(shld);
FASTOP3WCL(shrd);

FASTOP2W(imul);

FASTOP1(not);
FASTOP1(neg);
FASTOP1(inc);
FASTOP1(dec);

FASTOP2CL(rol);
FASTOP2CL(ror);
FASTOP2CL(rcl);