exec.c 70.7 KB
Newer Older
bellard's avatar
bellard committed
1
/*
bellard's avatar
bellard committed
2
 *  virtual page mapping and translated block handling
bellard's avatar
bellard committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
 * 
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
bellard's avatar
bellard committed
20
#include "config.h"
bellard's avatar
bellard committed
21
22
23
#ifdef _WIN32
#include <windows.h>
#else
bellard's avatar
bellard committed
24
#include <sys/types.h>
bellard's avatar
bellard committed
25
26
#include <sys/mman.h>
#endif
bellard's avatar
bellard committed
27
28
29
30
31
32
33
34
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <inttypes.h>

bellard's avatar
bellard committed
35
36
#include "cpu.h"
#include "exec-all.h"
37
38
39
#if defined(CONFIG_USER_ONLY)
#include <qemu.h>
#endif
bellard's avatar
bellard committed
40

bellard's avatar
bellard committed
41
//#define DEBUG_TB_INVALIDATE
bellard's avatar
bellard committed
42
//#define DEBUG_FLUSH
43
//#define DEBUG_TLB
pbrook's avatar
pbrook committed
44
//#define DEBUG_UNASSIGNED
bellard's avatar
bellard committed
45
46
47

/* make various TB consistency checks */
//#define DEBUG_TB_CHECK 
bellard's avatar
bellard committed
48
//#define DEBUG_TLB_CHECK 
bellard's avatar
bellard committed
49

50
51
52
53
54
#if !defined(CONFIG_USER_ONLY)
/* TB consistency checks only implemented for usermode emulation.  */
#undef DEBUG_TB_CHECK
#endif

bellard's avatar
bellard committed
55
56
57
/* threshold to flush the translated code buffer */
#define CODE_GEN_BUFFER_MAX_SIZE (CODE_GEN_BUFFER_SIZE - CODE_GEN_MAX_SIZE)

58
59
60
61
#define SMC_BITMAP_USE_THRESHOLD 10

#define MMAP_AREA_START        0x00000000
#define MMAP_AREA_END          0xa8000000
bellard's avatar
bellard committed
62

63
64
65
66
67
68
69
70
71
#if defined(TARGET_SPARC64)
#define TARGET_PHYS_ADDR_SPACE_BITS 41
#elif defined(TARGET_PPC64)
#define TARGET_PHYS_ADDR_SPACE_BITS 42
#else
/* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
#define TARGET_PHYS_ADDR_SPACE_BITS 32
#endif

bellard's avatar
bellard committed
72
TranslationBlock tbs[CODE_GEN_MAX_BLOCKS];
73
TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
bellard's avatar
bellard committed
74
int nb_tbs;
bellard's avatar
bellard committed
75
76
/* any access to the tbs or the page table must use this lock */
spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
bellard's avatar
bellard committed
77

bellard's avatar
bellard committed
78
uint8_t code_gen_buffer[CODE_GEN_BUFFER_SIZE] __attribute__((aligned (32)));
bellard's avatar
bellard committed
79
80
uint8_t *code_gen_ptr;

81
82
83
int phys_ram_size;
int phys_ram_fd;
uint8_t *phys_ram_base;
84
uint8_t *phys_ram_dirty;
85

bellard's avatar
bellard committed
86
87
88
89
90
CPUState *first_cpu;
/* current CPU in the current thread. It is only valid inside
   cpu_exec() */
CPUState *cpu_single_env; 

bellard's avatar
bellard committed
91
typedef struct PageDesc {
92
    /* list of TBs intersecting this ram page */
bellard's avatar
bellard committed
93
    TranslationBlock *first_tb;
94
95
96
97
98
99
100
    /* in order to optimize self modifying code, we count the number
       of lookups we do to a given page to use a bitmap */
    unsigned int code_write_count;
    uint8_t *code_bitmap;
#if defined(CONFIG_USER_ONLY)
    unsigned long flags;
#endif
bellard's avatar
bellard committed
101
102
} PageDesc;

103
104
typedef struct PhysPageDesc {
    /* offset in host memory of the page + io_index in the low 12 bits */
bellard's avatar
bellard committed
105
    uint32_t phys_offset;
106
107
} PhysPageDesc;

bellard's avatar
bellard committed
108
109
110
111
112
113
#define L2_BITS 10
#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)

#define L1_SIZE (1 << L1_BITS)
#define L2_SIZE (1 << L2_BITS)

114
static void io_mem_init(void);
bellard's avatar
bellard committed
115

116
117
118
119
unsigned long qemu_real_host_page_size;
unsigned long qemu_host_page_bits;
unsigned long qemu_host_page_size;
unsigned long qemu_host_page_mask;
bellard's avatar
bellard committed
120

121
/* XXX: for system emulation, it could just be an array */
bellard's avatar
bellard committed
122
static PageDesc *l1_map[L1_SIZE];
bellard's avatar
bellard committed
123
PhysPageDesc **l1_phys_map;
bellard's avatar
bellard committed
124

125
126
127
/* io memory support */
CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
128
void *io_mem_opaque[IO_MEM_NB_ENTRIES];
129
130
static int io_mem_nb;

131
132
133
134
135
/* log support */
char *logfilename = "/tmp/qemu.log";
FILE *logfile;
int loglevel;

bellard's avatar
bellard committed
136
137
138
139
140
/* statistics */
static int tlb_flush_count;
static int tb_flush_count;
static int tb_phys_invalidate_count;

bellard's avatar
bellard committed
141
static void page_init(void)
bellard's avatar
bellard committed
142
{
143
    /* NOTE: we can always suppose that qemu_host_page_size >=
bellard's avatar
bellard committed
144
       TARGET_PAGE_SIZE */
bellard's avatar
bellard committed
145
#ifdef _WIN32
bellard's avatar
bellard committed
146
147
148
149
150
151
152
153
154
155
    {
        SYSTEM_INFO system_info;
        DWORD old_protect;
        
        GetSystemInfo(&system_info);
        qemu_real_host_page_size = system_info.dwPageSize;
        
        VirtualProtect(code_gen_buffer, sizeof(code_gen_buffer),
                       PAGE_EXECUTE_READWRITE, &old_protect);
    }
bellard's avatar
bellard committed
156
#else
157
    qemu_real_host_page_size = getpagesize();
bellard's avatar
bellard committed
158
159
160
161
162
163
164
165
166
167
168
169
170
    {
        unsigned long start, end;

        start = (unsigned long)code_gen_buffer;
        start &= ~(qemu_real_host_page_size - 1);
        
        end = (unsigned long)code_gen_buffer + sizeof(code_gen_buffer);
        end += qemu_real_host_page_size - 1;
        end &= ~(qemu_real_host_page_size - 1);
        
        mprotect((void *)start, end - start, 
                 PROT_READ | PROT_WRITE | PROT_EXEC);
    }
bellard's avatar
bellard committed
171
#endif
bellard's avatar
bellard committed
172

173
174
175
176
177
178
179
180
    if (qemu_host_page_size == 0)
        qemu_host_page_size = qemu_real_host_page_size;
    if (qemu_host_page_size < TARGET_PAGE_SIZE)
        qemu_host_page_size = TARGET_PAGE_SIZE;
    qemu_host_page_bits = 0;
    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
        qemu_host_page_bits++;
    qemu_host_page_mask = ~(qemu_host_page_size - 1);
181
182
    l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
    memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
bellard's avatar
bellard committed
183
184
}

bellard's avatar
bellard committed
185
static inline PageDesc *page_find_alloc(unsigned int index)
bellard's avatar
bellard committed
186
187
188
189
190
191
192
{
    PageDesc **lp, *p;

    lp = &l1_map[index >> L2_BITS];
    p = *lp;
    if (!p) {
        /* allocate if not found */
193
        p = qemu_malloc(sizeof(PageDesc) * L2_SIZE);
bellard's avatar
bellard committed
194
        memset(p, 0, sizeof(PageDesc) * L2_SIZE);
bellard's avatar
bellard committed
195
196
197
198
199
        *lp = p;
    }
    return p + (index & (L2_SIZE - 1));
}

bellard's avatar
bellard committed
200
static inline PageDesc *page_find(unsigned int index)
bellard's avatar
bellard committed
201
202
203
204
205
206
{
    PageDesc *p;

    p = l1_map[index >> L2_BITS];
    if (!p)
        return 0;
bellard's avatar
bellard committed
207
208
209
    return p + (index & (L2_SIZE - 1));
}

210
static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
211
{
212
    void **lp, **p;
213
    PhysPageDesc *pd;
214

215
216
217
218
219
220
221
    p = (void **)l1_phys_map;
#if TARGET_PHYS_ADDR_SPACE_BITS > 32

#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
#endif
    lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
222
223
224
    p = *lp;
    if (!p) {
        /* allocate if not found */
225
226
227
228
229
230
231
232
        if (!alloc)
            return NULL;
        p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
        memset(p, 0, sizeof(void *) * L1_SIZE);
        *lp = p;
    }
#endif
    lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
233
234
235
    pd = *lp;
    if (!pd) {
        int i;
236
237
238
        /* allocate if not found */
        if (!alloc)
            return NULL;
239
240
241
242
        pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
        *lp = pd;
        for (i = 0; i < L2_SIZE; i++)
          pd[i].phys_offset = IO_MEM_UNASSIGNED;
243
    }
244
    return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
245
246
}

247
static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
248
{
249
    return phys_page_find_alloc(index, 0);
250
251
}

252
#if !defined(CONFIG_USER_ONLY)
bellard's avatar
bellard committed
253
static void tlb_protect_code(ram_addr_t ram_addr);
254
255
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr, 
                                    target_ulong vaddr);
256
#endif
bellard's avatar
bellard committed
257

bellard's avatar
bellard committed
258
void cpu_exec_init(CPUState *env)
bellard's avatar
bellard committed
259
{
bellard's avatar
bellard committed
260
261
262
    CPUState **penv;
    int cpu_index;

bellard's avatar
bellard committed
263
264
    if (!code_gen_ptr) {
        code_gen_ptr = code_gen_buffer;
bellard's avatar
bellard committed
265
        page_init();
266
        io_mem_init();
bellard's avatar
bellard committed
267
    }
bellard's avatar
bellard committed
268
269
270
271
272
273
274
275
276
    env->next_cpu = NULL;
    penv = &first_cpu;
    cpu_index = 0;
    while (*penv != NULL) {
        penv = (CPUState **)&(*penv)->next_cpu;
        cpu_index++;
    }
    env->cpu_index = cpu_index;
    *penv = env;
bellard's avatar
bellard committed
277
278
}

279
280
281
static inline void invalidate_page_bitmap(PageDesc *p)
{
    if (p->code_bitmap) {
282
        qemu_free(p->code_bitmap);
283
284
285
286
287
        p->code_bitmap = NULL;
    }
    p->code_write_count = 0;
}

bellard's avatar
bellard committed
288
289
290
291
292
293
294
295
296
/* set to NULL all the 'first_tb' fields in all PageDescs */
static void page_flush_tb(void)
{
    int i, j;
    PageDesc *p;

    for(i = 0; i < L1_SIZE; i++) {
        p = l1_map[i];
        if (p) {
297
298
299
300
301
            for(j = 0; j < L2_SIZE; j++) {
                p->first_tb = NULL;
                invalidate_page_bitmap(p);
                p++;
            }
bellard's avatar
bellard committed
302
303
304
305
306
        }
    }
}

/* flush all the translation blocks */
bellard's avatar
bellard committed
307
/* XXX: tb_flush is currently not thread safe */
bellard's avatar
bellard committed
308
void tb_flush(CPUState *env1)
bellard's avatar
bellard committed
309
{
bellard's avatar
bellard committed
310
    CPUState *env;
311
#if defined(DEBUG_FLUSH)
bellard's avatar
bellard committed
312
313
314
    printf("qemu: flush code_size=%d nb_tbs=%d avg_tb_size=%d\n", 
           code_gen_ptr - code_gen_buffer, 
           nb_tbs, 
315
           nb_tbs > 0 ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0);
bellard's avatar
bellard committed
316
317
#endif
    nb_tbs = 0;
bellard's avatar
bellard committed
318
319
320
321
    
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
    }
322

bellard's avatar
bellard committed
323
    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
bellard's avatar
bellard committed
324
    page_flush_tb();
325

bellard's avatar
bellard committed
326
    code_gen_ptr = code_gen_buffer;
bellard's avatar
bellard committed
327
328
    /* XXX: flush processor icache at this point if cache flush is
       expensive */
bellard's avatar
bellard committed
329
    tb_flush_count++;
bellard's avatar
bellard committed
330
331
332
333
334
335
336
337
338
}

#ifdef DEBUG_TB_CHECK

static void tb_invalidate_check(unsigned long address)
{
    TranslationBlock *tb;
    int i;
    address &= TARGET_PAGE_MASK;
339
340
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
bellard's avatar
bellard committed
341
342
343
            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                  address >= tb->pc + tb->size)) {
                printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
344
                       address, (long)tb->pc, tb->size);
bellard's avatar
bellard committed
345
346
347
348
349
350
351
352
353
354
355
            }
        }
    }
}

/* verify that all the pages have correct rights for code */
static void tb_page_check(void)
{
    TranslationBlock *tb;
    int i, flags1, flags2;
    
356
357
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
bellard's avatar
bellard committed
358
359
360
361
            flags1 = page_get_flags(tb->pc);
            flags2 = page_get_flags(tb->pc + tb->size - 1);
            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
362
                       (long)tb->pc, tb->size, flags1, flags2);
bellard's avatar
bellard committed
363
364
365
366
367
            }
        }
    }
}

bellard's avatar
bellard committed
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
void tb_jmp_check(TranslationBlock *tb)
{
    TranslationBlock *tb1;
    unsigned int n1;

    /* suppress any remaining jumps to this TB */
    tb1 = tb->jmp_first;
    for(;;) {
        n1 = (long)tb1 & 3;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        if (n1 == 2)
            break;
        tb1 = tb1->jmp_next[n1];
    }
    /* check end of list */
    if (tb1 != tb) {
        printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
    }
}

bellard's avatar
bellard committed
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
#endif

/* invalidate one TB */
static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
                             int next_offset)
{
    TranslationBlock *tb1;
    for(;;) {
        tb1 = *ptb;
        if (tb1 == tb) {
            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
            break;
        }
        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
    }
}

405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
{
    TranslationBlock *tb1;
    unsigned int n1;

    for(;;) {
        tb1 = *ptb;
        n1 = (long)tb1 & 3;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        if (tb1 == tb) {
            *ptb = tb1->page_next[n1];
            break;
        }
        ptb = &tb1->page_next[n1];
    }
}

bellard's avatar
bellard committed
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
static inline void tb_jmp_remove(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, **ptb;
    unsigned int n1;

    ptb = &tb->jmp_next[n];
    tb1 = *ptb;
    if (tb1) {
        /* find tb(n) in circular list */
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            if (n1 == 2) {
                ptb = &tb1->jmp_first;
            } else {
                ptb = &tb1->jmp_next[n1];
            }
        }
        /* now we can suppress tb(n) from the list */
        *ptb = tb->jmp_next[n];

        tb->jmp_next[n] = NULL;
    }
}

/* reset the jump entry 'n' of a TB so that it is not chained to
   another TB */
static inline void tb_reset_jump(TranslationBlock *tb, int n)
{
    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
}

457
static inline void tb_phys_invalidate(TranslationBlock *tb, unsigned int page_addr)
bellard's avatar
bellard committed
458
{
bellard's avatar
bellard committed
459
    CPUState *env;
460
    PageDesc *p;
bellard's avatar
bellard committed
461
    unsigned int h, n1;
462
463
    target_ulong phys_pc;
    TranslationBlock *tb1, *tb2;
bellard's avatar
bellard committed
464
    
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
    /* remove the TB from the hash list */
    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
    h = tb_phys_hash_func(phys_pc);
    tb_remove(&tb_phys_hash[h], tb, 
              offsetof(TranslationBlock, phys_hash_next));

    /* remove the TB from the page list */
    if (tb->page_addr[0] != page_addr) {
        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }
    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }

483
    tb_invalidated_flag = 1;
484

bellard's avatar
bellard committed
485
    /* remove the TB from the hash list */
486
    h = tb_jmp_cache_hash_func(tb->pc);
bellard's avatar
bellard committed
487
488
489
490
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        if (env->tb_jmp_cache[h] == tb)
            env->tb_jmp_cache[h] = NULL;
    }
bellard's avatar
bellard committed
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508

    /* suppress this TB from the two jump lists */
    tb_jmp_remove(tb, 0);
    tb_jmp_remove(tb, 1);

    /* suppress any remaining jumps to this TB */
    tb1 = tb->jmp_first;
    for(;;) {
        n1 = (long)tb1 & 3;
        if (n1 == 2)
            break;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        tb2 = tb1->jmp_next[n1];
        tb_reset_jump(tb1, n1);
        tb1->jmp_next[n1] = NULL;
        tb1 = tb2;
    }
    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
509

bellard's avatar
bellard committed
510
    tb_phys_invalidate_count++;
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
}

static inline void set_bits(uint8_t *tab, int start, int len)
{
    int end, mask, end1;

    end = start + len;
    tab += start >> 3;
    mask = 0xff << (start & 7);
    if ((start & ~7) == (end & ~7)) {
        if (start < end) {
            mask &= ~(0xff << (end & 7));
            *tab |= mask;
        }
    } else {
        *tab++ |= mask;
        start = (start + 8) & ~7;
        end1 = end & ~7;
        while (start < end1) {
            *tab++ = 0xff;
            start += 8;
        }
        if (start < end) {
            mask = ~(0xff << (end & 7));
            *tab |= mask;
        }
    }
}

static void build_page_bitmap(PageDesc *p)
{
    int n, tb_start, tb_end;
    TranslationBlock *tb;
    
545
    p->code_bitmap = qemu_malloc(TARGET_PAGE_SIZE / 8);
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
    if (!p->code_bitmap)
        return;
    memset(p->code_bitmap, 0, TARGET_PAGE_SIZE / 8);

    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->pc & ~TARGET_PAGE_MASK;
            tb_end = tb_start + tb->size;
            if (tb_end > TARGET_PAGE_SIZE)
                tb_end = TARGET_PAGE_SIZE;
        } else {
            tb_start = 0;
            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
        tb = tb->page_next[n];
    }
}

bellard's avatar
bellard committed
571
572
573
574
575
576
577
578
579
580
581
#ifdef TARGET_HAS_PRECISE_SMC

static void tb_gen_code(CPUState *env, 
                        target_ulong pc, target_ulong cs_base, int flags,
                        int cflags)
{
    TranslationBlock *tb;
    uint8_t *tc_ptr;
    target_ulong phys_pc, phys_page2, virt_page2;
    int code_gen_size;

bellard's avatar
bellard committed
582
583
    phys_pc = get_phys_addr_code(env, pc);
    tb = tb_alloc(pc);
bellard's avatar
bellard committed
584
585
586
587
    if (!tb) {
        /* flush must be done */
        tb_flush(env);
        /* cannot fail at this point */
bellard's avatar
bellard committed
588
        tb = tb_alloc(pc);
bellard's avatar
bellard committed
589
590
591
592
593
594
595
596
597
598
    }
    tc_ptr = code_gen_ptr;
    tb->tc_ptr = tc_ptr;
    tb->cs_base = cs_base;
    tb->flags = flags;
    tb->cflags = cflags;
    cpu_gen_code(env, tb, CODE_GEN_MAX_SIZE, &code_gen_size);
    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
    
    /* check next page if needed */
bellard's avatar
bellard committed
599
    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
bellard's avatar
bellard committed
600
    phys_page2 = -1;
bellard's avatar
bellard committed
601
    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
bellard's avatar
bellard committed
602
603
604
605
606
607
        phys_page2 = get_phys_addr_code(env, virt_page2);
    }
    tb_link_phys(tb, phys_pc, phys_page2);
}
#endif
    
608
609
/* invalidate all TBs which intersect with the target physical page
   starting in range [start;end[. NOTE: start and end must refer to
bellard's avatar
bellard committed
610
611
612
613
614
615
616
617
   the same physical page. 'is_cpu_write_access' should be true if called
   from a real cpu write access: the virtual CPU will exit the current
   TB if code is modified inside this TB. */
void tb_invalidate_phys_page_range(target_ulong start, target_ulong end, 
                                   int is_cpu_write_access)
{
    int n, current_tb_modified, current_tb_not_found, current_flags;
    CPUState *env = cpu_single_env;
618
    PageDesc *p;
619
    TranslationBlock *tb, *tb_next, *current_tb, *saved_tb;
620
    target_ulong tb_start, tb_end;
bellard's avatar
bellard committed
621
    target_ulong current_pc, current_cs_base;
622
623
624
625
626

    p = page_find(start >> TARGET_PAGE_BITS);
    if (!p) 
        return;
    if (!p->code_bitmap && 
bellard's avatar
bellard committed
627
628
        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
        is_cpu_write_access) {
629
630
631
632
633
634
        /* build code bitmap */
        build_page_bitmap(p);
    }

    /* we remove all the TBs in the range [start, end[ */
    /* XXX: see if in some cases it could be faster to invalidate all the code */
bellard's avatar
bellard committed
635
636
637
638
639
640
    current_tb_not_found = is_cpu_write_access;
    current_tb_modified = 0;
    current_tb = NULL; /* avoid warning */
    current_pc = 0; /* avoid warning */
    current_cs_base = 0; /* avoid warning */
    current_flags = 0; /* avoid warning */
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        tb_next = tb->page_next[n];
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
            tb_end = tb_start + tb->size;
        } else {
            tb_start = tb->page_addr[1];
            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        if (!(tb_end <= start || tb_start >= end)) {
bellard's avatar
bellard committed
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
#ifdef TARGET_HAS_PRECISE_SMC
            if (current_tb_not_found) {
                current_tb_not_found = 0;
                current_tb = NULL;
                if (env->mem_write_pc) {
                    /* now we have a real cpu fault */
                    current_tb = tb_find_pc(env->mem_write_pc);
                }
            }
            if (current_tb == tb &&
                !(current_tb->cflags & CF_SINGLE_INSN)) {
                /* If we are modifying the current TB, we must stop
                its execution. We could be more precise by checking
                that the modification is after the current PC, but it
                would require a specialized function to partially
                restore the CPU state */
                
                current_tb_modified = 1;
                cpu_restore_state(current_tb, env, 
                                  env->mem_write_pc, NULL);
#if defined(TARGET_I386)
                current_flags = env->hflags;
                current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
                current_cs_base = (target_ulong)env->segs[R_CS].base;
                current_pc = current_cs_base + env->eip;
#else
#error unsupported CPU
#endif
            }
#endif /* TARGET_HAS_PRECISE_SMC */
687
688
689
690
691
692
693
            /* we need to do that to handle the case where a signal
               occurs while doing tb_phys_invalidate() */
            saved_tb = NULL;
            if (env) {
                saved_tb = env->current_tb;
                env->current_tb = NULL;
            }
694
            tb_phys_invalidate(tb, -1);
695
696
697
698
699
            if (env) {
                env->current_tb = saved_tb;
                if (env->interrupt_request && env->current_tb)
                    cpu_interrupt(env, env->interrupt_request);
            }
700
701
702
703
704
705
706
        }
        tb = tb_next;
    }
#if !defined(CONFIG_USER_ONLY)
    /* if no code remaining, no need to continue to use slow writes */
    if (!p->first_tb) {
        invalidate_page_bitmap(p);
bellard's avatar
bellard committed
707
708
709
710
711
712
713
714
715
716
        if (is_cpu_write_access) {
            tlb_unprotect_code_phys(env, start, env->mem_write_vaddr);
        }
    }
#endif
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
717
        env->current_tb = NULL;
bellard's avatar
bellard committed
718
719
720
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 
                    CF_SINGLE_INSN);
        cpu_resume_from_signal(env, NULL);
721
    }
bellard's avatar
bellard committed
722
#endif
723
}
bellard's avatar
bellard committed
724

725
/* len must be <= 8 and start must be a multiple of len */
bellard's avatar
bellard committed
726
static inline void tb_invalidate_phys_page_fast(target_ulong start, int len)
727
728
729
{
    PageDesc *p;
    int offset, b;
730
#if 0
731
732
733
734
735
736
737
    if (1) {
        if (loglevel) {
            fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 
                   cpu_single_env->mem_write_vaddr, len, 
                   cpu_single_env->eip, 
                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
        }
738
739
    }
#endif
740
741
742
743
744
745
746
747
748
749
    p = page_find(start >> TARGET_PAGE_BITS);
    if (!p) 
        return;
    if (p->code_bitmap) {
        offset = start & ~TARGET_PAGE_MASK;
        b = p->code_bitmap[offset >> 3] >> (offset & 7);
        if (b & ((1 << len) - 1))
            goto do_invalidate;
    } else {
    do_invalidate:
bellard's avatar
bellard committed
750
        tb_invalidate_phys_page_range(start, start + len, 1);
751
752
753
754
    }
}

#if !defined(CONFIG_SOFTMMU)
bellard's avatar
bellard committed
755
756
static void tb_invalidate_phys_page(target_ulong addr, 
                                    unsigned long pc, void *puc)
757
{
bellard's avatar
bellard committed
758
759
    int n, current_flags, current_tb_modified;
    target_ulong current_pc, current_cs_base;
760
    PageDesc *p;
bellard's avatar
bellard committed
761
762
763
764
    TranslationBlock *tb, *current_tb;
#ifdef TARGET_HAS_PRECISE_SMC
    CPUState *env = cpu_single_env;
#endif
765
766
767
768
769
770

    addr &= TARGET_PAGE_MASK;
    p = page_find(addr >> TARGET_PAGE_BITS);
    if (!p) 
        return;
    tb = p->first_tb;
bellard's avatar
bellard committed
771
772
773
774
775
776
777
778
779
780
    current_tb_modified = 0;
    current_tb = NULL;
    current_pc = 0; /* avoid warning */
    current_cs_base = 0; /* avoid warning */
    current_flags = 0; /* avoid warning */
#ifdef TARGET_HAS_PRECISE_SMC
    if (tb && pc != 0) {
        current_tb = tb_find_pc(pc);
    }
#endif
781
782
783
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
bellard's avatar
bellard committed
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
#ifdef TARGET_HAS_PRECISE_SMC
        if (current_tb == tb &&
            !(current_tb->cflags & CF_SINGLE_INSN)) {
                /* If we are modifying the current TB, we must stop
                   its execution. We could be more precise by checking
                   that the modification is after the current PC, but it
                   would require a specialized function to partially
                   restore the CPU state */
            
            current_tb_modified = 1;
            cpu_restore_state(current_tb, env, pc, puc);
#if defined(TARGET_I386)
            current_flags = env->hflags;
            current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
            current_cs_base = (target_ulong)env->segs[R_CS].base;
            current_pc = current_cs_base + env->eip;
#else
#error unsupported CPU
#endif
        }
#endif /* TARGET_HAS_PRECISE_SMC */
805
806
807
        tb_phys_invalidate(tb, addr);
        tb = tb->page_next[n];
    }
bellard's avatar
bellard committed
808
    p->first_tb = NULL;
bellard's avatar
bellard committed
809
810
811
812
813
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
814
        env->current_tb = NULL;
bellard's avatar
bellard committed
815
816
817
818
819
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 
                    CF_SINGLE_INSN);
        cpu_resume_from_signal(env, puc);
    }
#endif
bellard's avatar
bellard committed
820
}
821
#endif
bellard's avatar
bellard committed
822
823

/* add the tb in the target page and protect it if necessary */
824
static inline void tb_alloc_page(TranslationBlock *tb, 
825
                                 unsigned int n, target_ulong page_addr)
bellard's avatar
bellard committed
826
827
{
    PageDesc *p;
828
829
830
    TranslationBlock *last_first_tb;

    tb->page_addr[n] = page_addr;
831
    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
832
833
834
835
    tb->page_next[n] = p->first_tb;
    last_first_tb = p->first_tb;
    p->first_tb = (TranslationBlock *)((long)tb | n);
    invalidate_page_bitmap(p);
bellard's avatar
bellard committed
836

837
#if defined(TARGET_HAS_SMC) || 1
bellard's avatar
bellard committed
838

839
#if defined(CONFIG_USER_ONLY)
bellard's avatar
bellard committed
840
    if (p->flags & PAGE_WRITE) {
841
842
        target_ulong addr;
        PageDesc *p2;
843
844
        int prot;

bellard's avatar
bellard committed
845
846
        /* force the host page as non writable (writes will have a
           page fault + mprotect overhead) */
847
        page_addr &= qemu_host_page_mask;
bellard's avatar
bellard committed
848
        prot = 0;
849
850
851
852
853
854
855
856
857
858
859
        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
            addr += TARGET_PAGE_SIZE) {

            p2 = page_find (addr >> TARGET_PAGE_BITS);
            if (!p2)
                continue;
            prot |= p2->flags;
            p2->flags &= ~PAGE_WRITE;
            page_get_flags(addr);
          }
        mprotect(g2h(page_addr), qemu_host_page_size, 
bellard's avatar
bellard committed
860
861
862
                 (prot & PAGE_BITS) & ~PAGE_WRITE);
#ifdef DEBUG_TB_INVALIDATE
        printf("protecting code page: 0x%08lx\n", 
863
               page_addr);
bellard's avatar
bellard committed
864
865
#endif
    }
866
867
868
869
870
#else
    /* if some code is already present, then the pages are already
       protected. So we handle the case where only the first TB is
       allocated in a physical page */
    if (!last_first_tb) {
bellard's avatar
bellard committed
871
        tlb_protect_code(page_addr);
872
873
    }
#endif
bellard's avatar
bellard committed
874
875

#endif /* TARGET_HAS_SMC */
bellard's avatar
bellard committed
876
877
878
879
}

/* Allocate a new translation block. Flush the translation buffer if
   too many translation blocks or too much generated code. */
bellard's avatar
bellard committed
880
TranslationBlock *tb_alloc(target_ulong pc)
bellard's avatar
bellard committed
881
882
883
884
885
{
    TranslationBlock *tb;

    if (nb_tbs >= CODE_GEN_MAX_BLOCKS || 
        (code_gen_ptr - code_gen_buffer) >= CODE_GEN_BUFFER_MAX_SIZE)
bellard's avatar
bellard committed
886
        return NULL;
bellard's avatar
bellard committed
887
888
    tb = &tbs[nb_tbs++];
    tb->pc = pc;
889
    tb->cflags = 0;
bellard's avatar
bellard committed
890
891
892
    return tb;
}

893
894
895
896
/* add a new TB and link it to the physical page tables. phys_page2 is
   (-1) to indicate that only one page contains the TB. */
void tb_link_phys(TranslationBlock *tb, 
                  target_ulong phys_pc, target_ulong phys_page2)
bellard's avatar
bellard committed
897
{
898
899
900
901
902
903
904
905
    unsigned int h;
    TranslationBlock **ptb;

    /* add in the physical hash table */
    h = tb_phys_hash_func(phys_pc);
    ptb = &tb_phys_hash[h];
    tb->phys_hash_next = *ptb;
    *ptb = tb;
bellard's avatar
bellard committed
906
907

    /* add in the page list */
908
909
910
911
912
913
    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
    if (phys_page2 != -1)
        tb_alloc_page(tb, 1, phys_page2);
    else
        tb->page_addr[1] = -1;

bellard's avatar
bellard committed
914
915
916
    tb->jmp_first = (TranslationBlock *)((long)tb | 2);
    tb->jmp_next[0] = NULL;
    tb->jmp_next[1] = NULL;
917
918
919
920
921
#ifdef USE_CODE_COPY
    tb->cflags &= ~CF_FP_USED;
    if (tb->cflags & CF_TB_FP_USED)
        tb->cflags |= CF_FP_USED;
#endif
bellard's avatar
bellard committed
922
923
924
925
926
927

    /* init original jump addresses */
    if (tb->tb_next_offset[0] != 0xffff)
        tb_reset_jump(tb, 0);
    if (tb->tb_next_offset[1] != 0xffff)
        tb_reset_jump(tb, 1);
928
929
930
931

#ifdef DEBUG_TB_CHECK
    tb_page_check();
#endif
bellard's avatar
bellard committed
932
933
}

934
935
936
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
   tb[1].tc_ptr. Return NULL if not found */
TranslationBlock *tb_find_pc(unsigned long tc_ptr)
bellard's avatar
bellard committed
937
{
938
939
940
    int m_min, m_max, m;
    unsigned long v;
    TranslationBlock *tb;
bellard's avatar
bellard committed
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963

    if (nb_tbs <= 0)
        return NULL;
    if (tc_ptr < (unsigned long)code_gen_buffer ||
        tc_ptr >= (unsigned long)code_gen_ptr)
        return NULL;
    /* binary search (cf Knuth) */
    m_min = 0;
    m_max = nb_tbs - 1;
    while (m_min <= m_max) {
        m = (m_min + m_max) >> 1;
        tb = &tbs[m];
        v = (unsigned long)tb->tc_ptr;
        if (v == tc_ptr)
            return tb;
        else if (tc_ptr < v) {
            m_max = m - 1;
        } else {
            m_min = m + 1;
        }
    } 
    return &tbs[m_max];
}
bellard's avatar
bellard committed
964

bellard's avatar
bellard committed
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
static void tb_reset_jump_recursive(TranslationBlock *tb);

static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, *tb_next, **ptb;
    unsigned int n1;

    tb1 = tb->jmp_next[n];
    if (tb1 != NULL) {
        /* find head of list */
        for(;;) {
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == 2)
                break;
            tb1 = tb1->jmp_next[n1];
        }
        /* we are now sure now that tb jumps to tb1 */
        tb_next = tb1;

        /* remove tb from the jmp_first list */
        ptb = &tb_next->jmp_first;
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            ptb = &tb1->jmp_next[n1];
        }
        *ptb = tb->jmp_next[n];
        tb->jmp_next[n] = NULL;
        
        /* suppress the jump to next tb in generated code */
        tb_reset_jump(tb, n);

For faster browsing, not all history is shown. View entire blame