mmap.c 17.7 KB
Newer Older
bellard's avatar
bellard committed
1 2
/*
 *  mmap support for qemu
3
 *
bellard's avatar
bellard committed
4 5 6 7 8 9 10 11 12 13 14 15 16 17
 *  Copyright (c) 2003 Fabrice Bellard
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
blueswir1's avatar
blueswir1 committed
18 19
 *  Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston,
 *  MA 02110-1301, USA.
bellard's avatar
bellard committed
20 21 22 23 24 25 26 27
 */
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
28 29
#include <linux/mman.h>
#include <linux/unistd.h>
bellard's avatar
bellard committed
30 31

#include "qemu.h"
32
#include "qemu-common.h"
bellard's avatar
bellard committed
33 34 35

//#define DEBUG_MMAP

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
#if defined(USE_NPTL)
pthread_mutex_t mmap_mutex;
static int __thread mmap_lock_count;

void mmap_lock(void)
{
    if (mmap_lock_count++ == 0) {
        pthread_mutex_lock(&mmap_mutex);
    }
}

void mmap_unlock(void)
{
    if (--mmap_lock_count == 0) {
        pthread_mutex_unlock(&mmap_mutex);
    }
}
pbrook's avatar
pbrook committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68

/* Grab lock to make sure things are in a consistent state after fork().  */
void mmap_fork_start(void)
{
    if (mmap_lock_count)
        abort();
    pthread_mutex_lock(&mmap_mutex);
}

void mmap_fork_end(int child)
{
    if (child)
        pthread_mutex_init(&mmap_mutex, NULL);
    else
        pthread_mutex_unlock(&mmap_mutex);
}
69 70 71 72 73 74 75 76 77 78 79
#else
/* We aren't threadsafe to start with, so no need to worry about locking.  */
void mmap_lock(void)
{
}

void mmap_unlock(void)
{
}
#endif

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
void *qemu_vmalloc(size_t size)
{
    void *p;
    unsigned long addr;
    mmap_lock();
    /* Use map and mark the pages as used.  */
    p = mmap(NULL, size, PROT_READ | PROT_WRITE,
             MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

    addr = (unsigned long)p;
    if (addr == (target_ulong) addr) {
        /* Allocated region overlaps guest address space.
           This may recurse.  */
        page_set_flags(addr & TARGET_PAGE_MASK, TARGET_PAGE_ALIGN(addr + size),
                       PAGE_RESERVED);
    }

    mmap_unlock();
    return p;
}

void *qemu_malloc(size_t size)
{
    char * p;
    size += 16;
    p = qemu_vmalloc(size);
    *(size_t *)p = size;
    return p + 16;
}

/* We use map, which is always zero initialized.  */
void * qemu_mallocz(size_t size)
{
    return qemu_malloc(size);
}

void qemu_free(void *ptr)
{
    /* FIXME: We should unmark the reserved pages here.  However this gets
       complicated when one target page spans multiple host pages, so we
       don't bother.  */
    size_t *p;
    p = (size_t *)((char *)ptr - 16);
    munmap(p, *p);
}

126
/* NOTE: all the constants are the HOST ones, but addresses are target. */
127
int target_mprotect(abi_ulong start, abi_ulong len, int prot)
bellard's avatar
bellard committed
128
{
129
    abi_ulong end, host_start, host_end, addr;
bellard's avatar
bellard committed
130 131 132
    int prot1, ret;

#ifdef DEBUG_MMAP
133
    printf("mprotect: start=0x" TARGET_FMT_lx
134
           "len=0x" TARGET_FMT_lx " prot=%c%c%c\n", start, len,
bellard's avatar
bellard committed
135 136 137 138 139 140 141 142 143 144 145
           prot & PROT_READ ? 'r' : '-',
           prot & PROT_WRITE ? 'w' : '-',
           prot & PROT_EXEC ? 'x' : '-');
#endif

    if ((start & ~TARGET_PAGE_MASK) != 0)
        return -EINVAL;
    len = TARGET_PAGE_ALIGN(len);
    end = start + len;
    if (end < start)
        return -EINVAL;
146
    prot &= PROT_READ | PROT_WRITE | PROT_EXEC;
bellard's avatar
bellard committed
147 148
    if (len == 0)
        return 0;
149

150
    mmap_lock();
151
    host_start = start & qemu_host_page_mask;
bellard's avatar
bellard committed
152 153 154 155 156 157 158
    host_end = HOST_PAGE_ALIGN(end);
    if (start > host_start) {
        /* handle host page containing start */
        prot1 = prot;
        for(addr = host_start; addr < start; addr += TARGET_PAGE_SIZE) {
            prot1 |= page_get_flags(addr);
        }
159
        if (host_end == host_start + qemu_host_page_size) {
bellard's avatar
bellard committed
160 161 162 163 164
            for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) {
                prot1 |= page_get_flags(addr);
            }
            end = host_end;
        }
165
        ret = mprotect(g2h(host_start), qemu_host_page_size, prot1 & PAGE_BITS);
bellard's avatar
bellard committed
166
        if (ret != 0)
167
            goto error;
168
        host_start += qemu_host_page_size;
bellard's avatar
bellard committed
169 170 171 172 173 174
    }
    if (end < host_end) {
        prot1 = prot;
        for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) {
            prot1 |= page_get_flags(addr);
        }
175
        ret = mprotect(g2h(host_end - qemu_host_page_size), qemu_host_page_size,
bellard's avatar
bellard committed
176 177
                       prot1 & PAGE_BITS);
        if (ret != 0)
178
            goto error;
179
        host_end -= qemu_host_page_size;
bellard's avatar
bellard committed
180
    }
181

bellard's avatar
bellard committed
182 183
    /* handle the pages in the middle */
    if (host_start < host_end) {
184
        ret = mprotect(g2h(host_start), host_end - host_start, prot);
bellard's avatar
bellard committed
185
        if (ret != 0)
186
            goto error;
bellard's avatar
bellard committed
187 188
    }
    page_set_flags(start, start + len, prot | PAGE_VALID);
189
    mmap_unlock();
bellard's avatar
bellard committed
190
    return 0;
191 192 193
error:
    mmap_unlock();
    return ret;
bellard's avatar
bellard committed
194 195 196
}

/* map an incomplete host page */
197 198 199
static int mmap_frag(abi_ulong real_start,
                     abi_ulong start, abi_ulong end,
                     int prot, int flags, int fd, abi_ulong offset)
bellard's avatar
bellard committed
200
{
201
    abi_ulong real_end, addr;
202
    void *host_start;
bellard's avatar
bellard committed
203 204
    int prot1, prot_new;

205 206
    real_end = real_start + qemu_host_page_size;
    host_start = g2h(real_start);
bellard's avatar
bellard committed
207 208 209

    /* get the protection of the target pages outside the mapping */
    prot1 = 0;
210
    for(addr = real_start; addr < real_end; addr++) {
bellard's avatar
bellard committed
211 212 213
        if (addr < start || addr >= end)
            prot1 |= page_get_flags(addr);
    }
214

bellard's avatar
bellard committed
215 216
    if (prot1 == 0) {
        /* no page was there, so we allocate one */
217 218 219 220
        void *p = mmap(host_start, qemu_host_page_size, prot,
                       flags | MAP_ANONYMOUS, -1, 0);
        if (p == MAP_FAILED)
            return -1;
221
        prot1 = prot;
bellard's avatar
bellard committed
222 223 224 225 226 227 228 229 230 231 232 233 234
    }
    prot1 &= PAGE_BITS;

    prot_new = prot | prot1;
    if (!(flags & MAP_ANONYMOUS)) {
        /* msync() won't work here, so we return an error if write is
           possible while it is a shared mapping */
        if ((flags & MAP_TYPE) == MAP_SHARED &&
            (prot & PROT_WRITE))
            return -EINVAL;

        /* adjust protection to be able to read */
        if (!(prot1 & PROT_WRITE))
235
            mprotect(host_start, qemu_host_page_size, prot1 | PROT_WRITE);
236

bellard's avatar
bellard committed
237
        /* read the corresponding file data */
238
        pread(fd, g2h(start), end - start, offset);
239

bellard's avatar
bellard committed
240 241
        /* put final protection */
        if (prot_new != (prot1 | PROT_WRITE))
242
            mprotect(host_start, qemu_host_page_size, prot_new);
bellard's avatar
bellard committed
243 244 245
    } else {
        /* just update the protection */
        if (prot_new != prot1) {
246
            mprotect(host_start, qemu_host_page_size, prot_new);
bellard's avatar
bellard committed
247 248 249 250 251
        }
    }
    return 0;
}

252 253 254 255 256 257 258
#if defined(__CYGWIN__)
/* Cygwin doesn't have a whole lot of address space.  */
static abi_ulong mmap_next_start = 0x18000000;
#else
static abi_ulong mmap_next_start = 0x40000000;
#endif

259 260
unsigned long last_brk;

261 262 263 264
/* find a free memory area of size 'size'. The search starts at
   'start'. If 'start' == 0, then a default start address is used.
   Return -1 if error.
*/
265
/* page_init() marks pages used by the host as reserved to be sure not
266 267 268 269 270
   to use them. */
static abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size)
{
    abi_ulong addr, addr1, addr_start;
    int prot;
271 272 273 274 275 276 277 278 279 280 281 282 283 284
    unsigned long new_brk;

    new_brk = (unsigned long)sbrk(0);
    if (last_brk && last_brk < new_brk && last_brk == (target_ulong)last_brk) {
        /* This is a hack to catch the host allocating memory with brk().
           If it uses mmap then we loose.
           FIXME: We really want to avoid the host allocating memory in
           the first place, and maybe leave some slack to avoid switching
           to mmap.  */
        page_set_flags(last_brk & TARGET_PAGE_MASK,
                       TARGET_PAGE_ALIGN(new_brk),
                       PAGE_RESERVED); 
    }
    last_brk = new_brk;
285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308

    size = HOST_PAGE_ALIGN(size);
    start = start & qemu_host_page_mask;
    addr = start;
    if (addr == 0)
        addr = mmap_next_start;
    addr_start = addr;
    for(;;) {
        prot = 0;
        for(addr1 = addr; addr1 < (addr + size); addr1 += TARGET_PAGE_SIZE) {
            prot |= page_get_flags(addr1);
        }
        if (prot == 0)
            break;
        addr += qemu_host_page_size;
        /* we found nothing */
        if (addr == addr_start)
            return (abi_ulong)-1;
    }
    if (start == 0)
        mmap_next_start = addr + size;
    return addr;
}

bellard's avatar
bellard committed
309
/* NOTE: all the constants are the HOST ones */
310 311
abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
                     int flags, int fd, abi_ulong offset)
bellard's avatar
bellard committed
312
{
313
    abi_ulong ret, end, real_start, real_end, retaddr, host_offset, host_len;
314
    unsigned long host_start;
bellard's avatar
bellard committed
315

316
    mmap_lock();
bellard's avatar
bellard committed
317 318
#ifdef DEBUG_MMAP
    {
319
        printf("mmap: start=0x" TARGET_FMT_lx
320
               " len=0x" TARGET_FMT_lx " prot=%c%c%c flags=",
321
               start, len,
bellard's avatar
bellard committed
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
               prot & PROT_READ ? 'r' : '-',
               prot & PROT_WRITE ? 'w' : '-',
               prot & PROT_EXEC ? 'x' : '-');
        if (flags & MAP_FIXED)
            printf("MAP_FIXED ");
        if (flags & MAP_ANONYMOUS)
            printf("MAP_ANON ");
        switch(flags & MAP_TYPE) {
        case MAP_PRIVATE:
            printf("MAP_PRIVATE ");
            break;
        case MAP_SHARED:
            printf("MAP_SHARED ");
            break;
        default:
            printf("[MAP_TYPE=0x%x] ", flags & MAP_TYPE);
            break;
        }
340
        printf("fd=%d offset=" TARGET_FMT_lx "\n", fd, offset);
bellard's avatar
bellard committed
341 342 343
    }
#endif

344 345
    if (offset & ~TARGET_PAGE_MASK) {
        errno = EINVAL;
346
        goto fail;
347
    }
bellard's avatar
bellard committed
348 349 350

    len = TARGET_PAGE_ALIGN(len);
    if (len == 0)
351
        goto the_end;
352
    real_start = start & qemu_host_page_mask;
bellard's avatar
bellard committed
353 354

    if (!(flags & MAP_FIXED)) {
355 356 357 358 359 360 361 362
        abi_ulong mmap_start;
        void *p;
        host_offset = offset & qemu_host_page_mask;
        host_len = len + offset - host_offset;
        host_len = HOST_PAGE_ALIGN(host_len);
        mmap_start = mmap_find_vma(real_start, host_len);
        if (mmap_start == (abi_ulong)-1) {
            errno = ENOMEM;
363
            goto fail;
bellard's avatar
bellard committed
364
        }
365 366 367 368 369 370
        /* Note: we prefer to control the mapping address. It is
           especially important if qemu_host_page_size >
           qemu_real_host_page_size */
        p = mmap(g2h(mmap_start),
                 host_len, prot, flags | MAP_FIXED, fd, host_offset);
        if (p == MAP_FAILED)
371
            goto fail;
372 373 374 375 376 377
        /* update start so that it points to the file position at 'offset' */
        host_start = (unsigned long)p;
        if (!(flags & MAP_ANONYMOUS))
            host_start += offset - host_offset;
        start = h2g(host_start);
    } else {
378 379 380
        int flg;
        target_ulong addr;

381
        if (start & ~TARGET_PAGE_MASK) {
382
            errno = EINVAL;
383
            goto fail;
384
        }
385 386
        end = start + len;
        real_end = HOST_PAGE_ALIGN(end);
387

388 389 390 391 392 393 394 395 396 397
	/*
	 * Test if requested memory area fits target address space
	 * It can fail only on 64-bit host with 32-bit target.
	 * On any other target/host host mmap() handles this error correctly.
	 */
        if ((unsigned long)start + len - 1 > (abi_ulong) -1) {
            errno = EINVAL;
            goto fail;
        }

398 399 400 401
        for(addr = real_start; addr < real_end; addr += TARGET_PAGE_SIZE) {
            flg = page_get_flags(addr);
            if (flg & PAGE_RESERVED) {
                errno = ENXIO;
402
                goto fail;
403 404 405
            }
        }

406 407 408 409 410 411 412 413 414
        /* worst case: we cannot map the file because the offset is not
           aligned, so we read it */
        if (!(flags & MAP_ANONYMOUS) &&
            (offset & ~qemu_host_page_mask) != (start & ~qemu_host_page_mask)) {
            /* msync() won't work here, so we return an error if write is
               possible while it is a shared mapping */
            if ((flags & MAP_TYPE) == MAP_SHARED &&
                (prot & PROT_WRITE)) {
                errno = EINVAL;
415
                goto fail;
416 417 418 419 420
            }
            retaddr = target_mmap(start, len, prot | PROT_WRITE,
                                  MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS,
                                  -1, 0);
            if (retaddr == -1)
421
                goto fail;
422 423 424
            pread(fd, g2h(start), len, offset);
            if (!(prot & PROT_WRITE)) {
                ret = target_mprotect(start, len, prot);
425 426 427 428
                if (ret != 0) {
                    start = ret;
                    goto the_end;
                }
429 430
            }
            goto the_end;
bellard's avatar
bellard committed
431
        }
432 433 434 435 436 437 438 439
        
        /* handle the start of the mapping */
        if (start > real_start) {
            if (real_end == real_start + qemu_host_page_size) {
                /* one single host page */
                ret = mmap_frag(real_start, start, end,
                                prot, flags, fd, offset);
                if (ret == -1)
440
                    goto fail;
441 442 443
                goto the_end1;
            }
            ret = mmap_frag(real_start, start, real_start + qemu_host_page_size,
bellard's avatar
bellard committed
444 445
                            prot, flags, fd, offset);
            if (ret == -1)
446
                goto fail;
447 448 449 450 451 452 453 454 455
            real_start += qemu_host_page_size;
        }
        /* handle the end of the mapping */
        if (end < real_end) {
            ret = mmap_frag(real_end - qemu_host_page_size,
                            real_end - qemu_host_page_size, real_end,
                            prot, flags, fd,
                            offset + real_end - qemu_host_page_size - start);
            if (ret == -1)
456
                goto fail;
457
            real_end -= qemu_host_page_size;
bellard's avatar
bellard committed
458
        }
459

460 461 462 463 464 465 466 467 468 469 470
        /* map the middle (easier) */
        if (real_start < real_end) {
            void *p;
            unsigned long offset1;
            if (flags & MAP_ANONYMOUS)
                offset1 = 0;
            else
                offset1 = offset + real_start - start;
            p = mmap(g2h(real_start), real_end - real_start,
                     prot, flags, fd, offset1);
            if (p == MAP_FAILED)
471
                goto fail;
472
        }
bellard's avatar
bellard committed
473 474 475 476 477
    }
 the_end1:
    page_set_flags(start, start + len, prot | PAGE_VALID);
 the_end:
#ifdef DEBUG_MMAP
edgar_igl's avatar
edgar_igl committed
478
    printf("ret=0x" TARGET_FMT_lx "\n", start);
bellard's avatar
bellard committed
479 480 481
    page_dump(stdout);
    printf("\n");
#endif
482
    mmap_unlock();
bellard's avatar
bellard committed
483
    return start;
484 485 486
fail:
    mmap_unlock();
    return -1;
bellard's avatar
bellard committed
487 488
}

489
int target_munmap(abi_ulong start, abi_ulong len)
bellard's avatar
bellard committed
490
{
491
    abi_ulong end, real_start, real_end, addr;
bellard's avatar
bellard committed
492 493 494 495 496 497 498 499 500 501
    int prot, ret;

#ifdef DEBUG_MMAP
    printf("munmap: start=0x%lx len=0x%lx\n", start, len);
#endif
    if (start & ~TARGET_PAGE_MASK)
        return -EINVAL;
    len = TARGET_PAGE_ALIGN(len);
    if (len == 0)
        return -EINVAL;
502
    mmap_lock();
bellard's avatar
bellard committed
503
    end = start + len;
504 505
    real_start = start & qemu_host_page_mask;
    real_end = HOST_PAGE_ALIGN(end);
bellard's avatar
bellard committed
506

507
    if (start > real_start) {
bellard's avatar
bellard committed
508 509
        /* handle host page containing start */
        prot = 0;
510
        for(addr = real_start; addr < start; addr += TARGET_PAGE_SIZE) {
bellard's avatar
bellard committed
511 512
            prot |= page_get_flags(addr);
        }
513 514
        if (real_end == real_start + qemu_host_page_size) {
            for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
bellard's avatar
bellard committed
515 516
                prot |= page_get_flags(addr);
            }
517
            end = real_end;
bellard's avatar
bellard committed
518
        }
bellard's avatar
bellard committed
519
        if (prot != 0)
520
            real_start += qemu_host_page_size;
bellard's avatar
bellard committed
521
    }
522
    if (end < real_end) {
bellard's avatar
bellard committed
523
        prot = 0;
524
        for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
bellard's avatar
bellard committed
525 526 527
            prot |= page_get_flags(addr);
        }
        if (prot != 0)
528
            real_end -= qemu_host_page_size;
bellard's avatar
bellard committed
529
    }
530

531
    ret = 0;
bellard's avatar
bellard committed
532
    /* unmap what we can */
533
    if (real_start < real_end) {
534
        ret = munmap(g2h(real_start), real_end - real_start);
bellard's avatar
bellard committed
535 536
    }

537 538 539 540
    if (ret == 0)
        page_set_flags(start, start + len, 0);
    mmap_unlock();
    return ret;
bellard's avatar
bellard committed
541 542
}

543 544 545
abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                       abi_ulong new_size, unsigned long flags,
                       abi_ulong new_addr)
bellard's avatar
bellard committed
546 547
{
    int prot;
548
    void *host_addr;
bellard's avatar
bellard committed
549

550
    mmap_lock();
551 552

    if (flags & MREMAP_FIXED)
553 554 555 556
        host_addr = (void *) syscall(__NR_mremap, g2h(old_addr),
                                     old_size, new_size,
                                     flags,
                                     new_addr);
557 558 559 560 561 562 563 564 565
    else if (flags & MREMAP_MAYMOVE) {
        abi_ulong mmap_start;

        mmap_start = mmap_find_vma(0, new_size);

        if (mmap_start == -1) {
            errno = ENOMEM;
            host_addr = MAP_FAILED;
        } else
566 567 568 569 570
            host_addr = (void *) syscall(__NR_mremap, g2h(old_addr),
                                         old_size, new_size,
                                         flags | MREMAP_FIXED,
                                         g2h(mmap_start));
    } else {
571 572 573 574 575 576 577 578 579 580 581
        host_addr = mremap(g2h(old_addr), old_size, new_size, flags);
        /* Check if address fits target address space */
        if ((unsigned long)host_addr + new_size > (abi_ulong)-1) {
            /* Revert mremap() changes */
            host_addr = mremap(g2h(old_addr), new_size, old_size, flags);
            errno = ENOMEM;
            host_addr = MAP_FAILED;
        }
    }

    if (host_addr == MAP_FAILED) {
582 583 584 585 586 587 588 589
        new_addr = -1;
    } else {
        new_addr = h2g(host_addr);
        prot = page_get_flags(old_addr);
        page_set_flags(old_addr, old_addr + old_size, 0);
        page_set_flags(new_addr, new_addr + new_size, prot | PAGE_VALID);
    }
    mmap_unlock();
bellard's avatar
bellard committed
590 591 592
    return new_addr;
}

593
int target_msync(abi_ulong start, abi_ulong len, int flags)
bellard's avatar
bellard committed
594
{
595
    abi_ulong end;
bellard's avatar
bellard committed
596 597 598 599 600

    if (start & ~TARGET_PAGE_MASK)
        return -EINVAL;
    len = TARGET_PAGE_ALIGN(len);
    end = start + len;
bellard's avatar
bellard committed
601 602 603 604
    if (end < start)
        return -EINVAL;
    if (end == start)
        return 0;
605

606
    start &= qemu_host_page_mask;
607
    return msync(g2h(start), end - start, flags);
bellard's avatar
bellard committed
608
}