PATCH: Re: Problem with virtual memory rlimits

[Marcus Meissner]

On Fri, Feb 24, 2006 at 02:12:46PM +0100, Marcus Meissner wrote:
> Hi,
> 
> I am having a problem with virtual ulimits/rlimits.
> 
> If you limit the virtual memory size using "ulimit -v 1400000" for
> instance, wine will only crash.
> 
> The problem is, that VIRTUAL_alloc_teb() tries to allocate a memory
> area within the acceptable userspace ... but it does so much mmap()s
> that it runs out of available mmap space before.
> 
> I was only able to reproduce this on AMD64 machines up to now, 
> which pass out memory starting from 0xf7fffffff somewhere.
> 
> Any clue how to solve this?

Got an idea...

It helps if the process is started with the ADDR_COMPAT_LAYOUT personality
flag set.

So if we are running without it, we can just set the personality and
execve ourselves.

Btw, this trick might even help execshield and avoid the need for the
preloader at all, but I do not have Fedora/Redhat here to test.

Ciao, Marcus

Changelog:
	Change to ADDR_COMPAT_LAYOUT personality to avoid problems
	with AMD64 memory layout and ulimits set.

Index: loader/preloader.c
===================================================================
RCS file: /home/wine/wine/loader/preloader.c,v
retrieving revision 1.20
diff -u -r1.20 preloader.c
--- loader/preloader.c	6 Feb 2006 13:35:58 -0000	1.20
+++ loader/preloader.c	17 Mar 2006 15:36:21 -0000
@@ -208,6 +208,14 @@
     return SYSCALL_RET(ret);
 }
 
+static inline int wld_personality( int perso )
+{
+    int ret;
+    __asm__ __volatile__( "pushl %%ebx; movl %2,%%ebx; int $0x80; popl %%ebx"
+                          : "=a" (ret) : "0" (SYS_personality), "r" (perso) );
+    return SYSCALL_RET(ret);
+}
+
 static inline ssize_t wld_read( int fd, void *buffer, size_t len )
 {
     int ret;
@@ -226,6 +234,14 @@
     return SYSCALL_RET(ret);
 }
 
+static inline int wld_execve( const char * path, char **argv, char ** envp )
+{
+    int ret;
+    __asm__ __volatile__( "pushl %%ebx; movl %2,%%ebx; int $0x80; popl %%ebx"
+                          : "=a" (ret) : "0" (SYS_execve), "r" (path), "c" (argv), "d" (envp) );
+    return SYSCALL_RET(ret);
+}
+
 static inline int wld_mprotect( const void *addr, size_t len, int prot )
 {
     int ret;
@@ -886,6 +902,7 @@
     ElfW(auxv_t) new_av[12], delete_av[3], *av;
     struct wld_link_map main_binary_map, ld_so_map;
     struct wine_preload_info **wine_main_preload_info;
+    unsigned int perso;
 
     pargc = *stack;
     argv = (char **)pargc + 1;
@@ -894,6 +911,14 @@
     /* skip over the parameters */
     p = argv + *pargc + 1;
 
+    perso = wld_personality(0xffffffff);
+    if (!(perso & 0x200000)) {
+	wld_personality(perso | 0x200000);
+    	wld_execve(argv[0],argv,p);
+	fatal_error( "Failed execve %s\n", argv[0] );
+	return NULL;
+    }
+
     /* skip over the environment */
     while (*p)
     {