wine-staging/patches/ntdll-BitmaskAllocAreaSearch/0001-ntdll-Fix-free-area-search-outside-of-reserved-area-.patch

From 19635dea9206b1606ad0ac53c99205d48bc2ee41 Mon Sep 17 00:00:00 2001
From: Paul Gofman <gofmanp@gmail.com>
Date: Thu, 24 Oct 2019 20:17:36 +0300
Subject: [PATCH] ntdll: Fix free area search outside of reserved area in
 map_view() for non-zero bitmask.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Patch includes logic changes suggested and edited by Rémi Bernon.
---
 dlls/ntdll/virtual.c | 90 ++++++++++++++++++++++++++++++++------------
 1 file changed, 66 insertions(+), 24 deletions(-)

diff --git a/dlls/ntdll/virtual.c b/dlls/ntdll/virtual.c
index 72ee2cade3..25fbe3b0b1 100644
--- a/dlls/ntdll/virtual.c
+++ b/dlls/ntdll/virtual.c
@@ -543,6 +543,24 @@ static struct file_view *find_view_range( const void *addr, size_t size )
     return NULL;
 }
 
+/***********************************************************************
+ *           try_map_free_area
+ *
+ * Try mmaping some expected free memory region, and return whether it
+ * succeeded as expected.
+ */
+static BOOL try_map_free_area( void *start, size_t size, int unix_prot )
+{
+    void *map_ptr;
+
+    if ((map_ptr = wine_anon_mmap(start, size, unix_prot, 0)) == start)
+        return TRUE;
+
+    if (map_ptr != (void *)-1)
+        munmap(map_ptr, size);
+
+    return FALSE;
+}
 
 /***********************************************************************
  *           find_free_area
@@ -550,7 +568,7 @@ static struct file_view *find_view_range( const void *addr, size_t size )
  * Find a free area between views inside the specified range.
  * The csVirtual section must be held by caller.
  */
-static void *find_free_area( void *base, void *end, size_t size, size_t mask, int top_down )
+static void *find_free_area( void *base, void *end, size_t size, size_t mask, int top_down, BOOL map, int unix_prot )
 {
     struct wine_rb_entry *first = NULL, *ptr = views_tree.root;
     void *start;
@@ -579,33 +597,53 @@ static void *find_free_area( void *base, void *end, size_t size, size_t mask, in
     if (top_down)
     {
         start = ROUND_ADDR( (char *)end - size, mask );
-        if (start >= end || start < base) return NULL;
 
-        while (first)
+        for (;;)
         {
-            struct file_view *view = WINE_RB_ENTRY_VALUE( first, struct file_view, entry );
+            if (start >= end || start < base) return NULL;
+
+            while (first)
+            {
+                struct file_view *view = WINE_RB_ENTRY_VALUE( first, struct file_view, entry );
+
+                if ((char *)view->base + view->size <= (char *)start) break;
+                start = ROUND_ADDR( (char *)view->base - size, mask );
+                /* stop if remaining space is not large enough */
+                if (!start || start >= end || start < base) return NULL;
+                first = wine_rb_prev( first );
+            }
 
-            if ((char *)view->base + view->size <= (char *)start) break;
-            start = ROUND_ADDR( (char *)view->base - size, mask );
-            /* stop if remaining space is not large enough */
-            if (!start || start >= end || start < base) return NULL;
-            first = wine_rb_prev( first );
+            if (!map || try_map_free_area(start, size, unix_prot))
+                break;
+
+            TRACE("Found free area is already mapped, start %p.\n", start);
+            start = ROUND_ADDR((char *)start - 1, mask);
         }
     }
     else
     {
         start = ROUND_ADDR( (char *)base + mask, mask );
-        if (!start || start >= end || (char *)end - (char *)start < size) return NULL;
 
-        while (first)
+        for (;;)
         {
-            struct file_view *view = WINE_RB_ENTRY_VALUE( first, struct file_view, entry );
-
-            if ((char *)view->base >= (char *)start + size) break;
-            start = ROUND_ADDR( (char *)view->base + view->size + mask, mask );
-            /* stop if remaining space is not large enough */
             if (!start || start >= end || (char *)end - (char *)start < size) return NULL;
-            first = wine_rb_next( first );
+
+            while (first)
+            {
+                struct file_view *view = WINE_RB_ENTRY_VALUE( first, struct file_view, entry );
+
+                if ((char *)view->base >= (char *)start + size) break;
+                start = ROUND_ADDR( (char *)view->base + view->size + mask, mask );
+                /* stop if remaining space is not large enough */
+                if (!start || start >= end || (char *)end - (char *)start < size) return NULL;
+                first = wine_rb_next( first );
+            }
+
+            if (!map || try_map_free_area(start, size, unix_prot))
+                break;
+
+            TRACE("Found free area is already mapped, start %p.\n", start);
+            start = (char *)start + mask + 1;
         }
     }
     return start;
@@ -1101,13 +1139,13 @@ static int alloc_reserved_area_callback( void *start, size_t size, void *arg )
         {
             /* range is split in two by the preloader reservation, try first part */
             if ((alloc->result = find_free_area( start, preload_reserve_start, alloc->size,
-                                                 alloc->mask, alloc->top_down )))
+                                                 alloc->mask, alloc->top_down, FALSE, 0 )))
                 return 1;
             /* then fall through to try second part */
             start = preload_reserve_end;
         }
     }
-    if ((alloc->result = find_free_area( start, end, alloc->size, alloc->mask, alloc->top_down )))
+    if ((alloc->result = find_free_area( start, end, alloc->size, alloc->mask, alloc->top_down, FALSE, 0 )))
         return 1;
 
     return 0;
@@ -1210,14 +1248,18 @@ static NTSTATUS map_view( struct file_view **view_ret, void *base, size_t size,
             goto done;
         }
 
-        for (;;)
+        if (zero_bits_64)
         {
-            if (!zero_bits_64)
-                ptr = NULL;
-            else if (!(ptr = find_free_area( (void*)0, alloc.limit, view_size, mask, top_down )))
+            if (!(ptr = find_free_area(address_space_start, alloc.limit, size,
+                    mask, top_down, TRUE, VIRTUAL_GetUnixProt(vprot))))
                 return STATUS_NO_MEMORY;
+            TRACE( "got mem with find_free_area %p-%p\n", ptr, (char *)ptr + size );
+            goto done;
+        }
 
-            if ((ptr = wine_anon_mmap( ptr, view_size, VIRTUAL_GetUnixProt(vprot), ptr ? MAP_FIXED : 0 )) == (void *)-1)
+        for (;;)
+        {
+            if ((ptr = wine_anon_mmap(NULL, view_size, VIRTUAL_GetUnixProt(vprot), 0 )) == (void *)-1)
             {
                 if (errno == ENOMEM) return STATUS_NO_MEMORY;
                 return STATUS_INVALID_PARAMETER;
-- 
2.23.0