allocate big mmap with unreserved memory

.travis.yml

@@ -26,6 +26,8 @@ script:
 - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cargo check --benches --manifest-path=benches/Cargo.toml; fi
 # Make sure `no_std` version checks.
 - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cargo +nightly check --no-default-features --features core; fi
+# Check that `vec_memory` feature works.
+- cargo check --features vec_memory
 - travis_wait 60 ./test.sh
 - ./doc.sh
 

Cargo.toml

@@ -17,6 +17,7 @@ memory_units = "0.3.0"
 libm = { version = "0.1.2", optional = true }
 num-rational = "0.2.2"
 num-traits = "0.2.8"
+libc = "0.2.58"
 
 [dev-dependencies]
 assert_matches = "1.1"
@@ -37,6 +38,12 @@ core = [
     "wasmi-validation/core",
     "libm"
 ]
+# Enforce using the linear memory implementation based on `Vec` instead of
+# mmap on unix systems.
+#
+# Useful for tests and if you need to minimize unsafe usage at the cost of performance on some
+# workloads.
+vec_memory = []
 
 [workspace]
 members = ["validation"]

src/memory/mmap_bytebuf.rs

@@ -0,0 +1,193 @@
+//! An implementation of a `ByteBuf` based on virtual memory.
+//!
+//! This implementation uses `mmap` on POSIX systems (and should use `VirtualAlloc` on windows).
+//! There are possibilities to improve the performance for the reallocating case by reserving
+//! memory up to maximum. This might be a problem for systems that don't have a lot of virtual
+//! memory (i.e. 32-bit platforms).
+
+use std::ptr::{self, NonNull};
+use std::slice;
+
+struct Mmap {
+    /// The pointer that points to the start of the mapping.
+    ///
+    /// This value doesn't change after creation.
+    ptr: NonNull<u8>,
+    /// The length of this mapping.
+    ///
+    /// Cannot be more than `isize::max_value()`. This value doesn't change after creation.
+    len: usize,
+}
+
+impl Mmap {
+    /// Create a new mmap mapping
+    ///
+    /// Returns `Err` if:
+    /// - `len` should not exceed `isize::max_value()`
+    /// - `len` should be greater than 0.
+    /// - `mmap` returns an error (almost certainly means out of memory).
+    fn new(len: usize) -> Result<Self, &'static str> {
+        if len >= isize::max_value() as usize {
+            return Err("`len` should not exceed `isize::max_value()`");
+        }
+        if len == 0 {
+            return Err("`len` should be greater than 0");
+        }
+
+        let ptr_or_err = unsafe {
+            // Safety Proof:
+            // There are not specific safety proofs are required for this call, since the call
+            // by itself can't invoke any safety problems (however, misusing its result can).
+            libc::mmap(
+                // `addr` - let the system to choose the address at which to create the mapping.
+                ptr::null_mut(),
+                // the length of the mapping in bytes.
+                len,
+                // `prot` - protection flags: READ WRITE !EXECUTE
+                libc::PROT_READ | libc::PROT_WRITE,
+                // `flags`
+                // `MAP_ANON` - mapping is not backed by any file and initial contents are
+                // initialized to zero.
+                // `MAP_PRIVATE` - the mapping is private to this process.
+                // `MAP_NORESERVE` - do not reserve swap space for this mapping.
+                libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_NORESERVE,
+                // `fildes` - a file descriptor. Pass -1 as this is required for some platforms
+                // when the `MAP_ANON` is passed.
+                -1,
+                // `offset` - offset from the file.
+                0,
+            )
+        };
+
+        match ptr_or_err as usize {
+            // `mmap` returns -1 in case of an error.
+            // `mmap` shouldn't return 0 since it has a special meaning for compilers.
+            //
+            // With the current parameters, the error can only be returned in case of insufficient
+            // memory.
+            x if x == 0 || x as isize == -1 => Err("mmap returned error"),
+            _ => {
+                let ptr = unsafe {
+                    // Safety Proof:
+                    // the ptr cannot be null as checked within the enclosing match.
+                    NonNull::new_unchecked(ptr_or_err as *mut u8)
+                };
+                Ok(Self { ptr, len })
+            }
+        }
+    }
+
+    fn as_slice(&self) -> &[u8] {
+        unsafe {
+            // Safety Proof:
+            // - Aliasing guarantees of `self.ptr` are not violated since `self` is the only owner.
+            // - This pointer was allocated for `self.len` bytes and thus is a valid slice.
+            // - `self.len` doesn't change throughout the lifetime of `self`.
+            // - The value is returned valid for the duration of lifetime of `self`.
+            //   `self` cannot be destroyed while the returned slice is alive.
+            // - `self.ptr` is of `NonNull` type and thus `.as_ptr()` can never return NULL.
+            // - `self.len` cannot be larger than `isize::max_value()`.
+            slice::from_raw_parts(self.ptr.as_ptr(), self.len)
+        }
+    }
+
+    fn as_slice_mut(&mut self) -> &mut [u8] {
+        unsafe {
+            // Safety Proof:
+            // - See the proof for `Self::as_slice`
+            // - Additionally, it is not possible to obtain two mutable references for `self.ptr`
+            slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len)
+        }
+    }
+}
+
+impl Drop for Mmap {
+    fn drop(&mut self) {
+        let ret_val = unsafe {
+            // Safety proof:
+            // - `self.ptr` was allocated by a call to `mmap`.
+            // - `self.len` was saved at the same time and it doesn't change throughout the lifetime
+            //   of `self`.
+            libc::munmap(self.ptr.as_ptr() as *mut libc::c_void, self.len)
+        };
+
+        // There is no reason for `munmap` to fail to deallocate a private annonymous mapping
+        // allocated by `mmap`.
+        // However, for the cases when it actually fails prefer to fail, in order to not leak
+        // and exhaust the virtual memory.
+        assert_eq!(ret_val, 0, "munmap failed");
+    }
+}
+
+pub struct ByteBuf {
+    mmap: Option<Mmap>,
+    len: usize,
+}
+
+// NOTE: we either make this an arbitrarily large value and use MAP_NORESERVE
+// (which means we can segfault when writing instead of the allocation
+// failing). or we need to figure out the maximum mem + swap available.
+const MMAP_SIZE: usize = 2 << 40;
+
+impl ByteBuf {
+    pub fn new(len: usize) -> Result<Self, &'static str> {
+        let mmap = if len == 0 {
+            None
+        } else {
+            Some(Mmap::new(MMAP_SIZE)?)
+        };
+
+        Ok(Self { mmap, len })
+    }
+
+    pub fn realloc(&mut self, new_len: usize) -> Result<(), &'static str> {
+        if new_len == 0 {
+            self.mmap = None
+        } else if let None = self.mmap {
+            self.mmap = Some(Mmap::new(MMAP_SIZE)?)
+        }
+
+        self.len = new_len;
+
+        Ok(())
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    pub fn as_slice(&self) -> &[u8] {
+        let len = self.len();
+        self.mmap
+            .as_ref()
+            .map(|m| m.as_slice().split_at(len).0)
+            .unwrap_or(&[])
+    }
+
+    pub fn as_slice_mut(&mut self) -> &mut [u8] {
+        let len = self.len();
+        self.mmap
+            .as_mut()
+            .map(|m| m.as_slice_mut().split_at_mut(len).0)
+            .unwrap_or(&mut [])
+    }
+
+    pub fn erase(&mut self) -> Result<(), &'static str> {
+        match self.mmap {
+            // Nothing to do here...
+            None => return Ok(()),
+            Some(Mmap { len: cur_len, .. }) => cur_len,
+        };
+
+        // The order is important.
+        //
+        // 1. First we clear, and thus drop, the current mmap if any.
+        // 2. And then we create a new one.
+        //
+        // Otherwise we double the peak memory consumption.
+        self.mmap = None;
+        self.mmap = Some(Mmap::new(MMAP_SIZE)?);
+
+        Ok(())
+    }
+}

src/memory/mod.rs

@@ -12,6 +12,16 @@ use parity_wasm::elements::ResizableLimits;
 use value::LittleEndianConvert;
 use Error;
 
+#[cfg(all(unix, not(feature = "vec_memory")))]
+#[path = "mmap_bytebuf.rs"]
+mod bytebuf;
+
+#[cfg(any(not(unix), feature = "vec_memory"))]
+#[path = "vec_bytebuf.rs"]
+mod bytebuf;
+
+use self::bytebuf::ByteBuf;
+
 /// Size of a page of [linear memory][`MemoryInstance`] - 64KiB.
 ///
 /// The size of a memory is always a integer multiple of a page size.
@@ -52,11 +62,10 @@ pub struct MemoryInstance {
     /// Memory limits.
     limits: ResizableLimits,
     /// Linear memory buffer with lazy allocation.
-    buffer: RefCell<Vec<u8>>,
+    buffer: RefCell<ByteBuf>,
     initial: Pages,
     current_size: Cell<usize>,
     maximum: Option<Pages>,
-    lowest_used: Cell<u32>,
 }
 
 impl fmt::Debug for MemoryInstance {
@@ -126,23 +135,24 @@ impl MemoryInstance {
             validation::validate_memory(initial_u32, maximum_u32).map_err(Error::Memory)?;
         }
 
-        let memory = MemoryInstance::new(initial, maximum);
+        let memory = MemoryInstance::new(initial, maximum)?;
         Ok(MemoryRef(Rc::new(memory)))
     }
 
     /// Create new linear memory instance.
-    fn new(initial: Pages, maximum: Option<Pages>) -> Self {
+    fn new(initial: Pages, maximum: Option<Pages>) -> Result<Self, Error> {
         let limits = ResizableLimits::new(initial.0 as u32, maximum.map(|p| p.0 as u32));
 
         let initial_size: Bytes = initial.into();
-        MemoryInstance {
+        Ok(MemoryInstance {
             limits: limits,
-            buffer: RefCell::new(Vec::with_capacity(4096)),
+            buffer: RefCell::new(
+                ByteBuf::new(initial_size.0).map_err(|err| Error::Memory(err.to_string()))?,
+            ),
             initial: initial,
             current_size: Cell::new(initial_size.0),
             maximum: maximum,
-            lowest_used: Cell::new(u32::max_value()),
+        })
-        }
     }
 
     /// Return linear memory limits.
@@ -163,16 +173,6 @@ impl MemoryInstance {
         self.maximum
     }
 
-    /// Returns lowest offset ever written or `u32::max_value()` if none.
-    pub fn lowest_used(&self) -> u32 {
-        self.lowest_used.get()
-    }
-
-    /// Resets tracked lowest offset.
-    pub fn reset_lowest_used(&self, addr: u32) {
-        self.lowest_used.set(addr)
-    }
-
     /// Returns current linear memory size.
     ///
     /// Maximum memory size cannot exceed `65536` pages or 4GiB.
@@ -193,13 +193,7 @@ impl MemoryInstance {
     /// );
     /// ```
     pub fn current_size(&self) -> Pages {
-        Bytes(self.current_size.get()).round_up_to()
+        Bytes(self.buffer.borrow().len()).round_up_to()
-    }
-
-    /// Returns current used memory size in bytes.
-    /// This is one more than the highest memory address that had been written to.
-    pub fn used_size(&self) -> Bytes {
-        Bytes(self.buffer.borrow().len())
     }
 
     /// Get value from memory at given offset.
@@ -207,7 +201,10 @@ impl MemoryInstance {
         let mut buffer = self.buffer.borrow_mut();
         let region =
             self.checked_region(&mut buffer, offset as usize, ::core::mem::size_of::<T>())?;
-        Ok(T::from_little_endian(&buffer[region.range()]).expect("Slice size is checked"))
+        Ok(
+            T::from_little_endian(&buffer.as_slice_mut()[region.range()])
+                .expect("Slice size is checked"),
+        )
     }
 
     /// Copy data from memory at given offset.
@@ -220,7 +217,7 @@ impl MemoryInstance {
         let mut buffer = self.buffer.borrow_mut();
         let region = self.checked_region(&mut buffer, offset as usize, size)?;
 
-        Ok(buffer[region.range()].to_vec())
+        Ok(buffer.as_slice_mut()[region.range()].to_vec())
     }
 
     /// Copy data from given offset in the memory into `target` slice.
@@ -232,7 +229,7 @@ impl MemoryInstance {
         let mut buffer = self.buffer.borrow_mut();
         let region = self.checked_region(&mut buffer, offset as usize, target.len())?;
 
-        target.copy_from_slice(&buffer[region.range()]);
+        target.copy_from_slice(&buffer.as_slice_mut()[region.range()]);
 
         Ok(())
     }
@@ -244,10 +241,7 @@ impl MemoryInstance {
             .checked_region(&mut buffer, offset as usize, value.len())?
             .range();
 
-        if offset < self.lowest_used.get() {
+        buffer.as_slice_mut()[range].copy_from_slice(value);
-            self.lowest_used.set(offset);
-        }
-        buffer[range].copy_from_slice(value);
 
         Ok(())
     }
@@ -258,10 +252,7 @@ impl MemoryInstance {
         let range = self
             .checked_region(&mut buffer, offset as usize, ::core::mem::size_of::<T>())?
             .range();
-        if offset < self.lowest_used.get() {
+        value.into_little_endian(&mut buffer.as_slice_mut()[range]);
-            self.lowest_used.set(offset);
-        }
-        value.into_little_endian(&mut buffer[range]);
         Ok(())
     }
 
@@ -295,19 +286,22 @@ impl MemoryInstance {
         }
 
         let new_buffer_length: Bytes = new_size.into();
+        self.buffer
+            .borrow_mut()
+            .realloc(new_buffer_length.0)
+            .map_err(|err| Error::Memory(err.to_string()))?;
+
         self.current_size.set(new_buffer_length.0);
+
         Ok(size_before_grow)
     }
 
-    fn checked_region<B>(
+    fn checked_region(
         &self,
-        buffer: &mut B,
+        buffer: &mut ByteBuf,
         offset: usize,
         size: usize,
-    ) -> Result<CheckedRegion, Error>
+    ) -> Result<CheckedRegion, Error> {
-    where
-        B: ::core::ops::DerefMut<Target = Vec<u8>>,
-    {
         let end = offset.checked_add(size).ok_or_else(|| {
             Error::Memory(format!(
                 "trying to access memory block of size {} from offset {}",
@@ -315,10 +309,6 @@ impl MemoryInstance {
             ))
         })?;
 
-        if end <= self.current_size.get() && buffer.len() < end {
-            buffer.resize(end, 0);
-        }
-
         if end > buffer.len() {
             return Err(Error::Memory(format!(
                 "trying to access region [{}..{}] in memory [0..{}]",
@@ -334,17 +324,14 @@ impl MemoryInstance {
         })
     }
 
-    fn checked_region_pair<B>(
+    fn checked_region_pair(
         &self,
-        buffer: &mut B,
+        buffer: &mut ByteBuf,
         offset1: usize,
         size1: usize,
         offset2: usize,
         size2: usize,
-    ) -> Result<(CheckedRegion, CheckedRegion), Error>
+    ) -> Result<(CheckedRegion, CheckedRegion), Error> {
-    where
-        B: ::core::ops::DerefMut<Target = Vec<u8>>,
-    {
         let end1 = offset1.checked_add(size1).ok_or_else(|| {
             Error::Memory(format!(
                 "trying to access memory block of size {} from offset {}",
@@ -359,11 +346,6 @@ impl MemoryInstance {
             ))
         })?;
 
-        let max = cmp::max(end1, end2);
-        if max <= self.current_size.get() && buffer.len() < max {
-            buffer.resize(max, 0);
-        }
-
         if end1 > buffer.len() {
             return Err(Error::Memory(format!(
                 "trying to access region [{}..{}] in memory [0..{}]",
@@ -407,14 +389,10 @@ impl MemoryInstance {
         let (read_region, write_region) =
             self.checked_region_pair(&mut buffer, src_offset, len, dst_offset, len)?;
 
-        if dst_offset < self.lowest_used.get() as usize {
-            self.lowest_used.set(dst_offset as u32);
-        }
-
         unsafe {
             ::core::ptr::copy(
-                buffer[read_region.range()].as_ptr(),
+                buffer.as_slice()[read_region.range()].as_ptr(),
-                buffer[write_region.range()].as_mut_ptr(),
+                buffer.as_slice_mut()[write_region.range()].as_mut_ptr(),
                 len,
             )
         }
@@ -450,14 +428,10 @@ impl MemoryInstance {
             )));
         }
 
-        if dst_offset < self.lowest_used.get() as usize {
-            self.lowest_used.set(dst_offset as u32);
-        }
-
         unsafe {
             ::core::ptr::copy_nonoverlapping(
-                buffer[read_region.range()].as_ptr(),
+                buffer.as_slice()[read_region.range()].as_ptr(),
-                buffer[write_region.range()].as_mut_ptr(),
+                buffer.as_slice_mut()[write_region.range()].as_mut_ptr(),
                 len,
             )
         }
@@ -493,11 +467,7 @@ impl MemoryInstance {
             .checked_region(&mut dst_buffer, dst_offset, len)?
             .range();
 
-        if dst_offset < dst.lowest_used.get() as usize {
+        dst_buffer.as_slice_mut()[dst_range].copy_from_slice(&src_buffer.as_slice()[src_range]);
-            dst.lowest_used.set(dst_offset as u32);
-        }
-
-        dst_buffer[dst_range].copy_from_slice(&src_buffer[src_range]);
 
         Ok(())
     }
@@ -514,11 +484,7 @@ impl MemoryInstance {
 
         let range = self.checked_region(&mut buffer, offset, len)?.range();
 
-        if offset < self.lowest_used.get() as usize {
+        for val in &mut buffer.as_slice_mut()[range] {
-            self.lowest_used.set(offset as u32);
-        }
-
-        for val in &mut buffer[range] {
             *val = new_val
         }
         Ok(())
@@ -533,34 +499,14 @@ impl MemoryInstance {
         self.clear(offset, 0, len)
     }
 
-    /// Provides direct access to the underlying memory buffer.
+    /// Set every byte in the entire linear memory to 0, preserving its size.
     ///
-    /// # Panics
+    /// Might be useful for some optimization shenanigans.
-    ///
+    pub fn erase(&self) -> Result<(), Error> {
-    /// Any call that requires write access to memory (such as [`set`], [`clear`], etc) made within
+        self.buffer
-    /// the closure will panic. Note that the buffer size may be arbitraty. Proceed with caution.
+            .borrow_mut()
-    ///
+            .erase()
-    /// [`set`]: #method.get
+            .map_err(|err| Error::Memory(err.to_string()))
-    /// [`clear`]: #method.set
-    pub fn with_direct_access<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
-        let buf = self.buffer.borrow();
-        f(&*buf)
-    }
-
-    /// Provides direct mutable access to the underlying memory buffer.
-    ///
-    /// # Panics
-    ///
-    /// Any calls that requires either read or write access to memory (such as [`get`], [`set`], [`copy`], etc) made
-    /// within the closure will panic. Note that the buffer size may be arbitraty.
-    /// The closure may however resize it. Proceed with caution.
-    ///
-    /// [`get`]: #method.get
-    /// [`set`]: #method.set
-    /// [`copy`]: #method.copy
-    pub fn with_direct_access_mut<R, F: FnOnce(&mut Vec<u8>) -> R>(&self, f: F) -> R {
-        let mut buf = self.buffer.borrow_mut();
-        f(&mut buf)
     }
 }
 
@@ -574,29 +520,21 @@ mod tests {
 
     #[test]
     fn alloc() {
-        #[cfg(target_pointer_width = "64")]
+        let mut fixtures = vec![
-        let fixtures = &[
             (0, None, true),
             (0, Some(0), true),
             (1, None, true),
             (1, Some(1), true),
             (0, Some(1), true),
             (1, Some(0), false),
-            (0, Some(65536), true),
+        ];
+
+        #[cfg(target_pointer_width = "64")]
+        fixtures.extend(&[
             (65536, Some(65536), true),
             (65536, Some(0), false),
             (65536, None, true),
-        ];
+        ]);
-
-        #[cfg(target_pointer_width = "32")]
-        let fixtures = &[
-            (0, None, true),
-            (0, Some(0), true),
-            (1, None, true),
-            (1, Some(1), true),
-            (0, Some(1), true),
-            (1, Some(0), false),
-        ];
 
         for (index, &(initial, maybe_max, expected_ok)) in fixtures.iter().enumerate() {
             let initial: Pages = Pages(initial);
@@ -618,7 +556,7 @@ mod tests {
     }
 
     fn create_memory(initial_content: &[u8]) -> MemoryInstance {
-        let mem = MemoryInstance::new(Pages(1), Some(Pages(1)));
+        let mem = MemoryInstance::new(Pages(1), Some(Pages(1))).unwrap();
         mem.set(0, initial_content)
             .expect("Successful initialize the memory");
         mem
@@ -731,7 +669,7 @@ mod tests {
 
     #[test]
     fn get_into() {
-        let mem = MemoryInstance::new(Pages(1), None);
+        let mem = MemoryInstance::new(Pages(1), None).unwrap();
         mem.set(6, &[13, 17, 129])
             .expect("memory set should not fail");
 
@@ -741,28 +679,4 @@ mod tests {
 
         assert_eq!(data, [17, 129]);
     }
-
-    #[test]
-    fn zero_copy() {
-        let mem = MemoryInstance::alloc(Pages(1), None).unwrap();
-        mem.set(100, &[0]).expect("memory set should not fail");
-        mem.with_direct_access_mut(|buf| {
-            assert_eq!(buf.len(), 101);
-            buf[..10].copy_from_slice(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
-        });
-        mem.with_direct_access(|buf| {
-            assert_eq!(buf.len(), 101);
-            assert_eq!(&buf[..10], &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
-        });
-    }
-
-    #[should_panic]
-    #[test]
-    fn zero_copy_panics_on_nested_access() {
-        let mem = MemoryInstance::alloc(Pages(1), None).unwrap();
-        let mem_inner = mem.clone();
-        mem.with_direct_access(move |_| {
-            let _ = mem_inner.set(0, &[11, 12, 13]);
-        });
-    }
 }

src/memory/vec_bytebuf.rs

@@ -0,0 +1,39 @@
+//! An implementation of `ByteBuf` based on a plain `Vec`.
+
+use alloc::prelude::v1::*;
+
+pub struct ByteBuf {
+    buf: Vec<u8>,
+}
+
+impl ByteBuf {
+    pub fn new(len: usize) -> Result<Self, &'static str> {
+        let mut buf = Vec::new();
+        buf.resize(len, 0u8);
+        Ok(Self { buf })
+    }
+
+    pub fn realloc(&mut self, new_len: usize) -> Result<(), &'static str> {
+        self.buf.resize(new_len, 0u8);
+        Ok(())
+    }
+
+    pub fn len(&self) -> usize {
+        self.buf.len()
+    }
+
+    pub fn as_slice(&self) -> &[u8] {
+        self.buf.as_ref()
+    }
+
+    pub fn as_slice_mut(&mut self) -> &mut [u8] {
+        self.buf.as_mut()
+    }
+
+    pub fn erase(&mut self) -> Result<(), &'static str> {
+        for v in &mut self.buf {
+            *v = 0;
+        }
+        Ok(())
+    }
+}