added multithreaded version, no mmap optimization

Cargo.toml

@@ -3,8 +3,6 @@ name = "onebrc"
 version = "0.1.0"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 fxhash = "0.2.1"
 rayon = "1.8.0"

README.md

@@ -2,5 +2,4 @@
 
 * Fastest single threaded runtime: 29.7s (4.5s to read the file into memory, 24s to process rows and ~150micros to generate the output)
 
-
+* Fastest multi threaded runtime on a machine with 12 cores: read = 4.682276171s processed = 2.057576429s output_gen = 188.113µs
-* Fastest multi threaded runtime on a machine with 12 cores: -

src/main.rs

@@ -1,10 +1,8 @@
 use fxhash::FxHashMap;
-use std::fs::File;
+use rayon::prelude::*;
-use std::io::Read;
+use std::{fs::File, io::Read, time::Instant};
-use std::time::Instant;
 
 fn main() {
-    let mut map = FxHashMap::default();
     let mut buf = Vec::new();
 
     let t1 = Instant::now();
@@ -18,36 +16,58 @@ fn main() {
 
     let t2 = Instant::now();
 
-    for line in buf.split(|&x| x == b'\n') {
+    let map: FxHashMap<&[u8], (i16, i32, i16, usize)> = buf
-        let l = line.len();
+        .par_split(|&x| x == b'\n')
-        if l == 0 {
+        .filter_map(|line| {
-            continue;
+            let l = line.len();
-        }
+            if l == 0 {
+                return None;
+            }
 
-        let loc = if line[l - 6] == b';' {
+            let loc = if line[l - 6] == b';' {
-            l - 6
+                l - 6
-        } else if line[l - 5] == b';' {
+            } else if line[l - 5] == b';' {
-            l - 5
+                l - 5
-        } else if line[l - 4] == b';' {
+            } else if line[l - 4] == b';' {
-            l - 4
+                l - 4
-        } else {
+            } else {
-            unreachable!();
+                unreachable!();
-        };
+            };
 
-        let (city, val) = line.split_at(loc);
+            let (city, val) = line.split_at(loc);
-        let val = &val[1..];
+            let val = &val[1..];
 
-        let val = parse_float(val);
+            let val = parse_float(val);
 
-        let (min, sum, max, counter) = map
+            Some((city, val))
-            .entry(city)
+        })
-            .or_insert_with(|| (std::i16::MAX, 0, std::i16::MIN, 0));
+        .fold(FxHashMap::default, |mut a, (city, val)| {
+            let (min, sum, max, counter) = a
+                .entry(city)
+                .or_insert_with(|| (std::i16::MAX, 0, std::i16::MIN, 0usize));
 
-        *min = val.min(*min);
+            *min = val.min(*min);
-        *max = val.max(*max);
+            *max = val.max(*max);
-        *counter += 1;
+            *counter += 1;
-        *sum += val as i32;
+            *sum += val as i32;
-    }
+
+            a
+        })
+        .reduce_with(|mut m1, m2| {
+            for (k, (min1, sum1, max1, counter1)) in m2 {
+                let (min, sum, max, counter) = m1
+                    .entry(k)
+                    .or_insert_with(|| (std::i16::MAX, 0, std::i16::MIN, 0usize));
+
+                *min = min1.min(*min);
+                *max = max1.max(*max);
+                *counter += counter1;
+                *sum += sum1;
+            }
+
+            m1
+        })
+        .unwrap();
 
     let t2_elapsed = t2.elapsed();