0
\$\begingroup\$

Working on a project of mine, I wanted to parallelize mapping operations on an input stream. I found pariter on crates.io but it did not fit my needs, because its generic's bounds require the iterator to be 'static, which is not what I want (the iterators I work with are not 'static).

I am not too experienced with parallel programming, since mostly I let frameworks do the heavy lifting for me, but here I felt compelled to write the needed parallelization on my own, since I did not find anything fitting my needs.

[package]
name = "threaded-map"
authors = ["Richard Neumann <[email protected]>"]
description = "Encode bytes as ANSI background colors"
license = "MIT"
homepage = "https://github.com/conqp/threaded-map/"
repository = "https://github.com/conqp/threaded-map/"
readme = "README.md"
documentation = "https://docs.rs/threaded-map"
keywords = [ "threaded", "mapping"]
version = "0.1.1"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
threadpool = "1.8.1"

src/lib.rs:

use std::sync::mpsc::{channel, Receiver};
use threadpool::{Builder, ThreadPool};

pub trait ThreadedMappable<F, O>
where
    Self: Iterator + Sized,
    F: Fn(<Self as Iterator>::Item) -> O + Send + Clone,
    <Self as Iterator>::Item: Send,
    O: Send + Sync,
{
    /// Maps items of an iterator in parallel while conserving their order
    /// # Examples
    /// ```
    /// use threaded_map::ThreadedMappable;
    /// let items = vec![1, 2, 3, 4, 5, 6];
    /// let target: Vec<_> = items.iter().map(i32::to_string).collect();
    ///
    /// let result: Vec<_> = items
    ///     .into_iter()
    ///     .parallel_map(|item| item.to_string(), None)
    ///     .collect();
    ///
    /// assert_eq!(result, target);
    /// ```
    fn parallel_map(self, f: F, num_threads: Option<usize>) -> ThreadedMap<Self, F, O> {
        ThreadedMap::new(self, f, num_threads)
    }
}

#[derive(Debug)]
pub struct ThreadedMap<I, F, O>
where
    I: Iterator,
    F: Fn(<I as Iterator>::Item) -> O + 'static,
    <I as Iterator>::Item: 'static,
    O: Send + 'static,
{
    iterator: I,
    function: F,
    thread_pool: ThreadPool,
    window: Vec<O>,
}

impl<I, F, O> ThreadedMap<I, F, O>
where
    I: Iterator,
    F: Fn(<I as Iterator>::Item) -> O + Send + Clone,
    <I as Iterator>::Item: Send,
    O: Send + Sync,
{
    pub fn new(iterator: I, function: F, num_threads: Option<usize>) -> Self {
        Self {
            iterator,
            function,
            thread_pool: num_threads.map_or_else(|| Builder::new().build(), ThreadPool::new),
            window: Vec::new(),
        }
    }

    fn send_items(&mut self) -> Receiver<(usize, O)> {
        let (tx, rx) = channel::<(usize, O)>();

        for (index, item) in (0..self.thread_pool.max_count())
            .map_while(|_| self.iterator.next())
            .enumerate()
        {
            let tx = tx.clone();
            let f = self.function.clone();
            self.thread_pool.execute(move || {
                tx.send((index, (f)(item)))
                    .expect("channel will be there waiting for the pool");
            });
        }

        rx
    }
}

impl<I, F, O> Iterator for ThreadedMap<I, F, O>
where
    I: Iterator,
    F: Fn(<I as Iterator>::Item) -> O + Send + Clone,
    <I as Iterator>::Item: Send,
    O: Send + Sync,
{
    type Item = O;

    fn next(&mut self) -> Option<Self::Item> {
        if let Some(item) = self.window.pop() {
            return Some(item);
        }

        let rx = self.send_items();
        let mut window: Vec<_> = rx.iter().collect();

        if window.is_empty() {
            return None;
        }

        window.sort_by(|(lhs, _), (rhs, _)| rhs.cmp(lhs));
        self.window = window.into_iter().map(|(_, item)| item).collect();
        self.window.pop()
    }
}

impl<I, F, O> ThreadedMappable<F, O> for I
where
    I: Iterator,
    F: Fn(<I as Iterator>::Item) -> O + Send + Clone,
    <I as Iterator>::Item: Send,
    O: Send + Sync,
{
}

The code currently works with the program I wrote it for.

However, I'd like to have feedback as to whether my code is idiomatic and whether it may have some flaws, especially regarding its multithreading efficiency.

\$\endgroup\$
1
  • 1
    \$\begingroup\$ The classic library for those tasks in Rust is rayon, which has many more downloads than the obscure library you've found and also doesn't require 'static. \$\endgroup\$ Jun 28 at 13:35

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Browse other questions tagged or ask your own question.