Erlang: pmap with max number of processes

Question

I'm working through Joe Armstrong's Programming Erlang 2nd E. The book has exercises at the end of each chapter. Chapter 26, Question 5 is:

Write a function called pmap(F, L, Max) that computes the list [F(I) || I <- L] in parallel but is subject to the restriction that no more than Max parallel processes run simultaneously.

My solution is:

-module(pmap_cap).
-export([pmap/3]).

pmap(F, L, Max) ->
    S = self(),
    Ref = make_ref(),
    Pids = lists:map(fun(SubL) ->
                             spawn(fun() -> do_f(S, Ref, F, SubL) end)
                     end,
                     partition(L, min(length(L), Max))),
    gather(Pids, Ref).

partition(L, N) ->
    M = length(L),
    if
        M =< N -> lists:map(fun(X) -> [X] end, L);
        true -> partition(L, M div N, M rem N)
    end.

partition([], _Q, _R) -> [];
partition(L, Q, R) ->
    Extra = if R > 0 -> 1; true -> 0 end,
    [lists:sublist(L, Q + Extra)|
     partition(lists:sublist(L, Q + Extra + 1, length(L)), Q, R-1)].

do_f(Parent, Ref, F, SubL) ->
    Parent ! {self(), Ref, lists:map(fun(X) -> catch F(X) end, SubL)}.

gather([Pid|T], Ref) ->
    receive
        {Pid, Ref, Ret} ->
            lists:append(Ret, gather(T, Ref))
    end;
gather([], _) -> [].

What could be improved? Are there performance issues? Can the code be written more idiomatically?

Answer 1

General

1. Use standard library as many as possible, usually it given better result therefore function partition can be easily rewritting with wrapper on lists:split/2 and it will be consider as perfomance improvement

   split(List,Max)->
       L = length(List),
       split_by(List,L rem Max + L div Max,[]).
   
   split_by([],_N,R)->lists:reverse(R);
   split_by(List,N,R)->
       {Part,NewList} = lists:split(N,List),
       split_by(NewList,N,[Part|R]).
   

Idiomaticness

1. Add to head of list and reverse instead append. For explanation - SO question.

2. Tail recursion better for readability and may be faster, so function gather become:

   gather([Pid|T], Ref,R) ->
       receive
           {Pid, Ref, Ret} -> gather(T, Ref,[Ret|R])
       end;
   gather([], _,R) -> lists:flatten(lists:reverse(R)).
   

3. List comprehensions instead lists:map/2:

   Min = min(length(L), Max),
   Pids = [spawn(
       fun() -> do_f(S, Ref, F, SubL) end) || SubL <- split(L, Min)],
   ...