Here's a first tweak: Tighten recursion so there's fewer lines to read that contain recursion.
go :: String
go = let vind = Prelude.map toInd [(4,4),(2,3),(4,3),(1,1),(1,3),(2,1),(3,1),(1,1)]
pool = V.fromList vind
cmp x y = indCmpFit x y 0
pool_s = poolSort pool cmp
in show $ rankSort pool_s
rankSort :: Pool -> Fronts
rankSort = foldl new_fronts start_front where
start_front = V.fromList [V.fromList [(pool V.! 0){ rank = 0 }]] -- first ind. always has rank 0
new_fronts :: IndFronts -> FrontsInd -> Fronts
new_fronts ind fronts ind = if rank' == V.length fronts
then V.snoc fronts (V.fromList [upd_ind]) -- add a new front with ind as member
else V.update fronts (V.fromList [(rank', upd_front)]) -- add individual to existing front
where
upd_ind = rankIndex ind fronts
rank' = rank upd_ind
upd_front = V.snoc (fronts V.! rank') upd_ind -- add individual to already existing front
Edit: Second: Shuffle arguments around so eta reduction is feasible. Also inline most stuff that's only used once.
indCmpFit :: Int -> Ind -> Ind -> Ordering
indCmpFit fitIdx x y
go :: String
go = let vind = Prelude.map toInd [(4,4),(2,3),(4,3),(1,1),(1,3),(2,1),(3,1),(1,1)]
in show $ rankSort $ poolSort (V.fromList vind) (indCmpFit 0)