# Storing large maps in memory

Context of the problem:

I have a very large graph that cost about 4GB to store. About 3M nodes and 34M edges. My program takes this large graph and recursively builds smaller graphs from it. At each level of the recursion I have two graphs - the original graph and the graph created from the original. The recursion continues until the graph is reduced to very small graph say about 10 nodes.

Since I need these graphs for the entire execution of the program, memory efficiency is critical for my application.

Here's the problem I'm currently having:

This is the algorithm for creating a smaller graph from the larger one:

public static Graph buildByTriples(Graph g, ArrayList<Integer> seeds) {
ArrayList<Edge> edges = new ArrayList(g.getEdgeCount());
for (int i = 0; i < g.size(); i++) {
for (Edge e : g.adj(i)) {
int v = e.getEndpoint(i);
if (i < v) {
}
}
}

Table<Integer, Integer, Double> coarseEgdes = HashBasedTable.create(seeds.size(),seeds.size());
//compute coarse weights
edges.stream().forEach((e) -> {
int v = e.getV();
int u = e.getU();
if (g.isC(u) && g.isC(v)) {
addToTable(coarseEgdes, u, v, e.getWeight());
}else if(!g.isC(u) && g.isC(v)){ //F-C
for(Edge cEdge: g.cAdj(u)){//get coarse neighbors of the fine edges
int nb = cEdge.getEndpoint(u);
if(nb != v){
addToTable(coarseEgdes, v, nb, cEdge.getPij() * e.getWeight());

}
}
}else if(g.isC(u) && !g.isC(v)){//C-F
for(Edge cEdge: g.cAdj(v)){//get coarse neighbors of the fine edges
int nb = cEdge.getEndpoint(v);
if(nb != u){
addToTable(coarseEgdes, u, nb, cEdge.getPij() * e.getWeight());
}
}
}else{//F-F
for(Edge cEdgeU: g.cAdj(u)){//get coarse neighbors of the fine edges
int uNb = cEdgeU.getEndpoint(u);
int vNb = cEdgeV.getEndpoint(v);
if(uNb != vNb){
addToTable(coarseEgdes, uNb, vNb, cEdgeU.getPij() * e.getWeight() * cEdgeV.getPij());
}
}
}
}
});

return createGraph(g, coarseEgdes); //use the edges to build new graph. Basically loops through coarseEdges and add edge and weight to the new graph.
}

private static void addToTable(Table<Integer, Integer,Double> tbl, int r, int c, double val){
int mn = Math.min(r, c);//the smaller of the two nodeIds
int mx = Math.min(r, c);//the largest of the two nodeId
if(tbl.contains(mn, mx)){
tbl.put(mn, mx, tbl.get(mn, mx) + val);
}else{
tbl.put(mn, mx,val);
}
}


Now when I do this, I quickly run out of memory. I profiled the application with YourKit, and memory usage is over the roof (>6GB before it runs out) and consequently CPU usage too. coarseEdges can get really large. Is there a better in-memory Map implementation out there that scales with large datasets? Or is there a better way to do this without storing coarseEdges?

Note that my graph cannot retrieve an edge(u,v) in constant time. It's basically a list of list and this serves performance of other critical part of my application better.

Also, see my graph implementation:

public class Graph{
private final int SIZE;
private final EdgeList[] nodes;
private final float[] volumes;
private final double[] weightedSum;
private final double[] weightedCoarseSum;
private final int[] nodeDegrees;
private final int[] c_nodeDegrees;
private int edge_count=0;
private final boolean[] coarse;
private final EdgeList[] coarse_neighbors;
public Graph(int SIZE){
this.SIZE =SIZE;
nodes = new EdgeList[SIZE];
coarse_neighbors = new EdgeList[SIZE];

volumes = new float[SIZE];
coarse = new boolean[SIZE];

//initialize data
weightedSum = new double[SIZE];
weightedCoarseSum = new double[SIZE];
nodeDegrees= new int[SIZE];
c_nodeDegrees = new int[SIZE];

for(int i=0;i<SIZE;i++){
nodes[i]=new EdgeList();
coarse_neighbors[i] = new EdgeList();
volumes[i]=1;
}
}

public void addEdge(int u, int v, double w){
//graph is undirected
//In order to traverse edges in order such that u < v. We store edge u,v such that u<v
Edge e=null;
if(u<v){
e = new Edge(u,v,w);
}else if(u>v){
e = new Edge(v,u,w);
}else{
throw new UnsupportedOperationException("Self loops not allowed in graph"); //TODO: Need a graph validation routine
}

//update the weighted sum of each edge
weightedSum[u] += w;
weightedSum[v] += w;

//update the degree of each edge
++nodeDegrees[u];
++nodeDegrees[v];

++edge_count;
}

public int size(){
return SIZE;
}

public EdgeList adj(int v){
return nodes[v];
}

public EdgeList cAdj(int v){
return coarse_neighbors[v];
}

public void sortAdj(int u, Comparator<Edge> c){
nodes[u].sort(c);
}

public void sortCoarseAdj(int u, Comparator<Edge> c){
coarse_neighbors[u].sort(c);
}

public void setCoarse(int node, boolean c){
coarse[node] = c;
if(c){
//update the neighborHood of node
int v = e.getEndpoint(node);
weightedCoarseSum[v] += e.getWeight();
++c_nodeDegrees[v];
}
}
}

public int getEdgeCount(){
return edge_count;
}

public boolean isC(int id){
return coarse[id];
}

public double weightedDegree(int node){
return weightedSum[node];
}

public double weightedCoarseDegree(int node){
return weightedCoarseSum[node];
}

public int degree(int u){
return nodeDegrees[u];
}

public int cDegree(int u){
return c_nodeDegrees[u];
}

public Edge getCNeighborAt(int u,int idx){
return coarse_neighbors[u].getAt(idx);
}

public float volume(int u){
return volumes[u];
}

public void setVolume(int node, float v){
volumes[node] = v;
}

@Override
public String toString() {
return "Graph[nodes:"+SIZE+",edges:"+edge_count+"]";
}

}

//Edges are first class objects.
public class Edge {
private boolean deleted=false;
private int u;
private int v;
private double weight;
private double pij;
private double algebraicDist = (1/Constants.EPSILON);

public Edge(int u, int v, double weight) {
this.u = u;
this.v = v;
this.weight = weight;
}

public Edge() {
}

public int getU() {
return u;
}

public void setU(int u) {
this.u = u;
}

public int getV() {
return v;
}

public void setV(int v) {
this.v = v;
}

public int getEndpoint(int from){
if(from == v){
return u;
}

return v;
}

public double getPij() {
return pij;
}

public void setPij(double pij) {
this.pij = pij;
}

public double getAlgebraicDist() {
return algebraicDist;
}

public void setAlgebraicDist(double algebraicDist) {
this.algebraicDist = algebraicDist;
}

public boolean isDeleted() {
return deleted;
}

public void setDeleted(boolean deleted) {
this.deleted = deleted;
}

public double getWeight() {
return weight;
}

public void setWeight(double weight) {
this.weight = weight;
}

@Override
public String toString() {
return "Edge[u:"+u+", v:"+v+"]";
}
}

// The Edge iterable
public class EdgeList implements Iterable<Edge>{
private final ArrayList<Edge> data= new ArrayList();

public void add(Edge e){
}

@Override
public Iterator<Edge> iterator() {
Iterator<Edge> it = new IteratorImpl();
return it;
}

private class IteratorImpl implements Iterator<Edge> {

public IteratorImpl() {
}
private int currentIndex = 0;
private final int N = data.size();
@Override
public boolean hasNext() {

//skip deleted
while(currentIndex < N && data.get(currentIndex).isDeleted()){
currentIndex++;
}

return currentIndex < N;
}

@Override
public Edge next() {
return data.get(currentIndex++);
}

@Override
public void remove() {
throw new UnsupportedOperationException();
}
}

public Edge getAt(int idx){
return data.get(idx);
}

public void sort(Comparator<Edge> c){
data.sort(c);
}
}


Renumbering the nodes I wanted to include in the smaller graph from 0 - n enabled me to use an ArrayList to store the edges the following way ArraList<HashMap<Integer, Double>>. This improved my performance some but I'm open to your suggestions and ways I can improve in general. Also, someone on SO mentioned a problem with my next and hasNext functions. What am I missing?