Convert impute2 files to mach format

Question

Here is a program for converting Impute2 files into MaCH format (related to genetics).

Source files include one xxx_haps file and one xxx_samples file, for example:

// chromosome21.gen_haps
--- rs62224609 16051249 T C 0 0 0 0 0 0
--- rs62224610 16051347 G C 0 1 0 1 1 0
--- rs143503259 16051453 A C 0 0 0 0 0 0
--- rs192339082 16051477 C A 1 0 0 0 0 0
--- rs191139082 16052477 ACCTT A 1 0 0 0 0 0
--- rs191139082 16052478 G GCCGG 0 1 0 0 0 0

// chromosome21.gen_samples
ID_1 ID_2 missing father mother sex plink_pheno
0 0 0 D D D B
RS3 10051 0 0 0 2 -9
RS3 10052 0 0 0 2 -9
RS3 10068 0 0 0 2 -9

Output should be two MaCH files, one xxx.mach.out and one xxx.data.dat, for example, the above input files should be converted to:

// chromosome21.mach.out
RS3->10051 HAPL01 TGAADR
RS3->10051 HAPL02 TCACRI
RS3->10052 HAPL01 TGACRR
RS3->10052 HAPL02 TCACRR
RS3->10068 HAPL01 TCACRR
RS3->10068 HAPL02 TGACRR

// chromosome21.data.dat
M   rs62224609
M   rs62224610
M   rs143503259
M   rs192339082
M   rs191139082
M   rs191139082

Java code:

import com.beust.jcommander.JCommander;
import com.google.common.base.Joiner;

import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.regex.Pattern;

public class Main {
    public static void main(String[] args) throws Exception {
        ArgParser argParser = new ArgParser();
        new JCommander(argParser, args);
        if (argParser.debug) {
            System.out.printf("Debugging mode on\n%-22s%s\n%-22s%s\n",
                    "File to convert:",
                    argParser.file,
                    "Output folder:",
                    argParser.outDirString);
        }

        Pattern pattern = Pattern.compile("_haps\\.gz$");
        String fileStem = pattern.matcher(argParser.file).replaceAll("");
        System.out.println("File stem: " + fileStem);
        System.out.println("File : " + argParser.file);
        File outDir;
        File machOutFile = new File(fileStem + ".mach.out");
        File snpsOutFile = new File(fileStem + ".data.dat");
        if(!argParser.outDirString.isEmpty()) {
            outDir = new File(argParser.outDirString);
            if(!outDir.exists()) {
                outDir.mkdir();
            }
            machOutFile = new File(outDir.toString(), machOutFile.getName().toString());
            snpsOutFile = new File(outDir.toString(), snpsOutFile.getName().toString());
        }

        // Read the _samples file
        File sampleFile = new File(fileStem + "_samples");
        BufferedReader sampleBuffer = new BufferedReader(new FileReader(sampleFile));
        String sampleLine;
        ArrayList<String> fidiidList = new ArrayList<String>();
        sampleBuffer.readLine();
        sampleBuffer.readLine();
        while ((sampleLine = sampleBuffer.readLine()) != null) {
            String[] cols = sampleLine.split(" +", 3);
            String combinedLine = cols[0] + "->" + cols[1];
            fidiidList.add(combinedLine);
            fidiidList.add(combinedLine);
        }
        sampleBuffer.close();
        String[] fidiid = fidiidList.toArray(new String[fidiidList.size()]);

        // get the HAPL01 / HAPL02 rows
        String[] hapl = new String[fidiid.length];
        for(int i=0; i<hapl.length; i++) {
            if(i % 2 == 0) {
                hapl[i] = "HAPL01";
            } else {
                hapl[i] = "HAPL02";
            }
        }

        ArrayList<String[]> outputLines = new ArrayList<String[]>();
        outputLines.add(fidiid);
        outputLines.add(hapl);


        // read the _haps file
        File hapsFile   = new File(fileStem + "_haps");
        BufferedReader hapsBuffer = new BufferedReader(new FileReader(hapsFile));
        ArrayList<String> snps = new ArrayList<String>();
        String hapsLine;
        String tmpGen;
        while((hapsLine = hapsBuffer.readLine()) != null) {
            String[] cols = hapsLine.split(" +");
            String snp = "M\t" + cols[1];
            snps.add(snp);
            String refGen = cols[3];
            String mutGen = cols[4];
            String[] genCols = Arrays.copyOfRange(cols, 5, cols.length);
            String[] genColsTrans = new String[genCols.length];

            if(refGen.length() == 1 && mutGen.length() == 1) {
                for(int i=0; i<genCols.length; i++) {
                    genColsTrans[i] = (genCols[i].equals("0")) ? refGen : mutGen;
                }
            } else if(refGen.length() > 1 && mutGen.length() == 1) {
                for(int i=0; i<genCols.length; i++) {
                    genColsTrans[i] = (genCols[i].equals("0")) ? "R" : "D";
                }
            } else if(refGen.length() == 1 && mutGen.length() > 1) {
                for(int i=0; i<genCols.length; i++) {
                    genColsTrans[i] = (genCols[i].equals("0")) ? "R" : "I";
                }
            } else {
                throw new Exception("Impute2 file mal-formatted.");
            }
            outputLines.add(genColsTrans);
        }

        // Transpose the output matrix
        int nSnp = outputLines.size();
        int nObserve = fidiid.length;
        String[][] outputTranspose = new String[nObserve][nSnp];
        for(int i=0; i<nSnp; i++) {
            for(int j=0; j<nObserve; j++) {
                outputTranspose[j][i] = outputLines.get(i)[j];
            }
        }

        // get rows of strings
        String[] outputStrings = new String[nObserve];
        for(int i=0; i<nObserve; i++) {
            String[] idAndHapl = Arrays.copyOfRange(outputTranspose[i], 0, 2);
            String idAndHaplString = Joiner.on(" ").join(idAndHapl);
            String[] genotypes = Arrays.copyOfRange(outputTranspose[i], 2, outputTranspose[i].length);
            String genotypesString = Joiner.on("").join(genotypes);
            outputStrings[i] = idAndHaplString + " " + genotypesString;
        }

        String machOutString = Joiner.on("\n").join(outputStrings);
        BufferedWriter machOutBuffer = null;
        try {
            machOutBuffer = new BufferedWriter(new FileWriter(machOutFile));
            machOutBuffer.write(machOutString);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            machOutBuffer.close();
        }

        String snpsString = Joiner.on("\n").join(snps);
        BufferedWriter snpsOutBuffer = null;
        try {
            snpsOutBuffer = new BufferedWriter(new FileWriter(snpsOutFile));
            snpsOutBuffer.write(snpsString);
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            snpsOutBuffer.close();
        }
    }
}

---

// parsing commandline args in java, (options, similar to argparse for python)
import com.beust.jcommander.Parameter;
import com.beust.jcommander.internal.Lists;

import java.util.List;


public class ArgParser {
    @Parameter
    public List<String> parameters = Lists.newArrayList();
    @Parameter(names = "-file", description = "_haps.gz file to convert")
    public String file;
    @Parameter(names="-outDir", description = "Output folder")
    public String outDirString;
    @Parameter(names="-debug", description = "Debug mode")
    public boolean debug = false;
}

Answer 1

My rule of thumb is that a method should not be more than 10 lines long. So your single method makes me weep.

My guiding principle is the "single responsibility principle" (aka "separation of concerns"). You should try to split your method in smaller methods where each method does one "logical" thing. You will probably spend of lot time changing your mind and doing a lot of copy-paste and that's normal. As you gain experience you will get better.

For example the code that reads/writes to a file should be completely separated from the code that manipulates the data. So if you get the data from the network instead of reading the files from the filesystem, you only have to modify the part that does data input.

Since Java is an OO language, you should probably use that too. You have to think carefully about what is a class here. Maybe you could define the two kinds of files you are manipulating as classes. And those two types files seem to contain two different data structures which are written in different formats in the two files. You can define two new class for those data types, and they sub classes for the way those are represented in the two different file types. You would also need to define some methods to convert between those formats.

If you do it this way, when you add a third file type, all you have to do is add some new sub-classes and new converting methods. Even if you don't plan to add a third file type, you should code as if you will since it will help you modularize the code. Modularized code might be longer overall than non-modularized code, but it is a lot clearer to read and modify.

Answer 2

interfaces vs implementation

You should always code against interfaces not against implementations as far as you only need the methods provided by the interface.

ArrayList fidiidList = new ArrayList();

can be changed to

List<String> fidiidList = new ArrayList<String>();

If you are using java 1.7+ you can use the diamond operator like

List<String> fidiidList = new ArrayList<>();

You have create at least 38 objects in this method which contains 127 lines of code. This is way to much (both).

Naming

Just horrible. Sorry, I would like to say the naming of your variables are meaningful and pretty readable, but that would be a lie.

Consider using meaningful names, which you or Mr.Maintainer will still understand in 2 months or half a year.

Refactoring

Some pointers

• File.getFile() returns a String so there is no need to call toString()
• if a directory exists File.mkDir() does not throw an exception
• File has an overloaded constructor where the first parameter can be a File
• List<T> has an addAll() method which takes an Collection<T> as input parameter

• you should extract code that belongs together to separate methods

  So you could replace this

  // Read the _samples file
  File sampleFile = new File(fileStem + "_samples");
  BufferedReader sampleBuffer = new BufferedReader(new FileReader(sampleFile));
  String sampleLine;
  ArrayList<String> fidiidList = new ArrayList<String>();
  sampleBuffer.readLine();
  sampleBuffer.readLine();
  while ((sampleLine = sampleBuffer.readLine()) != null) {
      String[] cols = sampleLine.split(" +", 3);
      String combinedLine = cols[0] + "->" + cols[1];
      fidiidList.add(combinedLine);
      fidiidList.add(combinedLine);
  }
  

  with this

  List<String> fidiidList = = readFidiidList(new File(fileStem + "_samples"));
  

  by calling this extracted method

  private static List<String> readFidiidList(File sampleFile)
  {
      BufferedReader sampleBuffer = new BufferedReader(new FileReader(sampleFile));
  
      String sampleLine;
  
      sampleBuffer.readLine();
      sampleBuffer.readLine();
  
      List<String> fidiidList = new ArrayList<String>();
  
      while ((sampleLine = sampleBuffer.readLine()) != null) {
          String[] cols = sampleLine.split(" +", 3);
          String combinedLine = cols[0] + "->" + cols[1];
          fidiidList.add(combinedLine);
          fidiidList.add(combinedLine);
      }
      sampleBuffer.close();
  
      return fidiidList;
  }
  

After you have refactored your method, feel free to post a follow up question.