Take the 2-minute tour ×
Code Review Stack Exchange is a question and answer site for peer programmer code reviews. It's 100% free, no registration required.

I have written a C# file reader that reads a file converts the bytes of the file to hex and writes it out to another file. It works fine but it takes 7.2GB of memory when converting a 300MB file.

What can I do to this code to make it take less memory?

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Hex__Reader
{
    class Program
    {
        static void Main(string[] args)
        {
            OpenFile();
        }

        private static void OpenFile()
        {
            uint j = 0;
            byte[] bytes = System.IO.File.ReadAllBytes(filename);
            char[] hex = BitConverter.ToString(bytes).Replace("-", " ").ToCharArray();
            //char[] hexchar = hex.ToCharArray();
            System.IO.StreamWriter writer = new System.IO.StreamWriter(filename,false);
            Console.WriteLine("Writing Bytes");
            for (uint i = 0; i < hex.Length; i++)
            {
                if (j == 50)
                {
                    writer.WriteLine();
                    writer.WriteLine();
                    j = 0;
                }
                else
                {
                    writer.Write(hex[i]);
                    j++;
                }
            }
        }
    }
}
share|improve this question

2 Answers 2

up vote 8 down vote accepted

Your problem is that you load the whole file into memory, convert each byte to three characters (that's the file size times 6), going through Replace and ToCharArray.

To avoid holding all in memory, you should stream the data, converting and writing each chunk. This way, you'll only hold a fraction of the file at each time:

using (System.IO.StreamWriter writer = new System.IO.StreamWriter(filename,false)) {
    using (FileStream fs = File.OpenRead(filename)) 
    {
        byte[] b = new byte[50];

        while (fs.Read(b,0,b.Length) > 0) 
        {
            writer.WriteLine(BitConverter.ToString(bytes).Replace("-", " "));
        }
    }
}

If (even when the buffer size is more optimal at 1024) you still have memory issues, it might be because of BitConverter.ToString(bytes).Replace("-", " ")... you could use some other methods to make to conversion, as suggested here, for example:

int itemsInLineCounter = 0;
using (System.IO.StreamWriter writer = new System.IO.StreamWriter(filename,false)) {
    using (FileStream fs = File.OpenRead(filename)) {
        byte[] b = new byte[1024];
        int read;

        while ((read = fs.Read(b,0,b.Length)) > 0) {
            for (int i = 0; i < read; i++) {
                writer.Write(bytes[i].ToString("X2"));
                if (itemsInLineCounter > NUM_OF_ITEMS_IN_LINE) {
                    writer.WriteLine();
                    itemsInLineCounter = 0;
                } else {
                    writer.Write(' ');
                    itemsInLineCounter++;
                }
            }
        }
    }
}
share|improve this answer
    
it uses more memory now and it also takes longer –  System.Object Mar 15 at 19:28
    
hmm... the buffer size is not optimal... try using buffer size 1024... –  Uri Agassi Mar 15 at 19:32
    
added some other options –  Uri Agassi Mar 15 at 20:38
    
Another option, which should create fewer temporary/short strings, would be to expand line-sized chunks of text from bytes into a StringBuilder. –  ChrisW Mar 15 at 20:42
    
Down To 320MB Now –  System.Object Mar 16 at 11:23

If you're interested in further optimization, this should maintain a minimal memory footprint and be about 25% faster:

    private static void OpenFile()
    {
        string[] digits = new string[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F" };
        using(System.IO.BinaryReader br = new System.IO.BinaryReader
            (new System.IO.FileStream
                (
                "ByteFile.dat", 
                System.IO.FileMode.Open, 
                System.IO.FileAccess.Read, 
                System.IO.FileShare.None, 
                1024)
                )
                )
        {
            using(System.IO.StreamWriter sw = new System.IO.StreamWriter("HexFile.txt"))
            {
                byte[] inbuff = new byte[0];
                int b = 0;
                while((inbuff = br.ReadBytes(50)).Length > 0)
                {
                    for(b = 0; b < inbuff.Length - 1; b++)
                    {
                        sw.Write(digits[(inbuff[b] / 16) % 16] + digits[inbuff[b] % 16] + " ");
                    }
                    sw.WriteLine(digits[(inbuff[b] / 16) % 16] + digits[inbuff[b] % 16]);

                }
            }
        }
    }

You'll notice that using a string array as a look up table for the hex digits eliminates any tostring conversions.

share|improve this answer

Your Answer

 
discard

By posting your answer, you agree to the privacy policy and terms of service.

Not the answer you're looking for? Browse other questions tagged or ask your own question.