1
\$\begingroup\$

I implemented a Column-based FLAT parser and generator, where the information is loaded from a base class, which contains the attributes that tell the column size and type. For each line of the flat file, a new instance of this class is generated which in turn is loaded with the file line data.

My question is, what can be improved in the code to make it more efficient?

Some kind of optimization in the reflection of the attributes maybe?

The source code:

DataType.cs

namespace FlatParser
{
    public enum DataType
    {
        NUMERIC,
        ALPHANUMERIC,
        DATETIME,
        IGNORE_FIELD
    }
}

FileReader.cs

using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace FlatParser
{
    public class FileReader : IDisposable
    {
        private StreamReader Reader { get; set; }
        public int CurrentLineNumber { get; private set; }
        public string CurrentLine { get; private set; }
        public bool EOF { get; private set; } = false;

        public FileReader(string path)
            : this(path, Encoding.GetEncoding(1252))
        {
        }

        public FileReader(string path, Encoding encoding)
        {
            Reader = new StreamReader(path, encoding);
        }

        public void Dispose()
        {
            Reader.Dispose();
        }

        public virtual bool ReadLine()
        {
            while(true)
            {
                CurrentLine = Reader.ReadLine();
                if(CurrentLine == null)
                {
                    Reader.Close();
                    CurrentLine = String.Empty;
                    EOF = true;
                    return false;
                }
                CurrentLineNumber++;
                if (CurrentLine.Length > 0)
                    break;
            }
            return true;
        }
    }
}

LayoutAttribute.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace FlatParser
{
    [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)]
    public sealed class LayoutAttribute : Attribute
    {
        public int LineSize { get; private set; }

        public LayoutAttribute(int lineSize)
        {
            LineSize = lineSize;
        }
    }
}

LayoutDetailsAttribute.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using FlatParser.Exceptions;

namespace FlatParser
{
    [AttributeUsage(AttributeTargets.Property, AllowMultiple = false, Inherited = false)]
    public sealed class LayoutDetailsAttribute : Attribute
    {
        public int StartPosition { get; private set; }
        public int EndPosition { get; private set; }
        public DataType _DataType { get; private set; }
        public string FormatoDateTime { get; private set; }
        public string DefaultValue { get; private set; }

        public LayoutDetailsAttribute(int startPosition, int endPosition, DataType datatype, string defaultValue = null)
            : this(startPosition, endPosition, datatype, null, defaultValue)
        {
        }

        public LayoutDetailsAttribute(int startPosition, int endPosition, DataType datatype, string dateTimeFormat, string defaultValue = null)
        {
            if (startPosition >= 0 && endPosition >= startPosition)
            {
                StartPosition = startPosition;
                EndPosition = endPosition;
                _DataType = datatype;

                if (datatype == DataType.DATETIME && (dateTimeFormat == null || dateTimeFormat.Trim().Length == 0))
                    throw new FlatParserException("Deve ser informado o formato da data quando utilizado datetime para o tipo do layout");
                else
                    FormatoDateTime = dateTimeFormat;

                DefaultValue = defaultValue;
            }
            else
                throw new FlatParserException("Posicao inicial deve ser maior ou igual a posicao final");
        }
    }
}

LayoutUtility.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;

using FlatParser.Exceptions;

namespace FlatParser
{
    public abstract class LayoutUtility
    {
        public void Parse(string input)
        {
            if (!String.IsNullOrEmpty(input))
            {
                // Get the class attributes...
                {
                    var classAttribute = GetType().GetCustomAttribute<LayoutAttribute>();
                    if (input.Length != classAttribute.LineSize)
                        throw new DefaultFieldException("Tamanho da linha invalido");
                }

                foreach (PropertyInfo property in GetType().GetProperties())
                {
                    Attribute attribute = null;

                    // para propriedades da classe sem atributos ou type IGNORED_FIELD (campo que não refletem posição do arquivo lido)
                    try
                    {
                        attribute = (Attribute)property.GetCustomAttributes(true).First();

                        var typeAttributeTest = attribute as LayoutDetailsAttribute;
                        if (typeAttributeTest._DataType == DataType.IGNORE_FIELD)
                            continue;
                    }
                    catch (Exception)
                    {
                        continue;
                    }


                    LayoutDetailsAttribute layoutAttribute = attribute as LayoutDetailsAttribute;
                    if (null != layoutAttribute)
                    {
                        string tmp = string.Empty;
                        if (layoutAttribute.StartPosition <= input.Length - 1)
                        {
                            tmp = input.Substring(layoutAttribute.StartPosition, Math.Min((layoutAttribute.EndPosition - layoutAttribute.StartPosition + 1), input.Length - layoutAttribute.StartPosition));
                        }
                        switch (layoutAttribute._DataType)
                        {
                            case DataType.ALPHANUMERIC:
                                tmp = tmp.TrimEnd();
                                break;
                        }

                        if (layoutAttribute.DefaultValue != null && layoutAttribute.DefaultValue.Trim().Length > 0)
                        {
                            string dfTmp = tmp;
                            if (dfTmp.Length < layoutAttribute.DefaultValue.Length)
                            {
                                if (layoutAttribute._DataType == DataType.ALPHANUMERIC)
                                    dfTmp = dfTmp.PadRight(layoutAttribute.EndPosition - layoutAttribute.StartPosition + 1, ' ');
                                else if (layoutAttribute._DataType == DataType.NUMERIC)
                                    dfTmp = dfTmp.PadLeft(layoutAttribute.EndPosition - layoutAttribute.StartPosition + 1, '0');
                            }

                            if (dfTmp != layoutAttribute.DefaultValue)
                            {
                                throw new DefaultFieldException(String.Format("O campo {0} não contem o valor \"{1}\"", property.Name, layoutAttribute.DefaultValue));
                            }
                        }

                        if (layoutAttribute._DataType != DataType.DATETIME)
                            property.SetValue(this, tmp, null);
                        else
                        {
                            try
                            {
                                DateTime dt;
                                if (DateTime.TryParseExact(tmp, layoutAttribute.FormatoDateTime, CultureInfo.InvariantCulture, DateTimeStyles.None, out dt))
                                    property.SetValue(this, dt, null);
                                else
                                    property.SetValue(this, null, null);
                            }
                            catch (Exception)
                            {
                                throw new DefaultFieldException(String.Format("O campo {0} não é válido para data", property.Name));
                            }
                        }
                    }
                }
            }
        }

        public override string ToString()
        {
            var classAttribute = GetType().GetCustomAttribute<LayoutAttribute>();
            StringBuilder result = new StringBuilder(String.Empty.PadLeft(classAttribute.LineSize, ' '));

            foreach (PropertyInfo property in GetType().GetProperties())
            {
                foreach (Attribute attribute in property.GetCustomAttributes(false))
                {

                    var typeAttributeTest = attribute as LayoutDetailsAttribute;
                    if (typeAttributeTest._DataType == DataType.IGNORE_FIELD)
                        continue;


                    LayoutDetailsAttribute layoutAttribute = attribute as LayoutDetailsAttribute;
                    if (null != layoutAttribute)
                    {
                        string propertyValue = String.Empty;
                        if (layoutAttribute._DataType == DataType.ALPHANUMERIC || layoutAttribute._DataType == DataType.NUMERIC)
                        {
                            var prop = property.GetValue(this, null);
                            if (prop != null)
                                propertyValue = (string)property.GetValue(this, null);
                        }
                        else
                        {
                            var prop = (DateTime?)property.GetValue(this, null);
                            if (prop == null)
                            {
                                propertyValue = (String.Empty).PadLeft(layoutAttribute.FormatoDateTime.Length, '0');
                            }
                            else if (prop.Value.Day == 1 && prop.Value.Month == 1 && prop.Value.Year == 1)
                            {
                                propertyValue = (String.Empty).PadLeft(layoutAttribute.FormatoDateTime.Length, '0');
                                //propertyValue = DateTime.Today.ToString(layoutAttribute.FormatoDateTime);
                            }
                            else
                                propertyValue = ((DateTime)prop).ToString(layoutAttribute.FormatoDateTime);
                        }

                        // Tem valor default
                        if (layoutAttribute.DefaultValue != null && layoutAttribute.DefaultValue.Trim().Length > 0)
                            propertyValue = layoutAttribute.DefaultValue;

                        switch (layoutAttribute._DataType)
                        {
                            case DataType.ALPHANUMERIC:
                                propertyValue = propertyValue.PadRight((layoutAttribute.EndPosition - layoutAttribute.StartPosition) + 1, ' ');
                                break;
                            case DataType.NUMERIC:
                                propertyValue = propertyValue.PadLeft((layoutAttribute.EndPosition - layoutAttribute.StartPosition) + 1, '0');
                                break;
                        }

                        for (int i = 0, j = layoutAttribute.StartPosition; i < propertyValue.Length; i++, j++)
                            result[j] = propertyValue[i];
                    }
                    break;
                }
            }
            return result.ToString().ToUpper();
        }

        public abstract bool IsValid();
    }
}

Test source file:

Header.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using FlatParser;

namespace FlatFileTest
{
    [Layout(lineSize: 1200)]
    public class Header : LayoutUtility
    {
        [LayoutDetails(0, 0, DataType.ALPHANUMERIC)]
        public string TIPO_DE_REGISTRO { get; set; }

        //[LayoutDetails(1, 16, DataType.ALPHANUMERIC)]
        //public string FILLER0 { get; set; }

        [LayoutDetails(17, 27, DataType.ALPHANUMERIC)]
        public string NOME_DO_ARQUIVO { get; set; }

        [LayoutDetails(28, 35, DataType.DATETIME, dateTimeFormat: "yyyyMMdd")]
        public DateTime DATA_DE_GRAVACAO { get; set; }

        [LayoutDetails(36, 43, DataType.NUMERIC)]
        public string NUMERO_DA_REMESSA { get; set; }

        //[LayoutDetails(44, 1198, DataType.ALPHANUMERIC)]
        //public string FILLER1 { get; set; }

        [LayoutDetails(1199, 1199, DataType.ALPHANUMERIC)]
        public string FIM { get; set; }

        public override bool IsValid()
        {
            throw new NotImplementedException();
        }
    }
}

main.cs

static void Teste()
{
    using (FileReader fr = new FileReader("file.txt", Encoding.GetEncoding(1252)))
    {
        var header = new Header();
        header.Parse(fr.CurrentLine);

        Console.WriteLine(header.DATA_DE_GRAVACAO.ToString("dd/MM/yyyy"));
    }
}
\$\endgroup\$
  • \$\begingroup\$ What is a flat parser, a parser of real estate? \$\endgroup\$ – Mast Oct 12 at 20:40
2
\$\begingroup\$

Let’s capture parsing/formatting logic in the attributes to simplify extensibility. You would need to inherit the following base attribute class to define new column types:

[AttributeUsage(AttributeTargets.Property)]
public abstract class FieldAttribute : Attribute
{
    protected FieldAttribute(int length, int order) => 
        Length = length;

    public int Length { get; }
    public int AbsLength => Math.Abs(Length);
    public int Order { get; }
    protected abstract string Format(object value);
    protected abstract object Parse(string text);

    public string this[PropertyInfo property, object record]
    {
        get => Clip(Align(Format(property.GetValue(record))));                
        set => property.SetValue(record, 
            Convert.ChangeType(
                Parse(value.Trim()), property.PropertyType));
    }

    string Align(string text) =>
        Length > 0 ? text.PadRight(AbsLength) : text.PadLeft(AbsLength);

    string Clip(string text) => 
        text.Substring(0, AbsLength);
}

As you can see the negative Length value right-aligns the content.

Now some concrete field type specializations:

public class TextAttribute : FieldAttribute
{
    public TextAttribute(int length, [CallerLineNumber] int order = 0)
        : base(length, order)
    {
    }

    protected override string Format(object value) => $"{value}";
    protected override object Parse(string text) => text;
}

And:

public class NumberAttribute : FieldAttribute
{
    public NumberAttribute(int length, [CallerLineNumber] int order = 0)
        : base(length, order)
    {
    }

    protected override string Format(object value) => $"{value}";
    protected override object Parse(string text) => decimal.Parse(text);
}

And:

public class DateAttribute : FieldAttribute
{
    public DateAttribute(int length, [CallerLineNumber] int order = 0) 
        : base(length, order)
    {
    }

    protected override string Format(object value) => $"{value:yyyyMMdd}";
    protected override object Parse(string text) => 
        DateTime.ParseExact(text, "yyyyMMdd", CultureInfo.InvariantCulture);
}

The next step would be to create a serializer which caches the schema provided by reflection:

public class Serializer<T> where T : new()
{
    static IEnumerable<(PropertyInfo Property, FieldAttribute Attribute)> Schema { get; } =
        typeof(T).GetProperties()
            .SelectMany(p => from a in p.GetCustomAttributes(true).OfType<FieldAttribute>()
                             orderby a.Order ascending
                             select (p, a))
            .ToArray();

    public static T Parse(string line)
    {
        var record = new T();
        var start = 0;
        foreach (var (property, attribute) in Schema)
        {
            attribute[property, record] = line.Substring(start, attribute.AbsLength);
            start += attribute.AbsLength;
        }

        return record;
    }

    public static string Format(T record) =>
        string.Join("", from f in Schema
                        select f.Attribute[f.Property, record]);
}

Now let’s define loading/saving:

public class File<THeader, TRecord> 
    where THeader: new()
    where TRecord: new()
{
    public static File<THeader, TRecord> Parse(string text) =>
        Load(new StringReader(text));

    public static File<THeader, TRecord> Load(string path) =>
        Load(new StreamReader(path));

    public static File<THeader, TRecord> Load(TextReader reader)
    {
        using (reader)
            return new File<THeader, TRecord>(
                Serializer<THeader>.Parse(reader.ReadLine()),
                Enumerable.Range(0, int.MaxValue)
                    .Select(i => reader.ReadLine())
                    .TakeWhile(s => s != null)
                    .Select(s => Serializer<TRecord>.Parse(s)));
    }

    public File()
        : this(new THeader(), new TRecord[0])
    {
    }

    File(THeader header, IEnumerable<TRecord> records) =>
        (Header, Records) = (header, records.ToList());

    public THeader Header { get; } 
    public IList<TRecord> Records { get; }

    public override string ToString() => 
        string.Join(Environment.NewLine,
            Records.Select(Serializer<TRecord>.Format)
            .Prepend(Serializer<THeader>.Format(Header)));

    public void Save(string path) =>
        Save(new StreamWriter(path));

    public void Save(TextWriter writer)
    {
        using (writer)
            writer.WriteLine(ToString());
    }
}

We are ready to test (note the localization support):

[TestClass]
public class File_Should
{
    [TestMethod]
    public void Serialize()
    {
        var file = new File<FileHeader, FileRecord>();
        file.Records.Add(new FileRecord
        {
            Name = "Thomas Jefferson",
            Born = new DateTime(1743, 04, 13),
            Age = 83
        });

        var s = file.ToString();
        var copy = File<FileHeader, FileRecord>.Parse(s);

        Assert.AreEqual("Имя", copy.Header.Name);
        Assert.AreEqual("ДР", copy.Header.Born);
        Assert.AreEqual("Возраст", copy.Header.Age);
        Assert.AreEqual("Thomas Jefferson", copy.Records[0].Name);
        Assert.AreEqual(new DateTime(1743, 04, 13), copy.Records[0].Born);
        Assert.AreEqual(83, copy.Records[0].Age);
    }
}

Where:

class FileHeader
{
    [Text(20)] public string Name { get; set; } = "Имя";
    [Text(10)] public string Born { get; set; } = "ДР";
    [Text(10)] public string Age { get; set; } = "Возраст";
}

And:

class FileRecord
{
    [Text(20)] public string Name { get; set; }
    [Date(-10)] public DateTime Born { get; set; }
    [Number(-10)] public int Age { get; set; }
}
\$\endgroup\$
  • \$\begingroup\$ thanks! you are awesome!!! I have some questions, let's suppose I have a file in the following format: (header: 0AAAANNN .....), (Record ( type 1 ): 1NNNNNDDD ...), (Record ( type 2 ): 2AAAAAAAA ....), (N records of type 1 and 2), (Trailler: 9SSSSSSS), how can this be implemented in your logic? \$\endgroup\$ – Alexandre Oct 14 at 13:10
  • \$\begingroup\$ @Alexandre Please see my new answer :) \$\endgroup\$ – Dmitry Nogin Oct 15 at 4:21
1
\$\begingroup\$

let's suppose I have a file in the following format: (header: 0AAAANNN .....), (Record ( type 1 ): 1NNNNNDDD ...), (Record ( type 2 ): 2AAAAAAAA ....), (N records of type 1 and 2), (Trailler: 9SSSSSSS), how can this be implemented in your logic?

You could use an extension method to parse:

static class RecordReader
{
    public static Record ReadRecord(this TextReader reader)
    {
        Record Parse<TRecord>(string s) where TRecord : Record, new() =>
            Serializer<TRecord>.Parse(s);

        switch (reader.ReadLine())
        {
            case string s when s.StartsWith("0"):
                return Parse<Record0>(s);
            case string s when s.StartsWith("1"):
                return Parse<Record1>(s);
            case string s when s.StartsWith("2"):
                return Parse<Record2>(s);
            case string s when s.StartsWith("9"):
                return Parse<Record9>(s);
            default:
                return null;
        }
    }
}

Where you have formats defined like:

abstract class Record { }
class Record0 : Record { ... }
class Record1 : Record { ... }
class Record2 : Record { ... }
class Record9 : Record { ... }
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.