QuickSearch for files and directories

Question

I have a project where I need to search for directories and files. In fact I'm searching for paths in other projects. I don't want to just find files but I also want to exclude some directories or files from the results to avoid false positves.

In particular I'd like to exclude the following paths/patterns (not regex):

var directories = new[]
{
    @".git",
    @".idea",
    @".vs",
    @"_nuget",
    @"*.Tests",
    @"packages",
    @"TestResults",
    @"bin",
    @"obj",
};

var files = new[]
{
    @".xml",
    @".config",
    @".csproj",
};

There are not many wild cards in these patterns because files are matched at the end of the sting by default and directories are starting with a backslash ´\´. But it supports * and ?.

---

The core type is the QuickSearch. It takes a path where it should start searching and optionally a QuickSearchCriteria. Internally it does not work recursively but with a stack. Based on the search criteria it is able to exclude/include both files and/or directories from/in the results.

public interface IQuickSearch
{
    IEnumerable<string> EnumeratePaths(string path, QuickSearchCriteria criteria = null);
}

public class QuickSearch : IQuickSearch
{
    public IEnumerable<string> EnumeratePaths(string path, QuickSearchCriteria criteria = null)
    {

        criteria = criteria ?? new QuickSearchCriteria();

        var directories = new Stack<IEnumerable<string>>
        {
            Directory.EnumerateDirectories(path)
        };

        while (directories.Any())
        {
            foreach (var directory in directories.Pop().Where(DirectoryMatches))
            {
                yield return directory;

                if (criteria.Options.HasFlag(QuickSearchOptions.Recursive))
                {
                    directories.Push(Directory.EnumerateDirectories(directory));
                }

                foreach (var file in Directory.EnumerateFiles(directory).Where(FileMatches))
                {
                    yield return file;
                }
            }
        }

        bool DirectoryMatches(string directory)
        {
            return
                criteria.ExcludeDirectory.Matches(directory) == false &&
                criteria.Options.HasFlag(QuickSearchOptions.IncludeDirectories) &&
                criteria.IncludeDirectory.Matches(directory);
        }

        bool FileMatches(string file)
        {
            return
                criteria.ExcludeFile.Matches(file) == false &&
                criteria.Options.HasFlag(QuickSearchOptions.IncludeFiles) &&
                criteria.IncludeFile.Matches(file);
        }
    }
}

public static class StackExtensions
{
    public static void Add<T>(this Stack<T> stack, T item)
    {
        stack.Push(item);
    }
}

---

The QuickSearchCriteria is just a simple class with several defaults:

public class QuickSearchCriteria
{
    public DirectoryFilter ExcludeDirectory { get; set; } = DirectoryFilter.None;

    public FileFilter ExcludeFile { get; set; } = FileFilter.None;

    public DirectoryFilter IncludeDirectory { get; set; } = DirectoryFilter.All;

    public FileFilter IncludeFile { get; set; } = FileFilter.All;

    public QuickSearchOptions Options { get; set; } = QuickSearchOptions.IncludeBoth | QuickSearchOptions.Recursive;
}

[Flags]
public enum QuickSearchOptions
{
    IncludeDirectories = 1,
    IncludeFiles = 2,
    Recursive = 4,
    IncludeBoth = IncludeDirectories | IncludeFiles
}

---

Filters are based on the PathFilter that provides some default implementations and stores patterns.

public abstract class PathFilter : IEnumerable<string>
{
    // Using just a few symbols Windows is showing when renaming files.
    public static readonly string InvalidFileNameChars = @"""<>|:*?\/";
    private readonly string[] _patterns;
    private readonly Lazy<Func<string, bool>> _matcher;

    protected PathFilter(params string[] patterns)
    {
        _patterns = patterns;
        _matcher = new Lazy<Func<string, bool>>(CreateMatcher);
    }

    protected PathFilter(Func<string, bool> matcher)
    {
        _patterns = new string[0];
        _matcher = new Lazy<Func<string, bool>>(() => matcher);
    }

    protected abstract Func<string, bool> CreateMatcher();

    public bool Matches(string path)
    {
        return path == null ? false : _matcher.Value(path);
    }

    public IEnumerator<string> GetEnumerator() => ((IEnumerable<string>)_patterns).GetEnumerator();

    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
}

---

Filters I currently use are regex-based so the use another base class the RegexPathFilter. This class encapsulates common conversion of patterns to regex. Patterns that start and and with a slash / are considered as regex.

public abstract class RegexPathFilter : PathFilter
{
    protected RegexPathFilter(params string[] patterns) : base(patterns) { }

    protected RegexPathFilter(Func<string, bool> matcher) : base(matcher) { }

    protected override Func<string, bool> CreateMatcher()
    {
        var regexPatterns = this.ToLookup(IsRegex);
        var expressions = regexPatterns[true].Select(p => p.TrimStart('/').TrimEnd('/')).Concat(CreateRegularExpressions(regexPatterns[false]));
        var regex = new Regex($"({string.Join("|", expressions)})", RegexOptions.IgnoreCase);

        return regex.IsMatch;

        bool IsRegex(string value) => value.StartsWith("/") && value.EndsWith("/");
    }

    protected virtual IEnumerable<string> CreateRegularExpressions(IEnumerable<string> patterns)
    {
        return 
            from pattern in patterns 
            select Regex.Replace(pattern, @"\.", @"\.") into pattern 
            select Regex.Replace(pattern, @"\?", @".") into pattern 
            // Hard to decide which one to use.
            //select Regex.Replace(pattern, @"\*", $@"[^{Regex.Escape(InvalidFileNameChars)}]*?") into pattern
            select Regex.Replace(pattern, @"\*", $@".*?");
    }
}

---

The two main filters are the DirectoryFilter and the FileFilter. Each of them customizes the patters further to create the final version that either must start with a \ or be at the end of the string so that I don't have to use .*? for common exclusions like *.cs.

public class DirectoryFilter : RegexPathFilter
{
    public DirectoryFilter(params string[] patterns) : base(patterns) { }

    private DirectoryFilter(Func<string, bool> matcher) : base(matcher) { }

    public static DirectoryFilter None => new DirectoryFilter(_ => false);

    public static DirectoryFilter All => new DirectoryFilter(_ => true);

    public static DirectoryFilter Create(params string[] patterns)
    {
        return new DirectoryFilter(patterns);
    }

    protected override IEnumerable<string> CreateRegularExpressions(IEnumerable<string> patterns)
    {
        // Directories are matched at the \
        return base.CreateRegularExpressions(patterns).Select(p => @"\\" + p);
    }
}

public class FileFilter : RegexPathFilter
{
    public FileFilter(params string[] patterns) : base(patterns) { }

    private FileFilter(Func<string, bool> matcher) : base(matcher) { }

    public static FileFilter None => new FileFilter(_ => false);

    public static FileFilter All => new FileFilter(_ => true);

    public static FileFilter Create(params string[] patterns)
    {
        return new FileFilter(patterns);
    }

    protected override IEnumerable<string> CreateRegularExpressions(IEnumerable<string> patterns)
    {
        // Files are matched at the end of the string.
        return base.CreateRegularExpressions(patterns).Select(p => p + "$");
    }
}

---

Example To use it I just need to instantiate the QuickSearch and give it some search criteria:

var ignore = (DirectoryFilter.Create(directories), FileFilter.Create(files));

var pathEnumerator = new QuickSearch();

pathEnumerator
    .EnumeratePaths(@"c:\temp\projects\testproject", new QuickSearchCriteria
    {
        ExcludeDirectory = ignore.Item1,
        ExcludeFile = ignore.Item2
    })
    .Count()
    .Dump();

---

It takes 0.040sec to enumerate a directory with 10.600 files and 1.400 folders yealding 417 results with the above ignore list on my notebook's SSD.

Answer 1

...many refactorings later.

I came to the conclustion that this soltion was totaly overengineered. I simplified it and made more LINQ-friendly by using Func instead of the custom type PathFilter and the QuickSearchCriteria.

The core type is now the IFileSytem that is easier to use with DI and easier to mock. EnumerateDirectories just needs one parameter to make it faster, this is exclude directories, everything else I can filter outside later.

public interface IFileSystem
{
    IEnumerable<string> EnumerateDirectories(string path, Func<string, bool> exclude, bool deep = true);

    IEnumerable<string> EnumerateFiles(string path);
}

Its implementation has also become simpler because the big EnumeratePaths method is now two methods.

public class FileSystem : IFileSystem
{
    public IEnumerable<string> EnumerateDirectories(string path, Func<string, bool> exclude, bool deep = true)
    {
        if (path == null) throw new ArgumentNullException(nameof(path));

        var directories = new Stack<string>
        {
            path,
        };

        while (directories.Any())
        {
            foreach (var directory in Directory.EnumerateDirectories(directories.Pop()).Skip(exclude))
            {
                yield return directory;

                if (deep)
                {
                    directories.Push(directory);
                }
            }
        }
    }

    public IEnumerable<string> EnumerateFiles(string path)
    {
        if (path == null) throw new ArgumentNullException(nameof(path));

        return
            from file in Directory.EnumerateFiles(path)
            select file;
    }
}

To get files I can just use SelectMany:

var nuspecFilter = FileFilterFactory.Default.Create(".nuspec");

fileSystem
  .EnumerateDirectories(
        @"c:\temp\projects\testproject",
        directoryExclusions
            .Select(DirectoryFilterFactory.Default.Create)
            .ToList()
            .ToAny(),
     deep: true)
  .SelectMany(pathEnumerator.EnumerateFiles)
  .Where(nuspecFilter)
  .Dump();

where ToAny is implemented as

public static Func<TElement, bool> ToAny<TElement>(this IEnumerable<Func<TElement, bool>> filters)
{
    return x => filters.Any(f => f(x));
}

---

The rest of the code now implements the IFilterFactory interface.

public interface IFilterFactory<TElement, TCondition>
{
    Func<TElement, bool> Create(TCondition condition);
}

Currently there is only one such filter, the PathFilterFactory. This can create filters from three kinds of patterns:

• /regex/ - matches with user regex
• *.wildcardex - matches with regex compiled from wildcards
• stringex - matches files by EndsWith(stringex) and directories by IndexOf("\" + stringex)

(for the sake of consistency I add the ex at the end).

public abstract class PathFilterFactory : IFilterFactory<string, string>
{
    // Using just a few symbols Windows is showing when renaming files.
    public static readonly string InvalidFileNameChars = @"""<>|:*?\/"; 

    public static Func<string, bool> None => _ => false;

    public static Func<string, bool> All => _ => true;

    private IEnumerable<(Func<string, bool> Condition, Func<string, Func<string, bool>> Factory)> _factories;

    protected PathFilterFactory()
    {
        _factories = new(Func<string, bool> Condition, System.Func<string, Func<string, bool>>)[]
        {
            (IsRegex, FromRegex),
            (IsWildcardex, FromWildcardex),
            (_ => true, FromStringex),
        };
    }

    public Func<string, bool> Create(string pattern)
    {
        return 
            _factories
                .First(t => t.Condition(pattern))
                .Factory(pattern);
    }

    protected virtual Func<string, bool> FromRegex(string pattern)
    {
        return new Regex(pattern.TrimStart('/').TrimEnd('/'), RegexOptions.IgnoreCase).IsMatch;
    }

    protected abstract Func<string, bool> FromWildcardex(string pattern);   

    protected abstract Func<string, bool> FromStringex(string pattern);

    private static bool IsRegex(string value)
    {
        return
            value.StartsWith("/") &&
            value.EndsWith("/");
    }

    private static bool IsWildcardex(string value)
    {
        return
            value.Contains('*') ||
            value.Contains('?');
    }
}

public class DirectoryFilterFactory : PathFilterFactory
{
    public static readonly IFilterFactory<string, string> Default = new DirectoryFilterFactory();

    protected override Func<string, bool> FromWildcardex(string pattern)
    {
        pattern = Regex.Replace(pattern, @"\.", @"\.");
        pattern = Regex.Replace(pattern, @"\?", @".");
        // Hard to decide which one to use.
        //result = Regex.Replace(result, @"\*", $@"[^{Regex.Escape(InvalidFileNameChars)}]*?");
        pattern = Regex.Replace(pattern, @"\*", $@".*?");

        // Directories are matched at the '\'
        pattern = @"\\" + pattern;

        return new Regex(pattern, RegexOptions.IgnoreCase).IsMatch;
    }

    protected override Func<string, bool> FromStringex(string pattern)
    {
        pattern = @"\" + pattern;
        return path => path.IndexOf(pattern, StringComparison.OrdinalIgnoreCase) >= 0;
    }
}

public class FileFilterFactory : PathFilterFactory
{
    public static readonly IFilterFactory<string, string> Default = new FileFilterFactory();

    protected override Func<string, bool> FromWildcardex(string pattern)
    {
        pattern = Regex.Replace(pattern, @"\.", @"\.");
        pattern = Regex.Replace(pattern, @"\?", @".");
        // Hard to decide which one to use.
        //result = Regex.Replace(result, @"\*", $@"[^{Regex.Escape(InvalidFileNameChars)}]*?");
        pattern = Regex.Replace(pattern, @"\*", $@".*?");

        // Files are matched at the end of the string.
        pattern = pattern + "$";

        return new Regex(pattern, RegexOptions.IgnoreCase).IsMatch;
    }

    protected override Func<string, bool> FromStringex(string pattern)
    {
        return path => path.EndsWith(pattern, StringComparison.OrdinalIgnoreCase);
    }
}