To keep your code clean, you could define delegates:

delegate int AxisLocationDelegate(Rectangle r, bool primaryAxis);
delegate int AxisSizeDelegate(Rectangle r, bool primaryAxis);

Then define a unified GroupRectanglesBy function that takes an implementation of each of these delegates as parameters:

private List<Rectangle[]> GroupRectanglesBy(Rectangle[] rectangles,
    AxisLocationDelegate getLocation, AxisSizeDelegate getSize, out int averageGap)
{
    // ...

Here in the main function you could refer to the delegate functions getLocation and getSize and specify whether you're interested in the primary or secondary axis, e.g.:

    int nTolerance = (int)Math.Ceiling(getSize(rectangles[bi], false) * BOX_LINEUP_TOLERANCE);
    Rectangle[] baSimiliarMagnitude = rectangles.Where(
        b => Math.Abs(getLocation(b, false) - getSize(rectangles[bi], false)) <= nTolerance
                && getLocation(b, true) > getLocation(rectangles[bi], true))
        .ToArray();

Then call with horizontal/vertical implementations of the delegates that pick out the relevant properties:

private List<Rectangle[]> GroupRectanglesHorizontally(Rectangle[] rectangles, out int averageGap)
{
    return GroupRectanglesBy(
        rectangles,
        (rect, primaryAxis) => primaryAxis ? rect.Location.X : rect.Location.Y,
        (rect, primaryAxis) => primaryAxis ? rect.Width : rect.Height,
        out averageGap);
}

private List<Rectangle[]> GroupRectanglesVertically(Rectangle[] rectangles, out int averageGap)
{
    return GroupRectanglesBy(
        rectangles,
        (rect, primaryAxis) => primaryAxis ? rect.Location.Y : rect.Location.X,
        (rect, primaryAxis) => primaryAxis ? rect.Height : rect.Width,
        out averageGap);
}

To keep your code clean, you could define delegates:

delegate int AxisLocationDelegate(Rectangle r, bool primaryAxis);
delegate int AxisSizeDelegate(Rectangle r, bool primaryAxis);

Then define a unified GroupRectanglesBy function that takes an implementation of each of these delegates as parameters:

private List<Rectangle[]> GroupRectanglesBy(Rectangle[] rectangles,
    AxisLocationDelegate getLocation, AxisSizeDelegate getSize, out int averageGap)
{
    // ...

Here in the main function you could refer to the delegate functions getLocation and getSize and specify whether you're interested in the primary or secondary axis, e.g.:

    int nTolerance = (int)Math.Ceiling(getSize(rectangles[bi], false) * BOX_LINEUP_TOLERANCE);
    Rectangle[] baSimiliarMagnitude = rectangles.Where(
        b => Math.Abs(getLocation(b, false) - getSize(rectangles[bi], false)) <= nTolerance
                && getLocation(b, true) > getLocation(rectangles[bi], true))
        .ToArray();

Then call with horizontal/vertical implementations of the delegates that pick out the relevant properties:

private List<Rectangle[]> GroupRectanglesHorizontally(Rectangle[] rectangles, out int averageGap)
{
    return GroupRectanglesBy(
        rectangles,
        (rect, primaryAxis) => primaryAxis ? rect.Location.X : rect.Location.Y,
        (rect, primaryAxis) => primaryAxis ? rect.Width : rect.Height,
        out averageGap);
}

private List<Rectangle[]> GroupRectanglesVertically(Rectangle[] rectangles, out int averageGap)
{
    return GroupRectanglesBy(
        rectangles,
        (rect, primaryAxis) => primaryAxis ? rect.Location.Y : rect.Location.X,
        (rect, primaryAxis) => primaryAxis ? rect.Height : rect.Width,
        out averageGap);
}

Edit: if you were feeling particularly evil and wanted to use the boolean XOR operator ^, you could even reduce this to one function:

private List<Rectangle[]> GroupRectangles(Rectangle[] rectangles, bool vertically, out int averageGap)
{
    return GroupRectanglesBy(
        rectangles,
        (rect, primaryAxis) => primaryAxis ^ vertically ? rect.Location.X : rect.Location.Y,
        (rect, primaryAxis) => primaryAxis ^ vertically ? rect.Width : rect.Height,
        out averageGap);
}