Skip to main content
1 of 8
misha
  • 297
  • 4
  • 14

Find and report all occurrences of duplicate lines in a text file

################################################################################
#
# Find and report all occurrences of duplicate lines in a text file.
#
#
# Usage: awk -f uniq.awk [FILE]
#
################################################################################
{
    x = lines[$0]["count"]++;
    lines[$0]["NR"][x] = NR;

    # Find the length of the longest line to make it the column width
    if (x > 0) {
        if (length($0) > max) {
            max = length($0);
        }
    }
}
END {
    # If the file contains no lines to process, that is, it's empty,
    # return an exit status code of 1 to indicate the fact.
    if (!(NR > 0)) {
        exit 1;
    }

    # Prepare the format string
    fmt_s = "%s: %" max "-s (%s)\n";

    for (i in lines) {
        if (lines[i]["count"] > 1) {
            for (j = 0; j < lines[i]["count"]; j++) {
                s = s lines[i]["NR"][j] ", ";
            }
            # Get rid of the trailing comma and space
            s = substr(s, 1, length(s) - 2);
            printf(fmt_s, lines[i]["count"], i, s);
            s = "";
        }
    }
}

Test:

$ cat -n data
     1  car
     2  baby
     3  car
     4  man
     5  woman
     6  woman
     7  key
     8  woman
$ 
$ 
$ awk -f uniq.awk data
2: car   (1, 3)
3: woman (5, 6, 8)
misha
  • 297
  • 4
  • 14