The file I'm processing here is an outputted file from a Java library called duke.
Its goal was go through all purchases made on a site and group them into a single customer. The output is a csv file in the format, you'll find an example of it below the code.
I want to know if I've written this in a readable way or if there are any problems with the way I've written it.
#!/usr/bin/env ruby
require 'thor'
require 'csv'
require 'json'
# This is the class we use to extend and use thor
class DukeFileProcessor < Thor
desc 'dedupe <FILE>', 'Will process a file generated by Duke.'
# Thor dedupe command
#
# @param csv_file_path [String]
#
# @return File
def dedupe(csv_file_path)
unmerged_ids = parse_csv(csv_file_path)
merged_ids = merge_ids(unmerged_ids)
File.open(csv_file_path + '.matches.json', 'w').write(merged_ids.to_json)
end
private
# Merges the unmerged ids so they are all consolidated under a single
# group that we can call a customer
#
# Process:
# - Iterate through all unmerged_ids
# - Assign the current value of what's being iterated over to matches
# - Iterate through the current value to find matches in unmerged_ids
# and then merge matches into the current matches
# - Finally add the current key and matches to the merged_ids variable
#
# @param unmerged_ids [Array]
#
# @return File
def merge_ids(unmerged_ids)
merged_ids = {}
unmerged_ids.each_pair do |key, value|
value.each do |match_id|
next if unmerged_ids[match_id].nil?
unmerged_ids[match_id].each do |policy_id|
value.push(policy_id) unless value.include?(policy_id)
end
end
merged_ids[key] = value - [key]
end
end
# Parses the given file and does a simple merge on ids to create
# an array of all secondary_ids that match the primary_id
#
# CSV Format: (is_match?(-/+),target,match,probability)
#
# @param csv_file_path [String]
#
# @return Hash
def parse_csv(csv_file_path)
unmerged_ids = {}
CSV.foreach(csv_file_path) do |row|
primary_id = row[1].to_i
secondary_id = row[2].to_i
unmerged_ids[primary_id] = [] if unmerged_ids[primary_id].nil?
unmerged_ids[primary_id].push(secondary_id)
end
unmerged_ids
end
end
DukeFileProcessor.start
Here's the first 500 lines from the file I'm using. The format of which is:
is_match?(-/+),target,match,probability+,6,7,0.9997260166738423 +,3,4,0.9997260166738423 +,8,9,0.9997260166738423 +,9,8,0.9997260166738423 +,7,6,0.9997260166738423 +,6,10,0.9997260166738423 +,4,3,0.9997260166738423 +,10,6,0.9997260166738423 +,8,296,0.9986244841815681 +,6,39,0.9983391066412223 +,8,299,0.9986244841815681 +,7,10,0.9997260166738423 +,6,40,0.9983391066412223 +,9,296,0.9986244841815681 +,8,1101,0.9986244841815681 +,10,7,0.9997260166738423 +,6,1081,0.9983391066412223 +,6,1083,0.9983391066412223 +,10,39,0.9983391066412223 +,10,40,0.9983391066412223 +,10,1081,0.9983391066412223 +,10,1083,0.9983391066412223 +,8,1125,0.9997260166738423 +,8,1128,0.9997260166738423 +,9,299,0.9986244841815681 +,8,1132,0.9997260166738423 +,9,1101,0.9986244841815681 +,9,1125,0.9997260166738423 +,9,1128,0.9997260166738423 +,7,39,0.9983391066412223 +,8,1144,0.9986244841815681 +,8,1149,0.9986244841815681 +,7,40,0.9983391066412223 +,9,1132,0.9997260166738423 +,7,1081,0.9983391066412223 +,7,1083,0.9983391066412223 +,9,1144,0.9986244841815681 +,9,1149,0.9986244841815681 +,12,24781,0.9997260166738423 +,11,16,0.9999872532595235 +,17,15,0.9997260166738423 +,16,11,0.9999872532595235 +,7,36,0.9977532413823246 +,6,36,0.9977532413823246 +,10,36,0.9977532413823246 +,15,17,0.9997260166738423 +,18,560,0.99892382632337 +,59,56,0.9997260166738423 +,37,36,0.9997260166738423 +,37,1333,0.9997260166738423 +,37,1341,0.9997260166738423 +,37,12479,0.9997260166738423 +,37,19462,0.9997260166738423 +,37,19466,0.9997260166738423 +,70,64,0.9997260166738423 +,70,106,0.9997260166738423 +,27,8200,0.9999217037269025 +,28,397,0.9981390956560382 +,27,8229,0.9999217037269025 +,49,145,0.9991644138608996 +,49,19596,0.9998336350736409 +,49,250925,0.9991644138608996 +,64,70,0.9997260166738423 +,64,106,0.9997260166738423 +,26,22,0.9999217037269025 +,27,66061,0.9990737249986892 +,27,69613,0.9990737249986892 +,27,69617,0.9990737249986892 +,27,70011,0.9990737249986892 +,23,613,0.9999217037269025 +,27,70885,0.999849094020817 +,23,1186,0.9999217037269025 +,77,87,0.9997260166738423 +,23,1274,0.9999217037269025 +,22,26,0.9999217037269025 +,23,7603,0.9999217037269025 +,27,70946,0.9990737249986892 +,23,7759,0.9996066089693157 +,23,7766,0.9996066089693157 +,95,100,0.9997260166738423 +,95,12510,0.9997260166738423 +,23,12437,0.9996066089693157 +,23,12455,0.9996066089693157 +,23,32083,0.9999217037269025 +,39,40,0.9997260166738423 +,39,1081,0.9997260166738423 +,40,39,0.9997260166738423 +,39,1083,0.9997260166738423 +,40,1081,0.9997260166738423 +,40,1083,0.9997260166738423 +,39,36,0.9996291897994778 +,39,37,0.9996291897994778 +,40,36,0.9996291897994778 +,39,1333,0.9996291897994778 +,40,37,0.9996291897994778 +,39,1341,0.9996291897994778 +,40,1333,0.9996291897994778 +,39,1352,0.9997851370603088 +,114,108,0.9997260166738423 +,40,1341,0.9996291897994778 +,40,1352,0.9997851370603088 +,46,48,0.9997260166738423 +,56,59,0.9997260166738423 +,126,123,0.9997260166738423 +,126,131,0.9997260166738423 +,126,132,0.9997260166738423 +,88,65,0.9997260166738423 +,87,77,0.9997260166738423 +,36,37,0.9997260166738423 +,36,1333,0.9997260166738423 +,36,1341,0.9997260166738423 +,150,148,0.9997260166738423 +,150,155,0.9997260166738423 +,36,12479,0.9997260166738423 +,36,19462,0.9997260166738423 +,36,19466,0.9997260166738423 +,162,153,0.9997260166738423 +,72,61,0.9999217037269025 +,72,8079,0.9999217037269025 +,65,88,0.9997260166738423 +,48,46,0.9997260166738423 +,100,95,0.9997260166738423 +,100,12510,0.9997260166738423 +,84,190,0.9997260166738423 +,189,223,0.9997878432419376 +,108,114,0.9997260166738423 +,104,24385,0.9997260166738423 +,104,24389,0.9997260166738423 +,90,89,0.9998336350736409 +,97,10059,0.9999478011222758 +,89,90,0.9998336350736409 +,61,72,0.9999217037269025 +,61,8079,0.9999217037269025 +,131,123,0.9997260166738423 +,103,110,0.9997260166738423 +,74,19142,0.9999217037269025 +,131,126,0.9997260166738423 +,131,132,0.9997260166738423 +,208,18729,0.998739204342771 +,148,150,0.9997260166738423 +,148,155,0.9997260166738423 +,106,64,0.9997260166738423 +,106,70,0.9997260166738423 +,161,159,0.9997260166738423 +,123,126,0.9997260166738423 +,123,131,0.9997260166738423 +,123,132,0.9997260166738423 +,169,166,0.9997260166738423 +,110,103,0.9997260166738423 +,139,164,0.9998336350736409 +,139,4468,0.9998336350736409 +,240,248,0.9997260166738423 +,153,162,0.9997260166738423 +,132,123,0.9997260166738423 +,132,126,0.9997260166738423 +,132,131,0.9997260166738423 +,254,1263,0.9999217037269025 +,107,6,0.9878882293158856 +,107,7,0.9878882293158856 +,107,10,0.9878882293158856 +,181,177,0.9997260166738423 +,145,49,0.9991644138608996 +,181,191,0.9997260166738423 +,156,146,0.9997260166738423 +,181,236,0.9986244841815681 +,156,230,0.9997260166738423 +,107,39,0.9878882293158856 +,181,657,0.9986244841815681 +,145,19596,0.9991644138608996 +,155,148,0.9997260166738423 +,107,40,0.9878882293158856 +,181,675,0.9986244841815681 +,145,250925,0.9998336350736409 +,181,912,0.9997260166738423 +,155,150,0.9997260166738423 +,107,1081,0.9878882293158856 +,124,217,0.9997260166738423 +,156,278,0.9997260166738423 +,190,84,0.9997260166738423 +,107,1083,0.9878882293158856 +,166,169,0.9997260166738423 +,107,36,0.9836761877555505 +,159,161,0.9997260166738423 +,199,219,0.9997260166738423 +,167,174,0.9997260166738423 +,193,2934,0.9997260166738423 +,193,2998,0.9997260166738423 +,193,3366,0.9997260166738423 +,204,214,0.9997260166738423 +,177,181,0.9997260166738423 +,177,191,0.9997260166738423 +,278,146,0.9997260166738423 +,207,213,0.9999217037269025 +,214,204,0.9997260166738423 +,278,156,0.9997260166738423 +,177,236,0.9986244841815681 +,278,230,0.9997260166738423 +,223,189,0.9997878432419376 +,146,156,0.9997260166738423 +,146,230,0.9997260166738423 +,146,278,0.9997260166738423 +,209,3572,0.9997260166738423 +,236,177,0.9986244841815681 +,236,181,0.9986244841815681 +,236,191,0.9986244841815681 +,236,657,0.9997260166738423 +,236,675,0.9997260166738423 +,177,657,0.9986244841815681 +,207,21998,0.9999217037269025 +,236,912,0.9986244841815681 +,177,675,0.9986244841815681 +,177,912,0.9997260166738423 +,298,300,0.9997260166738423 +,245,244,0.9997260166738423 +,298,307,0.9997260166738423 +,213,207,0.9999217037269025 +,213,21998,0.9999217037269025 +,306,304,0.9997260166738423 +,164,139,0.9998336350736409 +,164,4468,0.9998336350736409 +,217,124,0.9997260166738423 +,160,7682,0.9998890838982443 +,160,7725,0.9998890838982443 +,160,7735,0.9998890838982443 +,160,66236,0.9998890838982443 +,160,69756,0.9998890838982443 +,174,167,0.9997260166738423 +,244,245,0.9997260166738423 +,219,199,0.9997260166738423 +,333,1259,0.9997260166738423 +,182,1530,0.9999729135649095 +,260,802,0.9986244841815681 +,226,216,0.9998336350736409 +,226,1089,0.9998336350736409 +,226,1090,0.9998336350736409 +,226,1092,0.9998336350736409 +,226,1093,0.9998336350736409 +,226,1094,0.9998336350736409 +,226,1095,0.9998336350736409 +,226,1096,0.9998336350736409 +,196,200,0.9997260166738423 +,272,285,0.9997260166738423 +,272,290,0.9997260166738423 +,191,177,0.9997260166738423 +,191,181,0.9997260166738423 +,191,236,0.9986244841815681 +,191,657,0.9986244841815681 +,191,675,0.9986244841815681 +,191,912,0.9997260166738423 +,206,405,0.9997260166738423 +,343,352,0.9997260166738423 +,200,196,0.9997260166738423 +,352,343,0.9997260166738423 +,216,226,0.9998336350736409 +,230,146,0.9997260166738423 +,216,1089,0.9998336350736409 +,230,156,0.9997260166738423 +,230,278,0.9997260166738423 +,361,350,0.9997260166738423 +,361,360,0.9997260166738423 +,310,313,0.9997260166738423 +,216,1090,0.9998336350736409 +,216,1092,0.9998336350736409 +,241,238,0.9997260166738423 +,216,1093,0.9998336350736409 +,216,1094,0.9998336350736409 +,216,1095,0.9998336350736409 +,216,1096,0.9998336350736409 +,319,344,0.9997260166738423 +,249,263,0.9997260166738423 +,328,322,0.9997260166738423 +,328,335,0.9997260166738423 +,328,1299,0.9997260166738423 +,328,9497,0.9997260166738423 +,225,215,0.9998336350736409 +,238,241,0.9997260166738423 +,220,237,0.9999478011222758 +,248,240,0.9997260166738423 +,347,345,0.9997260166738423 +,357,353,0.9997260166738423 +,309,369,0.9997260166738423 +,215,225,0.9998336350736409 +,367,368,0.9997260166738423 +,384,373,0.9997260166738423 +,269,291,0.9997260166738423 +,269,294,0.9997260166738423 +,269,1272,0.9997260166738423 +,269,12493,0.9997260166738423 +,269,12494,0.9997260166738423 +,269,12495,0.9997260166738423 +,269,12496,0.9997260166738423 +,269,12498,0.9997260166738423 +,269,12499,0.9997260166738423 +,403,12841,0.9997260166738423 +,390,433,0.9997260166738423 +,403,12876,0.9997260166738423 +,285,272,0.9997260166738423 +,285,290,0.9997260166738423 +,296,8,0.9986244841815681 +,296,9,0.9986244841815681 +,296,299,0.9997260166738423 +,296,1101,0.9997260166738423 +,296,1125,0.9986244841815681 +,296,1128,0.9986244841815681 +,296,1132,0.9986244841815681 +,291,269,0.9997260166738423 +,296,1144,0.9997260166738423 +,291,294,0.9997260166738423 +,291,1272,0.9997260166738423 +,291,12493,0.9997260166738423 +,291,12494,0.9997260166738423 +,291,12495,0.9997260166738423 +,291,12496,0.9997260166738423 +,291,12498,0.9997260166738423 +,291,12499,0.9997260166738423 +,296,1149,0.9997260166738423 +,237,220,0.9999478011222758 +,417,8102,0.9998336350736409 +,304,306,0.9997260166738423 +,300,298,0.9997260166738423 +,290,272,0.9997260166738423 +,449,450,0.9997260166738423 +,300,307,0.9997260166738423 +,449,459,0.999071786152359 +,290,285,0.9997260166738423 +,426,382,0.9999217037269025 +,426,416,0.9999217037269025 +,426,480,0.9999217037269025 +,426,482,0.9999217037269025 +,426,485,0.9999217037269025 +,426,486,0.9999217037269025 +,426,489,0.9999217037269025 +,426,490,0.9999217037269025 +,426,492,0.9999217037269025 +,263,249,0.9997260166738423 +,436,400,0.9986244841815681 +,445,600,0.9999217037269025 +,454,439,0.9997260166738423 +,454,442,0.9997260166738423 +,458,452,0.9997260166738423 +,374,387,0.9997260166738423 +,299,8,0.9986244841815681 +,299,9,0.9986244841815681 +,299,296,0.9997260166738423 +,299,1101,0.9997260166738423 +,274,284,0.9997260166738423 +,299,1125,0.9986244841815681 +,299,1128,0.9986244841815681 +,299,1132,0.9986244841815681 +,299,1144,0.9997260166738423 +,299,1149,0.9997260166738423 +,307,298,0.9997260166738423 +,307,300,0.9997260166738423 +,322,328,0.9997260166738423 +,322,335,0.9997260166738423 +,322,1299,0.9997260166738423 +,322,9497,0.9997260166738423 +,476,470,0.9997260166738423 +,476,471,0.9997260166738423 +,284,274,0.9997260166738423 +,313,310,0.9997260166738423 +,485,382,0.9999217037269025 +,485,416,0.9999217037269025 +,485,426,0.9999217037269025 +,485,480,0.9999217037269025 +,485,482,0.9999217037269025 +,485,486,0.9999217037269025 +,485,489,0.9999217037269025 +,485,490,0.9999217037269025 +,485,492,0.9999217037269025 +,330,334,0.9997260166738423 +,387,374,0.9997260166738423 +,316,320,0.9999729135649095 +,480,382,0.9999217037269025 +,350,360,0.9997260166738423 +,493,382,0.9999217037269025 +,350,361,0.9997260166738423 +,480,416,0.9999217037269025 +,493,416,0.9999217037269025 +,397,28,0.9981390956560382 +,480,426,0.9999217037269025 +,480,482,0.9999217037269025 +,480,485,0.9999217037269025 +,294,269,0.9997260166738423 +,480,486,0.9999217037269025 +,294,291,0.9997260166738423 +,480,489,0.9999217037269025 +,359,1064,0.9998890838982443 +,294,1272,0.9997260166738423 +,480,490,0.9999217037269025 +,294,12493,0.9997260166738423 +,480,492,0.9999217037269025 +,294,12494,0.9997260166738423 +,294,12495,0.9997260166738423 +,369,309,0.9997260166738423 +,294,12496,0.9997260166738423 +,294,12498,0.9997260166738423 +,294,12499,0.9997260166738423 +,334,330,0.9997260166738423 +,380,391,0.9997260166738423 +,380,404,0.9986244841815681 +,380,410,0.9986244841815681 +,380,411,0.9986244841815681 +,351,9570,0.9997260166738423 +,351,18765,0.9997260166738423 +,351,18820,0.9997260166738423 +,351,64976,0.9997260166738423 +,489,382,0.9999217037269025 +,489,416,0.9999217037269025 +,416,382,0.9999217037269025 +,489,426,0.9999217037269025 +,344,319,0.9997260166738423 +,416,426,0.9999217037269025 +,489,480,0.9999217037269025 +,416,480,0.9999217037269025 +,489,482,0.9999217037269025 +,416,482,0.9999217037269025 +,489,485,0.9999217037269025 +,416,485,0.9999217037269025 +,489,486,0.9999217037269025 +,416,486,0.9999217037269025 +,489,490,0.9999217037269025 +,416,489,0.9999217037269025 +,416,490,0.9999217037269025 +,489,492,0.9999217037269025 +,416,492,0.9999217037269025 +,410,380,0.9986244841815681 +,353,357,0.9997260166738423 +,410,391,0.9986244841815681 +,360,350,0.9997260166738423 +,410,404,0.9997260166738423 +,410,411,0.9997260166738423 +,497,504,0.9997260166738423 +,419,427,0.9981390956560382 +,438,453,0.9997260166738423 +,438,435,0.9997363863461571 +,360,361,0.9997260166738423 +,326,348,0.9997260166738423 +,435,438,0.9997363863461571 +,524,533,0.9997260166738423 +,493,426,0.9999217037269025 +,435,453,0.9997363863461571 +,524,924,0.9997260166738423 +,493,480,0.9999217037269025 +,326,335704,0.9997260166738423 +,524,5542,0.9997512856021227 +,493,482,0.9999217037269025 +,326,335732,0.9997260166738423 +,524,5612,0.9997512856021227 +,493,485,0.9999217037269025 +,493,486,0.9999217037269025 +,524,5652,0.9997512856021227 +,493,489,0.9999217037269025 +,524,5691,0.9997512856021227 +,493,490,0.9999217037269025 +,493,492,0.9999217037269025 +,524,7343,0.9997512856021227 +,524,7345,0.9997512856021227 +,524,7346,0.9997512856021227 +,456,466,0.9999217037269025 +,466,456,0.9999217037269025 +,335,322,0.9997260166738423 +,509,508,0.9997260166738423 +,335,328,0.9997260166738423 +,335,1299,0.9997260166738423 +,335,9497,0.9997260166738423 +,453,438,0.9997260166738423 +,483,496,0.9997260166738423 +,483,521,0.9997260166738423 +,483,815,0.9997260166738423 +,320,316,0.9999729135649095 +,453,435,0.9997363863461571 +,382,416,0.9999217037269025 +,382,426,0.9999217037269025 +,382,480,0.9999217037269025 +,382,482,0.9999217037269025 +,382,485,0.9999217037269025 +,382,486,0.9999217037269025 +,382,489,0.9999217037269025 +,382,490,0.9999217037269025 +,544,539,0.9997260166738423 +,382,492,0.9999217037269025 +,544,541,0.9997260166738423 +,345,347,0.9997260166738423 +,553,551,0.9997260166738423 +,471,470,0.9997260166738423 +,471,476,0.9997260166738423 +,561,76267,0.9997260166738423 +,404,380,0.9986244841815681 +,404,391,0.9986244841815681 +,348,326,0.9997260166738423 +,404,410,0.9997260166738423 +,348,335704,0.9997260166738423 +,404,411,0.9997260166738423 +,348,335732,0.9997260166738423 +,565,566,0.9997260166738423 +,534,537,0.9997260166738423 +,534,538,0.9997260166738423 +,542,855,0.9997260166738423 +,542,349839,0.9997260166738423
