2
\$\begingroup\$

Previously, I built a scraper to get ranking data for ATP # 1 over the years. See Webscraping tennis data, and follow ups.

Now, I use the data to rank the players on the basis of how many weeks they've spent at #1.

I'd appreciate general feedback on code style and structure. The Scraper class is unchanged, so really everything else is up for review.


Code

Note: if you run the code below, it'll create a directory, and a bunch of files in it.

scraper package

package scraper;

// A POJO that encapsulates a ranking week and the name of the corresponding No.1 player
public class WeeklyResult {
    private final String week;
    private final String playerName;

    public WeeklyResult(final String week, final String playerName) {
        this.week = week;
        this.playerName = playerName;
    }
    public String getWeek() {
        return week;
    }
    public String getPlayerName() {
        return playerName;
    }
}

Scraper.java

package scraper;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.time.Duration;
import java.time.LocalDate;
import java.util.*;
import java.util.stream.Collectors;

public class Scraper {
    private static final Logger logger = LogManager.getLogger(Scraper.class);
    private final String urlPrefix;
    private final String urlSuffix;
    private final Duration timeout;
    private final int totalTries;
    private WeeklyResult latestResult;

    public Scraper(final String urlPrefix, final String urlSuffix, final Duration timeout, final int totalTries) {
        this.urlPrefix = urlPrefix;
        this.urlSuffix = urlSuffix;
        this.timeout = timeout;
        this.totalTries = totalTries;
        this.latestResult = new WeeklyResult("1973-08-16", "N/A");
    }

    public WeeklyResult scrape(final String week) throws ScraperException {
        // in the case the latest scraped data returns an "empty" weekly result, simply retain the latest No.1
        // since it is likely he wouldn't have changed. A weekly result is deemed empty if no player or week info
        // can be found on the ATP page.
        this.latestResult = scrapeWeekly(week)
                .orElse(new WeeklyResult(updateLatestWeekByOne(), this.latestResult.getPlayerName()));
        return this.latestResult;
    }

    private Optional<WeeklyResult> scrapeWeekly(final String week) throws ScraperException {
        final Document document = loadDocument(weeklyResultUrl(week));
        final boolean numberOneDataExists = selectNumberOneRankCell(document).isPresent();
        final Element playerCell = numberOneDataExists ? selectPlayerCellElement(document) : null;

        return Optional.ofNullable(playerCell)
                .map(element -> new WeeklyResult(week, element.text()));
    }

    public List<String> loadWeeks() throws ScraperException {
        final Document document = loadDocument(urlPrefix);
        final Elements elements = selectRankingWeeksElements(document);
        final List<String> weeks = extractWeeks(elements);

        return noEmptyElseThrow(weeks);
    }

    private Document loadDocument(final String url) throws ScraperException {
        for (int tries = 0; tries < this.totalTries; tries++) {
            try {
                return Jsoup.connect(url).timeout((int) timeout.toMillis()).get();
            } catch (IOException e) {
                if (tries == this.totalTries) {
                    throw new ScraperException("Error loading ATP website: ", e);
                }
            }
        }
        return null;
    }

    private static Elements selectRankingWeeksElements(final Document document) {
        // extract ranking weeks from the dropdown menu
        final Elements result = document.getElementsByAttributeValue("data-value", "rankDate")
                .select("ul li");

        Collections.reverse(result);
        return result;
    }

    private static List<String> extractWeeks(final Collection<Element> elements) {
        // refer to https://winterbe.com/posts/2014/07/31/java8-stream-tutorial-examples/
        // and https://www.baeldung.com/java-maps-streams.
        return elements.stream()
                .map(Scraper::extractWeek)
                .collect(Collectors.toList());
    }

    private static List<String> noEmptyElseThrow(final List<String> weeks) throws ScraperException {
        if (weeks.isEmpty()) {
            throw new ScraperException("Cannot process empty data from the weeks calendar!");
        } else {
            return weeks;
        }
    }

    private String weeklyResultUrl(final String week) {
        return urlPrefix + "rankDate=" + week + urlSuffix;
    }

    private static Optional<Element> selectNumberOneRankCell(final Document document) {
        final Element rankCell = selectPlayerRankCell(document);
        return Optional.ofNullable(rankCell).filter(element -> numberOneRankCellExists(element));
    }

    private static Element selectPlayerCellElement(final Document document) {
        return document.getElementsByClass("player-cell").first();
    }

    private static boolean numberOneRankCellExists(final Element rankCell) {
        return rankCell.text().equals("1");
    }

    private static Element selectPlayerRankCell(final Document document) {
        return document.getElementsByClass("rank-cell").first();
    }

    private static String extractWeek(final Element li) {
        return li.text().replaceAll("\\.", "-");
    }

    private String updateLatestWeekByOne() {
        return LocalDate.parse(this.latestResult.getWeek()).plusWeeks(1).toString();
    }
}

ScraperException.java

package scraper;

public class ScraperException extends Exception {
    final String message;
    public ScraperException (String message) {
        this.message = message;
    }
    public ScraperException (String message, Throwable cause) {
        super(cause);
        this.message = message;
    }
    @Override
    public String toString() {
        return this.message;
    }
}

rankallocator package

RankAllocator.java

package rankallocator;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import scraper.WeeklyResult;

import java.util.*;
import java.util.stream.Collectors;

// This class combines all the collective actions of parsing a weekly result
// and generating a weekly ranking of the No1 players for a given week.
public class RankAllocator {
    private static final Logger logger = LogManager.getLogger(RankAllocator.class);
    private final PlayerScoring trackScores;
    private final int top_N; // display the top-N players for Weekly rankings.

    public RankAllocator(final int N) {
        this.top_N = N;
        this.trackScores = new PlayerScoring();
    }
    public PlayerScoring getTrackScores() { return this.trackScores; }

    private void updatePlayerScore(final WeeklyResult weeklyResult) {
        // given a weekly result, update the trackScores hashmap
        this.trackScores.updatePlayerValue(weeklyResult.getWeek(), weeklyResult.getPlayerName());
    }

    private List<PlayerRank> collatePlayerScores() {
        // for each entry in trackScores, construct a new PlayerRank
        // then return a list sorted in descending order by PlayerValue
        List<PlayerRank> ranks =
                (trackScores.getPlayerScorer().entrySet().stream()
                        .map(entry -> new PlayerRank(entry.getKey(), entry.getValue())))
                        .sorted(Comparator.comparingInt((PlayerRank player) ->
                                player.getCurrentValue().getWeeksAtNumberOne()).reversed() //get descending order by weeks @ No.1
                                .thenComparing(player -> player.getCurrentValue().getFirstReached())) //break ties based on who reached earlier
                        .collect(Collectors.toList());
        return ranks;
    }

    public WeeklyRanking rank (final WeeklyResult weeklyResult) {
        Configurator.setLevel(logger.getName(), Level.ERROR);
        updatePlayerScore(weeklyResult);
        logger.debug(this.trackScores.getPlayerScorer());
        List<PlayerRank> weeklyScores = collatePlayerScores();
        return new WeeklyRanking(weeklyResult.getWeek(), weeklyScores, this.top_N);
    }
}

PlayerValue.java

package rankallocator;

// POJO class to hold the "score" of a player,
// which consists of the weeks they've spent at No.1,
// and the week they first attained No.1 (to break ties deterministically)

public class PlayerValue {
    private final String firstReached;
    private int weeksAtNumberOne;

    public PlayerValue(final String week) {
        this.firstReached = week;
        this.weeksAtNumberOne = 1;
    }

    public int getWeeksAtNumberOne() {
        return weeksAtNumberOne;
    }

    public String getFirstReached() {
        return firstReached;
    }

    public void updateWeeksAtNumberOne() {
        this.weeksAtNumberOne++;
    }
}

PlayerScoring.java

package rankallocator;

// A wrapper class that uses a hashmap and additional metadata to maintain the
// score of each No.1 player. The score is maintained using the PlayerValue abstraction
import java.util.HashMap;
import java.util.Map;

public class PlayerScoring {
    private Map<String, PlayerValue> playerScorer;

    public PlayerScoring() {
        playerScorer = new HashMap<>();
    }

    public Map<String, PlayerValue> getPlayerScorer() {
        return playerScorer;
    }

    public PlayerValue findPlayerValue(final String player) {
        return this.playerScorer.get(player);
    }

    public void updatePlayerValue (final String week, final String player) {
        // if the player is a new No.1, construct a new entry
        // otherwise, update (by one) his week tally
        if (!this.playerScorer.containsKey(player)) {
            this.playerScorer.put(player, new PlayerValue(week));
        } else {
            // Test to make sure values are updated properly
            this.playerScorer.get(player).updateWeeksAtNumberOne();
        }
    }
}

PlayerRank.java

package rankallocator;

//POJO coupling playerName and PlayerValue passed in to a WeeklyRanking's PriorityQueue.
public class PlayerRank {
    private final String playerName;
    private final PlayerValue currentValue;

    public PlayerRank(final String name, final PlayerValue value) {
        this.playerName = name;
        this.currentValue = value;
    }

    public String getPlayerName() { return playerName; }
    public PlayerValue getCurrentValue() { return currentValue; }
}

WeeklyRanking.java

package rankallocator;

// Given a specific week, this class builds a
// list of the top ten (or the size of the queue, if lesser)
// players for that week

import java.util.ArrayList;
import java.util.List;

public class WeeklyRanking {
    private final String week;
    private final List<PlayerRank> ranks;
    public WeeklyRanking(final String week, final List<PlayerRank> playerRanks, final int N) {
        this.week = week;
        // construct a list of the top N (max) players based on weeks at No.1
        this.ranks = new ArrayList<>();
        for (PlayerRank rank : playerRanks) {
            this.ranks.add(rank);
            if (this.ranks.size() == N) {
                break;
            }
        }
    }
    
    public String getWeek() { return week; }
    public List<PlayerRank> getRanks() { return ranks; }
}

legacyvisualizer package

LegacyVisualizer.java

package legacyvisualizer;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import rankallocator.PlayerRank;
import rankallocator.WeeklyRanking;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

// A class to write out a list of the "GOAT" status for each week.
public class LegacyVisualizer {
    private final String dirName;
    private static final Logger logger = LogManager.getLogger(LegacyVisualizer.class);
    
    public LegacyVisualizer(final String path) {
        this.dirName = path;
        File dir = new File(dirName);
        if (!dir.exists()) {
            dir.mkdir();
        }
    }

    private void createListFile(WeeklyRanking weeklyRanking) throws IOException {
        List<String> lines = generateLegacyList(weeklyRanking);
        Path file = Paths.get(dirName + "/" + weeklyRanking.getWeek()+".txt");
        Files.write(file, lines, StandardCharsets.UTF_8);
    }

    private static List<String> generateLegacyList (WeeklyRanking weeklyRanking) {
        final List<PlayerRank> ranks = weeklyRanking.getRanks();

        List<String> lines = new ArrayList<>();
        lines.add("Rank\t\tPlayer\t\tWeeks At #1\t\tFirst Reached On");
        for (int i = 0; i < ranks.size(); i++) {
            final String player = ranks.get(i).getPlayerName();
            final int weeksTally = ranks.get(i).getCurrentValue().getWeeksAtNumberOne();
            final String firstReached = ranks.get(i).getCurrentValue().getFirstReached();
            final String line = Integer.toString(i+1) + "." + "\t\t" + player + "\t\t" + Integer.toString(weeksTally) + "\t\t" + firstReached;
            lines.add(line);
        }
        return lines;
    }

    public void visualize(final WeeklyRanking weeklyRanking) {
        // create a file for that week. No need to check if that file
        // for the week exists - if we are retrying, it's probably due to a valid
        // reason (timeout, re-running the visualization etc) and the data we get
        // is at least the same or newer than the previous try,
        // so overwriting the exist file would be fine.
        Configurator.setLevel(logger.getName(), Level.DEBUG);
        try {
            createListFile(weeklyRanking);
        } catch (IOException e) {
            System.out.println("Failed to create file. " + e);
        }
    }
}

Main package

package project;

import legacyvisualizer.LegacyVisualizer;
import rankallocator.RankAllocator;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import rankallocator.WeeklyRanking;
import scraper.Scraper;
import scraper.ScraperException;
import scraper.WeeklyResult;

import java.time.Duration;
import java.util.List;

// Main class to manage the visualization of player's legacy rankings
public class Project {
    private static final Logger logger = LogManager.getRootLogger();
    private static final RankAllocator allocator = new RankAllocator(5);
    private static final String dirName = System.getProperty("user.dir")+"/legacy";
    private static final LegacyVisualizer visualizer = new LegacyVisualizer(dirName);

    private static void generateRankingAllocation(WeeklyResult weeklyResult) {
        // pass the scraped result to the next stage of the visualization logic.
        logger.debug("Week: " + weeklyResult.getWeek() + " No.1: " + weeklyResult.getPlayerName());
        WeeklyRanking weeklyRanking = allocator.rank(weeklyResult);
        visualizer.visualize(weeklyRanking);
    }

    public static void main(String[] args) {

        Configurator.setRootLevel(Level.INFO);

        final Scraper scraper =
                new Scraper("https://www.atptour.com/en/rankings/singles?",
                        "&rankRange=0-100", Duration.ofSeconds(90), 3);

        // The flow is as follows: scrape the latest weekly results (starting from 1973),
        // then pass it to the ranking logic (IPR). Rinse and repeat
        try {
            final List<String> weeks = scraper.loadWeeks();
            for (String week : weeks) {
                logger.debug(week);
                WeeklyResult weeklyResult =  scraper.scrape(week);
                generateRankingAllocation(weeklyResult);
            }
        } catch (ScraperException e) {
            logger.error(e.toString());
        }
    }
}


Testing

RankingTests.java

package ranking;

import junit.framework.TestSuite;

public class RankingTests {
    public static TestSuite suite() {
        TestSuite rankTest = new TestSuite("Generation of Accurate Weekly Rankings Suite");
        rankTest.addTestSuite(TrackScoresTest.class);
        rankTest.addTestSuite(WeeklyRankTest.class);
        return rankTest;
    }
}

TrackScoresTest.java

package ranking;

import rankallocator.RankAllocator;
import junit.framework.TestCase;
import org.junit.Test;
import scraper.WeeklyResult;

import java.util.ArrayList;
import java.util.List;

/** Given a series of WeeklyResults, ensure that the tracking/updating of
 *  player's scores (given be a PlayerValue type) is achieved correctly.
 *  This class will test the classes PlayerScoring and PlayerValue for the following:
 *  1) That the hashmap is correctly initializing scores for new No.1s
 *  2) That the hashmap is updating the player values when a No.1 retains his position
 *  3) That the hashmap's PlayerValues can be used to deterministically break a tie
 *  between players with the same tally, by favouring the player who reached No.1 earlier.
 */

public class TrackScoresTest extends TestCase {
    private RankAllocator allocator;
    private List<WeeklyResult> weeklyResults;

    public void setUp() {
        this.allocator = new RankAllocator(5);

        weeklyResults = setupResults();

        for (WeeklyResult weeklyResult : weeklyResults) {
            allocator.rank(weeklyResult);
        }
    }

    private List<WeeklyResult> setupResults() {
        List<WeeklyResult> results = new ArrayList<>();
        results.add(new WeeklyResult("2000-11-06", "Pete Sampras"));
        results.add(new WeeklyResult("2000-11-13", "Pete Sampras"));
        results.add(new WeeklyResult("2000-11-20", "Marat Safin"));
        results.add(new WeeklyResult("2000-11-27", "Marat Safin"));
        results.add(new WeeklyResult("2000-12-04", "Gustavo Kuerten"));
        return results;
    }

    public void tearDown() {
    }
    
    @Test
    public void testTotalTallies() {
        assertEquals(allocator.getTrackScores().findPlayerValue("Pete Sampras").getWeeksAtNumberOne(), 2);
        assertEquals(allocator.getTrackScores().findPlayerValue("Marat Safin").getWeeksAtNumberOne(), 2);
        assertEquals(allocator.getTrackScores().findPlayerValue("Gustavo Kuerten").getWeeksAtNumberOne(), 1);
    }

    @Test
    public void testFirstReached() {
        assertEquals(allocator.getTrackScores().findPlayerValue("Pete Sampras").getFirstReached(), "2000-11-06");
        assertFalse(allocator.getTrackScores().findPlayerValue("Marat Safin").getFirstReached().equals("2000-11-27"));
    }

    @Test
    public void testNewNumberOne() {
        assertEquals(allocator.getTrackScores().findPlayerValue("Gustavo Kuerten").getFirstReached(), "2000-12-04");
        assertEquals(allocator.getTrackScores().findPlayerValue("Gustavo Kuerten").getWeeksAtNumberOne(), 1);
    }
}

WeeklyRankTest.java

package ranking;

// Test class to see that players get properly ranked.
// Note that since we are counting tallies on
// a weekly basis, it is impossible for players to "jump"
// over the other. i.e. if Sam was ranked 3rd by weeks at No.1,
// he has to become 2nd and then become 1st.
// This means that we only need to test for the following cases:

// 1. Players are being ranked *in descending order* by total weeks at No.1
// 2. Ties are broken by using the week first reached
// 3. Weekly Ranking top-N lists are generated correctly when a) len(list) < N and b) >= N.
// 4. When len(list) >= N, changes in the Nth ranked player are reflected correctly.

import junit.framework.TestCase;
import org.junit.Test;
import rankallocator.RankAllocator;
import rankallocator.WeeklyRanking;
import scraper.WeeklyResult;

import java.util.ArrayList;
import java.util.List;

public class WeeklyRankTest extends TestCase {
    private RankAllocator allocator;
    private List<WeeklyResult> weeklyResults;
    private List<WeeklyRanking> weeklyRankings;

    public void setUp() {
        this.allocator = new RankAllocator(3);
        this.weeklyResults = setUpInitialResults();

        this.weeklyRankings = new ArrayList<>();
        for (WeeklyResult weeklyResult : weeklyResults) {
            weeklyRankings.add(allocator.rank(weeklyResult));
        }
    }

    private List<WeeklyResult> setUpInitialResults() {
        List<WeeklyResult> results = new ArrayList<>();
        results.add(new WeeklyResult("1", "Max"));
        results.add(new WeeklyResult("2", "James"));
        results.add(new WeeklyResult("3", "Max"));
        results.add(new WeeklyResult("4", "Max"));
        results.add(new WeeklyResult("5", "James"));
        return results;
    }

    public void tearDown() {
    }

    @Test
    public void testRankingOrderAndLessThanN() {
        // Cases 1, 2 & 3a).
        // We'll show the list as [<week, <name, PlayerValue>>]
        // After setup:
        // check Max is ranked 1 for all weeks,
        for (WeeklyRanking ranking : weeklyRankings) {
            assertEquals(ranking.getRanks().get(0).getPlayerName(), "Max");
        }

        final int numWeeks = weeklyRankings.size();
        final WeeklyRanking fifthWeekRanking = weeklyRankings.get(numWeeks-1);
        // check weeks have been assigned correctly to weekly ranking
        assertEquals(fifthWeekRanking.getWeek(), "5");
        assertEquals(fifthWeekRanking.getRanks().size(), 2);

        // check Max and James' tallies after week 5 to be 3 and 2.
        assertEquals(fifthWeekRanking.getRanks().get(0).getCurrentValue().getWeeksAtNumberOne(), 3);
        assertEquals(fifthWeekRanking.getRanks().get(1).getPlayerName(), "James");
        assertEquals(fifthWeekRanking.getRanks().get(1).getCurrentValue().getFirstReached(), "2");
        assertEquals(fifthWeekRanking.getRanks().get(1).getCurrentValue().getWeeksAtNumberOne(), 2);
    }

    @Test
    public void testNPlusPlayers() {
        // Case 3 b) & Case 4
        // Populate with more weekly results and rankings
        populateWithN();
    }

    private void populateWithN() {
        weeklyResults.add(new WeeklyResult("6", "Will"));
        weeklyResults.add(new WeeklyResult("7", "Mark"));
        weeklyResults.add(new WeeklyResult("8", "Mark"));

        for (WeeklyResult result : weeklyResults.subList(5, weeklyResults.size())) {
            weeklyRankings.add(allocator.rank(result));
        }
        assertEquals(weeklyRankings.size(), 8);

        // for week 7, Will should be number 3 with 1 week at No.1.
        final WeeklyRanking seventhWeekRanking = weeklyRankings.get(6);
        assertEquals(seventhWeekRanking.getRanks().get(2).getPlayerName(), "Will");
        assertEquals(seventhWeekRanking.getRanks().get(2).getCurrentValue().getFirstReached(), "6");
        assertEquals(seventhWeekRanking.getRanks().get(2).getCurrentValue().getWeeksAtNumberOne(), 1);

        // for week 8, Mark should be number 3, but have the same weeks tally as James
        final WeeklyRanking eighthWeekRanking = weeklyRankings.get(7);
        assertEquals(eighthWeekRanking.getRanks().get(2).getPlayerName(), "Mark");
        assertEquals(eighthWeekRanking.getRanks().get(2).getCurrentValue().getWeeksAtNumberOne(),
                eighthWeekRanking.getRanks().get(1).getCurrentValue().getWeeksAtNumberOne());
    }
}

Based on the testing, I'm inclined to say my code achieves what I wanted. Is there a better way to do so?

One think I'm considering is refactoring the main class, because currently I have a convoluted structure where I scrape and then pass it to rank allocation, and pass that to visualization, and so on. Is there a way to clean that up?

\$\endgroup\$

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.