2
\$\begingroup\$

Previously, I built a scraper to get ranking data for ATP # 1 over the years. See Webscraping tennis data, and follow ups.

Now, I use the data to rank the players on the basis of how many weeks they've spent at #1.

I'd appreciate general feedback on code style and structure. The Scraper class is unchanged, so really everything else is up for review.


Code

Note: if you run the code below, it'll create a directory, and a bunch of files in it.

scraper package

package scraper;

// A POJO that encapsulates a ranking week and the name of the corresponding No.1 player
public class WeeklyResult {
    private final String week;
    private final String playerName;

    public WeeklyResult(final String week, final String playerName) {
        this.week = week;
        this.playerName = playerName;
    }
    public String getWeek() {
        return week;
    }
    public String getPlayerName() {
        return playerName;
    }
}

Scraper.java

package scraper;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.time.Duration;
import java.time.LocalDate;
import java.util.*;
import java.util.stream.Collectors;

public class Scraper {
    private static final Logger logger = LogManager.getLogger(Scraper.class);
    private final String urlPrefix;
    private final String urlSuffix;
    private final Duration timeout;
    private final int totalTries;
    private WeeklyResult latestResult;

    public Scraper(final String urlPrefix, final String urlSuffix, final Duration timeout, final int totalTries) {
        this.urlPrefix = urlPrefix;
        this.urlSuffix = urlSuffix;
        this.timeout = timeout;
        this.totalTries = totalTries;
        this.latestResult = new WeeklyResult("1973-08-16", "N/A");
    }

    public WeeklyResult scrape(final String week) throws ScraperException {
        // in the case the latest scraped data returns an "empty" weekly result, simply retain the latest No.1
        // since it is likely he wouldn't have changed. A weekly result is deemed empty if no player or week info
        // can be found on the ATP page.
        this.latestResult = scrapeWeekly(week)
                .orElse(new WeeklyResult(updateLatestWeekByOne(), this.latestResult.getPlayerName()));
        return this.latestResult;
    }

    private Optional<WeeklyResult> scrapeWeekly(final String week) throws ScraperException {
        final Document document = loadDocument(weeklyResultUrl(week));
        final boolean numberOneDataExists = selectNumberOneRankCell(document).isPresent();
        final Element playerCell = numberOneDataExists ? selectPlayerCellElement(document) : null;

        return Optional.ofNullable(playerCell)
                .map(element -> new WeeklyResult(week, element.text()));
    }

    public List<String> loadWeeks() throws ScraperException {
        final Document document = loadDocument(urlPrefix);
        final Elements elements = selectRankingWeeksElements(document);
        final List<String> weeks = extractWeeks(elements);

        return noEmptyElseThrow(weeks);
    }

    private Document loadDocument(final String url) throws ScraperException {
        for (int tries = 0; tries < this.totalTries; tries++) {
            try {
                return Jsoup.connect(url).timeout((int) timeout.toMillis()).get();
            } catch (IOException e) {
                if (tries == this.totalTries) {
                    throw new ScraperException("Error loading ATP website: ", e);
                }
            }
        }
        return null;
    }

    private static Elements selectRankingWeeksElements(final Document document) {
        // extract ranking weeks from the dropdown menu
        final Elements result = document.getElementsByAttributeValue("data-value", "rankDate")
                .select("ul li");

        Collections.reverse(result);
        return result;
    }

    private static List<String> extractWeeks(final Collection<Element> elements) {
        // refer to https://winterbe.com/posts/2014/07/31/java8-stream-tutorial-examples/
        // and https://www.baeldung.com/java-maps-streams.
        return elements.stream()
                .map(Scraper::extractWeek)
                .collect(Collectors.toList());
    }

    private static List<String> noEmptyElseThrow(final List<String> weeks) throws ScraperException {
        if (weeks.isEmpty()) {
            throw new ScraperException("Cannot process empty data from the weeks calendar!");
        } else {
            return weeks;
        }
    }

    private String weeklyResultUrl(final String week) {
        return urlPrefix + "rankDate=" + week + urlSuffix;
    }

    private static Optional<Element> selectNumberOneRankCell(final Document document) {
        final Element rankCell = selectPlayerRankCell(document);
        return Optional.ofNullable(rankCell).filter(element -> numberOneRankCellExists(element));
    }

    private static Element selectPlayerCellElement(final Document document) {
        return document.getElementsByClass("player-cell").first();
    }

    private static boolean numberOneRankCellExists(final Element rankCell) {
        return rankCell.text().equals("1");
    }

    private static Element selectPlayerRankCell(final Document document) {
        return document.getElementsByClass("rank-cell").first();
    }

    private static String extractWeek(final Element li) {
        return li.text().replaceAll("\\.", "-");
    }

    private String updateLatestWeekByOne() {
        return LocalDate.parse(this.latestResult.getWeek()).plusWeeks(1).toString();
    }
}

ScraperException.java

package scraper;

public class ScraperException extends Exception {
    final String message;
    public ScraperException (String message) {
        this.message = message;
    }
    public ScraperException (String message, Throwable cause) {
        super(cause);
        this.message = message;
    }
    @Override
    public String toString() {
        return this.message;
    }
}

rankallocator package

RankAllocator.java

package rankallocator;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import scraper.WeeklyResult;

import java.util.*;
import java.util.stream.Collectors;

// This class combines all the collective actions of parsing a weekly result
// and generating a weekly ranking of the No1 players for a given week.
public class RankAllocator {
    private static final Logger logger = LogManager.getLogger(RankAllocator.class);
    private final PlayerScoring trackScores;
    private final int top_N; // display the top-N players for Weekly rankings.

    public RankAllocator(final int N) {
        this.top_N = N;
        this.trackScores = new PlayerScoring();
    }
    public PlayerScoring getTrackScores() { return this.trackScores; }

    private void updatePlayerScore(final WeeklyResult weeklyResult) {
        // given a weekly result, update the trackScores hashmap
        this.trackScores.updatePlayerValue(weeklyResult.getWeek(), weeklyResult.getPlayerName());
    }

    private List<PlayerRank> collatePlayerScores() {
        // for each entry in trackScores, construct a new PlayerRank
        // then return a list sorted in descending order by PlayerValue
        List<PlayerRank> ranks =
                (trackScores.getPlayerScorer().entrySet().stream()
                        .map(entry -> new PlayerRank(entry.getKey(), entry.getValue())))
                        .sorted(Comparator.comparingInt((PlayerRank player) ->
                                player.getCurrentValue().getWeeksAtNumberOne()).reversed() //get descending order by weeks @ No.1
                                .thenComparing(player -> player.getCurrentValue().getFirstReached())) //break ties based on who reached earlier
                        .collect(Collectors.toList());
        return ranks;
    }

    public WeeklyRanking rank (final WeeklyResult weeklyResult) {
        Configurator.setLevel(logger.getName(), Level.ERROR);
        updatePlayerScore(weeklyResult);
        logger.debug(this.trackScores.getPlayerScorer());
        List<PlayerRank> weeklyScores = collatePlayerScores();
        return new WeeklyRanking(weeklyResult.getWeek(), weeklyScores, this.top_N);
    }
}

PlayerValue.java

package rankallocator;

// POJO class to hold the "score" of a player,
// which consists of the weeks they've spent at No.1,
// and the week they first attained No.1 (to break ties deterministically)

public class PlayerValue {
    private final String firstReached;
    private int weeksAtNumberOne;

    public PlayerValue(final String week) {
        this.firstReached = week;
        this.weeksAtNumberOne = 1;
    }

    public int getWeeksAtNumberOne() {
        return weeksAtNumberOne;
    }

    public String getFirstReached() {
        return firstReached;
    }

    public void updateWeeksAtNumberOne() {
        this.weeksAtNumberOne++;
    }
}

PlayerScoring.java

package rankallocator;

// A wrapper class that uses a hashmap and additional metadata to maintain the
// score of each No.1 player. The score is maintained using the PlayerValue abstraction
import java.util.HashMap;
import java.util.Map;

public class PlayerScoring {
    private Map<String, PlayerValue> playerScorer;

    public PlayerScoring() {
        playerScorer = new HashMap<>();
    }

    public Map<String, PlayerValue> getPlayerScorer() {
        return playerScorer;
    }

    public PlayerValue findPlayerValue(final String player) {
        return this.playerScorer.get(player);
    }

    public void updatePlayerValue (final String week, final String player) {
        // if the player is a new No.1, construct a new entry
        // otherwise, update (by one) his week tally
        if (!this.playerScorer.containsKey(player)) {
            this.playerScorer.put(player, new PlayerValue(week));
        } else {
            // Test to make sure values are updated properly
            this.playerScorer.get(player).updateWeeksAtNumberOne();
        }
    }
}

PlayerRank.java

package rankallocator;

//POJO coupling playerName and PlayerValue passed in to a WeeklyRanking's PriorityQueue.
public class PlayerRank {
    private final String playerName;
    private final PlayerValue currentValue;

    public PlayerRank(final String name, final PlayerValue value) {
        this.playerName = name;
        this.currentValue = value;
    }

    public String getPlayerName() { return playerName; }
    public PlayerValue getCurrentValue() { return currentValue; }
}

WeeklyRanking.java

package rankallocator;

// Given a specific week, this class builds a
// list of the top ten (or the size of the queue, if lesser)
// players for that week

import java.util.ArrayList;
import java.util.List;

public class WeeklyRanking {
    private final String week;
    private final List<PlayerRank> ranks;
    public WeeklyRanking(final String week, final List<PlayerRank> playerRanks, final int N) {
        this.week = week;
        // construct a list of the top N (max) players based on weeks at No.1
        this.ranks = new ArrayList<>();
        for (PlayerRank rank : playerRanks) {
            this.ranks.add(rank);
            if (this.ranks.size() == N) {
                break;
            }
        }
    }
    
    public String getWeek() { return week; }
    public List<PlayerRank> getRanks() { return ranks; }
}

legacyvisualizer package

LegacyVisualizer.java

package legacyvisualizer;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import rankallocator.PlayerRank;
import rankallocator.WeeklyRanking;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

// A class to write out a list of the "GOAT" status for each week.
public class LegacyVisualizer {
    private final String dirName;
    private static final Logger logger = LogManager.getLogger(LegacyVisualizer.class);
    
    public LegacyVisualizer(final String path) {
        this.dirName = path;
        File dir = new File(dirName);
        if (!dir.exists()) {
            dir.mkdir();
        }
    }

    private void createListFile(WeeklyRanking weeklyRanking) throws IOException {
        List<String> lines = generateLegacyList(weeklyRanking);
        Path file = Paths.get(dirName + "/" + weeklyRanking.getWeek()+".txt");
        Files.write(file, lines, StandardCharsets.UTF_8);
    }

    private static List<String> generateLegacyList (WeeklyRanking weeklyRanking) {
        final List<PlayerRank> ranks = weeklyRanking.getRanks();

        List<String> lines = new ArrayList<>();
        lines.add("Rank\t\tPlayer\t\tWeeks At #1\t\tFirst Reached On");
        for (int i = 0; i < ranks.size(); i++) {
            final String player = ranks.get(i).getPlayerName();
            final int weeksTally = ranks.get(i).getCurrentValue().getWeeksAtNumberOne();
            final String firstReached = ranks.get(i).getCurrentValue().getFirstReached();
            final String line = Integer.toString(i+1) + "." + "\t\t" + player + "\t\t" + Integer.toString(weeksTally) + "\t\t" + firstReached;
            lines.add(line);
        }
        return lines;
    }

    public void visualize(final WeeklyRanking weeklyRanking) {
        // create a file for that week. No need to check if that file
        // for the week exists - if we are retrying, it's probably due to a valid
        // reason (timeout, re-running the visualization etc) and the data we get
        // is at least the same or newer than the previous try,
        // so overwriting the exist file would be fine.
        Configurator.setLevel(logger.getName(), Level.DEBUG);
        try {
            createListFile(weeklyRanking);
        } catch (IOException e) {
            System.out.println("Failed to create file. " + e);
        }
    }
}

Main package

package project;

import legacyvisualizer.LegacyVisualizer;
import rankallocator.RankAllocator;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configurator;
import rankallocator.WeeklyRanking;
import scraper.Scraper;
import scraper.ScraperException;
import scraper.WeeklyResult;

import java.time.Duration;
import java.util.List;

// Main class to manage the visualization of player's legacy rankings
public class Project {
    private static final Logger logger = LogManager.getRootLogger();
    private static final RankAllocator allocator = new RankAllocator(5);
    private static final String dirName = System.getProperty("user.dir")+"/legacy";
    private static final LegacyVisualizer visualizer = new LegacyVisualizer(dirName);

    private static void generateRankingAllocation(WeeklyResult weeklyResult) {
        // pass the scraped result to the next stage of the visualization logic.
        logger.debug("Week: " + weeklyResult.getWeek() + " No.1: " + weeklyResult.getPlayerName());
        WeeklyRanking weeklyRanking = allocator.rank(weeklyResult);
        visualizer.visualize(weeklyRanking);
    }

    public static void main(String[] args) {

        Configurator.setRootLevel(Level.INFO);

        final Scraper scraper =
                new Scraper("https://www.atptour.com/en/rankings/singles?",
                        "&rankRange=0-100", Duration.ofSeconds(90), 3);

        // The flow is as follows: scrape the latest weekly results (starting from 1973),
        // then pass it to the ranking logic (IPR). Rinse and repeat
        try {
            final List<String> weeks = scraper.loadWeeks();
            for (String week : weeks) {
                logger.debug(week);
                WeeklyResult weeklyResult =  scraper.scrape(week);
                generateRankingAllocation(weeklyResult);
            }
        } catch (ScraperException e) {
            logger.error(e.toString());
        }
    }
}


Testing

RankingTests.java

package ranking;

import junit.framework.TestSuite;

public class RankingTests {
    public static TestSuite suite() {
        TestSuite rankTest = new TestSuite("Generation of Accurate Weekly Rankings Suite");
        rankTest.addTestSuite(TrackScoresTest.class);
        rankTest.addTestSuite(WeeklyRankTest.class);
        return rankTest;
    }
}

TrackScoresTest.java

package ranking;

import rankallocator.RankAllocator;
import junit.framework.TestCase;
import org.junit.Test;
import scraper.WeeklyResult;

import java.util.ArrayList;
import java.util.List;

/** Given a series of WeeklyResults, ensure that the tracking/updating of
 *  player's scores (given be a PlayerValue type) is achieved correctly.
 *  This class will test the classes PlayerScoring and PlayerValue for the following:
 *  1) That the hashmap is correctly initializing scores for new No.1s
 *  2) That the hashmap is updating the player values when a No.1 retains his position
 *  3) That the hashmap's PlayerValues can be used to deterministically break a tie
 *  between players with the same tally, by favouring the player who reached No.1 earlier.
 */

public class TrackScoresTest extends TestCase {
    private RankAllocator allocator;
    private List<WeeklyResult> weeklyResults;

    public void setUp() {
        this.allocator = new RankAllocator(5);

        weeklyResults = setupResults();

        for (WeeklyResult weeklyResult : weeklyResults) {
            allocator.rank(weeklyResult);
        }
    }

    private List<WeeklyResult> setupResults() {
        List<WeeklyResult> results = new ArrayList<>();
        results.add(new WeeklyResult("2000-11-06", "Pete Sampras"));
        results.add(new WeeklyResult("2000-11-13", "Pete Sampras"));
        results.add(new WeeklyResult("2000-11-20", "Marat Safin"));
        results.add(new WeeklyResult("2000-11-27", "Marat Safin"));
        results.add(new WeeklyResult("2000-12-04", "Gustavo Kuerten"));
        return results;
    }

    public void tearDown() {
    }
    
    @Test
    public void testTotalTallies() {
        assertEquals(allocator.getTrackScores().findPlayerValue("Pete Sampras").getWeeksAtNumberOne(), 2);
        assertEquals(allocator.getTrackScores().findPlayerValue("Marat Safin").getWeeksAtNumberOne(), 2);
        assertEquals(allocator.getTrackScores().findPlayerValue("Gustavo Kuerten").getWeeksAtNumberOne(), 1);
    }

    @Test
    public void testFirstReached() {
        assertEquals(allocator.getTrackScores().findPlayerValue("Pete Sampras").getFirstReached(), "2000-11-06");
        assertFalse(allocator.getTrackScores().findPlayerValue("Marat Safin").getFirstReached().equals("2000-11-27"));
    }

    @Test
    public void testNewNumberOne() {
        assertEquals(allocator.getTrackScores().findPlayerValue("Gustavo Kuerten").getFirstReached(), "2000-12-04");
        assertEquals(allocator.getTrackScores().findPlayerValue("Gustavo Kuerten").getWeeksAtNumberOne(), 1);
    }
}

WeeklyRankTest.java

package ranking;

// Test class to see that players get properly ranked.
// Note that since we are counting tallies on
// a weekly basis, it is impossible for players to "jump"
// over the other. i.e. if Sam was ranked 3rd by weeks at No.1,
// he has to become 2nd and then become 1st.
// This means that we only need to test for the following cases:

// 1. Players are being ranked *in descending order* by total weeks at No.1
// 2. Ties are broken by using the week first reached
// 3. Weekly Ranking top-N lists are generated correctly when a) len(list) < N and b) >= N.
// 4. When len(list) >= N, changes in the Nth ranked player are reflected correctly.

import junit.framework.TestCase;
import org.junit.Test;
import rankallocator.RankAllocator;
import rankallocator.WeeklyRanking;
import scraper.WeeklyResult;

import java.util.ArrayList;
import java.util.List;

public class WeeklyRankTest extends TestCase {
    private RankAllocator allocator;
    private List<WeeklyResult> weeklyResults;
    private List<WeeklyRanking> weeklyRankings;

    public void setUp() {
        this.allocator = new RankAllocator(3);
        this.weeklyResults = setUpInitialResults();

        this.weeklyRankings = new ArrayList<>();
        for (WeeklyResult weeklyResult : weeklyResults) {
            weeklyRankings.add(allocator.rank(weeklyResult));
        }
    }

    private List<WeeklyResult> setUpInitialResults() {
        List<WeeklyResult> results = new ArrayList<>();
        results.add(new WeeklyResult("1", "Max"));
        results.add(new WeeklyResult("2", "James"));
        results.add(new WeeklyResult("3", "Max"));
        results.add(new WeeklyResult("4", "Max"));
        results.add(new WeeklyResult("5", "James"));
        return results;
    }

    public void tearDown() {
    }

    @Test
    public void testRankingOrderAndLessThanN() {
        // Cases 1, 2 & 3a).
        // We'll show the list as [<week, <name, PlayerValue>>]
        // After setup:
        // check Max is ranked 1 for all weeks,
        for (WeeklyRanking ranking : weeklyRankings) {
            assertEquals(ranking.getRanks().get(0).getPlayerName(), "Max");
        }

        final int numWeeks = weeklyRankings.size();
        final WeeklyRanking fifthWeekRanking = weeklyRankings.get(numWeeks-1);
        // check weeks have been assigned correctly to weekly ranking
        assertEquals(fifthWeekRanking.getWeek(), "5");
        assertEquals(fifthWeekRanking.getRanks().size(), 2);

        // check Max and James' tallies after week 5 to be 3 and 2.
        assertEquals(fifthWeekRanking.getRanks().get(0).getCurrentValue().getWeeksAtNumberOne(), 3);
        assertEquals(fifthWeekRanking.getRanks().get(1).getPlayerName(), "James");
        assertEquals(fifthWeekRanking.getRanks().get(1).getCurrentValue().getFirstReached(), "2");
        assertEquals(fifthWeekRanking.getRanks().get(1).getCurrentValue().getWeeksAtNumberOne(), 2);
    }

    @Test
    public void testNPlusPlayers() {
        // Case 3 b) & Case 4
        // Populate with more weekly results and rankings
        populateWithN();
    }

    private void populateWithN() {
        weeklyResults.add(new WeeklyResult("6", "Will"));
        weeklyResults.add(new WeeklyResult("7", "Mark"));
        weeklyResults.add(new WeeklyResult("8", "Mark"));

        for (WeeklyResult result : weeklyResults.subList(5, weeklyResults.size())) {
            weeklyRankings.add(allocator.rank(result));
        }
        assertEquals(weeklyRankings.size(), 8);

        // for week 7, Will should be number 3 with 1 week at No.1.
        final WeeklyRanking seventhWeekRanking = weeklyRankings.get(6);
        assertEquals(seventhWeekRanking.getRanks().get(2).getPlayerName(), "Will");
        assertEquals(seventhWeekRanking.getRanks().get(2).getCurrentValue().getFirstReached(), "6");
        assertEquals(seventhWeekRanking.getRanks().get(2).getCurrentValue().getWeeksAtNumberOne(), 1);

        // for week 8, Mark should be number 3, but have the same weeks tally as James
        final WeeklyRanking eighthWeekRanking = weeklyRankings.get(7);
        assertEquals(eighthWeekRanking.getRanks().get(2).getPlayerName(), "Mark");
        assertEquals(eighthWeekRanking.getRanks().get(2).getCurrentValue().getWeeksAtNumberOne(),
                eighthWeekRanking.getRanks().get(1).getCurrentValue().getWeeksAtNumberOne());
    }
}

Based on the testing, I'm inclined to say my code achieves what I wanted. Is there a better way to do so?

One think I'm considering is refactoring the main class, because currently I have a convoluted structure where I scrape and then pass it to rank allocation, and pass that to visualization, and so on. Is there a way to clean that up?

\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.