Python - Get Unique frames of video containing object

Question

I am analyzing videos from the popular social media platform, TikTok, and extracting the songs into a Spotify playlist. Here is a gif of one of these videos: https://i.imgur.com/I0dVoac.mp4. Full video: https://vm.tiktok.com/ZMJwqpTPq

Basically, after the video is downloaded, I get unique frames in 2 steps:

1. Get frames with a play button in them.

2. Get frames where the difference with the previous frame is significant.

My code thus far is:

import cv2
from skimage.metrics import structural_similarity  as ssim
import timeit

MATCHES = [
    {
        'img': cv2.imread('spotify_play.jpg', cv2.IMREAD_GRAYSCALE),
        'name': 'spotify',
        'bounds': [.8, .94]
    },
    {
        'img': cv2.imread('apple_play_2.jpg', cv2.IMREAD_GRAYSCALE),
        'name': 'apple',
        'bounds': [.7, .82]
    }
]

def time_method(func):
    def wrapper(*args):
        start = timeit.default_timer()
        val = func(*args)
        elapsed = timeit.default_timer() - start
        print(f'Elapsed time of {func.__name__}:', elapsed)
        return val
    return wrapper

@time_method
def get_music_frames(file, matches, min_sim=0.5):
    video = cv2.VideoCapture(file)
    phase2 = []
    types = []
    y = None
    while True:
        ret,frame = video.read()
        if ret:
            gray_img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            _, threshed = cv2.threshold(gray_img, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
            for i, match in enumerate(matches):
                matched_result = cv2.matchTemplate(match['img'], threshed , cv2.TM_SQDIFF_NORMED)
                c,_,loc,_ = cv2.minMaxLoc(matched_result)
                if c < min_sim:
                    if y is None:
                        y = loc[1]
                    phase2.append(frame[:y, :])
                    types.append(i)
                    break
        else:
            break
    video.release()
    if len(phase2) == 0:
        return [], None
    n = round(sum(types) / len(types))
    best_match = matches[n]
    m = best_match['bounds']
    final = []
    for my_frame in phase2:
        height, _,_ = my_frame.shape
        final.append(my_frame[int(height*m[0]):int(height*m[1]), :])
    return final, best_match 
@time_method
def remove_duplicate_frames(frames, max_sim=0.97):
    pframe = frames[0]
    non_duplicate_frames = []
    for frame in frames[1:]:
        sim = ssim(frame, pframe, multichannel=True)
        if sim < max_sim:
            gray_frame = cv2.cvtColor(pframe, cv2.COLOR_BGR2GRAY)
            _, bw_frame = cv2.threshold(gray_frame, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
            non_duplicate_frames.append(255 - bw_frame)
        pframe = frame
    return non_duplicate_frames

if __name__ == '__main__':
    frames, best_type = get_music_frames('input.mp4', MATCHES)
    if len(frames) == 0:
        exit()
    frames = remove_duplicate_frames(frames)

My code runs really slowly:

retep@desktop:~/repos/tiktok-analyze-js$ python3 analyze.py
Elapsed time of get_binary_from: ...
Elapsed time of get_music_frames: 18.29
Elapsed time of remove_duplicate_frames: 2.16
Elapsed time of recognize_text_in_frames: ...

I am looking for some optimizations to lower the time to process below 20 seconds.