I am analyzing videos from the popular social media platform, TikTok, and extracting the songs into a Spotify playlist. Here is a gif of one of these videos: https://i.imgur.com/I0dVoac.mp4. Full video: https://vm.tiktok.com/ZMJwqpTPq

enter image description here

Basically, after the video is downloaded, I get unique frames in 2 steps:

  1. Get frames with a play button in them. enter image description here

  2. Get frames where the difference with the previous frame is significant.

My code thus far is:

import cv2
from skimage.metrics import structural_similarity  as ssim
import timeit

        'img': cv2.imread('spotify_play.jpg', cv2.IMREAD_GRAYSCALE),
        'name': 'spotify',
        'bounds': [.8, .94]
        'img': cv2.imread('apple_play_2.jpg', cv2.IMREAD_GRAYSCALE),
        'name': 'apple',
        'bounds': [.7, .82]

def time_method(func):
    def wrapper(*args):
        start = timeit.default_timer()
        val = func(*args)
        elapsed = timeit.default_timer() - start
        print(f'Elapsed time of {func.__name__}:', elapsed)
        return val
    return wrapper

def get_music_frames(file, matches, min_sim=0.5):
    video = cv2.VideoCapture(file)
    phase2 = []
    types = []
    y = None
    while True:
        ret,frame = video.read()
        if ret:
            gray_img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            _, threshed = cv2.threshold(gray_img, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
            for i, match in enumerate(matches):
                matched_result = cv2.matchTemplate(match['img'], threshed , cv2.TM_SQDIFF_NORMED)
                c,_,loc,_ = cv2.minMaxLoc(matched_result)
                if c < min_sim:
                    if y is None:
                        y = loc[1]
                    phase2.append(frame[:y, :])
    if len(phase2) == 0:
        return [], None
    n = round(sum(types) / len(types))
    best_match = matches[n]
    m = best_match['bounds']
    final = []
    for my_frame in phase2:
        height, _,_ = my_frame.shape
        final.append(my_frame[int(height*m[0]):int(height*m[1]), :])
    return final, best_match 
def remove_duplicate_frames(frames, max_sim=0.97):
    pframe = frames[0]
    non_duplicate_frames = []
    for frame in frames[1:]:
        sim = ssim(frame, pframe, multichannel=True)
        if sim < max_sim:
            gray_frame = cv2.cvtColor(pframe, cv2.COLOR_BGR2GRAY)
            _, bw_frame = cv2.threshold(gray_frame, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
            non_duplicate_frames.append(255 - bw_frame)
        pframe = frame
    return non_duplicate_frames

if __name__ == '__main__':
    frames, best_type = get_music_frames('input.mp4', MATCHES)
    if len(frames) == 0:
    frames = remove_duplicate_frames(frames)

My code runs really slowly:

retep@desktop:~/repos/tiktok-analyze-js$ python3 analyze.py
Elapsed time of get_binary_from: ...
Elapsed time of get_music_frames: 18.29
Elapsed time of remove_duplicate_frames: 2.16
Elapsed time of recognize_text_in_frames: ...

I am looking for some optimizations to lower the time to process below 20 seconds.

  • 1
    \$\begingroup\$ Do you really need to write each frame to 'out.jpg'? That sounds like you will be very dependent on disk IO, but overwrite it every loop iteration. \$\endgroup\$
    – Graipher
    Jan 22, 2021 at 7:09


Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Browse other questions tagged or ask your own question.