From 22e19557d81121badfe7fbeee3b33e16a48653b9 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Wed, 10 Jul 2024 15:42:15 +0300 Subject: [PATCH] tmp --- common.py | 8 +++- libResolver.py | 97 +++++++++++++++++++++++++++++++++++++++++++ main.py | 6 --- spotifyFetch.py | 26 ++++++++++++ spotifyJsonCleanUp.py | 32 ++++++++++++++ 5 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 libResolver.py create mode 100644 spotifyJsonCleanUp.py diff --git a/common.py b/common.py index bb08d3e..ba9c296 100644 --- a/common.py +++ b/common.py @@ -22,4 +22,10 @@ def save_data(data, name): file_path = os.path.join(data_dir, f"{name}.json") with open(file_path, 'w') as file: json.dump(data, file, indent=4) - return f"Data saved to {name}.json." \ No newline at end of file + return f"Data saved to {name}.json." + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists diff --git a/libResolver.py b/libResolver.py new file mode 100644 index 0000000..9d89ffe --- /dev/null +++ b/libResolver.py @@ -0,0 +1,97 @@ +import common +import os +import fnmatch +import json +from fuzzywuzzy import fuzz + + +def get_local_library(root_dir): + music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') + music_files_map = {} + for dir_path, dir_names, filenames in os.walk(root_dir): + for pattern in music_extensions: + for filename in fnmatch.filter(filenames, pattern): + song_name = os.path.splitext(filename)[0] + relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) + music_files_map[song_name] = relative_path + return music_files_map + + +def get_matched(pattern, patterns): + result = {} + for target_pattern in patterns: + similarity_score = fuzz.token_set_ratio(pattern, target_pattern) + result[target_pattern] = similarity_score / 100 + + sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) + return sorted_results + + +def resolve_tracks(source_patterns, target_patterns): + from tqdm import tqdm + + pattern_map = {} + + total = len(source_patterns) + with tqdm(total=total, desc='Progress') as pbar: + for pattern in source_patterns: + res = get_matched(pattern, target_patterns) + matched = {"score": 0, "items": []} + if len(res) and res[0][1] > 0.9: + matched['items'] = res[0:min(len(res), 2)] + matched['score'] = res[0][1] + pattern_map[pattern] = matched + pbar.update(1) + + return pattern_map + + +def resolver1(local_library): + track_descriptions = common.get_data("tracks") + + source_patterns = [] + for track in track_descriptions: + artists = common.get_artists_str(track['track']['artists']) + pattern = f"{track['track']['name']} {artists}" + source_patterns.append(pattern) + + target_patterns = [local for _, local in local_library.items()] + + pattern_map = resolve_tracks(source_patterns, target_patterns) + + with open('prefetched/resolved.json', 'w') as file: + json.dump(pattern_map, file, indent=2) + + +def resolver2(local_library): + found_tracks = common.get_data("local_library_on_spotify") + track_descriptions = common.get_data("tracks") + + user_tracks_ids = {track['track']['id']: track for track in track_descriptions} + + resolved_map = {} + + for name, found in found_tracks.items(): + if not len(found): + continue + track_id = found[0]['id'] + if track_id in user_tracks_ids: + resolved_map[name] = track_id + + with open('prefetched/resolved2.json', 'w') as file: + json.dump(resolved_map, file, indent=2) + +def resolve(root_directory): + local_library = get_local_library(root_directory) + with open('prefetched/local_library.json', 'w') as file: + json.dump(local_library, file, indent=2) + + resolver2(local_library) + + +if __name__ == "__main__": + # root_directory = "/mnt/main/data/music/raw" + #root_directory = input("root: ") + resolve("/mnt/main/data/music/raw") + + diff --git a/main.py b/main.py index 49af365..f74ae1c 100644 --- a/main.py +++ b/main.py @@ -12,12 +12,6 @@ def get_playlist_names(): return [name for name, items in playlists.items()] -def get_artists_str(artists_pack): - track_artists = [artist['name'] for artist in artists_pack] - artists = " ".join(track_artists) - return artists - - def print_playlist_tracks(name): print(name) for track in playlists[name]: diff --git a/spotifyFetch.py b/spotifyFetch.py index ec590bc..6827049 100644 --- a/spotifyFetch.py +++ b/spotifyFetch.py @@ -129,10 +129,36 @@ def get_user_data(): return user_id +def find_song(song_pattern): + tracks = fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + return tracks + + +def find_local_library(): + with open('prefetched/local_library.json', 'r') as file: + local_library = json.load(file) + + from tqdm import tqdm + + pattern_map = {} + + total = len(local_library) + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + found_tracks = {} + for track, path in local_library.items(): + found_tracks[track] = find_song(track) + pbar.update(1) + + with open('prefetched/local_library_on_spotify.json', 'w') as file: + json.dump(found_tracks, file, indent=2) + + def fetch_data(): global token token = authenticate() + find_local_library() + user_id = get_user_data() fetch_tracks(user_id) diff --git a/spotifyJsonCleanUp.py b/spotifyJsonCleanUp.py new file mode 100644 index 0000000..7501fbb --- /dev/null +++ b/spotifyJsonCleanUp.py @@ -0,0 +1,32 @@ +import os +import json + + +def process_json_files(directory): + for filename in os.listdir(directory): + if filename.endswith('.json') and ("_mod.json" not in filename): + filepath = os.path.join(directory, filename) + + with open(filepath, 'r') as f: + try: + data = json.load(f) + except json.JSONDecodeError: + continue + + def remove_available_markets(obj): + if isinstance(obj, dict): + return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} + elif isinstance(obj, list): + return [remove_available_markets(item) for item in obj] + else: + return obj + + modified_data = remove_available_markets(data) + + modified_filepath = os.path.join(directory, f"{os.path.splitext(filename)[0]}_mod.json") + with open(modified_filepath, 'w') as f: + json.dump(modified_data, f, indent=4) + + +if __name__ == "__main__": + process_json_files("prefetched")