From bb5453b59eb87d2d8c4d6b9a46852247019b7cf4 Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:14:29 +0300 Subject: [PATCH 01/18] initial --- .gitignore | 4 ++ README.md | 3 +- common.py | 25 ++++++++ loadArtwork.py | 45 +++++++++++++++ main.py | 106 ++++++++++++++++++++++++++++++++++ spotifyAuth.py | 122 ++++++++++++++++++++++++++++++++++++++++ spotifyFetch.py | 147 ++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 common.py create mode 100644 loadArtwork.py create mode 100644 main.py create mode 100644 spotifyAuth.py create mode 100644 spotifyFetch.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2803b17 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +prefetched* +__pycache__ +.idea +secret diff --git a/README.md b/README.md index 508172a..68b9ba5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # spotifyFetcher -loads all data through web api +loads all data through web api
+client secret - ***REDACTED*** diff --git a/common.py b/common.py new file mode 100644 index 0000000..bb08d3e --- /dev/null +++ b/common.py @@ -0,0 +1,25 @@ +import json +import os + +data_dir = 'prefetched' + + +def get_data(name): + file_path = os.path.join(data_dir, f"{name}.json") + try: + with open(file_path, 'r') as file: + data = json.load(file) + return data + except FileNotFoundError: + return f"File {name}.json not found." + except json.JSONDecodeError: + return f"Error decoding JSON from {name}.json." + + +def save_data(data, name): + os.makedirs(data_dir, exist_ok=True) + + file_path = os.path.join(data_dir, f"{name}.json") + with open(file_path, 'w') as file: + json.dump(data, file, indent=4) + return f"Data saved to {name}.json." \ No newline at end of file diff --git a/loadArtwork.py b/loadArtwork.py new file mode 100644 index 0000000..4d2dc38 --- /dev/null +++ b/loadArtwork.py @@ -0,0 +1,45 @@ +import requests +import os +from PIL import Image +from io import BytesIO + +from common import * + + +def save_image_from_url(url, name, image_dir="playlist_covers"): + response = requests.get(url) + + if response.status_code != 200: + raise "Cannot fetch the playlist cover" + + image = Image.open(BytesIO(response.content)) + + directory = os.path.join(data_dir, image_dir) + os.makedirs(directory, exist_ok=True) + file_path = os.path.join(directory, f"{name}.jpg") + print(f"Image saved {file_path}") + image.save(file_path) + + +def load_playlist_covers(): + playlists = get_data('playlists') + + for pl in playlists: + if len(pl['images']): + url = pl['images'][0]['url'] + save_image_from_url(url, pl['name']) + + +def load_user_cover(): + user_data = get_data("user_data") + url = user_data['images'][1]['url'] + save_image_from_url(url, "user", ".") + + +def load(): + load_user_cover() + load_playlist_covers() + + +if __name__ == "__main__": + load() diff --git a/main.py b/main.py new file mode 100644 index 0000000..c88776b --- /dev/null +++ b/main.py @@ -0,0 +1,106 @@ +from common import * +from spotifyFetch import fetch_data +import cmd + +playlists = get_data('playlistTracks') +top_artists = get_data('top_artists') +top_tracks = get_data('top_tracks') +user_tracks = get_data('tracks') + + +def get_playlist_names(): + return [name for name, items in playlists.items()] + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists + + +def print_playlist_tracks(name): + print(name) + for track in playlists[name]: + track_data = track['track'] + track_name = track_data['name'] + artists = get_artists_str(track_data['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_top_artists(): + for artist in top_artists: + name = artist['name'] + print(f" '{name}'") + + +def print_top_tracks(): + for track in top_tracks: + track_name = track['name'] + artists = get_artists_str(track['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_stats(): + print(f" tracks - {len(user_tracks)}") + print(f" playlists - {len(playlists)}") + + for name, tracks in playlists.items(): + print(f" '{name}' - {len(tracks)}") + + print(f" top tracks - {len(top_tracks)}") + print(f" top artists - {len(top_artists)}") + + +def print_tracks(): + for trackItem in user_tracks: + track = trackItem['track'] + print(f" '{track['name']}' - {get_artists_str(track['artists'])}") + + +class Interpreter(cmd.Cmd): + intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" + prompt = "(spotify) " + + def do_playlists(self, arg): + """prints user playlists""" + print("\n".join(get_playlist_names())) + + def do_playlist_tracks(self, arg): + """prints playlist [name]""" + try: + name = arg.split()[0] + print_playlist_tracks(name) + except IndexError: + print("Invalid input") + + def do_top_artists(self, arg): + """Top artists""" + print_top_artists() + + def do_top_tracks(self, arg): + """Top tracks""" + print_top_tracks() + + def do_stat(self, arg): + """Print stats""" + print_stats() + + def do_tracks(self, arg): + """Print tracks""" + print_tracks() + + def do_fetch(self, arg): + """Fetch all spotify data""" + try: + fetch_data() + except ValueError: + print("Can not fetch the data.") + + def do_exit(self, arg): + """Exit the command loop: exit""" + print("Goodbye!") + return True + + +if __name__ == '__main__': + Interpreter().cmdloop() diff --git a/spotifyAuth.py b/spotifyAuth.py new file mode 100644 index 0000000..64a4b2c --- /dev/null +++ b/spotifyAuth.py @@ -0,0 +1,122 @@ +import base64 + +import requests +import webbrowser +from http.server import BaseHTTPRequestHandler, HTTPServer +import urllib.parse +import threading +import time +import os.path + +client_id = '***REDACTED***' +client_secret = None +token = None + + +def resolve_client_secret(): + global client_secret + if os.path.exists('secret'): + with open('secret', 'r') as file: + client_secret = file.readline().strip() + else: + client_secret = input("Enter the application secret: ") + with open('secret', 'w') as file: + file.write(client_secret) + + +def retrieve_web_api_token(authorization_code): + global token + + token_url = 'https://accounts.spotify.com/api/token' + + token_data = { + "grant_type": "authorization_code", + "client_secret": client_secret, + "client_id": client_id, + "code": authorization_code, + 'redirect_uri': 'http://localhost:8888', + } + + print("\nAuthenticating WEB API\n") + + auth_string = f'{client_id}:{client_secret}' + b64_auth_string = base64.b64encode(auth_string.encode()).decode() + + # Define the headers + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': f'Basic {b64_auth_string}' + } + + token_response = requests.post(token_url, data=token_data, headers=headers) + + if token_response.status_code not in range(200, 299): + print(f"Failed to retrieve access token: {token_response.status_code}") + print(token_response.json()) + raise ValueError + + token = token_response.json()['access_token'] + + +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + + parsed_url = urllib.parse.urlparse(self.path) + query_params = urllib.parse.parse_qs(parsed_url.query) + + if 'code' in query_params: + authorization_code = query_params['code'][0] + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code received. You can close this window.') + + retrieve_web_api_token(authorization_code) + + threading.Thread(target=self.server.shutdown).start() + + else: + self.send_response(400) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code not found.') + + +def run_redirect_server(): + httpd = HTTPServer(('localhost', 8888), RequestHandler) + print('Starting HTTP redirection server') + httpd.serve_forever() + + +def open_authenticate_page(): + authorize_url = 'https://accounts.spotify.com/authorize' + + params = { + 'client_id': client_id, + 'response_type': 'code', + 'redirect_uri': 'http://localhost:8888', + "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", + } + + auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}" + webbrowser.open(auth_url) + + +def authenticate(): + resolve_client_secret() + + redirect_server = threading.Thread(target=run_redirect_server) + redirect_server.start() + + time.sleep(1) + open_authenticate_page() + + redirect_server.join() + + return token + + +if __name__ == "__main__": + authenticate() + diff --git a/spotifyFetch.py b/spotifyFetch.py new file mode 100644 index 0000000..ec590bc --- /dev/null +++ b/spotifyFetch.py @@ -0,0 +1,147 @@ +import requests +from loadArtwork import load +from spotifyAuth import authenticate +from common import * + +FETCH_STEP = 50 +token = '' + + +def fetch(endpoint, method='GET', body=None): + url = f'https://api.spotify.com/{endpoint}' + headers = { + 'Authorization': f'Bearer {token}', + } + + response = requests.request(method, url, headers=headers, json=body) + response.raise_for_status() + return response.json() + + +def fetch_playlists(user_id): + print("Fetching playlists") + endpoint = f"v1/me/playlists" + pl_total = fetch(f'{endpoint}?limit=1')['total'] + pl_count = 0 + playlists = [] + + while pl_count < pl_total: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}') + playlists += res['items'] + pl_count += FETCH_STEP + + return playlists + + +def fetch_playlist_items(playlist_id): + endpoint = f"v1/playlists/{playlist_id}/tracks" + items = fetch(f'{endpoint}?fields=total')['total'] + loaded = 0 + tracks = [] + + while loaded < items: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}') + + tracks += res['items'] + loaded += FETCH_STEP + + return tracks + + +def fetch_tracks(user_id): + print("Fetching tracks") + endpoint = f"v1/me/tracks" + total = fetch(f"{endpoint}?limit=1&market=ES")['total'] + current = 0 + tracks = [] + + while total > current: + res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES") + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "tracks") + return tracks + + +def fetch_tracks_top(): + print("Fetching top tracks") + total = fetch(f'v1/me/top/tracks?fields=total')['total'] + current = 0 + tracks = [] + + while current < total: + res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term') + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "top_tracks") + return tracks + + +def fetch_artists_top(): + print("Fetching top artists") + + total = fetch(f'v1/me/top/artists?fields=total')['total'] + current = 0 + artists = [] + + while current < total: + res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term') + artists += res['items'] + current += FETCH_STEP + + save_data(artists, "top_artists") + return artists + + +def fetch_all_playlist_data(user_id): + playlists = fetch_playlists(user_id) + + save_data(playlists, 'playlists') + + print(len(playlists)) + + playlist_items = {} + + print("Fetching playlists items") + + index = 0 + for pl in playlists: + pl_id = pl['id'] + name = pl['name'] + playlist_items[name] = fetch_playlist_items(pl_id) + print(f"{index} - {len(playlist_items[name])}") + index += 1 + + save_data(playlist_items, "playlistTracks") + + +def get_user_data(): + print("Fetching user data") + + user_profile = fetch(f'v1/me') + user_id = user_profile['id'] + user_data = fetch(f'v1/users/{user_id}') + + save_data(user_profile, "user_profile") + save_data(user_data, "user_data") + return user_id + + +def fetch_data(): + global token + token = authenticate() + + user_id = get_user_data() + + fetch_tracks(user_id) + fetch_all_playlist_data(user_id) + fetch_tracks_top() + fetch_artists_top() + + load() + + +if __name__ == "__main__": + fetch_data() From 366860a1ad3efd32572eb16e6c71d3a85cfd6425 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Mon, 8 Jul 2024 23:24:51 +0300 Subject: [PATCH 02/18] atmop --- main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.py b/main.py index c88776b..49af365 100644 --- a/main.py +++ b/main.py @@ -93,6 +93,14 @@ class Interpreter(cmd.Cmd): """Fetch all spotify data""" try: fetch_data() + + global playlists, top_artists, top_tracks, user_tracks + + playlists = get_data('playlistTracks') + top_artists = get_data('top_artists') + top_tracks = get_data('top_tracks') + user_tracks = get_data('tracks') + except ValueError: print("Can not fetch the data.") From 11e3b06824590d679b233e002e5881784e1838d3 Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:14:29 +0300 Subject: [PATCH 03/18] stable need run main fetch command then run indexer then LinkerPatternGenerator then link_pattern command in the main --- .gitignore | 4 + DataBase.py | 49 ++++++++++ LinkerPatternGenerator.py | 137 ++++++++++++++++++++++++++ LocalLibraryIndexer.py | 80 ++++++++++++++++ ParseItunesToJson.py | 101 +++++++++++++++++++ README.md | 3 +- SpotifyAuthenticator.py | 122 +++++++++++++++++++++++ SpotifyWebAPI.py | 190 ++++++++++++++++++++++++++++++++++++ env | 36 +++++++ main.py | 197 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 918 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 DataBase.py create mode 100644 LinkerPatternGenerator.py create mode 100644 LocalLibraryIndexer.py create mode 100755 ParseItunesToJson.py create mode 100644 SpotifyAuthenticator.py create mode 100644 SpotifyWebAPI.py create mode 100755 env create mode 100644 main.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2803b17 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +prefetched* +__pycache__ +.idea +secret diff --git a/DataBase.py b/DataBase.py new file mode 100644 index 0000000..c0f230b --- /dev/null +++ b/DataBase.py @@ -0,0 +1,49 @@ +import json +import os + +data_dir = 'prefetched' + + +class SongTags: + def __init__(self, title='', artist='', album=''): + self.artist = artist + self.title = title + self.album = album + + +def get_data(name): + file_path = os.path.join(data_dir, f"{name}.json") + try: + with open(file_path, 'r') as file: + data = json.load(file) + return data + except FileNotFoundError: + print(f"File {name}.json not found.") + return {} + except json.JSONDecodeError: + print(f"Error decoding JSON from {name}.json.") + return {} + +def save_data(data, name): + def remove_available_markets(obj): + if isinstance(obj, dict): + return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} + elif isinstance(obj, list): + return [remove_available_markets(item) for item in obj] + else: + return obj + + data = remove_available_markets(data) + + os.makedirs(data_dir, exist_ok=True) + + file_path = os.path.join(data_dir, f"{name}.json") + with open(file_path, 'w') as file: + json.dump(data, file, indent=4) + return f"Data saved to {name}.json." + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py new file mode 100644 index 0000000..2f6eba2 --- /dev/null +++ b/LinkerPatternGenerator.py @@ -0,0 +1,137 @@ +from DataBase import * +from SpotifyWebAPI import find_song, update_access_token + +from rapidfuzz import fuzz +from tqdm import tqdm + + +def spotify_pattern_generator(): + update_access_token() + + local_library = get_data("local_library") + spotify_library = get_data("tracks") + + def find_local_songs_on_spotify(local_lib): + total = len(local_library) + found_tracks_map = {} + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + for track_id, track in local_lib.items(): + search_pattern = f"{track['name']} {track['artist']} {track['album']}" + found_tracks = find_song(search_pattern) + found_tracks_map[track_id] = found_tracks[0:1] + pbar.update(1) + return found_tracks_map + + spotify_found_tracks = find_local_songs_on_spotify(local_library) + spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library} + + spotify_pattern = {} + for local_id, found_items in spotify_found_tracks.items(): + spotify_pattern[local_id] = {"score": 0.0, "items": []} + + if not len(found_items): + continue + + for found_item in found_items: + spotify_track_id = found_item['id'] + if spotify_track_id in spotify_user_track: + spotify_pattern[local_id]['items'].append(spotify_track_id) + spotify_user_track[spotify_track_id]['local_mappings'].append(local_id) + + for spotify_id, track_local_mappings in spotify_user_track.items(): + score = len(track_local_mappings['local_mappings']) + for local_id in track_local_mappings['local_mappings']: + spotify_pattern[local_id]['score'] = 1 / score + + save_data(spotify_pattern, "link_pattern_spotify") + + +def fuzzy_pattern_generator(): + local_library = get_data("local_library") + track_descriptions = get_data("tracks") + + def get_matched(src_pattern, patterns): + result = {} + for target_pattern in patterns: + similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern) + result[target_pattern] = similarity_score / 100 + + sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) + return sorted_results + + def resolve_tracks(s_patterns, t_patterns): + + pattern_map = {} + + total = len(s_patterns) + with tqdm(total=total, desc='Progress') as pbar: + for s_pattern in s_patterns: + res = get_matched(s_pattern, t_patterns) + matched = {"score": 0, "items": []} + if len(res): + matched['items'] = res[0:min(len(res), 4)] + matched['score'] = res[0][1] + pattern_map[s_pattern] = matched + pbar.update(1) + + return pattern_map + + source_patterns = [] + for track in track_descriptions: + artists = get_artists_str(track['track']['artists']) + pattern = f"{track['track']['name']} {artists}" + source_patterns.append(pattern) + + target_patterns = [local['path'] for _, local in local_library.items()] + + fuzzy_pattern = resolve_tracks(source_patterns, target_patterns) + save_data(fuzzy_pattern, "link_pattern_fuzzy") + + +def fuzzy_tag_pattern_generator(): + local_library = get_data("local_library") + spotify_library = get_data("tracks") + mappings = {} + + def fuzzy_tags_ratio(first: SongTags, second: SongTags): + title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100 + artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100 + album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100 + return title_ratio, artist_ratio, album_ratio + + with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar: + for local_track_id, local_track in local_library.items(): + mapping = mappings.get(local_track_id, {"score": 1.0, "items": []}) + ratios = [] + for remote_track in spotify_library: + track = remote_track['track'] + + tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album']) + tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name']) + + ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) + + filtered_ratios = [] + threshold = 0.8 + for ratio in ratios: + if all([val > threshold for val in ratio[0]]): + filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1])) + + filtered_ratios.sort(reverse=True, key=lambda x: x[0]) + if len(filtered_ratios): + mapping['items'] = [filtered_ratios[0][1]] + + mappings[local_track_id] = mapping + pbar.update(1) + + save_data(mappings, "link_pattern_tags") + + +def run_pattern_generators(): + #fuzzy_pattern_generator() + #spotify_pattern_generator() + fuzzy_tag_pattern_generator() + + +if __name__ == "__main__": + run_pattern_generators() diff --git a/LocalLibraryIndexer.py b/LocalLibraryIndexer.py new file mode 100644 index 0000000..eecdd58 --- /dev/null +++ b/LocalLibraryIndexer.py @@ -0,0 +1,80 @@ +import os +import fnmatch + +import mutagen + +from DataBase import * +import eyed3 +from mutagen.flac import FLAC + + +music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') +exclude_directories = ("*mary--*", "*example-word*") + + +def update_local_library(root_dir): + old_library = get_data("local_library") + known_paths = {track['path']: track_id for track_id, track in old_library.items()} + new_library = {} + + song_id = '0' + + def get_new_song_id(prev_id, song_path): + if song_path in known_paths: + return int(known_paths[song_path]) + while (prev_id in old_library) or (prev_id in new_library): + prev_id = str(int(prev_id) + 1) + return prev_id + + for dir_path, dir_names, filenames in os.walk(root_dir): + dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)] + filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)] + + for filename in filtered_filenames: + song_name, track_type = os.path.splitext(filename) + relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) + song_id = get_new_song_id(song_id, relative_path) + new_library[song_id] = { + "name": song_name, + "path": relative_path, + "type": track_type, + "artist": "", + "album": "" + } + + for track_id, track in new_library.items(): + abs_path = os.path.join(root_dir, track['path']) + match track['type']: + case ".mp3": + mp3track = eyed3.load(abs_path) + if not mp3track: + print(f"Failed to load mp3 {abs_path} using just name of the file as track name") + track['name'] = track['path'] + else: + tag = mp3track.tag + if not tag: + print(f"Invalid mp3 header {abs_path} using just name of the file as track name") + track['name'] = track['path'] + else: + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album + + case ".flac": + try: + metadata = FLAC(abs_path).tags + if 'artist' in metadata: + track['artist'] = " ".join(metadata['artist']) + if 'TITLE' in metadata: + track['name'] = " ".join(metadata['TITLE']) + if 'album' in metadata: + track['album'] = " ".join(metadata['album']) + except mutagen.MutagenError: + print(f"Invalid flac header {abs_path} using just name of the file as track name") + track['name'] = track['path'] + + save_data(new_library, "local_library") + + +if __name__ == "__main__": + update_local_library("/home/auser/Music/") diff --git a/ParseItunesToJson.py b/ParseItunesToJson.py new file mode 100755 index 0000000..1505a36 --- /dev/null +++ b/ParseItunesToJson.py @@ -0,0 +1,101 @@ +import xmltodict +import json + + +def flatten_dict(d): + + out = [] + for song in d: + newSong = {} + + strCount = 0 + intCount = 0 + dateCount = 0 + + def getStr(id): + nonlocal strCount + if id not in song["key"]: + return "undef" + if len(song["string"]) <= strCount: + return "error" + strCount += 1 + return song["string"][strCount - 1] + + def getInt(id): + nonlocal intCount + if id not in song["key"]: + return -1 + if len(song["integer"]) <= intCount: + return -1 + intCount += 1 + return song["integer"][intCount - 1] + + def getDate(id): + nonlocal dateCount + if id not in song["key"]: + return "undef" + if len(song["date"]) <= dateCount: + return "error" + dateCount += 1 + return song["date"][dateCount - 1] + + newSong["Track ID"] = getInt("Track ID") + newSong["Name"] = getStr("Name") + newSong["Artist"] = getStr("Artist") + newSong["Album Artist"] = getStr("Album Artist") + newSong["Composer"] = getStr("Composer") + newSong["Album"] = getStr("Album") + newSong["Genre"] = getStr("Genre") + newSong["Kind"] = getStr("Kind") + newSong["Size"] = getInt("Size") + newSong["Total Time"] = getInt("Total Time") + newSong["Disc Number"] = getInt("Disc Number") + newSong["Disc Count"] = getInt("Disc Count") + newSong["Track Number"] = getInt("Track Number") + newSong["Track Count"] = getInt("Track Count") + newSong["Year"] = getInt("Year") + newSong["Date Modified"] = getDate("Date Modified") + newSong["Date Added"] = getDate("Date Added") + newSong["Bit Rate"] = getInt("Bit Rate") + newSong["Sample Rate"] = getInt("Sample Rate") + newSong["Play Count"] = getInt("Play Count") + newSong["Play Date"] = getInt("Play Date") + newSong["Play Date UTC"] = getDate("Play Date UTC") + newSong["Skip Count"] = getInt("Skip Count") + newSong["Skip Date"] = getDate("Skip Date") + newSong["Release Date"] = getDate("Release Date") + newSong["Album Rating"] = getInt("Album Rating") + newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"] + newSong["Loved"] = "Loved" in song["key"] + newSong["Album Loved"] = "Album Loved" in song["key"] + newSong["Explicit"] = "Explicit" in song["key"] + newSong["Compilation"] = "Compilation" in song["key"] + newSong["Artwork Count"] = getInt("Artwork Count") + newSong["Sort Album"] = getStr("Sort Album") + newSong["Sort Artist"] = getStr("Sort Artist") + newSong["Sort Name"] = getStr("Sort Name") + newSong["Persistent ID"] = getStr("Persistent ID") + newSong["Track Type"] = getStr("Track Type") + + out.append(newSong) + + return out + + +def convert(filename, out_path): + with open(filename, 'r', encoding='utf-8') as xml_file: + data_dict = xmltodict.parse(xml_file.read()) + + # Extract the "Tracks" dictionary to be flattened + tracks_dict = data_dict['plist']['dict']['dict'] + + flat_tracks_dict = flatten_dict(tracks_dict["dict"]) + + json_data = json.dumps(flat_tracks_dict, indent=2) + + with open(out_path, 'w', encoding='utf-8') as json_file: + json_file.write(json_data) + + +if __name__ == "__main__": + convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json') diff --git a/README.md b/README.md index 508172a..68b9ba5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # spotifyFetcher -loads all data through web api +loads all data through web api
+client secret - ***REDACTED*** diff --git a/SpotifyAuthenticator.py b/SpotifyAuthenticator.py new file mode 100644 index 0000000..86705d0 --- /dev/null +++ b/SpotifyAuthenticator.py @@ -0,0 +1,122 @@ +import base64 + +import requests +import webbrowser +from http.server import BaseHTTPRequestHandler, HTTPServer +import urllib.parse +import threading +import time +import os.path + +client_id = '***REDACTED***' +client_secret = None +token = None + + +def resolve_client_secret(): + global client_secret + if os.path.exists('secret'): + with open('secret', 'r') as file: + client_secret = file.readline().strip() + else: + client_secret = input("Enter the application secret: ") + with open('secret', 'w') as file: + file.write(client_secret) + + +def retrieve_web_api_token(authorization_code): + global token + + token_url = 'https://accounts.spotify.com/api/token' + + token_data = { + "grant_type": "authorization_code", + "client_secret": client_secret, + "client_id": client_id, + "code": authorization_code, + 'redirect_uri': 'http://127.0.0.1:8888', + } + + print("\nAuthenticating WEB API\n") + + auth_string = f'{client_id}:{client_secret}' + b64_auth_string = base64.b64encode(auth_string.encode()).decode() + + # Define the headers + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': f'Basic {b64_auth_string}' + } + + token_response = requests.post(token_url, data=token_data, headers=headers) + + if token_response.status_code not in range(200, 299): + print(f"Failed to retrieve access token: {token_response.status_code}") + print(token_response.json()) + raise ValueError + + token = token_response.json()['access_token'] + + +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + + parsed_url = urllib.parse.urlparse(self.path) + query_params = urllib.parse.parse_qs(parsed_url.query) + + if 'code' in query_params: + authorization_code = query_params['code'][0] + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code received. You can close this window.') + + retrieve_web_api_token(authorization_code) + + threading.Thread(target=self.server.shutdown).start() + + else: + self.send_response(400) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code not found.') + + +def run_redirect_server(): + httpd = HTTPServer(('localhost', 8888), RequestHandler) + print('Starting HTTP redirection server') + httpd.serve_forever() + + +def open_authenticate_page(): + authorize_url = 'https://accounts.spotify.com/authorize' + + params = { + 'client_id': client_id, + 'response_type': 'code', + 'redirect_uri': 'http://127.0.0.1:8888', + "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", + } + + auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}" + webbrowser.open(auth_url) + + +def authenticate(): + resolve_client_secret() + + redirect_server = threading.Thread(target=run_redirect_server) + redirect_server.start() + + time.sleep(1) + open_authenticate_page() + + redirect_server.join() + + return token + + +if __name__ == "__main__": + authenticate() + diff --git a/SpotifyWebAPI.py b/SpotifyWebAPI.py new file mode 100644 index 0000000..71ec2f7 --- /dev/null +++ b/SpotifyWebAPI.py @@ -0,0 +1,190 @@ +import requests +from PIL import Image +from io import BytesIO +from SpotifyAuthenticator import authenticate +from DataBase import * + +FETCH_STEP = 50 +token = '' + + +def fetch(endpoint, method='GET', body=None): + url = f'https://api.spotify.com/{endpoint}' + headers = { + 'Authorization': f'Bearer {token}', + } + + response = requests.request(method, url, headers=headers, json=body) + response.raise_for_status() + return response.json() + + +def find_song(song_pattern): + return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + + +def fetch_playlists(user_id): + print("Fetching playlists") + endpoint = f"v1/me/playlists" + pl_total = fetch(f'{endpoint}?limit=1')['total'] + pl_count = 0 + playlists = [] + + while pl_count < pl_total: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}') + playlists += res['items'] + pl_count += FETCH_STEP + + return playlists + + +def fetch_playlist_items(playlist_id): + endpoint = f"v1/playlists/{playlist_id}/tracks" + items = fetch(f'{endpoint}?fields=total')['total'] + loaded = 0 + tracks = [] + + while loaded < items: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}') + + tracks += res['items'] + loaded += FETCH_STEP + + return tracks + + +def fetch_tracks(user_id): + print("Fetching tracks") + endpoint = f"v1/me/tracks" + total = fetch(f"{endpoint}?limit=1&market=ES")['total'] + current = 0 + tracks = [] + + while total > current: + res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES") + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "tracks") + return tracks + + +def fetch_tracks_top(): + print("Fetching top tracks") + total = fetch(f'v1/me/top/tracks?fields=total')['total'] + current = 0 + tracks = [] + + while current < total: + res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term') + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "top_tracks") + return tracks + + +def fetch_artists_top(): + print("Fetching top artists") + + total = fetch(f'v1/me/top/artists?fields=total')['total'] + current = 0 + artists = [] + + while current < total: + res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term') + artists += res['items'] + current += FETCH_STEP + + save_data(artists, "top_artists") + return artists + + +def fetch_all_playlist_data(user_id): + playlists = fetch_playlists(user_id) + + save_data(playlists, 'playlists') + + print(len(playlists)) + + playlist_items = {} + + print("Fetching playlists items") + + index = 0 + for pl in playlists: + pl_id = pl['id'] + name = pl['name'] + playlist_items[name] = fetch_playlist_items(pl_id) + print(f"{index} - {len(playlist_items[name])}") + index += 1 + + save_data(playlist_items, "playlistTracks") + + +def get_user_data(): + print("Fetching user data") + + user_profile = fetch(f'v1/me') + user_id = user_profile['id'] + user_data = fetch(f'v1/users/{user_id}') + + save_data(user_profile, "user_profile") + save_data(user_data, "user_data") + return user_id + + +def save_image_from_url(url, name, image_dir="playlist_covers"): + response = requests.get(url) + + if response.status_code != 200: + raise "Cannot fetch the playlist cover" + + image = Image.open(BytesIO(response.content)) + + directory = os.path.join(data_dir, image_dir) + os.makedirs(directory, exist_ok=True) + file_path = os.path.join(directory, f"{name}.jpg") + print(f"Image saved {file_path}") + image.save(file_path) + + +def load_playlist_covers(): + playlists = get_data('playlists') + + for pl in playlists: + if len(pl['images']): + url = pl['images'][0]['url'] + save_image_from_url(url, pl['name']) + + +def load_user_cover(): + user_data = get_data("user_data") + url = user_data['images'][1]['url'] + save_image_from_url(url, "user", ".") + + +def load_artworks(): + load_user_cover() + load_playlist_covers() + + +def update_access_token(): + global token + token = authenticate() + + +def fetch_data(): + user_id = get_user_data() + + fetch_tracks(user_id) + fetch_all_playlist_data(user_id) + fetch_tracks_top() + fetch_artists_top() + + load_artworks() + + +if __name__ == "__main__": + update_access_token() + fetch_data() diff --git a/env b/env new file mode 100755 index 0000000..8955572 --- /dev/null +++ b/env @@ -0,0 +1,36 @@ + +PS1=" > " + +#export PATH="/usr/local/bin/:$PATH" +export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH" +#export PATH="$HOME/home/auser/.local/bin:$PATH" + +export SIP="185.238.170.251" + +alias v=nvim +alias ll="ls -l -a" +alias gl="git log --oneline" +alias ssh_server="ssh auser@185.238.170.251" + +#eval "$(zoxide init bash --cmd zd)" + +#source $PMAIN/.scripts/bashmarks.sh + +fe() { + local result=$(command tere "$@") + [ -n "$result" ] && cd -- "$result" +} + +vim_configure() { + pwd=$(pwd) + cd ~/.config/nvim/ + nvim + cd $pwd +} + +pyenv() { + source ~/bin/python/env311/bin/activate +} + +source ~/src/scripts/remotes.sh +source /usr/share/fzf/key-bindings.bash diff --git a/main.py b/main.py new file mode 100644 index 0000000..9709e93 --- /dev/null +++ b/main.py @@ -0,0 +1,197 @@ +from DataBase import * +from SpotifyWebAPI import fetch_data, update_access_token +import cmd + +playlists = get_data('playlistTracks') +top_artists = get_data('top_artists') +top_tracks = get_data('top_tracks') +user_tracks = get_data('tracks') +link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')] +local_library = get_data("local_library") +user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} + + +def get_playlist_names(pl): + return [name for name, items in pl.items()] + + +def print_playlist_tracks(pl_id: int): + pls = list(playlists) + name = pls[pl_id] + pl = playlists[name] + print(name) + for track in pl: + track_data = track['track'] + track_name = track_data['name'] + artists = get_artists_str(track_data['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_top_artists(): + for artist in top_artists: + name = artist['name'] + print(f" '{name}'") + + +def print_top_tracks(): + for track in top_tracks: + track_name = track['name'] + artists = get_artists_str(track['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_stats(): + print(f" tracks - {len(user_tracks)}") + print(f" playlists - {len(playlists)}") + + for name, tracks in playlists.items(): + print(f" '{name}' - {len(tracks)}") + + print(f" top tracks - {len(top_tracks)}") + print(f" top artists - {len(top_artists)}") + + +def print_tracks(): + track_idx = 0 + for trackItem in user_tracks: + track = trackItem['track'] + print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'") + track_idx += 1 + + +def sort_remote_tracks_by_popularity(track_ids): + out = [] + for top_track in top_tracks: + if top_track['id'] in track_ids: + out.append(top_track['id']) + return out + + +def print_link_stats(link_pattern): + resolved_reversed = {} + resolved = [] + unresolved_local = [] + unresolved_remote = [] + + for local_id, links in link_pattern.items(): + if len(links['items']): + resolved.append(local_id) + resolved_reversed[links['items'][0]] = local_id + else: + unresolved_local.append(local_id) + + for track in user_tracks: + track_id = track['track']['id'] + if track_id not in resolved_reversed: + unresolved_remote.append(track_id) + + print(f"Local Tracks: {len(local_library)}") + print(f"Remote Tracks: {len(user_tracks)}") + + unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) + unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] + + print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") + index = 0 + for remote_id in unresolved_remote: + remote_track = user_tracks_by_id[remote_id] + print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + print(f"\nResolved tracks: {len(resolved)}") + index = 0 + for link in resolved: + remote_id = link_pattern[link]['items'][0] + local_track = local_library[link] + remote_track = user_tracks_by_id[remote_id] + + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') + print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + return + print(f"\nUnresolved tracks from the local: {len(unresolved_local)}") + index = 0 + for link in unresolved_local: + local_track = local_library[link] + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'") + index += 1 + + +class Interpreter(cmd.Cmd): + intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" + prompt = "(spotify) " + + def do_link_patterns(self, arg): + """prints available linking patterns""" + print(f"0 - fuzzy tags") + print(f"1 - spotify") + print(f"2 - fuzzy") + + def do_link_pattern(self, arg): + if not len(arg.split()): + print("expected name of the pattern") + return + + """prints available linking patterns""" + pattern_id = int(arg.split()[0]) + print_link_stats(link_patterns[pattern_id]) + + def do_playlists(self, arg): + """prints user playlists""" + pl_id = 0 + for pl in get_playlist_names(playlists): + print(f"{pl_id} - {pl}") + pl_id += 1 + + def do_playlist_tracks(self, arg): + """prints playlist [name]""" + try: + pl_id = int(arg.split()[0]) + print_playlist_tracks(pl_id) + except IndexError: + print("Invalid input") + + def do_top_artists(self, arg): + """Top artists""" + print_top_artists() + + def do_top_tracks(self, arg): + """Top tracks""" + print_top_tracks() + + def do_stat(self, arg): + """Print stats""" + print_stats() + + def do_tracks(self, arg): + """Print tracks""" + print_tracks() + + def do_fetch(self, arg): + """Fetch all spotify data""" + try: + update_access_token() + fetch_data() + + global playlists, top_artists, top_tracks, user_tracks + + playlists = get_data('playlistTracks') + top_artists = get_data('top_artists') + top_tracks = get_data('top_tracks') + user_tracks = get_data('tracks') + + except ValueError: + print("Can not fetch the data.") + + def do_exit(self, arg): + """Exit the command loop: exit""" + print("Goodbye!") + return True + + +if __name__ == '__main__': + try: + Interpreter().cmdloop() + except KeyboardInterrupt as kb: + print("process terminated") \ No newline at end of file From 892568d668b487414c17dc4ffbb46bcc30c95e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sun, 4 Jan 2026 01:37:47 +0300 Subject: [PATCH 04/18] stable - generate local playlists & add default options --- LinkerPatternGenerator.py | 28 ++++++++++----- LocalPLaylistGenerator.py | 50 +++++++++++++++++++++++++++ TODO | 7 ++++ main.py | 73 +++++++++++++++++++++++++++++---------- 4 files changed, 132 insertions(+), 26 deletions(-) create mode 100644 LocalPLaylistGenerator.py create mode 100644 TODO diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index 2f6eba2..8576da8 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -3,7 +3,7 @@ from SpotifyWebAPI import find_song, update_access_token from rapidfuzz import fuzz from tqdm import tqdm - +import requests def spotify_pattern_generator(): update_access_token() @@ -11,15 +11,29 @@ def spotify_pattern_generator(): local_library = get_data("local_library") spotify_library = get_data("tracks") + def find_local_songs_on_spotify(local_lib): + search_log = [] + total = len(local_library) found_tracks_map = {} with tqdm(total=total, desc='Searching local songs on spotify') as pbar: for track_id, track in local_lib.items(): - search_pattern = f"{track['name']} {track['artist']} {track['album']}" - found_tracks = find_song(search_pattern) - found_tracks_map[track_id] = found_tracks[0:1] + parts = [track["name"], track["artist"], track.get("album")] + search_pattern = " ".join(p for p in parts if p) + + if search_pattern == "": + print(f"cannot generate search pattern for the track - {track["path"]}") + pbar.update(1) + continue + + found_tracks = find_song(search_pattern)[0:1] + found_tracks_map[track_id] = found_tracks + search_log.append({"path" : track["path"], "search_pattern": search_pattern, "result" : found_tracks_map[track_id]}) pbar.update(1) + + save_data(search_log, "spotify_search_log") + return found_tracks_map spotify_found_tracks = find_local_songs_on_spotify(local_library) @@ -127,10 +141,8 @@ def fuzzy_tag_pattern_generator(): save_data(mappings, "link_pattern_tags") -def run_pattern_generators(): - #fuzzy_pattern_generator() - #spotify_pattern_generator() - fuzzy_tag_pattern_generator() +def run_pattern_generators(type_id = 0): + spotify_pattern_generator() if __name__ == "__main__": diff --git a/LocalPLaylistGenerator.py b/LocalPLaylistGenerator.py new file mode 100644 index 0000000..0299498 --- /dev/null +++ b/LocalPLaylistGenerator.py @@ -0,0 +1,50 @@ + +from pathlib import Path + +def generate_playlists(local_library, library_path, remote_playlists, link_pattern, output_dir): + output_dir = Path(output_dir) + output_playlists = [] + + remote_to_local_map = {} + + for local_id, links in link_pattern.items(): + if len(links['items']): + remote_to_local_map[links['items'][0]] = local_id + + for name, tracks in remote_playlists.items(): + local_tracks = [] + + for track in tracks: + track = track["track"] + path = "error" + error = "" + if track is None: + error = f"error: track is none" + elif track["id"] in remote_to_local_map: + path = local_library[remote_to_local_map[track["id"]]]["path"] + else: + error = f"error: not_found - {track["name"]}" + + if error != "": + local_tracks.append("# " + error) + else: + path = (output_dir / ".." ).resolve() / Path(path) + local_tracks.append(path) + + pl = { + "name": name, + "tracks": local_tracks, + } + + output_playlists.append(pl) + + + for pl in output_playlists: + output_dir.mkdir(parents=True, exist_ok=True) + + playlist_path = Path(output_dir) / f"{pl['name']}.m3u" + + with (playlist_path.open("w", encoding="utf-8") as f): + for track in pl["tracks"]: + f.write(str(track)) + f.write("\n") \ No newline at end of file diff --git a/TODO b/TODO new file mode 100644 index 0000000..f285a22 --- /dev/null +++ b/TODO @@ -0,0 +1,7 @@ +skip playlists that are not created by user + +create one big playlist with the order of the spotify and dont skip missing + +generate percentage like hierarchy of the missing items + +generate detailed log of the missing items and search results diff --git a/main.py b/main.py index 9709e93..10d378a 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,19 @@ from DataBase import * +from LocalLibraryIndexer import update_local_library +from LinkerPatternGenerator import run_pattern_generators +from LocalPLaylistGenerator import generate_playlists + from SpotifyWebAPI import fetch_data, update_access_token import cmd +local_library_path = "/home/auser/Music/" +rel_autogen_dir = "0autogen" + playlists = get_data('playlistTracks') top_artists = get_data('top_artists') top_tracks = get_data('top_tracks') user_tracks = get_data('tracks') -link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')] +link_pattern = get_data('link_pattern_spotify') local_library = get_data("local_library") user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} @@ -109,33 +116,20 @@ def print_link_stats(link_pattern): print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") index += 1 - return - print(f"\nUnresolved tracks from the local: {len(unresolved_local)}") - index = 0 - for link in unresolved_local: - local_track = local_library[link] - print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'") - index += 1 - class Interpreter(cmd.Cmd): intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" prompt = "(spotify) " - def do_link_patterns(self, arg): + def do_list_stat_vs_types(self, arg): """prints available linking patterns""" print(f"0 - fuzzy tags") print(f"1 - spotify") print(f"2 - fuzzy") - def do_link_pattern(self, arg): - if not len(arg.split()): - print("expected name of the pattern") - return - - """prints available linking patterns""" - pattern_id = int(arg.split()[0]) - print_link_stats(link_patterns[pattern_id]) + def do_stat_local_library_vs_spotify(self, arg): + """compares local library and remote with the link pattern generated by the generate_pattern_command""" + print_link_stats(link_pattern) def do_playlists(self, arg): """prints user playlists""" @@ -168,6 +162,24 @@ class Interpreter(cmd.Cmd): """Print tracks""" print_tracks() + def do_generate_local_playlists(self, arg): + """generates local playlists based on the vs comparison type""" + generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir) + + + def do_index_local_library(self, arg): + """find local tracks and fetches the data from them for further analysis""" + global local_library + update_local_library(local_library_path) + local_library = get_data("local_library") + + def do_generate_vs_stat(self, arg): + """compares local and remote library and generates the vs stats""" + + global link_pattern + run_pattern_generators() + link_pattern = get_data('link_pattern_spotify') + def do_fetch(self, arg): """Fetch all spotify data""" try: @@ -184,6 +196,31 @@ class Interpreter(cmd.Cmd): except ValueError: print("Can not fetch the data.") + def do_generate_todo(self, arg): + """generates todos for the pirates""" + print("sorry not implemented yet.") + + def do_run_all_default(self, arg): + """runs general flow""" + + local_lib_path = "/home/auser/Music/" + print("Fetching spotify data...") + self.do_fetch([]) + + print("Fetching local data...") + self.do_index_local_library(f"{local_lib_path}/organized") + + print("Generating vs patterns...") + self.do_generate_vs_stat("0") + + print("Generating todos...") + self.do_generate_todo(f"{local_lib_path}") + + print("Generating local playlists...") + self.do_generate_local_playlists(f"{local_lib_path}") + + return True + def do_exit(self, arg): """Exit the command loop: exit""" print("Goodbye!") From a620e7a4f565e0f795af1b2450416294d6b043e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sun, 4 Jan 2026 19:25:42 +0300 Subject: [PATCH 05/18] refactor initial not fully recovered --- Env.py | 7 + main.py | 241 +++--------------- env => old/env | 0 old/main.py | 234 +++++++++++++++++ src/Library.py | 159 ++++++++++++ src/LibrarySerializer.py | 199 +++++++++++++++ .../MapLocalToSpotify.py | 61 ++--- src/StatGenerator.py | 42 +++ DataBase.py => src/helpers.py | 16 +- .../itunes/ParseItunesToJson.py | 0 .../local/LocalLibraryIndexer.py | 14 +- .../local/LocalPLaylistGenerator.py | 0 src/local/ParseLocal.py | 40 +++ src/spotify/ParseSpotify.py | 139 ++++++++++ .../spotify/SpotifyAuthenticator.py | 22 +- .../spotify/SpotifyWebAPI.py | 18 +- 16 files changed, 925 insertions(+), 267 deletions(-) create mode 100644 Env.py rename env => old/env (100%) create mode 100644 old/main.py create mode 100644 src/Library.py create mode 100644 src/LibrarySerializer.py rename LinkerPatternGenerator.py => src/MapLocalToSpotify.py (71%) create mode 100644 src/StatGenerator.py rename DataBase.py => src/helpers.py (85%) rename ParseItunesToJson.py => src/itunes/ParseItunesToJson.py (100%) rename LocalLibraryIndexer.py => src/local/LocalLibraryIndexer.py (95%) rename LocalPLaylistGenerator.py => src/local/LocalPLaylistGenerator.py (100%) create mode 100644 src/local/ParseLocal.py create mode 100644 src/spotify/ParseSpotify.py rename SpotifyAuthenticator.py => src/spotify/SpotifyAuthenticator.py (88%) rename SpotifyWebAPI.py => src/spotify/SpotifyWebAPI.py (91%) diff --git a/Env.py b/Env.py new file mode 100644 index 0000000..6549c6b --- /dev/null +++ b/Env.py @@ -0,0 +1,7 @@ +from pathlib import Path + +local_library_path = Path("/home/auser/Music/") +work_dir = Path("./work_dir/new") + +client_id = '***REDACTED***' +client_secret = "***REDACTED***" diff --git a/main.py b/main.py index 10d378a..32682e3 100644 --- a/main.py +++ b/main.py @@ -1,234 +1,77 @@ -from DataBase import * -from LocalLibraryIndexer import update_local_library -from LinkerPatternGenerator import run_pattern_generators -from LocalPLaylistGenerator import generate_playlists -from SpotifyWebAPI import fetch_data, update_access_token import cmd -local_library_path = "/home/auser/Music/" -rel_autogen_dir = "0autogen" +from Env import client_id, client_secret, local_library_path +from Env import work_dir -playlists = get_data('playlistTracks') -top_artists = get_data('top_artists') -top_tracks = get_data('top_tracks') -user_tracks = get_data('tracks') -link_pattern = get_data('link_pattern_spotify') -local_library = get_data("local_library") -user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} +from src.LibrarySerializer import LibrarySaver, LibraryLoader +from src.spotify.SpotifyWebAPI import fetch_all +from src.spotify.ParseSpotify import Parser -def get_playlist_names(pl): - return [name for name, items in pl.items()] +from src.local.LocalLibraryIndexer import update_local_library +from src.local.ParseLocal import LocalParser - -def print_playlist_tracks(pl_id: int): - pls = list(playlists) - name = pls[pl_id] - pl = playlists[name] - print(name) - for track in pl: - track_data = track['track'] - track_name = track_data['name'] - artists = get_artists_str(track_data['artists']) - print(f" '{track_name}' - '{artists}' ") - - -def print_top_artists(): - for artist in top_artists: - name = artist['name'] - print(f" '{name}'") - - -def print_top_tracks(): - for track in top_tracks: - track_name = track['name'] - artists = get_artists_str(track['artists']) - print(f" '{track_name}' - '{artists}' ") - - -def print_stats(): - print(f" tracks - {len(user_tracks)}") - print(f" playlists - {len(playlists)}") - - for name, tracks in playlists.items(): - print(f" '{name}' - {len(tracks)}") - - print(f" top tracks - {len(top_tracks)}") - print(f" top artists - {len(top_artists)}") - - -def print_tracks(): - track_idx = 0 - for trackItem in user_tracks: - track = trackItem['track'] - print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'") - track_idx += 1 - - -def sort_remote_tracks_by_popularity(track_ids): - out = [] - for top_track in top_tracks: - if top_track['id'] in track_ids: - out.append(top_track['id']) - return out - - -def print_link_stats(link_pattern): - resolved_reversed = {} - resolved = [] - unresolved_local = [] - unresolved_remote = [] - - for local_id, links in link_pattern.items(): - if len(links['items']): - resolved.append(local_id) - resolved_reversed[links['items'][0]] = local_id - else: - unresolved_local.append(local_id) - - for track in user_tracks: - track_id = track['track']['id'] - if track_id not in resolved_reversed: - unresolved_remote.append(track_id) - - print(f"Local Tracks: {len(local_library)}") - print(f"Remote Tracks: {len(user_tracks)}") - - unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) - unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] - - print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") - index = 0 - for remote_id in unresolved_remote: - remote_track = user_tracks_by_id[remote_id] - print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") - index += 1 - - print(f"\nResolved tracks: {len(resolved)}") - index = 0 - for link in resolved: - remote_id = link_pattern[link]['items'][0] - local_track = local_library[link] - remote_track = user_tracks_by_id[remote_id] - - print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') - print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") - index += 1 +from src.MapLocalToSpotify import map_local_to_spotify class Interpreter(cmd.Cmd): intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" - prompt = "(spotify) " + prompt = "(kuw) " - def do_list_stat_vs_types(self, arg): - """prints available linking patterns""" - print(f"0 - fuzzy tags") - print(f"1 - spotify") - print(f"2 - fuzzy") + spotify_library = None + local_library = None - def do_stat_local_library_vs_spotify(self, arg): - """compares local library and remote with the link pattern generated by the generate_pattern_command""" - print_link_stats(link_pattern) - - def do_playlists(self, arg): - """prints user playlists""" - pl_id = 0 - for pl in get_playlist_names(playlists): - print(f"{pl_id} - {pl}") - pl_id += 1 - - def do_playlist_tracks(self, arg): - """prints playlist [name]""" - try: - pl_id = int(arg.split()[0]) - print_playlist_tracks(pl_id) - except IndexError: - print("Invalid input") - - def do_top_artists(self, arg): - """Top artists""" - print_top_artists() - - def do_top_tracks(self, arg): - """Top tracks""" - print_top_tracks() - - def do_stat(self, arg): - """Print stats""" - print_stats() - - def do_tracks(self, arg): - """Print tracks""" - print_tracks() - - def do_generate_local_playlists(self, arg): - """generates local playlists based on the vs comparison type""" - generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir) + @staticmethod + def do_fetch_spotify(self): + fetch_all(client_id, client_secret, work_dir) - def do_index_local_library(self, arg): - """find local tracks and fetches the data from them for further analysis""" - global local_library - update_local_library(local_library_path) - local_library = get_data("local_library") + @staticmethod + def do_fetch_local(): + update_local_library(local_library_path, work_dir) - def do_generate_vs_stat(self, arg): - """compares local and remote library and generates the vs stats""" - global link_pattern - run_pattern_generators() - link_pattern = get_data('link_pattern_spotify') + def do_parse_spotify_library(self): + parser = Parser() + library_parsed = parser.parse(work_dir / "spotify" / "playlistTracks.json", work_dir / "spotify" / "tracks.json") + LibrarySaver(library_parsed).save(work_dir / "spotify_library.json") - def do_fetch(self, arg): - """Fetch all spotify data""" - try: - update_access_token() - fetch_data() + def do_parse_local_library(self): + parser = LocalParser() + library_parsed = parser.parse(work_dir / "local" / "local_library.json") + LibrarySaver(library_parsed).save(work_dir / "local_library.json") - global playlists, top_artists, top_tracks, user_tracks + def do_load_libraries(self): + self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json") + self.local_library = LibraryLoader().load(work_dir / "local_library.json") - playlists = get_data('playlistTracks') - top_artists = get_data('top_artists') - top_tracks = get_data('top_tracks') - user_tracks = get_data('tracks') + def do_map_local_to_spotify(self): + map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir) - except ValueError: - print("Can not fetch the data.") - def do_generate_todo(self, arg): - """generates todos for the pirates""" - print("sorry not implemented yet.") + def do_run(self, arg): + # self.do_fetch_local() + # self.do_fetch_spotify() - def do_run_all_default(self, arg): - """runs general flow""" + # self.do_parse_local_library() + # self.do_parse_spotify_library() - local_lib_path = "/home/auser/Music/" - print("Fetching spotify data...") - self.do_fetch([]) + self.do_load_libraries() - print("Fetching local data...") - self.do_index_local_library(f"{local_lib_path}/organized") - - print("Generating vs patterns...") - self.do_generate_vs_stat("0") - - print("Generating todos...") - self.do_generate_todo(f"{local_lib_path}") - - print("Generating local playlists...") - self.do_generate_local_playlists(f"{local_lib_path}") + self.do_map_local_to_spotify() return True + @staticmethod def do_exit(self, arg): - """Exit the command loop: exit""" - print("Goodbye!") return True if __name__ == '__main__': try: - Interpreter().cmdloop() + Interpreter().do_run("") + # Interpreter().cmdloop() + except KeyboardInterrupt as kb: - print("process terminated") \ No newline at end of file + print("process terminated") diff --git a/env b/old/env similarity index 100% rename from env rename to old/env diff --git a/old/main.py b/old/main.py new file mode 100644 index 0000000..10d378a --- /dev/null +++ b/old/main.py @@ -0,0 +1,234 @@ +from DataBase import * +from LocalLibraryIndexer import update_local_library +from LinkerPatternGenerator import run_pattern_generators +from LocalPLaylistGenerator import generate_playlists + +from SpotifyWebAPI import fetch_data, update_access_token +import cmd + +local_library_path = "/home/auser/Music/" +rel_autogen_dir = "0autogen" + +playlists = get_data('playlistTracks') +top_artists = get_data('top_artists') +top_tracks = get_data('top_tracks') +user_tracks = get_data('tracks') +link_pattern = get_data('link_pattern_spotify') +local_library = get_data("local_library") +user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} + + +def get_playlist_names(pl): + return [name for name, items in pl.items()] + + +def print_playlist_tracks(pl_id: int): + pls = list(playlists) + name = pls[pl_id] + pl = playlists[name] + print(name) + for track in pl: + track_data = track['track'] + track_name = track_data['name'] + artists = get_artists_str(track_data['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_top_artists(): + for artist in top_artists: + name = artist['name'] + print(f" '{name}'") + + +def print_top_tracks(): + for track in top_tracks: + track_name = track['name'] + artists = get_artists_str(track['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_stats(): + print(f" tracks - {len(user_tracks)}") + print(f" playlists - {len(playlists)}") + + for name, tracks in playlists.items(): + print(f" '{name}' - {len(tracks)}") + + print(f" top tracks - {len(top_tracks)}") + print(f" top artists - {len(top_artists)}") + + +def print_tracks(): + track_idx = 0 + for trackItem in user_tracks: + track = trackItem['track'] + print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'") + track_idx += 1 + + +def sort_remote_tracks_by_popularity(track_ids): + out = [] + for top_track in top_tracks: + if top_track['id'] in track_ids: + out.append(top_track['id']) + return out + + +def print_link_stats(link_pattern): + resolved_reversed = {} + resolved = [] + unresolved_local = [] + unresolved_remote = [] + + for local_id, links in link_pattern.items(): + if len(links['items']): + resolved.append(local_id) + resolved_reversed[links['items'][0]] = local_id + else: + unresolved_local.append(local_id) + + for track in user_tracks: + track_id = track['track']['id'] + if track_id not in resolved_reversed: + unresolved_remote.append(track_id) + + print(f"Local Tracks: {len(local_library)}") + print(f"Remote Tracks: {len(user_tracks)}") + + unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) + unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] + + print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") + index = 0 + for remote_id in unresolved_remote: + remote_track = user_tracks_by_id[remote_id] + print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + print(f"\nResolved tracks: {len(resolved)}") + index = 0 + for link in resolved: + remote_id = link_pattern[link]['items'][0] + local_track = local_library[link] + remote_track = user_tracks_by_id[remote_id] + + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') + print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + +class Interpreter(cmd.Cmd): + intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" + prompt = "(spotify) " + + def do_list_stat_vs_types(self, arg): + """prints available linking patterns""" + print(f"0 - fuzzy tags") + print(f"1 - spotify") + print(f"2 - fuzzy") + + def do_stat_local_library_vs_spotify(self, arg): + """compares local library and remote with the link pattern generated by the generate_pattern_command""" + print_link_stats(link_pattern) + + def do_playlists(self, arg): + """prints user playlists""" + pl_id = 0 + for pl in get_playlist_names(playlists): + print(f"{pl_id} - {pl}") + pl_id += 1 + + def do_playlist_tracks(self, arg): + """prints playlist [name]""" + try: + pl_id = int(arg.split()[0]) + print_playlist_tracks(pl_id) + except IndexError: + print("Invalid input") + + def do_top_artists(self, arg): + """Top artists""" + print_top_artists() + + def do_top_tracks(self, arg): + """Top tracks""" + print_top_tracks() + + def do_stat(self, arg): + """Print stats""" + print_stats() + + def do_tracks(self, arg): + """Print tracks""" + print_tracks() + + def do_generate_local_playlists(self, arg): + """generates local playlists based on the vs comparison type""" + generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir) + + + def do_index_local_library(self, arg): + """find local tracks and fetches the data from them for further analysis""" + global local_library + update_local_library(local_library_path) + local_library = get_data("local_library") + + def do_generate_vs_stat(self, arg): + """compares local and remote library and generates the vs stats""" + + global link_pattern + run_pattern_generators() + link_pattern = get_data('link_pattern_spotify') + + def do_fetch(self, arg): + """Fetch all spotify data""" + try: + update_access_token() + fetch_data() + + global playlists, top_artists, top_tracks, user_tracks + + playlists = get_data('playlistTracks') + top_artists = get_data('top_artists') + top_tracks = get_data('top_tracks') + user_tracks = get_data('tracks') + + except ValueError: + print("Can not fetch the data.") + + def do_generate_todo(self, arg): + """generates todos for the pirates""" + print("sorry not implemented yet.") + + def do_run_all_default(self, arg): + """runs general flow""" + + local_lib_path = "/home/auser/Music/" + print("Fetching spotify data...") + self.do_fetch([]) + + print("Fetching local data...") + self.do_index_local_library(f"{local_lib_path}/organized") + + print("Generating vs patterns...") + self.do_generate_vs_stat("0") + + print("Generating todos...") + self.do_generate_todo(f"{local_lib_path}") + + print("Generating local playlists...") + self.do_generate_local_playlists(f"{local_lib_path}") + + return True + + def do_exit(self, arg): + """Exit the command loop: exit""" + print("Goodbye!") + return True + + +if __name__ == '__main__': + try: + Interpreter().cmdloop() + except KeyboardInterrupt as kb: + print("process terminated") \ No newline at end of file diff --git a/src/Library.py b/src/Library.py new file mode 100644 index 0000000..dc2efa3 --- /dev/null +++ b/src/Library.py @@ -0,0 +1,159 @@ +from __future__ import annotations +import json +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Any + + +# ========================= +# Core models +# ========================= + +class Track: + def __init__( + self, + *, + id: str, + title: str, + artists: List[str], + album: Optional[str] = None, + duration_ms: Optional[int] = None, + added_at: Optional[datetime] = None, + spotify: Optional["SpotifyData"] = None, + local: Optional["LocalData"] = None, + ): + self.id = id + self.title = title + self.artists = artists + self.album = album + self.duration_ms = duration_ms + self.added_at = added_at + self.spotify = spotify + self.local = local + + +class Playlist: + def __init__( + self, + *, + name: str, + tracks: Optional[List[Track]] = None, + created_at: Optional[datetime] = None, + description: Optional[str] = None, + spotify: Optional["SpotifyData"] = None, + ): + self.name = name + self.tracks = tracks or [] + self.created_at = created_at + self.description = description + self.spotify = spotify + + +class Album: + def __init__( + self, + *, + title: str, + release_date: Optional[str] = None, + tracks: Optional[List[Track]] = None, + spotify: Optional["SpotifyData"] = None, + ): + self.title = title + self.release_date = release_date + self.tracks = tracks or [] + self.spotify = spotify + + +class TrackList: + def __init__(self, *, tracks: Optional[List[Track]] = None): + self.tracks = tracks or [] + + def total_duration_ms(self) -> int: + return sum(t.duration_ms or 0 for t in self.tracks) + + +# ========================= +# Spotify models +# ========================= + +class SpotifyArtist: + def __init__(self, *, id: str, name: str, uri: str, url: str): + self.id = id + self.name = name + self.uri = uri + self.url = url + + +class SpotifyAlbum: + def __init__( + self, + *, + id: str, + name: str, + release_date: str, + total_tracks: int, + uri: str, + url: str, + ): + self.id = id + self.name = name + self.release_date = release_date + self.total_tracks = total_tracks + self.uri = uri + self.url = url + + +class SpotifyData: + def __init__( + self, + *, + id: str, + uri: str, + url: str, + href: str, + file_path: Optional[Path] = None, + popularity: Optional[int] = None, + is_playable: Optional[bool] = None, + is_local: Optional[bool] = None, + artists: Optional[List[SpotifyArtist]] = None, + album: Optional[SpotifyAlbum] = None, + external_ids: Optional[dict[str, str]] = None, + ): + self.id = id + self.uri = uri + self.url = url + self.href = href + self.file_path = file_path + self.popularity = popularity + self.is_playable = is_playable + self.is_local = is_local + self.artists = artists or [] + self.album = album + self.external_ids = external_ids or {} + +class LocalData: + def __init__( + self, + *, + path: str, + ): + self.path = path + +# ========================= +# Library container +# ========================= + +class Library: + def __init__( + self, + *, + playlists: Optional[List[Playlist]] = None, + tracks: Optional[List[Track]] = None, + tracklists: Optional[List[TrackList]] = None, + albums: Optional[List[Album]] = None, + ): + self.playlists = playlists or [] + self.tracks = tracks or [] + self.tracklists = tracklists or [] + self.albums = albums or [] + diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py new file mode 100644 index 0000000..06ea57b --- /dev/null +++ b/src/LibrarySerializer.py @@ -0,0 +1,199 @@ + +from src.Library import * + +import json +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Any + +def _dt_to_str(dt: Optional[datetime]) -> Optional[str]: + return dt.isoformat() if dt else None + + +def _str_to_dt(s: Optional[str]) -> Optional[datetime]: + return datetime.fromisoformat(s) if s else None + + +def _path_to_str(p: Optional[Path]) -> Optional[str]: + return str(p) if p else None + + +def _str_to_path(s: Optional[str]) -> Optional[Path]: + return Path(s) if s else None + + +class LibrarySaver: + def __init__(self, library: Library): + self.library = library + + def save(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + json.dump(self.to_dict(), f, indent=2, ensure_ascii=False) + + def to_dict(self) -> dict[str, Any]: + return { + "tracks": [self._track_to_dict(t) for t in self.library.tracks], + "playlists": [self._playlist_to_dict(p) for p in self.library.playlists], + "tracklists": [self._tracklist_to_dict(tl) for tl in self.library.tracklists], + "albums": [self._album_to_dict(a) for a in self.library.albums], + } + + def _track_to_dict(self, t: Track) -> dict[str, Any]: + return { + "id": t.id, + "title": t.title, + "artists": t.artists, + "album": t.album, + "duration_ms": t.duration_ms, + "added_at": _dt_to_str(t.added_at), + "spotify": self._spotify_to_dict(t.spotify), + "local": self._local_to_dict(t.local), + } + + def _playlist_to_dict(self, p: Playlist) -> dict[str, Any]: + return { + "name": p.name, + "tracks": [t.id for t in p.tracks], + "created_at": _dt_to_str(p.created_at), + "description": p.description, + "spotify": self._spotify_to_dict(p.spotify), + } + + def _tracklist_to_dict(self, tl: TrackList) -> dict[str, Any]: + return {"tracks": [t.id for t in tl.tracks]} + + def _album_to_dict(self, a: Album) -> dict[str, Any]: + return { + "title": a.title, + "release_date": a.release_date, + "tracks": [t.id for t in a.tracks], + "spotify": self._spotify_to_dict(a.spotify), + } + + def _local_to_dict(self, local): + if not local: + return None + return {"path": local.path} + + def _spotify_to_dict(self, s: Optional[SpotifyData]) -> Optional[dict[str, Any]]: + if not s: + return None + return { + "id": s.id, + "uri": s.uri, + "url": s.url, + "href": s.href, + "file_path": _path_to_str(s.file_path), + "popularity": s.popularity, + "is_playable": s.is_playable, + "is_local": s.is_local, + "external_ids": s.external_ids, + "artists": [{"id": a.id, "name": a.name, "uri": a.uri, "url": a.url} for a in s.artists], + "album": { + "id": s.album.id, + "name": s.album.name, + "release_date": s.album.release_date, + "total_tracks": s.album.total_tracks, + "uri": s.album.uri, + "url": s.album.url, + } if s.album else None, + } + + +class LibraryLoader: + def __init__(self): + # Map of track id -> Track object + self.track_map: dict[str, Track] = {} + + def load(self, path: Path) -> Library: + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + + # Load tracks first + tracks = [] + for t in data.get("tracks", []): + track = self._track_from_dict(t) + tracks.append(track) + self.track_map[track.id] = track + + # Load playlists using track_map + playlists = [] + for p in data.get("playlists", []): + pl_tracks = [] + for tid in p.get("tracks", []): + if tid not in self.track_map: + raise KeyError(f"Track ID '{tid}' not found in track map") + pl_tracks.append(self.track_map[tid]) + + playlists.append(Playlist( + name=p["name"], + tracks=pl_tracks, + created_at=_str_to_dt(p.get("created_at")), + description=p.get("description"), + spotify=self._spotify_from_dict(p.get("spotify")), + )) + + + # Load tracklists + tracklists = [] + for tl in data.get("tracklists", []): + tl_tracks = [] + for tid in tl.get("tracks", []): + if tid not in self.track_map: + raise KeyError(f"Track ID '{tid}' not found in track map (tracklist)") + tl_tracks.append(self.track_map[tid]) + tracklists.append(TrackList(tracks=tl_tracks)) + + # Load albums + albums = [] + for a in data.get("albums", []): + alb_tracks = [] + for tid in a.get("tracks", []): + if tid not in self.track_map: + raise KeyError(f"Track ID '{tid}' not found in track map (album '{a.get('title')}')") + alb_tracks.append(self.track_map[tid]) + albums.append(Album( + title=a["title"], + release_date=a.get("release_date"), + tracks=alb_tracks, + spotify=self._spotify_from_dict(a.get("spotify")), + )) + + return Library(playlists=playlists, tracks=tracks, tracklists=tracklists, albums=albums) + + def _track_from_dict(self, d: dict[str, Any]) -> Track: + return Track( + id=d["id"], + title=d["title"], + artists=d["artists"], + album=d.get("album"), + duration_ms=d.get("duration_ms"), + added_at=_str_to_dt(d.get("added_at")), + spotify=self._spotify_from_dict(d.get("spotify")), + local=self._local_from_dict(d.get("local")), + ) + + def _local_from_dict(self, d): + if not d: + return None + return LocalData( + path=d["path"], + ) + + def _spotify_from_dict(self, d: Optional[dict[str, Any]]) -> Optional[SpotifyData]: + if not d: + return None + return SpotifyData( + id=d["id"], + uri=d["uri"], + url=d["url"], + href=d["href"], + file_path=_str_to_path(d.get("file_path")), + popularity=d.get("popularity"), + is_playable=d.get("is_playable"), + is_local=d.get("is_local"), + external_ids=d.get("external_ids"), + artists=[SpotifyArtist(**a) for a in d.get("artists", [])], + album=SpotifyAlbum(**d["album"]) if d.get("album") else None, + ) diff --git a/LinkerPatternGenerator.py b/src/MapLocalToSpotify.py similarity index 71% rename from LinkerPatternGenerator.py rename to src/MapLocalToSpotify.py index 8576da8..dae5252 100644 --- a/LinkerPatternGenerator.py +++ b/src/MapLocalToSpotify.py @@ -1,47 +1,52 @@ -from DataBase import * -from SpotifyWebAPI import find_song, update_access_token from rapidfuzz import fuzz from tqdm import tqdm import requests -def spotify_pattern_generator(): - update_access_token() - - local_library = get_data("local_library") - spotify_library = get_data("tracks") +from src.Library import * +from src.helpers import * +from src.spotify.SpotifyWebAPI import find_song, update_access_token - def find_local_songs_on_spotify(local_lib): +def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir): + + set_workdir(output_dir) + + update_access_token(client_id, client_secret) + + def find_local_songs_on_spotify(): search_log = [] - total = len(local_library) + total = len(local_lib.tracks) found_tracks_map = {} with tqdm(total=total, desc='Searching local songs on spotify') as pbar: - for track_id, track in local_lib.items(): - parts = [track["name"], track["artist"], track.get("album")] + for track in local_lib.tracks: + + parts = [track.title, track.artists[0] if len(track.artists) else "", track.album] search_pattern = " ".join(p for p in parts if p) if search_pattern == "": - print(f"cannot generate search pattern for the track - {track["path"]}") + print(f"cannot generate search pattern for the track - {track.local.path}") pbar.update(1) continue - found_tracks = find_song(search_pattern)[0:1] - found_tracks_map[track_id] = found_tracks - search_log.append({"path" : track["path"], "search_pattern": search_pattern, "result" : found_tracks_map[track_id]}) + found_tracks = find_song(search_pattern)[0:10] + found_tracks_map[track.id] = found_tracks + search_log.append({"path" : track.local.path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]}) pbar.update(1) save_data(search_log, "spotify_search_log") + save_data(found_tracks_map, "spotify_found_map") return found_tracks_map - spotify_found_tracks = find_local_songs_on_spotify(local_library) - spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library} + spotify_found_tracks = find_local_songs_on_spotify() + + spotify_user_track = {track.id for track in spotify_lib.tracks} spotify_pattern = {} for local_id, found_items in spotify_found_tracks.items(): - spotify_pattern[local_id] = {"score": 0.0, "items": []} + spotify_pattern[local_id] = {"score": 1.0, "items": []} if not len(found_items): continue @@ -50,14 +55,8 @@ def spotify_pattern_generator(): spotify_track_id = found_item['id'] if spotify_track_id in spotify_user_track: spotify_pattern[local_id]['items'].append(spotify_track_id) - spotify_user_track[spotify_track_id]['local_mappings'].append(local_id) - for spotify_id, track_local_mappings in spotify_user_track.items(): - score = len(track_local_mappings['local_mappings']) - for local_id in track_local_mappings['local_mappings']: - spotify_pattern[local_id]['score'] = 1 / score - - save_data(spotify_pattern, "link_pattern_spotify") + save_data(spotify_pattern, "local_to_spotify_id_map") def fuzzy_pattern_generator(): @@ -126,7 +125,7 @@ def fuzzy_tag_pattern_generator(): ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) filtered_ratios = [] - threshold = 0.8 + threshold = 0.81 for ratio in ratios: if all([val > threshold for val in ratio[0]]): filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1])) @@ -138,12 +137,4 @@ def fuzzy_tag_pattern_generator(): mappings[local_track_id] = mapping pbar.update(1) - save_data(mappings, "link_pattern_tags") - - -def run_pattern_generators(type_id = 0): - spotify_pattern_generator() - - -if __name__ == "__main__": - run_pattern_generators() + save_data(mappings, "link_pattern_tags") \ No newline at end of file diff --git a/src/StatGenerator.py b/src/StatGenerator.py new file mode 100644 index 0000000..c40bebb --- /dev/null +++ b/src/StatGenerator.py @@ -0,0 +1,42 @@ + +def print_link_stats(link_pattern): + resolved_reversed = {} + resolved = [] + unresolved_local = [] + unresolved_remote = [] + + for local_id, links in link_pattern.items(): + if len(links['items']): + resolved.append(local_id) + resolved_reversed[links['items'][0]] = local_id + else: + unresolved_local.append(local_id) + + for track in user_tracks: + track_id = track['track']['id'] + if track_id not in resolved_reversed: + unresolved_remote.append(track_id) + + print(f"Local Tracks: {len(local_library)}") + print(f"Remote Tracks: {len(user_tracks)}") + + unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) + unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] + + print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") + index = 0 + for remote_id in unresolved_remote: + remote_track = user_tracks_by_id[remote_id] + print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + print(f"\nResolved tracks: {len(resolved)}") + index = 0 + for link in resolved: + remote_id = link_pattern[link]['items'][0] + local_track = local_library[link] + remote_track = user_tracks_by_id[remote_id] + + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') + print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 \ No newline at end of file diff --git a/DataBase.py b/src/helpers.py similarity index 85% rename from DataBase.py rename to src/helpers.py index c0f230b..8f81bbf 100644 --- a/DataBase.py +++ b/src/helpers.py @@ -3,6 +3,13 @@ import os data_dir = 'prefetched' +def set_workdir(dir): + global data_dir + data_dir = str(dir) + +def get_workdir(): + global data_dir + return data_dir class SongTags: def __init__(self, title='', artist='', album=''): @@ -40,10 +47,9 @@ def save_data(data, name): file_path = os.path.join(data_dir, f"{name}.json") with open(file_path, 'w') as file: json.dump(data, file, indent=4) - return f"Data saved to {name}.json." -def get_artists_str(artists_pack): - track_artists = [artist['name'] for artist in artists_pack] - artists = " ".join(track_artists) - return artists + print(f"Data saved to {file_path}.") + + + diff --git a/ParseItunesToJson.py b/src/itunes/ParseItunesToJson.py similarity index 100% rename from ParseItunesToJson.py rename to src/itunes/ParseItunesToJson.py diff --git a/LocalLibraryIndexer.py b/src/local/LocalLibraryIndexer.py similarity index 95% rename from LocalLibraryIndexer.py rename to src/local/LocalLibraryIndexer.py index eecdd58..d3a7ce2 100644 --- a/LocalLibraryIndexer.py +++ b/src/local/LocalLibraryIndexer.py @@ -1,18 +1,20 @@ import os import fnmatch - import mutagen - -from DataBase import * import eyed3 from mutagen.flac import FLAC +from src.helpers import * + music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') exclude_directories = ("*mary--*", "*example-word*") -def update_local_library(root_dir): +def update_local_library(root_dir, output_dir): + + set_workdir(output_dir / "local") + old_library = get_data("local_library") known_paths = {track['path']: track_id for track_id, track in old_library.items()} new_library = {} @@ -74,7 +76,3 @@ def update_local_library(root_dir): track['name'] = track['path'] save_data(new_library, "local_library") - - -if __name__ == "__main__": - update_local_library("/home/auser/Music/") diff --git a/LocalPLaylistGenerator.py b/src/local/LocalPLaylistGenerator.py similarity index 100% rename from LocalPLaylistGenerator.py rename to src/local/LocalPLaylistGenerator.py diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py new file mode 100644 index 0000000..4991ede --- /dev/null +++ b/src/local/ParseLocal.py @@ -0,0 +1,40 @@ +from datetime import datetime +from typing import Any, Iterable +from pathlib import Path +import json + +from src.Library import * + +class LocalParser: + def __init__(self): + pass + + def parse_track_item(self, id, track: dict[str, Any]) -> Track: + track = Track( + title=track["name"], + artists=[track.get("artist", [])], + album=track["album"] if track.get("album") else None, + duration_ms=None, + added_at=None, + spotify=None, + id=id, + local=LocalData(path=track["path"]) + ) + return track + + + def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]: + return [self.parse_track_item(id, track) for id, track in items.items()] + + def build_library_from_tracks(self, tracks_content_json: dict[str, list[dict[str, Any]]]) -> Library: + library = Library() + + tracks = self.parse_all_tracks(tracks_content_json) + library.tracklists.append(TrackList(tracks=list(tracks))) + library.tracks.extend(tracks) + + return library + + def parse(self, tracks_file: Path) -> Library: + tracks_content = json.loads(tracks_file.read_text()) + return self.build_library_from_tracks(tracks_content) diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py new file mode 100644 index 0000000..535caa0 --- /dev/null +++ b/src/spotify/ParseSpotify.py @@ -0,0 +1,139 @@ +from datetime import datetime +from typing import Any, Iterable +from pathlib import Path +import json + +from src.Library import * + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists + +class Parser: + def __init__(self): + # state: Spotify track ID -> Track object + self.track_map: dict[str, Track] = {} + + # ------------------------- + # Low-level helpers + # ------------------------- + @staticmethod + def _parse_spotify_datetime(value: str | None) -> datetime | None: + if not value: + return None + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + @staticmethod + def _parse_spotify_artist(a: dict[str, Any]) -> SpotifyArtist: + return SpotifyArtist( + id=a["id"], + name=a["name"], + uri=a["uri"], + url=a["external_urls"]["spotify"], + ) + + @staticmethod + def _parse_spotify_album(a: dict[str, Any]) -> SpotifyAlbum: + return SpotifyAlbum( + id=a["id"], + name=a["name"], + release_date=a.get("release_date"), + total_tracks=a.get("total_tracks", 0), + uri=a["uri"], + url=a["external_urls"]["spotify"], + ) + + def _parse_spotify_data(self, track_json: dict[str, Any]) -> SpotifyData: + return SpotifyData( + id=track_json["id"], + uri=track_json["uri"], + url=track_json["external_urls"]["spotify"], + href=track_json["href"], + popularity=track_json.get("popularity"), + is_playable=track_json.get("is_playable"), + is_local=track_json.get("is_local"), + external_ids=track_json.get("external_ids", {}), + artists=[self._parse_spotify_artist(a) for a in track_json.get("artists", [])], + album=self._parse_spotify_album(track_json["album"]) + if track_json.get("album") + else None, + ) + + # ------------------------- + # Track parsers + # ------------------------- + def parse_track_item(self, item: dict[str, Any]) -> Track: + """ + Parses a single track item and returns Track object. + Reuses existing Track from track_map if already parsed. + """ + t = item["track"] + track_id = t["id"] + + # return existing track if already parsed + if track_id in self.track_map: + return self.track_map[track_id] + + track = Track( + title=t["name"], + artists=[a["name"] for a in t.get("artists", [])], + album=t["album"]["name"] if t.get("album") else None, + duration_ms=t.get("duration_ms"), + added_at=self._parse_spotify_datetime(item.get("added_at")), + spotify=self._parse_spotify_data(t), + id=self._parse_spotify_data(t).id + ) + + self.track_map[track_id] = track + return track + + def parse_track_items(self, items: Iterable[dict[str, Any]]) -> list[Track]: + return [self.parse_track_item(i) for i in items if i.get("track")] + + # ------------------------- + # Playlist parsers + # ------------------------- + def parse_playlist_content(self, playlist_name: str, items: list[dict[str, Any]]) -> Playlist: + tracks = self.parse_track_items(items) + return Playlist(name=playlist_name, tracks=tracks) + + def parse_all_playlist_content(self, data: dict[str, list[dict[str, Any]]]) -> list[Playlist]: + playlists: list[Playlist] = [] + for name, items in data.items(): + playlists.append(self.parse_playlist_content(name, items)) + return playlists + + def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]: + return [self.parse_track_item(i) for i in items] + + # ------------------------- + # Library builder + # ------------------------- + def build_library_from_playlist_content( + self, playlist_content_json: dict[str, list[dict[str, Any]]], + tracks_content_json: dict[str, list[dict[str, Any]]] + ) -> Library: + library = Library() + + self.parse_all_tracks(tracks_content_json) + + library.tracklists.append(TrackList(tracks=list((self.track_map.values())))) + + library.playlists = self.parse_all_playlist_content(playlist_content_json) + library.tracks.extend(self.track_map.values()) + + return library + + # ------------------------- + # Convenience method + # ------------------------- + def parse( + self, + playlist_content_file: Path, + tracks_file: Path, + ) -> Library: + playlist_content = json.loads(playlist_content_file.read_text()) + tracks_content = json.loads(tracks_file.read_text()) + return self.build_library_from_playlist_content(playlist_content, tracks_content) diff --git a/SpotifyAuthenticator.py b/src/spotify/SpotifyAuthenticator.py similarity index 88% rename from SpotifyAuthenticator.py rename to src/spotify/SpotifyAuthenticator.py index 86705d0..5e27f7d 100644 --- a/SpotifyAuthenticator.py +++ b/src/spotify/SpotifyAuthenticator.py @@ -8,22 +8,11 @@ import threading import time import os.path -client_id = '***REDACTED***' +client_id = None client_secret = None token = None -def resolve_client_secret(): - global client_secret - if os.path.exists('secret'): - with open('secret', 'r') as file: - client_secret = file.readline().strip() - else: - client_secret = input("Enter the application secret: ") - with open('secret', 'w') as file: - file.write(client_secret) - - def retrieve_web_api_token(authorization_code): global token @@ -103,8 +92,13 @@ def open_authenticate_page(): webbrowser.open(auth_url) -def authenticate(): - resolve_client_secret() +def authenticate(id, secret): + + global client_secret + global client_id + + client_secret=secret + client_id=id redirect_server = threading.Thread(target=run_redirect_server) redirect_server.start() diff --git a/SpotifyWebAPI.py b/src/spotify/SpotifyWebAPI.py similarity index 91% rename from SpotifyWebAPI.py rename to src/spotify/SpotifyWebAPI.py index 71ec2f7..5ddc12f 100644 --- a/SpotifyWebAPI.py +++ b/src/spotify/SpotifyWebAPI.py @@ -1,8 +1,10 @@ import requests from PIL import Image from io import BytesIO -from SpotifyAuthenticator import authenticate -from DataBase import * + +from src.spotify.SpotifyAuthenticator import authenticate + +from src.helpers import * FETCH_STEP = 50 token = '' @@ -142,7 +144,7 @@ def save_image_from_url(url, name, image_dir="playlist_covers"): image = Image.open(BytesIO(response.content)) - directory = os.path.join(data_dir, image_dir) + directory = os.path.join(get_workdir(), image_dir) os.makedirs(directory, exist_ok=True) file_path = os.path.join(directory, f"{name}.jpg") print(f"Image saved {file_path}") @@ -169,9 +171,9 @@ def load_artworks(): load_playlist_covers() -def update_access_token(): +def update_access_token(id, secret): global token - token = authenticate() + token = authenticate(id, secret) def fetch_data(): @@ -184,7 +186,11 @@ def fetch_data(): load_artworks() +def fetch_all(id, secret, workdir): + set_workdir(workdir / "spotify") + update_access_token(id, secret) + fetch_data() if __name__ == "__main__": - update_access_token() + update_access_token("", "") fetch_data() From 5f4bbbaa592a92a9b94455793421aada76108ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sun, 4 Jan 2026 23:40:41 +0300 Subject: [PATCH 06/18] refactor 2 --- Flow.py | 65 ++++++++++ main.py | 86 ++----------- src/Library.py | 176 +++++-------------------- src/LibrarySerializer.py | 248 ++++++++++++++---------------------- src/MapLocalToSpotify.py | 6 +- src/local/ParseLocal.py | 151 ++++++++++++++++++---- src/spotify/ParseSpotify.py | 197 ++++++++++++++-------------- 7 files changed, 426 insertions(+), 503 deletions(-) create mode 100644 Flow.py diff --git a/Flow.py b/Flow.py new file mode 100644 index 0000000..bb35731 --- /dev/null +++ b/Flow.py @@ -0,0 +1,65 @@ + +from Env import client_id, client_secret, local_library_path +from Env import work_dir + +from src.LibrarySerializer import LibrarySaver, LibraryLoader + +from src.spotify.SpotifyWebAPI import fetch_all +from src.spotify.ParseSpotify import Parser + +from src.local.LocalLibraryIndexer import update_local_library +from src.local.ParseLocal import LocalParser + +from src.MapLocalToSpotify import map_local_to_spotify + + +class Flow: + spotify_library = None + local_library = None + + @staticmethod + def fetch_spotify(self): + fetch_all(client_id, client_secret, work_dir) + + @staticmethod + def fetch_local(): + update_local_library(local_library_path, work_dir) + + @staticmethod + def parse_spotify_library(): + parser = Parser() + + library_parsed = parser.parse( + work_dir / "spotify" / "playlistTracks.json", + work_dir / "spotify" / "tracks.json", + work_dir / "spotify" / "top_tracks.json", + work_dir / "spotify" / "top_artists.json", + ) + + LibrarySaver(library_parsed).save(work_dir / "spotify_library.json") + + @staticmethod + def parse_local_library(): + parser = LocalParser() + library_parsed = parser.parse(work_dir / "local" / "local_library.json") + LibrarySaver(library_parsed).save(work_dir / "local_library.json") + + def load_libraries(self): + self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json") + self.local_library = LibraryLoader().load(work_dir / "local_library.json") + + def map_local_to_spotify(self): + map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir) + + def run(self, arg): + self.fetch_local() + # self.do_fetch_spotify() + + self.parse_local_library() + self.parse_spotify_library() + + self.load_libraries() + + # self.map_local_to_spotify() + + return True \ No newline at end of file diff --git a/main.py b/main.py index 32682e3..98b4efa 100644 --- a/main.py +++ b/main.py @@ -1,77 +1,17 @@ -import cmd - -from Env import client_id, client_secret, local_library_path -from Env import work_dir - -from src.LibrarySerializer import LibrarySaver, LibraryLoader - -from src.spotify.SpotifyWebAPI import fetch_all -from src.spotify.ParseSpotify import Parser - -from src.local.LocalLibraryIndexer import update_local_library -from src.local.ParseLocal import LocalParser - -from src.MapLocalToSpotify import map_local_to_spotify - - -class Interpreter(cmd.Cmd): - intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" - prompt = "(kuw) " - - spotify_library = None - local_library = None - - @staticmethod - def do_fetch_spotify(self): - fetch_all(client_id, client_secret, work_dir) - - - @staticmethod - def do_fetch_local(): - update_local_library(local_library_path, work_dir) - - - def do_parse_spotify_library(self): - parser = Parser() - library_parsed = parser.parse(work_dir / "spotify" / "playlistTracks.json", work_dir / "spotify" / "tracks.json") - LibrarySaver(library_parsed).save(work_dir / "spotify_library.json") - - def do_parse_local_library(self): - parser = LocalParser() - library_parsed = parser.parse(work_dir / "local" / "local_library.json") - LibrarySaver(library_parsed).save(work_dir / "local_library.json") - - def do_load_libraries(self): - self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json") - self.local_library = LibraryLoader().load(work_dir / "local_library.json") - - def do_map_local_to_spotify(self): - map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir) - - - def do_run(self, arg): - # self.do_fetch_local() - # self.do_fetch_spotify() - - # self.do_parse_local_library() - # self.do_parse_spotify_library() - - self.do_load_libraries() - - self.do_map_local_to_spotify() - - return True - - @staticmethod - def do_exit(self, arg): - return True - +import Flow if __name__ == '__main__': - try: - Interpreter().do_run("") - # Interpreter().cmdloop() + flow = Flow.Flow() - except KeyboardInterrupt as kb: - print("process terminated") + # flow.fetch_spotify() + flow.fetch_local() + + flow.parse_spotify_library() + flow.parse_local_library() + + flow.load_libraries() + + flow.map_local_to_spotify() + + print("asd") diff --git a/src/Library.py b/src/Library.py index dc2efa3..c3287b8 100644 --- a/src/Library.py +++ b/src/Library.py @@ -1,159 +1,51 @@ from __future__ import annotations -import json + from datetime import datetime from pathlib import Path from typing import List, Optional, Any +class Artist: + def __init__(self): + self.id : str = None + self.title : str = None + self.tracks : List[Track] = [] -# ========================= -# Core models -# ========================= class Track: - def __init__( - self, - *, - id: str, - title: str, - artists: List[str], - album: Optional[str] = None, - duration_ms: Optional[int] = None, - added_at: Optional[datetime] = None, - spotify: Optional["SpotifyData"] = None, - local: Optional["LocalData"] = None, - ): - self.id = id - self.title = title - self.artists = artists - self.album = album - self.duration_ms = duration_ms - self.added_at = added_at - self.spotify = spotify - self.local = local - - -class Playlist: - def __init__( - self, - *, - name: str, - tracks: Optional[List[Track]] = None, - created_at: Optional[datetime] = None, - description: Optional[str] = None, - spotify: Optional["SpotifyData"] = None, - ): - self.name = name - self.tracks = tracks or [] - self.created_at = created_at - self.description = description - self.spotify = spotify + def __init__(self): + self.id : str = None + self.title : str = None + self.artists : List[Artist] = [] + self.album : Album = None + self.duration_ms : int = None + self.added_at : datetime = None + self.local_path : Path = None class Album: - def __init__( - self, - *, - title: str, - release_date: Optional[str] = None, - tracks: Optional[List[Track]] = None, - spotify: Optional["SpotifyData"] = None, - ): - self.title = title - self.release_date = release_date - self.tracks = tracks or [] - self.spotify = spotify + def __init__(self): + self.id : str = None + self.title : str = None + self.release_date : datetime = None + self.tracks : List[Track] = [] + self.artists : List[Artist] = [] -class TrackList: - def __init__(self, *, tracks: Optional[List[Track]] = None): - self.tracks = tracks or [] +class Playlist: + def __init__(self): + self.title : str = None + self.tracks : List[Track] = [] + self.created_at : datetime = None + self.description : str = None - def total_duration_ms(self) -> int: - return sum(t.duration_ms or 0 for t in self.tracks) - - -# ========================= -# Spotify models -# ========================= - -class SpotifyArtist: - def __init__(self, *, id: str, name: str, uri: str, url: str): - self.id = id - self.name = name - self.uri = uri - self.url = url - - -class SpotifyAlbum: - def __init__( - self, - *, - id: str, - name: str, - release_date: str, - total_tracks: int, - uri: str, - url: str, - ): - self.id = id - self.name = name - self.release_date = release_date - self.total_tracks = total_tracks - self.uri = uri - self.url = url - - -class SpotifyData: - def __init__( - self, - *, - id: str, - uri: str, - url: str, - href: str, - file_path: Optional[Path] = None, - popularity: Optional[int] = None, - is_playable: Optional[bool] = None, - is_local: Optional[bool] = None, - artists: Optional[List[SpotifyArtist]] = None, - album: Optional[SpotifyAlbum] = None, - external_ids: Optional[dict[str, str]] = None, - ): - self.id = id - self.uri = uri - self.url = url - self.href = href - self.file_path = file_path - self.popularity = popularity - self.is_playable = is_playable - self.is_local = is_local - self.artists = artists or [] - self.album = album - self.external_ids = external_ids or {} - -class LocalData: - def __init__( - self, - *, - path: str, - ): - self.path = path - -# ========================= -# Library container -# ========================= class Library: - def __init__( - self, - *, - playlists: Optional[List[Playlist]] = None, - tracks: Optional[List[Track]] = None, - tracklists: Optional[List[TrackList]] = None, - albums: Optional[List[Album]] = None, - ): - self.playlists = playlists or [] - self.tracks = tracks or [] - self.tracklists = tracklists or [] - self.albums = albums or [] + def __init__(self): + self.artists : List[Artist] = [] + self.playlists : List[Playlist] = [] + self.tracks : List[Track] = [] + self.albums : List[Album] = [] + self.liked_tracks : List[Track] = [] + self.top_tracks : List[Track]= [] + self.top_artists : List[Artist] = [] diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py index 06ea57b..bc968ea 100644 --- a/src/LibrarySerializer.py +++ b/src/LibrarySerializer.py @@ -26,174 +26,112 @@ class LibrarySaver: def __init__(self, library: Library): self.library = library + @staticmethod + def _artist_to_dict(artist: Artist) -> dict[str, Any]: + return { + "id": artist.id, + "title": artist.title, + "tracks": [track.id for track in artist.tracks], + } + + @staticmethod + def _track_to_dict(track: Track) -> dict[str, Any]: + return { + "id": track.id, + "title": track.title, + "artists": [artist.id for artist in track.artists], + "album": track.album.id, + "duration_ms": track.duration_ms, + "added_at": _dt_to_str(track.added_at), + "local_path": track.local_path, + } + + @staticmethod + def _playlist_to_dict(playlist: Playlist) -> dict[str, Any]: + return { + "title": playlist.title, + "tracks": [track.id for track in playlist.tracks], + "created_at": _dt_to_str(playlist.created_at), + "description": playlist.description, + } + + @staticmethod + def _album_to_dict(album: Album) -> dict[str, Any]: + return { + "id": album.id, + "title": album.title, + "release_date": album.release_date, + "tracks": [track.id for track in album.tracks], + "artists": [artist.id for artist in album.artists] + } + def save(self, path: Path) -> None: + + data = { + "tracks": [self._track_to_dict(track) for track in self.library.tracks], + "albums": [self._album_to_dict(album) for album in self.library.albums], + "artists": [self._artist_to_dict(artist) for artist in self.library.artists], + "playlists": [self._playlist_to_dict(playlist) for playlist in self.library.playlists], + "top_tracks": [track.id for track in self.library.top_tracks], + "top_artists": [artist.id for artist in self.library.top_artists], + "liked_tracks": [track.id for track in self.library.liked_tracks], + } + path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as f: - json.dump(self.to_dict(), f, indent=2, ensure_ascii=False) - - def to_dict(self) -> dict[str, Any]: - return { - "tracks": [self._track_to_dict(t) for t in self.library.tracks], - "playlists": [self._playlist_to_dict(p) for p in self.library.playlists], - "tracklists": [self._tracklist_to_dict(tl) for tl in self.library.tracklists], - "albums": [self._album_to_dict(a) for a in self.library.albums], - } - - def _track_to_dict(self, t: Track) -> dict[str, Any]: - return { - "id": t.id, - "title": t.title, - "artists": t.artists, - "album": t.album, - "duration_ms": t.duration_ms, - "added_at": _dt_to_str(t.added_at), - "spotify": self._spotify_to_dict(t.spotify), - "local": self._local_to_dict(t.local), - } - - def _playlist_to_dict(self, p: Playlist) -> dict[str, Any]: - return { - "name": p.name, - "tracks": [t.id for t in p.tracks], - "created_at": _dt_to_str(p.created_at), - "description": p.description, - "spotify": self._spotify_to_dict(p.spotify), - } - - def _tracklist_to_dict(self, tl: TrackList) -> dict[str, Any]: - return {"tracks": [t.id for t in tl.tracks]} - - def _album_to_dict(self, a: Album) -> dict[str, Any]: - return { - "title": a.title, - "release_date": a.release_date, - "tracks": [t.id for t in a.tracks], - "spotify": self._spotify_to_dict(a.spotify), - } - - def _local_to_dict(self, local): - if not local: - return None - return {"path": local.path} - - def _spotify_to_dict(self, s: Optional[SpotifyData]) -> Optional[dict[str, Any]]: - if not s: - return None - return { - "id": s.id, - "uri": s.uri, - "url": s.url, - "href": s.href, - "file_path": _path_to_str(s.file_path), - "popularity": s.popularity, - "is_playable": s.is_playable, - "is_local": s.is_local, - "external_ids": s.external_ids, - "artists": [{"id": a.id, "name": a.name, "uri": a.uri, "url": a.url} for a in s.artists], - "album": { - "id": s.album.id, - "name": s.album.name, - "release_date": s.album.release_date, - "total_tracks": s.album.total_tracks, - "uri": s.album.uri, - "url": s.album.url, - } if s.album else None, - } + json.dump(data, f, indent=2, ensure_ascii=False) class LibraryLoader: def __init__(self): - # Map of track id -> Track object self.track_map: dict[str, Track] = {} + self.album_map: dict[str, Album] = {} + self.artist_map: dict[str, Artist] = {} + + def load_track(self, data: dict[str, Any]) -> Track: + track = self.track_map.get(data.get("id"), Track()) + track.id = data.get("id") + track.title = data.get("title") + track.duration_ms = data.get("duration_ms") + track.added_at = _str_to_dt(data.get("added_at")) + track.local_path = data.get("local_path") + track.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")] + track.album = self.album_map.get(data.get("album"), Album()) + + self.track_map[track.id] = track + return track + + def load_artist(self, data: dict[str, Any]): + artist = self.artist_map.get(data.get("id"), Artist()) + artist.id = data.get("id") + artist.title = data.get("title") + artist.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")] + + self.artist_map[artist.id] = artist + return artist + + def load_album(self, data: dict[str, Any]): + album = self.album_map.get(data.get("id"), Album()) + album.id = data.get("id") + album.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")] + album.title = data.get("title") + album.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")] + + self.album_map[album.id] = album + return album def load(self, path: Path) -> Library: with path.open("r", encoding="utf-8") as f: data = json.load(f) - # Load tracks first - tracks = [] - for t in data.get("tracks", []): - track = self._track_from_dict(t) - tracks.append(track) - self.track_map[track.id] = track + library = Library() - # Load playlists using track_map - playlists = [] - for p in data.get("playlists", []): - pl_tracks = [] - for tid in p.get("tracks", []): - if tid not in self.track_map: - raise KeyError(f"Track ID '{tid}' not found in track map") - pl_tracks.append(self.track_map[tid]) + library.tracks = [self.load_track(track) for track in data.get("tracks")] + library.albums = [self.load_album(album) for album in data.get("albums")] + library.artists = [self.load_artist(artist) for artist in data.get("artists")] - playlists.append(Playlist( - name=p["name"], - tracks=pl_tracks, - created_at=_str_to_dt(p.get("created_at")), - description=p.get("description"), - spotify=self._spotify_from_dict(p.get("spotify")), - )) + library.top_artists = [self.artist_map[artist] for artist in data.get("top_artists")] + library.top_tracks = [self.track_map[track] for track in data.get("top_tracks")] + library.liked_tracks = [self.track_map[track] for track in data.get("liked_tracks")] - - # Load tracklists - tracklists = [] - for tl in data.get("tracklists", []): - tl_tracks = [] - for tid in tl.get("tracks", []): - if tid not in self.track_map: - raise KeyError(f"Track ID '{tid}' not found in track map (tracklist)") - tl_tracks.append(self.track_map[tid]) - tracklists.append(TrackList(tracks=tl_tracks)) - - # Load albums - albums = [] - for a in data.get("albums", []): - alb_tracks = [] - for tid in a.get("tracks", []): - if tid not in self.track_map: - raise KeyError(f"Track ID '{tid}' not found in track map (album '{a.get('title')}')") - alb_tracks.append(self.track_map[tid]) - albums.append(Album( - title=a["title"], - release_date=a.get("release_date"), - tracks=alb_tracks, - spotify=self._spotify_from_dict(a.get("spotify")), - )) - - return Library(playlists=playlists, tracks=tracks, tracklists=tracklists, albums=albums) - - def _track_from_dict(self, d: dict[str, Any]) -> Track: - return Track( - id=d["id"], - title=d["title"], - artists=d["artists"], - album=d.get("album"), - duration_ms=d.get("duration_ms"), - added_at=_str_to_dt(d.get("added_at")), - spotify=self._spotify_from_dict(d.get("spotify")), - local=self._local_from_dict(d.get("local")), - ) - - def _local_from_dict(self, d): - if not d: - return None - return LocalData( - path=d["path"], - ) - - def _spotify_from_dict(self, d: Optional[dict[str, Any]]) -> Optional[SpotifyData]: - if not d: - return None - return SpotifyData( - id=d["id"], - uri=d["uri"], - url=d["url"], - href=d["href"], - file_path=_str_to_path(d.get("file_path")), - popularity=d.get("popularity"), - is_playable=d.get("is_playable"), - is_local=d.get("is_local"), - external_ids=d.get("external_ids"), - artists=[SpotifyArtist(**a) for a in d.get("artists", [])], - album=SpotifyAlbum(**d["album"]) if d.get("album") else None, - ) + return library diff --git a/src/MapLocalToSpotify.py b/src/MapLocalToSpotify.py index dae5252..84e37c5 100644 --- a/src/MapLocalToSpotify.py +++ b/src/MapLocalToSpotify.py @@ -22,17 +22,17 @@ def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_ with tqdm(total=total, desc='Searching local songs on spotify') as pbar: for track in local_lib.tracks: - parts = [track.title, track.artists[0] if len(track.artists) else "", track.album] + parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title] search_pattern = " ".join(p for p in parts if p) if search_pattern == "": - print(f"cannot generate search pattern for the track - {track.local.path}") + print(f"cannot generate search pattern for the track - {track.local_path}") pbar.update(1) continue found_tracks = find_song(search_pattern)[0:10] found_tracks_map[track.id] = found_tracks - search_log.append({"path" : track.local.path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]}) + search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]}) pbar.update(1) save_data(search_log, "spotify_search_log") diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py index 4991ede..602e89e 100644 --- a/src/local/ParseLocal.py +++ b/src/local/ParseLocal.py @@ -1,40 +1,139 @@ -from datetime import datetime -from typing import Any, Iterable -from pathlib import Path import json +import re from src.Library import * +import re + +PROTECTED_ARTISTS = { + "Tyler, The Creator", + "The Good, The Bad & The Queen", +} + +def split_artists(artist_tag: str) -> list[str]: + if not artist_tag: + return [] + + placeholder_comma = "§COMMA§" + placeholder_amp = "§AMP§" + + protected_map = {} + + # Protect known artist names + for artist in PROTECTED_ARTISTS: + protected = ( + artist + .replace(",", placeholder_comma) + .replace("&", placeholder_amp) + ) + protected_map[protected] = artist + artist_tag = artist_tag.replace(artist, protected) + + # Split remaining separators + parts = re.split(r"[,&]", artist_tag) + + # Restore protected names and trim spaces + result = [] + for part in parts: + part = part.strip() + if not part: + continue + + part = ( + part + .replace(placeholder_comma, ",") + .replace(placeholder_amp, "&") + ) + + result.append(part) + + return result + + class LocalParser: def __init__(self): - pass - - def parse_track_item(self, id, track: dict[str, Any]) -> Track: - track = Track( - title=track["name"], - artists=[track.get("artist", [])], - album=track["album"] if track.get("album") else None, - duration_ms=None, - added_at=None, - spotify=None, - id=id, - local=LocalData(path=track["path"]) - ) - return track + self.track_map: dict[str, Track] = {} + self.artist_map: dict[str, Artist] = {} + self.album_map: dict[str, Album] = {} + self.library = Library() - def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]: - return [self.parse_track_item(id, track) for id, track in items.items()] + def parse_artists(self, data: str) -> List[Artist]: + if not data: + data = "unknown" - def build_library_from_tracks(self, tracks_content_json: dict[str, list[dict[str, Any]]]) -> Library: - library = Library() + data = split_artists(data) - tracks = self.parse_all_tracks(tracks_content_json) - library.tracklists.append(TrackList(tracks=list(tracks))) - library.tracks.extend(tracks) + artists = [] + + for artist_data in data: + name = artist_data + artist_id = name + + if artist_id in self.artist_map: + artists.append(self.artist_map[artist_id]) + continue + + artist = Artist() + artist.id = artist_id + artist.title = name + + self.artist_map[artist_id] = artist + self.library.artists.append(artist) + + artists.append(artist) + + return artists + + def parse_album(self, data: str) -> Album: + if not data: + data = "unknown" + + name = data + album_id = name + + if album_id in self.album_map: + return self.album_map[album_id] + + album = Album() + album.id = album_id + album.title = name + + self.album_map[album_id] = album + self.library.albums.append(album) + + return album + + def add_tracks_to_albums_and_artists(self): + for track_id, track in self.track_map.items(): + for artist in track.artists: + if track not in artist.tracks: + artist.tracks.append(track) + + if track not in track.album.tracks: + track.album.tracks.append(track) - return library def parse(self, tracks_file: Path) -> Library: tracks_content = json.loads(tracks_file.read_text()) - return self.build_library_from_tracks(tracks_content) + + for track_id, data in tracks_content.items(): + track = Track() + + track.id = track_id + track.title = data.get("name") + track.artists = self.parse_artists(data.get("artist")) + track.album = self.parse_album(data.get("album")) + track.local_path = data.get("path") + + self.track_map[track.id] = track + self.library.tracks.append(track) + + self.add_tracks_to_albums_and_artists() + + return self.library + + + def parse_track_item(self, id, track: dict[str, Any]) -> Track: + + return track diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py index 535caa0..49882cb 100644 --- a/src/spotify/ParseSpotify.py +++ b/src/spotify/ParseSpotify.py @@ -1,9 +1,10 @@ -from datetime import datetime +from src.Library import * + from typing import Any, Iterable -from pathlib import Path import json -from src.Library import * +from datetime import datetime +from pathlib import Path def get_artists_str(artists_pack): @@ -11,129 +12,117 @@ def get_artists_str(artists_pack): artists = " ".join(track_artists) return artists + class Parser: def __init__(self): - # state: Spotify track ID -> Track object self.track_map: dict[str, Track] = {} + self.artist_map: dict[str, Artist] = {} + self.album_map: dict[str, Album] = {} + self.library = Library() - # ------------------------- - # Low-level helpers - # ------------------------- - @staticmethod - def _parse_spotify_datetime(value: str | None) -> datetime | None: - if not value: - return None - return datetime.fromisoformat(value.replace("Z", "+00:00")) + def parse(self, playlists: Path, tracks: Path, top_tracks: Path, top_artists: Path) -> Library: + self.library = Library() - @staticmethod - def _parse_spotify_artist(a: dict[str, Any]) -> SpotifyArtist: - return SpotifyArtist( - id=a["id"], - name=a["name"], - uri=a["uri"], - url=a["external_urls"]["spotify"], - ) + self.parse_tracks(json.loads(tracks.read_text())) + self.parse_playlists(json.loads(playlists.read_text())) - @staticmethod - def _parse_spotify_album(a: dict[str, Any]) -> SpotifyAlbum: - return SpotifyAlbum( - id=a["id"], - name=a["name"], - release_date=a.get("release_date"), - total_tracks=a.get("total_tracks", 0), - uri=a["uri"], - url=a["external_urls"]["spotify"], - ) + self.parse_top_artists(json.loads(top_artists.read_text())) + self.parse_top_tracks(json.loads(top_tracks.read_text())) - def _parse_spotify_data(self, track_json: dict[str, Any]) -> SpotifyData: - return SpotifyData( - id=track_json["id"], - uri=track_json["uri"], - url=track_json["external_urls"]["spotify"], - href=track_json["href"], - popularity=track_json.get("popularity"), - is_playable=track_json.get("is_playable"), - is_local=track_json.get("is_local"), - external_ids=track_json.get("external_ids", {}), - artists=[self._parse_spotify_artist(a) for a in track_json.get("artists", [])], - album=self._parse_spotify_album(track_json["album"]) - if track_json.get("album") - else None, - ) + self.add_tracks_to_albums_and_artists() - # ------------------------- - # Track parsers - # ------------------------- - def parse_track_item(self, item: dict[str, Any]) -> Track: - """ - Parses a single track item and returns Track object. - Reuses existing Track from track_map if already parsed. - """ - t = item["track"] - track_id = t["id"] + return self.library + + def add_tracks_to_albums_and_artists(self): + for track_id, track in self.track_map.items(): + for artist in track.artists: + if track not in artist.tracks: + artist.tracks.append(track) + + if track not in track.album.tracks: + track.album.tracks.append(track) + + + def parse_playlists(self, data): + for name, tracks in data.items(): + playlist = Playlist() + playlist.title = name + for track in tracks: + track_data = track.get("track") + if track_data: + track_id = track_data.get("id") + playlist.tracks.append(self.parse_track(track_data, track.get("added_at"))) + + self.library.playlists.append(playlist) + + def parse_top_tracks(self, data): + for track in data: + if track.get("id") in self.track_map: + self.library.top_tracks.append(self.track_map.get(track.get("id"))) + + def parse_top_artists(self, data): + for artist in data: + if artist.get("id") in self.artist_map: + self.library.top_artists.append(self.artist_map.get(artist.get("id"))) + + def parse_tracks(self, data): + for track in data: + self.parse_track(track.get("track"), track.get("added_at")) + + for track in data: + self.library.liked_tracks.append(self.track_map.get(track.get("track").get("id"))) + + def parse_track(self, data: dict[str, Any], added_time : datetime) -> Track: + track_id = data.get("id") - # return existing track if already parsed if track_id in self.track_map: return self.track_map[track_id] - track = Track( - title=t["name"], - artists=[a["name"] for a in t.get("artists", [])], - album=t["album"]["name"] if t.get("album") else None, - duration_ms=t.get("duration_ms"), - added_at=self._parse_spotify_datetime(item.get("added_at")), - spotify=self._parse_spotify_data(t), - id=self._parse_spotify_data(t).id - ) + track = Track() self.track_map[track_id] = track + + track.id = track_id + track.title = data.get("name") + track.artists = [self.parse_artist(artist) for artist in data.get("artists")] + track.album = self.parse_album(data.get("album")) + track.duration_ms = data.get("duration_ms") + + self.library.tracks.append(track) + return track - def parse_track_items(self, items: Iterable[dict[str, Any]]) -> list[Track]: - return [self.parse_track_item(i) for i in items if i.get("track")] + def parse_album(self, data): + album_id = data.get("id") - # ------------------------- - # Playlist parsers - # ------------------------- - def parse_playlist_content(self, playlist_name: str, items: list[dict[str, Any]]) -> Playlist: - tracks = self.parse_track_items(items) - return Playlist(name=playlist_name, tracks=tracks) + if album_id in self.album_map: + return self.album_map[album_id] - def parse_all_playlist_content(self, data: dict[str, list[dict[str, Any]]]) -> list[Playlist]: - playlists: list[Playlist] = [] - for name, items in data.items(): - playlists.append(self.parse_playlist_content(name, items)) - return playlists + album = Album() - def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]: - return [self.parse_track_item(i) for i in items] + self.album_map[album_id] = album - # ------------------------- - # Library builder - # ------------------------- - def build_library_from_playlist_content( - self, playlist_content_json: dict[str, list[dict[str, Any]]], - tracks_content_json: dict[str, list[dict[str, Any]]] - ) -> Library: - library = Library() + album.id = data.get("id") + album.title = data.get("name") + album.artists = [self.parse_artist(artist) for artist in data.get("artists")] - self.parse_all_tracks(tracks_content_json) + self.library.albums.append(album) - library.tracklists.append(TrackList(tracks=list((self.track_map.values())))) + return album - library.playlists = self.parse_all_playlist_content(playlist_content_json) - library.tracks.extend(self.track_map.values()) + def parse_artist(self, data: dict[str, Any]) -> Artist: + artist_id = data.get("id") - return library + if artist_id in self.artist_map: + return self.artist_map[artist_id] - # ------------------------- - # Convenience method - # ------------------------- - def parse( - self, - playlist_content_file: Path, - tracks_file: Path, - ) -> Library: - playlist_content = json.loads(playlist_content_file.read_text()) - tracks_content = json.loads(tracks_file.read_text()) - return self.build_library_from_playlist_content(playlist_content, tracks_content) + artist = Artist() + + self.artist_map[artist_id] = artist + + artist.id = data.get("id") + artist.title = data.get("name") + + self.library.artists.append(artist) + + return artist From 20c7b28ed69e4cafe58c2584e9df9d5cb92009f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 5 Jan 2026 01:19:00 +0300 Subject: [PATCH 07/18] fixes --- .gitignore | 3 + Flow.py | 5 + main.py | 10 +- old/env | 36 -------- old/main.py | 195 +-------------------------------------- src/LibrarySerializer.py | 37 ++++++-- src/StatGenerator.py | 81 +++++++++------- 7 files changed, 91 insertions(+), 276 deletions(-) delete mode 100755 old/env diff --git a/.gitignore b/.gitignore index 2803b17..880f9c3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ prefetched* __pycache__ .idea secret +work_dir +old +tmp \ No newline at end of file diff --git a/Flow.py b/Flow.py index bb35731..e982d39 100644 --- a/Flow.py +++ b/Flow.py @@ -11,6 +11,7 @@ from src.local.LocalLibraryIndexer import update_local_library from src.local.ParseLocal import LocalParser from src.MapLocalToSpotify import map_local_to_spotify +from src.StatGenerator import log_stats class Flow: @@ -51,6 +52,10 @@ class Flow: def map_local_to_spotify(self): map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir) + def log_stats(self): + log_stats(work_dir / "local_to_spotify_id_map.json", work_dir / "stats.json", self.local_library, self.spotify_library) + + def run(self, arg): self.fetch_local() # self.do_fetch_spotify() diff --git a/main.py b/main.py index 98b4efa..1069171 100644 --- a/main.py +++ b/main.py @@ -5,13 +5,15 @@ if __name__ == '__main__': flow = Flow.Flow() # flow.fetch_spotify() - flow.fetch_local() + # flow.fetch_local() - flow.parse_spotify_library() - flow.parse_local_library() + # flow.parse_spotify_library() + # flow.parse_local_library() flow.load_libraries() - flow.map_local_to_spotify() + # flow.map_local_to_spotify() + + flow.log_stats() print("asd") diff --git a/old/env b/old/env deleted file mode 100755 index 8955572..0000000 --- a/old/env +++ /dev/null @@ -1,36 +0,0 @@ - -PS1=" > " - -#export PATH="/usr/local/bin/:$PATH" -export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH" -#export PATH="$HOME/home/auser/.local/bin:$PATH" - -export SIP="185.238.170.251" - -alias v=nvim -alias ll="ls -l -a" -alias gl="git log --oneline" -alias ssh_server="ssh auser@185.238.170.251" - -#eval "$(zoxide init bash --cmd zd)" - -#source $PMAIN/.scripts/bashmarks.sh - -fe() { - local result=$(command tere "$@") - [ -n "$result" ] && cd -- "$result" -} - -vim_configure() { - pwd=$(pwd) - cd ~/.config/nvim/ - nvim - cd $pwd -} - -pyenv() { - source ~/bin/python/env311/bin/activate -} - -source ~/src/scripts/remotes.sh -source /usr/share/fzf/key-bindings.bash diff --git a/old/main.py b/old/main.py index 10d378a..0b2a991 100644 --- a/old/main.py +++ b/old/main.py @@ -1,79 +1,3 @@ -from DataBase import * -from LocalLibraryIndexer import update_local_library -from LinkerPatternGenerator import run_pattern_generators -from LocalPLaylistGenerator import generate_playlists - -from SpotifyWebAPI import fetch_data, update_access_token -import cmd - -local_library_path = "/home/auser/Music/" -rel_autogen_dir = "0autogen" - -playlists = get_data('playlistTracks') -top_artists = get_data('top_artists') -top_tracks = get_data('top_tracks') -user_tracks = get_data('tracks') -link_pattern = get_data('link_pattern_spotify') -local_library = get_data("local_library") -user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} - - -def get_playlist_names(pl): - return [name for name, items in pl.items()] - - -def print_playlist_tracks(pl_id: int): - pls = list(playlists) - name = pls[pl_id] - pl = playlists[name] - print(name) - for track in pl: - track_data = track['track'] - track_name = track_data['name'] - artists = get_artists_str(track_data['artists']) - print(f" '{track_name}' - '{artists}' ") - - -def print_top_artists(): - for artist in top_artists: - name = artist['name'] - print(f" '{name}'") - - -def print_top_tracks(): - for track in top_tracks: - track_name = track['name'] - artists = get_artists_str(track['artists']) - print(f" '{track_name}' - '{artists}' ") - - -def print_stats(): - print(f" tracks - {len(user_tracks)}") - print(f" playlists - {len(playlists)}") - - for name, tracks in playlists.items(): - print(f" '{name}' - {len(tracks)}") - - print(f" top tracks - {len(top_tracks)}") - print(f" top artists - {len(top_artists)}") - - -def print_tracks(): - track_idx = 0 - for trackItem in user_tracks: - track = trackItem['track'] - print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'") - track_idx += 1 - - -def sort_remote_tracks_by_popularity(track_ids): - out = [] - for top_track in top_tracks: - if top_track['id'] in track_ids: - out.append(top_track['id']) - return out - - def print_link_stats(link_pattern): resolved_reversed = {} resolved = [] @@ -114,121 +38,4 @@ def print_link_stats(link_pattern): print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") - index += 1 - - -class Interpreter(cmd.Cmd): - intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" - prompt = "(spotify) " - - def do_list_stat_vs_types(self, arg): - """prints available linking patterns""" - print(f"0 - fuzzy tags") - print(f"1 - spotify") - print(f"2 - fuzzy") - - def do_stat_local_library_vs_spotify(self, arg): - """compares local library and remote with the link pattern generated by the generate_pattern_command""" - print_link_stats(link_pattern) - - def do_playlists(self, arg): - """prints user playlists""" - pl_id = 0 - for pl in get_playlist_names(playlists): - print(f"{pl_id} - {pl}") - pl_id += 1 - - def do_playlist_tracks(self, arg): - """prints playlist [name]""" - try: - pl_id = int(arg.split()[0]) - print_playlist_tracks(pl_id) - except IndexError: - print("Invalid input") - - def do_top_artists(self, arg): - """Top artists""" - print_top_artists() - - def do_top_tracks(self, arg): - """Top tracks""" - print_top_tracks() - - def do_stat(self, arg): - """Print stats""" - print_stats() - - def do_tracks(self, arg): - """Print tracks""" - print_tracks() - - def do_generate_local_playlists(self, arg): - """generates local playlists based on the vs comparison type""" - generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir) - - - def do_index_local_library(self, arg): - """find local tracks and fetches the data from them for further analysis""" - global local_library - update_local_library(local_library_path) - local_library = get_data("local_library") - - def do_generate_vs_stat(self, arg): - """compares local and remote library and generates the vs stats""" - - global link_pattern - run_pattern_generators() - link_pattern = get_data('link_pattern_spotify') - - def do_fetch(self, arg): - """Fetch all spotify data""" - try: - update_access_token() - fetch_data() - - global playlists, top_artists, top_tracks, user_tracks - - playlists = get_data('playlistTracks') - top_artists = get_data('top_artists') - top_tracks = get_data('top_tracks') - user_tracks = get_data('tracks') - - except ValueError: - print("Can not fetch the data.") - - def do_generate_todo(self, arg): - """generates todos for the pirates""" - print("sorry not implemented yet.") - - def do_run_all_default(self, arg): - """runs general flow""" - - local_lib_path = "/home/auser/Music/" - print("Fetching spotify data...") - self.do_fetch([]) - - print("Fetching local data...") - self.do_index_local_library(f"{local_lib_path}/organized") - - print("Generating vs patterns...") - self.do_generate_vs_stat("0") - - print("Generating todos...") - self.do_generate_todo(f"{local_lib_path}") - - print("Generating local playlists...") - self.do_generate_local_playlists(f"{local_lib_path}") - - return True - - def do_exit(self, arg): - """Exit the command loop: exit""" - print("Goodbye!") - return True - - -if __name__ == '__main__': - try: - Interpreter().cmdloop() - except KeyboardInterrupt as kb: - print("process terminated") \ No newline at end of file + index += 1 \ No newline at end of file diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py index bc968ea..cbccd9f 100644 --- a/src/LibrarySerializer.py +++ b/src/LibrarySerializer.py @@ -88,34 +88,55 @@ class LibraryLoader: self.album_map: dict[str, Album] = {} self.artist_map: dict[str, Artist] = {} + def get_artist(self, artist_id) -> Artist: + artist = self.artist_map.get(artist_id) + if artist is None: + artist = Artist() + self.artist_map[artist_id] = artist + return artist + + def get_album(self, id) -> Album: + item = self.album_map.get(id) + if item is None: + item = Album() + self.album_map[id] = item + return item + + def get_track(self, id) -> Track: + item = self.track_map.get(id) + if item is None: + item = Track() + self.track_map[id] = item + return item + def load_track(self, data: dict[str, Any]) -> Track: - track = self.track_map.get(data.get("id"), Track()) + track = self.get_track(data.get("id")) track.id = data.get("id") track.title = data.get("title") track.duration_ms = data.get("duration_ms") track.added_at = _str_to_dt(data.get("added_at")) track.local_path = data.get("local_path") - track.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")] - track.album = self.album_map.get(data.get("album"), Album()) + track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] + track.album = self.get_album(data.get("album")) self.track_map[track.id] = track return track def load_artist(self, data: dict[str, Any]): - artist = self.artist_map.get(data.get("id"), Artist()) + artist = self.get_artist(data.get("id")) artist.id = data.get("id") artist.title = data.get("title") - artist.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")] + artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] self.artist_map[artist.id] = artist return artist def load_album(self, data: dict[str, Any]): - album = self.album_map.get(data.get("id"), Album()) + album = self.get_album(data.get("id")) album.id = data.get("id") - album.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")] + album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] album.title = data.get("title") - album.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")] + album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] self.album_map[album.id] = album return album diff --git a/src/StatGenerator.py b/src/StatGenerator.py index c40bebb..d6ebc41 100644 --- a/src/StatGenerator.py +++ b/src/StatGenerator.py @@ -1,42 +1,55 @@ +from src.Library import * -def print_link_stats(link_pattern): - resolved_reversed = {} - resolved = [] - unresolved_local = [] - unresolved_remote = [] +from pathlib import Path +import json - for local_id, links in link_pattern.items(): - if len(links['items']): - resolved.append(local_id) - resolved_reversed[links['items'][0]] = local_id - else: - unresolved_local.append(local_id) +out = { + "local_count": 0, + "resolved_count": 0, + "unresolved_tracks": [], +} - for track in user_tracks: - track_id = track['track']['id'] - if track_id not in resolved_reversed: - unresolved_remote.append(track_id) +def get_mapping(mapping_path) -> dict[str, Any]: + data = {} - print(f"Local Tracks: {len(local_library)}") - print(f"Remote Tracks: {len(user_tracks)}") + with Path(mapping_path).open("r", encoding="utf-8") as f: + data = json.load(f) - unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) - unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] + out = {} - print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") - index = 0 - for remote_id in unresolved_remote: - remote_track = user_tracks_by_id[remote_id] - print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") - index += 1 + for local, remotes in data.items(): + if len(remotes["items"]): + out[local] = remotes["items"][0] - print(f"\nResolved tracks: {len(resolved)}") - index = 0 - for link in resolved: - remote_id = link_pattern[link]['items'][0] - local_track = local_library[link] - remote_track = user_tracks_by_id[remote_id] + return out - print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') - print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") - index += 1 \ No newline at end of file + +def find_add_local_paths(mapping_path: Path, local_lib: Library, remote_lib: Library): + mapping = get_mapping(mapping_path) + + out["resolved_count"] = len(mapping) + + remote_id_map = {track.id: track for track in remote_lib.tracks} + local_id_map = {track.id: track for track in local_lib.tracks} + + for local, remote in mapping.items(): + remote_id_map[remote].local_path = local_id_map[local].local_path + + +def generate_stats(local_lib: Library, remote_lib: Library, output_path : Path): + + out["local_count"] = len(local_lib.tracks) + + for track in remote_lib.top_tracks: + if not track.local_path: + out["unresolved_tracks"].append( + f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}") + + + with output_path.open("w", encoding="utf-8") as f: + json.dump(out, f, indent=2, ensure_ascii=False) + + +def log_stats(mapping_path : Path, output_path : Path, local_lib: Library, remote_lib: Library): + find_add_local_paths(mapping_path, local_lib, remote_lib) + generate_stats(local_lib, remote_lib, output_path) From b1c96511a3ed0ba75f9c69a43c1e7b55d5264399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 5 Jan 2026 14:53:06 +0300 Subject: [PATCH 08/18] nice stat generator --- src/StatGenerator.py | 207 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 183 insertions(+), 24 deletions(-) diff --git a/src/StatGenerator.py b/src/StatGenerator.py index d6ebc41..b6dcaee 100644 --- a/src/StatGenerator.py +++ b/src/StatGenerator.py @@ -3,11 +3,10 @@ from src.Library import * from pathlib import Path import json -out = { - "local_count": 0, - "resolved_count": 0, - "unresolved_tracks": [], -} + +def is_resolved(track: Track): + return track.local_path is not None + def get_mapping(mapping_path) -> dict[str, Any]: data = {} @@ -24,32 +23,192 @@ def get_mapping(mapping_path) -> dict[str, Any]: return out -def find_add_local_paths(mapping_path: Path, local_lib: Library, remote_lib: Library): - mapping = get_mapping(mapping_path) +class AlbumStat: + def __init__(self): + self.album: Album = Album() + self.score: int = 0 + self.resolved_precent: float = 0 - out["resolved_count"] = len(mapping) + def calc_resolved_precent(self): + self.resolved_precent = (sum(1 for track in self.album.tracks if is_resolved(track)) / len( + self.album.tracks)) * 100 - remote_id_map = {track.id: track for track in remote_lib.tracks} - local_id_map = {track.id: track for track in local_lib.tracks} - - for local, remote in mapping.items(): - remote_id_map[remote].local_path = local_id_map[local].local_path + def get_str(self): + return (f"{str(int(self.resolved_precent))} % - " + f"{self.album.title} - " + f"{str(self.album.artists[0].title) if len(self.album.artists) else " "}") -def generate_stats(local_lib: Library, remote_lib: Library, output_path : Path): +class ArtistStat: + def __init__(self): + self.artist: Artist = Artist() + self.score: int = 0 + self.resolved_precent: float = 0 + self.albums_stats: list[AlbumStat] = [] - out["local_count"] = len(local_lib.tracks) + def calc_resolved_precent(self): + if len(self.artist.tracks) == 0: + self.resolved_precent = 0 + return - for track in remote_lib.top_tracks: - if not track.local_path: - out["unresolved_tracks"].append( - f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}") + self.resolved_precent = (sum(1 for track in self.artist.tracks if is_resolved(track)) / len( + self.artist.tracks)) * 100 + + def get_str(self): + return (f"{str(int(self.resolved_precent))} % - " + f"{self.artist.title}") - with output_path.open("w", encoding="utf-8") as f: - json.dump(out, f, indent=2, ensure_ascii=False) +class StatGenerator: + def __init__(self): + self.out = { + "local_count": 0, + "resolved_count": 0, + + "popular_artists": [], + "popular_albums": [], + + "unresolved_albums": [], + "unresolved_tracks": [], + } + + self.max_popular_albums = 100 + self.max_popular_artists = 100 + + self.max_unresolved_tracks = 100 + self.max_unresolved_albums = 20 + self.resolved_percentage_threshold = 30 + + self.albums: List[AlbumStat] = [] + self.artists: List[ArtistStat] = [] + + self.albums_map: dict[str, AlbumStat] = {} + self.artists_map: dict[str, ArtistStat] = {} + + self.local_lib: Library = Library() + self.remote_lib: Library = Library() + + self.mapping: dict[str, Any] = Any + + self.remote_id_map: dict[str, Track] = {} + self.local_id_map: dict[str, Track] = {} + + def find_add_local_paths(self, mapping_path: Path): + self.mapping = get_mapping(mapping_path) + + self.remote_id_map = {track.id: track for track in self.remote_lib.tracks} + self.local_id_map = {track.id: track for track in self.local_lib.tracks} + + for local, remote in self.mapping.items(): + self.remote_id_map[remote].local_path = self.local_id_map[local].local_path + + def score_artists(self): + for artist in self.remote_lib.artists: + artist_stat = ArtistStat() + artist_stat.artist = artist + artist_stat.score = 0 + self.artists_map[artist_stat.artist.id] = artist_stat + self.artists.append(artist_stat) + + for idx, track in enumerate(self.remote_lib.top_tracks): + score = len(self.remote_lib.top_tracks) - idx + for artist in track.artists: + self.artists_map[artist.id].score += score + + for artist_stat in self.artists: + artist_stat.calc_resolved_precent() + + self.artists = sorted(self.artists, key=lambda item: item.score, reverse=True) + + def score_albums(self): + for album in self.remote_lib.albums: + stat = AlbumStat() + stat.album = album + self.albums_map[album.id] = stat + + for idx, track in enumerate(self.remote_lib.top_tracks): + score = len(self.remote_lib.top_tracks) - idx + self.albums_map[track.album.id].score += score + + for _, album in self.albums_map.items(): + album.calc_resolved_precent() + + self.albums = [item[1] for item in + sorted(self.albums_map.items(), key=lambda item: item[1].score, reverse=True)] + + for album_stat in self.albums: + for artist in album_stat.album.artists: + self.artists_map[artist.id].albums_stats.append(album_stat) + + def stat_generic(self): + self.out["resolved_count"] = len(self.mapping) + + def gen_album_songs(album_stat): + tracks = [] + + for track in album_stat.album.tracks: + tracks.append(f"{"✔" if is_resolved(track) else "𐄂"} {track.title} -> {track.local_path}") + + return { + "album" : album_stat.get_str(), + "tracks" : tracks + } + + for artist_stat in self.artists: + artist_log = { + "artist": artist_stat.get_str(), + "albums": [gen_album_songs(album) for album in artist_stat.albums_stats] + } + + if len(self.out["popular_artists"]) < self.max_popular_artists: + self.out["popular_artists"].append(artist_log) + + for album_data in self.albums: + log = album_data.get_str() + + if len(self.out["popular_albums"]) < self.max_popular_albums: + self.out["popular_albums"].append(log) + + def stat_unresolved_albums(self): + for album_data in self.albums: + if album_data.resolved_precent > self.resolved_percentage_threshold: + continue + + log = album_data.get_str() + + if len(self.out["unresolved_albums"]) < self.max_unresolved_albums: + self.out["unresolved_albums"].append(log) + + def stat_unresolved_tracks(self): + self.out["local_count"] = len(self.local_lib.tracks) + + for track in self.remote_lib.top_tracks: + if not track.local_path: + + log = f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}" + if len(self.out["unresolved_tracks"]) < self.max_unresolved_tracks: + self.out["unresolved_tracks"].append(log) + + def save_stats(self, output_path: Path): + with output_path.open("w", encoding="utf-8") as f: + json.dump(self.out, f, indent=2, ensure_ascii=False) + + def log_stats(self, mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library): + self.remote_lib = remote_lib + self.local_lib = local_lib + + self.find_add_local_paths(mapping_path) + + self.score_artists() + self.score_albums() + + self.stat_generic() + self.stat_unresolved_tracks() + self.stat_unresolved_albums() + + self.save_stats(output_path) -def log_stats(mapping_path : Path, output_path : Path, local_lib: Library, remote_lib: Library): - find_add_local_paths(mapping_path, local_lib, remote_lib) - generate_stats(local_lib, remote_lib, output_path) +def log_stats(mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library): + stat_gen = StatGenerator() + stat_gen.log_stats(mapping_path, output_path, local_lib, remote_lib) From 11fb2816e34f5cbe3dec1f5dab4f41deb909b41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 5 Jan 2026 17:42:56 +0300 Subject: [PATCH 09/18] stable stat --- Flow.py | 6 +- main.py | 6 +- src/Library.py | 29 ++++++ src/LibrarySerializer.py | 4 + src/MapLocalToSpotify.py | 140 --------------------------- src/StatGenerator.py | 10 +- src/local/ParseLocal.py | 16 ++- src/mappers/FuzzyMapper.py | 68 +++++++++++++ src/mappers/RemoteLibraryResolver.py | 9 ++ src/mappers/SpotifySearchMapping.py | 56 +++++++++++ src/spotify/ParseSpotify.py | 11 +-- 11 files changed, 181 insertions(+), 174 deletions(-) delete mode 100644 src/MapLocalToSpotify.py create mode 100644 src/mappers/FuzzyMapper.py create mode 100644 src/mappers/RemoteLibraryResolver.py create mode 100644 src/mappers/SpotifySearchMapping.py diff --git a/Flow.py b/Flow.py index e982d39..79c1759 100644 --- a/Flow.py +++ b/Flow.py @@ -10,7 +10,7 @@ from src.spotify.ParseSpotify import Parser from src.local.LocalLibraryIndexer import update_local_library from src.local.ParseLocal import LocalParser -from src.MapLocalToSpotify import map_local_to_spotify +from src.mappers.RemoteLibraryResolver import resolve_remote_tracks from src.StatGenerator import log_stats @@ -50,10 +50,10 @@ class Flow: self.local_library = LibraryLoader().load(work_dir / "local_library.json") def map_local_to_spotify(self): - map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir) + resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir) def log_stats(self): - log_stats(work_dir / "local_to_spotify_id_map.json", work_dir / "stats.json", self.local_library, self.spotify_library) + log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats.json", self.local_library, self.spotify_library) def run(self, arg): diff --git a/main.py b/main.py index 1069171..f811d33 100644 --- a/main.py +++ b/main.py @@ -7,12 +7,12 @@ if __name__ == '__main__': # flow.fetch_spotify() # flow.fetch_local() - # flow.parse_spotify_library() - # flow.parse_local_library() + flow.parse_spotify_library() + flow.parse_local_library() flow.load_libraries() - # flow.map_local_to_spotify() + flow.map_local_to_spotify() flow.log_stats() diff --git a/src/Library.py b/src/Library.py index c3287b8..68e2c63 100644 --- a/src/Library.py +++ b/src/Library.py @@ -4,11 +4,13 @@ from datetime import datetime from pathlib import Path from typing import List, Optional, Any + class Artist: def __init__(self): self.id : str = None self.title : str = None self.tracks : List[Track] = [] + self.albums : List[Album] = [] class Track: @@ -49,3 +51,30 @@ class Library: self.top_tracks : List[Track]= [] self.top_artists : List[Artist] = [] + self.artist_map : dict[str, Artist] = None + self.album_map : dict[str, Album] = None + self.track_map : dict[str, Track] = None + + + def update_cache(self): + self.update_id_maps() + self.create_reverse_links() + + def update_id_maps(self): + self.artist_map : dict[str, Artist] = { artist.id : artist for artist in self.artists } + self.album_map : dict[str, Album] = { album.id : album for album in self.albums } + self.track_map : dict[str, Track] = { track.id : track for track in self.tracks } + + def create_reverse_links(self): + for track_id, track in self.track_map.items(): + for artist in track.artists: + if track not in artist.tracks: + artist.tracks.append(track) + + if track not in track.album.tracks: + track.album.tracks.append(track) + + for album_id, album in self.album_map.items(): + for artist in album.artists: + if album not in artist.albums: + artist.albums.append(album) diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py index cbccd9f..e94a0f7 100644 --- a/src/LibrarySerializer.py +++ b/src/LibrarySerializer.py @@ -32,6 +32,7 @@ class LibrarySaver: "id": artist.id, "title": artist.title, "tracks": [track.id for track in artist.tracks], + "albums": [album.id for album in artist.albums], } @staticmethod @@ -127,6 +128,7 @@ class LibraryLoader: artist.id = data.get("id") artist.title = data.get("title") artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] + artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else [] self.artist_map[artist.id] = artist return artist @@ -155,4 +157,6 @@ class LibraryLoader: library.top_tracks = [self.track_map[track] for track in data.get("top_tracks")] library.liked_tracks = [self.track_map[track] for track in data.get("liked_tracks")] + library.update_cache() + return library diff --git a/src/MapLocalToSpotify.py b/src/MapLocalToSpotify.py deleted file mode 100644 index 84e37c5..0000000 --- a/src/MapLocalToSpotify.py +++ /dev/null @@ -1,140 +0,0 @@ - -from rapidfuzz import fuzz -from tqdm import tqdm -import requests - -from src.Library import * -from src.helpers import * -from src.spotify.SpotifyWebAPI import find_song, update_access_token - - -def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir): - - set_workdir(output_dir) - - update_access_token(client_id, client_secret) - - def find_local_songs_on_spotify(): - search_log = [] - - total = len(local_lib.tracks) - found_tracks_map = {} - with tqdm(total=total, desc='Searching local songs on spotify') as pbar: - for track in local_lib.tracks: - - parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title] - search_pattern = " ".join(p for p in parts if p) - - if search_pattern == "": - print(f"cannot generate search pattern for the track - {track.local_path}") - pbar.update(1) - continue - - found_tracks = find_song(search_pattern)[0:10] - found_tracks_map[track.id] = found_tracks - search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]}) - pbar.update(1) - - save_data(search_log, "spotify_search_log") - save_data(found_tracks_map, "spotify_found_map") - - return found_tracks_map - - spotify_found_tracks = find_local_songs_on_spotify() - - spotify_user_track = {track.id for track in spotify_lib.tracks} - - spotify_pattern = {} - for local_id, found_items in spotify_found_tracks.items(): - spotify_pattern[local_id] = {"score": 1.0, "items": []} - - if not len(found_items): - continue - - for found_item in found_items: - spotify_track_id = found_item['id'] - if spotify_track_id in spotify_user_track: - spotify_pattern[local_id]['items'].append(spotify_track_id) - - save_data(spotify_pattern, "local_to_spotify_id_map") - - -def fuzzy_pattern_generator(): - local_library = get_data("local_library") - track_descriptions = get_data("tracks") - - def get_matched(src_pattern, patterns): - result = {} - for target_pattern in patterns: - similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern) - result[target_pattern] = similarity_score / 100 - - sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) - return sorted_results - - def resolve_tracks(s_patterns, t_patterns): - - pattern_map = {} - - total = len(s_patterns) - with tqdm(total=total, desc='Progress') as pbar: - for s_pattern in s_patterns: - res = get_matched(s_pattern, t_patterns) - matched = {"score": 0, "items": []} - if len(res): - matched['items'] = res[0:min(len(res), 4)] - matched['score'] = res[0][1] - pattern_map[s_pattern] = matched - pbar.update(1) - - return pattern_map - - source_patterns = [] - for track in track_descriptions: - artists = get_artists_str(track['track']['artists']) - pattern = f"{track['track']['name']} {artists}" - source_patterns.append(pattern) - - target_patterns = [local['path'] for _, local in local_library.items()] - - fuzzy_pattern = resolve_tracks(source_patterns, target_patterns) - save_data(fuzzy_pattern, "link_pattern_fuzzy") - - -def fuzzy_tag_pattern_generator(): - local_library = get_data("local_library") - spotify_library = get_data("tracks") - mappings = {} - - def fuzzy_tags_ratio(first: SongTags, second: SongTags): - title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100 - artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100 - album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100 - return title_ratio, artist_ratio, album_ratio - - with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar: - for local_track_id, local_track in local_library.items(): - mapping = mappings.get(local_track_id, {"score": 1.0, "items": []}) - ratios = [] - for remote_track in spotify_library: - track = remote_track['track'] - - tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album']) - tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name']) - - ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) - - filtered_ratios = [] - threshold = 0.81 - for ratio in ratios: - if all([val > threshold for val in ratio[0]]): - filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1])) - - filtered_ratios.sort(reverse=True, key=lambda x: x[0]) - if len(filtered_ratios): - mapping['items'] = [filtered_ratios[0][1]] - - mappings[local_track_id] = mapping - pbar.update(1) - - save_data(mappings, "link_pattern_tags") \ No newline at end of file diff --git a/src/StatGenerator.py b/src/StatGenerator.py index b6dcaee..f597e17 100644 --- a/src/StatGenerator.py +++ b/src/StatGenerator.py @@ -14,13 +14,7 @@ def get_mapping(mapping_path) -> dict[str, Any]: with Path(mapping_path).open("r", encoding="utf-8") as f: data = json.load(f) - out = {} - - for local, remotes in data.items(): - if len(remotes["items"]): - out[local] = remotes["items"][0] - - return out + return data class AlbumStat: @@ -99,7 +93,7 @@ class StatGenerator: self.remote_id_map = {track.id: track for track in self.remote_lib.tracks} self.local_id_map = {track.id: track for track in self.local_lib.tracks} - for local, remote in self.mapping.items(): + for remote, local in self.mapping.items(): self.remote_id_map[remote].local_path = self.local_id_map[local].local_path def score_artists(self): diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py index 602e89e..084bbce 100644 --- a/src/local/ParseLocal.py +++ b/src/local/ParseLocal.py @@ -104,15 +104,6 @@ class LocalParser: return album - def add_tracks_to_albums_and_artists(self): - for track_id, track in self.track_map.items(): - for artist in track.artists: - if track not in artist.tracks: - artist.tracks.append(track) - - if track not in track.album.tracks: - track.album.tracks.append(track) - def parse(self, tracks_file: Path) -> Library: tracks_content = json.loads(tracks_file.read_text()) @@ -129,7 +120,12 @@ class LocalParser: self.track_map[track.id] = track self.library.tracks.append(track) - self.add_tracks_to_albums_and_artists() + for track_id, track in self.track_map.items(): + for artist in track.artists: + if artist not in track.album.artists: + track.album.artists.append(artist) + + self.library.update_cache() return self.library diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py new file mode 100644 index 0000000..ca78db9 --- /dev/null +++ b/src/mappers/FuzzyMapper.py @@ -0,0 +1,68 @@ + +from rapidfuzz import fuzz +from tqdm import tqdm + +from src.Library import * +from src.helpers import * + + +def get_score(first: str, second: str) -> float: + return fuzz.token_set_ratio(first, second) + +def is_close_enough(first: float) -> bool: + return first > 50 + +def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path): + + artist_map : dict[str, tuple[str, float]] = {} + album_map : dict[str, tuple[str, float]] = {} + track_map : dict[str, tuple[str, float]] = {} + + with (tqdm(total=len(remote_lib.artists), desc='searching remote artists in local library') as process_bar): + for remote_artist in remote_lib.artists: + for local_artist in local_lib.artists: + score = get_score(remote_artist.title, local_artist.title) + if not is_close_enough(score): + continue + + if remote_artist.id not in artist_map or score > artist_map[remote_artist.id][1]: + artist_map[remote_artist.id] = (local_artist.id, score) + + process_bar.update(1) + + + with tqdm(total=len(artist_map), desc='searching remote albums for each artist in local library') as process_bar: + for remote_artist_id, (local_artist_id, _) in artist_map.items(): + remote_artist = remote_lib.artist_map[remote_artist_id] + local_artist = local_lib.artist_map[local_artist_id] + + for remote_album in remote_artist.albums: + for local_album in local_artist.albums: + score = get_score(remote_album.title, local_album.title) + if not is_close_enough(score): + continue + + if remote_album.id not in artist_map or score > album_map[remote_album.id][1]: + album_map[remote_album.id] = (local_album.id, score) + + process_bar.update(1) + + with tqdm(total=len(album_map), desc='searching remote tracks for each album in local library') as process_bar: + for remote_album_id, (local_album_id, _) in album_map.items(): + remote_album = remote_lib.album_map[remote_album_id] + local_album = local_lib.album_map[local_album_id] + + for remote_track in remote_album.tracks: + for local_track in local_album.tracks: + score = get_score(remote_track.title, local_track.title) + if not is_close_enough(score): + continue + + if remote_track.id not in artist_map or score > track_map[remote_track.id][1]: + track_map[remote_track.id] = (local_track.id, score) + + process_bar.update(1) + + out = { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() } + set_workdir(output_dir) + save_data(out, "remote_to_local_map") diff --git a/src/mappers/RemoteLibraryResolver.py b/src/mappers/RemoteLibraryResolver.py new file mode 100644 index 0000000..7342848 --- /dev/null +++ b/src/mappers/RemoteLibraryResolver.py @@ -0,0 +1,9 @@ + +import src.mappers.FuzzyMapper as Fuzzy +import src.mappers.SpotifySearchMapping as Spotify + +from src.Library import * + +def resolve_remote_tracks(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir): + Fuzzy.generate_map(local_lib, spotify_lib, output_dir) + # Spotify.generate_map(client_id, client_secret, local_lib, spotify_lib, output_dir) \ No newline at end of file diff --git a/src/mappers/SpotifySearchMapping.py b/src/mappers/SpotifySearchMapping.py new file mode 100644 index 0000000..41c5bbb --- /dev/null +++ b/src/mappers/SpotifySearchMapping.py @@ -0,0 +1,56 @@ + +from tqdm import tqdm + +from src.Library import * +from src.helpers import * +from src.spotify.SpotifyWebAPI import find_song, update_access_token + +def generate_map(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir): + + set_workdir(output_dir) + + update_access_token(client_id, client_secret) + + def find_local_songs_on_spotify(): + search_log = [] + + total = len(local_lib.tracks) + found_tracks_map = {} + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + for track in local_lib.tracks: + + parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title] + search_pattern = " ".join(p for p in parts if p) + + if search_pattern == "": + print(f"cannot generate search pattern for the track - {track.local_path}") + pbar.update(1) + continue + + found_tracks = find_song(search_pattern)[0:10] + found_tracks_map[track.id] = found_tracks + search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]}) + pbar.update(1) + + save_data(search_log, "spotify_search_log") + save_data(found_tracks_map, "spotify_found_map") + + return found_tracks_map + + spotify_found_tracks = find_local_songs_on_spotify() + + spotify_user_track = {track.id for track in spotify_lib.tracks} + + spotify_pattern = {} + for local_id, found_items in spotify_found_tracks.items(): + spotify_pattern[local_id] = {"score": 1.0, "items": []} + + if not len(found_items): + continue + + for found_item in found_items: + spotify_track_id = found_item['id'] + if spotify_track_id in spotify_user_track: + spotify_pattern[local_id]['items'].append(spotify_track_id) + + save_data(spotify_pattern, "local_to_spotify_id_map") \ No newline at end of file diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py index 49882cb..50806bf 100644 --- a/src/spotify/ParseSpotify.py +++ b/src/spotify/ParseSpotify.py @@ -29,19 +29,10 @@ class Parser: self.parse_top_artists(json.loads(top_artists.read_text())) self.parse_top_tracks(json.loads(top_tracks.read_text())) - self.add_tracks_to_albums_and_artists() + self.library.update_cache() return self.library - def add_tracks_to_albums_and_artists(self): - for track_id, track in self.track_map.items(): - for artist in track.artists: - if track not in artist.tracks: - artist.tracks.append(track) - - if track not in track.album.tracks: - track.album.tracks.append(track) - def parse_playlists(self, data): for name, tracks in data.items(): From 9c9ddb7a9ed278ac06e9858a9bf786c6f4fd0736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Tue, 6 Jan 2026 02:34:04 +0300 Subject: [PATCH 10/18] nice stats and stable --- Flow.py | 2 +- main.py | 2 +- src/StatGenerator.py | 155 ++++++++++++++++++++++--------------------- src/helpers.py | 2 +- 4 files changed, 81 insertions(+), 80 deletions(-) diff --git a/Flow.py b/Flow.py index 79c1759..bcbdd16 100644 --- a/Flow.py +++ b/Flow.py @@ -53,7 +53,7 @@ class Flow: resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir) def log_stats(self): - log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats.json", self.local_library, self.spotify_library) + log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats", self.local_library, self.spotify_library) def run(self, arg): diff --git a/main.py b/main.py index f811d33..5e395a4 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ if __name__ == '__main__': flow = Flow.Flow() # flow.fetch_spotify() - # flow.fetch_local() + flow.fetch_local() flow.parse_spotify_library() flow.parse_local_library() diff --git a/src/StatGenerator.py b/src/StatGenerator.py index f597e17..ac16622 100644 --- a/src/StatGenerator.py +++ b/src/StatGenerator.py @@ -1,4 +1,5 @@ from src.Library import * +from src.helpers import * from pathlib import Path import json @@ -52,20 +53,8 @@ class ArtistStat: return (f"{str(int(self.resolved_precent))} % - " f"{self.artist.title}") - -class StatGenerator: - def __init__(self): - self.out = { - "local_count": 0, - "resolved_count": 0, - - "popular_artists": [], - "popular_albums": [], - - "unresolved_albums": [], - "unresolved_tracks": [], - } - +class LibraryStats: + def __init__(self, mapping_path : Path, local_lib: Library, remote_lib: Library): self.max_popular_albums = 100 self.max_popular_artists = 100 @@ -79,17 +68,19 @@ class StatGenerator: self.albums_map: dict[str, AlbumStat] = {} self.artists_map: dict[str, ArtistStat] = {} - self.local_lib: Library = Library() - self.remote_lib: Library = Library() + self.local_lib: Library = local_lib + self.remote_lib: Library = remote_lib - self.mapping: dict[str, Any] = Any + self.mapping: dict[str, Any] = get_mapping(mapping_path) self.remote_id_map: dict[str, Track] = {} self.local_id_map: dict[str, Track] = {} - def find_add_local_paths(self, mapping_path: Path): - self.mapping = get_mapping(mapping_path) + self.find_add_local_paths() + self.score_artists() + self.score_albums() + def find_add_local_paths(self): self.remote_id_map = {track.id: track for track in self.remote_lib.tracks} self.local_id_map = {track.id: track for track in self.local_lib.tracks} @@ -134,75 +125,85 @@ class StatGenerator: for artist in album_stat.album.artists: self.artists_map[artist.id].albums_stats.append(album_stat) - def stat_generic(self): - self.out["resolved_count"] = len(self.mapping) - def gen_album_songs(album_stat): - tracks = [] + def gen_album(self, album_stat, condition): + tracks = [] - for track in album_stat.album.tracks: + for track in album_stat.album.tracks: + if condition.should_add_track(track): tracks.append(f"{"✔" if is_resolved(track) else "𐄂"} {track.title} -> {track.local_path}") - return { - "album" : album_stat.get_str(), - "tracks" : tracks - } + return { + "album" : album_stat.get_str(), + "tracks" : tracks + } + + def gen_artist(self, artist_stat, condition): + albums = [] + + for album_stat in artist_stat.albums_stats: + if condition.should_add_album(album_stat): + albums.append(self.gen_album(album_stat, condition)) + + return { + "artist" : artist_stat.get_str(), + "albums" : albums, + } + + def generate_stat_tree(self, condition): + artists = [] for artist_stat in self.artists: - artist_log = { - "artist": artist_stat.get_str(), - "albums": [gen_album_songs(album) for album in artist_stat.albums_stats] - } + if condition.should_add_artist(artist_stat): + artists.append(self.gen_artist(artist_stat, condition)) - if len(self.out["popular_artists"]) < self.max_popular_artists: - self.out["popular_artists"].append(artist_log) + return { + "artists" : artists + } - for album_data in self.albums: - log = album_data.get_str() + def save_tree(self, output_path, condition): + save_data(self.generate_stat_tree(condition), output_path) - if len(self.out["popular_albums"]) < self.max_popular_albums: - self.out["popular_albums"].append(log) + def save_all(self, output_path): - def stat_unresolved_albums(self): - for album_data in self.albums: - if album_data.resolved_precent > self.resolved_percentage_threshold: - continue + class Condition: + def should_add_artist(self, item): + return True - log = album_data.get_str() + def should_add_album(self, item): + return True - if len(self.out["unresolved_albums"]) < self.max_unresolved_albums: - self.out["unresolved_albums"].append(log) - - def stat_unresolved_tracks(self): - self.out["local_count"] = len(self.local_lib.tracks) - - for track in self.remote_lib.top_tracks: - if not track.local_path: - - log = f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}" - if len(self.out["unresolved_tracks"]) < self.max_unresolved_tracks: - self.out["unresolved_tracks"].append(log) - - def save_stats(self, output_path: Path): - with output_path.open("w", encoding="utf-8") as f: - json.dump(self.out, f, indent=2, ensure_ascii=False) - - def log_stats(self, mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library): - self.remote_lib = remote_lib - self.local_lib = local_lib - - self.find_add_local_paths(mapping_path) - - self.score_artists() - self.score_albums() - - self.stat_generic() - self.stat_unresolved_tracks() - self.stat_unresolved_albums() - - self.save_stats(output_path) + def should_add_track(self, item): + return True -def log_stats(mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library): - stat_gen = StatGenerator() - stat_gen.log_stats(mapping_path, output_path, local_lib, remote_lib) + self.save_tree(output_path, Condition()) + + + def save_threshold(self, output_path, threshold = 80): + + class Condition: + def should_add_artist(self, item : ArtistStat): + for album in item.albums_stats: + if self.should_add_album(album): + return True + + return False + + def should_add_album(self, item: AlbumStat): + return item.resolved_precent < threshold and len(item.album.tracks) > 3 + + def should_add_track(self, item: Track): + return not is_resolved(item) + + self.save_tree(output_path, Condition()) + + + +def log_stats(mapping_path: Path, output_dir: Path, local_lib: Library, remote_lib: Library): + set_workdir(output_dir) + + stat = LibraryStats(mapping_path, local_lib, remote_lib) + + stat.save_all("stats_all") + stat.save_threshold("stats_missing") diff --git a/src/helpers.py b/src/helpers.py index 8f81bbf..3c7db3e 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -46,7 +46,7 @@ def save_data(data, name): file_path = os.path.join(data_dir, f"{name}.json") with open(file_path, 'w') as file: - json.dump(data, file, indent=4) + json.dump(data, file, indent=4, ensure_ascii=False) print(f"Data saved to {file_path}.") From 5d38a146a1232cbe34325bb2cb5509b4867c8a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sat, 17 Jan 2026 14:24:05 +0300 Subject: [PATCH 11/18] lyrics fetcher --- Flow.py | 51 ++++++++++++- main.py | 20 ++++- src/Library.py | 1 + src/LibrarySerializer.py | 2 + src/local/LocalLibraryIndexer.py | 111 +++++++++++++++++++++++----- src/local/LocalPLaylistGenerator.py | 2 +- src/local/ParseLocal.py | 2 + src/lyrics/lyrics.py | 111 ++++++++++++++++++++++++++++ 8 files changed, 277 insertions(+), 23 deletions(-) create mode 100644 src/lyrics/lyrics.py diff --git a/Flow.py b/Flow.py index bcbdd16..4731f1c 100644 --- a/Flow.py +++ b/Flow.py @@ -1,7 +1,11 @@ +from tqdm import tqdm +import src.lyrics.lyrics as Lyrics + from Env import client_id, client_secret, local_library_path from Env import work_dir +from src.Library import * from src.LibrarySerializer import LibrarySaver, LibraryLoader from src.spotify.SpotifyWebAPI import fetch_all @@ -15,8 +19,8 @@ from src.StatGenerator import log_stats class Flow: - spotify_library = None - local_library = None + spotify_library : Library = None + local_library : Library = None @staticmethod def fetch_spotify(self): @@ -45,6 +49,49 @@ class Flow: library_parsed = parser.parse(work_dir / "local" / "local_library.json") LibrarySaver(library_parsed).save(work_dir / "local_library.json") + + def fetch_and_save_lyrics(self): + """ + Fetches and writes lyrics for tracks that don't have lyrics yet. + Displays stats and uses a progress bar. + """ + + # ---------- Step 1: Collect tracks without lyrics ---------- + tracks_missing_lyrics = [ + track for track in self.local_library.tracks + if len(track.artists) > 0 and not track.has_lyrics + ] + + total_tracks = len(self.local_library.tracks) + missing_count = len(tracks_missing_lyrics) + + print(f"[INFO] Total tracks in library: {total_tracks}") + print(f"[INFO] Tracks missing lyrics: {missing_count}") + + if missing_count == 0: + print("[INFO] No tracks need lyrics. Exiting.") + return + + # ---------- Step 2: Fetch and write lyrics with progress bar ---------- + for track in tqdm(tracks_missing_lyrics, desc="Fetching lyrics", unit="track"): + artist_name = track.artists[0].title + album_title = track.album.title if track.album else "Unknown Album" + print(f"\n[INFO] Processing: '{track.title}' by '{artist_name}' on '{album_title}'") + + lyrics_text = Lyrics.fetch_lyrics(artist_name, album_title, track.title, track.duration_ms) + + if lyrics_text: + print("[SUCCESS] Lyrics found, writing to file...") + file_path = local_library_path / track.local_path + try: + Lyrics.write_lyrics(file_path, lyrics_text) + print(f"[SUCCESS] Lyrics written for '{track.title}'") + except Exception as e: + print(f"[ERROR] Failed to write lyrics for '{track.title}': {e}") + else: + print(f"[WARNING] Lyrics not found for '{track.title}'") + + def load_libraries(self): self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json") self.local_library = LibraryLoader().load(work_dir / "local_library.json") diff --git a/main.py b/main.py index 5e395a4..8499c07 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,21 @@ + import Flow -if __name__ == '__main__': +def test1(): + flow = Flow.Flow() + + # flow.fetch_local() + flow.parse_local_library() + flow.load_libraries() + + # flow.map_local_to_spotify() + # flow.log_stats() + + flow.fetch_and_save_lyrics() + + +def main(): flow = Flow.Flow() # flow.fetch_spotify() @@ -17,3 +31,7 @@ if __name__ == '__main__': flow.log_stats() print("asd") + + +test1() +# main() diff --git a/src/Library.py b/src/Library.py index 68e2c63..0de72ce 100644 --- a/src/Library.py +++ b/src/Library.py @@ -22,6 +22,7 @@ class Track: self.duration_ms : int = None self.added_at : datetime = None self.local_path : Path = None + self.has_lyrics : bool = False class Album: diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py index e94a0f7..84f6a46 100644 --- a/src/LibrarySerializer.py +++ b/src/LibrarySerializer.py @@ -45,6 +45,7 @@ class LibrarySaver: "duration_ms": track.duration_ms, "added_at": _dt_to_str(track.added_at), "local_path": track.local_path, + "has_lyrics": track.has_lyrics, } @staticmethod @@ -119,6 +120,7 @@ class LibraryLoader: track.local_path = data.get("local_path") track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] track.album = self.get_album(data.get("album")) + track.has_lyrics = data.get("has_lyrics") self.track_map[track.id] = track return track diff --git a/src/local/LocalLibraryIndexer.py b/src/local/LocalLibraryIndexer.py index d3a7ce2..1a8eaab 100644 --- a/src/local/LocalLibraryIndexer.py +++ b/src/local/LocalLibraryIndexer.py @@ -1,8 +1,8 @@ -import os import fnmatch import mutagen import eyed3 from mutagen.flac import FLAC +from mutagen import File as MutagenFile from src.helpers import * @@ -11,6 +11,52 @@ music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') exclude_directories = ("*mary--*", "*example-word*") +from mutagen.id3 import ID3 +from mutagen.id3 import ID3NoHeaderError + + +def detect_lyrics(abs_path, ext): + ext = ext.lower() + + # ---------- MP3 ---------- + if ext == ".mp3": + try: + tags = ID3(abs_path) + except ID3NoHeaderError: + return False + + return bool( + tags.getall("SYLT") or + tags.getall("USLT") + ) + + # ---------- FLAC / OGG / OPUS ---------- + if ext in {".flac", ".ogg", ".opus"}: + try: + audio = MutagenFile(abs_path) + if not audio or not audio.tags: + return False + + for key in audio.tags.keys(): + k = key.upper() + if k in {"LYRICS", "LRC", "SYNCEDLYRICS"}: + if audio.tags[key]: + return True + except Exception: + return False + + # ---------- Fallback ---------- + try: + audio = MutagenFile(abs_path) + if audio and audio.tags: + for key in audio.tags.keys(): + if "LYRIC" in key.upper(): + return True + except Exception: + pass + + return False + def update_local_library(root_dir, output_dir): set_workdir(output_dir / "local") @@ -29,50 +75,77 @@ def update_local_library(root_dir, output_dir): return prev_id for dir_path, dir_names, filenames in os.walk(root_dir): - dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)] - filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)] + dir_names[:] = [ + d for d in dir_names + if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories) + ] + + filtered_filenames = [ + filename for filename in filenames + if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions) + ] for filename in filtered_filenames: song_name, track_type = os.path.splitext(filename) relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) song_id = get_new_song_id(song_id, relative_path) + new_library[song_id] = { "name": song_name, "path": relative_path, "type": track_type, "artist": "", - "album": "" + "album": "", + "duration": None } for track_id, track in new_library.items(): abs_path = os.path.join(root_dir, track['path']) - match track['type']: + + track_abs_path = os.path.join(root_dir, track["path"]) + track["has_lyrics"] = detect_lyrics(track_abs_path, track['type'].lower()) + + match track['type'].lower(): case ".mp3": mp3track = eyed3.load(abs_path) - if not mp3track: - print(f"Failed to load mp3 {abs_path} using just name of the file as track name") + if not mp3track or not mp3track.info: + print(f"Failed to load mp3 {abs_path}") track['name'] = track['path'] else: + track['duration'] = mp3track.info.time_secs + tag = mp3track.tag - if not tag: - print(f"Invalid mp3 header {abs_path} using just name of the file as track name") - track['name'] = track['path'] - else: + if tag: track['artist'] = tag.artist track['name'] = tag.title track['album'] = tag.album case ".flac": try: - metadata = FLAC(abs_path).tags - if 'artist' in metadata: - track['artist'] = " ".join(metadata['artist']) - if 'TITLE' in metadata: - track['name'] = " ".join(metadata['TITLE']) - if 'album' in metadata: - track['album'] = " ".join(metadata['album']) + audio = FLAC(abs_path) + + if audio.info: + track['duration'] = audio.info.length + + metadata = audio.tags + if metadata: + if 'artist' in metadata: + track['artist'] = " ".join(metadata['artist']) + if 'TITLE' in metadata: + track['name'] = " ".join(metadata['TITLE']) + if 'album' in metadata: + track['album'] = " ".join(metadata['album']) + except mutagen.MutagenError: - print(f"Invalid flac header {abs_path} using just name of the file as track name") + print(f"Invalid flac header {abs_path}") track['name'] = track['path'] + case _: + try: + audio = MutagenFile(abs_path) + if audio and audio.info: + track['duration'] = audio.info.length + except Exception: + pass + save_data(new_library, "local_library") diff --git a/src/local/LocalPLaylistGenerator.py b/src/local/LocalPLaylistGenerator.py index 0299498..1c0f1e1 100644 --- a/src/local/LocalPLaylistGenerator.py +++ b/src/local/LocalPLaylistGenerator.py @@ -1,7 +1,7 @@ from pathlib import Path -def generate_playlists(local_library, library_path, remote_playlists, link_pattern, output_dir): +def (): output_dir = Path(output_dir) output_playlists = [] diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py index 084bbce..49b3f55 100644 --- a/src/local/ParseLocal.py +++ b/src/local/ParseLocal.py @@ -116,6 +116,8 @@ class LocalParser: track.artists = self.parse_artists(data.get("artist")) track.album = self.parse_album(data.get("album")) track.local_path = data.get("path") + track.duration_ms = data.get("duration") + track.has_lyrics = data.get("has_lyrics") self.track_map[track.id] = track self.library.tracks.append(track) diff --git a/src/lyrics/lyrics.py b/src/lyrics/lyrics.py new file mode 100644 index 0000000..fb427de --- /dev/null +++ b/src/lyrics/lyrics.py @@ -0,0 +1,111 @@ +import requests +from mutagen import File +from mutagen.id3 import ID3, USLT, ID3NoHeaderError +import os + +def fetch_lyrics(artist: str, album: str, track: str, duration: int = 0) -> str: + + skip_artists = [ + "shurupis", + "unknown" + ] + + skip_album = [ + "unknown" + ] + + if artist in skip_artists or album in skip_album: + return "" + + + url = "https://lrclib.net/api/get" + params = { + "artist_name": artist, + "track_name": track, + "album_name": album, + "duration": duration, + } + try: + resp = requests.get(url, params=params, timeout=10) + if resp.status_code != 200: + try: + err = resp.json() + print(err.get("error"), "-", err.get("message")) + except ValueError: + print("HTTPError", resp.status_code) + return "" + data = resp.json() + if "error" in data: + print(data.get("error"), "-", data.get("message")) + return "" + return data.get("syncedLyrics", "") + except requests.RequestException as e: + print(type(e).__name__, "-", str(e)) + return "" + except ValueError as e: + print(type(e).__name__, "-", str(e)) + return "" + + +def write_lyrics(file_path, lyrics_text, lang="eng"): + """ + Writes lyrics text into tags recognized by Navidrome. + Provides detailed logs and success/failure info. + """ + + ext = os.path.splitext(file_path)[1].lower() + print(f"[INFO] Processing file: {file_path} (extension: {ext})") + + try: + # ---------- MP3 ---------- + if ext == ".mp3": + print("[INFO] Detected MP3 file") + try: + tags = ID3(file_path) + print("[INFO] Existing ID3 tags loaded") + except ID3NoHeaderError: + tags = ID3() + print("[INFO] No ID3 header found, creating new tags") + + tags.delall("USLT") + print("[INFO] Removed existing USLT frames (lyrics)") + + tags.add( + USLT( + encoding=3, # UTF-8 + lang=lang, + desc="", + text=lyrics_text + ) + ) + tags.save(file_path) + print("[SUCCESS] Lyrics written to MP3 tags") + return + + # ---------- FLAC / OGG / OPUS ---------- + audio = File(file_path) + if audio is None: + print(f"[ERROR] Unsupported or unrecognized file format: {file_path}") + return + + if ext in {".flac", ".ogg", ".opus"}: + print(f"[INFO] Detected {ext.upper()} file") + audio["LRC"] = lyrics_text + audio["SYNCEDLYRICS"] = lyrics_text + audio.save() + print(f"[SUCCESS] Lyrics written to {ext.upper()} tags") + return + + # ---------- Fallback ---------- + if audio.tags is not None: + print("[INFO] Using fallback tag writing") + audio["LYRICS"] = lyrics_text + audio.save() + print("[SUCCESS] Lyrics written using fallback tags") + return + + print(f"[WARNING] No suitable tag frame found for {file_path}") + + except Exception as e: + print(f"[ERROR] Failed to write lyrics to {file_path}: {e}") + From 35905ea9428cc78d575551daa6a00b3701aa8728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Wed, 4 Feb 2026 21:21:19 +0300 Subject: [PATCH 12/18] stable --- Flow.py | 8 ++++---- main.py | 2 +- src/{ => backends}/itunes/ParseItunesToJson.py | 0 src/{ => backends}/local/LocalLibraryIndexer.py | 0 src/{ => backends}/local/LocalPLaylistGenerator.py | 0 src/{ => backends}/local/ParseLocal.py | 0 src/{ => backends}/spotify/ParseSpotify.py | 0 src/{ => backends}/spotify/SpotifyAuthenticator.py | 0 src/{ => backends}/spotify/SpotifyWebAPI.py | 2 +- src/lyrics/lyrics.py | 6 ++++-- src/mappers/SpotifySearchMapping.py | 2 +- 11 files changed, 11 insertions(+), 9 deletions(-) rename src/{ => backends}/itunes/ParseItunesToJson.py (100%) rename src/{ => backends}/local/LocalLibraryIndexer.py (100%) rename src/{ => backends}/local/LocalPLaylistGenerator.py (100%) rename src/{ => backends}/local/ParseLocal.py (100%) rename src/{ => backends}/spotify/ParseSpotify.py (100%) rename src/{ => backends}/spotify/SpotifyAuthenticator.py (100%) rename src/{ => backends}/spotify/SpotifyWebAPI.py (98%) diff --git a/Flow.py b/Flow.py index 4731f1c..e1ac33d 100644 --- a/Flow.py +++ b/Flow.py @@ -8,11 +8,11 @@ from Env import work_dir from src.Library import * from src.LibrarySerializer import LibrarySaver, LibraryLoader -from src.spotify.SpotifyWebAPI import fetch_all -from src.spotify.ParseSpotify import Parser +from src.backends.spotify.SpotifyWebAPI import fetch_all +from src.backends.spotify.ParseSpotify import Parser -from src.local.LocalLibraryIndexer import update_local_library -from src.local.ParseLocal import LocalParser +from src.backends.local.LocalLibraryIndexer import update_local_library +from src.backends.local.ParseLocal import LocalParser from src.mappers.RemoteLibraryResolver import resolve_remote_tracks from src.StatGenerator import log_stats diff --git a/main.py b/main.py index 8499c07..af9ed31 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ import Flow def test1(): flow = Flow.Flow() - # flow.fetch_local() + flow.fetch_local() flow.parse_local_library() flow.load_libraries() diff --git a/src/itunes/ParseItunesToJson.py b/src/backends/itunes/ParseItunesToJson.py similarity index 100% rename from src/itunes/ParseItunesToJson.py rename to src/backends/itunes/ParseItunesToJson.py diff --git a/src/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py similarity index 100% rename from src/local/LocalLibraryIndexer.py rename to src/backends/local/LocalLibraryIndexer.py diff --git a/src/local/LocalPLaylistGenerator.py b/src/backends/local/LocalPLaylistGenerator.py similarity index 100% rename from src/local/LocalPLaylistGenerator.py rename to src/backends/local/LocalPLaylistGenerator.py diff --git a/src/local/ParseLocal.py b/src/backends/local/ParseLocal.py similarity index 100% rename from src/local/ParseLocal.py rename to src/backends/local/ParseLocal.py diff --git a/src/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py similarity index 100% rename from src/spotify/ParseSpotify.py rename to src/backends/spotify/ParseSpotify.py diff --git a/src/spotify/SpotifyAuthenticator.py b/src/backends/spotify/SpotifyAuthenticator.py similarity index 100% rename from src/spotify/SpotifyAuthenticator.py rename to src/backends/spotify/SpotifyAuthenticator.py diff --git a/src/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py similarity index 98% rename from src/spotify/SpotifyWebAPI.py rename to src/backends/spotify/SpotifyWebAPI.py index 5ddc12f..8f4807f 100644 --- a/src/spotify/SpotifyWebAPI.py +++ b/src/backends/spotify/SpotifyWebAPI.py @@ -2,7 +2,7 @@ import requests from PIL import Image from io import BytesIO -from src.spotify.SpotifyAuthenticator import authenticate +from src.backends.spotify.SpotifyAuthenticator import authenticate from src.helpers import * diff --git a/src/lyrics/lyrics.py b/src/lyrics/lyrics.py index fb427de..ae6ed6d 100644 --- a/src/lyrics/lyrics.py +++ b/src/lyrics/lyrics.py @@ -7,11 +7,13 @@ def fetch_lyrics(artist: str, album: str, track: str, duration: int = 0) -> str: skip_artists = [ "shurupis", - "unknown" + "unknown", + "Unknown" ] skip_album = [ - "unknown" + "unknown", + "Unknown" ] if artist in skip_artists or album in skip_album: diff --git a/src/mappers/SpotifySearchMapping.py b/src/mappers/SpotifySearchMapping.py index 41c5bbb..e8878e9 100644 --- a/src/mappers/SpotifySearchMapping.py +++ b/src/mappers/SpotifySearchMapping.py @@ -3,7 +3,7 @@ from tqdm import tqdm from src.Library import * from src.helpers import * -from src.spotify.SpotifyWebAPI import find_song, update_access_token +from src.backends.spotify.SpotifyWebAPI import find_song, update_access_token def generate_map(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir): From 99cd872227fa01bd237a5994ef10b3712e489141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sat, 7 Feb 2026 00:48:46 +0300 Subject: [PATCH 13/18] nice --- Flow.py | 4 + Gui.py | 206 ++++++++++++++++++++++ error.log | 2 + gui.sh | 1 + main.py | 14 +- src/Library.py | 123 +++++++++---- src/LibrarySerializer.py | 31 +++- src/StatGenerator.py | 21 +-- src/backends/local/LocalLibraryIndexer.py | 194 +++++++++++--------- src/backends/local/ParseLocal.py | 53 +----- src/backends/navidrome/navidrome.py | 110 ++++++++++++ src/backends/spotify/ParseSpotify.py | 7 +- src/mappers/FuzzyMapper.py | 54 +++++- 13 files changed, 630 insertions(+), 190 deletions(-) create mode 100644 Gui.py create mode 100644 error.log create mode 100755 gui.sh create mode 100644 src/backends/navidrome/navidrome.py diff --git a/Flow.py b/Flow.py index e1ac33d..5e813e0 100644 --- a/Flow.py +++ b/Flow.py @@ -96,6 +96,10 @@ class Flow: self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json") self.local_library = LibraryLoader().load(work_dir / "local_library.json") + def save_libraries(self): + LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json") + LibrarySaver(self.local_library).save(work_dir / "local_library.json") + def map_local_to_spotify(self): resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir) diff --git a/Gui.py b/Gui.py new file mode 100644 index 0000000..85fd6e3 --- /dev/null +++ b/Gui.py @@ -0,0 +1,206 @@ +import time + +import streamlit as st +from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode +import pandas as pd + +import Flow + +from src.Library import Library, Artist, Album, Track, Playlist + +def track_to_row(track: Track): + return { + "Title": track.title, + "Album": track.album.title if track.album else "", + "Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists), + "Lyrics": track.has_lyrics, + "AutoScore": track.auto_score, + "Added": track.added_at.strftime("%Y-%m-%d") if track.added_at else "", + "LocalId": str(track.local_id) if track.local_id else "", + "Path": str(track.local_path) if track.local_path else "", + } + +def artist_to_row(artist: Artist): + return { + "Title": artist.title, + "AutoScore": artist.auto_score, + "Resolved": int(artist.resolved_percentage * 100), + "LocalId": str(artist.local_id) if artist.local_id else "", + } + +def album_to_row(album: Album): + return { + "Title": album.title, + "AutoScore": album.auto_score, + "Resolved": int(album.resolved_percentage * 100), + "LocalId": str(album.local_id) if album.local_id else "", + } + + +class LibraryView: + def __init__(self): + self.lib : Library = None + self.libs : list[Library] = None + self.view = None + self.lyrics_only = False + self.query = None + + def add_libraries(self, libs : list[Library]): + self.libs = libs + + def render(self): + st.set_page_config(layout="wide") + + with st.sidebar: + lib = st.selectbox( + "Library", + options=[lib.name for lib in self.libs], + ) + + for iter in self.libs: + if iter.name == lib: + self.lib = iter + + st.write(f"Showing library '{self.lib.name}'") + + self.view = st.radio( + "View", + ["Tree"] + ) + + if self.view == "Tree": + self.render_artists_albums_tracks() + # elif self.view == "Tracks": + # self.render_tracks() + # elif self.view == "Playlists": + # self.render_playlists() + + def track_visible( + self, + track: Track, + artist: Artist | None, + album: Album | None, + ): + if self.lyrics_only and not track.has_lyrics: + return False + + if self.query and self.query.lower() not in track.title.lower(): + return False + + if artist and all(a.id != artist.id for a in track.artists): + return False + + if album and (not track.album or track.album.id != album.id): + return False + + return True + + def render_artists_albums_tracks(self): + # --- ARTISTS TABLE --- + st.subheader("Artists") + artist_rows = [artist_to_row(a) for a in self.lib.artists] + df_artists = pd.DataFrame(artist_rows) + + gb_artists = GridOptionsBuilder.from_dataframe(df_artists) + gb_artists.configure_selection(selection_mode="multiple", use_checkbox=True) + gb_artists.configure_column("Title", filter="agTextColumnFilter") + gb_artists.configure_column("AutoScore", sort="desc") + grid_options_artists = gb_artists.build() + + grid_response_artists = AgGrid( + df_artists, + gridOptions=grid_options_artists, + update_mode=GridUpdateMode.SELECTION_CHANGED, + allow_unsafe_jscode=True, + enable_enterprise_modules=False, + fit_columns_on_grid_load=True, + ) + + selected_rows_df = grid_response_artists.get("selected_rows") + selected_artist_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else [] + selected_artists = [a for a in self.lib.artists if a.title in selected_artist_titles] + + # --- ALBUMS TABLE --- + st.subheader("Albums") + if selected_artists: + albums_list = [] + for artist in selected_artists: + albums_list.extend(artist.albums) + albums_list = list({al.id: al for al in albums_list}.values()) + else: + albums_list = self.lib.albums + + album_rows = [album_to_row(a) for a in albums_list] + df_albums = pd.DataFrame(album_rows) + + gb_albums = GridOptionsBuilder.from_dataframe(df_albums) + gb_albums.configure_selection(selection_mode="multiple", use_checkbox=True) + gb_albums.configure_column("Title", filter="agTextColumnFilter") + gb_albums.configure_column("AutoScore", sort="desc") + grid_options_albums = gb_albums.build() + + grid_response_albums = AgGrid( + df_albums, + gridOptions=grid_options_albums, + update_mode=GridUpdateMode.SELECTION_CHANGED, + allow_unsafe_jscode=True, + enable_enterprise_modules=False, + fit_columns_on_grid_load=True, + ) + + selected_rows_df = grid_response_albums.get("selected_rows") + selected_album_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else [] + selected_albums = [al for al in albums_list if al.title in selected_album_titles] + + # --- TRACKS TABLE --- + st.subheader("Tracks") + if selected_albums: + tracks_list = [] + for album in selected_albums: + tracks_list.extend(album.tracks) + elif selected_artists: + tracks_list = [] + for artist in selected_artists: + tracks_list.extend([t for t in self.lib.tracks if artist in t.artists]) + else: + tracks_list = self.lib.tracks + + track_rows = [track_to_row(t) for t in tracks_list] + df_tracks = pd.DataFrame(track_rows) + + gb_tracks = GridOptionsBuilder.from_dataframe(df_tracks) + gb_tracks.configure_selection(selection_mode="multiple", use_checkbox=True) + gb_tracks.configure_column("Title", filter="agTextColumnFilter") + gb_tracks.configure_column("AutoScore", sort="desc") + grid_options_tracks = gb_tracks.build() + + AgGrid( + df_tracks, + gridOptions=grid_options_tracks, + update_mode=GridUpdateMode.SELECTION_CHANGED, + allow_unsafe_jscode=True, + enable_enterprise_modules=False, + fit_columns_on_grid_load=True, + ) + + +def run(): + flow = Flow.Flow() + + # flow.fetch_spotify() + # flow.fetch_local() + + # flow.parse_spotify_library() + # flow.parse_local_library() + + flow.load_libraries() + + # flow.map_local_to_spotify() + + # flow.log_stats() + + gui = LibraryView() + gui.add_libraries([flow.spotify_library, flow.local_library]) + gui.render() + +run() \ No newline at end of file diff --git a/error.log b/error.log new file mode 100644 index 0000000..7c830a2 --- /dev/null +++ b/error.log @@ -0,0 +1,2 @@ + +(process:38049): GLib-GIO-CRITICAL **: 14:06:16.561: g_dbus_connection_emit_signal: assertion 'G_IS_DBUS_CONNECTION (connection)' failed diff --git a/gui.sh b/gui.sh new file mode 100755 index 0000000..eb55739 --- /dev/null +++ b/gui.sh @@ -0,0 +1 @@ +streamlit run Gui.py diff --git a/main.py b/main.py index af9ed31..646539a 100644 --- a/main.py +++ b/main.py @@ -5,15 +5,14 @@ import Flow def test1(): flow = Flow.Flow() - flow.fetch_local() + # flow.fetch_local() flow.parse_local_library() flow.load_libraries() # flow.map_local_to_spotify() # flow.log_stats() - flow.fetch_and_save_lyrics() - + # flow.fetch_and_save_lyrics() def main(): flow = Flow.Flow() @@ -27,11 +26,12 @@ def main(): flow.load_libraries() flow.map_local_to_spotify() + flow.save_libraries() - flow.log_stats() + # flow.log_stats() - print("asd") + # print("asd") -test1() -# main() +# test1() +main() diff --git a/src/Library.py b/src/Library.py index 0de72ce..46d7566 100644 --- a/src/Library.py +++ b/src/Library.py @@ -7,64 +7,86 @@ from typing import List, Optional, Any class Artist: def __init__(self): - self.id : str = None - self.title : str = None - self.tracks : List[Track] = [] - self.albums : List[Album] = [] + self.id: str = None + self.title: str = None + self.tracks: List[Track] = [] + self.albums: List[Album] = [] + self.local_id: str = None + self.auto_score: float = 0 + self.fav: bool = False + self.rating: int = 0 + self.resolved_percentage : float = 0 class Track: def __init__(self): - self.id : str = None - self.title : str = None - self.artists : List[Artist] = [] - self.album : Album = None - self.duration_ms : int = None - self.added_at : datetime = None - self.local_path : Path = None - self.has_lyrics : bool = False + self.id: str = None + self.title: str = None + self.artists: List[Artist] = [] + self.album: Album = None + self.duration_ms: int = None + self.added_at: datetime = None + self.local_path: Path = None + self.has_lyrics: bool = False + self.local_id: str = None + self.auto_score: float = 0 + self.fav: bool = False + self.rating: int = 0 class Album: def __init__(self): - self.id : str = None - self.title : str = None - self.release_date : datetime = None - self.tracks : List[Track] = [] - self.artists : List[Artist] = [] + self.id: str = None + self.title: str = None + self.release_date: datetime = None + self.tracks: List[Track] = [] + self.artists: List[Artist] = [] + self.local_id: str = None + self.auto_score: float = 0 + self.fav: bool = False + self.rating: int = 0 + self.resolved_percentage : float = 0 class Playlist: def __init__(self): - self.title : str = None - self.tracks : List[Track] = [] - self.created_at : datetime = None - self.description : str = None + self.title: str = None + self.tracks: List[Track] = [] + self.created_at: datetime = None + self.description: str = None + self.auto_score: float = 0 + self.fav: bool = False + self.rating: int = 0 + self.resolved_percentage : float = 0 class Library: def __init__(self): - self.artists : List[Artist] = [] - self.playlists : List[Playlist] = [] - self.tracks : List[Track] = [] - self.albums : List[Album] = [] - self.liked_tracks : List[Track] = [] - self.top_tracks : List[Track]= [] - self.top_artists : List[Artist] = [] + self.artists: List[Artist] = [] + self.playlists: List[Playlist] = [] + self.tracks: List[Track] = [] + self.albums: List[Album] = [] - self.artist_map : dict[str, Artist] = None - self.album_map : dict[str, Album] = None - self.track_map : dict[str, Track] = None + self.liked_tracks: List[Track] = [] + self.top_tracks: List[Track] = [] + self.top_artists: List[Artist] = [] + self.artist_map: dict[str, Artist] = None + self.album_map: dict[str, Album] = None + self.track_map: dict[str, Track] = None + + self.name = "Unnamed" def update_cache(self): self.update_id_maps() self.create_reverse_links() + self.score_albums() + self.calc_resolved_percentage() def update_id_maps(self): - self.artist_map : dict[str, Artist] = { artist.id : artist for artist in self.artists } - self.album_map : dict[str, Album] = { album.id : album for album in self.albums } - self.track_map : dict[str, Track] = { track.id : track for track in self.tracks } + self.artist_map: dict[str, Artist] = {artist.id: artist for artist in self.artists} + self.album_map: dict[str, Album] = {album.id: album for album in self.albums} + self.track_map: dict[str, Track] = {track.id: track for track in self.tracks} def create_reverse_links(self): for track_id, track in self.track_map.items(): @@ -79,3 +101,36 @@ class Library: for artist in album.artists: if album not in artist.albums: artist.albums.append(album) + + def auto_score(self): + self.score_tracks() + self.score_artists() + self.score_albums() + + def score_tracks(self): + for idx, track in enumerate(self.top_tracks): + score = len(self.top_tracks) - idx + track.auto_score = score + + def score_artists(self): + for track in self.tracks: + for artist in track.artists: + artist.auto_score += track.auto_score + + def score_albums(self): + for track in self.tracks: + track.album.auto_score += track.auto_score + + def calc_resolved_percentage(self): + for artist in self.artists: + artist_count_resolved = 0 + artist_count = 0 + for album in artist.albums: + album_count_resolved = 0 + for track in album.tracks: + if track.local_id: + album_count_resolved += 1 + album.resolved_percentage = album_count_resolved / len(album.tracks) + artist_count_resolved += album_count_resolved + artist_count += len(album.tracks) + artist.resolved_percentage = artist_count_resolved / artist_count \ No newline at end of file diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py index 84f6a46..f331824 100644 --- a/src/LibrarySerializer.py +++ b/src/LibrarySerializer.py @@ -33,6 +33,10 @@ class LibrarySaver: "title": artist.title, "tracks": [track.id for track in artist.tracks], "albums": [album.id for album in artist.albums], + "local_id": artist.local_id, + "auto_score": artist.auto_score, + "fav" : artist.fav, + "rating" : artist.rating, } @staticmethod @@ -46,6 +50,10 @@ class LibrarySaver: "added_at": _dt_to_str(track.added_at), "local_path": track.local_path, "has_lyrics": track.has_lyrics, + "local_id": track.local_id, + "auto_score": track.auto_score, + "fav" : track.fav, + "rating" : track.rating, } @staticmethod @@ -55,6 +63,9 @@ class LibrarySaver: "tracks": [track.id for track in playlist.tracks], "created_at": _dt_to_str(playlist.created_at), "description": playlist.description, + "auto_score": playlist.auto_score, + "fav" : playlist.fav, + "rating" : playlist.rating, } @staticmethod @@ -64,12 +75,17 @@ class LibrarySaver: "title": album.title, "release_date": album.release_date, "tracks": [track.id for track in album.tracks], - "artists": [artist.id for artist in album.artists] + "artists": [artist.id for artist in album.artists], + "local_id": album.local_id, + "auto_score": album.auto_score, + "fav" : album.fav, + "rating" : album.rating, } def save(self, path: Path) -> None: data = { + "name": self.library.name, "tracks": [self._track_to_dict(track) for track in self.library.tracks], "albums": [self._album_to_dict(album) for album in self.library.albums], "artists": [self._artist_to_dict(artist) for artist in self.library.artists], @@ -121,6 +137,10 @@ class LibraryLoader: track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] track.album = self.get_album(data.get("album")) track.has_lyrics = data.get("has_lyrics") + track.local_id = data.get("local_id") + track.auto_score = data.get("auto_score") + track.fav = data.get("fav") + track.rating = data.get("rating") self.track_map[track.id] = track return track @@ -131,6 +151,10 @@ class LibraryLoader: artist.title = data.get("title") artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else [] + artist.local_id = data.get("local_id") + artist.auto_score = data.get("auto_score") + artist.fav = data.get("fav") + artist.rating = data.get("rating") self.artist_map[artist.id] = artist return artist @@ -141,6 +165,10 @@ class LibraryLoader: album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] album.title = data.get("title") album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] + album.local_id = data.get("local_id") + album.auto_score = data.get("auto_score") + album.fav = data.get("fav") + album.rating = data.get("rating") self.album_map[album.id] = album return album @@ -151,6 +179,7 @@ class LibraryLoader: library = Library() + library.name = data.get("name") library.tracks = [self.load_track(track) for track in data.get("tracks")] library.albums = [self.load_album(album) for album in data.get("albums")] library.artists = [self.load_artist(artist) for artist in data.get("artists")] diff --git a/src/StatGenerator.py b/src/StatGenerator.py index ac16622..7c17ab5 100644 --- a/src/StatGenerator.py +++ b/src/StatGenerator.py @@ -15,13 +15,12 @@ def get_mapping(mapping_path) -> dict[str, Any]: with Path(mapping_path).open("r", encoding="utf-8") as f: data = json.load(f) - return data + return data["track_map"] class AlbumStat: def __init__(self): self.album: Album = Album() - self.score: int = 0 self.resolved_precent: float = 0 def calc_resolved_precent(self): @@ -37,7 +36,6 @@ class AlbumStat: class ArtistStat: def __init__(self): self.artist: Artist = Artist() - self.score: int = 0 self.resolved_precent: float = 0 self.albums_stats: list[AlbumStat] = [] @@ -87,18 +85,22 @@ class LibraryStats: for remote, local in self.mapping.items(): self.remote_id_map[remote].local_path = self.local_id_map[local].local_path + def score_tracks(self): + for idx, track in enumerate(self.remote_lib.top_tracks): + score = len(self.remote_lib.top_tracks) - idx + track.auto_score = score + def score_artists(self): for artist in self.remote_lib.artists: artist_stat = ArtistStat() artist_stat.artist = artist - artist_stat.score = 0 + artist_stat.artist.auto_score = 0 self.artists_map[artist_stat.artist.id] = artist_stat self.artists.append(artist_stat) - for idx, track in enumerate(self.remote_lib.top_tracks): - score = len(self.remote_lib.top_tracks) - idx + for track in self.remote_lib.tracks: for artist in track.artists: - self.artists_map[artist.id].score += score + self.artists_map[artist.id].artist.auto_score += track.auto_score for artist_stat in self.artists: artist_stat.calc_resolved_precent() @@ -111,9 +113,8 @@ class LibraryStats: stat.album = album self.albums_map[album.id] = stat - for idx, track in enumerate(self.remote_lib.top_tracks): - score = len(self.remote_lib.top_tracks) - idx - self.albums_map[track.album.id].score += score + for track in self.remote_lib.tracks: + self.albums_map[track.album.id].album.auto_score += track.auto_score for _, album in self.albums_map.items(): album.calc_resolved_precent() diff --git a/src/backends/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py index 1a8eaab..6f132d4 100644 --- a/src/backends/local/LocalLibraryIndexer.py +++ b/src/backends/local/LocalLibraryIndexer.py @@ -1,64 +1,97 @@ +import os import fnmatch -import mutagen -import eyed3 -from mutagen.flac import FLAC from mutagen import File as MutagenFile +from mutagen.id3 import ID3, ID3NoHeaderError +from mutagen.flac import FLAC +from mutagen.mp4 import MP4 +from mutagen.oggvorbis import OggVorbis +from tqdm import tqdm from src.helpers import * - music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') exclude_directories = ("*mary--*", "*example-word*") -from mutagen.id3 import ID3 -from mutagen.id3 import ID3NoHeaderError - - def detect_lyrics(abs_path, ext): ext = ext.lower() - # ---------- MP3 ---------- - if ext == ".mp3": - try: - tags = ID3(abs_path) - except ID3NoHeaderError: - return False - - return bool( - tags.getall("SYLT") or - tags.getall("USLT") - ) - - # ---------- FLAC / OGG / OPUS ---------- - if ext in {".flac", ".ogg", ".opus"}: - try: - audio = MutagenFile(abs_path) - if not audio or not audio.tags: - return False - - for key in audio.tags.keys(): - k = key.upper() - if k in {"LYRICS", "LRC", "SYNCEDLYRICS"}: - if audio.tags[key]: - return True - except Exception: - return False - - # ---------- Fallback ---------- try: audio = MutagenFile(abs_path) - if audio and audio.tags: - for key in audio.tags.keys(): - if "LYRIC" in key.upper(): - return True + if not audio or not audio.tags: + return False + + # MP3-specific SYLT/USLT detection + if ext == ".mp3" and isinstance(audio, ID3): + return bool(audio.getall("SYLT") or audio.getall("USLT")) + + # FLAC/OGG/OPUS lyrics detection + for key, value in audio.tags.items(): + k = key.upper() + if k in {"LYRICS", "LRC", "SYNCEDLYRICS"} and value: + return True + + # Generic fallback + for key in audio.tags.keys(): + if "LYRIC" in key.upper(): + return True + except Exception: - pass + return False return False -def update_local_library(root_dir, output_dir): +def get_artist(tags, fallback=None) -> list : + # FLAC / OGG / general multiple artist fields + artist_fields = ["ARTISTS", "TXXX:ARTISTS", "©ART", "artist"] + for field in artist_fields: + if field in tags and tags[field]: + values = tags.get(field) + if not isinstance(values, list): + return values.text + return values + + # M4A + if '\xa9ART' in tags and tags['\xa9ART']: + values = tags['\xa9ART'] + return values + + if tags.get("TPE1"): + text = getattr(tags.get("TPE1"), "text", None) + if text: + return text + + return fallback + +def safe_tag_get(tags, key, index=0): + try: + value = tags.get(key, None) + if isinstance(value, list): + # filter out None/empty, convert to str + value = [str(v) for v in value if v] + return value[index] if len(value) > index else fallback + return str(value) + except (KeyError, ValueError, TypeError): + return None + +def get_title(tags): + ids = ["TIT2", "TITLE", "title", "\xa9nam"] + for key in ids: + val = safe_tag_get(tags, key) + if val and val != "None": + return val + return None + +def get_album(tags): + ids = ["TALB", "album", "\xa9alb"] + for key in ids: + val = safe_tag_get(tags, key) + if val and val != "None": + return val + return None + +def update_local_library(root_dir, output_dir): set_workdir(output_dir / "local") old_library = get_data("local_library") @@ -91,61 +124,48 @@ def update_local_library(root_dir, output_dir): song_id = get_new_song_id(song_id, relative_path) new_library[song_id] = { - "name": song_name, + "name": None, "path": relative_path, "type": track_type, - "artist": "", - "album": "", + "artists": None, + "album": None, "duration": None } - for track_id, track in new_library.items(): - abs_path = os.path.join(root_dir, track['path']) + to_remove = [] - track_abs_path = os.path.join(root_dir, track["path"]) - track["has_lyrics"] = detect_lyrics(track_abs_path, track['type'].lower()) + with tqdm(total=len(new_library.items()), desc='Indexing local library') as process_bar: + for track_id, track in new_library.items(): + process_bar.update(1) + + abs_path = os.path.join(root_dir, track['path']) + track["has_lyrics"] = detect_lyrics(abs_path, track['type'].lower()) + + try: + audio = MutagenFile(abs_path) + if audio and audio.tags: + tags = audio.tags + + track['artists'] = get_artist(tags) + track['name'] = get_title(tags) + track['album'] = get_album(tags) + + if not len(track['artists']) or not track['name'] or not track['album']: + print(track) + raise "Cannot index track" - match track['type'].lower(): - case ".mp3": - mp3track = eyed3.load(abs_path) - if not mp3track or not mp3track.info: - print(f"Failed to load mp3 {abs_path}") - track['name'] = track['path'] else: - track['duration'] = mp3track.info.time_secs + print(track) + to_remove.append(track_id) + # raise "Cannot index track" - tag = mp3track.tag - if tag: - track['artist'] = tag.artist - track['name'] = tag.title - track['album'] = tag.album + except Exception as e: + print(e) + print(abs_path) + raise e - case ".flac": - try: - audio = FLAC(abs_path) - if audio.info: - track['duration'] = audio.info.length - - metadata = audio.tags - if metadata: - if 'artist' in metadata: - track['artist'] = " ".join(metadata['artist']) - if 'TITLE' in metadata: - track['name'] = " ".join(metadata['TITLE']) - if 'album' in metadata: - track['album'] = " ".join(metadata['album']) - - except mutagen.MutagenError: - print(f"Invalid flac header {abs_path}") - track['name'] = track['path'] - - case _: - try: - audio = MutagenFile(abs_path) - if audio and audio.info: - track['duration'] = audio.info.length - except Exception: - pass + for rem in to_remove: + new_library.pop(rem) save_data(new_library, "local_library") diff --git a/src/backends/local/ParseLocal.py b/src/backends/local/ParseLocal.py index 49b3f55..b1fbba3 100644 --- a/src/backends/local/ParseLocal.py +++ b/src/backends/local/ParseLocal.py @@ -10,46 +10,6 @@ PROTECTED_ARTISTS = { "The Good, The Bad & The Queen", } -def split_artists(artist_tag: str) -> list[str]: - if not artist_tag: - return [] - - placeholder_comma = "§COMMA§" - placeholder_amp = "§AMP§" - - protected_map = {} - - # Protect known artist names - for artist in PROTECTED_ARTISTS: - protected = ( - artist - .replace(",", placeholder_comma) - .replace("&", placeholder_amp) - ) - protected_map[protected] = artist - artist_tag = artist_tag.replace(artist, protected) - - # Split remaining separators - parts = re.split(r"[,&]", artist_tag) - - # Restore protected names and trim spaces - result = [] - for part in parts: - part = part.strip() - if not part: - continue - - part = ( - part - .replace(placeholder_comma, ",") - .replace(placeholder_amp, "&") - ) - - result.append(part) - - return result - - class LocalParser: def __init__(self): self.track_map: dict[str, Track] = {} @@ -60,9 +20,7 @@ class LocalParser: def parse_artists(self, data: str) -> List[Artist]: if not data: - data = "unknown" - - data = split_artists(data) + raise "No artist found" artists = [] @@ -87,7 +45,7 @@ class LocalParser: def parse_album(self, data: str) -> Album: if not data: - data = "unknown" + raise "album is none" name = data album_id = name @@ -113,12 +71,15 @@ class LocalParser: track.id = track_id track.title = data.get("name") - track.artists = self.parse_artists(data.get("artist")) + track.artists = self.parse_artists(data.get("artists")) track.album = self.parse_album(data.get("album")) track.local_path = data.get("path") track.duration_ms = data.get("duration") track.has_lyrics = data.get("has_lyrics") + if not track.album or not len(track.artists) or not track.title: + raise "error parsing" + self.track_map[track.id] = track self.library.tracks.append(track) @@ -129,6 +90,8 @@ class LocalParser: self.library.update_cache() + self.library.name = "Local" + return self.library diff --git a/src/backends/navidrome/navidrome.py b/src/backends/navidrome/navidrome.py new file mode 100644 index 0000000..c7d04f4 --- /dev/null +++ b/src/backends/navidrome/navidrome.py @@ -0,0 +1,110 @@ +import sqlite3 +import os +from dataclasses import dataclass, field +from typing import List, Optional + +@dataclass +class Track: + id: int + title: str + path: str + favorite: bool + rating: Optional[int] + play_count: int + last_played: Optional[str] + playlists: List[str] = field(default_factory=list) + +@dataclass +class Album: + id: int + title: str + tracks: List[Track] = field(default_factory=list) + +@dataclass +class Artist: + id: str + name: str + albums: List[Album] = field(default_factory=list) + +class NavidromeDB: + def __init__(self, db_path: str): + if not os.path.isfile(db_path): + raise FileNotFoundError(f"Navidrome DB not found: {db_path}") + self.conn = sqlite3.connect(db_path) + self.conn.row_factory = sqlite3.Row + + def close(self): + self.conn.close() + + def get_artists(self) -> List[Artist]: + artists = [] + c = self.conn.cursor() + c.execute("SELECT id, name FROM artist ORDER BY name") + for row in c.fetchall(): + artist = Artist(id=row["id"], name=row["name"]) + artist.albums = self.get_albums(artist.id) + artists.append(artist) + return artists + + def get_albums(self, artist_id: str) -> List[Album]: + albums = [] + c = self.conn.cursor() + c.execute( + "SELECT id, name FROM album WHERE album_artist_id=? ORDER BY name", + (artist_id,) + ) + for row in c.fetchall(): + album = Album(id=row["id"], title=row["name"]) + album.tracks = self.get_tracks(album.id) + albums.append(album) + return albums + + def get_tracks(self, album_id: int) -> List[Track]: + tracks = [] + c = self.conn.cursor() + c.execute( + "SELECT t.id, t.name AS Title, t.path, ut.favorite, ut.rating, ut.play_count AS PlayCount, ut.last_played AS LastPlayed " + "FROM track t " + "LEFT JOIN usertrack ut ON ut.track_id = t.id " + "WHERE t.album_id=? ORDER BY t.track_number", + (album_id,) + ) + for row in c.fetchall(): + track = Track( + id=row["id"], + title=row["Title"], + path=row["path"], + favorite=bool(row["favorite"]), + rating=row["rating"], + play_count=row["PlayCount"] or 0, + last_played=row["LastPlayed"], + playlists=self.get_playlists_for_track(row["id"]) + ) + tracks.append(track) + return tracks + + def get_playlists_for_track(self, track_id: int) -> List[str]: + c = self.conn.cursor() + c.execute( + "SELECT p.name FROM playlist p " + "JOIN playlisttrack pt ON pt.playlist_id=p.id " + "WHERE pt.track_id=?", + (track_id,) + ) + return [row["name"] for row in c.fetchall()] + +# Example usage +if __name__ == "__main__": + db_path = "/mnt/main/data/music/navidrome.db" + navidb = NavidromeDB(db_path) + + try: + artists = navidb.get_artists() + for artist in artists: + print(f"Artist: {artist.name}") + for album in artist.albums: + print(f" Album: {album.title}") + for track in album.tracks: + print(f" Track: {track.title} | Path: {track.path} | Fav: {track.favorite} | Plays: {track.play_count}") + finally: + navidb.close() diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py index 50806bf..29c24e7 100644 --- a/src/backends/spotify/ParseSpotify.py +++ b/src/backends/spotify/ParseSpotify.py @@ -30,6 +30,8 @@ class Parser: self.parse_top_tracks(json.loads(top_tracks.read_text())) self.library.update_cache() + self.library.name = "Spotify" + self.library.auto_score() return self.library @@ -75,7 +77,7 @@ class Parser: track.id = track_id track.title = data.get("name") - track.artists = [self.parse_artist(artist) for artist in data.get("artists")] + track.artists = [] + self.parse_album(data.get("album")).artists track.album = self.parse_album(data.get("album")) track.duration_ms = data.get("duration_ms") @@ -114,6 +116,9 @@ class Parser: artist.id = data.get("id") artist.title = data.get("name") + if artist.title is None: + artist.title = "err_no_title" + self.library.artists.append(artist) return artist diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py index ca78db9..c6413d0 100644 --- a/src/mappers/FuzzyMapper.py +++ b/src/mappers/FuzzyMapper.py @@ -5,12 +5,46 @@ from tqdm import tqdm from src.Library import * from src.helpers import * +import unicodedata + +user_def_artist_map = { + "donda" : "kanye west", +} + +def normalize(s): + return unicodedata.normalize("NFKC", s).replace("‐", "-") def get_score(first: str, second: str) -> float: + first_norm = normalize(first).lower() + second_norm = normalize(second).lower() + return float(first_norm == second_norm) * 100 return fuzz.token_set_ratio(first, second) +def get_album_score(first: str, second: str) -> float: + return get_score(first, second) + +def get_track_score(first: str, second: str) -> float: + return get_score(first, second) + +def get_artist_score(first: str, second: str) -> float: + if not first or not second: + return 0.0 + + first_lower = first.lower() + second_lower = second.lower() + + for key, value in user_def_artist_map.items(): + key_lower = key.lower() + value_lower = value.lower() + if (first_lower == key_lower and second_lower == value_lower) or \ + (first_lower == value_lower and second_lower == key_lower): + return 100.0 + + return get_score(first, second) + + def is_close_enough(first: float) -> bool: - return first > 50 + return first > 90 def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path): @@ -21,12 +55,13 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path): with (tqdm(total=len(remote_lib.artists), desc='searching remote artists in local library') as process_bar): for remote_artist in remote_lib.artists: for local_artist in local_lib.artists: - score = get_score(remote_artist.title, local_artist.title) + score = get_artist_score(remote_artist.title, local_artist.title) if not is_close_enough(score): continue if remote_artist.id not in artist_map or score > artist_map[remote_artist.id][1]: artist_map[remote_artist.id] = (local_artist.id, score) + remote_artist.local_id = local_artist.id process_bar.update(1) @@ -38,12 +73,13 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path): for remote_album in remote_artist.albums: for local_album in local_artist.albums: - score = get_score(remote_album.title, local_album.title) + score = get_album_score(remote_album.title, local_album.title) if not is_close_enough(score): continue if remote_album.id not in artist_map or score > album_map[remote_album.id][1]: album_map[remote_album.id] = (local_album.id, score) + remote_album.local_id = local_album.id process_bar.update(1) @@ -54,15 +90,23 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path): for remote_track in remote_album.tracks: for local_track in local_album.tracks: - score = get_score(remote_track.title, local_track.title) + score = get_track_score(remote_track.title, local_track.title) if not is_close_enough(score): continue if remote_track.id not in artist_map or score > track_map[remote_track.id][1]: track_map[remote_track.id] = (local_track.id, score) + remote_track.local_id = local_track.id process_bar.update(1) - out = { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() } + remote_lib.calc_resolved_percentage() + + out = { + "artist_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in artist_map.items() }, + "album_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in album_map.items() }, + "track_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() }, + } + set_workdir(output_dir) save_data(out, "remote_to_local_map") From b6c59e81adfdf30c9cbcb94908b70245d10a4b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sat, 7 Feb 2026 11:24:59 +0300 Subject: [PATCH 14/18] tmp --- Flow.py | 13 ++ main.py | 7 +- src/Library.py | 17 ++ src/backends/itunes/ParseItunesToJson.py | 101 ------------ src/backends/itunes/ParseItunesXml.py | 192 +++++++++++++++++++++++ 5 files changed, 226 insertions(+), 104 deletions(-) delete mode 100755 src/backends/itunes/ParseItunesToJson.py create mode 100755 src/backends/itunes/ParseItunesXml.py diff --git a/Flow.py b/Flow.py index 5e813e0..0031017 100644 --- a/Flow.py +++ b/Flow.py @@ -13,6 +13,8 @@ from src.backends.spotify.ParseSpotify import Parser from src.backends.local.LocalLibraryIndexer import update_local_library from src.backends.local.ParseLocal import LocalParser +from src.backends.itunes.ParseItunesXml import ParseItunesXml + from src.mappers.RemoteLibraryResolver import resolve_remote_tracks from src.StatGenerator import log_stats @@ -43,6 +45,17 @@ class Flow: LibrarySaver(library_parsed).save(work_dir / "spotify_library.json") + @staticmethod + def parse_itunes_library(): + parser = ParseItunesXml() + + library_parsed = parser.parse( + work_dir / "itunes" / "Library.xml", + work_dir / "itunes" / "Library.json" + ) + + LibrarySaver(library_parsed).save(work_dir / "itunes_library.json") + @staticmethod def parse_local_library(): parser = LocalParser() diff --git a/main.py b/main.py index 646539a..26588d1 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,6 @@ - - import Flow + def test1(): flow = Flow.Flow() @@ -14,13 +13,15 @@ def test1(): # flow.fetch_and_save_lyrics() + def main(): flow = Flow.Flow() # flow.fetch_spotify() - flow.fetch_local() + # flow.fetch_local() flow.parse_spotify_library() + flow.parse_itunes_library() flow.parse_local_library() flow.load_libraries() diff --git a/src/Library.py b/src/Library.py index 46d7566..7bcefac 100644 --- a/src/Library.py +++ b/src/Library.py @@ -4,6 +4,23 @@ from datetime import datetime from pathlib import Path from typing import List, Optional, Any +class Entity: + def __init__(self): + self.id: str = None + self.mbid : str = None + + self.title: str = None + + self.fav: bool = False + self.rating: int = 0 + self.play_count : int = 0 + self.date_added : datetime = None + self.date_released : datetime = None + self.date_last_play : datetime = None + self.auto_score: float = 0 + + self.local_id: str = None + self.resolved_percentage : float = 0 class Artist: def __init__(self): diff --git a/src/backends/itunes/ParseItunesToJson.py b/src/backends/itunes/ParseItunesToJson.py deleted file mode 100755 index 1505a36..0000000 --- a/src/backends/itunes/ParseItunesToJson.py +++ /dev/null @@ -1,101 +0,0 @@ -import xmltodict -import json - - -def flatten_dict(d): - - out = [] - for song in d: - newSong = {} - - strCount = 0 - intCount = 0 - dateCount = 0 - - def getStr(id): - nonlocal strCount - if id not in song["key"]: - return "undef" - if len(song["string"]) <= strCount: - return "error" - strCount += 1 - return song["string"][strCount - 1] - - def getInt(id): - nonlocal intCount - if id not in song["key"]: - return -1 - if len(song["integer"]) <= intCount: - return -1 - intCount += 1 - return song["integer"][intCount - 1] - - def getDate(id): - nonlocal dateCount - if id not in song["key"]: - return "undef" - if len(song["date"]) <= dateCount: - return "error" - dateCount += 1 - return song["date"][dateCount - 1] - - newSong["Track ID"] = getInt("Track ID") - newSong["Name"] = getStr("Name") - newSong["Artist"] = getStr("Artist") - newSong["Album Artist"] = getStr("Album Artist") - newSong["Composer"] = getStr("Composer") - newSong["Album"] = getStr("Album") - newSong["Genre"] = getStr("Genre") - newSong["Kind"] = getStr("Kind") - newSong["Size"] = getInt("Size") - newSong["Total Time"] = getInt("Total Time") - newSong["Disc Number"] = getInt("Disc Number") - newSong["Disc Count"] = getInt("Disc Count") - newSong["Track Number"] = getInt("Track Number") - newSong["Track Count"] = getInt("Track Count") - newSong["Year"] = getInt("Year") - newSong["Date Modified"] = getDate("Date Modified") - newSong["Date Added"] = getDate("Date Added") - newSong["Bit Rate"] = getInt("Bit Rate") - newSong["Sample Rate"] = getInt("Sample Rate") - newSong["Play Count"] = getInt("Play Count") - newSong["Play Date"] = getInt("Play Date") - newSong["Play Date UTC"] = getDate("Play Date UTC") - newSong["Skip Count"] = getInt("Skip Count") - newSong["Skip Date"] = getDate("Skip Date") - newSong["Release Date"] = getDate("Release Date") - newSong["Album Rating"] = getInt("Album Rating") - newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"] - newSong["Loved"] = "Loved" in song["key"] - newSong["Album Loved"] = "Album Loved" in song["key"] - newSong["Explicit"] = "Explicit" in song["key"] - newSong["Compilation"] = "Compilation" in song["key"] - newSong["Artwork Count"] = getInt("Artwork Count") - newSong["Sort Album"] = getStr("Sort Album") - newSong["Sort Artist"] = getStr("Sort Artist") - newSong["Sort Name"] = getStr("Sort Name") - newSong["Persistent ID"] = getStr("Persistent ID") - newSong["Track Type"] = getStr("Track Type") - - out.append(newSong) - - return out - - -def convert(filename, out_path): - with open(filename, 'r', encoding='utf-8') as xml_file: - data_dict = xmltodict.parse(xml_file.read()) - - # Extract the "Tracks" dictionary to be flattened - tracks_dict = data_dict['plist']['dict']['dict'] - - flat_tracks_dict = flatten_dict(tracks_dict["dict"]) - - json_data = json.dumps(flat_tracks_dict, indent=2) - - with open(out_path, 'w', encoding='utf-8') as json_file: - json_file.write(json_data) - - -if __name__ == "__main__": - convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json') diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py new file mode 100755 index 0000000..0af465f --- /dev/null +++ b/src/backends/itunes/ParseItunesXml.py @@ -0,0 +1,192 @@ +import xmltodict +import json +import xmltodict +import json +import re + +from src.Library import * + +class XmlToJson: + def flatten_dict(self, d): + + out = [] + for song in d: + newSong = {} + + strCount = 0 + intCount = 0 + dateCount = 0 + + def getStr(id): + nonlocal strCount + if id not in song["key"]: + return "undef" + if len(song["string"]) <= strCount: + return "error" + strCount += 1 + return song["string"][strCount - 1] + + def getInt(id): + nonlocal intCount + if id not in song["key"]: + return -1 + if len(song["integer"]) <= intCount: + return -1 + intCount += 1 + return song["integer"][intCount - 1] + + def getDate(id): + nonlocal dateCount + if id not in song["key"]: + return "undef" + if len(song["date"]) <= dateCount: + return "error" + dateCount += 1 + return song["date"][dateCount - 1] + + newSong["Track ID"] = getInt("Track ID") + newSong["Name"] = getStr("Name") + newSong["Artist"] = getStr("Artist") + newSong["Album Artist"] = getStr("Album Artist") + newSong["Composer"] = getStr("Composer") + newSong["Album"] = getStr("Album") + newSong["Genre"] = getStr("Genre") + newSong["Kind"] = getStr("Kind") + newSong["Size"] = getInt("Size") + newSong["Total Time"] = getInt("Total Time") + newSong["Disc Number"] = getInt("Disc Number") + newSong["Disc Count"] = getInt("Disc Count") + newSong["Track Number"] = getInt("Track Number") + newSong["Track Count"] = getInt("Track Count") + newSong["Year"] = getInt("Year") + newSong["Date Modified"] = getDate("Date Modified") + newSong["Date Added"] = getDate("Date Added") + newSong["Bit Rate"] = getInt("Bit Rate") + newSong["Sample Rate"] = getInt("Sample Rate") + newSong["Play Count"] = getInt("Play Count") + newSong["Play Date"] = getInt("Play Date") + newSong["Play Date UTC"] = getDate("Play Date UTC") + newSong["Skip Count"] = getInt("Skip Count") + newSong["Skip Date"] = getDate("Skip Date") + newSong["Release Date"] = getDate("Release Date") + newSong["Album Rating"] = getInt("Album Rating") + newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"] + newSong["Loved"] = "Loved" in song["key"] + newSong["Album Loved"] = "Album Loved" in song["key"] + newSong["Explicit"] = "Explicit" in song["key"] + newSong["Compilation"] = "Compilation" in song["key"] + newSong["Artwork Count"] = getInt("Artwork Count") + newSong["Sort Album"] = getStr("Sort Album") + newSong["Sort Artist"] = getStr("Sort Artist") + newSong["Sort Name"] = getStr("Sort Name") + newSong["Persistent ID"] = getStr("Persistent ID") + newSong["Track Type"] = getStr("Track Type") + + out.append(newSong) + + return out + + + def convert(self, filename, out_path): + with open(filename, 'r', encoding='utf-8') as xml_file: + data_dict = xmltodict.parse(xml_file.read()) + + # Extract the "Tracks" dictionary to be flattened + tracks_dict = data_dict['plist']['dict']['dict'] + + flat_tracks_dict = self.flatten_dict(tracks_dict["dict"]) + + json_data = json.dumps(flat_tracks_dict, indent=2) + + with open(out_path, 'w', encoding='utf-8') as json_file: + json_file.write(json_data) + + +class ParseItunesXml: + def __init__(self): + self.track_map: dict[str, Track] = {} + self.artist_map: dict[str, Artist] = {} + self.album_map: dict[str, Album] = {} + self.library = Library() + + + def parse_artists(self, data: list) -> List[Artist]: + if not data: + raise "No artist found" + + artists = [] + + for artist_data in data: + name = artist_data + artist_id = name + + if artist_id in self.artist_map: + artists.append(self.artist_map[artist_id]) + continue + + artist = Artist() + artist.id = artist_id + artist.title = name + + self.artist_map[artist_id] = artist + self.library.artists.append(artist) + + artists.append(artist) + + return artists + + def parse_album(self, data: str) -> Album: + if not data: + raise "album is none" + + name = data + album_id = name + + if album_id in self.album_map: + return self.album_map[album_id] + + album = Album() + album.id = album_id + album.title = name + + self.album_map[album_id] = album + self.library.albums.append(album) + + return album + + + def parse(self, tracks_file_xml: Path, tracks_file_json: Path) -> Library: + XmlToJson().convert(tracks_file_xml, tracks_file_json) + + tracks_content = json.loads(tracks_file_json.read_text()) + + for data in tracks_content: + track = Track() + + track.id = data.get("Track ID") + track.title = data.get("Name") + track.artists = self.parse_artists([data.get("Artist")]) + track.album = self.parse_album(data.get("Album")) + + if not track.album or not len(track.artists) or not track.title: + raise "error parsing" + + self.track_map[track.id] = track + self.library.tracks.append(track) + + for track_id, track in self.track_map.items(): + for artist in track.artists: + if artist not in track.album.artists: + track.album.artists.append(artist) + + self.library.update_cache() + + self.library.name = "Itunes" + + return self.library + + + def parse_track_item(self, id, track: dict[str, Any]) -> Track: + + return track + From 9cd2c9189b706b0360da20c7be4e76264148009b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sat, 7 Feb 2026 15:17:59 +0300 Subject: [PATCH 15/18] nice --- Flow.py | 2 + Gui.py | 38 +++++---- src/Library.py | 69 +++++---------- src/LibrarySerializer.py | 116 +++++++++++++------------- src/backends/itunes/ParseItunesXml.py | 8 +- src/backends/spotify/ParseSpotify.py | 7 ++ 6 files changed, 112 insertions(+), 128 deletions(-) diff --git a/Flow.py b/Flow.py index 0031017..9cd7a04 100644 --- a/Flow.py +++ b/Flow.py @@ -23,6 +23,7 @@ from src.StatGenerator import log_stats class Flow: spotify_library : Library = None local_library : Library = None + itunes_library : Library = None @staticmethod def fetch_spotify(self): @@ -107,6 +108,7 @@ class Flow: def load_libraries(self): self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json") + self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json") self.local_library = LibraryLoader().load(work_dir / "local_library.json") def save_libraries(self): diff --git a/Gui.py b/Gui.py index 85fd6e3..3c0ecc8 100644 --- a/Gui.py +++ b/Gui.py @@ -6,34 +6,36 @@ import pandas as pd import Flow -from src.Library import Library, Artist, Album, Track, Playlist +from src.Library import * + +def entity_to_row(entity: Entity): + return { + "id": entity.id, + "mbid": entity.mbid, + "Title": entity.title, + "PLay Count": entity.play_count, + "Resolved": int(entity.resolved_percentage * 100), + "AutoScore": entity.auto_score, + "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None, + "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None, + "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None, + "LocalId": str(entity.local_id) if entity.local_id else None, + "Path": str(entity.local_path) if entity.local_path else None, + } def track_to_row(track: Track): - return { - "Title": track.title, + return entity_to_row(track) | { "Album": track.album.title if track.album else "", "Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists), "Lyrics": track.has_lyrics, - "AutoScore": track.auto_score, - "Added": track.added_at.strftime("%Y-%m-%d") if track.added_at else "", - "LocalId": str(track.local_id) if track.local_id else "", - "Path": str(track.local_path) if track.local_path else "", } def artist_to_row(artist: Artist): - return { - "Title": artist.title, - "AutoScore": artist.auto_score, - "Resolved": int(artist.resolved_percentage * 100), - "LocalId": str(artist.local_id) if artist.local_id else "", + return entity_to_row(artist) | { } def album_to_row(album: Album): - return { - "Title": album.title, - "AutoScore": album.auto_score, - "Resolved": int(album.resolved_percentage * 100), - "LocalId": str(album.local_id) if album.local_id else "", + return entity_to_row(album) | { } @@ -200,7 +202,7 @@ def run(): # flow.log_stats() gui = LibraryView() - gui.add_libraries([flow.spotify_library, flow.local_library]) + gui.add_libraries([flow.spotify_library, flow.itunes_library, flow.local_library]) gui.render() run() \ No newline at end of file diff --git a/src/Library.py b/src/Library.py index 7bcefac..246d581 100644 --- a/src/Library.py +++ b/src/Library.py @@ -4,77 +4,58 @@ from datetime import datetime from pathlib import Path from typing import List, Optional, Any + class Entity: def __init__(self): self.id: str = None - self.mbid : str = None + self.mbid: str = None self.title: str = None + self.duration_ms: int = None self.fav: bool = False self.rating: int = 0 - self.play_count : int = 0 - self.date_added : datetime = None - self.date_released : datetime = None - self.date_last_play : datetime = None + self.play_count: int = 0 + self.date_added: datetime = None + self.date_released: datetime = None + self.date_last_play: datetime = None self.auto_score: float = 0 self.local_id: str = None - self.resolved_percentage : float = 0 + self.local_path: Path = None + self.resolved_percentage: float = 0 -class Artist: + +class Artist(Entity): def __init__(self): - self.id: str = None - self.title: str = None + super().__init__() self.tracks: List[Track] = [] self.albums: List[Album] = [] - self.local_id: str = None - self.auto_score: float = 0 - self.fav: bool = False - self.rating: int = 0 - self.resolved_percentage : float = 0 -class Track: +class Track(Entity): def __init__(self): - self.id: str = None - self.title: str = None + super().__init__() + self.artists: List[Artist] = [] self.album: Album = None - self.duration_ms: int = None - self.added_at: datetime = None - self.local_path: Path = None + self.has_lyrics: bool = False - self.local_id: str = None - self.auto_score: float = 0 - self.fav: bool = False - self.rating: int = 0 -class Album: +class Album(Entity): def __init__(self): - self.id: str = None - self.title: str = None - self.release_date: datetime = None + super().__init__() self.tracks: List[Track] = [] self.artists: List[Artist] = [] - self.local_id: str = None - self.auto_score: float = 0 - self.fav: bool = False - self.rating: int = 0 - self.resolved_percentage : float = 0 -class Playlist: +class Playlist(Entity): def __init__(self): - self.title: str = None + super().__init__() self.tracks: List[Track] = [] self.created_at: datetime = None self.description: str = None - self.auto_score: float = 0 - self.fav: bool = False - self.rating: int = 0 - self.resolved_percentage : float = 0 class Library: @@ -120,14 +101,8 @@ class Library: artist.albums.append(album) def auto_score(self): - self.score_tracks() - self.score_artists() self.score_albums() - - def score_tracks(self): - for idx, track in enumerate(self.top_tracks): - score = len(self.top_tracks) - idx - track.auto_score = score + self.score_artists() def score_artists(self): for track in self.tracks: @@ -150,4 +125,4 @@ class Library: album.resolved_percentage = album_count_resolved / len(album.tracks) artist_count_resolved += album_count_resolved artist_count += len(album.tracks) - artist.resolved_percentage = artist_count_resolved / artist_count \ No newline at end of file + artist.resolved_percentage = artist_count_resolved / artist_count diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py index f331824..278776a 100644 --- a/src/LibrarySerializer.py +++ b/src/LibrarySerializer.py @@ -26,60 +26,51 @@ class LibrarySaver: def __init__(self, library: Library): self.library = library - @staticmethod - def _artist_to_dict(artist: Artist) -> dict[str, Any]: + def _entity_to_dict(self, entity: Entity) -> dict[str, Any]: return { - "id": artist.id, - "title": artist.title, - "tracks": [track.id for track in artist.tracks], - "albums": [album.id for album in artist.albums], - "local_id": artist.local_id, - "auto_score": artist.auto_score, - "fav" : artist.fav, - "rating" : artist.rating, + "id": entity.id, + "mbid": entity.mbid, + + "title": entity.title, + "duration": entity.duration_ms, + + "fav" : entity.fav, + "rating" : entity.rating, + "play_count": entity.play_count, + "date_added": entity.date_added, + "date_released": entity.date_released, + "date_last_play": entity.date_last_play, + "auto_score": entity.auto_score, + + "local_id": entity.local_id, + "local_path": entity.local_path, + "resolved_percentage": entity.resolved_percentage, } - @staticmethod - def _track_to_dict(track: Track) -> dict[str, Any]: - return { - "id": track.id, - "title": track.title, + def _artist_to_dict(self, artist: Artist) -> dict[str, Any]: + return self._entity_to_dict(artist) | { + "tracks": [track.id for track in artist.tracks], + "albums": [album.id for album in artist.albums], + } + + def _track_to_dict(self, track: Track) -> dict[str, Any]: + return self._entity_to_dict(track) | { "artists": [artist.id for artist in track.artists], "album": track.album.id, "duration_ms": track.duration_ms, - "added_at": _dt_to_str(track.added_at), - "local_path": track.local_path, "has_lyrics": track.has_lyrics, - "local_id": track.local_id, - "auto_score": track.auto_score, - "fav" : track.fav, - "rating" : track.rating, } - @staticmethod - def _playlist_to_dict(playlist: Playlist) -> dict[str, Any]: - return { - "title": playlist.title, + def _playlist_to_dict(self, playlist: Playlist) -> dict[str, Any]: + return self._entity_to_dict(playlist) | { "tracks": [track.id for track in playlist.tracks], - "created_at": _dt_to_str(playlist.created_at), "description": playlist.description, - "auto_score": playlist.auto_score, - "fav" : playlist.fav, - "rating" : playlist.rating, } - @staticmethod - def _album_to_dict(album: Album) -> dict[str, Any]: - return { - "id": album.id, - "title": album.title, - "release_date": album.release_date, + def _album_to_dict(self ,album: Album) -> dict[str, Any]: + return self._entity_to_dict(album) | { "tracks": [track.id for track in album.tracks], "artists": [artist.id for artist in album.artists], - "local_id": album.local_id, - "auto_score": album.auto_score, - "fav" : album.fav, - "rating" : album.rating, } def save(self, path: Path) -> None: @@ -100,6 +91,7 @@ class LibrarySaver: json.dump(data, f, indent=2, ensure_ascii=False) + class LibraryLoader: def __init__(self): self.track_map: dict[str, Track] = {} @@ -127,48 +119,54 @@ class LibraryLoader: self.track_map[id] = item return item + def load_entity(self, entity : Entity, data: dict[str, Any]): + entity.id = data.get("id") + entity.mbid = data.get("mbid") + + entity.title = data.get("title") + entity.duration_ms = data.get("duration_ms") + + entity.fav = data.get("fav") + entity.rating = data.get("rating") + entity.play_count = data.get("play_count") + entity.date_added = _str_to_dt(data.get("date_added")) + entity.date_released = _str_to_dt(data.get("date_released")) + entity.date_last_play = _str_to_dt(data.get("date_last_play")) + entity.auto_score = data.get("auto_score") + + entity.local_id = data.get("local_id") + entity.local_path = data.get("local_path") + entity.resolved_percentage = data.get("resolved_percentage") + + def load_track(self, data: dict[str, Any]) -> Track: track = self.get_track(data.get("id")) - track.id = data.get("id") - track.title = data.get("title") + self.load_entity(track, data) + track.duration_ms = data.get("duration_ms") - track.added_at = _str_to_dt(data.get("added_at")) - track.local_path = data.get("local_path") track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] track.album = self.get_album(data.get("album")) track.has_lyrics = data.get("has_lyrics") - track.local_id = data.get("local_id") - track.auto_score = data.get("auto_score") - track.fav = data.get("fav") - track.rating = data.get("rating") self.track_map[track.id] = track return track def load_artist(self, data: dict[str, Any]): artist = self.get_artist(data.get("id")) - artist.id = data.get("id") - artist.title = data.get("title") + self.load_entity(artist, data) + artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else [] - artist.local_id = data.get("local_id") - artist.auto_score = data.get("auto_score") - artist.fav = data.get("fav") - artist.rating = data.get("rating") self.artist_map[artist.id] = artist return artist def load_album(self, data: dict[str, Any]): album = self.get_album(data.get("id")) - album.id = data.get("id") + self.load_entity(album, data) + album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")] - album.title = data.get("title") album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")] - album.local_id = data.get("local_id") - album.auto_score = data.get("auto_score") - album.fav = data.get("fav") - album.rating = data.get("rating") self.album_map[album.id] = album return album diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py index 0af465f..71eb2d7 100755 --- a/src/backends/itunes/ParseItunesXml.py +++ b/src/backends/itunes/ParseItunesXml.py @@ -167,6 +167,7 @@ class ParseItunesXml: track.title = data.get("Name") track.artists = self.parse_artists([data.get("Artist")]) track.album = self.parse_album(data.get("Album")) + track.play_count = int(data.get("Play Count")) if not track.album or not len(track.artists) or not track.title: raise "error parsing" @@ -183,10 +184,9 @@ class ParseItunesXml: self.library.name = "Itunes" + for track in self.library.tracks: + track.auto_score = track.play_count + return self.library - def parse_track_item(self, id, track: dict[str, Any]) -> Track: - - return track - diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py index 29c24e7..cfca39d 100644 --- a/src/backends/spotify/ParseSpotify.py +++ b/src/backends/spotify/ParseSpotify.py @@ -33,6 +33,8 @@ class Parser: self.library.name = "Spotify" self.library.auto_score() + self.score_tracks() + return self.library @@ -122,3 +124,8 @@ class Parser: self.library.artists.append(artist) return artist + + def score_tracks(self): + for idx, track in enumerate(self.library.top_tracks): + score = len(self.library.top_tracks) - idx + track.auto_score = score \ No newline at end of file From 16b55b0674d3cb8cc77b1759144079ea2cc439f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 00:03:10 +0300 Subject: [PATCH 16/18] spotify: update fetching to current Web API - OAuth: URL-encode authorize params (scope spaces were unencoded), bind loopback redirect server to 127.0.0.1 to match the redirect_uri per Spotify's 2025 loopback rules, capture refresh_token. - Search: URL-encode the query and drop the bogus track=1 param. - Pagination: follow the paging object's `next` field instead of manual offset/total bookkeeping (Spotify's recommended approach). - Fix invalid `raise "string"` in save_image_from_url. Co-Authored-By: Claude Opus 4.8 --- src/backends/spotify/SpotifyAuthenticator.py | 11 ++- src/backends/spotify/SpotifyWebAPI.py | 80 ++++++-------------- 2 files changed, 33 insertions(+), 58 deletions(-) diff --git a/src/backends/spotify/SpotifyAuthenticator.py b/src/backends/spotify/SpotifyAuthenticator.py index 5e27f7d..28cce0a 100644 --- a/src/backends/spotify/SpotifyAuthenticator.py +++ b/src/backends/spotify/SpotifyAuthenticator.py @@ -11,6 +11,7 @@ import os.path client_id = None client_secret = None token = None +refresh_token = None def retrieve_web_api_token(authorization_code): @@ -44,7 +45,10 @@ def retrieve_web_api_token(authorization_code): print(token_response.json()) raise ValueError - token = token_response.json()['access_token'] + global refresh_token + response_json = token_response.json() + token = response_json['access_token'] + refresh_token = response_json.get('refresh_token') class RequestHandler(BaseHTTPRequestHandler): @@ -73,7 +77,7 @@ class RequestHandler(BaseHTTPRequestHandler): def run_redirect_server(): - httpd = HTTPServer(('localhost', 8888), RequestHandler) + httpd = HTTPServer(('127.0.0.1', 8888), RequestHandler) print('Starting HTTP redirection server') httpd.serve_forever() @@ -88,8 +92,9 @@ def open_authenticate_page(): "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", } - auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}" + auth_url = f"{authorize_url}?{urllib.parse.urlencode(params)}" webbrowser.open(auth_url) + print(f"If the browser did not open, visit:\n{auth_url}") def authenticate(id, secret): diff --git a/src/backends/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py index 8f4807f..73fe61b 100644 --- a/src/backends/spotify/SpotifyWebAPI.py +++ b/src/backends/spotify/SpotifyWebAPI.py @@ -1,4 +1,5 @@ import requests +import urllib.parse from PIL import Image from io import BytesIO @@ -11,7 +12,7 @@ token = '' def fetch(endpoint, method='GET', body=None): - url = f'https://api.spotify.com/{endpoint}' + url = endpoint if endpoint.startswith('http') else f'https://api.spotify.com/{endpoint}' headers = { 'Authorization': f'Bearer {token}', } @@ -21,83 +22,52 @@ def fetch(endpoint, method='GET', body=None): return response.json() +def fetch_paginated(endpoint): + """Collect every item of a paging object by following its `next` field, + which is Spotify's recommended way to page (more robust than tracking + offset/total manually).""" + items = [] + page = fetch(endpoint) + while True: + items += page.get('items', []) + next_url = page.get('next') + if not next_url: + break + page = fetch(next_url) + return items + + def find_song(song_pattern): - return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + query = urllib.parse.quote(song_pattern) + return fetch(f"v1/search?type=track&limit=10&q={query}")['tracks']['items'] def fetch_playlists(user_id): print("Fetching playlists") - endpoint = f"v1/me/playlists" - pl_total = fetch(f'{endpoint}?limit=1')['total'] - pl_count = 0 - playlists = [] - - while pl_count < pl_total: - res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}') - playlists += res['items'] - pl_count += FETCH_STEP - - return playlists + return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}") def fetch_playlist_items(playlist_id): - endpoint = f"v1/playlists/{playlist_id}/tracks" - items = fetch(f'{endpoint}?fields=total')['total'] - loaded = 0 - tracks = [] - - while loaded < items: - res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}') - - tracks += res['items'] - loaded += FETCH_STEP - - return tracks + return fetch_paginated(f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}") def fetch_tracks(user_id): print("Fetching tracks") - endpoint = f"v1/me/tracks" - total = fetch(f"{endpoint}?limit=1&market=ES")['total'] - current = 0 - tracks = [] - - while total > current: - res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES") - tracks += res['items'] - current += FETCH_STEP - + tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES") save_data(tracks, "tracks") return tracks def fetch_tracks_top(): print("Fetching top tracks") - total = fetch(f'v1/me/top/tracks?fields=total')['total'] - current = 0 - tracks = [] - - while current < total: - res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term') - tracks += res['items'] - current += FETCH_STEP - + tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term") save_data(tracks, "top_tracks") return tracks def fetch_artists_top(): print("Fetching top artists") - - total = fetch(f'v1/me/top/artists?fields=total')['total'] - current = 0 - artists = [] - - while current < total: - res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term') - artists += res['items'] - current += FETCH_STEP - + artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term") save_data(artists, "top_artists") return artists @@ -140,7 +110,7 @@ def save_image_from_url(url, name, image_dir="playlist_covers"): response = requests.get(url) if response.status_code != 200: - raise "Cannot fetch the playlist cover" + raise RuntimeError("Cannot fetch the playlist cover") image = Image.open(BytesIO(response.content)) From 36058c318a6c3556855723aa57eccb1278af9bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 00:39:59 +0300 Subject: [PATCH 17/18] spotify: resilient fetching, progress bars, venv launcher - fetch() uses a retrying session (connection resets, 5xx, 429 Retry-After) with backoff, so a transient drop mid-fetch no longer aborts the run. - tqdm progress bars on all paginated fetches (driven by paging `total`). - Flow.fetch_spotify(): drop stray `self` from the @staticmethod. - gui.sh runs the venv's streamlit from the script dir; add requirements.txt. Co-Authored-By: Claude Opus 4.8 --- Flow.py | 2 +- gui.sh | 6 ++- main.py | 12 ++--- requirements.txt | 9 ++++ src/backends/spotify/SpotifyWebAPI.py | 68 ++++++++++++++++++++------- 5 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 requirements.txt diff --git a/Flow.py b/Flow.py index 9cd7a04..9ced6ae 100644 --- a/Flow.py +++ b/Flow.py @@ -26,7 +26,7 @@ class Flow: itunes_library : Library = None @staticmethod - def fetch_spotify(self): + def fetch_spotify(): fetch_all(client_id, client_secret, work_dir) @staticmethod diff --git a/gui.sh b/gui.sh index eb55739..9be5f9d 100755 --- a/gui.sh +++ b/gui.sh @@ -1 +1,5 @@ -streamlit run Gui.py +#!/usr/bin/env bash +# Run from the script's directory using the project virtualenv, +# so it works regardless of whether the venv is activated / on PATH. +cd "$(dirname "$0")" +exec .venv/bin/streamlit run Gui.py "$@" diff --git a/main.py b/main.py index 26588d1..68707c7 100644 --- a/main.py +++ b/main.py @@ -17,17 +17,17 @@ def test1(): def main(): flow = Flow.Flow() - # flow.fetch_spotify() + flow.fetch_spotify() # flow.fetch_local() flow.parse_spotify_library() - flow.parse_itunes_library() - flow.parse_local_library() + #flow.parse_itunes_library() + #flow.parse_local_library() - flow.load_libraries() + #flow.load_libraries() - flow.map_local_to_spotify() - flow.save_libraries() + #flow.map_local_to_spotify() + #flow.save_libraries() # flow.log_stats() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9a1d950 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +streamlit==1.58.0 +streamlit-aggrid==1.2.1.post2 +pandas==3.0.3 +Pillow==12.2.0 +mutagen==1.47.0 +rapidfuzz==3.14.5 +requests==2.34.2 +tqdm==4.68.2 +xmltodict==1.0.4 diff --git a/src/backends/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py index 73fe61b..e795d8a 100644 --- a/src/backends/spotify/SpotifyWebAPI.py +++ b/src/backends/spotify/SpotifyWebAPI.py @@ -1,7 +1,10 @@ import requests import urllib.parse +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from PIL import Image from io import BytesIO +from tqdm import tqdm from src.backends.spotify.SpotifyAuthenticator import authenticate @@ -9,6 +12,31 @@ from src.helpers import * FETCH_STEP = 50 token = '' +REQUEST_TIMEOUT = 30 + + +def _build_session(): + """Session that retries transient failures (dropped connections, 5xx) and + honours Spotify's 429 Retry-After, so a single reset mid-fetch doesn't + abort the whole run.""" + session = requests.Session() + retry = Retry( + total=6, + connect=6, + read=6, + backoff_factor=1.5, # sleeps ~0, 1.5, 3, 6, 12, 24s between attempts + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET", "POST"}), + respect_retry_after_header=True, + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +_session = _build_session() def fetch(endpoint, method='GET', body=None): @@ -17,23 +45,27 @@ def fetch(endpoint, method='GET', body=None): 'Authorization': f'Bearer {token}', } - response = requests.request(method, url, headers=headers, json=body) + response = _session.request(method, url, headers=headers, json=body, timeout=REQUEST_TIMEOUT) response.raise_for_status() return response.json() -def fetch_paginated(endpoint): +def fetch_paginated(endpoint, desc="Fetching", leave=True): """Collect every item of a paging object by following its `next` field, which is Spotify's recommended way to page (more robust than tracking - offset/total manually).""" + offset/total manually). Shows a tqdm progress bar driven by the paging + object's `total`.""" items = [] page = fetch(endpoint) - while True: - items += page.get('items', []) - next_url = page.get('next') - if not next_url: - break - page = fetch(next_url) + with tqdm(total=page.get('total'), desc=desc, unit="item", leave=leave) as pbar: + while True: + batch = page.get('items', []) + items += batch + pbar.update(len(batch)) + next_url = page.get('next') + if not next_url: + break + page = fetch(next_url) return items @@ -43,31 +75,31 @@ def find_song(song_pattern): def fetch_playlists(user_id): - print("Fetching playlists") - return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}") + return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}", desc="Fetching playlists") def fetch_playlist_items(playlist_id): - return fetch_paginated(f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}") + return fetch_paginated( + f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}", + desc="Fetching playlist tracks", + leave=False, + ) def fetch_tracks(user_id): - print("Fetching tracks") - tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES") + tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES", desc="Fetching liked tracks") save_data(tracks, "tracks") return tracks def fetch_tracks_top(): - print("Fetching top tracks") - tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term") + tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top tracks") save_data(tracks, "top_tracks") return tracks def fetch_artists_top(): - print("Fetching top artists") - artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term") + artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top artists") save_data(artists, "top_artists") return artists From 83f782d125b2c3801e783b4c44b295b8510ee7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 00:39:59 +0300 Subject: [PATCH 18/18] Remove hardcoded credentials; read them from the environment - Env.py now reads SPOTIFY_CLIENT_ID/SPOTIFY_CLIENT_SECRET (and optional path overrides) from the environment instead of hardcoding them. - Add .env.example and ignore .env / .venv. - Rewrite README with setup, usage and layout (drops the leaked secret). Note: secrets are also purged from prior history in this push. Co-Authored-By: Claude Opus 4.8 --- .env.example | 9 +++++ .gitignore | 2 + Env.py | 14 +++++-- README.md | 103 +++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..27f5a90 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# Copy to `.env` and fill in, then `source .env` before running. +# Get these from https://developer.spotify.com/dashboard (your app's settings). +# Register `http://127.0.0.1:8888` as a Redirect URI in that app. +export SPOTIFY_CLIENT_ID=your_client_id_here +export SPOTIFY_CLIENT_SECRET=your_client_secret_here + +# Optional overrides: +# export MUSIC_LIBRARY_PATH=~/Music +# export WORK_DIR=./work_dir/new diff --git a/.gitignore b/.gitignore index 880f9c3..7b75b45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ prefetched* __pycache__ .idea secret +.env +.venv work_dir old tmp \ No newline at end of file diff --git a/Env.py b/Env.py index 6549c6b..c609347 100644 --- a/Env.py +++ b/Env.py @@ -1,7 +1,13 @@ +import os from pathlib import Path -local_library_path = Path("/home/auser/Music/") -work_dir = Path("./work_dir/new") +# Local config. Paths can be overridden via environment variables. +local_library_path = Path(os.environ.get("MUSIC_LIBRARY_PATH", "~/Music")).expanduser() +work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new")) -client_id = '***REDACTED***' -client_secret = "***REDACTED***" +# Spotify app credentials. NEVER hardcode these here — set them in the +# environment (see .env.example) so they don't end up in version control. +# export SPOTIFY_CLIENT_ID=... +# export SPOTIFY_CLIENT_SECRET=... +client_id = os.environ.get("SPOTIFY_CLIENT_ID") +client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET") diff --git a/README.md b/README.md index 68b9ba5..62d8bd7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,100 @@ -# spotifyFetcher -loads all data through web api
-client secret - ***REDACTED*** +# MusicIndexer + +Pull your Spotify library and listening stats via the Spotify Web API, index a +local music library (audio tags + iTunes XML), and match the two so you can see +which tracks you own, what's missing, and your top artists/tracks/playlists — in +a Streamlit GUI. + +## Features + +- **Spotify backend** — OAuth (Authorization Code) login, then fetch the user + profile, liked tracks, all playlists and their tracks, long-term top tracks + and top artists, and playlist/profile cover art. Resilient paginated fetching + (follows `next`, retries dropped connections, honours `429` rate limits) with + progress bars. +- **Local backend** — index a folder of audio files via `mutagen` tags. +- **iTunes backend** — parse an iTunes Library XML export. +- **Matching** — map local tracks to Spotify tracks (fuzzy matcher; ISRC/MBID + matching planned). +- **Stats & lyrics** — generate listening stats and fetch lyrics. +- **GUI** — browse libraries as artist → album → track tables. + +## Setup + +Requires Python 3.13+. + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +### Credentials + +Create an app at https://developer.spotify.com/dashboard and add +`http://127.0.0.1:8888` as a **Redirect URI**. Credentials are read from the +environment — never commit them. + +```bash +cp .env.example .env +# edit .env with your client id/secret +source .env +``` + +`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (plus optional +`MUSIC_LIBRARY_PATH` and `WORK_DIR`) from the environment. + +## Usage + +Edit `main.py` to uncomment the steps you want, then run: + +```bash +python main.py +``` + +The flow steps (see `Flow.py`) include: + +| Step | What it does | +|------|--------------| +| `fetch_spotify()` | OAuth login + download all Spotify data into `work_dir/spotify/` | +| `fetch_local()` | index local audio files into `work_dir/local/` | +| `parse_spotify_library()` / `parse_itunes_library()` / `parse_local_library()` | build normalized libraries | +| `load_libraries()` / `save_libraries()` | (de)serialize libraries to `work_dir` | +| `map_local_to_spotify()` | match local tracks against the Spotify library | +| `fetch_and_save_lyrics()` | fetch lyrics for tracks | +| `log_stats()` | generate stats | + +On first `fetch_spotify()` a browser opens for the Spotify login; a local server +on `127.0.0.1:8888` catches the redirect and exchanges the code for a token. + +### GUI + +```bash +./gui.sh # runs: .venv/bin/streamlit run Gui.py +``` + +## Layout + +``` +main.py entry point (CLI flow) +Gui.py / gui.sh Streamlit GUI +Flow.py orchestrates fetch / parse / map / stats +Env.py config (reads credentials from the environment) +src/ + Library.py data model (Library / Artist / Album / Track) + LibrarySerializer.py load/save libraries + backends/ + spotify/ Web API client, OAuth, parser + local/ local-file indexer, parser, playlist generator + itunes/ iTunes XML parser + navidrome/ Navidrome backend + mappers/ local↔Spotify matching (fuzzy / search) + lyrics/ lyrics fetching +work_dir/ fetched data and serialized libraries (gitignored) +``` + +## Notes + +- Data is cached as JSON under `work_dir/` (gitignored); re-running reuses it. +- The deprecated Spotify endpoints (audio-features, recommendations, etc.) are + not used — only currently-supported endpoints.