From 506258f9b12ad3ddccdeb637ebdc3b84d111133c Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:14:12 +0300 Subject: [PATCH 01/20] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..508172a --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# spotifyFetcher +loads all data through web api From 04da522b64e32a10a32243f807a797bafe25fbd1 Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:14:12 +0300 Subject: [PATCH 02/20] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..508172a --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# spotifyFetcher +loads all data through web api From 6703fdd9681e80ca07adcbc7f28a98a107cd0086 Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:14:29 +0300 Subject: [PATCH 03/20] initial --- .gitignore | 4 ++ README.md | 3 +- common.py | 25 ++++++++ loadArtwork.py | 45 +++++++++++++++ main.py | 106 ++++++++++++++++++++++++++++++++++ spotifyAuth.py | 122 ++++++++++++++++++++++++++++++++++++++++ spotifyFetch.py | 147 ++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 common.py create mode 100644 loadArtwork.py create mode 100644 main.py create mode 100644 spotifyAuth.py create mode 100644 spotifyFetch.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2803b17 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +prefetched* +__pycache__ +.idea +secret diff --git a/README.md b/README.md index 508172a..4a45af3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # spotifyFetcher -loads all data through web api +loads all data through web api
+client secret - 4ced4e4575094b749834a60996680c6c diff --git a/common.py b/common.py new file mode 100644 index 0000000..bb08d3e --- /dev/null +++ b/common.py @@ -0,0 +1,25 @@ +import json +import os + +data_dir = 'prefetched' + + +def get_data(name): + file_path = os.path.join(data_dir, f"{name}.json") + try: + with open(file_path, 'r') as file: + data = json.load(file) + return data + except FileNotFoundError: + return f"File {name}.json not found." + except json.JSONDecodeError: + return f"Error decoding JSON from {name}.json." + + +def save_data(data, name): + os.makedirs(data_dir, exist_ok=True) + + file_path = os.path.join(data_dir, f"{name}.json") + with open(file_path, 'w') as file: + json.dump(data, file, indent=4) + return f"Data saved to {name}.json." \ No newline at end of file diff --git a/loadArtwork.py b/loadArtwork.py new file mode 100644 index 0000000..4d2dc38 --- /dev/null +++ b/loadArtwork.py @@ -0,0 +1,45 @@ +import requests +import os +from PIL import Image +from io import BytesIO + +from common import * + + +def save_image_from_url(url, name, image_dir="playlist_covers"): + response = requests.get(url) + + if response.status_code != 200: + raise "Cannot fetch the playlist cover" + + image = Image.open(BytesIO(response.content)) + + directory = os.path.join(data_dir, image_dir) + os.makedirs(directory, exist_ok=True) + file_path = os.path.join(directory, f"{name}.jpg") + print(f"Image saved {file_path}") + image.save(file_path) + + +def load_playlist_covers(): + playlists = get_data('playlists') + + for pl in playlists: + if len(pl['images']): + url = pl['images'][0]['url'] + save_image_from_url(url, pl['name']) + + +def load_user_cover(): + user_data = get_data("user_data") + url = user_data['images'][1]['url'] + save_image_from_url(url, "user", ".") + + +def load(): + load_user_cover() + load_playlist_covers() + + +if __name__ == "__main__": + load() diff --git a/main.py b/main.py new file mode 100644 index 0000000..c88776b --- /dev/null +++ b/main.py @@ -0,0 +1,106 @@ +from common import * +from spotifyFetch import fetch_data +import cmd + +playlists = get_data('playlistTracks') +top_artists = get_data('top_artists') +top_tracks = get_data('top_tracks') +user_tracks = get_data('tracks') + + +def get_playlist_names(): + return [name for name, items in playlists.items()] + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists + + +def print_playlist_tracks(name): + print(name) + for track in playlists[name]: + track_data = track['track'] + track_name = track_data['name'] + artists = get_artists_str(track_data['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_top_artists(): + for artist in top_artists: + name = artist['name'] + print(f" '{name}'") + + +def print_top_tracks(): + for track in top_tracks: + track_name = track['name'] + artists = get_artists_str(track['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_stats(): + print(f" tracks - {len(user_tracks)}") + print(f" playlists - {len(playlists)}") + + for name, tracks in playlists.items(): + print(f" '{name}' - {len(tracks)}") + + print(f" top tracks - {len(top_tracks)}") + print(f" top artists - {len(top_artists)}") + + +def print_tracks(): + for trackItem in user_tracks: + track = trackItem['track'] + print(f" '{track['name']}' - {get_artists_str(track['artists'])}") + + +class Interpreter(cmd.Cmd): + intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" + prompt = "(spotify) " + + def do_playlists(self, arg): + """prints user playlists""" + print("\n".join(get_playlist_names())) + + def do_playlist_tracks(self, arg): + """prints playlist [name]""" + try: + name = arg.split()[0] + print_playlist_tracks(name) + except IndexError: + print("Invalid input") + + def do_top_artists(self, arg): + """Top artists""" + print_top_artists() + + def do_top_tracks(self, arg): + """Top tracks""" + print_top_tracks() + + def do_stat(self, arg): + """Print stats""" + print_stats() + + def do_tracks(self, arg): + """Print tracks""" + print_tracks() + + def do_fetch(self, arg): + """Fetch all spotify data""" + try: + fetch_data() + except ValueError: + print("Can not fetch the data.") + + def do_exit(self, arg): + """Exit the command loop: exit""" + print("Goodbye!") + return True + + +if __name__ == '__main__': + Interpreter().cmdloop() diff --git a/spotifyAuth.py b/spotifyAuth.py new file mode 100644 index 0000000..529c702 --- /dev/null +++ b/spotifyAuth.py @@ -0,0 +1,122 @@ +import base64 + +import requests +import webbrowser +from http.server import BaseHTTPRequestHandler, HTTPServer +import urllib.parse +import threading +import time +import os.path + +client_id = '3b4db45bb66b45e9a2532989c8034332' +client_secret = None +token = None + + +def resolve_client_secret(): + global client_secret + if os.path.exists('secret'): + with open('secret', 'r') as file: + client_secret = file.readline().strip() + else: + client_secret = input("Enter the application secret: ") + with open('secret', 'w') as file: + file.write(client_secret) + + +def retrieve_web_api_token(authorization_code): + global token + + token_url = 'https://accounts.spotify.com/api/token' + + token_data = { + "grant_type": "authorization_code", + "client_secret": client_secret, + "client_id": client_id, + "code": authorization_code, + 'redirect_uri': 'http://localhost:8888', + } + + print("\nAuthenticating WEB API\n") + + auth_string = f'{client_id}:{client_secret}' + b64_auth_string = base64.b64encode(auth_string.encode()).decode() + + # Define the headers + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': f'Basic {b64_auth_string}' + } + + token_response = requests.post(token_url, data=token_data, headers=headers) + + if token_response.status_code not in range(200, 299): + print(f"Failed to retrieve access token: {token_response.status_code}") + print(token_response.json()) + raise ValueError + + token = token_response.json()['access_token'] + + +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + + parsed_url = urllib.parse.urlparse(self.path) + query_params = urllib.parse.parse_qs(parsed_url.query) + + if 'code' in query_params: + authorization_code = query_params['code'][0] + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code received. You can close this window.') + + retrieve_web_api_token(authorization_code) + + threading.Thread(target=self.server.shutdown).start() + + else: + self.send_response(400) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code not found.') + + +def run_redirect_server(): + httpd = HTTPServer(('localhost', 8888), RequestHandler) + print('Starting HTTP redirection server') + httpd.serve_forever() + + +def open_authenticate_page(): + authorize_url = 'https://accounts.spotify.com/authorize' + + params = { + 'client_id': client_id, + 'response_type': 'code', + 'redirect_uri': 'http://localhost:8888', + "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", + } + + auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}" + webbrowser.open(auth_url) + + +def authenticate(): + resolve_client_secret() + + redirect_server = threading.Thread(target=run_redirect_server) + redirect_server.start() + + time.sleep(1) + open_authenticate_page() + + redirect_server.join() + + return token + + +if __name__ == "__main__": + authenticate() + diff --git a/spotifyFetch.py b/spotifyFetch.py new file mode 100644 index 0000000..ec590bc --- /dev/null +++ b/spotifyFetch.py @@ -0,0 +1,147 @@ +import requests +from loadArtwork import load +from spotifyAuth import authenticate +from common import * + +FETCH_STEP = 50 +token = '' + + +def fetch(endpoint, method='GET', body=None): + url = f'https://api.spotify.com/{endpoint}' + headers = { + 'Authorization': f'Bearer {token}', + } + + response = requests.request(method, url, headers=headers, json=body) + response.raise_for_status() + return response.json() + + +def fetch_playlists(user_id): + print("Fetching playlists") + endpoint = f"v1/me/playlists" + pl_total = fetch(f'{endpoint}?limit=1')['total'] + pl_count = 0 + playlists = [] + + while pl_count < pl_total: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}') + playlists += res['items'] + pl_count += FETCH_STEP + + return playlists + + +def fetch_playlist_items(playlist_id): + endpoint = f"v1/playlists/{playlist_id}/tracks" + items = fetch(f'{endpoint}?fields=total')['total'] + loaded = 0 + tracks = [] + + while loaded < items: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}') + + tracks += res['items'] + loaded += FETCH_STEP + + return tracks + + +def fetch_tracks(user_id): + print("Fetching tracks") + endpoint = f"v1/me/tracks" + total = fetch(f"{endpoint}?limit=1&market=ES")['total'] + current = 0 + tracks = [] + + while total > current: + res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES") + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "tracks") + return tracks + + +def fetch_tracks_top(): + print("Fetching top tracks") + total = fetch(f'v1/me/top/tracks?fields=total')['total'] + current = 0 + tracks = [] + + while current < total: + res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term') + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "top_tracks") + return tracks + + +def fetch_artists_top(): + print("Fetching top artists") + + total = fetch(f'v1/me/top/artists?fields=total')['total'] + current = 0 + artists = [] + + while current < total: + res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term') + artists += res['items'] + current += FETCH_STEP + + save_data(artists, "top_artists") + return artists + + +def fetch_all_playlist_data(user_id): + playlists = fetch_playlists(user_id) + + save_data(playlists, 'playlists') + + print(len(playlists)) + + playlist_items = {} + + print("Fetching playlists items") + + index = 0 + for pl in playlists: + pl_id = pl['id'] + name = pl['name'] + playlist_items[name] = fetch_playlist_items(pl_id) + print(f"{index} - {len(playlist_items[name])}") + index += 1 + + save_data(playlist_items, "playlistTracks") + + +def get_user_data(): + print("Fetching user data") + + user_profile = fetch(f'v1/me') + user_id = user_profile['id'] + user_data = fetch(f'v1/users/{user_id}') + + save_data(user_profile, "user_profile") + save_data(user_data, "user_data") + return user_id + + +def fetch_data(): + global token + token = authenticate() + + user_id = get_user_data() + + fetch_tracks(user_id) + fetch_all_playlist_data(user_id) + fetch_tracks_top() + fetch_artists_top() + + load() + + +if __name__ == "__main__": + fetch_data() From bb5453b59eb87d2d8c4d6b9a46852247019b7cf4 Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:14:29 +0300 Subject: [PATCH 04/20] initial --- .gitignore | 4 ++ README.md | 3 +- common.py | 25 ++++++++ loadArtwork.py | 45 +++++++++++++++ main.py | 106 ++++++++++++++++++++++++++++++++++ spotifyAuth.py | 122 ++++++++++++++++++++++++++++++++++++++++ spotifyFetch.py | 147 ++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 common.py create mode 100644 loadArtwork.py create mode 100644 main.py create mode 100644 spotifyAuth.py create mode 100644 spotifyFetch.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2803b17 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +prefetched* +__pycache__ +.idea +secret diff --git a/README.md b/README.md index 508172a..68b9ba5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # spotifyFetcher -loads all data through web api +loads all data through web api
+client secret - ***REDACTED*** diff --git a/common.py b/common.py new file mode 100644 index 0000000..bb08d3e --- /dev/null +++ b/common.py @@ -0,0 +1,25 @@ +import json +import os + +data_dir = 'prefetched' + + +def get_data(name): + file_path = os.path.join(data_dir, f"{name}.json") + try: + with open(file_path, 'r') as file: + data = json.load(file) + return data + except FileNotFoundError: + return f"File {name}.json not found." + except json.JSONDecodeError: + return f"Error decoding JSON from {name}.json." + + +def save_data(data, name): + os.makedirs(data_dir, exist_ok=True) + + file_path = os.path.join(data_dir, f"{name}.json") + with open(file_path, 'w') as file: + json.dump(data, file, indent=4) + return f"Data saved to {name}.json." \ No newline at end of file diff --git a/loadArtwork.py b/loadArtwork.py new file mode 100644 index 0000000..4d2dc38 --- /dev/null +++ b/loadArtwork.py @@ -0,0 +1,45 @@ +import requests +import os +from PIL import Image +from io import BytesIO + +from common import * + + +def save_image_from_url(url, name, image_dir="playlist_covers"): + response = requests.get(url) + + if response.status_code != 200: + raise "Cannot fetch the playlist cover" + + image = Image.open(BytesIO(response.content)) + + directory = os.path.join(data_dir, image_dir) + os.makedirs(directory, exist_ok=True) + file_path = os.path.join(directory, f"{name}.jpg") + print(f"Image saved {file_path}") + image.save(file_path) + + +def load_playlist_covers(): + playlists = get_data('playlists') + + for pl in playlists: + if len(pl['images']): + url = pl['images'][0]['url'] + save_image_from_url(url, pl['name']) + + +def load_user_cover(): + user_data = get_data("user_data") + url = user_data['images'][1]['url'] + save_image_from_url(url, "user", ".") + + +def load(): + load_user_cover() + load_playlist_covers() + + +if __name__ == "__main__": + load() diff --git a/main.py b/main.py new file mode 100644 index 0000000..c88776b --- /dev/null +++ b/main.py @@ -0,0 +1,106 @@ +from common import * +from spotifyFetch import fetch_data +import cmd + +playlists = get_data('playlistTracks') +top_artists = get_data('top_artists') +top_tracks = get_data('top_tracks') +user_tracks = get_data('tracks') + + +def get_playlist_names(): + return [name for name, items in playlists.items()] + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists + + +def print_playlist_tracks(name): + print(name) + for track in playlists[name]: + track_data = track['track'] + track_name = track_data['name'] + artists = get_artists_str(track_data['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_top_artists(): + for artist in top_artists: + name = artist['name'] + print(f" '{name}'") + + +def print_top_tracks(): + for track in top_tracks: + track_name = track['name'] + artists = get_artists_str(track['artists']) + print(f" '{track_name}' - '{artists}' ") + + +def print_stats(): + print(f" tracks - {len(user_tracks)}") + print(f" playlists - {len(playlists)}") + + for name, tracks in playlists.items(): + print(f" '{name}' - {len(tracks)}") + + print(f" top tracks - {len(top_tracks)}") + print(f" top artists - {len(top_artists)}") + + +def print_tracks(): + for trackItem in user_tracks: + track = trackItem['track'] + print(f" '{track['name']}' - {get_artists_str(track['artists'])}") + + +class Interpreter(cmd.Cmd): + intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" + prompt = "(spotify) " + + def do_playlists(self, arg): + """prints user playlists""" + print("\n".join(get_playlist_names())) + + def do_playlist_tracks(self, arg): + """prints playlist [name]""" + try: + name = arg.split()[0] + print_playlist_tracks(name) + except IndexError: + print("Invalid input") + + def do_top_artists(self, arg): + """Top artists""" + print_top_artists() + + def do_top_tracks(self, arg): + """Top tracks""" + print_top_tracks() + + def do_stat(self, arg): + """Print stats""" + print_stats() + + def do_tracks(self, arg): + """Print tracks""" + print_tracks() + + def do_fetch(self, arg): + """Fetch all spotify data""" + try: + fetch_data() + except ValueError: + print("Can not fetch the data.") + + def do_exit(self, arg): + """Exit the command loop: exit""" + print("Goodbye!") + return True + + +if __name__ == '__main__': + Interpreter().cmdloop() diff --git a/spotifyAuth.py b/spotifyAuth.py new file mode 100644 index 0000000..64a4b2c --- /dev/null +++ b/spotifyAuth.py @@ -0,0 +1,122 @@ +import base64 + +import requests +import webbrowser +from http.server import BaseHTTPRequestHandler, HTTPServer +import urllib.parse +import threading +import time +import os.path + +client_id = '***REDACTED***' +client_secret = None +token = None + + +def resolve_client_secret(): + global client_secret + if os.path.exists('secret'): + with open('secret', 'r') as file: + client_secret = file.readline().strip() + else: + client_secret = input("Enter the application secret: ") + with open('secret', 'w') as file: + file.write(client_secret) + + +def retrieve_web_api_token(authorization_code): + global token + + token_url = 'https://accounts.spotify.com/api/token' + + token_data = { + "grant_type": "authorization_code", + "client_secret": client_secret, + "client_id": client_id, + "code": authorization_code, + 'redirect_uri': 'http://localhost:8888', + } + + print("\nAuthenticating WEB API\n") + + auth_string = f'{client_id}:{client_secret}' + b64_auth_string = base64.b64encode(auth_string.encode()).decode() + + # Define the headers + headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': f'Basic {b64_auth_string}' + } + + token_response = requests.post(token_url, data=token_data, headers=headers) + + if token_response.status_code not in range(200, 299): + print(f"Failed to retrieve access token: {token_response.status_code}") + print(token_response.json()) + raise ValueError + + token = token_response.json()['access_token'] + + +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + + parsed_url = urllib.parse.urlparse(self.path) + query_params = urllib.parse.parse_qs(parsed_url.query) + + if 'code' in query_params: + authorization_code = query_params['code'][0] + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code received. You can close this window.') + + retrieve_web_api_token(authorization_code) + + threading.Thread(target=self.server.shutdown).start() + + else: + self.send_response(400) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'Authorization code not found.') + + +def run_redirect_server(): + httpd = HTTPServer(('localhost', 8888), RequestHandler) + print('Starting HTTP redirection server') + httpd.serve_forever() + + +def open_authenticate_page(): + authorize_url = 'https://accounts.spotify.com/authorize' + + params = { + 'client_id': client_id, + 'response_type': 'code', + 'redirect_uri': 'http://localhost:8888', + "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", + } + + auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}" + webbrowser.open(auth_url) + + +def authenticate(): + resolve_client_secret() + + redirect_server = threading.Thread(target=run_redirect_server) + redirect_server.start() + + time.sleep(1) + open_authenticate_page() + + redirect_server.join() + + return token + + +if __name__ == "__main__": + authenticate() + diff --git a/spotifyFetch.py b/spotifyFetch.py new file mode 100644 index 0000000..ec590bc --- /dev/null +++ b/spotifyFetch.py @@ -0,0 +1,147 @@ +import requests +from loadArtwork import load +from spotifyAuth import authenticate +from common import * + +FETCH_STEP = 50 +token = '' + + +def fetch(endpoint, method='GET', body=None): + url = f'https://api.spotify.com/{endpoint}' + headers = { + 'Authorization': f'Bearer {token}', + } + + response = requests.request(method, url, headers=headers, json=body) + response.raise_for_status() + return response.json() + + +def fetch_playlists(user_id): + print("Fetching playlists") + endpoint = f"v1/me/playlists" + pl_total = fetch(f'{endpoint}?limit=1')['total'] + pl_count = 0 + playlists = [] + + while pl_count < pl_total: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}') + playlists += res['items'] + pl_count += FETCH_STEP + + return playlists + + +def fetch_playlist_items(playlist_id): + endpoint = f"v1/playlists/{playlist_id}/tracks" + items = fetch(f'{endpoint}?fields=total')['total'] + loaded = 0 + tracks = [] + + while loaded < items: + res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}') + + tracks += res['items'] + loaded += FETCH_STEP + + return tracks + + +def fetch_tracks(user_id): + print("Fetching tracks") + endpoint = f"v1/me/tracks" + total = fetch(f"{endpoint}?limit=1&market=ES")['total'] + current = 0 + tracks = [] + + while total > current: + res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES") + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "tracks") + return tracks + + +def fetch_tracks_top(): + print("Fetching top tracks") + total = fetch(f'v1/me/top/tracks?fields=total')['total'] + current = 0 + tracks = [] + + while current < total: + res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term') + tracks += res['items'] + current += FETCH_STEP + + save_data(tracks, "top_tracks") + return tracks + + +def fetch_artists_top(): + print("Fetching top artists") + + total = fetch(f'v1/me/top/artists?fields=total')['total'] + current = 0 + artists = [] + + while current < total: + res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term') + artists += res['items'] + current += FETCH_STEP + + save_data(artists, "top_artists") + return artists + + +def fetch_all_playlist_data(user_id): + playlists = fetch_playlists(user_id) + + save_data(playlists, 'playlists') + + print(len(playlists)) + + playlist_items = {} + + print("Fetching playlists items") + + index = 0 + for pl in playlists: + pl_id = pl['id'] + name = pl['name'] + playlist_items[name] = fetch_playlist_items(pl_id) + print(f"{index} - {len(playlist_items[name])}") + index += 1 + + save_data(playlist_items, "playlistTracks") + + +def get_user_data(): + print("Fetching user data") + + user_profile = fetch(f'v1/me') + user_id = user_profile['id'] + user_data = fetch(f'v1/users/{user_id}') + + save_data(user_profile, "user_profile") + save_data(user_data, "user_data") + return user_id + + +def fetch_data(): + global token + token = authenticate() + + user_id = get_user_data() + + fetch_tracks(user_id) + fetch_all_playlist_data(user_id) + fetch_tracks_top() + fetch_artists_top() + + load() + + +if __name__ == "__main__": + fetch_data() From 6eea2ec040c53f86ac907651567c1394fc5fb96e Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Mon, 8 Jul 2024 23:24:51 +0300 Subject: [PATCH 05/20] atmop --- main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.py b/main.py index c88776b..49af365 100644 --- a/main.py +++ b/main.py @@ -93,6 +93,14 @@ class Interpreter(cmd.Cmd): """Fetch all spotify data""" try: fetch_data() + + global playlists, top_artists, top_tracks, user_tracks + + playlists = get_data('playlistTracks') + top_artists = get_data('top_artists') + top_tracks = get_data('top_tracks') + user_tracks = get_data('tracks') + except ValueError: print("Can not fetch the data.") From 366860a1ad3efd32572eb16e6c71d3a85cfd6425 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Mon, 8 Jul 2024 23:24:51 +0300 Subject: [PATCH 06/20] atmop --- main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/main.py b/main.py index c88776b..49af365 100644 --- a/main.py +++ b/main.py @@ -93,6 +93,14 @@ class Interpreter(cmd.Cmd): """Fetch all spotify data""" try: fetch_data() + + global playlists, top_artists, top_tracks, user_tracks + + playlists = get_data('playlistTracks') + top_artists = get_data('top_artists') + top_tracks = get_data('top_tracks') + user_tracks = get_data('tracks') + except ValueError: print("Can not fetch the data.") From 1ed118b31baee3eacab7a99d2d97d644421b85dd Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Wed, 10 Jul 2024 15:42:15 +0300 Subject: [PATCH 07/20] tmp --- common.py | 8 +++- libResolver.py | 97 +++++++++++++++++++++++++++++++++++++++++++ main.py | 6 --- spotifyFetch.py | 26 ++++++++++++ spotifyJsonCleanUp.py | 32 ++++++++++++++ 5 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 libResolver.py create mode 100644 spotifyJsonCleanUp.py diff --git a/common.py b/common.py index bb08d3e..ba9c296 100644 --- a/common.py +++ b/common.py @@ -22,4 +22,10 @@ def save_data(data, name): file_path = os.path.join(data_dir, f"{name}.json") with open(file_path, 'w') as file: json.dump(data, file, indent=4) - return f"Data saved to {name}.json." \ No newline at end of file + return f"Data saved to {name}.json." + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists diff --git a/libResolver.py b/libResolver.py new file mode 100644 index 0000000..9d89ffe --- /dev/null +++ b/libResolver.py @@ -0,0 +1,97 @@ +import common +import os +import fnmatch +import json +from fuzzywuzzy import fuzz + + +def get_local_library(root_dir): + music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') + music_files_map = {} + for dir_path, dir_names, filenames in os.walk(root_dir): + for pattern in music_extensions: + for filename in fnmatch.filter(filenames, pattern): + song_name = os.path.splitext(filename)[0] + relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) + music_files_map[song_name] = relative_path + return music_files_map + + +def get_matched(pattern, patterns): + result = {} + for target_pattern in patterns: + similarity_score = fuzz.token_set_ratio(pattern, target_pattern) + result[target_pattern] = similarity_score / 100 + + sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) + return sorted_results + + +def resolve_tracks(source_patterns, target_patterns): + from tqdm import tqdm + + pattern_map = {} + + total = len(source_patterns) + with tqdm(total=total, desc='Progress') as pbar: + for pattern in source_patterns: + res = get_matched(pattern, target_patterns) + matched = {"score": 0, "items": []} + if len(res) and res[0][1] > 0.9: + matched['items'] = res[0:min(len(res), 2)] + matched['score'] = res[0][1] + pattern_map[pattern] = matched + pbar.update(1) + + return pattern_map + + +def resolver1(local_library): + track_descriptions = common.get_data("tracks") + + source_patterns = [] + for track in track_descriptions: + artists = common.get_artists_str(track['track']['artists']) + pattern = f"{track['track']['name']} {artists}" + source_patterns.append(pattern) + + target_patterns = [local for _, local in local_library.items()] + + pattern_map = resolve_tracks(source_patterns, target_patterns) + + with open('prefetched/resolved.json', 'w') as file: + json.dump(pattern_map, file, indent=2) + + +def resolver2(local_library): + found_tracks = common.get_data("local_library_on_spotify") + track_descriptions = common.get_data("tracks") + + user_tracks_ids = {track['track']['id']: track for track in track_descriptions} + + resolved_map = {} + + for name, found in found_tracks.items(): + if not len(found): + continue + track_id = found[0]['id'] + if track_id in user_tracks_ids: + resolved_map[name] = track_id + + with open('prefetched/resolved2.json', 'w') as file: + json.dump(resolved_map, file, indent=2) + +def resolve(root_directory): + local_library = get_local_library(root_directory) + with open('prefetched/local_library.json', 'w') as file: + json.dump(local_library, file, indent=2) + + resolver2(local_library) + + +if __name__ == "__main__": + # root_directory = "/mnt/main/data/music/raw" + #root_directory = input("root: ") + resolve("/mnt/main/data/music/raw") + + diff --git a/main.py b/main.py index 49af365..f74ae1c 100644 --- a/main.py +++ b/main.py @@ -12,12 +12,6 @@ def get_playlist_names(): return [name for name, items in playlists.items()] -def get_artists_str(artists_pack): - track_artists = [artist['name'] for artist in artists_pack] - artists = " ".join(track_artists) - return artists - - def print_playlist_tracks(name): print(name) for track in playlists[name]: diff --git a/spotifyFetch.py b/spotifyFetch.py index ec590bc..6827049 100644 --- a/spotifyFetch.py +++ b/spotifyFetch.py @@ -129,10 +129,36 @@ def get_user_data(): return user_id +def find_song(song_pattern): + tracks = fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + return tracks + + +def find_local_library(): + with open('prefetched/local_library.json', 'r') as file: + local_library = json.load(file) + + from tqdm import tqdm + + pattern_map = {} + + total = len(local_library) + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + found_tracks = {} + for track, path in local_library.items(): + found_tracks[track] = find_song(track) + pbar.update(1) + + with open('prefetched/local_library_on_spotify.json', 'w') as file: + json.dump(found_tracks, file, indent=2) + + def fetch_data(): global token token = authenticate() + find_local_library() + user_id = get_user_data() fetch_tracks(user_id) diff --git a/spotifyJsonCleanUp.py b/spotifyJsonCleanUp.py new file mode 100644 index 0000000..7501fbb --- /dev/null +++ b/spotifyJsonCleanUp.py @@ -0,0 +1,32 @@ +import os +import json + + +def process_json_files(directory): + for filename in os.listdir(directory): + if filename.endswith('.json') and ("_mod.json" not in filename): + filepath = os.path.join(directory, filename) + + with open(filepath, 'r') as f: + try: + data = json.load(f) + except json.JSONDecodeError: + continue + + def remove_available_markets(obj): + if isinstance(obj, dict): + return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} + elif isinstance(obj, list): + return [remove_available_markets(item) for item in obj] + else: + return obj + + modified_data = remove_available_markets(data) + + modified_filepath = os.path.join(directory, f"{os.path.splitext(filename)[0]}_mod.json") + with open(modified_filepath, 'w') as f: + json.dump(modified_data, f, indent=4) + + +if __name__ == "__main__": + process_json_files("prefetched") From 22e19557d81121badfe7fbeee3b33e16a48653b9 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Wed, 10 Jul 2024 15:42:15 +0300 Subject: [PATCH 08/20] tmp --- common.py | 8 +++- libResolver.py | 97 +++++++++++++++++++++++++++++++++++++++++++ main.py | 6 --- spotifyFetch.py | 26 ++++++++++++ spotifyJsonCleanUp.py | 32 ++++++++++++++ 5 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 libResolver.py create mode 100644 spotifyJsonCleanUp.py diff --git a/common.py b/common.py index bb08d3e..ba9c296 100644 --- a/common.py +++ b/common.py @@ -22,4 +22,10 @@ def save_data(data, name): file_path = os.path.join(data_dir, f"{name}.json") with open(file_path, 'w') as file: json.dump(data, file, indent=4) - return f"Data saved to {name}.json." \ No newline at end of file + return f"Data saved to {name}.json." + + +def get_artists_str(artists_pack): + track_artists = [artist['name'] for artist in artists_pack] + artists = " ".join(track_artists) + return artists diff --git a/libResolver.py b/libResolver.py new file mode 100644 index 0000000..9d89ffe --- /dev/null +++ b/libResolver.py @@ -0,0 +1,97 @@ +import common +import os +import fnmatch +import json +from fuzzywuzzy import fuzz + + +def get_local_library(root_dir): + music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') + music_files_map = {} + for dir_path, dir_names, filenames in os.walk(root_dir): + for pattern in music_extensions: + for filename in fnmatch.filter(filenames, pattern): + song_name = os.path.splitext(filename)[0] + relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) + music_files_map[song_name] = relative_path + return music_files_map + + +def get_matched(pattern, patterns): + result = {} + for target_pattern in patterns: + similarity_score = fuzz.token_set_ratio(pattern, target_pattern) + result[target_pattern] = similarity_score / 100 + + sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) + return sorted_results + + +def resolve_tracks(source_patterns, target_patterns): + from tqdm import tqdm + + pattern_map = {} + + total = len(source_patterns) + with tqdm(total=total, desc='Progress') as pbar: + for pattern in source_patterns: + res = get_matched(pattern, target_patterns) + matched = {"score": 0, "items": []} + if len(res) and res[0][1] > 0.9: + matched['items'] = res[0:min(len(res), 2)] + matched['score'] = res[0][1] + pattern_map[pattern] = matched + pbar.update(1) + + return pattern_map + + +def resolver1(local_library): + track_descriptions = common.get_data("tracks") + + source_patterns = [] + for track in track_descriptions: + artists = common.get_artists_str(track['track']['artists']) + pattern = f"{track['track']['name']} {artists}" + source_patterns.append(pattern) + + target_patterns = [local for _, local in local_library.items()] + + pattern_map = resolve_tracks(source_patterns, target_patterns) + + with open('prefetched/resolved.json', 'w') as file: + json.dump(pattern_map, file, indent=2) + + +def resolver2(local_library): + found_tracks = common.get_data("local_library_on_spotify") + track_descriptions = common.get_data("tracks") + + user_tracks_ids = {track['track']['id']: track for track in track_descriptions} + + resolved_map = {} + + for name, found in found_tracks.items(): + if not len(found): + continue + track_id = found[0]['id'] + if track_id in user_tracks_ids: + resolved_map[name] = track_id + + with open('prefetched/resolved2.json', 'w') as file: + json.dump(resolved_map, file, indent=2) + +def resolve(root_directory): + local_library = get_local_library(root_directory) + with open('prefetched/local_library.json', 'w') as file: + json.dump(local_library, file, indent=2) + + resolver2(local_library) + + +if __name__ == "__main__": + # root_directory = "/mnt/main/data/music/raw" + #root_directory = input("root: ") + resolve("/mnt/main/data/music/raw") + + diff --git a/main.py b/main.py index 49af365..f74ae1c 100644 --- a/main.py +++ b/main.py @@ -12,12 +12,6 @@ def get_playlist_names(): return [name for name, items in playlists.items()] -def get_artists_str(artists_pack): - track_artists = [artist['name'] for artist in artists_pack] - artists = " ".join(track_artists) - return artists - - def print_playlist_tracks(name): print(name) for track in playlists[name]: diff --git a/spotifyFetch.py b/spotifyFetch.py index ec590bc..6827049 100644 --- a/spotifyFetch.py +++ b/spotifyFetch.py @@ -129,10 +129,36 @@ def get_user_data(): return user_id +def find_song(song_pattern): + tracks = fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + return tracks + + +def find_local_library(): + with open('prefetched/local_library.json', 'r') as file: + local_library = json.load(file) + + from tqdm import tqdm + + pattern_map = {} + + total = len(local_library) + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + found_tracks = {} + for track, path in local_library.items(): + found_tracks[track] = find_song(track) + pbar.update(1) + + with open('prefetched/local_library_on_spotify.json', 'w') as file: + json.dump(found_tracks, file, indent=2) + + def fetch_data(): global token token = authenticate() + find_local_library() + user_id = get_user_data() fetch_tracks(user_id) diff --git a/spotifyJsonCleanUp.py b/spotifyJsonCleanUp.py new file mode 100644 index 0000000..7501fbb --- /dev/null +++ b/spotifyJsonCleanUp.py @@ -0,0 +1,32 @@ +import os +import json + + +def process_json_files(directory): + for filename in os.listdir(directory): + if filename.endswith('.json') and ("_mod.json" not in filename): + filepath = os.path.join(directory, filename) + + with open(filepath, 'r') as f: + try: + data = json.load(f) + except json.JSONDecodeError: + continue + + def remove_available_markets(obj): + if isinstance(obj, dict): + return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} + elif isinstance(obj, list): + return [remove_available_markets(item) for item in obj] + else: + return obj + + modified_data = remove_available_markets(data) + + modified_filepath = os.path.join(directory, f"{os.path.splitext(filename)[0]}_mod.json") + with open(modified_filepath, 'w') as f: + json.dump(modified_data, f, indent=4) + + +if __name__ == "__main__": + process_json_files("prefetched") From b68bda2e1bc185476cad5d6aa23d25845ae38a74 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Wed, 10 Jul 2024 20:54:40 +0300 Subject: [PATCH 09/20] tmp --- common.py => DataBase.py | 10 +++ Linker.py | 2 + LinkerPatternGenerator.py | 98 +++++++++++++++++++++++ LocalLibrary.py | 70 ++++++++++++++++ spotifyAuth.py => SpotifyAuthenticator.py | 0 spotifyFetch.py => SpotifyWebAPI.py | 67 ++++++++++------ libResolver.py | 97 ---------------------- loadArtwork.py | 45 ----------- main.py | 10 +-- spotifyJsonCleanUp.py | 32 -------- 10 files changed, 227 insertions(+), 204 deletions(-) rename common.py => DataBase.py (68%) create mode 100644 Linker.py create mode 100644 LinkerPatternGenerator.py create mode 100644 LocalLibrary.py rename spotifyAuth.py => SpotifyAuthenticator.py (100%) rename spotifyFetch.py => SpotifyWebAPI.py (74%) delete mode 100644 libResolver.py delete mode 100644 loadArtwork.py delete mode 100644 spotifyJsonCleanUp.py diff --git a/common.py b/DataBase.py similarity index 68% rename from common.py rename to DataBase.py index ba9c296..5eabc68 100644 --- a/common.py +++ b/DataBase.py @@ -17,6 +17,16 @@ def get_data(name): def save_data(data, name): + def remove_available_markets(obj): + if isinstance(obj, dict): + return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} + elif isinstance(obj, list): + return [remove_available_markets(item) for item in obj] + else: + return obj + + data = remove_available_markets(data) + os.makedirs(data_dir, exist_ok=True) file_path = os.path.join(data_dir, f"{name}.json") diff --git a/Linker.py b/Linker.py new file mode 100644 index 0000000..139597f --- /dev/null +++ b/Linker.py @@ -0,0 +1,2 @@ + + diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py new file mode 100644 index 0000000..e81a423 --- /dev/null +++ b/LinkerPatternGenerator.py @@ -0,0 +1,98 @@ +from DataBase import * +from SpotifyWebAPI import find_song, update_access_token + +from fuzzywuzzy import fuzz + + +def spotify_pattern_generator(): + update_access_token() + + local_library = get_data("local_library") + spotify_library = get_data("tracks") + + def find_local_songs_on_spotify(local_lib): + from tqdm import tqdm + total = len(local_library) + found_tracks_map = {} + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + for track_id, track in local_lib.items(): + search_pattern = f"{track['name']} {track['artist']} {track['album']}" + found_tracks = find_song(search_pattern) + found_tracks_map[track_id] = found_tracks[0:1] + pbar.update(1) + return found_tracks_map + + spotify_found_tracks = find_local_songs_on_spotify(local_library) + spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library} + + spotify_pattern = {} + for local_id, found_items in spotify_found_tracks.items(): + spotify_pattern[local_id] = {"score": 0.0, "items": []} + + if not len(found_items): + continue + + for found_item in found_items: + spotify_track_id = found_item['id'] + if spotify_track_id in spotify_user_track: + spotify_pattern[local_id]['items'].append(spotify_track_id) + spotify_user_track[spotify_track_id]['local_mappings'].append(local_id) + + for spotify_id, track_local_mappings in spotify_user_track.items(): + score = len(track_local_mappings['local_mappings']) + for local_id in track_local_mappings['local_mappings']: + spotify_pattern[local_id]['score'] = 1 / score + + save_data(spotify_pattern, "link_pattern_spotify") + + +def fuzzy_pattern_generator(): + local_library = get_data("local_library") + track_descriptions = get_data("tracks") + + def get_matched(src_pattern, patterns): + result = {} + for target_pattern in patterns: + similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern) + result[target_pattern] = similarity_score / 100 + + sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) + return sorted_results + + def resolve_tracks(s_patterns, t_patterns): + from tqdm import tqdm + + pattern_map = {} + + total = len(s_patterns) + with tqdm(total=total, desc='Progress') as pbar: + for s_pattern in s_patterns: + res = get_matched(s_pattern, t_patterns) + matched = {"score": 0, "items": []} + if len(res): + matched['items'] = res[0:min(len(res), 4)] + matched['score'] = res[0][1] + pattern_map[s_pattern] = matched + pbar.update(1) + + return pattern_map + + source_patterns = [] + for track in track_descriptions: + artists = get_artists_str(track['track']['artists']) + pattern = f"{track['track']['name']} {artists}" + source_patterns.append(pattern) + + target_patterns = [local['path'] for _, local in local_library.items()] + + fuzzy_pattern = resolve_tracks(source_patterns, target_patterns) + save_data(fuzzy_pattern, "link_pattern_fuzzy") + + +def run_pattern_generators(): + #fuzzy_pattern_generator() + spotify_pattern_generator() + + +if __name__ == "__main__": + run_pattern_generators() \ No newline at end of file diff --git a/LocalLibrary.py b/LocalLibrary.py new file mode 100644 index 0000000..2d8791d --- /dev/null +++ b/LocalLibrary.py @@ -0,0 +1,70 @@ +import os +import fnmatch + +import mutagen + +from DataBase import * +import eyed3 +from mutagen.flac import FLAC + + +music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') +exclude_directories = ("*mary--*", "*example-word*") + + +def update_local_library(root_dir): + old_library = get_data("local_library") + known_paths = {track['path']: track_id for track_id, track in old_library.items()} + new_library = {} + + song_id = '0' + + def get_new_song_id(prev_id, song_path): + if song_path in known_paths: + return int(known_paths[song_path]) + while (prev_id in old_library) or (prev_id in new_library): + prev_id = str(int(prev_id) + 1) + return prev_id + + for dir_path, dir_names, filenames in os.walk(root_dir): + dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)] + filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)] + + for filename in filtered_filenames: + song_name, track_type = os.path.splitext(filename) + relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) + song_id = get_new_song_id(song_id, relative_path) + new_library[song_id] = { + "name": song_name, + "path": relative_path, + "type": track_type, + "artist": "", + "album": "" + } + + for track_id, track in new_library.items(): + abs_path = os.path.join(root_dir, track['path']) + match track['type']: + case ".mp3": + tag = eyed3.load(abs_path).tag + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album + + case ".flac": + try: + metadata = FLAC(abs_path).tags + if 'artist' in metadata: + track['artist'] = " ".join(metadata['artist']) + if 'TITLE' in metadata: + track['name'] = " ".join(metadata['TITLE']) + if 'album' in metadata: + track['album'] = " ".join(metadata['album']) + except mutagen.MutagenError: + print(f"Invalid flac header {abs_path}") + + save_data(new_library, "local_library") + + +if __name__ == "__main__": + update_local_library("/mnt/main/data/music/raw") diff --git a/spotifyAuth.py b/SpotifyAuthenticator.py similarity index 100% rename from spotifyAuth.py rename to SpotifyAuthenticator.py diff --git a/spotifyFetch.py b/SpotifyWebAPI.py similarity index 74% rename from spotifyFetch.py rename to SpotifyWebAPI.py index 6827049..71ec2f7 100644 --- a/spotifyFetch.py +++ b/SpotifyWebAPI.py @@ -1,7 +1,8 @@ import requests -from loadArtwork import load -from spotifyAuth import authenticate -from common import * +from PIL import Image +from io import BytesIO +from SpotifyAuthenticator import authenticate +from DataBase import * FETCH_STEP = 50 token = '' @@ -18,6 +19,10 @@ def fetch(endpoint, method='GET', body=None): return response.json() +def find_song(song_pattern): + return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + + def fetch_playlists(user_id): print("Fetching playlists") endpoint = f"v1/me/playlists" @@ -129,36 +134,47 @@ def get_user_data(): return user_id -def find_song(song_pattern): - tracks = fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] - return tracks +def save_image_from_url(url, name, image_dir="playlist_covers"): + response = requests.get(url) + + if response.status_code != 200: + raise "Cannot fetch the playlist cover" + + image = Image.open(BytesIO(response.content)) + + directory = os.path.join(data_dir, image_dir) + os.makedirs(directory, exist_ok=True) + file_path = os.path.join(directory, f"{name}.jpg") + print(f"Image saved {file_path}") + image.save(file_path) -def find_local_library(): - with open('prefetched/local_library.json', 'r') as file: - local_library = json.load(file) +def load_playlist_covers(): + playlists = get_data('playlists') - from tqdm import tqdm - - pattern_map = {} - - total = len(local_library) - with tqdm(total=total, desc='Searching local songs on spotify') as pbar: - found_tracks = {} - for track, path in local_library.items(): - found_tracks[track] = find_song(track) - pbar.update(1) - - with open('prefetched/local_library_on_spotify.json', 'w') as file: - json.dump(found_tracks, file, indent=2) + for pl in playlists: + if len(pl['images']): + url = pl['images'][0]['url'] + save_image_from_url(url, pl['name']) -def fetch_data(): +def load_user_cover(): + user_data = get_data("user_data") + url = user_data['images'][1]['url'] + save_image_from_url(url, "user", ".") + + +def load_artworks(): + load_user_cover() + load_playlist_covers() + + +def update_access_token(): global token token = authenticate() - find_local_library() +def fetch_data(): user_id = get_user_data() fetch_tracks(user_id) @@ -166,8 +182,9 @@ def fetch_data(): fetch_tracks_top() fetch_artists_top() - load() + load_artworks() if __name__ == "__main__": + update_access_token() fetch_data() diff --git a/libResolver.py b/libResolver.py deleted file mode 100644 index 9d89ffe..0000000 --- a/libResolver.py +++ /dev/null @@ -1,97 +0,0 @@ -import common -import os -import fnmatch -import json -from fuzzywuzzy import fuzz - - -def get_local_library(root_dir): - music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') - music_files_map = {} - for dir_path, dir_names, filenames in os.walk(root_dir): - for pattern in music_extensions: - for filename in fnmatch.filter(filenames, pattern): - song_name = os.path.splitext(filename)[0] - relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) - music_files_map[song_name] = relative_path - return music_files_map - - -def get_matched(pattern, patterns): - result = {} - for target_pattern in patterns: - similarity_score = fuzz.token_set_ratio(pattern, target_pattern) - result[target_pattern] = similarity_score / 100 - - sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) - return sorted_results - - -def resolve_tracks(source_patterns, target_patterns): - from tqdm import tqdm - - pattern_map = {} - - total = len(source_patterns) - with tqdm(total=total, desc='Progress') as pbar: - for pattern in source_patterns: - res = get_matched(pattern, target_patterns) - matched = {"score": 0, "items": []} - if len(res) and res[0][1] > 0.9: - matched['items'] = res[0:min(len(res), 2)] - matched['score'] = res[0][1] - pattern_map[pattern] = matched - pbar.update(1) - - return pattern_map - - -def resolver1(local_library): - track_descriptions = common.get_data("tracks") - - source_patterns = [] - for track in track_descriptions: - artists = common.get_artists_str(track['track']['artists']) - pattern = f"{track['track']['name']} {artists}" - source_patterns.append(pattern) - - target_patterns = [local for _, local in local_library.items()] - - pattern_map = resolve_tracks(source_patterns, target_patterns) - - with open('prefetched/resolved.json', 'w') as file: - json.dump(pattern_map, file, indent=2) - - -def resolver2(local_library): - found_tracks = common.get_data("local_library_on_spotify") - track_descriptions = common.get_data("tracks") - - user_tracks_ids = {track['track']['id']: track for track in track_descriptions} - - resolved_map = {} - - for name, found in found_tracks.items(): - if not len(found): - continue - track_id = found[0]['id'] - if track_id in user_tracks_ids: - resolved_map[name] = track_id - - with open('prefetched/resolved2.json', 'w') as file: - json.dump(resolved_map, file, indent=2) - -def resolve(root_directory): - local_library = get_local_library(root_directory) - with open('prefetched/local_library.json', 'w') as file: - json.dump(local_library, file, indent=2) - - resolver2(local_library) - - -if __name__ == "__main__": - # root_directory = "/mnt/main/data/music/raw" - #root_directory = input("root: ") - resolve("/mnt/main/data/music/raw") - - diff --git a/loadArtwork.py b/loadArtwork.py deleted file mode 100644 index 4d2dc38..0000000 --- a/loadArtwork.py +++ /dev/null @@ -1,45 +0,0 @@ -import requests -import os -from PIL import Image -from io import BytesIO - -from common import * - - -def save_image_from_url(url, name, image_dir="playlist_covers"): - response = requests.get(url) - - if response.status_code != 200: - raise "Cannot fetch the playlist cover" - - image = Image.open(BytesIO(response.content)) - - directory = os.path.join(data_dir, image_dir) - os.makedirs(directory, exist_ok=True) - file_path = os.path.join(directory, f"{name}.jpg") - print(f"Image saved {file_path}") - image.save(file_path) - - -def load_playlist_covers(): - playlists = get_data('playlists') - - for pl in playlists: - if len(pl['images']): - url = pl['images'][0]['url'] - save_image_from_url(url, pl['name']) - - -def load_user_cover(): - user_data = get_data("user_data") - url = user_data['images'][1]['url'] - save_image_from_url(url, "user", ".") - - -def load(): - load_user_cover() - load_playlist_covers() - - -if __name__ == "__main__": - load() diff --git a/main.py b/main.py index f74ae1c..b1b1c79 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ -from common import * -from spotifyFetch import fetch_data +from DataBase import * +from SpotifyWebAPI import fetch_data import cmd playlists = get_data('playlistTracks') @@ -8,8 +8,8 @@ top_tracks = get_data('top_tracks') user_tracks = get_data('tracks') -def get_playlist_names(): - return [name for name, items in playlists.items()] +def get_playlist_names(pl): + return [name for name, items in pl.items()] def print_playlist_tracks(name): @@ -57,7 +57,7 @@ class Interpreter(cmd.Cmd): def do_playlists(self, arg): """prints user playlists""" - print("\n".join(get_playlist_names())) + print("\n".join(get_playlist_names(playlists))) def do_playlist_tracks(self, arg): """prints playlist [name]""" diff --git a/spotifyJsonCleanUp.py b/spotifyJsonCleanUp.py deleted file mode 100644 index 7501fbb..0000000 --- a/spotifyJsonCleanUp.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import json - - -def process_json_files(directory): - for filename in os.listdir(directory): - if filename.endswith('.json') and ("_mod.json" not in filename): - filepath = os.path.join(directory, filename) - - with open(filepath, 'r') as f: - try: - data = json.load(f) - except json.JSONDecodeError: - continue - - def remove_available_markets(obj): - if isinstance(obj, dict): - return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} - elif isinstance(obj, list): - return [remove_available_markets(item) for item in obj] - else: - return obj - - modified_data = remove_available_markets(data) - - modified_filepath = os.path.join(directory, f"{os.path.splitext(filename)[0]}_mod.json") - with open(modified_filepath, 'w') as f: - json.dump(modified_data, f, indent=4) - - -if __name__ == "__main__": - process_json_files("prefetched") From 59a3be110822d256a17d6d7e8e8e20318666b621 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Wed, 10 Jul 2024 20:54:40 +0300 Subject: [PATCH 10/20] tmp --- common.py => DataBase.py | 10 +++ Linker.py | 2 + LinkerPatternGenerator.py | 98 +++++++++++++++++++++++ LocalLibrary.py | 70 ++++++++++++++++ spotifyAuth.py => SpotifyAuthenticator.py | 0 spotifyFetch.py => SpotifyWebAPI.py | 67 ++++++++++------ libResolver.py | 97 ---------------------- loadArtwork.py | 45 ----------- main.py | 10 +-- spotifyJsonCleanUp.py | 32 -------- 10 files changed, 227 insertions(+), 204 deletions(-) rename common.py => DataBase.py (68%) create mode 100644 Linker.py create mode 100644 LinkerPatternGenerator.py create mode 100644 LocalLibrary.py rename spotifyAuth.py => SpotifyAuthenticator.py (100%) rename spotifyFetch.py => SpotifyWebAPI.py (74%) delete mode 100644 libResolver.py delete mode 100644 loadArtwork.py delete mode 100644 spotifyJsonCleanUp.py diff --git a/common.py b/DataBase.py similarity index 68% rename from common.py rename to DataBase.py index ba9c296..5eabc68 100644 --- a/common.py +++ b/DataBase.py @@ -17,6 +17,16 @@ def get_data(name): def save_data(data, name): + def remove_available_markets(obj): + if isinstance(obj, dict): + return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} + elif isinstance(obj, list): + return [remove_available_markets(item) for item in obj] + else: + return obj + + data = remove_available_markets(data) + os.makedirs(data_dir, exist_ok=True) file_path = os.path.join(data_dir, f"{name}.json") diff --git a/Linker.py b/Linker.py new file mode 100644 index 0000000..139597f --- /dev/null +++ b/Linker.py @@ -0,0 +1,2 @@ + + diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py new file mode 100644 index 0000000..e81a423 --- /dev/null +++ b/LinkerPatternGenerator.py @@ -0,0 +1,98 @@ +from DataBase import * +from SpotifyWebAPI import find_song, update_access_token + +from fuzzywuzzy import fuzz + + +def spotify_pattern_generator(): + update_access_token() + + local_library = get_data("local_library") + spotify_library = get_data("tracks") + + def find_local_songs_on_spotify(local_lib): + from tqdm import tqdm + total = len(local_library) + found_tracks_map = {} + with tqdm(total=total, desc='Searching local songs on spotify') as pbar: + for track_id, track in local_lib.items(): + search_pattern = f"{track['name']} {track['artist']} {track['album']}" + found_tracks = find_song(search_pattern) + found_tracks_map[track_id] = found_tracks[0:1] + pbar.update(1) + return found_tracks_map + + spotify_found_tracks = find_local_songs_on_spotify(local_library) + spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library} + + spotify_pattern = {} + for local_id, found_items in spotify_found_tracks.items(): + spotify_pattern[local_id] = {"score": 0.0, "items": []} + + if not len(found_items): + continue + + for found_item in found_items: + spotify_track_id = found_item['id'] + if spotify_track_id in spotify_user_track: + spotify_pattern[local_id]['items'].append(spotify_track_id) + spotify_user_track[spotify_track_id]['local_mappings'].append(local_id) + + for spotify_id, track_local_mappings in spotify_user_track.items(): + score = len(track_local_mappings['local_mappings']) + for local_id in track_local_mappings['local_mappings']: + spotify_pattern[local_id]['score'] = 1 / score + + save_data(spotify_pattern, "link_pattern_spotify") + + +def fuzzy_pattern_generator(): + local_library = get_data("local_library") + track_descriptions = get_data("tracks") + + def get_matched(src_pattern, patterns): + result = {} + for target_pattern in patterns: + similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern) + result[target_pattern] = similarity_score / 100 + + sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) + return sorted_results + + def resolve_tracks(s_patterns, t_patterns): + from tqdm import tqdm + + pattern_map = {} + + total = len(s_patterns) + with tqdm(total=total, desc='Progress') as pbar: + for s_pattern in s_patterns: + res = get_matched(s_pattern, t_patterns) + matched = {"score": 0, "items": []} + if len(res): + matched['items'] = res[0:min(len(res), 4)] + matched['score'] = res[0][1] + pattern_map[s_pattern] = matched + pbar.update(1) + + return pattern_map + + source_patterns = [] + for track in track_descriptions: + artists = get_artists_str(track['track']['artists']) + pattern = f"{track['track']['name']} {artists}" + source_patterns.append(pattern) + + target_patterns = [local['path'] for _, local in local_library.items()] + + fuzzy_pattern = resolve_tracks(source_patterns, target_patterns) + save_data(fuzzy_pattern, "link_pattern_fuzzy") + + +def run_pattern_generators(): + #fuzzy_pattern_generator() + spotify_pattern_generator() + + +if __name__ == "__main__": + run_pattern_generators() \ No newline at end of file diff --git a/LocalLibrary.py b/LocalLibrary.py new file mode 100644 index 0000000..2d8791d --- /dev/null +++ b/LocalLibrary.py @@ -0,0 +1,70 @@ +import os +import fnmatch + +import mutagen + +from DataBase import * +import eyed3 +from mutagen.flac import FLAC + + +music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') +exclude_directories = ("*mary--*", "*example-word*") + + +def update_local_library(root_dir): + old_library = get_data("local_library") + known_paths = {track['path']: track_id for track_id, track in old_library.items()} + new_library = {} + + song_id = '0' + + def get_new_song_id(prev_id, song_path): + if song_path in known_paths: + return int(known_paths[song_path]) + while (prev_id in old_library) or (prev_id in new_library): + prev_id = str(int(prev_id) + 1) + return prev_id + + for dir_path, dir_names, filenames in os.walk(root_dir): + dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)] + filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)] + + for filename in filtered_filenames: + song_name, track_type = os.path.splitext(filename) + relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) + song_id = get_new_song_id(song_id, relative_path) + new_library[song_id] = { + "name": song_name, + "path": relative_path, + "type": track_type, + "artist": "", + "album": "" + } + + for track_id, track in new_library.items(): + abs_path = os.path.join(root_dir, track['path']) + match track['type']: + case ".mp3": + tag = eyed3.load(abs_path).tag + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album + + case ".flac": + try: + metadata = FLAC(abs_path).tags + if 'artist' in metadata: + track['artist'] = " ".join(metadata['artist']) + if 'TITLE' in metadata: + track['name'] = " ".join(metadata['TITLE']) + if 'album' in metadata: + track['album'] = " ".join(metadata['album']) + except mutagen.MutagenError: + print(f"Invalid flac header {abs_path}") + + save_data(new_library, "local_library") + + +if __name__ == "__main__": + update_local_library("/mnt/main/data/music/raw") diff --git a/spotifyAuth.py b/SpotifyAuthenticator.py similarity index 100% rename from spotifyAuth.py rename to SpotifyAuthenticator.py diff --git a/spotifyFetch.py b/SpotifyWebAPI.py similarity index 74% rename from spotifyFetch.py rename to SpotifyWebAPI.py index 6827049..71ec2f7 100644 --- a/spotifyFetch.py +++ b/SpotifyWebAPI.py @@ -1,7 +1,8 @@ import requests -from loadArtwork import load -from spotifyAuth import authenticate -from common import * +from PIL import Image +from io import BytesIO +from SpotifyAuthenticator import authenticate +from DataBase import * FETCH_STEP = 50 token = '' @@ -18,6 +19,10 @@ def fetch(endpoint, method='GET', body=None): return response.json() +def find_song(song_pattern): + return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + + def fetch_playlists(user_id): print("Fetching playlists") endpoint = f"v1/me/playlists" @@ -129,36 +134,47 @@ def get_user_data(): return user_id -def find_song(song_pattern): - tracks = fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] - return tracks +def save_image_from_url(url, name, image_dir="playlist_covers"): + response = requests.get(url) + + if response.status_code != 200: + raise "Cannot fetch the playlist cover" + + image = Image.open(BytesIO(response.content)) + + directory = os.path.join(data_dir, image_dir) + os.makedirs(directory, exist_ok=True) + file_path = os.path.join(directory, f"{name}.jpg") + print(f"Image saved {file_path}") + image.save(file_path) -def find_local_library(): - with open('prefetched/local_library.json', 'r') as file: - local_library = json.load(file) +def load_playlist_covers(): + playlists = get_data('playlists') - from tqdm import tqdm - - pattern_map = {} - - total = len(local_library) - with tqdm(total=total, desc='Searching local songs on spotify') as pbar: - found_tracks = {} - for track, path in local_library.items(): - found_tracks[track] = find_song(track) - pbar.update(1) - - with open('prefetched/local_library_on_spotify.json', 'w') as file: - json.dump(found_tracks, file, indent=2) + for pl in playlists: + if len(pl['images']): + url = pl['images'][0]['url'] + save_image_from_url(url, pl['name']) -def fetch_data(): +def load_user_cover(): + user_data = get_data("user_data") + url = user_data['images'][1]['url'] + save_image_from_url(url, "user", ".") + + +def load_artworks(): + load_user_cover() + load_playlist_covers() + + +def update_access_token(): global token token = authenticate() - find_local_library() +def fetch_data(): user_id = get_user_data() fetch_tracks(user_id) @@ -166,8 +182,9 @@ def fetch_data(): fetch_tracks_top() fetch_artists_top() - load() + load_artworks() if __name__ == "__main__": + update_access_token() fetch_data() diff --git a/libResolver.py b/libResolver.py deleted file mode 100644 index 9d89ffe..0000000 --- a/libResolver.py +++ /dev/null @@ -1,97 +0,0 @@ -import common -import os -import fnmatch -import json -from fuzzywuzzy import fuzz - - -def get_local_library(root_dir): - music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a') - music_files_map = {} - for dir_path, dir_names, filenames in os.walk(root_dir): - for pattern in music_extensions: - for filename in fnmatch.filter(filenames, pattern): - song_name = os.path.splitext(filename)[0] - relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir) - music_files_map[song_name] = relative_path - return music_files_map - - -def get_matched(pattern, patterns): - result = {} - for target_pattern in patterns: - similarity_score = fuzz.token_set_ratio(pattern, target_pattern) - result[target_pattern] = similarity_score / 100 - - sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True) - return sorted_results - - -def resolve_tracks(source_patterns, target_patterns): - from tqdm import tqdm - - pattern_map = {} - - total = len(source_patterns) - with tqdm(total=total, desc='Progress') as pbar: - for pattern in source_patterns: - res = get_matched(pattern, target_patterns) - matched = {"score": 0, "items": []} - if len(res) and res[0][1] > 0.9: - matched['items'] = res[0:min(len(res), 2)] - matched['score'] = res[0][1] - pattern_map[pattern] = matched - pbar.update(1) - - return pattern_map - - -def resolver1(local_library): - track_descriptions = common.get_data("tracks") - - source_patterns = [] - for track in track_descriptions: - artists = common.get_artists_str(track['track']['artists']) - pattern = f"{track['track']['name']} {artists}" - source_patterns.append(pattern) - - target_patterns = [local for _, local in local_library.items()] - - pattern_map = resolve_tracks(source_patterns, target_patterns) - - with open('prefetched/resolved.json', 'w') as file: - json.dump(pattern_map, file, indent=2) - - -def resolver2(local_library): - found_tracks = common.get_data("local_library_on_spotify") - track_descriptions = common.get_data("tracks") - - user_tracks_ids = {track['track']['id']: track for track in track_descriptions} - - resolved_map = {} - - for name, found in found_tracks.items(): - if not len(found): - continue - track_id = found[0]['id'] - if track_id in user_tracks_ids: - resolved_map[name] = track_id - - with open('prefetched/resolved2.json', 'w') as file: - json.dump(resolved_map, file, indent=2) - -def resolve(root_directory): - local_library = get_local_library(root_directory) - with open('prefetched/local_library.json', 'w') as file: - json.dump(local_library, file, indent=2) - - resolver2(local_library) - - -if __name__ == "__main__": - # root_directory = "/mnt/main/data/music/raw" - #root_directory = input("root: ") - resolve("/mnt/main/data/music/raw") - - diff --git a/loadArtwork.py b/loadArtwork.py deleted file mode 100644 index 4d2dc38..0000000 --- a/loadArtwork.py +++ /dev/null @@ -1,45 +0,0 @@ -import requests -import os -from PIL import Image -from io import BytesIO - -from common import * - - -def save_image_from_url(url, name, image_dir="playlist_covers"): - response = requests.get(url) - - if response.status_code != 200: - raise "Cannot fetch the playlist cover" - - image = Image.open(BytesIO(response.content)) - - directory = os.path.join(data_dir, image_dir) - os.makedirs(directory, exist_ok=True) - file_path = os.path.join(directory, f"{name}.jpg") - print(f"Image saved {file_path}") - image.save(file_path) - - -def load_playlist_covers(): - playlists = get_data('playlists') - - for pl in playlists: - if len(pl['images']): - url = pl['images'][0]['url'] - save_image_from_url(url, pl['name']) - - -def load_user_cover(): - user_data = get_data("user_data") - url = user_data['images'][1]['url'] - save_image_from_url(url, "user", ".") - - -def load(): - load_user_cover() - load_playlist_covers() - - -if __name__ == "__main__": - load() diff --git a/main.py b/main.py index f74ae1c..b1b1c79 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ -from common import * -from spotifyFetch import fetch_data +from DataBase import * +from SpotifyWebAPI import fetch_data import cmd playlists = get_data('playlistTracks') @@ -8,8 +8,8 @@ top_tracks = get_data('top_tracks') user_tracks = get_data('tracks') -def get_playlist_names(): - return [name for name, items in playlists.items()] +def get_playlist_names(pl): + return [name for name, items in pl.items()] def print_playlist_tracks(name): @@ -57,7 +57,7 @@ class Interpreter(cmd.Cmd): def do_playlists(self, arg): """prints user playlists""" - print("\n".join(get_playlist_names())) + print("\n".join(get_playlist_names(playlists))) def do_playlist_tracks(self, arg): """prints playlist [name]""" diff --git a/spotifyJsonCleanUp.py b/spotifyJsonCleanUp.py deleted file mode 100644 index 7501fbb..0000000 --- a/spotifyJsonCleanUp.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import json - - -def process_json_files(directory): - for filename in os.listdir(directory): - if filename.endswith('.json') and ("_mod.json" not in filename): - filepath = os.path.join(directory, filename) - - with open(filepath, 'r') as f: - try: - data = json.load(f) - except json.JSONDecodeError: - continue - - def remove_available_markets(obj): - if isinstance(obj, dict): - return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'} - elif isinstance(obj, list): - return [remove_available_markets(item) for item in obj] - else: - return obj - - modified_data = remove_available_markets(data) - - modified_filepath = os.path.join(directory, f"{os.path.splitext(filename)[0]}_mod.json") - with open(modified_filepath, 'w') as f: - json.dump(modified_data, f, indent=4) - - -if __name__ == "__main__": - process_json_files("prefetched") From afc020fdbf0ab01d6f3cd10be91abe229059a916 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 11 Jul 2024 10:26:27 +0300 Subject: [PATCH 11/20] tmp --- DataBase.py | 7 +++++++ LinkerPatternGenerator.py | 39 ++++++++++++++++++++++++++++++++++++--- main.py | 20 ++++++++++++++------ 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/DataBase.py b/DataBase.py index 5eabc68..3132b7b 100644 --- a/DataBase.py +++ b/DataBase.py @@ -4,6 +4,13 @@ import os data_dir = 'prefetched' +class SongTags: + def __init__(self, title='', artist='', album=''): + self.artist = artist + self.title = title + self.album = album + + def get_data(name): file_path = os.path.join(data_dir, f"{name}.json") try: diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index e81a423..bb16839 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -2,6 +2,7 @@ from DataBase import * from SpotifyWebAPI import find_song, update_access_token from fuzzywuzzy import fuzz +from tqdm import tqdm def spotify_pattern_generator(): @@ -11,7 +12,6 @@ def spotify_pattern_generator(): spotify_library = get_data("tracks") def find_local_songs_on_spotify(local_lib): - from tqdm import tqdm total = len(local_library) found_tracks_map = {} with tqdm(total=total, desc='Searching local songs on spotify') as pbar: @@ -60,7 +60,6 @@ def fuzzy_pattern_generator(): return sorted_results def resolve_tracks(s_patterns, t_patterns): - from tqdm import tqdm pattern_map = {} @@ -89,9 +88,43 @@ def fuzzy_pattern_generator(): save_data(fuzzy_pattern, "link_pattern_fuzzy") +def fuzzy_tag_pattern_generator(): + local_library = get_data("local_library") + spotify_library = get_data("tracks") + mappings = {} + + def fuzzy_tags_ratio(first: SongTags, second: SongTags): + title_ratio = fuzz.ratio(first.title, second.title) / 100 + artist_ratio = fuzz.ratio(first.artist, second.artist) / 100 + album_ratio = fuzz.ratio(first.album, second.album) / 100 + return sum([title_ratio, artist_ratio, album_ratio]) / 3 + + with tqdm(total=len(local_library), desc='Searching local songs on spotify') as pbar: + for local_track_id, local_track in local_library.items(): + mapping = mappings.get(local_track_id, {"score": 1.0, "items": []}) + ratios = [] + for remote_track in spotify_library: + track = remote_track['track'] + + tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album']) + tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name']) + + ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) + + ratios.sort(reverse=True, key=lambda x: x[0]) + if ratios[0][0] > 0.8: + mapping['items'] = [ratios[0][1]] + + mappings[local_track_id] = mapping + pbar.update(1) + + save_data(mappings, "link_pattern_tags") + + def run_pattern_generators(): #fuzzy_pattern_generator() - spotify_pattern_generator() + #spotify_pattern_generator() + fuzzy_tag_pattern_generator() if __name__ == "__main__": diff --git a/main.py b/main.py index b1b1c79..339152f 100644 --- a/main.py +++ b/main.py @@ -12,9 +12,12 @@ def get_playlist_names(pl): return [name for name, items in pl.items()] -def print_playlist_tracks(name): +def print_playlist_tracks(pl_id: int): + pls = list(playlists) + name = pls[pl_id] + pl = playlists[name] print(name) - for track in playlists[name]: + for track in pl: track_data = track['track'] track_name = track_data['name'] artists = get_artists_str(track_data['artists']) @@ -46,9 +49,11 @@ def print_stats(): def print_tracks(): + track_idx = 0 for trackItem in user_tracks: track = trackItem['track'] - print(f" '{track['name']}' - {get_artists_str(track['artists'])}") + print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'") + track_idx += 1 class Interpreter(cmd.Cmd): @@ -57,13 +62,16 @@ class Interpreter(cmd.Cmd): def do_playlists(self, arg): """prints user playlists""" - print("\n".join(get_playlist_names(playlists))) + pl_id = 0 + for pl in get_playlist_names(playlists): + print(f"{pl_id} - {pl}") + pl_id += 1 def do_playlist_tracks(self, arg): """prints playlist [name]""" try: - name = arg.split()[0] - print_playlist_tracks(name) + pl_id = int(arg.split()[0]) + print_playlist_tracks(pl_id) except IndexError: print("Invalid input") From 3744bd1c56473cd36e73716e9df99b028e62f697 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 11 Jul 2024 10:26:27 +0300 Subject: [PATCH 12/20] tmp --- DataBase.py | 7 +++++++ LinkerPatternGenerator.py | 39 ++++++++++++++++++++++++++++++++++++--- main.py | 20 ++++++++++++++------ 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/DataBase.py b/DataBase.py index 5eabc68..3132b7b 100644 --- a/DataBase.py +++ b/DataBase.py @@ -4,6 +4,13 @@ import os data_dir = 'prefetched' +class SongTags: + def __init__(self, title='', artist='', album=''): + self.artist = artist + self.title = title + self.album = album + + def get_data(name): file_path = os.path.join(data_dir, f"{name}.json") try: diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index e81a423..bb16839 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -2,6 +2,7 @@ from DataBase import * from SpotifyWebAPI import find_song, update_access_token from fuzzywuzzy import fuzz +from tqdm import tqdm def spotify_pattern_generator(): @@ -11,7 +12,6 @@ def spotify_pattern_generator(): spotify_library = get_data("tracks") def find_local_songs_on_spotify(local_lib): - from tqdm import tqdm total = len(local_library) found_tracks_map = {} with tqdm(total=total, desc='Searching local songs on spotify') as pbar: @@ -60,7 +60,6 @@ def fuzzy_pattern_generator(): return sorted_results def resolve_tracks(s_patterns, t_patterns): - from tqdm import tqdm pattern_map = {} @@ -89,9 +88,43 @@ def fuzzy_pattern_generator(): save_data(fuzzy_pattern, "link_pattern_fuzzy") +def fuzzy_tag_pattern_generator(): + local_library = get_data("local_library") + spotify_library = get_data("tracks") + mappings = {} + + def fuzzy_tags_ratio(first: SongTags, second: SongTags): + title_ratio = fuzz.ratio(first.title, second.title) / 100 + artist_ratio = fuzz.ratio(first.artist, second.artist) / 100 + album_ratio = fuzz.ratio(first.album, second.album) / 100 + return sum([title_ratio, artist_ratio, album_ratio]) / 3 + + with tqdm(total=len(local_library), desc='Searching local songs on spotify') as pbar: + for local_track_id, local_track in local_library.items(): + mapping = mappings.get(local_track_id, {"score": 1.0, "items": []}) + ratios = [] + for remote_track in spotify_library: + track = remote_track['track'] + + tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album']) + tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name']) + + ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) + + ratios.sort(reverse=True, key=lambda x: x[0]) + if ratios[0][0] > 0.8: + mapping['items'] = [ratios[0][1]] + + mappings[local_track_id] = mapping + pbar.update(1) + + save_data(mappings, "link_pattern_tags") + + def run_pattern_generators(): #fuzzy_pattern_generator() - spotify_pattern_generator() + #spotify_pattern_generator() + fuzzy_tag_pattern_generator() if __name__ == "__main__": diff --git a/main.py b/main.py index b1b1c79..339152f 100644 --- a/main.py +++ b/main.py @@ -12,9 +12,12 @@ def get_playlist_names(pl): return [name for name, items in pl.items()] -def print_playlist_tracks(name): +def print_playlist_tracks(pl_id: int): + pls = list(playlists) + name = pls[pl_id] + pl = playlists[name] print(name) - for track in playlists[name]: + for track in pl: track_data = track['track'] track_name = track_data['name'] artists = get_artists_str(track_data['artists']) @@ -46,9 +49,11 @@ def print_stats(): def print_tracks(): + track_idx = 0 for trackItem in user_tracks: track = trackItem['track'] - print(f" '{track['name']}' - {get_artists_str(track['artists'])}") + print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'") + track_idx += 1 class Interpreter(cmd.Cmd): @@ -57,13 +62,16 @@ class Interpreter(cmd.Cmd): def do_playlists(self, arg): """prints user playlists""" - print("\n".join(get_playlist_names(playlists))) + pl_id = 0 + for pl in get_playlist_names(playlists): + print(f"{pl_id} - {pl}") + pl_id += 1 def do_playlist_tracks(self, arg): """prints playlist [name]""" try: - name = arg.split()[0] - print_playlist_tracks(name) + pl_id = int(arg.split()[0]) + print_playlist_tracks(pl_id) except IndexError: print("Invalid input") From 9fe80257a85662126364f157bee0560eb350a7a2 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 11 Jul 2024 11:57:06 +0300 Subject: [PATCH 13/20] tmp --- LinkerPatternGenerator.py | 20 +++++++---- main.py | 73 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index bb16839..c037899 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -94,10 +94,10 @@ def fuzzy_tag_pattern_generator(): mappings = {} def fuzzy_tags_ratio(first: SongTags, second: SongTags): - title_ratio = fuzz.ratio(first.title, second.title) / 100 - artist_ratio = fuzz.ratio(first.artist, second.artist) / 100 - album_ratio = fuzz.ratio(first.album, second.album) / 100 - return sum([title_ratio, artist_ratio, album_ratio]) / 3 + title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100 + artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100 + album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100 + return title_ratio, artist_ratio, album_ratio with tqdm(total=len(local_library), desc='Searching local songs on spotify') as pbar: for local_track_id, local_track in local_library.items(): @@ -111,9 +111,15 @@ def fuzzy_tag_pattern_generator(): ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) - ratios.sort(reverse=True, key=lambda x: x[0]) - if ratios[0][0] > 0.8: - mapping['items'] = [ratios[0][1]] + filtered_ratios = [] + threshold = 0.8 + for ratio in ratios: + if all([val > threshold for val in ratio[0]]): + filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1])) + + filtered_ratios.sort(reverse=True, key=lambda x: x[0]) + if len(filtered_ratios): + mapping['items'] = [filtered_ratios[0][1]] mappings[local_track_id] = mapping pbar.update(1) diff --git a/main.py b/main.py index 339152f..6e04234 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,9 @@ playlists = get_data('playlistTracks') top_artists = get_data('top_artists') top_tracks = get_data('top_tracks') user_tracks = get_data('tracks') +link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')] +local_library = get_data("local_library") +user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} def get_playlist_names(pl): @@ -56,10 +59,80 @@ def print_tracks(): track_idx += 1 +def sort_remote_tracks_by_popularity(track_ids): + out = [] + for top_track in top_tracks: + if top_track['id'] in track_ids: + out.append(top_track['id']) + return out + + +def print_link_stats(link_pattern): + resolved_reversed = {} + resolved = [] + unresolved_local = [] + unresolved_remote = [] + + for local_id, links in link_pattern.items(): + if len(links['items']): + resolved.append(local_id) + resolved_reversed[links['items'][0]] = local_id + else: + unresolved_local.append(local_id) + + for track in user_tracks: + track_id = track['track']['id'] + if track_id not in resolved_reversed: + unresolved_remote.append(track_id) + + print(f"Local Tracks: {len(local_library)}") + print(f"Remote Tracks: {len(user_tracks)}") + + unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) + unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] + + print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") + index = 0 + for remote_id in unresolved_remote: + remote_track = user_tracks_by_id[remote_id] + print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + print(f"\nResolved tracks: {len(resolved)}") + index = 0 + for link in resolved: + remote_id = link_pattern[link]['items'][0] + local_track = local_library[link] + remote_track = user_tracks_by_id[remote_id] + + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') + print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + return + print(f"\nUnresolved tracks from the local: {len(unresolved_local)}") + index = 0 + for link in unresolved_local: + local_track = local_library[link] + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'") + index += 1 + + class Interpreter(cmd.Cmd): intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" prompt = "(spotify) " + def do_link_patterns(self, arg): + """prints available linking patterns""" + print(f"0 - fuzzy tags") + print(f"1 - spotify") + print(f"2 - fuzzy") + + def do_link_pattern(self, arg): + """prints available linking patterns""" + pattern_id = int(arg.split()[0]) + print_link_stats(link_patterns[pattern_id]) + def do_playlists(self, arg): """prints user playlists""" pl_id = 0 From e044b7ffd4bcfd7e6ec2afc3a934968216008ef6 Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 11 Jul 2024 11:57:06 +0300 Subject: [PATCH 14/20] tmp --- LinkerPatternGenerator.py | 20 +++++++---- main.py | 73 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index bb16839..c037899 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -94,10 +94,10 @@ def fuzzy_tag_pattern_generator(): mappings = {} def fuzzy_tags_ratio(first: SongTags, second: SongTags): - title_ratio = fuzz.ratio(first.title, second.title) / 100 - artist_ratio = fuzz.ratio(first.artist, second.artist) / 100 - album_ratio = fuzz.ratio(first.album, second.album) / 100 - return sum([title_ratio, artist_ratio, album_ratio]) / 3 + title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100 + artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100 + album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100 + return title_ratio, artist_ratio, album_ratio with tqdm(total=len(local_library), desc='Searching local songs on spotify') as pbar: for local_track_id, local_track in local_library.items(): @@ -111,9 +111,15 @@ def fuzzy_tag_pattern_generator(): ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id'])) - ratios.sort(reverse=True, key=lambda x: x[0]) - if ratios[0][0] > 0.8: - mapping['items'] = [ratios[0][1]] + filtered_ratios = [] + threshold = 0.8 + for ratio in ratios: + if all([val > threshold for val in ratio[0]]): + filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1])) + + filtered_ratios.sort(reverse=True, key=lambda x: x[0]) + if len(filtered_ratios): + mapping['items'] = [filtered_ratios[0][1]] mappings[local_track_id] = mapping pbar.update(1) diff --git a/main.py b/main.py index 339152f..6e04234 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,9 @@ playlists = get_data('playlistTracks') top_artists = get_data('top_artists') top_tracks = get_data('top_tracks') user_tracks = get_data('tracks') +link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')] +local_library = get_data("local_library") +user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks} def get_playlist_names(pl): @@ -56,10 +59,80 @@ def print_tracks(): track_idx += 1 +def sort_remote_tracks_by_popularity(track_ids): + out = [] + for top_track in top_tracks: + if top_track['id'] in track_ids: + out.append(top_track['id']) + return out + + +def print_link_stats(link_pattern): + resolved_reversed = {} + resolved = [] + unresolved_local = [] + unresolved_remote = [] + + for local_id, links in link_pattern.items(): + if len(links['items']): + resolved.append(local_id) + resolved_reversed[links['items'][0]] = local_id + else: + unresolved_local.append(local_id) + + for track in user_tracks: + track_id = track['track']['id'] + if track_id not in resolved_reversed: + unresolved_remote.append(track_id) + + print(f"Local Tracks: {len(local_library)}") + print(f"Remote Tracks: {len(user_tracks)}") + + unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote) + unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)] + + print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}") + index = 0 + for remote_id in unresolved_remote: + remote_track = user_tracks_by_id[remote_id] + print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + print(f"\nResolved tracks: {len(resolved)}") + index = 0 + for link in resolved: + remote_id = link_pattern[link]['items'][0] + local_track = local_library[link] + remote_track = user_tracks_by_id[remote_id] + + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ') + print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'") + index += 1 + + return + print(f"\nUnresolved tracks from the local: {len(unresolved_local)}") + index = 0 + for link in unresolved_local: + local_track = local_library[link] + print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'") + index += 1 + + class Interpreter(cmd.Cmd): intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n" prompt = "(spotify) " + def do_link_patterns(self, arg): + """prints available linking patterns""" + print(f"0 - fuzzy tags") + print(f"1 - spotify") + print(f"2 - fuzzy") + + def do_link_pattern(self, arg): + """prints available linking patterns""" + pattern_id = int(arg.split()[0]) + print_link_stats(link_patterns[pattern_id]) + def do_playlists(self, arg): """prints user playlists""" pl_id = 0 From d4d31d6752880e40c56847561009e8601c7a01ca Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 11 Jul 2024 13:36:53 +0300 Subject: [PATCH 15/20] tmp --- LinkerPatternGenerator.py | 2 +- LocalLibrary.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index c037899..11fdf4e 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -134,4 +134,4 @@ def run_pattern_generators(): if __name__ == "__main__": - run_pattern_generators() \ No newline at end of file + run_pattern_generators() diff --git a/LocalLibrary.py b/LocalLibrary.py index 2d8791d..2cc220b 100644 --- a/LocalLibrary.py +++ b/LocalLibrary.py @@ -47,9 +47,12 @@ def update_local_library(root_dir): match track['type']: case ".mp3": tag = eyed3.load(abs_path).tag - track['artist'] = tag.artist - track['name'] = tag.title - track['album'] = tag.album + if not tag: + print(f"Invalid mp3 header {abs_path}") + else: + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album case ".flac": try: From b2e582c56654b87ac4eb103169390c260f9be97a Mon Sep 17 00:00:00 2001 From: IlyaShurupov Date: Thu, 11 Jul 2024 13:36:53 +0300 Subject: [PATCH 16/20] tmp --- LinkerPatternGenerator.py | 2 +- LocalLibrary.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index c037899..11fdf4e 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -134,4 +134,4 @@ def run_pattern_generators(): if __name__ == "__main__": - run_pattern_generators() \ No newline at end of file + run_pattern_generators() diff --git a/LocalLibrary.py b/LocalLibrary.py index 2d8791d..2cc220b 100644 --- a/LocalLibrary.py +++ b/LocalLibrary.py @@ -47,9 +47,12 @@ def update_local_library(root_dir): match track['type']: case ".mp3": tag = eyed3.load(abs_path).tag - track['artist'] = tag.artist - track['name'] = tag.title - track['album'] = tag.album + if not tag: + print(f"Invalid mp3 header {abs_path}") + else: + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album case ".flac": try: From e68e29fa4adae3cebf501c2935299d90e84385b8 Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:26:51 +0300 Subject: [PATCH 17/20] tmp --- Linker.py | 2 - LocalLibrary.py => LocalLibraryIndexer.py | 0 ParseItunesToJson.py | 101 ++++++++++++++++++++++ env | 36 ++++++++ main.py | 3 +- 5 files changed, 139 insertions(+), 3 deletions(-) delete mode 100644 Linker.py rename LocalLibrary.py => LocalLibraryIndexer.py (100%) create mode 100755 ParseItunesToJson.py create mode 100755 env diff --git a/Linker.py b/Linker.py deleted file mode 100644 index 139597f..0000000 --- a/Linker.py +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/LocalLibrary.py b/LocalLibraryIndexer.py similarity index 100% rename from LocalLibrary.py rename to LocalLibraryIndexer.py diff --git a/ParseItunesToJson.py b/ParseItunesToJson.py new file mode 100755 index 0000000..1505a36 --- /dev/null +++ b/ParseItunesToJson.py @@ -0,0 +1,101 @@ +import xmltodict +import json + + +def flatten_dict(d): + + out = [] + for song in d: + newSong = {} + + strCount = 0 + intCount = 0 + dateCount = 0 + + def getStr(id): + nonlocal strCount + if id not in song["key"]: + return "undef" + if len(song["string"]) <= strCount: + return "error" + strCount += 1 + return song["string"][strCount - 1] + + def getInt(id): + nonlocal intCount + if id not in song["key"]: + return -1 + if len(song["integer"]) <= intCount: + return -1 + intCount += 1 + return song["integer"][intCount - 1] + + def getDate(id): + nonlocal dateCount + if id not in song["key"]: + return "undef" + if len(song["date"]) <= dateCount: + return "error" + dateCount += 1 + return song["date"][dateCount - 1] + + newSong["Track ID"] = getInt("Track ID") + newSong["Name"] = getStr("Name") + newSong["Artist"] = getStr("Artist") + newSong["Album Artist"] = getStr("Album Artist") + newSong["Composer"] = getStr("Composer") + newSong["Album"] = getStr("Album") + newSong["Genre"] = getStr("Genre") + newSong["Kind"] = getStr("Kind") + newSong["Size"] = getInt("Size") + newSong["Total Time"] = getInt("Total Time") + newSong["Disc Number"] = getInt("Disc Number") + newSong["Disc Count"] = getInt("Disc Count") + newSong["Track Number"] = getInt("Track Number") + newSong["Track Count"] = getInt("Track Count") + newSong["Year"] = getInt("Year") + newSong["Date Modified"] = getDate("Date Modified") + newSong["Date Added"] = getDate("Date Added") + newSong["Bit Rate"] = getInt("Bit Rate") + newSong["Sample Rate"] = getInt("Sample Rate") + newSong["Play Count"] = getInt("Play Count") + newSong["Play Date"] = getInt("Play Date") + newSong["Play Date UTC"] = getDate("Play Date UTC") + newSong["Skip Count"] = getInt("Skip Count") + newSong["Skip Date"] = getDate("Skip Date") + newSong["Release Date"] = getDate("Release Date") + newSong["Album Rating"] = getInt("Album Rating") + newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"] + newSong["Loved"] = "Loved" in song["key"] + newSong["Album Loved"] = "Album Loved" in song["key"] + newSong["Explicit"] = "Explicit" in song["key"] + newSong["Compilation"] = "Compilation" in song["key"] + newSong["Artwork Count"] = getInt("Artwork Count") + newSong["Sort Album"] = getStr("Sort Album") + newSong["Sort Artist"] = getStr("Sort Artist") + newSong["Sort Name"] = getStr("Sort Name") + newSong["Persistent ID"] = getStr("Persistent ID") + newSong["Track Type"] = getStr("Track Type") + + out.append(newSong) + + return out + + +def convert(filename, out_path): + with open(filename, 'r', encoding='utf-8') as xml_file: + data_dict = xmltodict.parse(xml_file.read()) + + # Extract the "Tracks" dictionary to be flattened + tracks_dict = data_dict['plist']['dict']['dict'] + + flat_tracks_dict = flatten_dict(tracks_dict["dict"]) + + json_data = json.dumps(flat_tracks_dict, indent=2) + + with open(out_path, 'w', encoding='utf-8') as json_file: + json_file.write(json_data) + + +if __name__ == "__main__": + convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json') diff --git a/env b/env new file mode 100755 index 0000000..8955572 --- /dev/null +++ b/env @@ -0,0 +1,36 @@ + +PS1=" > " + +#export PATH="/usr/local/bin/:$PATH" +export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH" +#export PATH="$HOME/home/auser/.local/bin:$PATH" + +export SIP="185.238.170.251" + +alias v=nvim +alias ll="ls -l -a" +alias gl="git log --oneline" +alias ssh_server="ssh auser@185.238.170.251" + +#eval "$(zoxide init bash --cmd zd)" + +#source $PMAIN/.scripts/bashmarks.sh + +fe() { + local result=$(command tere "$@") + [ -n "$result" ] && cd -- "$result" +} + +vim_configure() { + pwd=$(pwd) + cd ~/.config/nvim/ + nvim + cd $pwd +} + +pyenv() { + source ~/bin/python/env311/bin/activate +} + +source ~/src/scripts/remotes.sh +source /usr/share/fzf/key-bindings.bash diff --git a/main.py b/main.py index 6e04234..204c0ab 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ from DataBase import * -from SpotifyWebAPI import fetch_data +from SpotifyWebAPI import fetch_data, update_access_token import cmd playlists = get_data('playlistTracks') @@ -167,6 +167,7 @@ class Interpreter(cmd.Cmd): def do_fetch(self, arg): """Fetch all spotify data""" try: + update_access_token() fetch_data() global playlists, top_artists, top_tracks, user_tracks From d8929d30647106771f575d2e3fab60715de7e76e Mon Sep 17 00:00:00 2001 From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com> Date: Mon, 16 Dec 2024 19:26:51 +0300 Subject: [PATCH 18/20] tmp --- Linker.py | 2 - LocalLibrary.py => LocalLibraryIndexer.py | 0 ParseItunesToJson.py | 101 ++++++++++++++++++++++ env | 36 ++++++++ main.py | 3 +- 5 files changed, 139 insertions(+), 3 deletions(-) delete mode 100644 Linker.py rename LocalLibrary.py => LocalLibraryIndexer.py (100%) create mode 100755 ParseItunesToJson.py create mode 100755 env diff --git a/Linker.py b/Linker.py deleted file mode 100644 index 139597f..0000000 --- a/Linker.py +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/LocalLibrary.py b/LocalLibraryIndexer.py similarity index 100% rename from LocalLibrary.py rename to LocalLibraryIndexer.py diff --git a/ParseItunesToJson.py b/ParseItunesToJson.py new file mode 100755 index 0000000..1505a36 --- /dev/null +++ b/ParseItunesToJson.py @@ -0,0 +1,101 @@ +import xmltodict +import json + + +def flatten_dict(d): + + out = [] + for song in d: + newSong = {} + + strCount = 0 + intCount = 0 + dateCount = 0 + + def getStr(id): + nonlocal strCount + if id not in song["key"]: + return "undef" + if len(song["string"]) <= strCount: + return "error" + strCount += 1 + return song["string"][strCount - 1] + + def getInt(id): + nonlocal intCount + if id not in song["key"]: + return -1 + if len(song["integer"]) <= intCount: + return -1 + intCount += 1 + return song["integer"][intCount - 1] + + def getDate(id): + nonlocal dateCount + if id not in song["key"]: + return "undef" + if len(song["date"]) <= dateCount: + return "error" + dateCount += 1 + return song["date"][dateCount - 1] + + newSong["Track ID"] = getInt("Track ID") + newSong["Name"] = getStr("Name") + newSong["Artist"] = getStr("Artist") + newSong["Album Artist"] = getStr("Album Artist") + newSong["Composer"] = getStr("Composer") + newSong["Album"] = getStr("Album") + newSong["Genre"] = getStr("Genre") + newSong["Kind"] = getStr("Kind") + newSong["Size"] = getInt("Size") + newSong["Total Time"] = getInt("Total Time") + newSong["Disc Number"] = getInt("Disc Number") + newSong["Disc Count"] = getInt("Disc Count") + newSong["Track Number"] = getInt("Track Number") + newSong["Track Count"] = getInt("Track Count") + newSong["Year"] = getInt("Year") + newSong["Date Modified"] = getDate("Date Modified") + newSong["Date Added"] = getDate("Date Added") + newSong["Bit Rate"] = getInt("Bit Rate") + newSong["Sample Rate"] = getInt("Sample Rate") + newSong["Play Count"] = getInt("Play Count") + newSong["Play Date"] = getInt("Play Date") + newSong["Play Date UTC"] = getDate("Play Date UTC") + newSong["Skip Count"] = getInt("Skip Count") + newSong["Skip Date"] = getDate("Skip Date") + newSong["Release Date"] = getDate("Release Date") + newSong["Album Rating"] = getInt("Album Rating") + newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"] + newSong["Loved"] = "Loved" in song["key"] + newSong["Album Loved"] = "Album Loved" in song["key"] + newSong["Explicit"] = "Explicit" in song["key"] + newSong["Compilation"] = "Compilation" in song["key"] + newSong["Artwork Count"] = getInt("Artwork Count") + newSong["Sort Album"] = getStr("Sort Album") + newSong["Sort Artist"] = getStr("Sort Artist") + newSong["Sort Name"] = getStr("Sort Name") + newSong["Persistent ID"] = getStr("Persistent ID") + newSong["Track Type"] = getStr("Track Type") + + out.append(newSong) + + return out + + +def convert(filename, out_path): + with open(filename, 'r', encoding='utf-8') as xml_file: + data_dict = xmltodict.parse(xml_file.read()) + + # Extract the "Tracks" dictionary to be flattened + tracks_dict = data_dict['plist']['dict']['dict'] + + flat_tracks_dict = flatten_dict(tracks_dict["dict"]) + + json_data = json.dumps(flat_tracks_dict, indent=2) + + with open(out_path, 'w', encoding='utf-8') as json_file: + json_file.write(json_data) + + +if __name__ == "__main__": + convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json') diff --git a/env b/env new file mode 100755 index 0000000..8955572 --- /dev/null +++ b/env @@ -0,0 +1,36 @@ + +PS1=" > " + +#export PATH="/usr/local/bin/:$PATH" +export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH" +#export PATH="$HOME/home/auser/.local/bin:$PATH" + +export SIP="185.238.170.251" + +alias v=nvim +alias ll="ls -l -a" +alias gl="git log --oneline" +alias ssh_server="ssh auser@185.238.170.251" + +#eval "$(zoxide init bash --cmd zd)" + +#source $PMAIN/.scripts/bashmarks.sh + +fe() { + local result=$(command tere "$@") + [ -n "$result" ] && cd -- "$result" +} + +vim_configure() { + pwd=$(pwd) + cd ~/.config/nvim/ + nvim + cd $pwd +} + +pyenv() { + source ~/bin/python/env311/bin/activate +} + +source ~/src/scripts/remotes.sh +source /usr/share/fzf/key-bindings.bash diff --git a/main.py b/main.py index 6e04234..204c0ab 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ from DataBase import * -from SpotifyWebAPI import fetch_data +from SpotifyWebAPI import fetch_data, update_access_token import cmd playlists = get_data('playlistTracks') @@ -167,6 +167,7 @@ class Interpreter(cmd.Cmd): def do_fetch(self, arg): """Fetch all spotify data""" try: + update_access_token() fetch_data() global playlists, top_artists, top_tracks, user_tracks From 589271ae209e7b844f9341860bfae321696aa08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sat, 3 Jan 2026 19:54:44 +0300 Subject: [PATCH 19/20] tmp --- DataBase.py | 7 ++++--- LinkerPatternGenerator.py | 4 ++-- LocalLibraryIndexer.py | 23 +++++++++++++++-------- SpotifyAuthenticator.py | 4 ++-- main.py | 9 ++++++++- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/DataBase.py b/DataBase.py index 3132b7b..c0f230b 100644 --- a/DataBase.py +++ b/DataBase.py @@ -18,10 +18,11 @@ def get_data(name): data = json.load(file) return data except FileNotFoundError: - return f"File {name}.json not found." + print(f"File {name}.json not found.") + return {} except json.JSONDecodeError: - return f"Error decoding JSON from {name}.json." - + print(f"Error decoding JSON from {name}.json.") + return {} def save_data(data, name): def remove_available_markets(obj): diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index 11fdf4e..2f6eba2 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -1,7 +1,7 @@ from DataBase import * from SpotifyWebAPI import find_song, update_access_token -from fuzzywuzzy import fuzz +from rapidfuzz import fuzz from tqdm import tqdm @@ -99,7 +99,7 @@ def fuzzy_tag_pattern_generator(): album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100 return title_ratio, artist_ratio, album_ratio - with tqdm(total=len(local_library), desc='Searching local songs on spotify') as pbar: + with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar: for local_track_id, local_track in local_library.items(): mapping = mappings.get(local_track_id, {"score": 1.0, "items": []}) ratios = [] diff --git a/LocalLibraryIndexer.py b/LocalLibraryIndexer.py index 2cc220b..eecdd58 100644 --- a/LocalLibraryIndexer.py +++ b/LocalLibraryIndexer.py @@ -46,13 +46,19 @@ def update_local_library(root_dir): abs_path = os.path.join(root_dir, track['path']) match track['type']: case ".mp3": - tag = eyed3.load(abs_path).tag - if not tag: - print(f"Invalid mp3 header {abs_path}") + mp3track = eyed3.load(abs_path) + if not mp3track: + print(f"Failed to load mp3 {abs_path} using just name of the file as track name") + track['name'] = track['path'] else: - track['artist'] = tag.artist - track['name'] = tag.title - track['album'] = tag.album + tag = mp3track.tag + if not tag: + print(f"Invalid mp3 header {abs_path} using just name of the file as track name") + track['name'] = track['path'] + else: + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album case ".flac": try: @@ -64,10 +70,11 @@ def update_local_library(root_dir): if 'album' in metadata: track['album'] = " ".join(metadata['album']) except mutagen.MutagenError: - print(f"Invalid flac header {abs_path}") + print(f"Invalid flac header {abs_path} using just name of the file as track name") + track['name'] = track['path'] save_data(new_library, "local_library") if __name__ == "__main__": - update_local_library("/mnt/main/data/music/raw") + update_local_library("/home/auser/Music/") diff --git a/SpotifyAuthenticator.py b/SpotifyAuthenticator.py index 529c702..e06d343 100644 --- a/SpotifyAuthenticator.py +++ b/SpotifyAuthenticator.py @@ -34,7 +34,7 @@ def retrieve_web_api_token(authorization_code): "client_secret": client_secret, "client_id": client_id, "code": authorization_code, - 'redirect_uri': 'http://localhost:8888', + 'redirect_uri': 'http://127.0.0.1:8888', } print("\nAuthenticating WEB API\n") @@ -95,7 +95,7 @@ def open_authenticate_page(): params = { 'client_id': client_id, 'response_type': 'code', - 'redirect_uri': 'http://localhost:8888', + 'redirect_uri': 'http://127.0.0.1:8888', "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", } diff --git a/main.py b/main.py index 204c0ab..9709e93 100644 --- a/main.py +++ b/main.py @@ -129,6 +129,10 @@ class Interpreter(cmd.Cmd): print(f"2 - fuzzy") def do_link_pattern(self, arg): + if not len(arg.split()): + print("expected name of the pattern") + return + """prints available linking patterns""" pattern_id = int(arg.split()[0]) print_link_stats(link_patterns[pattern_id]) @@ -187,4 +191,7 @@ class Interpreter(cmd.Cmd): if __name__ == '__main__': - Interpreter().cmdloop() + try: + Interpreter().cmdloop() + except KeyboardInterrupt as kb: + print("process terminated") \ No newline at end of file From a4524d73842e83bcc925baecb7fd8f14881e08c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sat, 3 Jan 2026 19:54:44 +0300 Subject: [PATCH 20/20] tmp --- DataBase.py | 7 ++++--- LinkerPatternGenerator.py | 4 ++-- LocalLibraryIndexer.py | 23 +++++++++++++++-------- SpotifyAuthenticator.py | 4 ++-- main.py | 9 ++++++++- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/DataBase.py b/DataBase.py index 3132b7b..c0f230b 100644 --- a/DataBase.py +++ b/DataBase.py @@ -18,10 +18,11 @@ def get_data(name): data = json.load(file) return data except FileNotFoundError: - return f"File {name}.json not found." + print(f"File {name}.json not found.") + return {} except json.JSONDecodeError: - return f"Error decoding JSON from {name}.json." - + print(f"Error decoding JSON from {name}.json.") + return {} def save_data(data, name): def remove_available_markets(obj): diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py index 11fdf4e..2f6eba2 100644 --- a/LinkerPatternGenerator.py +++ b/LinkerPatternGenerator.py @@ -1,7 +1,7 @@ from DataBase import * from SpotifyWebAPI import find_song, update_access_token -from fuzzywuzzy import fuzz +from rapidfuzz import fuzz from tqdm import tqdm @@ -99,7 +99,7 @@ def fuzzy_tag_pattern_generator(): album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100 return title_ratio, artist_ratio, album_ratio - with tqdm(total=len(local_library), desc='Searching local songs on spotify') as pbar: + with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar: for local_track_id, local_track in local_library.items(): mapping = mappings.get(local_track_id, {"score": 1.0, "items": []}) ratios = [] diff --git a/LocalLibraryIndexer.py b/LocalLibraryIndexer.py index 2cc220b..eecdd58 100644 --- a/LocalLibraryIndexer.py +++ b/LocalLibraryIndexer.py @@ -46,13 +46,19 @@ def update_local_library(root_dir): abs_path = os.path.join(root_dir, track['path']) match track['type']: case ".mp3": - tag = eyed3.load(abs_path).tag - if not tag: - print(f"Invalid mp3 header {abs_path}") + mp3track = eyed3.load(abs_path) + if not mp3track: + print(f"Failed to load mp3 {abs_path} using just name of the file as track name") + track['name'] = track['path'] else: - track['artist'] = tag.artist - track['name'] = tag.title - track['album'] = tag.album + tag = mp3track.tag + if not tag: + print(f"Invalid mp3 header {abs_path} using just name of the file as track name") + track['name'] = track['path'] + else: + track['artist'] = tag.artist + track['name'] = tag.title + track['album'] = tag.album case ".flac": try: @@ -64,10 +70,11 @@ def update_local_library(root_dir): if 'album' in metadata: track['album'] = " ".join(metadata['album']) except mutagen.MutagenError: - print(f"Invalid flac header {abs_path}") + print(f"Invalid flac header {abs_path} using just name of the file as track name") + track['name'] = track['path'] save_data(new_library, "local_library") if __name__ == "__main__": - update_local_library("/mnt/main/data/music/raw") + update_local_library("/home/auser/Music/") diff --git a/SpotifyAuthenticator.py b/SpotifyAuthenticator.py index 64a4b2c..86705d0 100644 --- a/SpotifyAuthenticator.py +++ b/SpotifyAuthenticator.py @@ -34,7 +34,7 @@ def retrieve_web_api_token(authorization_code): "client_secret": client_secret, "client_id": client_id, "code": authorization_code, - 'redirect_uri': 'http://localhost:8888', + 'redirect_uri': 'http://127.0.0.1:8888', } print("\nAuthenticating WEB API\n") @@ -95,7 +95,7 @@ def open_authenticate_page(): params = { 'client_id': client_id, 'response_type': 'code', - 'redirect_uri': 'http://localhost:8888', + 'redirect_uri': 'http://127.0.0.1:8888', "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", } diff --git a/main.py b/main.py index 204c0ab..9709e93 100644 --- a/main.py +++ b/main.py @@ -129,6 +129,10 @@ class Interpreter(cmd.Cmd): print(f"2 - fuzzy") def do_link_pattern(self, arg): + if not len(arg.split()): + print("expected name of the pattern") + return + """prints available linking patterns""" pattern_id = int(arg.split()[0]) print_link_stats(link_patterns[pattern_id]) @@ -187,4 +191,7 @@ class Interpreter(cmd.Cmd): if __name__ == '__main__': - Interpreter().cmdloop() + try: + Interpreter().cmdloop() + except KeyboardInterrupt as kb: + print("process terminated") \ No newline at end of file