From 506258f9b12ad3ddccdeb637ebdc3b84d111133c Mon Sep 17 00:00:00 2001
From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com>
Date: Mon, 8 Jul 2024 23:14:12 +0300
Subject: [PATCH 01/30] Initial commit
---
README.md | 2 ++
1 file changed, 2 insertions(+)
create mode 100644 README.md
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..508172a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+# spotifyFetcher
+loads all data through web api
From 04da522b64e32a10a32243f807a797bafe25fbd1 Mon Sep 17 00:00:00 2001
From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com>
Date: Mon, 8 Jul 2024 23:14:12 +0300
Subject: [PATCH 02/30] Initial commit
---
README.md | 2 ++
1 file changed, 2 insertions(+)
create mode 100644 README.md
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..508172a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+# spotifyFetcher
+loads all data through web api
From 4734befe984c168abf7b8679b8a542bfa7e6fdc5 Mon Sep 17 00:00:00 2001
From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com>
Date: Mon, 8 Jul 2024 23:14:29 +0300
Subject: [PATCH 03/30] stable
need run main fetch command
then run indexer
then LinkerPatternGenerator
then link_pattern command in the main
---
.gitignore | 4 +
DataBase.py | 49 ++++++++++
LinkerPatternGenerator.py | 137 ++++++++++++++++++++++++++
LocalLibraryIndexer.py | 80 ++++++++++++++++
ParseItunesToJson.py | 101 +++++++++++++++++++
README.md | 3 +-
SpotifyAuthenticator.py | 122 +++++++++++++++++++++++
SpotifyWebAPI.py | 190 ++++++++++++++++++++++++++++++++++++
env | 36 +++++++
main.py | 197 ++++++++++++++++++++++++++++++++++++++
10 files changed, 918 insertions(+), 1 deletion(-)
create mode 100644 .gitignore
create mode 100644 DataBase.py
create mode 100644 LinkerPatternGenerator.py
create mode 100644 LocalLibraryIndexer.py
create mode 100755 ParseItunesToJson.py
create mode 100644 SpotifyAuthenticator.py
create mode 100644 SpotifyWebAPI.py
create mode 100755 env
create mode 100644 main.py
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2803b17
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+prefetched*
+__pycache__
+.idea
+secret
diff --git a/DataBase.py b/DataBase.py
new file mode 100644
index 0000000..c0f230b
--- /dev/null
+++ b/DataBase.py
@@ -0,0 +1,49 @@
+import json
+import os
+
+data_dir = 'prefetched'
+
+
+class SongTags:
+ def __init__(self, title='', artist='', album=''):
+ self.artist = artist
+ self.title = title
+ self.album = album
+
+
+def get_data(name):
+ file_path = os.path.join(data_dir, f"{name}.json")
+ try:
+ with open(file_path, 'r') as file:
+ data = json.load(file)
+ return data
+ except FileNotFoundError:
+ print(f"File {name}.json not found.")
+ return {}
+ except json.JSONDecodeError:
+ print(f"Error decoding JSON from {name}.json.")
+ return {}
+
+def save_data(data, name):
+ def remove_available_markets(obj):
+ if isinstance(obj, dict):
+ return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'}
+ elif isinstance(obj, list):
+ return [remove_available_markets(item) for item in obj]
+ else:
+ return obj
+
+ data = remove_available_markets(data)
+
+ os.makedirs(data_dir, exist_ok=True)
+
+ file_path = os.path.join(data_dir, f"{name}.json")
+ with open(file_path, 'w') as file:
+ json.dump(data, file, indent=4)
+ return f"Data saved to {name}.json."
+
+
+def get_artists_str(artists_pack):
+ track_artists = [artist['name'] for artist in artists_pack]
+ artists = " ".join(track_artists)
+ return artists
diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py
new file mode 100644
index 0000000..2f6eba2
--- /dev/null
+++ b/LinkerPatternGenerator.py
@@ -0,0 +1,137 @@
+from DataBase import *
+from SpotifyWebAPI import find_song, update_access_token
+
+from rapidfuzz import fuzz
+from tqdm import tqdm
+
+
+def spotify_pattern_generator():
+ update_access_token()
+
+ local_library = get_data("local_library")
+ spotify_library = get_data("tracks")
+
+ def find_local_songs_on_spotify(local_lib):
+ total = len(local_library)
+ found_tracks_map = {}
+ with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
+ for track_id, track in local_lib.items():
+ search_pattern = f"{track['name']} {track['artist']} {track['album']}"
+ found_tracks = find_song(search_pattern)
+ found_tracks_map[track_id] = found_tracks[0:1]
+ pbar.update(1)
+ return found_tracks_map
+
+ spotify_found_tracks = find_local_songs_on_spotify(local_library)
+ spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library}
+
+ spotify_pattern = {}
+ for local_id, found_items in spotify_found_tracks.items():
+ spotify_pattern[local_id] = {"score": 0.0, "items": []}
+
+ if not len(found_items):
+ continue
+
+ for found_item in found_items:
+ spotify_track_id = found_item['id']
+ if spotify_track_id in spotify_user_track:
+ spotify_pattern[local_id]['items'].append(spotify_track_id)
+ spotify_user_track[spotify_track_id]['local_mappings'].append(local_id)
+
+ for spotify_id, track_local_mappings in spotify_user_track.items():
+ score = len(track_local_mappings['local_mappings'])
+ for local_id in track_local_mappings['local_mappings']:
+ spotify_pattern[local_id]['score'] = 1 / score
+
+ save_data(spotify_pattern, "link_pattern_spotify")
+
+
+def fuzzy_pattern_generator():
+ local_library = get_data("local_library")
+ track_descriptions = get_data("tracks")
+
+ def get_matched(src_pattern, patterns):
+ result = {}
+ for target_pattern in patterns:
+ similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern)
+ result[target_pattern] = similarity_score / 100
+
+ sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True)
+ return sorted_results
+
+ def resolve_tracks(s_patterns, t_patterns):
+
+ pattern_map = {}
+
+ total = len(s_patterns)
+ with tqdm(total=total, desc='Progress') as pbar:
+ for s_pattern in s_patterns:
+ res = get_matched(s_pattern, t_patterns)
+ matched = {"score": 0, "items": []}
+ if len(res):
+ matched['items'] = res[0:min(len(res), 4)]
+ matched['score'] = res[0][1]
+ pattern_map[s_pattern] = matched
+ pbar.update(1)
+
+ return pattern_map
+
+ source_patterns = []
+ for track in track_descriptions:
+ artists = get_artists_str(track['track']['artists'])
+ pattern = f"{track['track']['name']} {artists}"
+ source_patterns.append(pattern)
+
+ target_patterns = [local['path'] for _, local in local_library.items()]
+
+ fuzzy_pattern = resolve_tracks(source_patterns, target_patterns)
+ save_data(fuzzy_pattern, "link_pattern_fuzzy")
+
+
+def fuzzy_tag_pattern_generator():
+ local_library = get_data("local_library")
+ spotify_library = get_data("tracks")
+ mappings = {}
+
+ def fuzzy_tags_ratio(first: SongTags, second: SongTags):
+ title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100
+ artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100
+ album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100
+ return title_ratio, artist_ratio, album_ratio
+
+ with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar:
+ for local_track_id, local_track in local_library.items():
+ mapping = mappings.get(local_track_id, {"score": 1.0, "items": []})
+ ratios = []
+ for remote_track in spotify_library:
+ track = remote_track['track']
+
+ tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album'])
+ tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name'])
+
+ ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id']))
+
+ filtered_ratios = []
+ threshold = 0.8
+ for ratio in ratios:
+ if all([val > threshold for val in ratio[0]]):
+ filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1]))
+
+ filtered_ratios.sort(reverse=True, key=lambda x: x[0])
+ if len(filtered_ratios):
+ mapping['items'] = [filtered_ratios[0][1]]
+
+ mappings[local_track_id] = mapping
+ pbar.update(1)
+
+ save_data(mappings, "link_pattern_tags")
+
+
+def run_pattern_generators():
+ #fuzzy_pattern_generator()
+ #spotify_pattern_generator()
+ fuzzy_tag_pattern_generator()
+
+
+if __name__ == "__main__":
+ run_pattern_generators()
diff --git a/LocalLibraryIndexer.py b/LocalLibraryIndexer.py
new file mode 100644
index 0000000..eecdd58
--- /dev/null
+++ b/LocalLibraryIndexer.py
@@ -0,0 +1,80 @@
+import os
+import fnmatch
+
+import mutagen
+
+from DataBase import *
+import eyed3
+from mutagen.flac import FLAC
+
+
+music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
+exclude_directories = ("*mary--*", "*example-word*")
+
+
+def update_local_library(root_dir):
+ old_library = get_data("local_library")
+ known_paths = {track['path']: track_id for track_id, track in old_library.items()}
+ new_library = {}
+
+ song_id = '0'
+
+ def get_new_song_id(prev_id, song_path):
+ if song_path in known_paths:
+ return int(known_paths[song_path])
+ while (prev_id in old_library) or (prev_id in new_library):
+ prev_id = str(int(prev_id) + 1)
+ return prev_id
+
+ for dir_path, dir_names, filenames in os.walk(root_dir):
+ dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)]
+ filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)]
+
+ for filename in filtered_filenames:
+ song_name, track_type = os.path.splitext(filename)
+ relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir)
+ song_id = get_new_song_id(song_id, relative_path)
+ new_library[song_id] = {
+ "name": song_name,
+ "path": relative_path,
+ "type": track_type,
+ "artist": "",
+ "album": ""
+ }
+
+ for track_id, track in new_library.items():
+ abs_path = os.path.join(root_dir, track['path'])
+ match track['type']:
+ case ".mp3":
+ mp3track = eyed3.load(abs_path)
+ if not mp3track:
+ print(f"Failed to load mp3 {abs_path} using just name of the file as track name")
+ track['name'] = track['path']
+ else:
+ tag = mp3track.tag
+ if not tag:
+ print(f"Invalid mp3 header {abs_path} using just name of the file as track name")
+ track['name'] = track['path']
+ else:
+ track['artist'] = tag.artist
+ track['name'] = tag.title
+ track['album'] = tag.album
+
+ case ".flac":
+ try:
+ metadata = FLAC(abs_path).tags
+ if 'artist' in metadata:
+ track['artist'] = " ".join(metadata['artist'])
+ if 'TITLE' in metadata:
+ track['name'] = " ".join(metadata['TITLE'])
+ if 'album' in metadata:
+ track['album'] = " ".join(metadata['album'])
+ except mutagen.MutagenError:
+ print(f"Invalid flac header {abs_path} using just name of the file as track name")
+ track['name'] = track['path']
+
+ save_data(new_library, "local_library")
+
+
+if __name__ == "__main__":
+ update_local_library("/home/auser/Music/")
diff --git a/ParseItunesToJson.py b/ParseItunesToJson.py
new file mode 100755
index 0000000..1505a36
--- /dev/null
+++ b/ParseItunesToJson.py
@@ -0,0 +1,101 @@
+import xmltodict
+import json
+
+
+def flatten_dict(d):
+
+ out = []
+ for song in d:
+ newSong = {}
+
+ strCount = 0
+ intCount = 0
+ dateCount = 0
+
+ def getStr(id):
+ nonlocal strCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["string"]) <= strCount:
+ return "error"
+ strCount += 1
+ return song["string"][strCount - 1]
+
+ def getInt(id):
+ nonlocal intCount
+ if id not in song["key"]:
+ return -1
+ if len(song["integer"]) <= intCount:
+ return -1
+ intCount += 1
+ return song["integer"][intCount - 1]
+
+ def getDate(id):
+ nonlocal dateCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["date"]) <= dateCount:
+ return "error"
+ dateCount += 1
+ return song["date"][dateCount - 1]
+
+ newSong["Track ID"] = getInt("Track ID")
+ newSong["Name"] = getStr("Name")
+ newSong["Artist"] = getStr("Artist")
+ newSong["Album Artist"] = getStr("Album Artist")
+ newSong["Composer"] = getStr("Composer")
+ newSong["Album"] = getStr("Album")
+ newSong["Genre"] = getStr("Genre")
+ newSong["Kind"] = getStr("Kind")
+ newSong["Size"] = getInt("Size")
+ newSong["Total Time"] = getInt("Total Time")
+ newSong["Disc Number"] = getInt("Disc Number")
+ newSong["Disc Count"] = getInt("Disc Count")
+ newSong["Track Number"] = getInt("Track Number")
+ newSong["Track Count"] = getInt("Track Count")
+ newSong["Year"] = getInt("Year")
+ newSong["Date Modified"] = getDate("Date Modified")
+ newSong["Date Added"] = getDate("Date Added")
+ newSong["Bit Rate"] = getInt("Bit Rate")
+ newSong["Sample Rate"] = getInt("Sample Rate")
+ newSong["Play Count"] = getInt("Play Count")
+ newSong["Play Date"] = getInt("Play Date")
+ newSong["Play Date UTC"] = getDate("Play Date UTC")
+ newSong["Skip Count"] = getInt("Skip Count")
+ newSong["Skip Date"] = getDate("Skip Date")
+ newSong["Release Date"] = getDate("Release Date")
+ newSong["Album Rating"] = getInt("Album Rating")
+ newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"]
+ newSong["Loved"] = "Loved" in song["key"]
+ newSong["Album Loved"] = "Album Loved" in song["key"]
+ newSong["Explicit"] = "Explicit" in song["key"]
+ newSong["Compilation"] = "Compilation" in song["key"]
+ newSong["Artwork Count"] = getInt("Artwork Count")
+ newSong["Sort Album"] = getStr("Sort Album")
+ newSong["Sort Artist"] = getStr("Sort Artist")
+ newSong["Sort Name"] = getStr("Sort Name")
+ newSong["Persistent ID"] = getStr("Persistent ID")
+ newSong["Track Type"] = getStr("Track Type")
+
+ out.append(newSong)
+
+ return out
+
+
+def convert(filename, out_path):
+ with open(filename, 'r', encoding='utf-8') as xml_file:
+ data_dict = xmltodict.parse(xml_file.read())
+
+ # Extract the "Tracks" dictionary to be flattened
+ tracks_dict = data_dict['plist']['dict']['dict']
+
+ flat_tracks_dict = flatten_dict(tracks_dict["dict"])
+
+ json_data = json.dumps(flat_tracks_dict, indent=2)
+
+ with open(out_path, 'w', encoding='utf-8') as json_file:
+ json_file.write(json_data)
+
+
+if __name__ == "__main__":
+ convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json')
diff --git a/README.md b/README.md
index 508172a..4a45af3 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,3 @@
# spotifyFetcher
-loads all data through web api
+loads all data through web api
+client secret - 4ced4e4575094b749834a60996680c6c
diff --git a/SpotifyAuthenticator.py b/SpotifyAuthenticator.py
new file mode 100644
index 0000000..e06d343
--- /dev/null
+++ b/SpotifyAuthenticator.py
@@ -0,0 +1,122 @@
+import base64
+
+import requests
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import urllib.parse
+import threading
+import time
+import os.path
+
+client_id = '3b4db45bb66b45e9a2532989c8034332'
+client_secret = None
+token = None
+
+
+def resolve_client_secret():
+ global client_secret
+ if os.path.exists('secret'):
+ with open('secret', 'r') as file:
+ client_secret = file.readline().strip()
+ else:
+ client_secret = input("Enter the application secret: ")
+ with open('secret', 'w') as file:
+ file.write(client_secret)
+
+
+def retrieve_web_api_token(authorization_code):
+ global token
+
+ token_url = 'https://accounts.spotify.com/api/token'
+
+ token_data = {
+ "grant_type": "authorization_code",
+ "client_secret": client_secret,
+ "client_id": client_id,
+ "code": authorization_code,
+ 'redirect_uri': 'http://127.0.0.1:8888',
+ }
+
+ print("\nAuthenticating WEB API\n")
+
+ auth_string = f'{client_id}:{client_secret}'
+ b64_auth_string = base64.b64encode(auth_string.encode()).decode()
+
+ # Define the headers
+ headers = {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ 'Authorization': f'Basic {b64_auth_string}'
+ }
+
+ token_response = requests.post(token_url, data=token_data, headers=headers)
+
+ if token_response.status_code not in range(200, 299):
+ print(f"Failed to retrieve access token: {token_response.status_code}")
+ print(token_response.json())
+ raise ValueError
+
+ token = token_response.json()['access_token']
+
+
+class RequestHandler(BaseHTTPRequestHandler):
+ def do_GET(self):
+
+ parsed_url = urllib.parse.urlparse(self.path)
+ query_params = urllib.parse.parse_qs(parsed_url.query)
+
+ if 'code' in query_params:
+ authorization_code = query_params['code'][0]
+
+ self.send_response(200)
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+ self.wfile.write(b'Authorization code received. You can close this window.')
+
+ retrieve_web_api_token(authorization_code)
+
+ threading.Thread(target=self.server.shutdown).start()
+
+ else:
+ self.send_response(400)
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+ self.wfile.write(b'Authorization code not found.')
+
+
+def run_redirect_server():
+ httpd = HTTPServer(('localhost', 8888), RequestHandler)
+ print('Starting HTTP redirection server')
+ httpd.serve_forever()
+
+
+def open_authenticate_page():
+ authorize_url = 'https://accounts.spotify.com/authorize'
+
+ params = {
+ 'client_id': client_id,
+ 'response_type': 'code',
+ 'redirect_uri': 'http://127.0.0.1:8888',
+ "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private",
+ }
+
+ auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}"
+ webbrowser.open(auth_url)
+
+
+def authenticate():
+ resolve_client_secret()
+
+ redirect_server = threading.Thread(target=run_redirect_server)
+ redirect_server.start()
+
+ time.sleep(1)
+ open_authenticate_page()
+
+ redirect_server.join()
+
+ return token
+
+
+if __name__ == "__main__":
+ authenticate()
+
diff --git a/SpotifyWebAPI.py b/SpotifyWebAPI.py
new file mode 100644
index 0000000..71ec2f7
--- /dev/null
+++ b/SpotifyWebAPI.py
@@ -0,0 +1,190 @@
+import requests
+from PIL import Image
+from io import BytesIO
+from SpotifyAuthenticator import authenticate
+from DataBase import *
+
+FETCH_STEP = 50
+token = ''
+
+
+def fetch(endpoint, method='GET', body=None):
+ url = f'https://api.spotify.com/{endpoint}'
+ headers = {
+ 'Authorization': f'Bearer {token}',
+ }
+
+ response = requests.request(method, url, headers=headers, json=body)
+ response.raise_for_status()
+ return response.json()
+
+
+def find_song(song_pattern):
+ return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items']
+
+
+def fetch_playlists(user_id):
+ print("Fetching playlists")
+ endpoint = f"v1/me/playlists"
+ pl_total = fetch(f'{endpoint}?limit=1')['total']
+ pl_count = 0
+ playlists = []
+
+ while pl_count < pl_total:
+ res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}')
+ playlists += res['items']
+ pl_count += FETCH_STEP
+
+ return playlists
+
+
+def fetch_playlist_items(playlist_id):
+ endpoint = f"v1/playlists/{playlist_id}/tracks"
+ items = fetch(f'{endpoint}?fields=total')['total']
+ loaded = 0
+ tracks = []
+
+ while loaded < items:
+ res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}')
+
+ tracks += res['items']
+ loaded += FETCH_STEP
+
+ return tracks
+
+
+def fetch_tracks(user_id):
+ print("Fetching tracks")
+ endpoint = f"v1/me/tracks"
+ total = fetch(f"{endpoint}?limit=1&market=ES")['total']
+ current = 0
+ tracks = []
+
+ while total > current:
+ res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES")
+ tracks += res['items']
+ current += FETCH_STEP
+
+ save_data(tracks, "tracks")
+ return tracks
+
+
+def fetch_tracks_top():
+ print("Fetching top tracks")
+ total = fetch(f'v1/me/top/tracks?fields=total')['total']
+ current = 0
+ tracks = []
+
+ while current < total:
+ res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term')
+ tracks += res['items']
+ current += FETCH_STEP
+
+ save_data(tracks, "top_tracks")
+ return tracks
+
+
+def fetch_artists_top():
+ print("Fetching top artists")
+
+ total = fetch(f'v1/me/top/artists?fields=total')['total']
+ current = 0
+ artists = []
+
+ while current < total:
+ res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term')
+ artists += res['items']
+ current += FETCH_STEP
+
+ save_data(artists, "top_artists")
+ return artists
+
+
+def fetch_all_playlist_data(user_id):
+ playlists = fetch_playlists(user_id)
+
+ save_data(playlists, 'playlists')
+
+ print(len(playlists))
+
+ playlist_items = {}
+
+ print("Fetching playlists items")
+
+ index = 0
+ for pl in playlists:
+ pl_id = pl['id']
+ name = pl['name']
+ playlist_items[name] = fetch_playlist_items(pl_id)
+ print(f"{index} - {len(playlist_items[name])}")
+ index += 1
+
+ save_data(playlist_items, "playlistTracks")
+
+
+def get_user_data():
+ print("Fetching user data")
+
+ user_profile = fetch(f'v1/me')
+ user_id = user_profile['id']
+ user_data = fetch(f'v1/users/{user_id}')
+
+ save_data(user_profile, "user_profile")
+ save_data(user_data, "user_data")
+ return user_id
+
+
+def save_image_from_url(url, name, image_dir="playlist_covers"):
+ response = requests.get(url)
+
+ if response.status_code != 200:
+ raise "Cannot fetch the playlist cover"
+
+ image = Image.open(BytesIO(response.content))
+
+ directory = os.path.join(data_dir, image_dir)
+ os.makedirs(directory, exist_ok=True)
+ file_path = os.path.join(directory, f"{name}.jpg")
+ print(f"Image saved {file_path}")
+ image.save(file_path)
+
+
+def load_playlist_covers():
+ playlists = get_data('playlists')
+
+ for pl in playlists:
+ if len(pl['images']):
+ url = pl['images'][0]['url']
+ save_image_from_url(url, pl['name'])
+
+
+def load_user_cover():
+ user_data = get_data("user_data")
+ url = user_data['images'][1]['url']
+ save_image_from_url(url, "user", ".")
+
+
+def load_artworks():
+ load_user_cover()
+ load_playlist_covers()
+
+
+def update_access_token():
+ global token
+ token = authenticate()
+
+
+def fetch_data():
+ user_id = get_user_data()
+
+ fetch_tracks(user_id)
+ fetch_all_playlist_data(user_id)
+ fetch_tracks_top()
+ fetch_artists_top()
+
+ load_artworks()
+
+
+if __name__ == "__main__":
+ update_access_token()
+ fetch_data()
diff --git a/env b/env
new file mode 100755
index 0000000..8955572
--- /dev/null
+++ b/env
@@ -0,0 +1,36 @@
+
+PS1=" > "
+
+#export PATH="/usr/local/bin/:$PATH"
+export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH"
+#export PATH="$HOME/home/auser/.local/bin:$PATH"
+
+export SIP="185.238.170.251"
+
+alias v=nvim
+alias ll="ls -l -a"
+alias gl="git log --oneline"
+alias ssh_server="ssh auser@185.238.170.251"
+
+#eval "$(zoxide init bash --cmd zd)"
+
+#source $PMAIN/.scripts/bashmarks.sh
+
+fe() {
+ local result=$(command tere "$@")
+ [ -n "$result" ] && cd -- "$result"
+}
+
+vim_configure() {
+ pwd=$(pwd)
+ cd ~/.config/nvim/
+ nvim
+ cd $pwd
+}
+
+pyenv() {
+ source ~/bin/python/env311/bin/activate
+}
+
+source ~/src/scripts/remotes.sh
+source /usr/share/fzf/key-bindings.bash
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..9709e93
--- /dev/null
+++ b/main.py
@@ -0,0 +1,197 @@
+from DataBase import *
+from SpotifyWebAPI import fetch_data, update_access_token
+import cmd
+
+playlists = get_data('playlistTracks')
+top_artists = get_data('top_artists')
+top_tracks = get_data('top_tracks')
+user_tracks = get_data('tracks')
+link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')]
+local_library = get_data("local_library")
+user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
+
+
+def get_playlist_names(pl):
+ return [name for name, items in pl.items()]
+
+
+def print_playlist_tracks(pl_id: int):
+ pls = list(playlists)
+ name = pls[pl_id]
+ pl = playlists[name]
+ print(name)
+ for track in pl:
+ track_data = track['track']
+ track_name = track_data['name']
+ artists = get_artists_str(track_data['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_top_artists():
+ for artist in top_artists:
+ name = artist['name']
+ print(f" '{name}'")
+
+
+def print_top_tracks():
+ for track in top_tracks:
+ track_name = track['name']
+ artists = get_artists_str(track['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_stats():
+ print(f" tracks - {len(user_tracks)}")
+ print(f" playlists - {len(playlists)}")
+
+ for name, tracks in playlists.items():
+ print(f" '{name}' - {len(tracks)}")
+
+ print(f" top tracks - {len(top_tracks)}")
+ print(f" top artists - {len(top_artists)}")
+
+
+def print_tracks():
+ track_idx = 0
+ for trackItem in user_tracks:
+ track = trackItem['track']
+ print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
+ track_idx += 1
+
+
+def sort_remote_tracks_by_popularity(track_ids):
+ out = []
+ for top_track in top_tracks:
+ if top_track['id'] in track_ids:
+ out.append(top_track['id'])
+ return out
+
+
+def print_link_stats(link_pattern):
+ resolved_reversed = {}
+ resolved = []
+ unresolved_local = []
+ unresolved_remote = []
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ resolved.append(local_id)
+ resolved_reversed[links['items'][0]] = local_id
+ else:
+ unresolved_local.append(local_id)
+
+ for track in user_tracks:
+ track_id = track['track']['id']
+ if track_id not in resolved_reversed:
+ unresolved_remote.append(track_id)
+
+ print(f"Local Tracks: {len(local_library)}")
+ print(f"Remote Tracks: {len(user_tracks)}")
+
+ unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
+ unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+
+ print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
+ index = 0
+ for remote_id in unresolved_remote:
+ remote_track = user_tracks_by_id[remote_id]
+ print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ print(f"\nResolved tracks: {len(resolved)}")
+ index = 0
+ for link in resolved:
+ remote_id = link_pattern[link]['items'][0]
+ local_track = local_library[link]
+ remote_track = user_tracks_by_id[remote_id]
+
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
+ print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ return
+ print(f"\nUnresolved tracks from the local: {len(unresolved_local)}")
+ index = 0
+ for link in unresolved_local:
+ local_track = local_library[link]
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'")
+ index += 1
+
+
+class Interpreter(cmd.Cmd):
+ intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
+ prompt = "(spotify) "
+
+ def do_link_patterns(self, arg):
+ """prints available linking patterns"""
+ print(f"0 - fuzzy tags")
+ print(f"1 - spotify")
+ print(f"2 - fuzzy")
+
+ def do_link_pattern(self, arg):
+ if not len(arg.split()):
+ print("expected name of the pattern")
+ return
+
+ """prints available linking patterns"""
+ pattern_id = int(arg.split()[0])
+ print_link_stats(link_patterns[pattern_id])
+
+ def do_playlists(self, arg):
+ """prints user playlists"""
+ pl_id = 0
+ for pl in get_playlist_names(playlists):
+ print(f"{pl_id} - {pl}")
+ pl_id += 1
+
+ def do_playlist_tracks(self, arg):
+ """prints playlist [name]"""
+ try:
+ pl_id = int(arg.split()[0])
+ print_playlist_tracks(pl_id)
+ except IndexError:
+ print("Invalid input")
+
+ def do_top_artists(self, arg):
+ """Top artists"""
+ print_top_artists()
+
+ def do_top_tracks(self, arg):
+ """Top tracks"""
+ print_top_tracks()
+
+ def do_stat(self, arg):
+ """Print stats"""
+ print_stats()
+
+ def do_tracks(self, arg):
+ """Print tracks"""
+ print_tracks()
+
+ def do_fetch(self, arg):
+ """Fetch all spotify data"""
+ try:
+ update_access_token()
+ fetch_data()
+
+ global playlists, top_artists, top_tracks, user_tracks
+
+ playlists = get_data('playlistTracks')
+ top_artists = get_data('top_artists')
+ top_tracks = get_data('top_tracks')
+ user_tracks = get_data('tracks')
+
+ except ValueError:
+ print("Can not fetch the data.")
+
+ def do_exit(self, arg):
+ """Exit the command loop: exit"""
+ print("Goodbye!")
+ return True
+
+
+if __name__ == '__main__':
+ try:
+ Interpreter().cmdloop()
+ except KeyboardInterrupt as kb:
+ print("process terminated")
\ No newline at end of file
From 11e3b06824590d679b233e002e5881784e1838d3 Mon Sep 17 00:00:00 2001
From: Ilya Shurupov <163508118+elushaX@users.noreply.github.com>
Date: Mon, 8 Jul 2024 23:14:29 +0300
Subject: [PATCH 04/30] stable
need run main fetch command
then run indexer
then LinkerPatternGenerator
then link_pattern command in the main
---
.gitignore | 4 +
DataBase.py | 49 ++++++++++
LinkerPatternGenerator.py | 137 ++++++++++++++++++++++++++
LocalLibraryIndexer.py | 80 ++++++++++++++++
ParseItunesToJson.py | 101 +++++++++++++++++++
README.md | 3 +-
SpotifyAuthenticator.py | 122 +++++++++++++++++++++++
SpotifyWebAPI.py | 190 ++++++++++++++++++++++++++++++++++++
env | 36 +++++++
main.py | 197 ++++++++++++++++++++++++++++++++++++++
10 files changed, 918 insertions(+), 1 deletion(-)
create mode 100644 .gitignore
create mode 100644 DataBase.py
create mode 100644 LinkerPatternGenerator.py
create mode 100644 LocalLibraryIndexer.py
create mode 100755 ParseItunesToJson.py
create mode 100644 SpotifyAuthenticator.py
create mode 100644 SpotifyWebAPI.py
create mode 100755 env
create mode 100644 main.py
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2803b17
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+prefetched*
+__pycache__
+.idea
+secret
diff --git a/DataBase.py b/DataBase.py
new file mode 100644
index 0000000..c0f230b
--- /dev/null
+++ b/DataBase.py
@@ -0,0 +1,49 @@
+import json
+import os
+
+data_dir = 'prefetched'
+
+
+class SongTags:
+ def __init__(self, title='', artist='', album=''):
+ self.artist = artist
+ self.title = title
+ self.album = album
+
+
+def get_data(name):
+ file_path = os.path.join(data_dir, f"{name}.json")
+ try:
+ with open(file_path, 'r') as file:
+ data = json.load(file)
+ return data
+ except FileNotFoundError:
+ print(f"File {name}.json not found.")
+ return {}
+ except json.JSONDecodeError:
+ print(f"Error decoding JSON from {name}.json.")
+ return {}
+
+def save_data(data, name):
+ def remove_available_markets(obj):
+ if isinstance(obj, dict):
+ return {key: remove_available_markets(value) for key, value in obj.items() if key != 'available_markets'}
+ elif isinstance(obj, list):
+ return [remove_available_markets(item) for item in obj]
+ else:
+ return obj
+
+ data = remove_available_markets(data)
+
+ os.makedirs(data_dir, exist_ok=True)
+
+ file_path = os.path.join(data_dir, f"{name}.json")
+ with open(file_path, 'w') as file:
+ json.dump(data, file, indent=4)
+ return f"Data saved to {name}.json."
+
+
+def get_artists_str(artists_pack):
+ track_artists = [artist['name'] for artist in artists_pack]
+ artists = " ".join(track_artists)
+ return artists
diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py
new file mode 100644
index 0000000..2f6eba2
--- /dev/null
+++ b/LinkerPatternGenerator.py
@@ -0,0 +1,137 @@
+from DataBase import *
+from SpotifyWebAPI import find_song, update_access_token
+
+from rapidfuzz import fuzz
+from tqdm import tqdm
+
+
+def spotify_pattern_generator():
+ update_access_token()
+
+ local_library = get_data("local_library")
+ spotify_library = get_data("tracks")
+
+ def find_local_songs_on_spotify(local_lib):
+ total = len(local_library)
+ found_tracks_map = {}
+ with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
+ for track_id, track in local_lib.items():
+ search_pattern = f"{track['name']} {track['artist']} {track['album']}"
+ found_tracks = find_song(search_pattern)
+ found_tracks_map[track_id] = found_tracks[0:1]
+ pbar.update(1)
+ return found_tracks_map
+
+ spotify_found_tracks = find_local_songs_on_spotify(local_library)
+ spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library}
+
+ spotify_pattern = {}
+ for local_id, found_items in spotify_found_tracks.items():
+ spotify_pattern[local_id] = {"score": 0.0, "items": []}
+
+ if not len(found_items):
+ continue
+
+ for found_item in found_items:
+ spotify_track_id = found_item['id']
+ if spotify_track_id in spotify_user_track:
+ spotify_pattern[local_id]['items'].append(spotify_track_id)
+ spotify_user_track[spotify_track_id]['local_mappings'].append(local_id)
+
+ for spotify_id, track_local_mappings in spotify_user_track.items():
+ score = len(track_local_mappings['local_mappings'])
+ for local_id in track_local_mappings['local_mappings']:
+ spotify_pattern[local_id]['score'] = 1 / score
+
+ save_data(spotify_pattern, "link_pattern_spotify")
+
+
+def fuzzy_pattern_generator():
+ local_library = get_data("local_library")
+ track_descriptions = get_data("tracks")
+
+ def get_matched(src_pattern, patterns):
+ result = {}
+ for target_pattern in patterns:
+ similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern)
+ result[target_pattern] = similarity_score / 100
+
+ sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True)
+ return sorted_results
+
+ def resolve_tracks(s_patterns, t_patterns):
+
+ pattern_map = {}
+
+ total = len(s_patterns)
+ with tqdm(total=total, desc='Progress') as pbar:
+ for s_pattern in s_patterns:
+ res = get_matched(s_pattern, t_patterns)
+ matched = {"score": 0, "items": []}
+ if len(res):
+ matched['items'] = res[0:min(len(res), 4)]
+ matched['score'] = res[0][1]
+ pattern_map[s_pattern] = matched
+ pbar.update(1)
+
+ return pattern_map
+
+ source_patterns = []
+ for track in track_descriptions:
+ artists = get_artists_str(track['track']['artists'])
+ pattern = f"{track['track']['name']} {artists}"
+ source_patterns.append(pattern)
+
+ target_patterns = [local['path'] for _, local in local_library.items()]
+
+ fuzzy_pattern = resolve_tracks(source_patterns, target_patterns)
+ save_data(fuzzy_pattern, "link_pattern_fuzzy")
+
+
+def fuzzy_tag_pattern_generator():
+ local_library = get_data("local_library")
+ spotify_library = get_data("tracks")
+ mappings = {}
+
+ def fuzzy_tags_ratio(first: SongTags, second: SongTags):
+ title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100
+ artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100
+ album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100
+ return title_ratio, artist_ratio, album_ratio
+
+ with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar:
+ for local_track_id, local_track in local_library.items():
+ mapping = mappings.get(local_track_id, {"score": 1.0, "items": []})
+ ratios = []
+ for remote_track in spotify_library:
+ track = remote_track['track']
+
+ tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album'])
+ tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name'])
+
+ ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id']))
+
+ filtered_ratios = []
+ threshold = 0.8
+ for ratio in ratios:
+ if all([val > threshold for val in ratio[0]]):
+ filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1]))
+
+ filtered_ratios.sort(reverse=True, key=lambda x: x[0])
+ if len(filtered_ratios):
+ mapping['items'] = [filtered_ratios[0][1]]
+
+ mappings[local_track_id] = mapping
+ pbar.update(1)
+
+ save_data(mappings, "link_pattern_tags")
+
+
+def run_pattern_generators():
+ #fuzzy_pattern_generator()
+ #spotify_pattern_generator()
+ fuzzy_tag_pattern_generator()
+
+
+if __name__ == "__main__":
+ run_pattern_generators()
diff --git a/LocalLibraryIndexer.py b/LocalLibraryIndexer.py
new file mode 100644
index 0000000..eecdd58
--- /dev/null
+++ b/LocalLibraryIndexer.py
@@ -0,0 +1,80 @@
+import os
+import fnmatch
+
+import mutagen
+
+from DataBase import *
+import eyed3
+from mutagen.flac import FLAC
+
+
+music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
+exclude_directories = ("*mary--*", "*example-word*")
+
+
+def update_local_library(root_dir):
+ old_library = get_data("local_library")
+ known_paths = {track['path']: track_id for track_id, track in old_library.items()}
+ new_library = {}
+
+ song_id = '0'
+
+ def get_new_song_id(prev_id, song_path):
+ if song_path in known_paths:
+ return int(known_paths[song_path])
+ while (prev_id in old_library) or (prev_id in new_library):
+ prev_id = str(int(prev_id) + 1)
+ return prev_id
+
+ for dir_path, dir_names, filenames in os.walk(root_dir):
+ dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)]
+ filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)]
+
+ for filename in filtered_filenames:
+ song_name, track_type = os.path.splitext(filename)
+ relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir)
+ song_id = get_new_song_id(song_id, relative_path)
+ new_library[song_id] = {
+ "name": song_name,
+ "path": relative_path,
+ "type": track_type,
+ "artist": "",
+ "album": ""
+ }
+
+ for track_id, track in new_library.items():
+ abs_path = os.path.join(root_dir, track['path'])
+ match track['type']:
+ case ".mp3":
+ mp3track = eyed3.load(abs_path)
+ if not mp3track:
+ print(f"Failed to load mp3 {abs_path} using just name of the file as track name")
+ track['name'] = track['path']
+ else:
+ tag = mp3track.tag
+ if not tag:
+ print(f"Invalid mp3 header {abs_path} using just name of the file as track name")
+ track['name'] = track['path']
+ else:
+ track['artist'] = tag.artist
+ track['name'] = tag.title
+ track['album'] = tag.album
+
+ case ".flac":
+ try:
+ metadata = FLAC(abs_path).tags
+ if 'artist' in metadata:
+ track['artist'] = " ".join(metadata['artist'])
+ if 'TITLE' in metadata:
+ track['name'] = " ".join(metadata['TITLE'])
+ if 'album' in metadata:
+ track['album'] = " ".join(metadata['album'])
+ except mutagen.MutagenError:
+ print(f"Invalid flac header {abs_path} using just name of the file as track name")
+ track['name'] = track['path']
+
+ save_data(new_library, "local_library")
+
+
+if __name__ == "__main__":
+ update_local_library("/home/auser/Music/")
diff --git a/ParseItunesToJson.py b/ParseItunesToJson.py
new file mode 100755
index 0000000..1505a36
--- /dev/null
+++ b/ParseItunesToJson.py
@@ -0,0 +1,101 @@
+import xmltodict
+import json
+
+
+def flatten_dict(d):
+
+ out = []
+ for song in d:
+ newSong = {}
+
+ strCount = 0
+ intCount = 0
+ dateCount = 0
+
+ def getStr(id):
+ nonlocal strCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["string"]) <= strCount:
+ return "error"
+ strCount += 1
+ return song["string"][strCount - 1]
+
+ def getInt(id):
+ nonlocal intCount
+ if id not in song["key"]:
+ return -1
+ if len(song["integer"]) <= intCount:
+ return -1
+ intCount += 1
+ return song["integer"][intCount - 1]
+
+ def getDate(id):
+ nonlocal dateCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["date"]) <= dateCount:
+ return "error"
+ dateCount += 1
+ return song["date"][dateCount - 1]
+
+ newSong["Track ID"] = getInt("Track ID")
+ newSong["Name"] = getStr("Name")
+ newSong["Artist"] = getStr("Artist")
+ newSong["Album Artist"] = getStr("Album Artist")
+ newSong["Composer"] = getStr("Composer")
+ newSong["Album"] = getStr("Album")
+ newSong["Genre"] = getStr("Genre")
+ newSong["Kind"] = getStr("Kind")
+ newSong["Size"] = getInt("Size")
+ newSong["Total Time"] = getInt("Total Time")
+ newSong["Disc Number"] = getInt("Disc Number")
+ newSong["Disc Count"] = getInt("Disc Count")
+ newSong["Track Number"] = getInt("Track Number")
+ newSong["Track Count"] = getInt("Track Count")
+ newSong["Year"] = getInt("Year")
+ newSong["Date Modified"] = getDate("Date Modified")
+ newSong["Date Added"] = getDate("Date Added")
+ newSong["Bit Rate"] = getInt("Bit Rate")
+ newSong["Sample Rate"] = getInt("Sample Rate")
+ newSong["Play Count"] = getInt("Play Count")
+ newSong["Play Date"] = getInt("Play Date")
+ newSong["Play Date UTC"] = getDate("Play Date UTC")
+ newSong["Skip Count"] = getInt("Skip Count")
+ newSong["Skip Date"] = getDate("Skip Date")
+ newSong["Release Date"] = getDate("Release Date")
+ newSong["Album Rating"] = getInt("Album Rating")
+ newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"]
+ newSong["Loved"] = "Loved" in song["key"]
+ newSong["Album Loved"] = "Album Loved" in song["key"]
+ newSong["Explicit"] = "Explicit" in song["key"]
+ newSong["Compilation"] = "Compilation" in song["key"]
+ newSong["Artwork Count"] = getInt("Artwork Count")
+ newSong["Sort Album"] = getStr("Sort Album")
+ newSong["Sort Artist"] = getStr("Sort Artist")
+ newSong["Sort Name"] = getStr("Sort Name")
+ newSong["Persistent ID"] = getStr("Persistent ID")
+ newSong["Track Type"] = getStr("Track Type")
+
+ out.append(newSong)
+
+ return out
+
+
+def convert(filename, out_path):
+ with open(filename, 'r', encoding='utf-8') as xml_file:
+ data_dict = xmltodict.parse(xml_file.read())
+
+ # Extract the "Tracks" dictionary to be flattened
+ tracks_dict = data_dict['plist']['dict']['dict']
+
+ flat_tracks_dict = flatten_dict(tracks_dict["dict"])
+
+ json_data = json.dumps(flat_tracks_dict, indent=2)
+
+ with open(out_path, 'w', encoding='utf-8') as json_file:
+ json_file.write(json_data)
+
+
+if __name__ == "__main__":
+ convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json')
diff --git a/README.md b/README.md
index 508172a..68b9ba5 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,3 @@
# spotifyFetcher
-loads all data through web api
+loads all data through web api
+client secret - ***REDACTED***
diff --git a/SpotifyAuthenticator.py b/SpotifyAuthenticator.py
new file mode 100644
index 0000000..86705d0
--- /dev/null
+++ b/SpotifyAuthenticator.py
@@ -0,0 +1,122 @@
+import base64
+
+import requests
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import urllib.parse
+import threading
+import time
+import os.path
+
+client_id = '***REDACTED***'
+client_secret = None
+token = None
+
+
+def resolve_client_secret():
+ global client_secret
+ if os.path.exists('secret'):
+ with open('secret', 'r') as file:
+ client_secret = file.readline().strip()
+ else:
+ client_secret = input("Enter the application secret: ")
+ with open('secret', 'w') as file:
+ file.write(client_secret)
+
+
+def retrieve_web_api_token(authorization_code):
+ global token
+
+ token_url = 'https://accounts.spotify.com/api/token'
+
+ token_data = {
+ "grant_type": "authorization_code",
+ "client_secret": client_secret,
+ "client_id": client_id,
+ "code": authorization_code,
+ 'redirect_uri': 'http://127.0.0.1:8888',
+ }
+
+ print("\nAuthenticating WEB API\n")
+
+ auth_string = f'{client_id}:{client_secret}'
+ b64_auth_string = base64.b64encode(auth_string.encode()).decode()
+
+ # Define the headers
+ headers = {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ 'Authorization': f'Basic {b64_auth_string}'
+ }
+
+ token_response = requests.post(token_url, data=token_data, headers=headers)
+
+ if token_response.status_code not in range(200, 299):
+ print(f"Failed to retrieve access token: {token_response.status_code}")
+ print(token_response.json())
+ raise ValueError
+
+ token = token_response.json()['access_token']
+
+
+class RequestHandler(BaseHTTPRequestHandler):
+ def do_GET(self):
+
+ parsed_url = urllib.parse.urlparse(self.path)
+ query_params = urllib.parse.parse_qs(parsed_url.query)
+
+ if 'code' in query_params:
+ authorization_code = query_params['code'][0]
+
+ self.send_response(200)
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+ self.wfile.write(b'Authorization code received. You can close this window.')
+
+ retrieve_web_api_token(authorization_code)
+
+ threading.Thread(target=self.server.shutdown).start()
+
+ else:
+ self.send_response(400)
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+ self.wfile.write(b'Authorization code not found.')
+
+
+def run_redirect_server():
+ httpd = HTTPServer(('localhost', 8888), RequestHandler)
+ print('Starting HTTP redirection server')
+ httpd.serve_forever()
+
+
+def open_authenticate_page():
+ authorize_url = 'https://accounts.spotify.com/authorize'
+
+ params = {
+ 'client_id': client_id,
+ 'response_type': 'code',
+ 'redirect_uri': 'http://127.0.0.1:8888',
+ "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private",
+ }
+
+ auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}"
+ webbrowser.open(auth_url)
+
+
+def authenticate():
+ resolve_client_secret()
+
+ redirect_server = threading.Thread(target=run_redirect_server)
+ redirect_server.start()
+
+ time.sleep(1)
+ open_authenticate_page()
+
+ redirect_server.join()
+
+ return token
+
+
+if __name__ == "__main__":
+ authenticate()
+
diff --git a/SpotifyWebAPI.py b/SpotifyWebAPI.py
new file mode 100644
index 0000000..71ec2f7
--- /dev/null
+++ b/SpotifyWebAPI.py
@@ -0,0 +1,190 @@
+import requests
+from PIL import Image
+from io import BytesIO
+from SpotifyAuthenticator import authenticate
+from DataBase import *
+
+FETCH_STEP = 50
+token = ''
+
+
+def fetch(endpoint, method='GET', body=None):
+ url = f'https://api.spotify.com/{endpoint}'
+ headers = {
+ 'Authorization': f'Bearer {token}',
+ }
+
+ response = requests.request(method, url, headers=headers, json=body)
+ response.raise_for_status()
+ return response.json()
+
+
+def find_song(song_pattern):
+ return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items']
+
+
+def fetch_playlists(user_id):
+ print("Fetching playlists")
+ endpoint = f"v1/me/playlists"
+ pl_total = fetch(f'{endpoint}?limit=1')['total']
+ pl_count = 0
+ playlists = []
+
+ while pl_count < pl_total:
+ res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}')
+ playlists += res['items']
+ pl_count += FETCH_STEP
+
+ return playlists
+
+
+def fetch_playlist_items(playlist_id):
+ endpoint = f"v1/playlists/{playlist_id}/tracks"
+ items = fetch(f'{endpoint}?fields=total')['total']
+ loaded = 0
+ tracks = []
+
+ while loaded < items:
+ res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}')
+
+ tracks += res['items']
+ loaded += FETCH_STEP
+
+ return tracks
+
+
+def fetch_tracks(user_id):
+ print("Fetching tracks")
+ endpoint = f"v1/me/tracks"
+ total = fetch(f"{endpoint}?limit=1&market=ES")['total']
+ current = 0
+ tracks = []
+
+ while total > current:
+ res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES")
+ tracks += res['items']
+ current += FETCH_STEP
+
+ save_data(tracks, "tracks")
+ return tracks
+
+
+def fetch_tracks_top():
+ print("Fetching top tracks")
+ total = fetch(f'v1/me/top/tracks?fields=total')['total']
+ current = 0
+ tracks = []
+
+ while current < total:
+ res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term')
+ tracks += res['items']
+ current += FETCH_STEP
+
+ save_data(tracks, "top_tracks")
+ return tracks
+
+
+def fetch_artists_top():
+ print("Fetching top artists")
+
+ total = fetch(f'v1/me/top/artists?fields=total')['total']
+ current = 0
+ artists = []
+
+ while current < total:
+ res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term')
+ artists += res['items']
+ current += FETCH_STEP
+
+ save_data(artists, "top_artists")
+ return artists
+
+
+def fetch_all_playlist_data(user_id):
+ playlists = fetch_playlists(user_id)
+
+ save_data(playlists, 'playlists')
+
+ print(len(playlists))
+
+ playlist_items = {}
+
+ print("Fetching playlists items")
+
+ index = 0
+ for pl in playlists:
+ pl_id = pl['id']
+ name = pl['name']
+ playlist_items[name] = fetch_playlist_items(pl_id)
+ print(f"{index} - {len(playlist_items[name])}")
+ index += 1
+
+ save_data(playlist_items, "playlistTracks")
+
+
+def get_user_data():
+ print("Fetching user data")
+
+ user_profile = fetch(f'v1/me')
+ user_id = user_profile['id']
+ user_data = fetch(f'v1/users/{user_id}')
+
+ save_data(user_profile, "user_profile")
+ save_data(user_data, "user_data")
+ return user_id
+
+
+def save_image_from_url(url, name, image_dir="playlist_covers"):
+ response = requests.get(url)
+
+ if response.status_code != 200:
+ raise "Cannot fetch the playlist cover"
+
+ image = Image.open(BytesIO(response.content))
+
+ directory = os.path.join(data_dir, image_dir)
+ os.makedirs(directory, exist_ok=True)
+ file_path = os.path.join(directory, f"{name}.jpg")
+ print(f"Image saved {file_path}")
+ image.save(file_path)
+
+
+def load_playlist_covers():
+ playlists = get_data('playlists')
+
+ for pl in playlists:
+ if len(pl['images']):
+ url = pl['images'][0]['url']
+ save_image_from_url(url, pl['name'])
+
+
+def load_user_cover():
+ user_data = get_data("user_data")
+ url = user_data['images'][1]['url']
+ save_image_from_url(url, "user", ".")
+
+
+def load_artworks():
+ load_user_cover()
+ load_playlist_covers()
+
+
+def update_access_token():
+ global token
+ token = authenticate()
+
+
+def fetch_data():
+ user_id = get_user_data()
+
+ fetch_tracks(user_id)
+ fetch_all_playlist_data(user_id)
+ fetch_tracks_top()
+ fetch_artists_top()
+
+ load_artworks()
+
+
+if __name__ == "__main__":
+ update_access_token()
+ fetch_data()
diff --git a/env b/env
new file mode 100755
index 0000000..8955572
--- /dev/null
+++ b/env
@@ -0,0 +1,36 @@
+
+PS1=" > "
+
+#export PATH="/usr/local/bin/:$PATH"
+export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH"
+#export PATH="$HOME/home/auser/.local/bin:$PATH"
+
+export SIP="185.238.170.251"
+
+alias v=nvim
+alias ll="ls -l -a"
+alias gl="git log --oneline"
+alias ssh_server="ssh auser@185.238.170.251"
+
+#eval "$(zoxide init bash --cmd zd)"
+
+#source $PMAIN/.scripts/bashmarks.sh
+
+fe() {
+ local result=$(command tere "$@")
+ [ -n "$result" ] && cd -- "$result"
+}
+
+vim_configure() {
+ pwd=$(pwd)
+ cd ~/.config/nvim/
+ nvim
+ cd $pwd
+}
+
+pyenv() {
+ source ~/bin/python/env311/bin/activate
+}
+
+source ~/src/scripts/remotes.sh
+source /usr/share/fzf/key-bindings.bash
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..9709e93
--- /dev/null
+++ b/main.py
@@ -0,0 +1,197 @@
+from DataBase import *
+from SpotifyWebAPI import fetch_data, update_access_token
+import cmd
+
+playlists = get_data('playlistTracks')
+top_artists = get_data('top_artists')
+top_tracks = get_data('top_tracks')
+user_tracks = get_data('tracks')
+link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')]
+local_library = get_data("local_library")
+user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
+
+
+def get_playlist_names(pl):
+ return [name for name, items in pl.items()]
+
+
+def print_playlist_tracks(pl_id: int):
+ pls = list(playlists)
+ name = pls[pl_id]
+ pl = playlists[name]
+ print(name)
+ for track in pl:
+ track_data = track['track']
+ track_name = track_data['name']
+ artists = get_artists_str(track_data['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_top_artists():
+ for artist in top_artists:
+ name = artist['name']
+ print(f" '{name}'")
+
+
+def print_top_tracks():
+ for track in top_tracks:
+ track_name = track['name']
+ artists = get_artists_str(track['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_stats():
+ print(f" tracks - {len(user_tracks)}")
+ print(f" playlists - {len(playlists)}")
+
+ for name, tracks in playlists.items():
+ print(f" '{name}' - {len(tracks)}")
+
+ print(f" top tracks - {len(top_tracks)}")
+ print(f" top artists - {len(top_artists)}")
+
+
+def print_tracks():
+ track_idx = 0
+ for trackItem in user_tracks:
+ track = trackItem['track']
+ print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
+ track_idx += 1
+
+
+def sort_remote_tracks_by_popularity(track_ids):
+ out = []
+ for top_track in top_tracks:
+ if top_track['id'] in track_ids:
+ out.append(top_track['id'])
+ return out
+
+
+def print_link_stats(link_pattern):
+ resolved_reversed = {}
+ resolved = []
+ unresolved_local = []
+ unresolved_remote = []
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ resolved.append(local_id)
+ resolved_reversed[links['items'][0]] = local_id
+ else:
+ unresolved_local.append(local_id)
+
+ for track in user_tracks:
+ track_id = track['track']['id']
+ if track_id not in resolved_reversed:
+ unresolved_remote.append(track_id)
+
+ print(f"Local Tracks: {len(local_library)}")
+ print(f"Remote Tracks: {len(user_tracks)}")
+
+ unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
+ unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+
+ print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
+ index = 0
+ for remote_id in unresolved_remote:
+ remote_track = user_tracks_by_id[remote_id]
+ print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ print(f"\nResolved tracks: {len(resolved)}")
+ index = 0
+ for link in resolved:
+ remote_id = link_pattern[link]['items'][0]
+ local_track = local_library[link]
+ remote_track = user_tracks_by_id[remote_id]
+
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
+ print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ return
+ print(f"\nUnresolved tracks from the local: {len(unresolved_local)}")
+ index = 0
+ for link in unresolved_local:
+ local_track = local_library[link]
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'")
+ index += 1
+
+
+class Interpreter(cmd.Cmd):
+ intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
+ prompt = "(spotify) "
+
+ def do_link_patterns(self, arg):
+ """prints available linking patterns"""
+ print(f"0 - fuzzy tags")
+ print(f"1 - spotify")
+ print(f"2 - fuzzy")
+
+ def do_link_pattern(self, arg):
+ if not len(arg.split()):
+ print("expected name of the pattern")
+ return
+
+ """prints available linking patterns"""
+ pattern_id = int(arg.split()[0])
+ print_link_stats(link_patterns[pattern_id])
+
+ def do_playlists(self, arg):
+ """prints user playlists"""
+ pl_id = 0
+ for pl in get_playlist_names(playlists):
+ print(f"{pl_id} - {pl}")
+ pl_id += 1
+
+ def do_playlist_tracks(self, arg):
+ """prints playlist [name]"""
+ try:
+ pl_id = int(arg.split()[0])
+ print_playlist_tracks(pl_id)
+ except IndexError:
+ print("Invalid input")
+
+ def do_top_artists(self, arg):
+ """Top artists"""
+ print_top_artists()
+
+ def do_top_tracks(self, arg):
+ """Top tracks"""
+ print_top_tracks()
+
+ def do_stat(self, arg):
+ """Print stats"""
+ print_stats()
+
+ def do_tracks(self, arg):
+ """Print tracks"""
+ print_tracks()
+
+ def do_fetch(self, arg):
+ """Fetch all spotify data"""
+ try:
+ update_access_token()
+ fetch_data()
+
+ global playlists, top_artists, top_tracks, user_tracks
+
+ playlists = get_data('playlistTracks')
+ top_artists = get_data('top_artists')
+ top_tracks = get_data('top_tracks')
+ user_tracks = get_data('tracks')
+
+ except ValueError:
+ print("Can not fetch the data.")
+
+ def do_exit(self, arg):
+ """Exit the command loop: exit"""
+ print("Goodbye!")
+ return True
+
+
+if __name__ == '__main__':
+ try:
+ Interpreter().cmdloop()
+ except KeyboardInterrupt as kb:
+ print("process terminated")
\ No newline at end of file
From 2824bd1a0e196848a8fa1c7a48d5d44cc0e0562c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 4 Jan 2026 01:37:47 +0300
Subject: [PATCH 05/30] stable - generate local playlists & add default options
---
LinkerPatternGenerator.py | 28 ++++++++++-----
LocalPLaylistGenerator.py | 50 +++++++++++++++++++++++++++
TODO | 7 ++++
main.py | 73 +++++++++++++++++++++++++++++----------
4 files changed, 132 insertions(+), 26 deletions(-)
create mode 100644 LocalPLaylistGenerator.py
create mode 100644 TODO
diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py
index 2f6eba2..8576da8 100644
--- a/LinkerPatternGenerator.py
+++ b/LinkerPatternGenerator.py
@@ -3,7 +3,7 @@ from SpotifyWebAPI import find_song, update_access_token
from rapidfuzz import fuzz
from tqdm import tqdm
-
+import requests
def spotify_pattern_generator():
update_access_token()
@@ -11,15 +11,29 @@ def spotify_pattern_generator():
local_library = get_data("local_library")
spotify_library = get_data("tracks")
+
def find_local_songs_on_spotify(local_lib):
+ search_log = []
+
total = len(local_library)
found_tracks_map = {}
with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
for track_id, track in local_lib.items():
- search_pattern = f"{track['name']} {track['artist']} {track['album']}"
- found_tracks = find_song(search_pattern)
- found_tracks_map[track_id] = found_tracks[0:1]
+ parts = [track["name"], track["artist"], track.get("album")]
+ search_pattern = " ".join(p for p in parts if p)
+
+ if search_pattern == "":
+ print(f"cannot generate search pattern for the track - {track["path"]}")
+ pbar.update(1)
+ continue
+
+ found_tracks = find_song(search_pattern)[0:1]
+ found_tracks_map[track_id] = found_tracks
+ search_log.append({"path" : track["path"], "search_pattern": search_pattern, "result" : found_tracks_map[track_id]})
pbar.update(1)
+
+ save_data(search_log, "spotify_search_log")
+
return found_tracks_map
spotify_found_tracks = find_local_songs_on_spotify(local_library)
@@ -127,10 +141,8 @@ def fuzzy_tag_pattern_generator():
save_data(mappings, "link_pattern_tags")
-def run_pattern_generators():
- #fuzzy_pattern_generator()
- #spotify_pattern_generator()
- fuzzy_tag_pattern_generator()
+def run_pattern_generators(type_id = 0):
+ spotify_pattern_generator()
if __name__ == "__main__":
diff --git a/LocalPLaylistGenerator.py b/LocalPLaylistGenerator.py
new file mode 100644
index 0000000..0299498
--- /dev/null
+++ b/LocalPLaylistGenerator.py
@@ -0,0 +1,50 @@
+
+from pathlib import Path
+
+def generate_playlists(local_library, library_path, remote_playlists, link_pattern, output_dir):
+ output_dir = Path(output_dir)
+ output_playlists = []
+
+ remote_to_local_map = {}
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ remote_to_local_map[links['items'][0]] = local_id
+
+ for name, tracks in remote_playlists.items():
+ local_tracks = []
+
+ for track in tracks:
+ track = track["track"]
+ path = "error"
+ error = ""
+ if track is None:
+ error = f"error: track is none"
+ elif track["id"] in remote_to_local_map:
+ path = local_library[remote_to_local_map[track["id"]]]["path"]
+ else:
+ error = f"error: not_found - {track["name"]}"
+
+ if error != "":
+ local_tracks.append("# " + error)
+ else:
+ path = (output_dir / ".." ).resolve() / Path(path)
+ local_tracks.append(path)
+
+ pl = {
+ "name": name,
+ "tracks": local_tracks,
+ }
+
+ output_playlists.append(pl)
+
+
+ for pl in output_playlists:
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ playlist_path = Path(output_dir) / f"{pl['name']}.m3u"
+
+ with (playlist_path.open("w", encoding="utf-8") as f):
+ for track in pl["tracks"]:
+ f.write(str(track))
+ f.write("\n")
\ No newline at end of file
diff --git a/TODO b/TODO
new file mode 100644
index 0000000..f285a22
--- /dev/null
+++ b/TODO
@@ -0,0 +1,7 @@
+skip playlists that are not created by user
+
+create one big playlist with the order of the spotify and dont skip missing
+
+generate percentage like hierarchy of the missing items
+
+generate detailed log of the missing items and search results
diff --git a/main.py b/main.py
index 9709e93..10d378a 100644
--- a/main.py
+++ b/main.py
@@ -1,12 +1,19 @@
from DataBase import *
+from LocalLibraryIndexer import update_local_library
+from LinkerPatternGenerator import run_pattern_generators
+from LocalPLaylistGenerator import generate_playlists
+
from SpotifyWebAPI import fetch_data, update_access_token
import cmd
+local_library_path = "/home/auser/Music/"
+rel_autogen_dir = "0autogen"
+
playlists = get_data('playlistTracks')
top_artists = get_data('top_artists')
top_tracks = get_data('top_tracks')
user_tracks = get_data('tracks')
-link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')]
+link_pattern = get_data('link_pattern_spotify')
local_library = get_data("local_library")
user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
@@ -109,33 +116,20 @@ def print_link_stats(link_pattern):
print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
index += 1
- return
- print(f"\nUnresolved tracks from the local: {len(unresolved_local)}")
- index = 0
- for link in unresolved_local:
- local_track = local_library[link]
- print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'")
- index += 1
-
class Interpreter(cmd.Cmd):
intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
prompt = "(spotify) "
- def do_link_patterns(self, arg):
+ def do_list_stat_vs_types(self, arg):
"""prints available linking patterns"""
print(f"0 - fuzzy tags")
print(f"1 - spotify")
print(f"2 - fuzzy")
- def do_link_pattern(self, arg):
- if not len(arg.split()):
- print("expected name of the pattern")
- return
-
- """prints available linking patterns"""
- pattern_id = int(arg.split()[0])
- print_link_stats(link_patterns[pattern_id])
+ def do_stat_local_library_vs_spotify(self, arg):
+ """compares local library and remote with the link pattern generated by the generate_pattern_command"""
+ print_link_stats(link_pattern)
def do_playlists(self, arg):
"""prints user playlists"""
@@ -168,6 +162,24 @@ class Interpreter(cmd.Cmd):
"""Print tracks"""
print_tracks()
+ def do_generate_local_playlists(self, arg):
+ """generates local playlists based on the vs comparison type"""
+ generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
+
+
+ def do_index_local_library(self, arg):
+ """find local tracks and fetches the data from them for further analysis"""
+ global local_library
+ update_local_library(local_library_path)
+ local_library = get_data("local_library")
+
+ def do_generate_vs_stat(self, arg):
+ """compares local and remote library and generates the vs stats"""
+
+ global link_pattern
+ run_pattern_generators()
+ link_pattern = get_data('link_pattern_spotify')
+
def do_fetch(self, arg):
"""Fetch all spotify data"""
try:
@@ -184,6 +196,31 @@ class Interpreter(cmd.Cmd):
except ValueError:
print("Can not fetch the data.")
+ def do_generate_todo(self, arg):
+ """generates todos for the pirates"""
+ print("sorry not implemented yet.")
+
+ def do_run_all_default(self, arg):
+ """runs general flow"""
+
+ local_lib_path = "/home/auser/Music/"
+ print("Fetching spotify data...")
+ self.do_fetch([])
+
+ print("Fetching local data...")
+ self.do_index_local_library(f"{local_lib_path}/organized")
+
+ print("Generating vs patterns...")
+ self.do_generate_vs_stat("0")
+
+ print("Generating todos...")
+ self.do_generate_todo(f"{local_lib_path}")
+
+ print("Generating local playlists...")
+ self.do_generate_local_playlists(f"{local_lib_path}")
+
+ return True
+
def do_exit(self, arg):
"""Exit the command loop: exit"""
print("Goodbye!")
From 892568d668b487414c17dc4ffbb46bcc30c95e29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 4 Jan 2026 01:37:47 +0300
Subject: [PATCH 06/30] stable - generate local playlists & add default options
---
LinkerPatternGenerator.py | 28 ++++++++++-----
LocalPLaylistGenerator.py | 50 +++++++++++++++++++++++++++
TODO | 7 ++++
main.py | 73 +++++++++++++++++++++++++++++----------
4 files changed, 132 insertions(+), 26 deletions(-)
create mode 100644 LocalPLaylistGenerator.py
create mode 100644 TODO
diff --git a/LinkerPatternGenerator.py b/LinkerPatternGenerator.py
index 2f6eba2..8576da8 100644
--- a/LinkerPatternGenerator.py
+++ b/LinkerPatternGenerator.py
@@ -3,7 +3,7 @@ from SpotifyWebAPI import find_song, update_access_token
from rapidfuzz import fuzz
from tqdm import tqdm
-
+import requests
def spotify_pattern_generator():
update_access_token()
@@ -11,15 +11,29 @@ def spotify_pattern_generator():
local_library = get_data("local_library")
spotify_library = get_data("tracks")
+
def find_local_songs_on_spotify(local_lib):
+ search_log = []
+
total = len(local_library)
found_tracks_map = {}
with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
for track_id, track in local_lib.items():
- search_pattern = f"{track['name']} {track['artist']} {track['album']}"
- found_tracks = find_song(search_pattern)
- found_tracks_map[track_id] = found_tracks[0:1]
+ parts = [track["name"], track["artist"], track.get("album")]
+ search_pattern = " ".join(p for p in parts if p)
+
+ if search_pattern == "":
+ print(f"cannot generate search pattern for the track - {track["path"]}")
+ pbar.update(1)
+ continue
+
+ found_tracks = find_song(search_pattern)[0:1]
+ found_tracks_map[track_id] = found_tracks
+ search_log.append({"path" : track["path"], "search_pattern": search_pattern, "result" : found_tracks_map[track_id]})
pbar.update(1)
+
+ save_data(search_log, "spotify_search_log")
+
return found_tracks_map
spotify_found_tracks = find_local_songs_on_spotify(local_library)
@@ -127,10 +141,8 @@ def fuzzy_tag_pattern_generator():
save_data(mappings, "link_pattern_tags")
-def run_pattern_generators():
- #fuzzy_pattern_generator()
- #spotify_pattern_generator()
- fuzzy_tag_pattern_generator()
+def run_pattern_generators(type_id = 0):
+ spotify_pattern_generator()
if __name__ == "__main__":
diff --git a/LocalPLaylistGenerator.py b/LocalPLaylistGenerator.py
new file mode 100644
index 0000000..0299498
--- /dev/null
+++ b/LocalPLaylistGenerator.py
@@ -0,0 +1,50 @@
+
+from pathlib import Path
+
+def generate_playlists(local_library, library_path, remote_playlists, link_pattern, output_dir):
+ output_dir = Path(output_dir)
+ output_playlists = []
+
+ remote_to_local_map = {}
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ remote_to_local_map[links['items'][0]] = local_id
+
+ for name, tracks in remote_playlists.items():
+ local_tracks = []
+
+ for track in tracks:
+ track = track["track"]
+ path = "error"
+ error = ""
+ if track is None:
+ error = f"error: track is none"
+ elif track["id"] in remote_to_local_map:
+ path = local_library[remote_to_local_map[track["id"]]]["path"]
+ else:
+ error = f"error: not_found - {track["name"]}"
+
+ if error != "":
+ local_tracks.append("# " + error)
+ else:
+ path = (output_dir / ".." ).resolve() / Path(path)
+ local_tracks.append(path)
+
+ pl = {
+ "name": name,
+ "tracks": local_tracks,
+ }
+
+ output_playlists.append(pl)
+
+
+ for pl in output_playlists:
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ playlist_path = Path(output_dir) / f"{pl['name']}.m3u"
+
+ with (playlist_path.open("w", encoding="utf-8") as f):
+ for track in pl["tracks"]:
+ f.write(str(track))
+ f.write("\n")
\ No newline at end of file
diff --git a/TODO b/TODO
new file mode 100644
index 0000000..f285a22
--- /dev/null
+++ b/TODO
@@ -0,0 +1,7 @@
+skip playlists that are not created by user
+
+create one big playlist with the order of the spotify and dont skip missing
+
+generate percentage like hierarchy of the missing items
+
+generate detailed log of the missing items and search results
diff --git a/main.py b/main.py
index 9709e93..10d378a 100644
--- a/main.py
+++ b/main.py
@@ -1,12 +1,19 @@
from DataBase import *
+from LocalLibraryIndexer import update_local_library
+from LinkerPatternGenerator import run_pattern_generators
+from LocalPLaylistGenerator import generate_playlists
+
from SpotifyWebAPI import fetch_data, update_access_token
import cmd
+local_library_path = "/home/auser/Music/"
+rel_autogen_dir = "0autogen"
+
playlists = get_data('playlistTracks')
top_artists = get_data('top_artists')
top_tracks = get_data('top_tracks')
user_tracks = get_data('tracks')
-link_patterns = [get_data('link_pattern_tags'), get_data('link_pattern_tags'), get_data('link_pattern_tags')]
+link_pattern = get_data('link_pattern_spotify')
local_library = get_data("local_library")
user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
@@ -109,33 +116,20 @@ def print_link_stats(link_pattern):
print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
index += 1
- return
- print(f"\nUnresolved tracks from the local: {len(unresolved_local)}")
- index = 0
- for link in unresolved_local:
- local_track = local_library[link]
- print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'")
- index += 1
-
class Interpreter(cmd.Cmd):
intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
prompt = "(spotify) "
- def do_link_patterns(self, arg):
+ def do_list_stat_vs_types(self, arg):
"""prints available linking patterns"""
print(f"0 - fuzzy tags")
print(f"1 - spotify")
print(f"2 - fuzzy")
- def do_link_pattern(self, arg):
- if not len(arg.split()):
- print("expected name of the pattern")
- return
-
- """prints available linking patterns"""
- pattern_id = int(arg.split()[0])
- print_link_stats(link_patterns[pattern_id])
+ def do_stat_local_library_vs_spotify(self, arg):
+ """compares local library and remote with the link pattern generated by the generate_pattern_command"""
+ print_link_stats(link_pattern)
def do_playlists(self, arg):
"""prints user playlists"""
@@ -168,6 +162,24 @@ class Interpreter(cmd.Cmd):
"""Print tracks"""
print_tracks()
+ def do_generate_local_playlists(self, arg):
+ """generates local playlists based on the vs comparison type"""
+ generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
+
+
+ def do_index_local_library(self, arg):
+ """find local tracks and fetches the data from them for further analysis"""
+ global local_library
+ update_local_library(local_library_path)
+ local_library = get_data("local_library")
+
+ def do_generate_vs_stat(self, arg):
+ """compares local and remote library and generates the vs stats"""
+
+ global link_pattern
+ run_pattern_generators()
+ link_pattern = get_data('link_pattern_spotify')
+
def do_fetch(self, arg):
"""Fetch all spotify data"""
try:
@@ -184,6 +196,31 @@ class Interpreter(cmd.Cmd):
except ValueError:
print("Can not fetch the data.")
+ def do_generate_todo(self, arg):
+ """generates todos for the pirates"""
+ print("sorry not implemented yet.")
+
+ def do_run_all_default(self, arg):
+ """runs general flow"""
+
+ local_lib_path = "/home/auser/Music/"
+ print("Fetching spotify data...")
+ self.do_fetch([])
+
+ print("Fetching local data...")
+ self.do_index_local_library(f"{local_lib_path}/organized")
+
+ print("Generating vs patterns...")
+ self.do_generate_vs_stat("0")
+
+ print("Generating todos...")
+ self.do_generate_todo(f"{local_lib_path}")
+
+ print("Generating local playlists...")
+ self.do_generate_local_playlists(f"{local_lib_path}")
+
+ return True
+
def do_exit(self, arg):
"""Exit the command loop: exit"""
print("Goodbye!")
From 72af96eada159f8516df82cacc3d549cb63c8310 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 4 Jan 2026 19:25:42 +0300
Subject: [PATCH 07/30] refactor initial not fully recovered
---
Env.py | 7 +
main.py | 241 +++---------------
env => old/env | 0
old/main.py | 234 +++++++++++++++++
src/Library.py | 159 ++++++++++++
src/LibrarySerializer.py | 199 +++++++++++++++
.../MapLocalToSpotify.py | 61 ++---
src/StatGenerator.py | 42 +++
DataBase.py => src/helpers.py | 16 +-
.../itunes/ParseItunesToJson.py | 0
.../local/LocalLibraryIndexer.py | 14 +-
.../local/LocalPLaylistGenerator.py | 0
src/local/ParseLocal.py | 40 +++
src/spotify/ParseSpotify.py | 139 ++++++++++
.../spotify/SpotifyAuthenticator.py | 22 +-
.../spotify/SpotifyWebAPI.py | 18 +-
16 files changed, 925 insertions(+), 267 deletions(-)
create mode 100644 Env.py
rename env => old/env (100%)
create mode 100644 old/main.py
create mode 100644 src/Library.py
create mode 100644 src/LibrarySerializer.py
rename LinkerPatternGenerator.py => src/MapLocalToSpotify.py (71%)
create mode 100644 src/StatGenerator.py
rename DataBase.py => src/helpers.py (85%)
rename ParseItunesToJson.py => src/itunes/ParseItunesToJson.py (100%)
rename LocalLibraryIndexer.py => src/local/LocalLibraryIndexer.py (95%)
rename LocalPLaylistGenerator.py => src/local/LocalPLaylistGenerator.py (100%)
create mode 100644 src/local/ParseLocal.py
create mode 100644 src/spotify/ParseSpotify.py
rename SpotifyAuthenticator.py => src/spotify/SpotifyAuthenticator.py (88%)
rename SpotifyWebAPI.py => src/spotify/SpotifyWebAPI.py (91%)
diff --git a/Env.py b/Env.py
new file mode 100644
index 0000000..f717d8b
--- /dev/null
+++ b/Env.py
@@ -0,0 +1,7 @@
+from pathlib import Path
+
+local_library_path = Path("/home/auser/Music/")
+work_dir = Path("./work_dir/new")
+
+client_id = '3b4db45bb66b45e9a2532989c8034332'
+client_secret = "ed03dce79ec049e59b1bf20967aba418"
diff --git a/main.py b/main.py
index 10d378a..32682e3 100644
--- a/main.py
+++ b/main.py
@@ -1,234 +1,77 @@
-from DataBase import *
-from LocalLibraryIndexer import update_local_library
-from LinkerPatternGenerator import run_pattern_generators
-from LocalPLaylistGenerator import generate_playlists
-from SpotifyWebAPI import fetch_data, update_access_token
import cmd
-local_library_path = "/home/auser/Music/"
-rel_autogen_dir = "0autogen"
+from Env import client_id, client_secret, local_library_path
+from Env import work_dir
-playlists = get_data('playlistTracks')
-top_artists = get_data('top_artists')
-top_tracks = get_data('top_tracks')
-user_tracks = get_data('tracks')
-link_pattern = get_data('link_pattern_spotify')
-local_library = get_data("local_library")
-user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
+from src.LibrarySerializer import LibrarySaver, LibraryLoader
+from src.spotify.SpotifyWebAPI import fetch_all
+from src.spotify.ParseSpotify import Parser
-def get_playlist_names(pl):
- return [name for name, items in pl.items()]
+from src.local.LocalLibraryIndexer import update_local_library
+from src.local.ParseLocal import LocalParser
-
-def print_playlist_tracks(pl_id: int):
- pls = list(playlists)
- name = pls[pl_id]
- pl = playlists[name]
- print(name)
- for track in pl:
- track_data = track['track']
- track_name = track_data['name']
- artists = get_artists_str(track_data['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_top_artists():
- for artist in top_artists:
- name = artist['name']
- print(f" '{name}'")
-
-
-def print_top_tracks():
- for track in top_tracks:
- track_name = track['name']
- artists = get_artists_str(track['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_stats():
- print(f" tracks - {len(user_tracks)}")
- print(f" playlists - {len(playlists)}")
-
- for name, tracks in playlists.items():
- print(f" '{name}' - {len(tracks)}")
-
- print(f" top tracks - {len(top_tracks)}")
- print(f" top artists - {len(top_artists)}")
-
-
-def print_tracks():
- track_idx = 0
- for trackItem in user_tracks:
- track = trackItem['track']
- print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
- track_idx += 1
-
-
-def sort_remote_tracks_by_popularity(track_ids):
- out = []
- for top_track in top_tracks:
- if top_track['id'] in track_ids:
- out.append(top_track['id'])
- return out
-
-
-def print_link_stats(link_pattern):
- resolved_reversed = {}
- resolved = []
- unresolved_local = []
- unresolved_remote = []
-
- for local_id, links in link_pattern.items():
- if len(links['items']):
- resolved.append(local_id)
- resolved_reversed[links['items'][0]] = local_id
- else:
- unresolved_local.append(local_id)
-
- for track in user_tracks:
- track_id = track['track']['id']
- if track_id not in resolved_reversed:
- unresolved_remote.append(track_id)
-
- print(f"Local Tracks: {len(local_library)}")
- print(f"Remote Tracks: {len(user_tracks)}")
-
- unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
- unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
-
- print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
- index = 0
- for remote_id in unresolved_remote:
- remote_track = user_tracks_by_id[remote_id]
- print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
-
- print(f"\nResolved tracks: {len(resolved)}")
- index = 0
- for link in resolved:
- remote_id = link_pattern[link]['items'][0]
- local_track = local_library[link]
- remote_track = user_tracks_by_id[remote_id]
-
- print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
- print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
+from src.MapLocalToSpotify import map_local_to_spotify
class Interpreter(cmd.Cmd):
intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
- prompt = "(spotify) "
+ prompt = "(kuw) "
- def do_list_stat_vs_types(self, arg):
- """prints available linking patterns"""
- print(f"0 - fuzzy tags")
- print(f"1 - spotify")
- print(f"2 - fuzzy")
+ spotify_library = None
+ local_library = None
- def do_stat_local_library_vs_spotify(self, arg):
- """compares local library and remote with the link pattern generated by the generate_pattern_command"""
- print_link_stats(link_pattern)
-
- def do_playlists(self, arg):
- """prints user playlists"""
- pl_id = 0
- for pl in get_playlist_names(playlists):
- print(f"{pl_id} - {pl}")
- pl_id += 1
-
- def do_playlist_tracks(self, arg):
- """prints playlist [name]"""
- try:
- pl_id = int(arg.split()[0])
- print_playlist_tracks(pl_id)
- except IndexError:
- print("Invalid input")
-
- def do_top_artists(self, arg):
- """Top artists"""
- print_top_artists()
-
- def do_top_tracks(self, arg):
- """Top tracks"""
- print_top_tracks()
-
- def do_stat(self, arg):
- """Print stats"""
- print_stats()
-
- def do_tracks(self, arg):
- """Print tracks"""
- print_tracks()
-
- def do_generate_local_playlists(self, arg):
- """generates local playlists based on the vs comparison type"""
- generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
+ @staticmethod
+ def do_fetch_spotify(self):
+ fetch_all(client_id, client_secret, work_dir)
- def do_index_local_library(self, arg):
- """find local tracks and fetches the data from them for further analysis"""
- global local_library
- update_local_library(local_library_path)
- local_library = get_data("local_library")
+ @staticmethod
+ def do_fetch_local():
+ update_local_library(local_library_path, work_dir)
- def do_generate_vs_stat(self, arg):
- """compares local and remote library and generates the vs stats"""
- global link_pattern
- run_pattern_generators()
- link_pattern = get_data('link_pattern_spotify')
+ def do_parse_spotify_library(self):
+ parser = Parser()
+ library_parsed = parser.parse(work_dir / "spotify" / "playlistTracks.json", work_dir / "spotify" / "tracks.json")
+ LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
- def do_fetch(self, arg):
- """Fetch all spotify data"""
- try:
- update_access_token()
- fetch_data()
+ def do_parse_local_library(self):
+ parser = LocalParser()
+ library_parsed = parser.parse(work_dir / "local" / "local_library.json")
+ LibrarySaver(library_parsed).save(work_dir / "local_library.json")
- global playlists, top_artists, top_tracks, user_tracks
+ def do_load_libraries(self):
+ self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
+ self.local_library = LibraryLoader().load(work_dir / "local_library.json")
- playlists = get_data('playlistTracks')
- top_artists = get_data('top_artists')
- top_tracks = get_data('top_tracks')
- user_tracks = get_data('tracks')
+ def do_map_local_to_spotify(self):
+ map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
- except ValueError:
- print("Can not fetch the data.")
- def do_generate_todo(self, arg):
- """generates todos for the pirates"""
- print("sorry not implemented yet.")
+ def do_run(self, arg):
+ # self.do_fetch_local()
+ # self.do_fetch_spotify()
- def do_run_all_default(self, arg):
- """runs general flow"""
+ # self.do_parse_local_library()
+ # self.do_parse_spotify_library()
- local_lib_path = "/home/auser/Music/"
- print("Fetching spotify data...")
- self.do_fetch([])
+ self.do_load_libraries()
- print("Fetching local data...")
- self.do_index_local_library(f"{local_lib_path}/organized")
-
- print("Generating vs patterns...")
- self.do_generate_vs_stat("0")
-
- print("Generating todos...")
- self.do_generate_todo(f"{local_lib_path}")
-
- print("Generating local playlists...")
- self.do_generate_local_playlists(f"{local_lib_path}")
+ self.do_map_local_to_spotify()
return True
+ @staticmethod
def do_exit(self, arg):
- """Exit the command loop: exit"""
- print("Goodbye!")
return True
if __name__ == '__main__':
try:
- Interpreter().cmdloop()
+ Interpreter().do_run("")
+ # Interpreter().cmdloop()
+
except KeyboardInterrupt as kb:
- print("process terminated")
\ No newline at end of file
+ print("process terminated")
diff --git a/env b/old/env
similarity index 100%
rename from env
rename to old/env
diff --git a/old/main.py b/old/main.py
new file mode 100644
index 0000000..10d378a
--- /dev/null
+++ b/old/main.py
@@ -0,0 +1,234 @@
+from DataBase import *
+from LocalLibraryIndexer import update_local_library
+from LinkerPatternGenerator import run_pattern_generators
+from LocalPLaylistGenerator import generate_playlists
+
+from SpotifyWebAPI import fetch_data, update_access_token
+import cmd
+
+local_library_path = "/home/auser/Music/"
+rel_autogen_dir = "0autogen"
+
+playlists = get_data('playlistTracks')
+top_artists = get_data('top_artists')
+top_tracks = get_data('top_tracks')
+user_tracks = get_data('tracks')
+link_pattern = get_data('link_pattern_spotify')
+local_library = get_data("local_library")
+user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
+
+
+def get_playlist_names(pl):
+ return [name for name, items in pl.items()]
+
+
+def print_playlist_tracks(pl_id: int):
+ pls = list(playlists)
+ name = pls[pl_id]
+ pl = playlists[name]
+ print(name)
+ for track in pl:
+ track_data = track['track']
+ track_name = track_data['name']
+ artists = get_artists_str(track_data['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_top_artists():
+ for artist in top_artists:
+ name = artist['name']
+ print(f" '{name}'")
+
+
+def print_top_tracks():
+ for track in top_tracks:
+ track_name = track['name']
+ artists = get_artists_str(track['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_stats():
+ print(f" tracks - {len(user_tracks)}")
+ print(f" playlists - {len(playlists)}")
+
+ for name, tracks in playlists.items():
+ print(f" '{name}' - {len(tracks)}")
+
+ print(f" top tracks - {len(top_tracks)}")
+ print(f" top artists - {len(top_artists)}")
+
+
+def print_tracks():
+ track_idx = 0
+ for trackItem in user_tracks:
+ track = trackItem['track']
+ print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
+ track_idx += 1
+
+
+def sort_remote_tracks_by_popularity(track_ids):
+ out = []
+ for top_track in top_tracks:
+ if top_track['id'] in track_ids:
+ out.append(top_track['id'])
+ return out
+
+
+def print_link_stats(link_pattern):
+ resolved_reversed = {}
+ resolved = []
+ unresolved_local = []
+ unresolved_remote = []
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ resolved.append(local_id)
+ resolved_reversed[links['items'][0]] = local_id
+ else:
+ unresolved_local.append(local_id)
+
+ for track in user_tracks:
+ track_id = track['track']['id']
+ if track_id not in resolved_reversed:
+ unresolved_remote.append(track_id)
+
+ print(f"Local Tracks: {len(local_library)}")
+ print(f"Remote Tracks: {len(user_tracks)}")
+
+ unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
+ unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+
+ print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
+ index = 0
+ for remote_id in unresolved_remote:
+ remote_track = user_tracks_by_id[remote_id]
+ print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ print(f"\nResolved tracks: {len(resolved)}")
+ index = 0
+ for link in resolved:
+ remote_id = link_pattern[link]['items'][0]
+ local_track = local_library[link]
+ remote_track = user_tracks_by_id[remote_id]
+
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
+ print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+
+class Interpreter(cmd.Cmd):
+ intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
+ prompt = "(spotify) "
+
+ def do_list_stat_vs_types(self, arg):
+ """prints available linking patterns"""
+ print(f"0 - fuzzy tags")
+ print(f"1 - spotify")
+ print(f"2 - fuzzy")
+
+ def do_stat_local_library_vs_spotify(self, arg):
+ """compares local library and remote with the link pattern generated by the generate_pattern_command"""
+ print_link_stats(link_pattern)
+
+ def do_playlists(self, arg):
+ """prints user playlists"""
+ pl_id = 0
+ for pl in get_playlist_names(playlists):
+ print(f"{pl_id} - {pl}")
+ pl_id += 1
+
+ def do_playlist_tracks(self, arg):
+ """prints playlist [name]"""
+ try:
+ pl_id = int(arg.split()[0])
+ print_playlist_tracks(pl_id)
+ except IndexError:
+ print("Invalid input")
+
+ def do_top_artists(self, arg):
+ """Top artists"""
+ print_top_artists()
+
+ def do_top_tracks(self, arg):
+ """Top tracks"""
+ print_top_tracks()
+
+ def do_stat(self, arg):
+ """Print stats"""
+ print_stats()
+
+ def do_tracks(self, arg):
+ """Print tracks"""
+ print_tracks()
+
+ def do_generate_local_playlists(self, arg):
+ """generates local playlists based on the vs comparison type"""
+ generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
+
+
+ def do_index_local_library(self, arg):
+ """find local tracks and fetches the data from them for further analysis"""
+ global local_library
+ update_local_library(local_library_path)
+ local_library = get_data("local_library")
+
+ def do_generate_vs_stat(self, arg):
+ """compares local and remote library and generates the vs stats"""
+
+ global link_pattern
+ run_pattern_generators()
+ link_pattern = get_data('link_pattern_spotify')
+
+ def do_fetch(self, arg):
+ """Fetch all spotify data"""
+ try:
+ update_access_token()
+ fetch_data()
+
+ global playlists, top_artists, top_tracks, user_tracks
+
+ playlists = get_data('playlistTracks')
+ top_artists = get_data('top_artists')
+ top_tracks = get_data('top_tracks')
+ user_tracks = get_data('tracks')
+
+ except ValueError:
+ print("Can not fetch the data.")
+
+ def do_generate_todo(self, arg):
+ """generates todos for the pirates"""
+ print("sorry not implemented yet.")
+
+ def do_run_all_default(self, arg):
+ """runs general flow"""
+
+ local_lib_path = "/home/auser/Music/"
+ print("Fetching spotify data...")
+ self.do_fetch([])
+
+ print("Fetching local data...")
+ self.do_index_local_library(f"{local_lib_path}/organized")
+
+ print("Generating vs patterns...")
+ self.do_generate_vs_stat("0")
+
+ print("Generating todos...")
+ self.do_generate_todo(f"{local_lib_path}")
+
+ print("Generating local playlists...")
+ self.do_generate_local_playlists(f"{local_lib_path}")
+
+ return True
+
+ def do_exit(self, arg):
+ """Exit the command loop: exit"""
+ print("Goodbye!")
+ return True
+
+
+if __name__ == '__main__':
+ try:
+ Interpreter().cmdloop()
+ except KeyboardInterrupt as kb:
+ print("process terminated")
\ No newline at end of file
diff --git a/src/Library.py b/src/Library.py
new file mode 100644
index 0000000..dc2efa3
--- /dev/null
+++ b/src/Library.py
@@ -0,0 +1,159 @@
+from __future__ import annotations
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Any
+
+
+# =========================
+# Core models
+# =========================
+
+class Track:
+ def __init__(
+ self,
+ *,
+ id: str,
+ title: str,
+ artists: List[str],
+ album: Optional[str] = None,
+ duration_ms: Optional[int] = None,
+ added_at: Optional[datetime] = None,
+ spotify: Optional["SpotifyData"] = None,
+ local: Optional["LocalData"] = None,
+ ):
+ self.id = id
+ self.title = title
+ self.artists = artists
+ self.album = album
+ self.duration_ms = duration_ms
+ self.added_at = added_at
+ self.spotify = spotify
+ self.local = local
+
+
+class Playlist:
+ def __init__(
+ self,
+ *,
+ name: str,
+ tracks: Optional[List[Track]] = None,
+ created_at: Optional[datetime] = None,
+ description: Optional[str] = None,
+ spotify: Optional["SpotifyData"] = None,
+ ):
+ self.name = name
+ self.tracks = tracks or []
+ self.created_at = created_at
+ self.description = description
+ self.spotify = spotify
+
+
+class Album:
+ def __init__(
+ self,
+ *,
+ title: str,
+ release_date: Optional[str] = None,
+ tracks: Optional[List[Track]] = None,
+ spotify: Optional["SpotifyData"] = None,
+ ):
+ self.title = title
+ self.release_date = release_date
+ self.tracks = tracks or []
+ self.spotify = spotify
+
+
+class TrackList:
+ def __init__(self, *, tracks: Optional[List[Track]] = None):
+ self.tracks = tracks or []
+
+ def total_duration_ms(self) -> int:
+ return sum(t.duration_ms or 0 for t in self.tracks)
+
+
+# =========================
+# Spotify models
+# =========================
+
+class SpotifyArtist:
+ def __init__(self, *, id: str, name: str, uri: str, url: str):
+ self.id = id
+ self.name = name
+ self.uri = uri
+ self.url = url
+
+
+class SpotifyAlbum:
+ def __init__(
+ self,
+ *,
+ id: str,
+ name: str,
+ release_date: str,
+ total_tracks: int,
+ uri: str,
+ url: str,
+ ):
+ self.id = id
+ self.name = name
+ self.release_date = release_date
+ self.total_tracks = total_tracks
+ self.uri = uri
+ self.url = url
+
+
+class SpotifyData:
+ def __init__(
+ self,
+ *,
+ id: str,
+ uri: str,
+ url: str,
+ href: str,
+ file_path: Optional[Path] = None,
+ popularity: Optional[int] = None,
+ is_playable: Optional[bool] = None,
+ is_local: Optional[bool] = None,
+ artists: Optional[List[SpotifyArtist]] = None,
+ album: Optional[SpotifyAlbum] = None,
+ external_ids: Optional[dict[str, str]] = None,
+ ):
+ self.id = id
+ self.uri = uri
+ self.url = url
+ self.href = href
+ self.file_path = file_path
+ self.popularity = popularity
+ self.is_playable = is_playable
+ self.is_local = is_local
+ self.artists = artists or []
+ self.album = album
+ self.external_ids = external_ids or {}
+
+class LocalData:
+ def __init__(
+ self,
+ *,
+ path: str,
+ ):
+ self.path = path
+
+# =========================
+# Library container
+# =========================
+
+class Library:
+ def __init__(
+ self,
+ *,
+ playlists: Optional[List[Playlist]] = None,
+ tracks: Optional[List[Track]] = None,
+ tracklists: Optional[List[TrackList]] = None,
+ albums: Optional[List[Album]] = None,
+ ):
+ self.playlists = playlists or []
+ self.tracks = tracks or []
+ self.tracklists = tracklists or []
+ self.albums = albums or []
+
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
new file mode 100644
index 0000000..06ea57b
--- /dev/null
+++ b/src/LibrarySerializer.py
@@ -0,0 +1,199 @@
+
+from src.Library import *
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Any
+
+def _dt_to_str(dt: Optional[datetime]) -> Optional[str]:
+ return dt.isoformat() if dt else None
+
+
+def _str_to_dt(s: Optional[str]) -> Optional[datetime]:
+ return datetime.fromisoformat(s) if s else None
+
+
+def _path_to_str(p: Optional[Path]) -> Optional[str]:
+ return str(p) if p else None
+
+
+def _str_to_path(s: Optional[str]) -> Optional[Path]:
+ return Path(s) if s else None
+
+
+class LibrarySaver:
+ def __init__(self, library: Library):
+ self.library = library
+
+ def save(self, path: Path) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", encoding="utf-8") as f:
+ json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "tracks": [self._track_to_dict(t) for t in self.library.tracks],
+ "playlists": [self._playlist_to_dict(p) for p in self.library.playlists],
+ "tracklists": [self._tracklist_to_dict(tl) for tl in self.library.tracklists],
+ "albums": [self._album_to_dict(a) for a in self.library.albums],
+ }
+
+ def _track_to_dict(self, t: Track) -> dict[str, Any]:
+ return {
+ "id": t.id,
+ "title": t.title,
+ "artists": t.artists,
+ "album": t.album,
+ "duration_ms": t.duration_ms,
+ "added_at": _dt_to_str(t.added_at),
+ "spotify": self._spotify_to_dict(t.spotify),
+ "local": self._local_to_dict(t.local),
+ }
+
+ def _playlist_to_dict(self, p: Playlist) -> dict[str, Any]:
+ return {
+ "name": p.name,
+ "tracks": [t.id for t in p.tracks],
+ "created_at": _dt_to_str(p.created_at),
+ "description": p.description,
+ "spotify": self._spotify_to_dict(p.spotify),
+ }
+
+ def _tracklist_to_dict(self, tl: TrackList) -> dict[str, Any]:
+ return {"tracks": [t.id for t in tl.tracks]}
+
+ def _album_to_dict(self, a: Album) -> dict[str, Any]:
+ return {
+ "title": a.title,
+ "release_date": a.release_date,
+ "tracks": [t.id for t in a.tracks],
+ "spotify": self._spotify_to_dict(a.spotify),
+ }
+
+ def _local_to_dict(self, local):
+ if not local:
+ return None
+ return {"path": local.path}
+
+ def _spotify_to_dict(self, s: Optional[SpotifyData]) -> Optional[dict[str, Any]]:
+ if not s:
+ return None
+ return {
+ "id": s.id,
+ "uri": s.uri,
+ "url": s.url,
+ "href": s.href,
+ "file_path": _path_to_str(s.file_path),
+ "popularity": s.popularity,
+ "is_playable": s.is_playable,
+ "is_local": s.is_local,
+ "external_ids": s.external_ids,
+ "artists": [{"id": a.id, "name": a.name, "uri": a.uri, "url": a.url} for a in s.artists],
+ "album": {
+ "id": s.album.id,
+ "name": s.album.name,
+ "release_date": s.album.release_date,
+ "total_tracks": s.album.total_tracks,
+ "uri": s.album.uri,
+ "url": s.album.url,
+ } if s.album else None,
+ }
+
+
+class LibraryLoader:
+ def __init__(self):
+ # Map of track id -> Track object
+ self.track_map: dict[str, Track] = {}
+
+ def load(self, path: Path) -> Library:
+ with path.open("r", encoding="utf-8") as f:
+ data = json.load(f)
+
+ # Load tracks first
+ tracks = []
+ for t in data.get("tracks", []):
+ track = self._track_from_dict(t)
+ tracks.append(track)
+ self.track_map[track.id] = track
+
+ # Load playlists using track_map
+ playlists = []
+ for p in data.get("playlists", []):
+ pl_tracks = []
+ for tid in p.get("tracks", []):
+ if tid not in self.track_map:
+ raise KeyError(f"Track ID '{tid}' not found in track map")
+ pl_tracks.append(self.track_map[tid])
+
+ playlists.append(Playlist(
+ name=p["name"],
+ tracks=pl_tracks,
+ created_at=_str_to_dt(p.get("created_at")),
+ description=p.get("description"),
+ spotify=self._spotify_from_dict(p.get("spotify")),
+ ))
+
+
+ # Load tracklists
+ tracklists = []
+ for tl in data.get("tracklists", []):
+ tl_tracks = []
+ for tid in tl.get("tracks", []):
+ if tid not in self.track_map:
+ raise KeyError(f"Track ID '{tid}' not found in track map (tracklist)")
+ tl_tracks.append(self.track_map[tid])
+ tracklists.append(TrackList(tracks=tl_tracks))
+
+ # Load albums
+ albums = []
+ for a in data.get("albums", []):
+ alb_tracks = []
+ for tid in a.get("tracks", []):
+ if tid not in self.track_map:
+ raise KeyError(f"Track ID '{tid}' not found in track map (album '{a.get('title')}')")
+ alb_tracks.append(self.track_map[tid])
+ albums.append(Album(
+ title=a["title"],
+ release_date=a.get("release_date"),
+ tracks=alb_tracks,
+ spotify=self._spotify_from_dict(a.get("spotify")),
+ ))
+
+ return Library(playlists=playlists, tracks=tracks, tracklists=tracklists, albums=albums)
+
+ def _track_from_dict(self, d: dict[str, Any]) -> Track:
+ return Track(
+ id=d["id"],
+ title=d["title"],
+ artists=d["artists"],
+ album=d.get("album"),
+ duration_ms=d.get("duration_ms"),
+ added_at=_str_to_dt(d.get("added_at")),
+ spotify=self._spotify_from_dict(d.get("spotify")),
+ local=self._local_from_dict(d.get("local")),
+ )
+
+ def _local_from_dict(self, d):
+ if not d:
+ return None
+ return LocalData(
+ path=d["path"],
+ )
+
+ def _spotify_from_dict(self, d: Optional[dict[str, Any]]) -> Optional[SpotifyData]:
+ if not d:
+ return None
+ return SpotifyData(
+ id=d["id"],
+ uri=d["uri"],
+ url=d["url"],
+ href=d["href"],
+ file_path=_str_to_path(d.get("file_path")),
+ popularity=d.get("popularity"),
+ is_playable=d.get("is_playable"),
+ is_local=d.get("is_local"),
+ external_ids=d.get("external_ids"),
+ artists=[SpotifyArtist(**a) for a in d.get("artists", [])],
+ album=SpotifyAlbum(**d["album"]) if d.get("album") else None,
+ )
diff --git a/LinkerPatternGenerator.py b/src/MapLocalToSpotify.py
similarity index 71%
rename from LinkerPatternGenerator.py
rename to src/MapLocalToSpotify.py
index 8576da8..dae5252 100644
--- a/LinkerPatternGenerator.py
+++ b/src/MapLocalToSpotify.py
@@ -1,47 +1,52 @@
-from DataBase import *
-from SpotifyWebAPI import find_song, update_access_token
from rapidfuzz import fuzz
from tqdm import tqdm
import requests
-def spotify_pattern_generator():
- update_access_token()
-
- local_library = get_data("local_library")
- spotify_library = get_data("tracks")
+from src.Library import *
+from src.helpers import *
+from src.spotify.SpotifyWebAPI import find_song, update_access_token
- def find_local_songs_on_spotify(local_lib):
+def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
+
+ set_workdir(output_dir)
+
+ update_access_token(client_id, client_secret)
+
+ def find_local_songs_on_spotify():
search_log = []
- total = len(local_library)
+ total = len(local_lib.tracks)
found_tracks_map = {}
with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
- for track_id, track in local_lib.items():
- parts = [track["name"], track["artist"], track.get("album")]
+ for track in local_lib.tracks:
+
+ parts = [track.title, track.artists[0] if len(track.artists) else "", track.album]
search_pattern = " ".join(p for p in parts if p)
if search_pattern == "":
- print(f"cannot generate search pattern for the track - {track["path"]}")
+ print(f"cannot generate search pattern for the track - {track.local.path}")
pbar.update(1)
continue
- found_tracks = find_song(search_pattern)[0:1]
- found_tracks_map[track_id] = found_tracks
- search_log.append({"path" : track["path"], "search_pattern": search_pattern, "result" : found_tracks_map[track_id]})
+ found_tracks = find_song(search_pattern)[0:10]
+ found_tracks_map[track.id] = found_tracks
+ search_log.append({"path" : track.local.path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
pbar.update(1)
save_data(search_log, "spotify_search_log")
+ save_data(found_tracks_map, "spotify_found_map")
return found_tracks_map
- spotify_found_tracks = find_local_songs_on_spotify(local_library)
- spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library}
+ spotify_found_tracks = find_local_songs_on_spotify()
+
+ spotify_user_track = {track.id for track in spotify_lib.tracks}
spotify_pattern = {}
for local_id, found_items in spotify_found_tracks.items():
- spotify_pattern[local_id] = {"score": 0.0, "items": []}
+ spotify_pattern[local_id] = {"score": 1.0, "items": []}
if not len(found_items):
continue
@@ -50,14 +55,8 @@ def spotify_pattern_generator():
spotify_track_id = found_item['id']
if spotify_track_id in spotify_user_track:
spotify_pattern[local_id]['items'].append(spotify_track_id)
- spotify_user_track[spotify_track_id]['local_mappings'].append(local_id)
- for spotify_id, track_local_mappings in spotify_user_track.items():
- score = len(track_local_mappings['local_mappings'])
- for local_id in track_local_mappings['local_mappings']:
- spotify_pattern[local_id]['score'] = 1 / score
-
- save_data(spotify_pattern, "link_pattern_spotify")
+ save_data(spotify_pattern, "local_to_spotify_id_map")
def fuzzy_pattern_generator():
@@ -126,7 +125,7 @@ def fuzzy_tag_pattern_generator():
ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id']))
filtered_ratios = []
- threshold = 0.8
+ threshold = 0.81
for ratio in ratios:
if all([val > threshold for val in ratio[0]]):
filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1]))
@@ -138,12 +137,4 @@ def fuzzy_tag_pattern_generator():
mappings[local_track_id] = mapping
pbar.update(1)
- save_data(mappings, "link_pattern_tags")
-
-
-def run_pattern_generators(type_id = 0):
- spotify_pattern_generator()
-
-
-if __name__ == "__main__":
- run_pattern_generators()
+ save_data(mappings, "link_pattern_tags")
\ No newline at end of file
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
new file mode 100644
index 0000000..c40bebb
--- /dev/null
+++ b/src/StatGenerator.py
@@ -0,0 +1,42 @@
+
+def print_link_stats(link_pattern):
+ resolved_reversed = {}
+ resolved = []
+ unresolved_local = []
+ unresolved_remote = []
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ resolved.append(local_id)
+ resolved_reversed[links['items'][0]] = local_id
+ else:
+ unresolved_local.append(local_id)
+
+ for track in user_tracks:
+ track_id = track['track']['id']
+ if track_id not in resolved_reversed:
+ unresolved_remote.append(track_id)
+
+ print(f"Local Tracks: {len(local_library)}")
+ print(f"Remote Tracks: {len(user_tracks)}")
+
+ unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
+ unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+
+ print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
+ index = 0
+ for remote_id in unresolved_remote:
+ remote_track = user_tracks_by_id[remote_id]
+ print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ print(f"\nResolved tracks: {len(resolved)}")
+ index = 0
+ for link in resolved:
+ remote_id = link_pattern[link]['items'][0]
+ local_track = local_library[link]
+ remote_track = user_tracks_by_id[remote_id]
+
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
+ print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
\ No newline at end of file
diff --git a/DataBase.py b/src/helpers.py
similarity index 85%
rename from DataBase.py
rename to src/helpers.py
index c0f230b..8f81bbf 100644
--- a/DataBase.py
+++ b/src/helpers.py
@@ -3,6 +3,13 @@ import os
data_dir = 'prefetched'
+def set_workdir(dir):
+ global data_dir
+ data_dir = str(dir)
+
+def get_workdir():
+ global data_dir
+ return data_dir
class SongTags:
def __init__(self, title='', artist='', album=''):
@@ -40,10 +47,9 @@ def save_data(data, name):
file_path = os.path.join(data_dir, f"{name}.json")
with open(file_path, 'w') as file:
json.dump(data, file, indent=4)
- return f"Data saved to {name}.json."
-def get_artists_str(artists_pack):
- track_artists = [artist['name'] for artist in artists_pack]
- artists = " ".join(track_artists)
- return artists
+ print(f"Data saved to {file_path}.")
+
+
+
diff --git a/ParseItunesToJson.py b/src/itunes/ParseItunesToJson.py
similarity index 100%
rename from ParseItunesToJson.py
rename to src/itunes/ParseItunesToJson.py
diff --git a/LocalLibraryIndexer.py b/src/local/LocalLibraryIndexer.py
similarity index 95%
rename from LocalLibraryIndexer.py
rename to src/local/LocalLibraryIndexer.py
index eecdd58..d3a7ce2 100644
--- a/LocalLibraryIndexer.py
+++ b/src/local/LocalLibraryIndexer.py
@@ -1,18 +1,20 @@
import os
import fnmatch
-
import mutagen
-
-from DataBase import *
import eyed3
from mutagen.flac import FLAC
+from src.helpers import *
+
music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
exclude_directories = ("*mary--*", "*example-word*")
-def update_local_library(root_dir):
+def update_local_library(root_dir, output_dir):
+
+ set_workdir(output_dir / "local")
+
old_library = get_data("local_library")
known_paths = {track['path']: track_id for track_id, track in old_library.items()}
new_library = {}
@@ -74,7 +76,3 @@ def update_local_library(root_dir):
track['name'] = track['path']
save_data(new_library, "local_library")
-
-
-if __name__ == "__main__":
- update_local_library("/home/auser/Music/")
diff --git a/LocalPLaylistGenerator.py b/src/local/LocalPLaylistGenerator.py
similarity index 100%
rename from LocalPLaylistGenerator.py
rename to src/local/LocalPLaylistGenerator.py
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
new file mode 100644
index 0000000..4991ede
--- /dev/null
+++ b/src/local/ParseLocal.py
@@ -0,0 +1,40 @@
+from datetime import datetime
+from typing import Any, Iterable
+from pathlib import Path
+import json
+
+from src.Library import *
+
+class LocalParser:
+ def __init__(self):
+ pass
+
+ def parse_track_item(self, id, track: dict[str, Any]) -> Track:
+ track = Track(
+ title=track["name"],
+ artists=[track.get("artist", [])],
+ album=track["album"] if track.get("album") else None,
+ duration_ms=None,
+ added_at=None,
+ spotify=None,
+ id=id,
+ local=LocalData(path=track["path"])
+ )
+ return track
+
+
+ def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
+ return [self.parse_track_item(id, track) for id, track in items.items()]
+
+ def build_library_from_tracks(self, tracks_content_json: dict[str, list[dict[str, Any]]]) -> Library:
+ library = Library()
+
+ tracks = self.parse_all_tracks(tracks_content_json)
+ library.tracklists.append(TrackList(tracks=list(tracks)))
+ library.tracks.extend(tracks)
+
+ return library
+
+ def parse(self, tracks_file: Path) -> Library:
+ tracks_content = json.loads(tracks_file.read_text())
+ return self.build_library_from_tracks(tracks_content)
diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py
new file mode 100644
index 0000000..535caa0
--- /dev/null
+++ b/src/spotify/ParseSpotify.py
@@ -0,0 +1,139 @@
+from datetime import datetime
+from typing import Any, Iterable
+from pathlib import Path
+import json
+
+from src.Library import *
+
+
+def get_artists_str(artists_pack):
+ track_artists = [artist['name'] for artist in artists_pack]
+ artists = " ".join(track_artists)
+ return artists
+
+class Parser:
+ def __init__(self):
+ # state: Spotify track ID -> Track object
+ self.track_map: dict[str, Track] = {}
+
+ # -------------------------
+ # Low-level helpers
+ # -------------------------
+ @staticmethod
+ def _parse_spotify_datetime(value: str | None) -> datetime | None:
+ if not value:
+ return None
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+
+ @staticmethod
+ def _parse_spotify_artist(a: dict[str, Any]) -> SpotifyArtist:
+ return SpotifyArtist(
+ id=a["id"],
+ name=a["name"],
+ uri=a["uri"],
+ url=a["external_urls"]["spotify"],
+ )
+
+ @staticmethod
+ def _parse_spotify_album(a: dict[str, Any]) -> SpotifyAlbum:
+ return SpotifyAlbum(
+ id=a["id"],
+ name=a["name"],
+ release_date=a.get("release_date"),
+ total_tracks=a.get("total_tracks", 0),
+ uri=a["uri"],
+ url=a["external_urls"]["spotify"],
+ )
+
+ def _parse_spotify_data(self, track_json: dict[str, Any]) -> SpotifyData:
+ return SpotifyData(
+ id=track_json["id"],
+ uri=track_json["uri"],
+ url=track_json["external_urls"]["spotify"],
+ href=track_json["href"],
+ popularity=track_json.get("popularity"),
+ is_playable=track_json.get("is_playable"),
+ is_local=track_json.get("is_local"),
+ external_ids=track_json.get("external_ids", {}),
+ artists=[self._parse_spotify_artist(a) for a in track_json.get("artists", [])],
+ album=self._parse_spotify_album(track_json["album"])
+ if track_json.get("album")
+ else None,
+ )
+
+ # -------------------------
+ # Track parsers
+ # -------------------------
+ def parse_track_item(self, item: dict[str, Any]) -> Track:
+ """
+ Parses a single track item and returns Track object.
+ Reuses existing Track from track_map if already parsed.
+ """
+ t = item["track"]
+ track_id = t["id"]
+
+ # return existing track if already parsed
+ if track_id in self.track_map:
+ return self.track_map[track_id]
+
+ track = Track(
+ title=t["name"],
+ artists=[a["name"] for a in t.get("artists", [])],
+ album=t["album"]["name"] if t.get("album") else None,
+ duration_ms=t.get("duration_ms"),
+ added_at=self._parse_spotify_datetime(item.get("added_at")),
+ spotify=self._parse_spotify_data(t),
+ id=self._parse_spotify_data(t).id
+ )
+
+ self.track_map[track_id] = track
+ return track
+
+ def parse_track_items(self, items: Iterable[dict[str, Any]]) -> list[Track]:
+ return [self.parse_track_item(i) for i in items if i.get("track")]
+
+ # -------------------------
+ # Playlist parsers
+ # -------------------------
+ def parse_playlist_content(self, playlist_name: str, items: list[dict[str, Any]]) -> Playlist:
+ tracks = self.parse_track_items(items)
+ return Playlist(name=playlist_name, tracks=tracks)
+
+ def parse_all_playlist_content(self, data: dict[str, list[dict[str, Any]]]) -> list[Playlist]:
+ playlists: list[Playlist] = []
+ for name, items in data.items():
+ playlists.append(self.parse_playlist_content(name, items))
+ return playlists
+
+ def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
+ return [self.parse_track_item(i) for i in items]
+
+ # -------------------------
+ # Library builder
+ # -------------------------
+ def build_library_from_playlist_content(
+ self, playlist_content_json: dict[str, list[dict[str, Any]]],
+ tracks_content_json: dict[str, list[dict[str, Any]]]
+ ) -> Library:
+ library = Library()
+
+ self.parse_all_tracks(tracks_content_json)
+
+ library.tracklists.append(TrackList(tracks=list((self.track_map.values()))))
+
+ library.playlists = self.parse_all_playlist_content(playlist_content_json)
+ library.tracks.extend(self.track_map.values())
+
+ return library
+
+ # -------------------------
+ # Convenience method
+ # -------------------------
+ def parse(
+ self,
+ playlist_content_file: Path,
+ tracks_file: Path,
+ ) -> Library:
+ playlist_content = json.loads(playlist_content_file.read_text())
+ tracks_content = json.loads(tracks_file.read_text())
+ return self.build_library_from_playlist_content(playlist_content, tracks_content)
diff --git a/SpotifyAuthenticator.py b/src/spotify/SpotifyAuthenticator.py
similarity index 88%
rename from SpotifyAuthenticator.py
rename to src/spotify/SpotifyAuthenticator.py
index e06d343..5e27f7d 100644
--- a/SpotifyAuthenticator.py
+++ b/src/spotify/SpotifyAuthenticator.py
@@ -8,22 +8,11 @@ import threading
import time
import os.path
-client_id = '3b4db45bb66b45e9a2532989c8034332'
+client_id = None
client_secret = None
token = None
-def resolve_client_secret():
- global client_secret
- if os.path.exists('secret'):
- with open('secret', 'r') as file:
- client_secret = file.readline().strip()
- else:
- client_secret = input("Enter the application secret: ")
- with open('secret', 'w') as file:
- file.write(client_secret)
-
-
def retrieve_web_api_token(authorization_code):
global token
@@ -103,8 +92,13 @@ def open_authenticate_page():
webbrowser.open(auth_url)
-def authenticate():
- resolve_client_secret()
+def authenticate(id, secret):
+
+ global client_secret
+ global client_id
+
+ client_secret=secret
+ client_id=id
redirect_server = threading.Thread(target=run_redirect_server)
redirect_server.start()
diff --git a/SpotifyWebAPI.py b/src/spotify/SpotifyWebAPI.py
similarity index 91%
rename from SpotifyWebAPI.py
rename to src/spotify/SpotifyWebAPI.py
index 71ec2f7..5ddc12f 100644
--- a/SpotifyWebAPI.py
+++ b/src/spotify/SpotifyWebAPI.py
@@ -1,8 +1,10 @@
import requests
from PIL import Image
from io import BytesIO
-from SpotifyAuthenticator import authenticate
-from DataBase import *
+
+from src.spotify.SpotifyAuthenticator import authenticate
+
+from src.helpers import *
FETCH_STEP = 50
token = ''
@@ -142,7 +144,7 @@ def save_image_from_url(url, name, image_dir="playlist_covers"):
image = Image.open(BytesIO(response.content))
- directory = os.path.join(data_dir, image_dir)
+ directory = os.path.join(get_workdir(), image_dir)
os.makedirs(directory, exist_ok=True)
file_path = os.path.join(directory, f"{name}.jpg")
print(f"Image saved {file_path}")
@@ -169,9 +171,9 @@ def load_artworks():
load_playlist_covers()
-def update_access_token():
+def update_access_token(id, secret):
global token
- token = authenticate()
+ token = authenticate(id, secret)
def fetch_data():
@@ -184,7 +186,11 @@ def fetch_data():
load_artworks()
+def fetch_all(id, secret, workdir):
+ set_workdir(workdir / "spotify")
+ update_access_token(id, secret)
+ fetch_data()
if __name__ == "__main__":
- update_access_token()
+ update_access_token("", "")
fetch_data()
From a620e7a4f565e0f795af1b2450416294d6b043e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 4 Jan 2026 19:25:42 +0300
Subject: [PATCH 08/30] refactor initial not fully recovered
---
Env.py | 7 +
main.py | 241 +++---------------
env => old/env | 0
old/main.py | 234 +++++++++++++++++
src/Library.py | 159 ++++++++++++
src/LibrarySerializer.py | 199 +++++++++++++++
.../MapLocalToSpotify.py | 61 ++---
src/StatGenerator.py | 42 +++
DataBase.py => src/helpers.py | 16 +-
.../itunes/ParseItunesToJson.py | 0
.../local/LocalLibraryIndexer.py | 14 +-
.../local/LocalPLaylistGenerator.py | 0
src/local/ParseLocal.py | 40 +++
src/spotify/ParseSpotify.py | 139 ++++++++++
.../spotify/SpotifyAuthenticator.py | 22 +-
.../spotify/SpotifyWebAPI.py | 18 +-
16 files changed, 925 insertions(+), 267 deletions(-)
create mode 100644 Env.py
rename env => old/env (100%)
create mode 100644 old/main.py
create mode 100644 src/Library.py
create mode 100644 src/LibrarySerializer.py
rename LinkerPatternGenerator.py => src/MapLocalToSpotify.py (71%)
create mode 100644 src/StatGenerator.py
rename DataBase.py => src/helpers.py (85%)
rename ParseItunesToJson.py => src/itunes/ParseItunesToJson.py (100%)
rename LocalLibraryIndexer.py => src/local/LocalLibraryIndexer.py (95%)
rename LocalPLaylistGenerator.py => src/local/LocalPLaylistGenerator.py (100%)
create mode 100644 src/local/ParseLocal.py
create mode 100644 src/spotify/ParseSpotify.py
rename SpotifyAuthenticator.py => src/spotify/SpotifyAuthenticator.py (88%)
rename SpotifyWebAPI.py => src/spotify/SpotifyWebAPI.py (91%)
diff --git a/Env.py b/Env.py
new file mode 100644
index 0000000..6549c6b
--- /dev/null
+++ b/Env.py
@@ -0,0 +1,7 @@
+from pathlib import Path
+
+local_library_path = Path("/home/auser/Music/")
+work_dir = Path("./work_dir/new")
+
+client_id = '***REDACTED***'
+client_secret = "***REDACTED***"
diff --git a/main.py b/main.py
index 10d378a..32682e3 100644
--- a/main.py
+++ b/main.py
@@ -1,234 +1,77 @@
-from DataBase import *
-from LocalLibraryIndexer import update_local_library
-from LinkerPatternGenerator import run_pattern_generators
-from LocalPLaylistGenerator import generate_playlists
-from SpotifyWebAPI import fetch_data, update_access_token
import cmd
-local_library_path = "/home/auser/Music/"
-rel_autogen_dir = "0autogen"
+from Env import client_id, client_secret, local_library_path
+from Env import work_dir
-playlists = get_data('playlistTracks')
-top_artists = get_data('top_artists')
-top_tracks = get_data('top_tracks')
-user_tracks = get_data('tracks')
-link_pattern = get_data('link_pattern_spotify')
-local_library = get_data("local_library")
-user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
+from src.LibrarySerializer import LibrarySaver, LibraryLoader
+from src.spotify.SpotifyWebAPI import fetch_all
+from src.spotify.ParseSpotify import Parser
-def get_playlist_names(pl):
- return [name for name, items in pl.items()]
+from src.local.LocalLibraryIndexer import update_local_library
+from src.local.ParseLocal import LocalParser
-
-def print_playlist_tracks(pl_id: int):
- pls = list(playlists)
- name = pls[pl_id]
- pl = playlists[name]
- print(name)
- for track in pl:
- track_data = track['track']
- track_name = track_data['name']
- artists = get_artists_str(track_data['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_top_artists():
- for artist in top_artists:
- name = artist['name']
- print(f" '{name}'")
-
-
-def print_top_tracks():
- for track in top_tracks:
- track_name = track['name']
- artists = get_artists_str(track['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_stats():
- print(f" tracks - {len(user_tracks)}")
- print(f" playlists - {len(playlists)}")
-
- for name, tracks in playlists.items():
- print(f" '{name}' - {len(tracks)}")
-
- print(f" top tracks - {len(top_tracks)}")
- print(f" top artists - {len(top_artists)}")
-
-
-def print_tracks():
- track_idx = 0
- for trackItem in user_tracks:
- track = trackItem['track']
- print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
- track_idx += 1
-
-
-def sort_remote_tracks_by_popularity(track_ids):
- out = []
- for top_track in top_tracks:
- if top_track['id'] in track_ids:
- out.append(top_track['id'])
- return out
-
-
-def print_link_stats(link_pattern):
- resolved_reversed = {}
- resolved = []
- unresolved_local = []
- unresolved_remote = []
-
- for local_id, links in link_pattern.items():
- if len(links['items']):
- resolved.append(local_id)
- resolved_reversed[links['items'][0]] = local_id
- else:
- unresolved_local.append(local_id)
-
- for track in user_tracks:
- track_id = track['track']['id']
- if track_id not in resolved_reversed:
- unresolved_remote.append(track_id)
-
- print(f"Local Tracks: {len(local_library)}")
- print(f"Remote Tracks: {len(user_tracks)}")
-
- unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
- unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
-
- print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
- index = 0
- for remote_id in unresolved_remote:
- remote_track = user_tracks_by_id[remote_id]
- print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
-
- print(f"\nResolved tracks: {len(resolved)}")
- index = 0
- for link in resolved:
- remote_id = link_pattern[link]['items'][0]
- local_track = local_library[link]
- remote_track = user_tracks_by_id[remote_id]
-
- print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
- print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
+from src.MapLocalToSpotify import map_local_to_spotify
class Interpreter(cmd.Cmd):
intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
- prompt = "(spotify) "
+ prompt = "(kuw) "
- def do_list_stat_vs_types(self, arg):
- """prints available linking patterns"""
- print(f"0 - fuzzy tags")
- print(f"1 - spotify")
- print(f"2 - fuzzy")
+ spotify_library = None
+ local_library = None
- def do_stat_local_library_vs_spotify(self, arg):
- """compares local library and remote with the link pattern generated by the generate_pattern_command"""
- print_link_stats(link_pattern)
-
- def do_playlists(self, arg):
- """prints user playlists"""
- pl_id = 0
- for pl in get_playlist_names(playlists):
- print(f"{pl_id} - {pl}")
- pl_id += 1
-
- def do_playlist_tracks(self, arg):
- """prints playlist [name]"""
- try:
- pl_id = int(arg.split()[0])
- print_playlist_tracks(pl_id)
- except IndexError:
- print("Invalid input")
-
- def do_top_artists(self, arg):
- """Top artists"""
- print_top_artists()
-
- def do_top_tracks(self, arg):
- """Top tracks"""
- print_top_tracks()
-
- def do_stat(self, arg):
- """Print stats"""
- print_stats()
-
- def do_tracks(self, arg):
- """Print tracks"""
- print_tracks()
-
- def do_generate_local_playlists(self, arg):
- """generates local playlists based on the vs comparison type"""
- generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
+ @staticmethod
+ def do_fetch_spotify(self):
+ fetch_all(client_id, client_secret, work_dir)
- def do_index_local_library(self, arg):
- """find local tracks and fetches the data from them for further analysis"""
- global local_library
- update_local_library(local_library_path)
- local_library = get_data("local_library")
+ @staticmethod
+ def do_fetch_local():
+ update_local_library(local_library_path, work_dir)
- def do_generate_vs_stat(self, arg):
- """compares local and remote library and generates the vs stats"""
- global link_pattern
- run_pattern_generators()
- link_pattern = get_data('link_pattern_spotify')
+ def do_parse_spotify_library(self):
+ parser = Parser()
+ library_parsed = parser.parse(work_dir / "spotify" / "playlistTracks.json", work_dir / "spotify" / "tracks.json")
+ LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
- def do_fetch(self, arg):
- """Fetch all spotify data"""
- try:
- update_access_token()
- fetch_data()
+ def do_parse_local_library(self):
+ parser = LocalParser()
+ library_parsed = parser.parse(work_dir / "local" / "local_library.json")
+ LibrarySaver(library_parsed).save(work_dir / "local_library.json")
- global playlists, top_artists, top_tracks, user_tracks
+ def do_load_libraries(self):
+ self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
+ self.local_library = LibraryLoader().load(work_dir / "local_library.json")
- playlists = get_data('playlistTracks')
- top_artists = get_data('top_artists')
- top_tracks = get_data('top_tracks')
- user_tracks = get_data('tracks')
+ def do_map_local_to_spotify(self):
+ map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
- except ValueError:
- print("Can not fetch the data.")
- def do_generate_todo(self, arg):
- """generates todos for the pirates"""
- print("sorry not implemented yet.")
+ def do_run(self, arg):
+ # self.do_fetch_local()
+ # self.do_fetch_spotify()
- def do_run_all_default(self, arg):
- """runs general flow"""
+ # self.do_parse_local_library()
+ # self.do_parse_spotify_library()
- local_lib_path = "/home/auser/Music/"
- print("Fetching spotify data...")
- self.do_fetch([])
+ self.do_load_libraries()
- print("Fetching local data...")
- self.do_index_local_library(f"{local_lib_path}/organized")
-
- print("Generating vs patterns...")
- self.do_generate_vs_stat("0")
-
- print("Generating todos...")
- self.do_generate_todo(f"{local_lib_path}")
-
- print("Generating local playlists...")
- self.do_generate_local_playlists(f"{local_lib_path}")
+ self.do_map_local_to_spotify()
return True
+ @staticmethod
def do_exit(self, arg):
- """Exit the command loop: exit"""
- print("Goodbye!")
return True
if __name__ == '__main__':
try:
- Interpreter().cmdloop()
+ Interpreter().do_run("")
+ # Interpreter().cmdloop()
+
except KeyboardInterrupt as kb:
- print("process terminated")
\ No newline at end of file
+ print("process terminated")
diff --git a/env b/old/env
similarity index 100%
rename from env
rename to old/env
diff --git a/old/main.py b/old/main.py
new file mode 100644
index 0000000..10d378a
--- /dev/null
+++ b/old/main.py
@@ -0,0 +1,234 @@
+from DataBase import *
+from LocalLibraryIndexer import update_local_library
+from LinkerPatternGenerator import run_pattern_generators
+from LocalPLaylistGenerator import generate_playlists
+
+from SpotifyWebAPI import fetch_data, update_access_token
+import cmd
+
+local_library_path = "/home/auser/Music/"
+rel_autogen_dir = "0autogen"
+
+playlists = get_data('playlistTracks')
+top_artists = get_data('top_artists')
+top_tracks = get_data('top_tracks')
+user_tracks = get_data('tracks')
+link_pattern = get_data('link_pattern_spotify')
+local_library = get_data("local_library")
+user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
+
+
+def get_playlist_names(pl):
+ return [name for name, items in pl.items()]
+
+
+def print_playlist_tracks(pl_id: int):
+ pls = list(playlists)
+ name = pls[pl_id]
+ pl = playlists[name]
+ print(name)
+ for track in pl:
+ track_data = track['track']
+ track_name = track_data['name']
+ artists = get_artists_str(track_data['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_top_artists():
+ for artist in top_artists:
+ name = artist['name']
+ print(f" '{name}'")
+
+
+def print_top_tracks():
+ for track in top_tracks:
+ track_name = track['name']
+ artists = get_artists_str(track['artists'])
+ print(f" '{track_name}' - '{artists}' ")
+
+
+def print_stats():
+ print(f" tracks - {len(user_tracks)}")
+ print(f" playlists - {len(playlists)}")
+
+ for name, tracks in playlists.items():
+ print(f" '{name}' - {len(tracks)}")
+
+ print(f" top tracks - {len(top_tracks)}")
+ print(f" top artists - {len(top_artists)}")
+
+
+def print_tracks():
+ track_idx = 0
+ for trackItem in user_tracks:
+ track = trackItem['track']
+ print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
+ track_idx += 1
+
+
+def sort_remote_tracks_by_popularity(track_ids):
+ out = []
+ for top_track in top_tracks:
+ if top_track['id'] in track_ids:
+ out.append(top_track['id'])
+ return out
+
+
+def print_link_stats(link_pattern):
+ resolved_reversed = {}
+ resolved = []
+ unresolved_local = []
+ unresolved_remote = []
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ resolved.append(local_id)
+ resolved_reversed[links['items'][0]] = local_id
+ else:
+ unresolved_local.append(local_id)
+
+ for track in user_tracks:
+ track_id = track['track']['id']
+ if track_id not in resolved_reversed:
+ unresolved_remote.append(track_id)
+
+ print(f"Local Tracks: {len(local_library)}")
+ print(f"Remote Tracks: {len(user_tracks)}")
+
+ unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
+ unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+
+ print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
+ index = 0
+ for remote_id in unresolved_remote:
+ remote_track = user_tracks_by_id[remote_id]
+ print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ print(f"\nResolved tracks: {len(resolved)}")
+ index = 0
+ for link in resolved:
+ remote_id = link_pattern[link]['items'][0]
+ local_track = local_library[link]
+ remote_track = user_tracks_by_id[remote_id]
+
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
+ print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+
+class Interpreter(cmd.Cmd):
+ intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
+ prompt = "(spotify) "
+
+ def do_list_stat_vs_types(self, arg):
+ """prints available linking patterns"""
+ print(f"0 - fuzzy tags")
+ print(f"1 - spotify")
+ print(f"2 - fuzzy")
+
+ def do_stat_local_library_vs_spotify(self, arg):
+ """compares local library and remote with the link pattern generated by the generate_pattern_command"""
+ print_link_stats(link_pattern)
+
+ def do_playlists(self, arg):
+ """prints user playlists"""
+ pl_id = 0
+ for pl in get_playlist_names(playlists):
+ print(f"{pl_id} - {pl}")
+ pl_id += 1
+
+ def do_playlist_tracks(self, arg):
+ """prints playlist [name]"""
+ try:
+ pl_id = int(arg.split()[0])
+ print_playlist_tracks(pl_id)
+ except IndexError:
+ print("Invalid input")
+
+ def do_top_artists(self, arg):
+ """Top artists"""
+ print_top_artists()
+
+ def do_top_tracks(self, arg):
+ """Top tracks"""
+ print_top_tracks()
+
+ def do_stat(self, arg):
+ """Print stats"""
+ print_stats()
+
+ def do_tracks(self, arg):
+ """Print tracks"""
+ print_tracks()
+
+ def do_generate_local_playlists(self, arg):
+ """generates local playlists based on the vs comparison type"""
+ generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
+
+
+ def do_index_local_library(self, arg):
+ """find local tracks and fetches the data from them for further analysis"""
+ global local_library
+ update_local_library(local_library_path)
+ local_library = get_data("local_library")
+
+ def do_generate_vs_stat(self, arg):
+ """compares local and remote library and generates the vs stats"""
+
+ global link_pattern
+ run_pattern_generators()
+ link_pattern = get_data('link_pattern_spotify')
+
+ def do_fetch(self, arg):
+ """Fetch all spotify data"""
+ try:
+ update_access_token()
+ fetch_data()
+
+ global playlists, top_artists, top_tracks, user_tracks
+
+ playlists = get_data('playlistTracks')
+ top_artists = get_data('top_artists')
+ top_tracks = get_data('top_tracks')
+ user_tracks = get_data('tracks')
+
+ except ValueError:
+ print("Can not fetch the data.")
+
+ def do_generate_todo(self, arg):
+ """generates todos for the pirates"""
+ print("sorry not implemented yet.")
+
+ def do_run_all_default(self, arg):
+ """runs general flow"""
+
+ local_lib_path = "/home/auser/Music/"
+ print("Fetching spotify data...")
+ self.do_fetch([])
+
+ print("Fetching local data...")
+ self.do_index_local_library(f"{local_lib_path}/organized")
+
+ print("Generating vs patterns...")
+ self.do_generate_vs_stat("0")
+
+ print("Generating todos...")
+ self.do_generate_todo(f"{local_lib_path}")
+
+ print("Generating local playlists...")
+ self.do_generate_local_playlists(f"{local_lib_path}")
+
+ return True
+
+ def do_exit(self, arg):
+ """Exit the command loop: exit"""
+ print("Goodbye!")
+ return True
+
+
+if __name__ == '__main__':
+ try:
+ Interpreter().cmdloop()
+ except KeyboardInterrupt as kb:
+ print("process terminated")
\ No newline at end of file
diff --git a/src/Library.py b/src/Library.py
new file mode 100644
index 0000000..dc2efa3
--- /dev/null
+++ b/src/Library.py
@@ -0,0 +1,159 @@
+from __future__ import annotations
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Any
+
+
+# =========================
+# Core models
+# =========================
+
+class Track:
+ def __init__(
+ self,
+ *,
+ id: str,
+ title: str,
+ artists: List[str],
+ album: Optional[str] = None,
+ duration_ms: Optional[int] = None,
+ added_at: Optional[datetime] = None,
+ spotify: Optional["SpotifyData"] = None,
+ local: Optional["LocalData"] = None,
+ ):
+ self.id = id
+ self.title = title
+ self.artists = artists
+ self.album = album
+ self.duration_ms = duration_ms
+ self.added_at = added_at
+ self.spotify = spotify
+ self.local = local
+
+
+class Playlist:
+ def __init__(
+ self,
+ *,
+ name: str,
+ tracks: Optional[List[Track]] = None,
+ created_at: Optional[datetime] = None,
+ description: Optional[str] = None,
+ spotify: Optional["SpotifyData"] = None,
+ ):
+ self.name = name
+ self.tracks = tracks or []
+ self.created_at = created_at
+ self.description = description
+ self.spotify = spotify
+
+
+class Album:
+ def __init__(
+ self,
+ *,
+ title: str,
+ release_date: Optional[str] = None,
+ tracks: Optional[List[Track]] = None,
+ spotify: Optional["SpotifyData"] = None,
+ ):
+ self.title = title
+ self.release_date = release_date
+ self.tracks = tracks or []
+ self.spotify = spotify
+
+
+class TrackList:
+ def __init__(self, *, tracks: Optional[List[Track]] = None):
+ self.tracks = tracks or []
+
+ def total_duration_ms(self) -> int:
+ return sum(t.duration_ms or 0 for t in self.tracks)
+
+
+# =========================
+# Spotify models
+# =========================
+
+class SpotifyArtist:
+ def __init__(self, *, id: str, name: str, uri: str, url: str):
+ self.id = id
+ self.name = name
+ self.uri = uri
+ self.url = url
+
+
+class SpotifyAlbum:
+ def __init__(
+ self,
+ *,
+ id: str,
+ name: str,
+ release_date: str,
+ total_tracks: int,
+ uri: str,
+ url: str,
+ ):
+ self.id = id
+ self.name = name
+ self.release_date = release_date
+ self.total_tracks = total_tracks
+ self.uri = uri
+ self.url = url
+
+
+class SpotifyData:
+ def __init__(
+ self,
+ *,
+ id: str,
+ uri: str,
+ url: str,
+ href: str,
+ file_path: Optional[Path] = None,
+ popularity: Optional[int] = None,
+ is_playable: Optional[bool] = None,
+ is_local: Optional[bool] = None,
+ artists: Optional[List[SpotifyArtist]] = None,
+ album: Optional[SpotifyAlbum] = None,
+ external_ids: Optional[dict[str, str]] = None,
+ ):
+ self.id = id
+ self.uri = uri
+ self.url = url
+ self.href = href
+ self.file_path = file_path
+ self.popularity = popularity
+ self.is_playable = is_playable
+ self.is_local = is_local
+ self.artists = artists or []
+ self.album = album
+ self.external_ids = external_ids or {}
+
+class LocalData:
+ def __init__(
+ self,
+ *,
+ path: str,
+ ):
+ self.path = path
+
+# =========================
+# Library container
+# =========================
+
+class Library:
+ def __init__(
+ self,
+ *,
+ playlists: Optional[List[Playlist]] = None,
+ tracks: Optional[List[Track]] = None,
+ tracklists: Optional[List[TrackList]] = None,
+ albums: Optional[List[Album]] = None,
+ ):
+ self.playlists = playlists or []
+ self.tracks = tracks or []
+ self.tracklists = tracklists or []
+ self.albums = albums or []
+
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
new file mode 100644
index 0000000..06ea57b
--- /dev/null
+++ b/src/LibrarySerializer.py
@@ -0,0 +1,199 @@
+
+from src.Library import *
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Any
+
+def _dt_to_str(dt: Optional[datetime]) -> Optional[str]:
+ return dt.isoformat() if dt else None
+
+
+def _str_to_dt(s: Optional[str]) -> Optional[datetime]:
+ return datetime.fromisoformat(s) if s else None
+
+
+def _path_to_str(p: Optional[Path]) -> Optional[str]:
+ return str(p) if p else None
+
+
+def _str_to_path(s: Optional[str]) -> Optional[Path]:
+ return Path(s) if s else None
+
+
+class LibrarySaver:
+ def __init__(self, library: Library):
+ self.library = library
+
+ def save(self, path: Path) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("w", encoding="utf-8") as f:
+ json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "tracks": [self._track_to_dict(t) for t in self.library.tracks],
+ "playlists": [self._playlist_to_dict(p) for p in self.library.playlists],
+ "tracklists": [self._tracklist_to_dict(tl) for tl in self.library.tracklists],
+ "albums": [self._album_to_dict(a) for a in self.library.albums],
+ }
+
+ def _track_to_dict(self, t: Track) -> dict[str, Any]:
+ return {
+ "id": t.id,
+ "title": t.title,
+ "artists": t.artists,
+ "album": t.album,
+ "duration_ms": t.duration_ms,
+ "added_at": _dt_to_str(t.added_at),
+ "spotify": self._spotify_to_dict(t.spotify),
+ "local": self._local_to_dict(t.local),
+ }
+
+ def _playlist_to_dict(self, p: Playlist) -> dict[str, Any]:
+ return {
+ "name": p.name,
+ "tracks": [t.id for t in p.tracks],
+ "created_at": _dt_to_str(p.created_at),
+ "description": p.description,
+ "spotify": self._spotify_to_dict(p.spotify),
+ }
+
+ def _tracklist_to_dict(self, tl: TrackList) -> dict[str, Any]:
+ return {"tracks": [t.id for t in tl.tracks]}
+
+ def _album_to_dict(self, a: Album) -> dict[str, Any]:
+ return {
+ "title": a.title,
+ "release_date": a.release_date,
+ "tracks": [t.id for t in a.tracks],
+ "spotify": self._spotify_to_dict(a.spotify),
+ }
+
+ def _local_to_dict(self, local):
+ if not local:
+ return None
+ return {"path": local.path}
+
+ def _spotify_to_dict(self, s: Optional[SpotifyData]) -> Optional[dict[str, Any]]:
+ if not s:
+ return None
+ return {
+ "id": s.id,
+ "uri": s.uri,
+ "url": s.url,
+ "href": s.href,
+ "file_path": _path_to_str(s.file_path),
+ "popularity": s.popularity,
+ "is_playable": s.is_playable,
+ "is_local": s.is_local,
+ "external_ids": s.external_ids,
+ "artists": [{"id": a.id, "name": a.name, "uri": a.uri, "url": a.url} for a in s.artists],
+ "album": {
+ "id": s.album.id,
+ "name": s.album.name,
+ "release_date": s.album.release_date,
+ "total_tracks": s.album.total_tracks,
+ "uri": s.album.uri,
+ "url": s.album.url,
+ } if s.album else None,
+ }
+
+
+class LibraryLoader:
+ def __init__(self):
+ # Map of track id -> Track object
+ self.track_map: dict[str, Track] = {}
+
+ def load(self, path: Path) -> Library:
+ with path.open("r", encoding="utf-8") as f:
+ data = json.load(f)
+
+ # Load tracks first
+ tracks = []
+ for t in data.get("tracks", []):
+ track = self._track_from_dict(t)
+ tracks.append(track)
+ self.track_map[track.id] = track
+
+ # Load playlists using track_map
+ playlists = []
+ for p in data.get("playlists", []):
+ pl_tracks = []
+ for tid in p.get("tracks", []):
+ if tid not in self.track_map:
+ raise KeyError(f"Track ID '{tid}' not found in track map")
+ pl_tracks.append(self.track_map[tid])
+
+ playlists.append(Playlist(
+ name=p["name"],
+ tracks=pl_tracks,
+ created_at=_str_to_dt(p.get("created_at")),
+ description=p.get("description"),
+ spotify=self._spotify_from_dict(p.get("spotify")),
+ ))
+
+
+ # Load tracklists
+ tracklists = []
+ for tl in data.get("tracklists", []):
+ tl_tracks = []
+ for tid in tl.get("tracks", []):
+ if tid not in self.track_map:
+ raise KeyError(f"Track ID '{tid}' not found in track map (tracklist)")
+ tl_tracks.append(self.track_map[tid])
+ tracklists.append(TrackList(tracks=tl_tracks))
+
+ # Load albums
+ albums = []
+ for a in data.get("albums", []):
+ alb_tracks = []
+ for tid in a.get("tracks", []):
+ if tid not in self.track_map:
+ raise KeyError(f"Track ID '{tid}' not found in track map (album '{a.get('title')}')")
+ alb_tracks.append(self.track_map[tid])
+ albums.append(Album(
+ title=a["title"],
+ release_date=a.get("release_date"),
+ tracks=alb_tracks,
+ spotify=self._spotify_from_dict(a.get("spotify")),
+ ))
+
+ return Library(playlists=playlists, tracks=tracks, tracklists=tracklists, albums=albums)
+
+ def _track_from_dict(self, d: dict[str, Any]) -> Track:
+ return Track(
+ id=d["id"],
+ title=d["title"],
+ artists=d["artists"],
+ album=d.get("album"),
+ duration_ms=d.get("duration_ms"),
+ added_at=_str_to_dt(d.get("added_at")),
+ spotify=self._spotify_from_dict(d.get("spotify")),
+ local=self._local_from_dict(d.get("local")),
+ )
+
+ def _local_from_dict(self, d):
+ if not d:
+ return None
+ return LocalData(
+ path=d["path"],
+ )
+
+ def _spotify_from_dict(self, d: Optional[dict[str, Any]]) -> Optional[SpotifyData]:
+ if not d:
+ return None
+ return SpotifyData(
+ id=d["id"],
+ uri=d["uri"],
+ url=d["url"],
+ href=d["href"],
+ file_path=_str_to_path(d.get("file_path")),
+ popularity=d.get("popularity"),
+ is_playable=d.get("is_playable"),
+ is_local=d.get("is_local"),
+ external_ids=d.get("external_ids"),
+ artists=[SpotifyArtist(**a) for a in d.get("artists", [])],
+ album=SpotifyAlbum(**d["album"]) if d.get("album") else None,
+ )
diff --git a/LinkerPatternGenerator.py b/src/MapLocalToSpotify.py
similarity index 71%
rename from LinkerPatternGenerator.py
rename to src/MapLocalToSpotify.py
index 8576da8..dae5252 100644
--- a/LinkerPatternGenerator.py
+++ b/src/MapLocalToSpotify.py
@@ -1,47 +1,52 @@
-from DataBase import *
-from SpotifyWebAPI import find_song, update_access_token
from rapidfuzz import fuzz
from tqdm import tqdm
import requests
-def spotify_pattern_generator():
- update_access_token()
-
- local_library = get_data("local_library")
- spotify_library = get_data("tracks")
+from src.Library import *
+from src.helpers import *
+from src.spotify.SpotifyWebAPI import find_song, update_access_token
- def find_local_songs_on_spotify(local_lib):
+def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
+
+ set_workdir(output_dir)
+
+ update_access_token(client_id, client_secret)
+
+ def find_local_songs_on_spotify():
search_log = []
- total = len(local_library)
+ total = len(local_lib.tracks)
found_tracks_map = {}
with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
- for track_id, track in local_lib.items():
- parts = [track["name"], track["artist"], track.get("album")]
+ for track in local_lib.tracks:
+
+ parts = [track.title, track.artists[0] if len(track.artists) else "", track.album]
search_pattern = " ".join(p for p in parts if p)
if search_pattern == "":
- print(f"cannot generate search pattern for the track - {track["path"]}")
+ print(f"cannot generate search pattern for the track - {track.local.path}")
pbar.update(1)
continue
- found_tracks = find_song(search_pattern)[0:1]
- found_tracks_map[track_id] = found_tracks
- search_log.append({"path" : track["path"], "search_pattern": search_pattern, "result" : found_tracks_map[track_id]})
+ found_tracks = find_song(search_pattern)[0:10]
+ found_tracks_map[track.id] = found_tracks
+ search_log.append({"path" : track.local.path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
pbar.update(1)
save_data(search_log, "spotify_search_log")
+ save_data(found_tracks_map, "spotify_found_map")
return found_tracks_map
- spotify_found_tracks = find_local_songs_on_spotify(local_library)
- spotify_user_track = {track['track']['id']: {"local_mappings": []} for track in spotify_library}
+ spotify_found_tracks = find_local_songs_on_spotify()
+
+ spotify_user_track = {track.id for track in spotify_lib.tracks}
spotify_pattern = {}
for local_id, found_items in spotify_found_tracks.items():
- spotify_pattern[local_id] = {"score": 0.0, "items": []}
+ spotify_pattern[local_id] = {"score": 1.0, "items": []}
if not len(found_items):
continue
@@ -50,14 +55,8 @@ def spotify_pattern_generator():
spotify_track_id = found_item['id']
if spotify_track_id in spotify_user_track:
spotify_pattern[local_id]['items'].append(spotify_track_id)
- spotify_user_track[spotify_track_id]['local_mappings'].append(local_id)
- for spotify_id, track_local_mappings in spotify_user_track.items():
- score = len(track_local_mappings['local_mappings'])
- for local_id in track_local_mappings['local_mappings']:
- spotify_pattern[local_id]['score'] = 1 / score
-
- save_data(spotify_pattern, "link_pattern_spotify")
+ save_data(spotify_pattern, "local_to_spotify_id_map")
def fuzzy_pattern_generator():
@@ -126,7 +125,7 @@ def fuzzy_tag_pattern_generator():
ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id']))
filtered_ratios = []
- threshold = 0.8
+ threshold = 0.81
for ratio in ratios:
if all([val > threshold for val in ratio[0]]):
filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1]))
@@ -138,12 +137,4 @@ def fuzzy_tag_pattern_generator():
mappings[local_track_id] = mapping
pbar.update(1)
- save_data(mappings, "link_pattern_tags")
-
-
-def run_pattern_generators(type_id = 0):
- spotify_pattern_generator()
-
-
-if __name__ == "__main__":
- run_pattern_generators()
+ save_data(mappings, "link_pattern_tags")
\ No newline at end of file
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
new file mode 100644
index 0000000..c40bebb
--- /dev/null
+++ b/src/StatGenerator.py
@@ -0,0 +1,42 @@
+
+def print_link_stats(link_pattern):
+ resolved_reversed = {}
+ resolved = []
+ unresolved_local = []
+ unresolved_remote = []
+
+ for local_id, links in link_pattern.items():
+ if len(links['items']):
+ resolved.append(local_id)
+ resolved_reversed[links['items'][0]] = local_id
+ else:
+ unresolved_local.append(local_id)
+
+ for track in user_tracks:
+ track_id = track['track']['id']
+ if track_id not in resolved_reversed:
+ unresolved_remote.append(track_id)
+
+ print(f"Local Tracks: {len(local_library)}")
+ print(f"Remote Tracks: {len(user_tracks)}")
+
+ unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
+ unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+
+ print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
+ index = 0
+ for remote_id in unresolved_remote:
+ remote_track = user_tracks_by_id[remote_id]
+ print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
+
+ print(f"\nResolved tracks: {len(resolved)}")
+ index = 0
+ for link in resolved:
+ remote_id = link_pattern[link]['items'][0]
+ local_track = local_library[link]
+ remote_track = user_tracks_by_id[remote_id]
+
+ print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
+ print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
+ index += 1
\ No newline at end of file
diff --git a/DataBase.py b/src/helpers.py
similarity index 85%
rename from DataBase.py
rename to src/helpers.py
index c0f230b..8f81bbf 100644
--- a/DataBase.py
+++ b/src/helpers.py
@@ -3,6 +3,13 @@ import os
data_dir = 'prefetched'
+def set_workdir(dir):
+ global data_dir
+ data_dir = str(dir)
+
+def get_workdir():
+ global data_dir
+ return data_dir
class SongTags:
def __init__(self, title='', artist='', album=''):
@@ -40,10 +47,9 @@ def save_data(data, name):
file_path = os.path.join(data_dir, f"{name}.json")
with open(file_path, 'w') as file:
json.dump(data, file, indent=4)
- return f"Data saved to {name}.json."
-def get_artists_str(artists_pack):
- track_artists = [artist['name'] for artist in artists_pack]
- artists = " ".join(track_artists)
- return artists
+ print(f"Data saved to {file_path}.")
+
+
+
diff --git a/ParseItunesToJson.py b/src/itunes/ParseItunesToJson.py
similarity index 100%
rename from ParseItunesToJson.py
rename to src/itunes/ParseItunesToJson.py
diff --git a/LocalLibraryIndexer.py b/src/local/LocalLibraryIndexer.py
similarity index 95%
rename from LocalLibraryIndexer.py
rename to src/local/LocalLibraryIndexer.py
index eecdd58..d3a7ce2 100644
--- a/LocalLibraryIndexer.py
+++ b/src/local/LocalLibraryIndexer.py
@@ -1,18 +1,20 @@
import os
import fnmatch
-
import mutagen
-
-from DataBase import *
import eyed3
from mutagen.flac import FLAC
+from src.helpers import *
+
music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
exclude_directories = ("*mary--*", "*example-word*")
-def update_local_library(root_dir):
+def update_local_library(root_dir, output_dir):
+
+ set_workdir(output_dir / "local")
+
old_library = get_data("local_library")
known_paths = {track['path']: track_id for track_id, track in old_library.items()}
new_library = {}
@@ -74,7 +76,3 @@ def update_local_library(root_dir):
track['name'] = track['path']
save_data(new_library, "local_library")
-
-
-if __name__ == "__main__":
- update_local_library("/home/auser/Music/")
diff --git a/LocalPLaylistGenerator.py b/src/local/LocalPLaylistGenerator.py
similarity index 100%
rename from LocalPLaylistGenerator.py
rename to src/local/LocalPLaylistGenerator.py
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
new file mode 100644
index 0000000..4991ede
--- /dev/null
+++ b/src/local/ParseLocal.py
@@ -0,0 +1,40 @@
+from datetime import datetime
+from typing import Any, Iterable
+from pathlib import Path
+import json
+
+from src.Library import *
+
+class LocalParser:
+ def __init__(self):
+ pass
+
+ def parse_track_item(self, id, track: dict[str, Any]) -> Track:
+ track = Track(
+ title=track["name"],
+ artists=[track.get("artist", [])],
+ album=track["album"] if track.get("album") else None,
+ duration_ms=None,
+ added_at=None,
+ spotify=None,
+ id=id,
+ local=LocalData(path=track["path"])
+ )
+ return track
+
+
+ def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
+ return [self.parse_track_item(id, track) for id, track in items.items()]
+
+ def build_library_from_tracks(self, tracks_content_json: dict[str, list[dict[str, Any]]]) -> Library:
+ library = Library()
+
+ tracks = self.parse_all_tracks(tracks_content_json)
+ library.tracklists.append(TrackList(tracks=list(tracks)))
+ library.tracks.extend(tracks)
+
+ return library
+
+ def parse(self, tracks_file: Path) -> Library:
+ tracks_content = json.loads(tracks_file.read_text())
+ return self.build_library_from_tracks(tracks_content)
diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py
new file mode 100644
index 0000000..535caa0
--- /dev/null
+++ b/src/spotify/ParseSpotify.py
@@ -0,0 +1,139 @@
+from datetime import datetime
+from typing import Any, Iterable
+from pathlib import Path
+import json
+
+from src.Library import *
+
+
+def get_artists_str(artists_pack):
+ track_artists = [artist['name'] for artist in artists_pack]
+ artists = " ".join(track_artists)
+ return artists
+
+class Parser:
+ def __init__(self):
+ # state: Spotify track ID -> Track object
+ self.track_map: dict[str, Track] = {}
+
+ # -------------------------
+ # Low-level helpers
+ # -------------------------
+ @staticmethod
+ def _parse_spotify_datetime(value: str | None) -> datetime | None:
+ if not value:
+ return None
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+
+ @staticmethod
+ def _parse_spotify_artist(a: dict[str, Any]) -> SpotifyArtist:
+ return SpotifyArtist(
+ id=a["id"],
+ name=a["name"],
+ uri=a["uri"],
+ url=a["external_urls"]["spotify"],
+ )
+
+ @staticmethod
+ def _parse_spotify_album(a: dict[str, Any]) -> SpotifyAlbum:
+ return SpotifyAlbum(
+ id=a["id"],
+ name=a["name"],
+ release_date=a.get("release_date"),
+ total_tracks=a.get("total_tracks", 0),
+ uri=a["uri"],
+ url=a["external_urls"]["spotify"],
+ )
+
+ def _parse_spotify_data(self, track_json: dict[str, Any]) -> SpotifyData:
+ return SpotifyData(
+ id=track_json["id"],
+ uri=track_json["uri"],
+ url=track_json["external_urls"]["spotify"],
+ href=track_json["href"],
+ popularity=track_json.get("popularity"),
+ is_playable=track_json.get("is_playable"),
+ is_local=track_json.get("is_local"),
+ external_ids=track_json.get("external_ids", {}),
+ artists=[self._parse_spotify_artist(a) for a in track_json.get("artists", [])],
+ album=self._parse_spotify_album(track_json["album"])
+ if track_json.get("album")
+ else None,
+ )
+
+ # -------------------------
+ # Track parsers
+ # -------------------------
+ def parse_track_item(self, item: dict[str, Any]) -> Track:
+ """
+ Parses a single track item and returns Track object.
+ Reuses existing Track from track_map if already parsed.
+ """
+ t = item["track"]
+ track_id = t["id"]
+
+ # return existing track if already parsed
+ if track_id in self.track_map:
+ return self.track_map[track_id]
+
+ track = Track(
+ title=t["name"],
+ artists=[a["name"] for a in t.get("artists", [])],
+ album=t["album"]["name"] if t.get("album") else None,
+ duration_ms=t.get("duration_ms"),
+ added_at=self._parse_spotify_datetime(item.get("added_at")),
+ spotify=self._parse_spotify_data(t),
+ id=self._parse_spotify_data(t).id
+ )
+
+ self.track_map[track_id] = track
+ return track
+
+ def parse_track_items(self, items: Iterable[dict[str, Any]]) -> list[Track]:
+ return [self.parse_track_item(i) for i in items if i.get("track")]
+
+ # -------------------------
+ # Playlist parsers
+ # -------------------------
+ def parse_playlist_content(self, playlist_name: str, items: list[dict[str, Any]]) -> Playlist:
+ tracks = self.parse_track_items(items)
+ return Playlist(name=playlist_name, tracks=tracks)
+
+ def parse_all_playlist_content(self, data: dict[str, list[dict[str, Any]]]) -> list[Playlist]:
+ playlists: list[Playlist] = []
+ for name, items in data.items():
+ playlists.append(self.parse_playlist_content(name, items))
+ return playlists
+
+ def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
+ return [self.parse_track_item(i) for i in items]
+
+ # -------------------------
+ # Library builder
+ # -------------------------
+ def build_library_from_playlist_content(
+ self, playlist_content_json: dict[str, list[dict[str, Any]]],
+ tracks_content_json: dict[str, list[dict[str, Any]]]
+ ) -> Library:
+ library = Library()
+
+ self.parse_all_tracks(tracks_content_json)
+
+ library.tracklists.append(TrackList(tracks=list((self.track_map.values()))))
+
+ library.playlists = self.parse_all_playlist_content(playlist_content_json)
+ library.tracks.extend(self.track_map.values())
+
+ return library
+
+ # -------------------------
+ # Convenience method
+ # -------------------------
+ def parse(
+ self,
+ playlist_content_file: Path,
+ tracks_file: Path,
+ ) -> Library:
+ playlist_content = json.loads(playlist_content_file.read_text())
+ tracks_content = json.loads(tracks_file.read_text())
+ return self.build_library_from_playlist_content(playlist_content, tracks_content)
diff --git a/SpotifyAuthenticator.py b/src/spotify/SpotifyAuthenticator.py
similarity index 88%
rename from SpotifyAuthenticator.py
rename to src/spotify/SpotifyAuthenticator.py
index 86705d0..5e27f7d 100644
--- a/SpotifyAuthenticator.py
+++ b/src/spotify/SpotifyAuthenticator.py
@@ -8,22 +8,11 @@ import threading
import time
import os.path
-client_id = '***REDACTED***'
+client_id = None
client_secret = None
token = None
-def resolve_client_secret():
- global client_secret
- if os.path.exists('secret'):
- with open('secret', 'r') as file:
- client_secret = file.readline().strip()
- else:
- client_secret = input("Enter the application secret: ")
- with open('secret', 'w') as file:
- file.write(client_secret)
-
-
def retrieve_web_api_token(authorization_code):
global token
@@ -103,8 +92,13 @@ def open_authenticate_page():
webbrowser.open(auth_url)
-def authenticate():
- resolve_client_secret()
+def authenticate(id, secret):
+
+ global client_secret
+ global client_id
+
+ client_secret=secret
+ client_id=id
redirect_server = threading.Thread(target=run_redirect_server)
redirect_server.start()
diff --git a/SpotifyWebAPI.py b/src/spotify/SpotifyWebAPI.py
similarity index 91%
rename from SpotifyWebAPI.py
rename to src/spotify/SpotifyWebAPI.py
index 71ec2f7..5ddc12f 100644
--- a/SpotifyWebAPI.py
+++ b/src/spotify/SpotifyWebAPI.py
@@ -1,8 +1,10 @@
import requests
from PIL import Image
from io import BytesIO
-from SpotifyAuthenticator import authenticate
-from DataBase import *
+
+from src.spotify.SpotifyAuthenticator import authenticate
+
+from src.helpers import *
FETCH_STEP = 50
token = ''
@@ -142,7 +144,7 @@ def save_image_from_url(url, name, image_dir="playlist_covers"):
image = Image.open(BytesIO(response.content))
- directory = os.path.join(data_dir, image_dir)
+ directory = os.path.join(get_workdir(), image_dir)
os.makedirs(directory, exist_ok=True)
file_path = os.path.join(directory, f"{name}.jpg")
print(f"Image saved {file_path}")
@@ -169,9 +171,9 @@ def load_artworks():
load_playlist_covers()
-def update_access_token():
+def update_access_token(id, secret):
global token
- token = authenticate()
+ token = authenticate(id, secret)
def fetch_data():
@@ -184,7 +186,11 @@ def fetch_data():
load_artworks()
+def fetch_all(id, secret, workdir):
+ set_workdir(workdir / "spotify")
+ update_access_token(id, secret)
+ fetch_data()
if __name__ == "__main__":
- update_access_token()
+ update_access_token("", "")
fetch_data()
From 39c88564ba9bc57c49796852f2791d586bec6614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 4 Jan 2026 23:40:41 +0300
Subject: [PATCH 09/30] refactor 2
---
Flow.py | 65 ++++++++++
main.py | 86 ++-----------
src/Library.py | 176 +++++--------------------
src/LibrarySerializer.py | 248 ++++++++++++++----------------------
src/MapLocalToSpotify.py | 6 +-
src/local/ParseLocal.py | 151 ++++++++++++++++++----
src/spotify/ParseSpotify.py | 197 ++++++++++++++--------------
7 files changed, 426 insertions(+), 503 deletions(-)
create mode 100644 Flow.py
diff --git a/Flow.py b/Flow.py
new file mode 100644
index 0000000..bb35731
--- /dev/null
+++ b/Flow.py
@@ -0,0 +1,65 @@
+
+from Env import client_id, client_secret, local_library_path
+from Env import work_dir
+
+from src.LibrarySerializer import LibrarySaver, LibraryLoader
+
+from src.spotify.SpotifyWebAPI import fetch_all
+from src.spotify.ParseSpotify import Parser
+
+from src.local.LocalLibraryIndexer import update_local_library
+from src.local.ParseLocal import LocalParser
+
+from src.MapLocalToSpotify import map_local_to_spotify
+
+
+class Flow:
+ spotify_library = None
+ local_library = None
+
+ @staticmethod
+ def fetch_spotify(self):
+ fetch_all(client_id, client_secret, work_dir)
+
+ @staticmethod
+ def fetch_local():
+ update_local_library(local_library_path, work_dir)
+
+ @staticmethod
+ def parse_spotify_library():
+ parser = Parser()
+
+ library_parsed = parser.parse(
+ work_dir / "spotify" / "playlistTracks.json",
+ work_dir / "spotify" / "tracks.json",
+ work_dir / "spotify" / "top_tracks.json",
+ work_dir / "spotify" / "top_artists.json",
+ )
+
+ LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
+
+ @staticmethod
+ def parse_local_library():
+ parser = LocalParser()
+ library_parsed = parser.parse(work_dir / "local" / "local_library.json")
+ LibrarySaver(library_parsed).save(work_dir / "local_library.json")
+
+ def load_libraries(self):
+ self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
+ self.local_library = LibraryLoader().load(work_dir / "local_library.json")
+
+ def map_local_to_spotify(self):
+ map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
+
+ def run(self, arg):
+ self.fetch_local()
+ # self.do_fetch_spotify()
+
+ self.parse_local_library()
+ self.parse_spotify_library()
+
+ self.load_libraries()
+
+ # self.map_local_to_spotify()
+
+ return True
\ No newline at end of file
diff --git a/main.py b/main.py
index 32682e3..98b4efa 100644
--- a/main.py
+++ b/main.py
@@ -1,77 +1,17 @@
-import cmd
-
-from Env import client_id, client_secret, local_library_path
-from Env import work_dir
-
-from src.LibrarySerializer import LibrarySaver, LibraryLoader
-
-from src.spotify.SpotifyWebAPI import fetch_all
-from src.spotify.ParseSpotify import Parser
-
-from src.local.LocalLibraryIndexer import update_local_library
-from src.local.ParseLocal import LocalParser
-
-from src.MapLocalToSpotify import map_local_to_spotify
-
-
-class Interpreter(cmd.Cmd):
- intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
- prompt = "(kuw) "
-
- spotify_library = None
- local_library = None
-
- @staticmethod
- def do_fetch_spotify(self):
- fetch_all(client_id, client_secret, work_dir)
-
-
- @staticmethod
- def do_fetch_local():
- update_local_library(local_library_path, work_dir)
-
-
- def do_parse_spotify_library(self):
- parser = Parser()
- library_parsed = parser.parse(work_dir / "spotify" / "playlistTracks.json", work_dir / "spotify" / "tracks.json")
- LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
-
- def do_parse_local_library(self):
- parser = LocalParser()
- library_parsed = parser.parse(work_dir / "local" / "local_library.json")
- LibrarySaver(library_parsed).save(work_dir / "local_library.json")
-
- def do_load_libraries(self):
- self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
- self.local_library = LibraryLoader().load(work_dir / "local_library.json")
-
- def do_map_local_to_spotify(self):
- map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
-
-
- def do_run(self, arg):
- # self.do_fetch_local()
- # self.do_fetch_spotify()
-
- # self.do_parse_local_library()
- # self.do_parse_spotify_library()
-
- self.do_load_libraries()
-
- self.do_map_local_to_spotify()
-
- return True
-
- @staticmethod
- def do_exit(self, arg):
- return True
-
+import Flow
if __name__ == '__main__':
- try:
- Interpreter().do_run("")
- # Interpreter().cmdloop()
+ flow = Flow.Flow()
- except KeyboardInterrupt as kb:
- print("process terminated")
+ # flow.fetch_spotify()
+ flow.fetch_local()
+
+ flow.parse_spotify_library()
+ flow.parse_local_library()
+
+ flow.load_libraries()
+
+ flow.map_local_to_spotify()
+
+ print("asd")
diff --git a/src/Library.py b/src/Library.py
index dc2efa3..c3287b8 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -1,159 +1,51 @@
from __future__ import annotations
-import json
+
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+class Artist:
+ def __init__(self):
+ self.id : str = None
+ self.title : str = None
+ self.tracks : List[Track] = []
-# =========================
-# Core models
-# =========================
class Track:
- def __init__(
- self,
- *,
- id: str,
- title: str,
- artists: List[str],
- album: Optional[str] = None,
- duration_ms: Optional[int] = None,
- added_at: Optional[datetime] = None,
- spotify: Optional["SpotifyData"] = None,
- local: Optional["LocalData"] = None,
- ):
- self.id = id
- self.title = title
- self.artists = artists
- self.album = album
- self.duration_ms = duration_ms
- self.added_at = added_at
- self.spotify = spotify
- self.local = local
-
-
-class Playlist:
- def __init__(
- self,
- *,
- name: str,
- tracks: Optional[List[Track]] = None,
- created_at: Optional[datetime] = None,
- description: Optional[str] = None,
- spotify: Optional["SpotifyData"] = None,
- ):
- self.name = name
- self.tracks = tracks or []
- self.created_at = created_at
- self.description = description
- self.spotify = spotify
+ def __init__(self):
+ self.id : str = None
+ self.title : str = None
+ self.artists : List[Artist] = []
+ self.album : Album = None
+ self.duration_ms : int = None
+ self.added_at : datetime = None
+ self.local_path : Path = None
class Album:
- def __init__(
- self,
- *,
- title: str,
- release_date: Optional[str] = None,
- tracks: Optional[List[Track]] = None,
- spotify: Optional["SpotifyData"] = None,
- ):
- self.title = title
- self.release_date = release_date
- self.tracks = tracks or []
- self.spotify = spotify
+ def __init__(self):
+ self.id : str = None
+ self.title : str = None
+ self.release_date : datetime = None
+ self.tracks : List[Track] = []
+ self.artists : List[Artist] = []
-class TrackList:
- def __init__(self, *, tracks: Optional[List[Track]] = None):
- self.tracks = tracks or []
+class Playlist:
+ def __init__(self):
+ self.title : str = None
+ self.tracks : List[Track] = []
+ self.created_at : datetime = None
+ self.description : str = None
- def total_duration_ms(self) -> int:
- return sum(t.duration_ms or 0 for t in self.tracks)
-
-
-# =========================
-# Spotify models
-# =========================
-
-class SpotifyArtist:
- def __init__(self, *, id: str, name: str, uri: str, url: str):
- self.id = id
- self.name = name
- self.uri = uri
- self.url = url
-
-
-class SpotifyAlbum:
- def __init__(
- self,
- *,
- id: str,
- name: str,
- release_date: str,
- total_tracks: int,
- uri: str,
- url: str,
- ):
- self.id = id
- self.name = name
- self.release_date = release_date
- self.total_tracks = total_tracks
- self.uri = uri
- self.url = url
-
-
-class SpotifyData:
- def __init__(
- self,
- *,
- id: str,
- uri: str,
- url: str,
- href: str,
- file_path: Optional[Path] = None,
- popularity: Optional[int] = None,
- is_playable: Optional[bool] = None,
- is_local: Optional[bool] = None,
- artists: Optional[List[SpotifyArtist]] = None,
- album: Optional[SpotifyAlbum] = None,
- external_ids: Optional[dict[str, str]] = None,
- ):
- self.id = id
- self.uri = uri
- self.url = url
- self.href = href
- self.file_path = file_path
- self.popularity = popularity
- self.is_playable = is_playable
- self.is_local = is_local
- self.artists = artists or []
- self.album = album
- self.external_ids = external_ids or {}
-
-class LocalData:
- def __init__(
- self,
- *,
- path: str,
- ):
- self.path = path
-
-# =========================
-# Library container
-# =========================
class Library:
- def __init__(
- self,
- *,
- playlists: Optional[List[Playlist]] = None,
- tracks: Optional[List[Track]] = None,
- tracklists: Optional[List[TrackList]] = None,
- albums: Optional[List[Album]] = None,
- ):
- self.playlists = playlists or []
- self.tracks = tracks or []
- self.tracklists = tracklists or []
- self.albums = albums or []
+ def __init__(self):
+ self.artists : List[Artist] = []
+ self.playlists : List[Playlist] = []
+ self.tracks : List[Track] = []
+ self.albums : List[Album] = []
+ self.liked_tracks : List[Track] = []
+ self.top_tracks : List[Track]= []
+ self.top_artists : List[Artist] = []
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index 06ea57b..bc968ea 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -26,174 +26,112 @@ class LibrarySaver:
def __init__(self, library: Library):
self.library = library
+ @staticmethod
+ def _artist_to_dict(artist: Artist) -> dict[str, Any]:
+ return {
+ "id": artist.id,
+ "title": artist.title,
+ "tracks": [track.id for track in artist.tracks],
+ }
+
+ @staticmethod
+ def _track_to_dict(track: Track) -> dict[str, Any]:
+ return {
+ "id": track.id,
+ "title": track.title,
+ "artists": [artist.id for artist in track.artists],
+ "album": track.album.id,
+ "duration_ms": track.duration_ms,
+ "added_at": _dt_to_str(track.added_at),
+ "local_path": track.local_path,
+ }
+
+ @staticmethod
+ def _playlist_to_dict(playlist: Playlist) -> dict[str, Any]:
+ return {
+ "title": playlist.title,
+ "tracks": [track.id for track in playlist.tracks],
+ "created_at": _dt_to_str(playlist.created_at),
+ "description": playlist.description,
+ }
+
+ @staticmethod
+ def _album_to_dict(album: Album) -> dict[str, Any]:
+ return {
+ "id": album.id,
+ "title": album.title,
+ "release_date": album.release_date,
+ "tracks": [track.id for track in album.tracks],
+ "artists": [artist.id for artist in album.artists]
+ }
+
def save(self, path: Path) -> None:
+
+ data = {
+ "tracks": [self._track_to_dict(track) for track in self.library.tracks],
+ "albums": [self._album_to_dict(album) for album in self.library.albums],
+ "artists": [self._artist_to_dict(artist) for artist in self.library.artists],
+ "playlists": [self._playlist_to_dict(playlist) for playlist in self.library.playlists],
+ "top_tracks": [track.id for track in self.library.top_tracks],
+ "top_artists": [artist.id for artist in self.library.top_artists],
+ "liked_tracks": [track.id for track in self.library.liked_tracks],
+ }
+
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
- json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
-
- def to_dict(self) -> dict[str, Any]:
- return {
- "tracks": [self._track_to_dict(t) for t in self.library.tracks],
- "playlists": [self._playlist_to_dict(p) for p in self.library.playlists],
- "tracklists": [self._tracklist_to_dict(tl) for tl in self.library.tracklists],
- "albums": [self._album_to_dict(a) for a in self.library.albums],
- }
-
- def _track_to_dict(self, t: Track) -> dict[str, Any]:
- return {
- "id": t.id,
- "title": t.title,
- "artists": t.artists,
- "album": t.album,
- "duration_ms": t.duration_ms,
- "added_at": _dt_to_str(t.added_at),
- "spotify": self._spotify_to_dict(t.spotify),
- "local": self._local_to_dict(t.local),
- }
-
- def _playlist_to_dict(self, p: Playlist) -> dict[str, Any]:
- return {
- "name": p.name,
- "tracks": [t.id for t in p.tracks],
- "created_at": _dt_to_str(p.created_at),
- "description": p.description,
- "spotify": self._spotify_to_dict(p.spotify),
- }
-
- def _tracklist_to_dict(self, tl: TrackList) -> dict[str, Any]:
- return {"tracks": [t.id for t in tl.tracks]}
-
- def _album_to_dict(self, a: Album) -> dict[str, Any]:
- return {
- "title": a.title,
- "release_date": a.release_date,
- "tracks": [t.id for t in a.tracks],
- "spotify": self._spotify_to_dict(a.spotify),
- }
-
- def _local_to_dict(self, local):
- if not local:
- return None
- return {"path": local.path}
-
- def _spotify_to_dict(self, s: Optional[SpotifyData]) -> Optional[dict[str, Any]]:
- if not s:
- return None
- return {
- "id": s.id,
- "uri": s.uri,
- "url": s.url,
- "href": s.href,
- "file_path": _path_to_str(s.file_path),
- "popularity": s.popularity,
- "is_playable": s.is_playable,
- "is_local": s.is_local,
- "external_ids": s.external_ids,
- "artists": [{"id": a.id, "name": a.name, "uri": a.uri, "url": a.url} for a in s.artists],
- "album": {
- "id": s.album.id,
- "name": s.album.name,
- "release_date": s.album.release_date,
- "total_tracks": s.album.total_tracks,
- "uri": s.album.uri,
- "url": s.album.url,
- } if s.album else None,
- }
+ json.dump(data, f, indent=2, ensure_ascii=False)
class LibraryLoader:
def __init__(self):
- # Map of track id -> Track object
self.track_map: dict[str, Track] = {}
+ self.album_map: dict[str, Album] = {}
+ self.artist_map: dict[str, Artist] = {}
+
+ def load_track(self, data: dict[str, Any]) -> Track:
+ track = self.track_map.get(data.get("id"), Track())
+ track.id = data.get("id")
+ track.title = data.get("title")
+ track.duration_ms = data.get("duration_ms")
+ track.added_at = _str_to_dt(data.get("added_at"))
+ track.local_path = data.get("local_path")
+ track.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
+ track.album = self.album_map.get(data.get("album"), Album())
+
+ self.track_map[track.id] = track
+ return track
+
+ def load_artist(self, data: dict[str, Any]):
+ artist = self.artist_map.get(data.get("id"), Artist())
+ artist.id = data.get("id")
+ artist.title = data.get("title")
+ artist.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+
+ self.artist_map[artist.id] = artist
+ return artist
+
+ def load_album(self, data: dict[str, Any]):
+ album = self.album_map.get(data.get("id"), Album())
+ album.id = data.get("id")
+ album.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+ album.title = data.get("title")
+ album.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
+
+ self.album_map[album.id] = album
+ return album
def load(self, path: Path) -> Library:
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
- # Load tracks first
- tracks = []
- for t in data.get("tracks", []):
- track = self._track_from_dict(t)
- tracks.append(track)
- self.track_map[track.id] = track
+ library = Library()
- # Load playlists using track_map
- playlists = []
- for p in data.get("playlists", []):
- pl_tracks = []
- for tid in p.get("tracks", []):
- if tid not in self.track_map:
- raise KeyError(f"Track ID '{tid}' not found in track map")
- pl_tracks.append(self.track_map[tid])
+ library.tracks = [self.load_track(track) for track in data.get("tracks")]
+ library.albums = [self.load_album(album) for album in data.get("albums")]
+ library.artists = [self.load_artist(artist) for artist in data.get("artists")]
- playlists.append(Playlist(
- name=p["name"],
- tracks=pl_tracks,
- created_at=_str_to_dt(p.get("created_at")),
- description=p.get("description"),
- spotify=self._spotify_from_dict(p.get("spotify")),
- ))
+ library.top_artists = [self.artist_map[artist] for artist in data.get("top_artists")]
+ library.top_tracks = [self.track_map[track] for track in data.get("top_tracks")]
+ library.liked_tracks = [self.track_map[track] for track in data.get("liked_tracks")]
-
- # Load tracklists
- tracklists = []
- for tl in data.get("tracklists", []):
- tl_tracks = []
- for tid in tl.get("tracks", []):
- if tid not in self.track_map:
- raise KeyError(f"Track ID '{tid}' not found in track map (tracklist)")
- tl_tracks.append(self.track_map[tid])
- tracklists.append(TrackList(tracks=tl_tracks))
-
- # Load albums
- albums = []
- for a in data.get("albums", []):
- alb_tracks = []
- for tid in a.get("tracks", []):
- if tid not in self.track_map:
- raise KeyError(f"Track ID '{tid}' not found in track map (album '{a.get('title')}')")
- alb_tracks.append(self.track_map[tid])
- albums.append(Album(
- title=a["title"],
- release_date=a.get("release_date"),
- tracks=alb_tracks,
- spotify=self._spotify_from_dict(a.get("spotify")),
- ))
-
- return Library(playlists=playlists, tracks=tracks, tracklists=tracklists, albums=albums)
-
- def _track_from_dict(self, d: dict[str, Any]) -> Track:
- return Track(
- id=d["id"],
- title=d["title"],
- artists=d["artists"],
- album=d.get("album"),
- duration_ms=d.get("duration_ms"),
- added_at=_str_to_dt(d.get("added_at")),
- spotify=self._spotify_from_dict(d.get("spotify")),
- local=self._local_from_dict(d.get("local")),
- )
-
- def _local_from_dict(self, d):
- if not d:
- return None
- return LocalData(
- path=d["path"],
- )
-
- def _spotify_from_dict(self, d: Optional[dict[str, Any]]) -> Optional[SpotifyData]:
- if not d:
- return None
- return SpotifyData(
- id=d["id"],
- uri=d["uri"],
- url=d["url"],
- href=d["href"],
- file_path=_str_to_path(d.get("file_path")),
- popularity=d.get("popularity"),
- is_playable=d.get("is_playable"),
- is_local=d.get("is_local"),
- external_ids=d.get("external_ids"),
- artists=[SpotifyArtist(**a) for a in d.get("artists", [])],
- album=SpotifyAlbum(**d["album"]) if d.get("album") else None,
- )
+ return library
diff --git a/src/MapLocalToSpotify.py b/src/MapLocalToSpotify.py
index dae5252..84e37c5 100644
--- a/src/MapLocalToSpotify.py
+++ b/src/MapLocalToSpotify.py
@@ -22,17 +22,17 @@ def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_
with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
for track in local_lib.tracks:
- parts = [track.title, track.artists[0] if len(track.artists) else "", track.album]
+ parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title]
search_pattern = " ".join(p for p in parts if p)
if search_pattern == "":
- print(f"cannot generate search pattern for the track - {track.local.path}")
+ print(f"cannot generate search pattern for the track - {track.local_path}")
pbar.update(1)
continue
found_tracks = find_song(search_pattern)[0:10]
found_tracks_map[track.id] = found_tracks
- search_log.append({"path" : track.local.path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
+ search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
pbar.update(1)
save_data(search_log, "spotify_search_log")
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
index 4991ede..602e89e 100644
--- a/src/local/ParseLocal.py
+++ b/src/local/ParseLocal.py
@@ -1,40 +1,139 @@
-from datetime import datetime
-from typing import Any, Iterable
-from pathlib import Path
import json
+import re
from src.Library import *
+import re
+
+PROTECTED_ARTISTS = {
+ "Tyler, The Creator",
+ "The Good, The Bad & The Queen",
+}
+
+def split_artists(artist_tag: str) -> list[str]:
+ if not artist_tag:
+ return []
+
+ placeholder_comma = "§COMMA§"
+ placeholder_amp = "§AMP§"
+
+ protected_map = {}
+
+ # Protect known artist names
+ for artist in PROTECTED_ARTISTS:
+ protected = (
+ artist
+ .replace(",", placeholder_comma)
+ .replace("&", placeholder_amp)
+ )
+ protected_map[protected] = artist
+ artist_tag = artist_tag.replace(artist, protected)
+
+ # Split remaining separators
+ parts = re.split(r"[,&]", artist_tag)
+
+ # Restore protected names and trim spaces
+ result = []
+ for part in parts:
+ part = part.strip()
+ if not part:
+ continue
+
+ part = (
+ part
+ .replace(placeholder_comma, ",")
+ .replace(placeholder_amp, "&")
+ )
+
+ result.append(part)
+
+ return result
+
+
class LocalParser:
def __init__(self):
- pass
-
- def parse_track_item(self, id, track: dict[str, Any]) -> Track:
- track = Track(
- title=track["name"],
- artists=[track.get("artist", [])],
- album=track["album"] if track.get("album") else None,
- duration_ms=None,
- added_at=None,
- spotify=None,
- id=id,
- local=LocalData(path=track["path"])
- )
- return track
+ self.track_map: dict[str, Track] = {}
+ self.artist_map: dict[str, Artist] = {}
+ self.album_map: dict[str, Album] = {}
+ self.library = Library()
- def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
- return [self.parse_track_item(id, track) for id, track in items.items()]
+ def parse_artists(self, data: str) -> List[Artist]:
+ if not data:
+ data = "unknown"
- def build_library_from_tracks(self, tracks_content_json: dict[str, list[dict[str, Any]]]) -> Library:
- library = Library()
+ data = split_artists(data)
- tracks = self.parse_all_tracks(tracks_content_json)
- library.tracklists.append(TrackList(tracks=list(tracks)))
- library.tracks.extend(tracks)
+ artists = []
+
+ for artist_data in data:
+ name = artist_data
+ artist_id = name
+
+ if artist_id in self.artist_map:
+ artists.append(self.artist_map[artist_id])
+ continue
+
+ artist = Artist()
+ artist.id = artist_id
+ artist.title = name
+
+ self.artist_map[artist_id] = artist
+ self.library.artists.append(artist)
+
+ artists.append(artist)
+
+ return artists
+
+ def parse_album(self, data: str) -> Album:
+ if not data:
+ data = "unknown"
+
+ name = data
+ album_id = name
+
+ if album_id in self.album_map:
+ return self.album_map[album_id]
+
+ album = Album()
+ album.id = album_id
+ album.title = name
+
+ self.album_map[album_id] = album
+ self.library.albums.append(album)
+
+ return album
+
+ def add_tracks_to_albums_and_artists(self):
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if track not in artist.tracks:
+ artist.tracks.append(track)
+
+ if track not in track.album.tracks:
+ track.album.tracks.append(track)
- return library
def parse(self, tracks_file: Path) -> Library:
tracks_content = json.loads(tracks_file.read_text())
- return self.build_library_from_tracks(tracks_content)
+
+ for track_id, data in tracks_content.items():
+ track = Track()
+
+ track.id = track_id
+ track.title = data.get("name")
+ track.artists = self.parse_artists(data.get("artist"))
+ track.album = self.parse_album(data.get("album"))
+ track.local_path = data.get("path")
+
+ self.track_map[track.id] = track
+ self.library.tracks.append(track)
+
+ self.add_tracks_to_albums_and_artists()
+
+ return self.library
+
+
+ def parse_track_item(self, id, track: dict[str, Any]) -> Track:
+
+ return track
diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py
index 535caa0..49882cb 100644
--- a/src/spotify/ParseSpotify.py
+++ b/src/spotify/ParseSpotify.py
@@ -1,9 +1,10 @@
-from datetime import datetime
+from src.Library import *
+
from typing import Any, Iterable
-from pathlib import Path
import json
-from src.Library import *
+from datetime import datetime
+from pathlib import Path
def get_artists_str(artists_pack):
@@ -11,129 +12,117 @@ def get_artists_str(artists_pack):
artists = " ".join(track_artists)
return artists
+
class Parser:
def __init__(self):
- # state: Spotify track ID -> Track object
self.track_map: dict[str, Track] = {}
+ self.artist_map: dict[str, Artist] = {}
+ self.album_map: dict[str, Album] = {}
+ self.library = Library()
- # -------------------------
- # Low-level helpers
- # -------------------------
- @staticmethod
- def _parse_spotify_datetime(value: str | None) -> datetime | None:
- if not value:
- return None
- return datetime.fromisoformat(value.replace("Z", "+00:00"))
+ def parse(self, playlists: Path, tracks: Path, top_tracks: Path, top_artists: Path) -> Library:
+ self.library = Library()
- @staticmethod
- def _parse_spotify_artist(a: dict[str, Any]) -> SpotifyArtist:
- return SpotifyArtist(
- id=a["id"],
- name=a["name"],
- uri=a["uri"],
- url=a["external_urls"]["spotify"],
- )
+ self.parse_tracks(json.loads(tracks.read_text()))
+ self.parse_playlists(json.loads(playlists.read_text()))
- @staticmethod
- def _parse_spotify_album(a: dict[str, Any]) -> SpotifyAlbum:
- return SpotifyAlbum(
- id=a["id"],
- name=a["name"],
- release_date=a.get("release_date"),
- total_tracks=a.get("total_tracks", 0),
- uri=a["uri"],
- url=a["external_urls"]["spotify"],
- )
+ self.parse_top_artists(json.loads(top_artists.read_text()))
+ self.parse_top_tracks(json.loads(top_tracks.read_text()))
- def _parse_spotify_data(self, track_json: dict[str, Any]) -> SpotifyData:
- return SpotifyData(
- id=track_json["id"],
- uri=track_json["uri"],
- url=track_json["external_urls"]["spotify"],
- href=track_json["href"],
- popularity=track_json.get("popularity"),
- is_playable=track_json.get("is_playable"),
- is_local=track_json.get("is_local"),
- external_ids=track_json.get("external_ids", {}),
- artists=[self._parse_spotify_artist(a) for a in track_json.get("artists", [])],
- album=self._parse_spotify_album(track_json["album"])
- if track_json.get("album")
- else None,
- )
+ self.add_tracks_to_albums_and_artists()
- # -------------------------
- # Track parsers
- # -------------------------
- def parse_track_item(self, item: dict[str, Any]) -> Track:
- """
- Parses a single track item and returns Track object.
- Reuses existing Track from track_map if already parsed.
- """
- t = item["track"]
- track_id = t["id"]
+ return self.library
+
+ def add_tracks_to_albums_and_artists(self):
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if track not in artist.tracks:
+ artist.tracks.append(track)
+
+ if track not in track.album.tracks:
+ track.album.tracks.append(track)
+
+
+ def parse_playlists(self, data):
+ for name, tracks in data.items():
+ playlist = Playlist()
+ playlist.title = name
+ for track in tracks:
+ track_data = track.get("track")
+ if track_data:
+ track_id = track_data.get("id")
+ playlist.tracks.append(self.parse_track(track_data, track.get("added_at")))
+
+ self.library.playlists.append(playlist)
+
+ def parse_top_tracks(self, data):
+ for track in data:
+ if track.get("id") in self.track_map:
+ self.library.top_tracks.append(self.track_map.get(track.get("id")))
+
+ def parse_top_artists(self, data):
+ for artist in data:
+ if artist.get("id") in self.artist_map:
+ self.library.top_artists.append(self.artist_map.get(artist.get("id")))
+
+ def parse_tracks(self, data):
+ for track in data:
+ self.parse_track(track.get("track"), track.get("added_at"))
+
+ for track in data:
+ self.library.liked_tracks.append(self.track_map.get(track.get("track").get("id")))
+
+ def parse_track(self, data: dict[str, Any], added_time : datetime) -> Track:
+ track_id = data.get("id")
- # return existing track if already parsed
if track_id in self.track_map:
return self.track_map[track_id]
- track = Track(
- title=t["name"],
- artists=[a["name"] for a in t.get("artists", [])],
- album=t["album"]["name"] if t.get("album") else None,
- duration_ms=t.get("duration_ms"),
- added_at=self._parse_spotify_datetime(item.get("added_at")),
- spotify=self._parse_spotify_data(t),
- id=self._parse_spotify_data(t).id
- )
+ track = Track()
self.track_map[track_id] = track
+
+ track.id = track_id
+ track.title = data.get("name")
+ track.artists = [self.parse_artist(artist) for artist in data.get("artists")]
+ track.album = self.parse_album(data.get("album"))
+ track.duration_ms = data.get("duration_ms")
+
+ self.library.tracks.append(track)
+
return track
- def parse_track_items(self, items: Iterable[dict[str, Any]]) -> list[Track]:
- return [self.parse_track_item(i) for i in items if i.get("track")]
+ def parse_album(self, data):
+ album_id = data.get("id")
- # -------------------------
- # Playlist parsers
- # -------------------------
- def parse_playlist_content(self, playlist_name: str, items: list[dict[str, Any]]) -> Playlist:
- tracks = self.parse_track_items(items)
- return Playlist(name=playlist_name, tracks=tracks)
+ if album_id in self.album_map:
+ return self.album_map[album_id]
- def parse_all_playlist_content(self, data: dict[str, list[dict[str, Any]]]) -> list[Playlist]:
- playlists: list[Playlist] = []
- for name, items in data.items():
- playlists.append(self.parse_playlist_content(name, items))
- return playlists
+ album = Album()
- def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
- return [self.parse_track_item(i) for i in items]
+ self.album_map[album_id] = album
- # -------------------------
- # Library builder
- # -------------------------
- def build_library_from_playlist_content(
- self, playlist_content_json: dict[str, list[dict[str, Any]]],
- tracks_content_json: dict[str, list[dict[str, Any]]]
- ) -> Library:
- library = Library()
+ album.id = data.get("id")
+ album.title = data.get("name")
+ album.artists = [self.parse_artist(artist) for artist in data.get("artists")]
- self.parse_all_tracks(tracks_content_json)
+ self.library.albums.append(album)
- library.tracklists.append(TrackList(tracks=list((self.track_map.values()))))
+ return album
- library.playlists = self.parse_all_playlist_content(playlist_content_json)
- library.tracks.extend(self.track_map.values())
+ def parse_artist(self, data: dict[str, Any]) -> Artist:
+ artist_id = data.get("id")
- return library
+ if artist_id in self.artist_map:
+ return self.artist_map[artist_id]
- # -------------------------
- # Convenience method
- # -------------------------
- def parse(
- self,
- playlist_content_file: Path,
- tracks_file: Path,
- ) -> Library:
- playlist_content = json.loads(playlist_content_file.read_text())
- tracks_content = json.loads(tracks_file.read_text())
- return self.build_library_from_playlist_content(playlist_content, tracks_content)
+ artist = Artist()
+
+ self.artist_map[artist_id] = artist
+
+ artist.id = data.get("id")
+ artist.title = data.get("name")
+
+ self.library.artists.append(artist)
+
+ return artist
From 5f4bbbaa592a92a9b94455793421aada76108ab4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 4 Jan 2026 23:40:41 +0300
Subject: [PATCH 10/30] refactor 2
---
Flow.py | 65 ++++++++++
main.py | 86 ++-----------
src/Library.py | 176 +++++--------------------
src/LibrarySerializer.py | 248 ++++++++++++++----------------------
src/MapLocalToSpotify.py | 6 +-
src/local/ParseLocal.py | 151 ++++++++++++++++++----
src/spotify/ParseSpotify.py | 197 ++++++++++++++--------------
7 files changed, 426 insertions(+), 503 deletions(-)
create mode 100644 Flow.py
diff --git a/Flow.py b/Flow.py
new file mode 100644
index 0000000..bb35731
--- /dev/null
+++ b/Flow.py
@@ -0,0 +1,65 @@
+
+from Env import client_id, client_secret, local_library_path
+from Env import work_dir
+
+from src.LibrarySerializer import LibrarySaver, LibraryLoader
+
+from src.spotify.SpotifyWebAPI import fetch_all
+from src.spotify.ParseSpotify import Parser
+
+from src.local.LocalLibraryIndexer import update_local_library
+from src.local.ParseLocal import LocalParser
+
+from src.MapLocalToSpotify import map_local_to_spotify
+
+
+class Flow:
+ spotify_library = None
+ local_library = None
+
+ @staticmethod
+ def fetch_spotify(self):
+ fetch_all(client_id, client_secret, work_dir)
+
+ @staticmethod
+ def fetch_local():
+ update_local_library(local_library_path, work_dir)
+
+ @staticmethod
+ def parse_spotify_library():
+ parser = Parser()
+
+ library_parsed = parser.parse(
+ work_dir / "spotify" / "playlistTracks.json",
+ work_dir / "spotify" / "tracks.json",
+ work_dir / "spotify" / "top_tracks.json",
+ work_dir / "spotify" / "top_artists.json",
+ )
+
+ LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
+
+ @staticmethod
+ def parse_local_library():
+ parser = LocalParser()
+ library_parsed = parser.parse(work_dir / "local" / "local_library.json")
+ LibrarySaver(library_parsed).save(work_dir / "local_library.json")
+
+ def load_libraries(self):
+ self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
+ self.local_library = LibraryLoader().load(work_dir / "local_library.json")
+
+ def map_local_to_spotify(self):
+ map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
+
+ def run(self, arg):
+ self.fetch_local()
+ # self.do_fetch_spotify()
+
+ self.parse_local_library()
+ self.parse_spotify_library()
+
+ self.load_libraries()
+
+ # self.map_local_to_spotify()
+
+ return True
\ No newline at end of file
diff --git a/main.py b/main.py
index 32682e3..98b4efa 100644
--- a/main.py
+++ b/main.py
@@ -1,77 +1,17 @@
-import cmd
-
-from Env import client_id, client_secret, local_library_path
-from Env import work_dir
-
-from src.LibrarySerializer import LibrarySaver, LibraryLoader
-
-from src.spotify.SpotifyWebAPI import fetch_all
-from src.spotify.ParseSpotify import Parser
-
-from src.local.LocalLibraryIndexer import update_local_library
-from src.local.ParseLocal import LocalParser
-
-from src.MapLocalToSpotify import map_local_to_spotify
-
-
-class Interpreter(cmd.Cmd):
- intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
- prompt = "(kuw) "
-
- spotify_library = None
- local_library = None
-
- @staticmethod
- def do_fetch_spotify(self):
- fetch_all(client_id, client_secret, work_dir)
-
-
- @staticmethod
- def do_fetch_local():
- update_local_library(local_library_path, work_dir)
-
-
- def do_parse_spotify_library(self):
- parser = Parser()
- library_parsed = parser.parse(work_dir / "spotify" / "playlistTracks.json", work_dir / "spotify" / "tracks.json")
- LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
-
- def do_parse_local_library(self):
- parser = LocalParser()
- library_parsed = parser.parse(work_dir / "local" / "local_library.json")
- LibrarySaver(library_parsed).save(work_dir / "local_library.json")
-
- def do_load_libraries(self):
- self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
- self.local_library = LibraryLoader().load(work_dir / "local_library.json")
-
- def do_map_local_to_spotify(self):
- map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
-
-
- def do_run(self, arg):
- # self.do_fetch_local()
- # self.do_fetch_spotify()
-
- # self.do_parse_local_library()
- # self.do_parse_spotify_library()
-
- self.do_load_libraries()
-
- self.do_map_local_to_spotify()
-
- return True
-
- @staticmethod
- def do_exit(self, arg):
- return True
-
+import Flow
if __name__ == '__main__':
- try:
- Interpreter().do_run("")
- # Interpreter().cmdloop()
+ flow = Flow.Flow()
- except KeyboardInterrupt as kb:
- print("process terminated")
+ # flow.fetch_spotify()
+ flow.fetch_local()
+
+ flow.parse_spotify_library()
+ flow.parse_local_library()
+
+ flow.load_libraries()
+
+ flow.map_local_to_spotify()
+
+ print("asd")
diff --git a/src/Library.py b/src/Library.py
index dc2efa3..c3287b8 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -1,159 +1,51 @@
from __future__ import annotations
-import json
+
from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+class Artist:
+ def __init__(self):
+ self.id : str = None
+ self.title : str = None
+ self.tracks : List[Track] = []
-# =========================
-# Core models
-# =========================
class Track:
- def __init__(
- self,
- *,
- id: str,
- title: str,
- artists: List[str],
- album: Optional[str] = None,
- duration_ms: Optional[int] = None,
- added_at: Optional[datetime] = None,
- spotify: Optional["SpotifyData"] = None,
- local: Optional["LocalData"] = None,
- ):
- self.id = id
- self.title = title
- self.artists = artists
- self.album = album
- self.duration_ms = duration_ms
- self.added_at = added_at
- self.spotify = spotify
- self.local = local
-
-
-class Playlist:
- def __init__(
- self,
- *,
- name: str,
- tracks: Optional[List[Track]] = None,
- created_at: Optional[datetime] = None,
- description: Optional[str] = None,
- spotify: Optional["SpotifyData"] = None,
- ):
- self.name = name
- self.tracks = tracks or []
- self.created_at = created_at
- self.description = description
- self.spotify = spotify
+ def __init__(self):
+ self.id : str = None
+ self.title : str = None
+ self.artists : List[Artist] = []
+ self.album : Album = None
+ self.duration_ms : int = None
+ self.added_at : datetime = None
+ self.local_path : Path = None
class Album:
- def __init__(
- self,
- *,
- title: str,
- release_date: Optional[str] = None,
- tracks: Optional[List[Track]] = None,
- spotify: Optional["SpotifyData"] = None,
- ):
- self.title = title
- self.release_date = release_date
- self.tracks = tracks or []
- self.spotify = spotify
+ def __init__(self):
+ self.id : str = None
+ self.title : str = None
+ self.release_date : datetime = None
+ self.tracks : List[Track] = []
+ self.artists : List[Artist] = []
-class TrackList:
- def __init__(self, *, tracks: Optional[List[Track]] = None):
- self.tracks = tracks or []
+class Playlist:
+ def __init__(self):
+ self.title : str = None
+ self.tracks : List[Track] = []
+ self.created_at : datetime = None
+ self.description : str = None
- def total_duration_ms(self) -> int:
- return sum(t.duration_ms or 0 for t in self.tracks)
-
-
-# =========================
-# Spotify models
-# =========================
-
-class SpotifyArtist:
- def __init__(self, *, id: str, name: str, uri: str, url: str):
- self.id = id
- self.name = name
- self.uri = uri
- self.url = url
-
-
-class SpotifyAlbum:
- def __init__(
- self,
- *,
- id: str,
- name: str,
- release_date: str,
- total_tracks: int,
- uri: str,
- url: str,
- ):
- self.id = id
- self.name = name
- self.release_date = release_date
- self.total_tracks = total_tracks
- self.uri = uri
- self.url = url
-
-
-class SpotifyData:
- def __init__(
- self,
- *,
- id: str,
- uri: str,
- url: str,
- href: str,
- file_path: Optional[Path] = None,
- popularity: Optional[int] = None,
- is_playable: Optional[bool] = None,
- is_local: Optional[bool] = None,
- artists: Optional[List[SpotifyArtist]] = None,
- album: Optional[SpotifyAlbum] = None,
- external_ids: Optional[dict[str, str]] = None,
- ):
- self.id = id
- self.uri = uri
- self.url = url
- self.href = href
- self.file_path = file_path
- self.popularity = popularity
- self.is_playable = is_playable
- self.is_local = is_local
- self.artists = artists or []
- self.album = album
- self.external_ids = external_ids or {}
-
-class LocalData:
- def __init__(
- self,
- *,
- path: str,
- ):
- self.path = path
-
-# =========================
-# Library container
-# =========================
class Library:
- def __init__(
- self,
- *,
- playlists: Optional[List[Playlist]] = None,
- tracks: Optional[List[Track]] = None,
- tracklists: Optional[List[TrackList]] = None,
- albums: Optional[List[Album]] = None,
- ):
- self.playlists = playlists or []
- self.tracks = tracks or []
- self.tracklists = tracklists or []
- self.albums = albums or []
+ def __init__(self):
+ self.artists : List[Artist] = []
+ self.playlists : List[Playlist] = []
+ self.tracks : List[Track] = []
+ self.albums : List[Album] = []
+ self.liked_tracks : List[Track] = []
+ self.top_tracks : List[Track]= []
+ self.top_artists : List[Artist] = []
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index 06ea57b..bc968ea 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -26,174 +26,112 @@ class LibrarySaver:
def __init__(self, library: Library):
self.library = library
+ @staticmethod
+ def _artist_to_dict(artist: Artist) -> dict[str, Any]:
+ return {
+ "id": artist.id,
+ "title": artist.title,
+ "tracks": [track.id for track in artist.tracks],
+ }
+
+ @staticmethod
+ def _track_to_dict(track: Track) -> dict[str, Any]:
+ return {
+ "id": track.id,
+ "title": track.title,
+ "artists": [artist.id for artist in track.artists],
+ "album": track.album.id,
+ "duration_ms": track.duration_ms,
+ "added_at": _dt_to_str(track.added_at),
+ "local_path": track.local_path,
+ }
+
+ @staticmethod
+ def _playlist_to_dict(playlist: Playlist) -> dict[str, Any]:
+ return {
+ "title": playlist.title,
+ "tracks": [track.id for track in playlist.tracks],
+ "created_at": _dt_to_str(playlist.created_at),
+ "description": playlist.description,
+ }
+
+ @staticmethod
+ def _album_to_dict(album: Album) -> dict[str, Any]:
+ return {
+ "id": album.id,
+ "title": album.title,
+ "release_date": album.release_date,
+ "tracks": [track.id for track in album.tracks],
+ "artists": [artist.id for artist in album.artists]
+ }
+
def save(self, path: Path) -> None:
+
+ data = {
+ "tracks": [self._track_to_dict(track) for track in self.library.tracks],
+ "albums": [self._album_to_dict(album) for album in self.library.albums],
+ "artists": [self._artist_to_dict(artist) for artist in self.library.artists],
+ "playlists": [self._playlist_to_dict(playlist) for playlist in self.library.playlists],
+ "top_tracks": [track.id for track in self.library.top_tracks],
+ "top_artists": [artist.id for artist in self.library.top_artists],
+ "liked_tracks": [track.id for track in self.library.liked_tracks],
+ }
+
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
- json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)
-
- def to_dict(self) -> dict[str, Any]:
- return {
- "tracks": [self._track_to_dict(t) for t in self.library.tracks],
- "playlists": [self._playlist_to_dict(p) for p in self.library.playlists],
- "tracklists": [self._tracklist_to_dict(tl) for tl in self.library.tracklists],
- "albums": [self._album_to_dict(a) for a in self.library.albums],
- }
-
- def _track_to_dict(self, t: Track) -> dict[str, Any]:
- return {
- "id": t.id,
- "title": t.title,
- "artists": t.artists,
- "album": t.album,
- "duration_ms": t.duration_ms,
- "added_at": _dt_to_str(t.added_at),
- "spotify": self._spotify_to_dict(t.spotify),
- "local": self._local_to_dict(t.local),
- }
-
- def _playlist_to_dict(self, p: Playlist) -> dict[str, Any]:
- return {
- "name": p.name,
- "tracks": [t.id for t in p.tracks],
- "created_at": _dt_to_str(p.created_at),
- "description": p.description,
- "spotify": self._spotify_to_dict(p.spotify),
- }
-
- def _tracklist_to_dict(self, tl: TrackList) -> dict[str, Any]:
- return {"tracks": [t.id for t in tl.tracks]}
-
- def _album_to_dict(self, a: Album) -> dict[str, Any]:
- return {
- "title": a.title,
- "release_date": a.release_date,
- "tracks": [t.id for t in a.tracks],
- "spotify": self._spotify_to_dict(a.spotify),
- }
-
- def _local_to_dict(self, local):
- if not local:
- return None
- return {"path": local.path}
-
- def _spotify_to_dict(self, s: Optional[SpotifyData]) -> Optional[dict[str, Any]]:
- if not s:
- return None
- return {
- "id": s.id,
- "uri": s.uri,
- "url": s.url,
- "href": s.href,
- "file_path": _path_to_str(s.file_path),
- "popularity": s.popularity,
- "is_playable": s.is_playable,
- "is_local": s.is_local,
- "external_ids": s.external_ids,
- "artists": [{"id": a.id, "name": a.name, "uri": a.uri, "url": a.url} for a in s.artists],
- "album": {
- "id": s.album.id,
- "name": s.album.name,
- "release_date": s.album.release_date,
- "total_tracks": s.album.total_tracks,
- "uri": s.album.uri,
- "url": s.album.url,
- } if s.album else None,
- }
+ json.dump(data, f, indent=2, ensure_ascii=False)
class LibraryLoader:
def __init__(self):
- # Map of track id -> Track object
self.track_map: dict[str, Track] = {}
+ self.album_map: dict[str, Album] = {}
+ self.artist_map: dict[str, Artist] = {}
+
+ def load_track(self, data: dict[str, Any]) -> Track:
+ track = self.track_map.get(data.get("id"), Track())
+ track.id = data.get("id")
+ track.title = data.get("title")
+ track.duration_ms = data.get("duration_ms")
+ track.added_at = _str_to_dt(data.get("added_at"))
+ track.local_path = data.get("local_path")
+ track.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
+ track.album = self.album_map.get(data.get("album"), Album())
+
+ self.track_map[track.id] = track
+ return track
+
+ def load_artist(self, data: dict[str, Any]):
+ artist = self.artist_map.get(data.get("id"), Artist())
+ artist.id = data.get("id")
+ artist.title = data.get("title")
+ artist.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+
+ self.artist_map[artist.id] = artist
+ return artist
+
+ def load_album(self, data: dict[str, Any]):
+ album = self.album_map.get(data.get("id"), Album())
+ album.id = data.get("id")
+ album.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+ album.title = data.get("title")
+ album.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
+
+ self.album_map[album.id] = album
+ return album
def load(self, path: Path) -> Library:
with path.open("r", encoding="utf-8") as f:
data = json.load(f)
- # Load tracks first
- tracks = []
- for t in data.get("tracks", []):
- track = self._track_from_dict(t)
- tracks.append(track)
- self.track_map[track.id] = track
+ library = Library()
- # Load playlists using track_map
- playlists = []
- for p in data.get("playlists", []):
- pl_tracks = []
- for tid in p.get("tracks", []):
- if tid not in self.track_map:
- raise KeyError(f"Track ID '{tid}' not found in track map")
- pl_tracks.append(self.track_map[tid])
+ library.tracks = [self.load_track(track) for track in data.get("tracks")]
+ library.albums = [self.load_album(album) for album in data.get("albums")]
+ library.artists = [self.load_artist(artist) for artist in data.get("artists")]
- playlists.append(Playlist(
- name=p["name"],
- tracks=pl_tracks,
- created_at=_str_to_dt(p.get("created_at")),
- description=p.get("description"),
- spotify=self._spotify_from_dict(p.get("spotify")),
- ))
+ library.top_artists = [self.artist_map[artist] for artist in data.get("top_artists")]
+ library.top_tracks = [self.track_map[track] for track in data.get("top_tracks")]
+ library.liked_tracks = [self.track_map[track] for track in data.get("liked_tracks")]
-
- # Load tracklists
- tracklists = []
- for tl in data.get("tracklists", []):
- tl_tracks = []
- for tid in tl.get("tracks", []):
- if tid not in self.track_map:
- raise KeyError(f"Track ID '{tid}' not found in track map (tracklist)")
- tl_tracks.append(self.track_map[tid])
- tracklists.append(TrackList(tracks=tl_tracks))
-
- # Load albums
- albums = []
- for a in data.get("albums", []):
- alb_tracks = []
- for tid in a.get("tracks", []):
- if tid not in self.track_map:
- raise KeyError(f"Track ID '{tid}' not found in track map (album '{a.get('title')}')")
- alb_tracks.append(self.track_map[tid])
- albums.append(Album(
- title=a["title"],
- release_date=a.get("release_date"),
- tracks=alb_tracks,
- spotify=self._spotify_from_dict(a.get("spotify")),
- ))
-
- return Library(playlists=playlists, tracks=tracks, tracklists=tracklists, albums=albums)
-
- def _track_from_dict(self, d: dict[str, Any]) -> Track:
- return Track(
- id=d["id"],
- title=d["title"],
- artists=d["artists"],
- album=d.get("album"),
- duration_ms=d.get("duration_ms"),
- added_at=_str_to_dt(d.get("added_at")),
- spotify=self._spotify_from_dict(d.get("spotify")),
- local=self._local_from_dict(d.get("local")),
- )
-
- def _local_from_dict(self, d):
- if not d:
- return None
- return LocalData(
- path=d["path"],
- )
-
- def _spotify_from_dict(self, d: Optional[dict[str, Any]]) -> Optional[SpotifyData]:
- if not d:
- return None
- return SpotifyData(
- id=d["id"],
- uri=d["uri"],
- url=d["url"],
- href=d["href"],
- file_path=_str_to_path(d.get("file_path")),
- popularity=d.get("popularity"),
- is_playable=d.get("is_playable"),
- is_local=d.get("is_local"),
- external_ids=d.get("external_ids"),
- artists=[SpotifyArtist(**a) for a in d.get("artists", [])],
- album=SpotifyAlbum(**d["album"]) if d.get("album") else None,
- )
+ return library
diff --git a/src/MapLocalToSpotify.py b/src/MapLocalToSpotify.py
index dae5252..84e37c5 100644
--- a/src/MapLocalToSpotify.py
+++ b/src/MapLocalToSpotify.py
@@ -22,17 +22,17 @@ def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_
with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
for track in local_lib.tracks:
- parts = [track.title, track.artists[0] if len(track.artists) else "", track.album]
+ parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title]
search_pattern = " ".join(p for p in parts if p)
if search_pattern == "":
- print(f"cannot generate search pattern for the track - {track.local.path}")
+ print(f"cannot generate search pattern for the track - {track.local_path}")
pbar.update(1)
continue
found_tracks = find_song(search_pattern)[0:10]
found_tracks_map[track.id] = found_tracks
- search_log.append({"path" : track.local.path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
+ search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
pbar.update(1)
save_data(search_log, "spotify_search_log")
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
index 4991ede..602e89e 100644
--- a/src/local/ParseLocal.py
+++ b/src/local/ParseLocal.py
@@ -1,40 +1,139 @@
-from datetime import datetime
-from typing import Any, Iterable
-from pathlib import Path
import json
+import re
from src.Library import *
+import re
+
+PROTECTED_ARTISTS = {
+ "Tyler, The Creator",
+ "The Good, The Bad & The Queen",
+}
+
+def split_artists(artist_tag: str) -> list[str]:
+ if not artist_tag:
+ return []
+
+ placeholder_comma = "§COMMA§"
+ placeholder_amp = "§AMP§"
+
+ protected_map = {}
+
+ # Protect known artist names
+ for artist in PROTECTED_ARTISTS:
+ protected = (
+ artist
+ .replace(",", placeholder_comma)
+ .replace("&", placeholder_amp)
+ )
+ protected_map[protected] = artist
+ artist_tag = artist_tag.replace(artist, protected)
+
+ # Split remaining separators
+ parts = re.split(r"[,&]", artist_tag)
+
+ # Restore protected names and trim spaces
+ result = []
+ for part in parts:
+ part = part.strip()
+ if not part:
+ continue
+
+ part = (
+ part
+ .replace(placeholder_comma, ",")
+ .replace(placeholder_amp, "&")
+ )
+
+ result.append(part)
+
+ return result
+
+
class LocalParser:
def __init__(self):
- pass
-
- def parse_track_item(self, id, track: dict[str, Any]) -> Track:
- track = Track(
- title=track["name"],
- artists=[track.get("artist", [])],
- album=track["album"] if track.get("album") else None,
- duration_ms=None,
- added_at=None,
- spotify=None,
- id=id,
- local=LocalData(path=track["path"])
- )
- return track
+ self.track_map: dict[str, Track] = {}
+ self.artist_map: dict[str, Artist] = {}
+ self.album_map: dict[str, Album] = {}
+ self.library = Library()
- def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
- return [self.parse_track_item(id, track) for id, track in items.items()]
+ def parse_artists(self, data: str) -> List[Artist]:
+ if not data:
+ data = "unknown"
- def build_library_from_tracks(self, tracks_content_json: dict[str, list[dict[str, Any]]]) -> Library:
- library = Library()
+ data = split_artists(data)
- tracks = self.parse_all_tracks(tracks_content_json)
- library.tracklists.append(TrackList(tracks=list(tracks)))
- library.tracks.extend(tracks)
+ artists = []
+
+ for artist_data in data:
+ name = artist_data
+ artist_id = name
+
+ if artist_id in self.artist_map:
+ artists.append(self.artist_map[artist_id])
+ continue
+
+ artist = Artist()
+ artist.id = artist_id
+ artist.title = name
+
+ self.artist_map[artist_id] = artist
+ self.library.artists.append(artist)
+
+ artists.append(artist)
+
+ return artists
+
+ def parse_album(self, data: str) -> Album:
+ if not data:
+ data = "unknown"
+
+ name = data
+ album_id = name
+
+ if album_id in self.album_map:
+ return self.album_map[album_id]
+
+ album = Album()
+ album.id = album_id
+ album.title = name
+
+ self.album_map[album_id] = album
+ self.library.albums.append(album)
+
+ return album
+
+ def add_tracks_to_albums_and_artists(self):
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if track not in artist.tracks:
+ artist.tracks.append(track)
+
+ if track not in track.album.tracks:
+ track.album.tracks.append(track)
- return library
def parse(self, tracks_file: Path) -> Library:
tracks_content = json.loads(tracks_file.read_text())
- return self.build_library_from_tracks(tracks_content)
+
+ for track_id, data in tracks_content.items():
+ track = Track()
+
+ track.id = track_id
+ track.title = data.get("name")
+ track.artists = self.parse_artists(data.get("artist"))
+ track.album = self.parse_album(data.get("album"))
+ track.local_path = data.get("path")
+
+ self.track_map[track.id] = track
+ self.library.tracks.append(track)
+
+ self.add_tracks_to_albums_and_artists()
+
+ return self.library
+
+
+ def parse_track_item(self, id, track: dict[str, Any]) -> Track:
+
+ return track
diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py
index 535caa0..49882cb 100644
--- a/src/spotify/ParseSpotify.py
+++ b/src/spotify/ParseSpotify.py
@@ -1,9 +1,10 @@
-from datetime import datetime
+from src.Library import *
+
from typing import Any, Iterable
-from pathlib import Path
import json
-from src.Library import *
+from datetime import datetime
+from pathlib import Path
def get_artists_str(artists_pack):
@@ -11,129 +12,117 @@ def get_artists_str(artists_pack):
artists = " ".join(track_artists)
return artists
+
class Parser:
def __init__(self):
- # state: Spotify track ID -> Track object
self.track_map: dict[str, Track] = {}
+ self.artist_map: dict[str, Artist] = {}
+ self.album_map: dict[str, Album] = {}
+ self.library = Library()
- # -------------------------
- # Low-level helpers
- # -------------------------
- @staticmethod
- def _parse_spotify_datetime(value: str | None) -> datetime | None:
- if not value:
- return None
- return datetime.fromisoformat(value.replace("Z", "+00:00"))
+ def parse(self, playlists: Path, tracks: Path, top_tracks: Path, top_artists: Path) -> Library:
+ self.library = Library()
- @staticmethod
- def _parse_spotify_artist(a: dict[str, Any]) -> SpotifyArtist:
- return SpotifyArtist(
- id=a["id"],
- name=a["name"],
- uri=a["uri"],
- url=a["external_urls"]["spotify"],
- )
+ self.parse_tracks(json.loads(tracks.read_text()))
+ self.parse_playlists(json.loads(playlists.read_text()))
- @staticmethod
- def _parse_spotify_album(a: dict[str, Any]) -> SpotifyAlbum:
- return SpotifyAlbum(
- id=a["id"],
- name=a["name"],
- release_date=a.get("release_date"),
- total_tracks=a.get("total_tracks", 0),
- uri=a["uri"],
- url=a["external_urls"]["spotify"],
- )
+ self.parse_top_artists(json.loads(top_artists.read_text()))
+ self.parse_top_tracks(json.loads(top_tracks.read_text()))
- def _parse_spotify_data(self, track_json: dict[str, Any]) -> SpotifyData:
- return SpotifyData(
- id=track_json["id"],
- uri=track_json["uri"],
- url=track_json["external_urls"]["spotify"],
- href=track_json["href"],
- popularity=track_json.get("popularity"),
- is_playable=track_json.get("is_playable"),
- is_local=track_json.get("is_local"),
- external_ids=track_json.get("external_ids", {}),
- artists=[self._parse_spotify_artist(a) for a in track_json.get("artists", [])],
- album=self._parse_spotify_album(track_json["album"])
- if track_json.get("album")
- else None,
- )
+ self.add_tracks_to_albums_and_artists()
- # -------------------------
- # Track parsers
- # -------------------------
- def parse_track_item(self, item: dict[str, Any]) -> Track:
- """
- Parses a single track item and returns Track object.
- Reuses existing Track from track_map if already parsed.
- """
- t = item["track"]
- track_id = t["id"]
+ return self.library
+
+ def add_tracks_to_albums_and_artists(self):
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if track not in artist.tracks:
+ artist.tracks.append(track)
+
+ if track not in track.album.tracks:
+ track.album.tracks.append(track)
+
+
+ def parse_playlists(self, data):
+ for name, tracks in data.items():
+ playlist = Playlist()
+ playlist.title = name
+ for track in tracks:
+ track_data = track.get("track")
+ if track_data:
+ track_id = track_data.get("id")
+ playlist.tracks.append(self.parse_track(track_data, track.get("added_at")))
+
+ self.library.playlists.append(playlist)
+
+ def parse_top_tracks(self, data):
+ for track in data:
+ if track.get("id") in self.track_map:
+ self.library.top_tracks.append(self.track_map.get(track.get("id")))
+
+ def parse_top_artists(self, data):
+ for artist in data:
+ if artist.get("id") in self.artist_map:
+ self.library.top_artists.append(self.artist_map.get(artist.get("id")))
+
+ def parse_tracks(self, data):
+ for track in data:
+ self.parse_track(track.get("track"), track.get("added_at"))
+
+ for track in data:
+ self.library.liked_tracks.append(self.track_map.get(track.get("track").get("id")))
+
+ def parse_track(self, data: dict[str, Any], added_time : datetime) -> Track:
+ track_id = data.get("id")
- # return existing track if already parsed
if track_id in self.track_map:
return self.track_map[track_id]
- track = Track(
- title=t["name"],
- artists=[a["name"] for a in t.get("artists", [])],
- album=t["album"]["name"] if t.get("album") else None,
- duration_ms=t.get("duration_ms"),
- added_at=self._parse_spotify_datetime(item.get("added_at")),
- spotify=self._parse_spotify_data(t),
- id=self._parse_spotify_data(t).id
- )
+ track = Track()
self.track_map[track_id] = track
+
+ track.id = track_id
+ track.title = data.get("name")
+ track.artists = [self.parse_artist(artist) for artist in data.get("artists")]
+ track.album = self.parse_album(data.get("album"))
+ track.duration_ms = data.get("duration_ms")
+
+ self.library.tracks.append(track)
+
return track
- def parse_track_items(self, items: Iterable[dict[str, Any]]) -> list[Track]:
- return [self.parse_track_item(i) for i in items if i.get("track")]
+ def parse_album(self, data):
+ album_id = data.get("id")
- # -------------------------
- # Playlist parsers
- # -------------------------
- def parse_playlist_content(self, playlist_name: str, items: list[dict[str, Any]]) -> Playlist:
- tracks = self.parse_track_items(items)
- return Playlist(name=playlist_name, tracks=tracks)
+ if album_id in self.album_map:
+ return self.album_map[album_id]
- def parse_all_playlist_content(self, data: dict[str, list[dict[str, Any]]]) -> list[Playlist]:
- playlists: list[Playlist] = []
- for name, items in data.items():
- playlists.append(self.parse_playlist_content(name, items))
- return playlists
+ album = Album()
- def parse_all_tracks(self, items: dict[str, list[dict[str, Any]]]) -> list[Track]:
- return [self.parse_track_item(i) for i in items]
+ self.album_map[album_id] = album
- # -------------------------
- # Library builder
- # -------------------------
- def build_library_from_playlist_content(
- self, playlist_content_json: dict[str, list[dict[str, Any]]],
- tracks_content_json: dict[str, list[dict[str, Any]]]
- ) -> Library:
- library = Library()
+ album.id = data.get("id")
+ album.title = data.get("name")
+ album.artists = [self.parse_artist(artist) for artist in data.get("artists")]
- self.parse_all_tracks(tracks_content_json)
+ self.library.albums.append(album)
- library.tracklists.append(TrackList(tracks=list((self.track_map.values()))))
+ return album
- library.playlists = self.parse_all_playlist_content(playlist_content_json)
- library.tracks.extend(self.track_map.values())
+ def parse_artist(self, data: dict[str, Any]) -> Artist:
+ artist_id = data.get("id")
- return library
+ if artist_id in self.artist_map:
+ return self.artist_map[artist_id]
- # -------------------------
- # Convenience method
- # -------------------------
- def parse(
- self,
- playlist_content_file: Path,
- tracks_file: Path,
- ) -> Library:
- playlist_content = json.loads(playlist_content_file.read_text())
- tracks_content = json.loads(tracks_file.read_text())
- return self.build_library_from_playlist_content(playlist_content, tracks_content)
+ artist = Artist()
+
+ self.artist_map[artist_id] = artist
+
+ artist.id = data.get("id")
+ artist.title = data.get("name")
+
+ self.library.artists.append(artist)
+
+ return artist
From 200b8b18635c7a066fb1bc0c704d0bf10d2b84eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Mon, 5 Jan 2026 01:19:00 +0300
Subject: [PATCH 11/30] fixes
---
.gitignore | 3 +
Flow.py | 5 +
main.py | 10 +-
old/env | 36 --------
old/main.py | 195 +--------------------------------------
src/LibrarySerializer.py | 37 ++++++--
src/StatGenerator.py | 81 +++++++++-------
7 files changed, 91 insertions(+), 276 deletions(-)
delete mode 100755 old/env
diff --git a/.gitignore b/.gitignore
index 2803b17..880f9c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,6 @@ prefetched*
__pycache__
.idea
secret
+work_dir
+old
+tmp
\ No newline at end of file
diff --git a/Flow.py b/Flow.py
index bb35731..e982d39 100644
--- a/Flow.py
+++ b/Flow.py
@@ -11,6 +11,7 @@ from src.local.LocalLibraryIndexer import update_local_library
from src.local.ParseLocal import LocalParser
from src.MapLocalToSpotify import map_local_to_spotify
+from src.StatGenerator import log_stats
class Flow:
@@ -51,6 +52,10 @@ class Flow:
def map_local_to_spotify(self):
map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
+ def log_stats(self):
+ log_stats(work_dir / "local_to_spotify_id_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
+
+
def run(self, arg):
self.fetch_local()
# self.do_fetch_spotify()
diff --git a/main.py b/main.py
index 98b4efa..1069171 100644
--- a/main.py
+++ b/main.py
@@ -5,13 +5,15 @@ if __name__ == '__main__':
flow = Flow.Flow()
# flow.fetch_spotify()
- flow.fetch_local()
+ # flow.fetch_local()
- flow.parse_spotify_library()
- flow.parse_local_library()
+ # flow.parse_spotify_library()
+ # flow.parse_local_library()
flow.load_libraries()
- flow.map_local_to_spotify()
+ # flow.map_local_to_spotify()
+
+ flow.log_stats()
print("asd")
diff --git a/old/env b/old/env
deleted file mode 100755
index 8955572..0000000
--- a/old/env
+++ /dev/null
@@ -1,36 +0,0 @@
-
-PS1=" > "
-
-#export PATH="/usr/local/bin/:$PATH"
-export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH"
-#export PATH="$HOME/home/auser/.local/bin:$PATH"
-
-export SIP="185.238.170.251"
-
-alias v=nvim
-alias ll="ls -l -a"
-alias gl="git log --oneline"
-alias ssh_server="ssh auser@185.238.170.251"
-
-#eval "$(zoxide init bash --cmd zd)"
-
-#source $PMAIN/.scripts/bashmarks.sh
-
-fe() {
- local result=$(command tere "$@")
- [ -n "$result" ] && cd -- "$result"
-}
-
-vim_configure() {
- pwd=$(pwd)
- cd ~/.config/nvim/
- nvim
- cd $pwd
-}
-
-pyenv() {
- source ~/bin/python/env311/bin/activate
-}
-
-source ~/src/scripts/remotes.sh
-source /usr/share/fzf/key-bindings.bash
diff --git a/old/main.py b/old/main.py
index 10d378a..0b2a991 100644
--- a/old/main.py
+++ b/old/main.py
@@ -1,79 +1,3 @@
-from DataBase import *
-from LocalLibraryIndexer import update_local_library
-from LinkerPatternGenerator import run_pattern_generators
-from LocalPLaylistGenerator import generate_playlists
-
-from SpotifyWebAPI import fetch_data, update_access_token
-import cmd
-
-local_library_path = "/home/auser/Music/"
-rel_autogen_dir = "0autogen"
-
-playlists = get_data('playlistTracks')
-top_artists = get_data('top_artists')
-top_tracks = get_data('top_tracks')
-user_tracks = get_data('tracks')
-link_pattern = get_data('link_pattern_spotify')
-local_library = get_data("local_library")
-user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
-
-
-def get_playlist_names(pl):
- return [name for name, items in pl.items()]
-
-
-def print_playlist_tracks(pl_id: int):
- pls = list(playlists)
- name = pls[pl_id]
- pl = playlists[name]
- print(name)
- for track in pl:
- track_data = track['track']
- track_name = track_data['name']
- artists = get_artists_str(track_data['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_top_artists():
- for artist in top_artists:
- name = artist['name']
- print(f" '{name}'")
-
-
-def print_top_tracks():
- for track in top_tracks:
- track_name = track['name']
- artists = get_artists_str(track['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_stats():
- print(f" tracks - {len(user_tracks)}")
- print(f" playlists - {len(playlists)}")
-
- for name, tracks in playlists.items():
- print(f" '{name}' - {len(tracks)}")
-
- print(f" top tracks - {len(top_tracks)}")
- print(f" top artists - {len(top_artists)}")
-
-
-def print_tracks():
- track_idx = 0
- for trackItem in user_tracks:
- track = trackItem['track']
- print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
- track_idx += 1
-
-
-def sort_remote_tracks_by_popularity(track_ids):
- out = []
- for top_track in top_tracks:
- if top_track['id'] in track_ids:
- out.append(top_track['id'])
- return out
-
-
def print_link_stats(link_pattern):
resolved_reversed = {}
resolved = []
@@ -114,121 +38,4 @@ def print_link_stats(link_pattern):
print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
-
-
-class Interpreter(cmd.Cmd):
- intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
- prompt = "(spotify) "
-
- def do_list_stat_vs_types(self, arg):
- """prints available linking patterns"""
- print(f"0 - fuzzy tags")
- print(f"1 - spotify")
- print(f"2 - fuzzy")
-
- def do_stat_local_library_vs_spotify(self, arg):
- """compares local library and remote with the link pattern generated by the generate_pattern_command"""
- print_link_stats(link_pattern)
-
- def do_playlists(self, arg):
- """prints user playlists"""
- pl_id = 0
- for pl in get_playlist_names(playlists):
- print(f"{pl_id} - {pl}")
- pl_id += 1
-
- def do_playlist_tracks(self, arg):
- """prints playlist [name]"""
- try:
- pl_id = int(arg.split()[0])
- print_playlist_tracks(pl_id)
- except IndexError:
- print("Invalid input")
-
- def do_top_artists(self, arg):
- """Top artists"""
- print_top_artists()
-
- def do_top_tracks(self, arg):
- """Top tracks"""
- print_top_tracks()
-
- def do_stat(self, arg):
- """Print stats"""
- print_stats()
-
- def do_tracks(self, arg):
- """Print tracks"""
- print_tracks()
-
- def do_generate_local_playlists(self, arg):
- """generates local playlists based on the vs comparison type"""
- generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
-
-
- def do_index_local_library(self, arg):
- """find local tracks and fetches the data from them for further analysis"""
- global local_library
- update_local_library(local_library_path)
- local_library = get_data("local_library")
-
- def do_generate_vs_stat(self, arg):
- """compares local and remote library and generates the vs stats"""
-
- global link_pattern
- run_pattern_generators()
- link_pattern = get_data('link_pattern_spotify')
-
- def do_fetch(self, arg):
- """Fetch all spotify data"""
- try:
- update_access_token()
- fetch_data()
-
- global playlists, top_artists, top_tracks, user_tracks
-
- playlists = get_data('playlistTracks')
- top_artists = get_data('top_artists')
- top_tracks = get_data('top_tracks')
- user_tracks = get_data('tracks')
-
- except ValueError:
- print("Can not fetch the data.")
-
- def do_generate_todo(self, arg):
- """generates todos for the pirates"""
- print("sorry not implemented yet.")
-
- def do_run_all_default(self, arg):
- """runs general flow"""
-
- local_lib_path = "/home/auser/Music/"
- print("Fetching spotify data...")
- self.do_fetch([])
-
- print("Fetching local data...")
- self.do_index_local_library(f"{local_lib_path}/organized")
-
- print("Generating vs patterns...")
- self.do_generate_vs_stat("0")
-
- print("Generating todos...")
- self.do_generate_todo(f"{local_lib_path}")
-
- print("Generating local playlists...")
- self.do_generate_local_playlists(f"{local_lib_path}")
-
- return True
-
- def do_exit(self, arg):
- """Exit the command loop: exit"""
- print("Goodbye!")
- return True
-
-
-if __name__ == '__main__':
- try:
- Interpreter().cmdloop()
- except KeyboardInterrupt as kb:
- print("process terminated")
\ No newline at end of file
+ index += 1
\ No newline at end of file
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index bc968ea..cbccd9f 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -88,34 +88,55 @@ class LibraryLoader:
self.album_map: dict[str, Album] = {}
self.artist_map: dict[str, Artist] = {}
+ def get_artist(self, artist_id) -> Artist:
+ artist = self.artist_map.get(artist_id)
+ if artist is None:
+ artist = Artist()
+ self.artist_map[artist_id] = artist
+ return artist
+
+ def get_album(self, id) -> Album:
+ item = self.album_map.get(id)
+ if item is None:
+ item = Album()
+ self.album_map[id] = item
+ return item
+
+ def get_track(self, id) -> Track:
+ item = self.track_map.get(id)
+ if item is None:
+ item = Track()
+ self.track_map[id] = item
+ return item
+
def load_track(self, data: dict[str, Any]) -> Track:
- track = self.track_map.get(data.get("id"), Track())
+ track = self.get_track(data.get("id"))
track.id = data.get("id")
track.title = data.get("title")
track.duration_ms = data.get("duration_ms")
track.added_at = _str_to_dt(data.get("added_at"))
track.local_path = data.get("local_path")
- track.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
- track.album = self.album_map.get(data.get("album"), Album())
+ track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
+ track.album = self.get_album(data.get("album"))
self.track_map[track.id] = track
return track
def load_artist(self, data: dict[str, Any]):
- artist = self.artist_map.get(data.get("id"), Artist())
+ artist = self.get_artist(data.get("id"))
artist.id = data.get("id")
artist.title = data.get("title")
- artist.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+ artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
self.artist_map[artist.id] = artist
return artist
def load_album(self, data: dict[str, Any]):
- album = self.album_map.get(data.get("id"), Album())
+ album = self.get_album(data.get("id"))
album.id = data.get("id")
- album.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+ album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
album.title = data.get("title")
- album.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
+ album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
self.album_map[album.id] = album
return album
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index c40bebb..d6ebc41 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -1,42 +1,55 @@
+from src.Library import *
-def print_link_stats(link_pattern):
- resolved_reversed = {}
- resolved = []
- unresolved_local = []
- unresolved_remote = []
+from pathlib import Path
+import json
- for local_id, links in link_pattern.items():
- if len(links['items']):
- resolved.append(local_id)
- resolved_reversed[links['items'][0]] = local_id
- else:
- unresolved_local.append(local_id)
+out = {
+ "local_count": 0,
+ "resolved_count": 0,
+ "unresolved_tracks": [],
+}
- for track in user_tracks:
- track_id = track['track']['id']
- if track_id not in resolved_reversed:
- unresolved_remote.append(track_id)
+def get_mapping(mapping_path) -> dict[str, Any]:
+ data = {}
- print(f"Local Tracks: {len(local_library)}")
- print(f"Remote Tracks: {len(user_tracks)}")
+ with Path(mapping_path).open("r", encoding="utf-8") as f:
+ data = json.load(f)
- unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
- unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+ out = {}
- print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
- index = 0
- for remote_id in unresolved_remote:
- remote_track = user_tracks_by_id[remote_id]
- print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
+ for local, remotes in data.items():
+ if len(remotes["items"]):
+ out[local] = remotes["items"][0]
- print(f"\nResolved tracks: {len(resolved)}")
- index = 0
- for link in resolved:
- remote_id = link_pattern[link]['items'][0]
- local_track = local_library[link]
- remote_track = user_tracks_by_id[remote_id]
+ return out
- print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
- print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
\ No newline at end of file
+
+def find_add_local_paths(mapping_path: Path, local_lib: Library, remote_lib: Library):
+ mapping = get_mapping(mapping_path)
+
+ out["resolved_count"] = len(mapping)
+
+ remote_id_map = {track.id: track for track in remote_lib.tracks}
+ local_id_map = {track.id: track for track in local_lib.tracks}
+
+ for local, remote in mapping.items():
+ remote_id_map[remote].local_path = local_id_map[local].local_path
+
+
+def generate_stats(local_lib: Library, remote_lib: Library, output_path : Path):
+
+ out["local_count"] = len(local_lib.tracks)
+
+ for track in remote_lib.top_tracks:
+ if not track.local_path:
+ out["unresolved_tracks"].append(
+ f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}")
+
+
+ with output_path.open("w", encoding="utf-8") as f:
+ json.dump(out, f, indent=2, ensure_ascii=False)
+
+
+def log_stats(mapping_path : Path, output_path : Path, local_lib: Library, remote_lib: Library):
+ find_add_local_paths(mapping_path, local_lib, remote_lib)
+ generate_stats(local_lib, remote_lib, output_path)
From 20c7b28ed69e4cafe58c2584e9df9d5cb92009f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Mon, 5 Jan 2026 01:19:00 +0300
Subject: [PATCH 12/30] fixes
---
.gitignore | 3 +
Flow.py | 5 +
main.py | 10 +-
old/env | 36 --------
old/main.py | 195 +--------------------------------------
src/LibrarySerializer.py | 37 ++++++--
src/StatGenerator.py | 81 +++++++++-------
7 files changed, 91 insertions(+), 276 deletions(-)
delete mode 100755 old/env
diff --git a/.gitignore b/.gitignore
index 2803b17..880f9c3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,6 @@ prefetched*
__pycache__
.idea
secret
+work_dir
+old
+tmp
\ No newline at end of file
diff --git a/Flow.py b/Flow.py
index bb35731..e982d39 100644
--- a/Flow.py
+++ b/Flow.py
@@ -11,6 +11,7 @@ from src.local.LocalLibraryIndexer import update_local_library
from src.local.ParseLocal import LocalParser
from src.MapLocalToSpotify import map_local_to_spotify
+from src.StatGenerator import log_stats
class Flow:
@@ -51,6 +52,10 @@ class Flow:
def map_local_to_spotify(self):
map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
+ def log_stats(self):
+ log_stats(work_dir / "local_to_spotify_id_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
+
+
def run(self, arg):
self.fetch_local()
# self.do_fetch_spotify()
diff --git a/main.py b/main.py
index 98b4efa..1069171 100644
--- a/main.py
+++ b/main.py
@@ -5,13 +5,15 @@ if __name__ == '__main__':
flow = Flow.Flow()
# flow.fetch_spotify()
- flow.fetch_local()
+ # flow.fetch_local()
- flow.parse_spotify_library()
- flow.parse_local_library()
+ # flow.parse_spotify_library()
+ # flow.parse_local_library()
flow.load_libraries()
- flow.map_local_to_spotify()
+ # flow.map_local_to_spotify()
+
+ flow.log_stats()
print("asd")
diff --git a/old/env b/old/env
deleted file mode 100755
index 8955572..0000000
--- a/old/env
+++ /dev/null
@@ -1,36 +0,0 @@
-
-PS1=" > "
-
-#export PATH="/usr/local/bin/:$PATH"
-export PATH="$HOME/bin/scripts:$HOME/bin/:$PATH"
-#export PATH="$HOME/home/auser/.local/bin:$PATH"
-
-export SIP="185.238.170.251"
-
-alias v=nvim
-alias ll="ls -l -a"
-alias gl="git log --oneline"
-alias ssh_server="ssh auser@185.238.170.251"
-
-#eval "$(zoxide init bash --cmd zd)"
-
-#source $PMAIN/.scripts/bashmarks.sh
-
-fe() {
- local result=$(command tere "$@")
- [ -n "$result" ] && cd -- "$result"
-}
-
-vim_configure() {
- pwd=$(pwd)
- cd ~/.config/nvim/
- nvim
- cd $pwd
-}
-
-pyenv() {
- source ~/bin/python/env311/bin/activate
-}
-
-source ~/src/scripts/remotes.sh
-source /usr/share/fzf/key-bindings.bash
diff --git a/old/main.py b/old/main.py
index 10d378a..0b2a991 100644
--- a/old/main.py
+++ b/old/main.py
@@ -1,79 +1,3 @@
-from DataBase import *
-from LocalLibraryIndexer import update_local_library
-from LinkerPatternGenerator import run_pattern_generators
-from LocalPLaylistGenerator import generate_playlists
-
-from SpotifyWebAPI import fetch_data, update_access_token
-import cmd
-
-local_library_path = "/home/auser/Music/"
-rel_autogen_dir = "0autogen"
-
-playlists = get_data('playlistTracks')
-top_artists = get_data('top_artists')
-top_tracks = get_data('top_tracks')
-user_tracks = get_data('tracks')
-link_pattern = get_data('link_pattern_spotify')
-local_library = get_data("local_library")
-user_tracks_by_id = {track['track']['id']: track['track'] for track in user_tracks}
-
-
-def get_playlist_names(pl):
- return [name for name, items in pl.items()]
-
-
-def print_playlist_tracks(pl_id: int):
- pls = list(playlists)
- name = pls[pl_id]
- pl = playlists[name]
- print(name)
- for track in pl:
- track_data = track['track']
- track_name = track_data['name']
- artists = get_artists_str(track_data['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_top_artists():
- for artist in top_artists:
- name = artist['name']
- print(f" '{name}'")
-
-
-def print_top_tracks():
- for track in top_tracks:
- track_name = track['name']
- artists = get_artists_str(track['artists'])
- print(f" '{track_name}' - '{artists}' ")
-
-
-def print_stats():
- print(f" tracks - {len(user_tracks)}")
- print(f" playlists - {len(playlists)}")
-
- for name, tracks in playlists.items():
- print(f" '{name}' - {len(tracks)}")
-
- print(f" top tracks - {len(top_tracks)}")
- print(f" top artists - {len(top_artists)}")
-
-
-def print_tracks():
- track_idx = 0
- for trackItem in user_tracks:
- track = trackItem['track']
- print(f" {track_idx}: '{track['name']}' by '{get_artists_str(track['artists'])}'")
- track_idx += 1
-
-
-def sort_remote_tracks_by_popularity(track_ids):
- out = []
- for top_track in top_tracks:
- if top_track['id'] in track_ids:
- out.append(top_track['id'])
- return out
-
-
def print_link_stats(link_pattern):
resolved_reversed = {}
resolved = []
@@ -114,121 +38,4 @@ def print_link_stats(link_pattern):
print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
-
-
-class Interpreter(cmd.Cmd):
- intro = "Welcome to the interactive command loop. Type help or ? to list commands.\n"
- prompt = "(spotify) "
-
- def do_list_stat_vs_types(self, arg):
- """prints available linking patterns"""
- print(f"0 - fuzzy tags")
- print(f"1 - spotify")
- print(f"2 - fuzzy")
-
- def do_stat_local_library_vs_spotify(self, arg):
- """compares local library and remote with the link pattern generated by the generate_pattern_command"""
- print_link_stats(link_pattern)
-
- def do_playlists(self, arg):
- """prints user playlists"""
- pl_id = 0
- for pl in get_playlist_names(playlists):
- print(f"{pl_id} - {pl}")
- pl_id += 1
-
- def do_playlist_tracks(self, arg):
- """prints playlist [name]"""
- try:
- pl_id = int(arg.split()[0])
- print_playlist_tracks(pl_id)
- except IndexError:
- print("Invalid input")
-
- def do_top_artists(self, arg):
- """Top artists"""
- print_top_artists()
-
- def do_top_tracks(self, arg):
- """Top tracks"""
- print_top_tracks()
-
- def do_stat(self, arg):
- """Print stats"""
- print_stats()
-
- def do_tracks(self, arg):
- """Print tracks"""
- print_tracks()
-
- def do_generate_local_playlists(self, arg):
- """generates local playlists based on the vs comparison type"""
- generate_playlists(local_library, local_library_path, playlists, link_pattern, local_library_path + rel_autogen_dir)
-
-
- def do_index_local_library(self, arg):
- """find local tracks and fetches the data from them for further analysis"""
- global local_library
- update_local_library(local_library_path)
- local_library = get_data("local_library")
-
- def do_generate_vs_stat(self, arg):
- """compares local and remote library and generates the vs stats"""
-
- global link_pattern
- run_pattern_generators()
- link_pattern = get_data('link_pattern_spotify')
-
- def do_fetch(self, arg):
- """Fetch all spotify data"""
- try:
- update_access_token()
- fetch_data()
-
- global playlists, top_artists, top_tracks, user_tracks
-
- playlists = get_data('playlistTracks')
- top_artists = get_data('top_artists')
- top_tracks = get_data('top_tracks')
- user_tracks = get_data('tracks')
-
- except ValueError:
- print("Can not fetch the data.")
-
- def do_generate_todo(self, arg):
- """generates todos for the pirates"""
- print("sorry not implemented yet.")
-
- def do_run_all_default(self, arg):
- """runs general flow"""
-
- local_lib_path = "/home/auser/Music/"
- print("Fetching spotify data...")
- self.do_fetch([])
-
- print("Fetching local data...")
- self.do_index_local_library(f"{local_lib_path}/organized")
-
- print("Generating vs patterns...")
- self.do_generate_vs_stat("0")
-
- print("Generating todos...")
- self.do_generate_todo(f"{local_lib_path}")
-
- print("Generating local playlists...")
- self.do_generate_local_playlists(f"{local_lib_path}")
-
- return True
-
- def do_exit(self, arg):
- """Exit the command loop: exit"""
- print("Goodbye!")
- return True
-
-
-if __name__ == '__main__':
- try:
- Interpreter().cmdloop()
- except KeyboardInterrupt as kb:
- print("process terminated")
\ No newline at end of file
+ index += 1
\ No newline at end of file
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index bc968ea..cbccd9f 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -88,34 +88,55 @@ class LibraryLoader:
self.album_map: dict[str, Album] = {}
self.artist_map: dict[str, Artist] = {}
+ def get_artist(self, artist_id) -> Artist:
+ artist = self.artist_map.get(artist_id)
+ if artist is None:
+ artist = Artist()
+ self.artist_map[artist_id] = artist
+ return artist
+
+ def get_album(self, id) -> Album:
+ item = self.album_map.get(id)
+ if item is None:
+ item = Album()
+ self.album_map[id] = item
+ return item
+
+ def get_track(self, id) -> Track:
+ item = self.track_map.get(id)
+ if item is None:
+ item = Track()
+ self.track_map[id] = item
+ return item
+
def load_track(self, data: dict[str, Any]) -> Track:
- track = self.track_map.get(data.get("id"), Track())
+ track = self.get_track(data.get("id"))
track.id = data.get("id")
track.title = data.get("title")
track.duration_ms = data.get("duration_ms")
track.added_at = _str_to_dt(data.get("added_at"))
track.local_path = data.get("local_path")
- track.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
- track.album = self.album_map.get(data.get("album"), Album())
+ track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
+ track.album = self.get_album(data.get("album"))
self.track_map[track.id] = track
return track
def load_artist(self, data: dict[str, Any]):
- artist = self.artist_map.get(data.get("id"), Artist())
+ artist = self.get_artist(data.get("id"))
artist.id = data.get("id")
artist.title = data.get("title")
- artist.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+ artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
self.artist_map[artist.id] = artist
return artist
def load_album(self, data: dict[str, Any]):
- album = self.album_map.get(data.get("id"), Album())
+ album = self.get_album(data.get("id"))
album.id = data.get("id")
- album.tracks = [self.track_map.get(track_id, Track()) for track_id in data.get("tracks")]
+ album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
album.title = data.get("title")
- album.artists = [self.artist_map.get(artist_id, Artist()) for artist_id in data.get("artists")]
+ album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
self.album_map[album.id] = album
return album
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index c40bebb..d6ebc41 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -1,42 +1,55 @@
+from src.Library import *
-def print_link_stats(link_pattern):
- resolved_reversed = {}
- resolved = []
- unresolved_local = []
- unresolved_remote = []
+from pathlib import Path
+import json
- for local_id, links in link_pattern.items():
- if len(links['items']):
- resolved.append(local_id)
- resolved_reversed[links['items'][0]] = local_id
- else:
- unresolved_local.append(local_id)
+out = {
+ "local_count": 0,
+ "resolved_count": 0,
+ "unresolved_tracks": [],
+}
- for track in user_tracks:
- track_id = track['track']['id']
- if track_id not in resolved_reversed:
- unresolved_remote.append(track_id)
+def get_mapping(mapping_path) -> dict[str, Any]:
+ data = {}
- print(f"Local Tracks: {len(local_library)}")
- print(f"Remote Tracks: {len(user_tracks)}")
+ with Path(mapping_path).open("r", encoding="utf-8") as f:
+ data = json.load(f)
- unresolved_remote = sort_remote_tracks_by_popularity(unresolved_remote)
- unresolved_remote = unresolved_remote[0:min(len(unresolved_remote), 100)]
+ out = {}
- print(f"\nUnresolved tracks from the remote: {len(unresolved_remote)}")
- index = 0
- for remote_id in unresolved_remote:
- remote_track = user_tracks_by_id[remote_id]
- print(f" {index}: '{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
+ for local, remotes in data.items():
+ if len(remotes["items"]):
+ out[local] = remotes["items"][0]
- print(f"\nResolved tracks: {len(resolved)}")
- index = 0
- for link in resolved:
- remote_id = link_pattern[link]['items'][0]
- local_track = local_library[link]
- remote_track = user_tracks_by_id[remote_id]
+ return out
- print(f" {index}: '{local_track['name']}' by '{local_track['artist']}'", end=' ----> ')
- print(f"'{remote_track['name']}' by '{get_artists_str(remote_track['artists'])}'")
- index += 1
\ No newline at end of file
+
+def find_add_local_paths(mapping_path: Path, local_lib: Library, remote_lib: Library):
+ mapping = get_mapping(mapping_path)
+
+ out["resolved_count"] = len(mapping)
+
+ remote_id_map = {track.id: track for track in remote_lib.tracks}
+ local_id_map = {track.id: track for track in local_lib.tracks}
+
+ for local, remote in mapping.items():
+ remote_id_map[remote].local_path = local_id_map[local].local_path
+
+
+def generate_stats(local_lib: Library, remote_lib: Library, output_path : Path):
+
+ out["local_count"] = len(local_lib.tracks)
+
+ for track in remote_lib.top_tracks:
+ if not track.local_path:
+ out["unresolved_tracks"].append(
+ f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}")
+
+
+ with output_path.open("w", encoding="utf-8") as f:
+ json.dump(out, f, indent=2, ensure_ascii=False)
+
+
+def log_stats(mapping_path : Path, output_path : Path, local_lib: Library, remote_lib: Library):
+ find_add_local_paths(mapping_path, local_lib, remote_lib)
+ generate_stats(local_lib, remote_lib, output_path)
From 050b28fde8e694128d4b576f9b9241ffb37c9323 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Mon, 5 Jan 2026 14:53:06 +0300
Subject: [PATCH 13/30] nice stat generator
---
src/StatGenerator.py | 207 ++++++++++++++++++++++++++++++++++++++-----
1 file changed, 183 insertions(+), 24 deletions(-)
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index d6ebc41..b6dcaee 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -3,11 +3,10 @@ from src.Library import *
from pathlib import Path
import json
-out = {
- "local_count": 0,
- "resolved_count": 0,
- "unresolved_tracks": [],
-}
+
+def is_resolved(track: Track):
+ return track.local_path is not None
+
def get_mapping(mapping_path) -> dict[str, Any]:
data = {}
@@ -24,32 +23,192 @@ def get_mapping(mapping_path) -> dict[str, Any]:
return out
-def find_add_local_paths(mapping_path: Path, local_lib: Library, remote_lib: Library):
- mapping = get_mapping(mapping_path)
+class AlbumStat:
+ def __init__(self):
+ self.album: Album = Album()
+ self.score: int = 0
+ self.resolved_precent: float = 0
- out["resolved_count"] = len(mapping)
+ def calc_resolved_precent(self):
+ self.resolved_precent = (sum(1 for track in self.album.tracks if is_resolved(track)) / len(
+ self.album.tracks)) * 100
- remote_id_map = {track.id: track for track in remote_lib.tracks}
- local_id_map = {track.id: track for track in local_lib.tracks}
-
- for local, remote in mapping.items():
- remote_id_map[remote].local_path = local_id_map[local].local_path
+ def get_str(self):
+ return (f"{str(int(self.resolved_precent))} % - "
+ f"{self.album.title} - "
+ f"{str(self.album.artists[0].title) if len(self.album.artists) else " "}")
-def generate_stats(local_lib: Library, remote_lib: Library, output_path : Path):
+class ArtistStat:
+ def __init__(self):
+ self.artist: Artist = Artist()
+ self.score: int = 0
+ self.resolved_precent: float = 0
+ self.albums_stats: list[AlbumStat] = []
- out["local_count"] = len(local_lib.tracks)
+ def calc_resolved_precent(self):
+ if len(self.artist.tracks) == 0:
+ self.resolved_precent = 0
+ return
- for track in remote_lib.top_tracks:
- if not track.local_path:
- out["unresolved_tracks"].append(
- f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}")
+ self.resolved_precent = (sum(1 for track in self.artist.tracks if is_resolved(track)) / len(
+ self.artist.tracks)) * 100
+
+ def get_str(self):
+ return (f"{str(int(self.resolved_precent))} % - "
+ f"{self.artist.title}")
- with output_path.open("w", encoding="utf-8") as f:
- json.dump(out, f, indent=2, ensure_ascii=False)
+class StatGenerator:
+ def __init__(self):
+ self.out = {
+ "local_count": 0,
+ "resolved_count": 0,
+
+ "popular_artists": [],
+ "popular_albums": [],
+
+ "unresolved_albums": [],
+ "unresolved_tracks": [],
+ }
+
+ self.max_popular_albums = 100
+ self.max_popular_artists = 100
+
+ self.max_unresolved_tracks = 100
+ self.max_unresolved_albums = 20
+ self.resolved_percentage_threshold = 30
+
+ self.albums: List[AlbumStat] = []
+ self.artists: List[ArtistStat] = []
+
+ self.albums_map: dict[str, AlbumStat] = {}
+ self.artists_map: dict[str, ArtistStat] = {}
+
+ self.local_lib: Library = Library()
+ self.remote_lib: Library = Library()
+
+ self.mapping: dict[str, Any] = Any
+
+ self.remote_id_map: dict[str, Track] = {}
+ self.local_id_map: dict[str, Track] = {}
+
+ def find_add_local_paths(self, mapping_path: Path):
+ self.mapping = get_mapping(mapping_path)
+
+ self.remote_id_map = {track.id: track for track in self.remote_lib.tracks}
+ self.local_id_map = {track.id: track for track in self.local_lib.tracks}
+
+ for local, remote in self.mapping.items():
+ self.remote_id_map[remote].local_path = self.local_id_map[local].local_path
+
+ def score_artists(self):
+ for artist in self.remote_lib.artists:
+ artist_stat = ArtistStat()
+ artist_stat.artist = artist
+ artist_stat.score = 0
+ self.artists_map[artist_stat.artist.id] = artist_stat
+ self.artists.append(artist_stat)
+
+ for idx, track in enumerate(self.remote_lib.top_tracks):
+ score = len(self.remote_lib.top_tracks) - idx
+ for artist in track.artists:
+ self.artists_map[artist.id].score += score
+
+ for artist_stat in self.artists:
+ artist_stat.calc_resolved_precent()
+
+ self.artists = sorted(self.artists, key=lambda item: item.score, reverse=True)
+
+ def score_albums(self):
+ for album in self.remote_lib.albums:
+ stat = AlbumStat()
+ stat.album = album
+ self.albums_map[album.id] = stat
+
+ for idx, track in enumerate(self.remote_lib.top_tracks):
+ score = len(self.remote_lib.top_tracks) - idx
+ self.albums_map[track.album.id].score += score
+
+ for _, album in self.albums_map.items():
+ album.calc_resolved_precent()
+
+ self.albums = [item[1] for item in
+ sorted(self.albums_map.items(), key=lambda item: item[1].score, reverse=True)]
+
+ for album_stat in self.albums:
+ for artist in album_stat.album.artists:
+ self.artists_map[artist.id].albums_stats.append(album_stat)
+
+ def stat_generic(self):
+ self.out["resolved_count"] = len(self.mapping)
+
+ def gen_album_songs(album_stat):
+ tracks = []
+
+ for track in album_stat.album.tracks:
+ tracks.append(f"{"✔" if is_resolved(track) else "𐄂"} {track.title} -> {track.local_path}")
+
+ return {
+ "album" : album_stat.get_str(),
+ "tracks" : tracks
+ }
+
+ for artist_stat in self.artists:
+ artist_log = {
+ "artist": artist_stat.get_str(),
+ "albums": [gen_album_songs(album) for album in artist_stat.albums_stats]
+ }
+
+ if len(self.out["popular_artists"]) < self.max_popular_artists:
+ self.out["popular_artists"].append(artist_log)
+
+ for album_data in self.albums:
+ log = album_data.get_str()
+
+ if len(self.out["popular_albums"]) < self.max_popular_albums:
+ self.out["popular_albums"].append(log)
+
+ def stat_unresolved_albums(self):
+ for album_data in self.albums:
+ if album_data.resolved_precent > self.resolved_percentage_threshold:
+ continue
+
+ log = album_data.get_str()
+
+ if len(self.out["unresolved_albums"]) < self.max_unresolved_albums:
+ self.out["unresolved_albums"].append(log)
+
+ def stat_unresolved_tracks(self):
+ self.out["local_count"] = len(self.local_lib.tracks)
+
+ for track in self.remote_lib.top_tracks:
+ if not track.local_path:
+
+ log = f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}"
+ if len(self.out["unresolved_tracks"]) < self.max_unresolved_tracks:
+ self.out["unresolved_tracks"].append(log)
+
+ def save_stats(self, output_path: Path):
+ with output_path.open("w", encoding="utf-8") as f:
+ json.dump(self.out, f, indent=2, ensure_ascii=False)
+
+ def log_stats(self, mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
+ self.remote_lib = remote_lib
+ self.local_lib = local_lib
+
+ self.find_add_local_paths(mapping_path)
+
+ self.score_artists()
+ self.score_albums()
+
+ self.stat_generic()
+ self.stat_unresolved_tracks()
+ self.stat_unresolved_albums()
+
+ self.save_stats(output_path)
-def log_stats(mapping_path : Path, output_path : Path, local_lib: Library, remote_lib: Library):
- find_add_local_paths(mapping_path, local_lib, remote_lib)
- generate_stats(local_lib, remote_lib, output_path)
+def log_stats(mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
+ stat_gen = StatGenerator()
+ stat_gen.log_stats(mapping_path, output_path, local_lib, remote_lib)
From b1c96511a3ed0ba75f9c69a43c1e7b55d5264399 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Mon, 5 Jan 2026 14:53:06 +0300
Subject: [PATCH 14/30] nice stat generator
---
src/StatGenerator.py | 207 ++++++++++++++++++++++++++++++++++++++-----
1 file changed, 183 insertions(+), 24 deletions(-)
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index d6ebc41..b6dcaee 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -3,11 +3,10 @@ from src.Library import *
from pathlib import Path
import json
-out = {
- "local_count": 0,
- "resolved_count": 0,
- "unresolved_tracks": [],
-}
+
+def is_resolved(track: Track):
+ return track.local_path is not None
+
def get_mapping(mapping_path) -> dict[str, Any]:
data = {}
@@ -24,32 +23,192 @@ def get_mapping(mapping_path) -> dict[str, Any]:
return out
-def find_add_local_paths(mapping_path: Path, local_lib: Library, remote_lib: Library):
- mapping = get_mapping(mapping_path)
+class AlbumStat:
+ def __init__(self):
+ self.album: Album = Album()
+ self.score: int = 0
+ self.resolved_precent: float = 0
- out["resolved_count"] = len(mapping)
+ def calc_resolved_precent(self):
+ self.resolved_precent = (sum(1 for track in self.album.tracks if is_resolved(track)) / len(
+ self.album.tracks)) * 100
- remote_id_map = {track.id: track for track in remote_lib.tracks}
- local_id_map = {track.id: track for track in local_lib.tracks}
-
- for local, remote in mapping.items():
- remote_id_map[remote].local_path = local_id_map[local].local_path
+ def get_str(self):
+ return (f"{str(int(self.resolved_precent))} % - "
+ f"{self.album.title} - "
+ f"{str(self.album.artists[0].title) if len(self.album.artists) else " "}")
-def generate_stats(local_lib: Library, remote_lib: Library, output_path : Path):
+class ArtistStat:
+ def __init__(self):
+ self.artist: Artist = Artist()
+ self.score: int = 0
+ self.resolved_precent: float = 0
+ self.albums_stats: list[AlbumStat] = []
- out["local_count"] = len(local_lib.tracks)
+ def calc_resolved_precent(self):
+ if len(self.artist.tracks) == 0:
+ self.resolved_precent = 0
+ return
- for track in remote_lib.top_tracks:
- if not track.local_path:
- out["unresolved_tracks"].append(
- f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}")
+ self.resolved_precent = (sum(1 for track in self.artist.tracks if is_resolved(track)) / len(
+ self.artist.tracks)) * 100
+
+ def get_str(self):
+ return (f"{str(int(self.resolved_precent))} % - "
+ f"{self.artist.title}")
- with output_path.open("w", encoding="utf-8") as f:
- json.dump(out, f, indent=2, ensure_ascii=False)
+class StatGenerator:
+ def __init__(self):
+ self.out = {
+ "local_count": 0,
+ "resolved_count": 0,
+
+ "popular_artists": [],
+ "popular_albums": [],
+
+ "unresolved_albums": [],
+ "unresolved_tracks": [],
+ }
+
+ self.max_popular_albums = 100
+ self.max_popular_artists = 100
+
+ self.max_unresolved_tracks = 100
+ self.max_unresolved_albums = 20
+ self.resolved_percentage_threshold = 30
+
+ self.albums: List[AlbumStat] = []
+ self.artists: List[ArtistStat] = []
+
+ self.albums_map: dict[str, AlbumStat] = {}
+ self.artists_map: dict[str, ArtistStat] = {}
+
+ self.local_lib: Library = Library()
+ self.remote_lib: Library = Library()
+
+ self.mapping: dict[str, Any] = Any
+
+ self.remote_id_map: dict[str, Track] = {}
+ self.local_id_map: dict[str, Track] = {}
+
+ def find_add_local_paths(self, mapping_path: Path):
+ self.mapping = get_mapping(mapping_path)
+
+ self.remote_id_map = {track.id: track for track in self.remote_lib.tracks}
+ self.local_id_map = {track.id: track for track in self.local_lib.tracks}
+
+ for local, remote in self.mapping.items():
+ self.remote_id_map[remote].local_path = self.local_id_map[local].local_path
+
+ def score_artists(self):
+ for artist in self.remote_lib.artists:
+ artist_stat = ArtistStat()
+ artist_stat.artist = artist
+ artist_stat.score = 0
+ self.artists_map[artist_stat.artist.id] = artist_stat
+ self.artists.append(artist_stat)
+
+ for idx, track in enumerate(self.remote_lib.top_tracks):
+ score = len(self.remote_lib.top_tracks) - idx
+ for artist in track.artists:
+ self.artists_map[artist.id].score += score
+
+ for artist_stat in self.artists:
+ artist_stat.calc_resolved_precent()
+
+ self.artists = sorted(self.artists, key=lambda item: item.score, reverse=True)
+
+ def score_albums(self):
+ for album in self.remote_lib.albums:
+ stat = AlbumStat()
+ stat.album = album
+ self.albums_map[album.id] = stat
+
+ for idx, track in enumerate(self.remote_lib.top_tracks):
+ score = len(self.remote_lib.top_tracks) - idx
+ self.albums_map[track.album.id].score += score
+
+ for _, album in self.albums_map.items():
+ album.calc_resolved_precent()
+
+ self.albums = [item[1] for item in
+ sorted(self.albums_map.items(), key=lambda item: item[1].score, reverse=True)]
+
+ for album_stat in self.albums:
+ for artist in album_stat.album.artists:
+ self.artists_map[artist.id].albums_stats.append(album_stat)
+
+ def stat_generic(self):
+ self.out["resolved_count"] = len(self.mapping)
+
+ def gen_album_songs(album_stat):
+ tracks = []
+
+ for track in album_stat.album.tracks:
+ tracks.append(f"{"✔" if is_resolved(track) else "𐄂"} {track.title} -> {track.local_path}")
+
+ return {
+ "album" : album_stat.get_str(),
+ "tracks" : tracks
+ }
+
+ for artist_stat in self.artists:
+ artist_log = {
+ "artist": artist_stat.get_str(),
+ "albums": [gen_album_songs(album) for album in artist_stat.albums_stats]
+ }
+
+ if len(self.out["popular_artists"]) < self.max_popular_artists:
+ self.out["popular_artists"].append(artist_log)
+
+ for album_data in self.albums:
+ log = album_data.get_str()
+
+ if len(self.out["popular_albums"]) < self.max_popular_albums:
+ self.out["popular_albums"].append(log)
+
+ def stat_unresolved_albums(self):
+ for album_data in self.albums:
+ if album_data.resolved_precent > self.resolved_percentage_threshold:
+ continue
+
+ log = album_data.get_str()
+
+ if len(self.out["unresolved_albums"]) < self.max_unresolved_albums:
+ self.out["unresolved_albums"].append(log)
+
+ def stat_unresolved_tracks(self):
+ self.out["local_count"] = len(self.local_lib.tracks)
+
+ for track in self.remote_lib.top_tracks:
+ if not track.local_path:
+
+ log = f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}"
+ if len(self.out["unresolved_tracks"]) < self.max_unresolved_tracks:
+ self.out["unresolved_tracks"].append(log)
+
+ def save_stats(self, output_path: Path):
+ with output_path.open("w", encoding="utf-8") as f:
+ json.dump(self.out, f, indent=2, ensure_ascii=False)
+
+ def log_stats(self, mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
+ self.remote_lib = remote_lib
+ self.local_lib = local_lib
+
+ self.find_add_local_paths(mapping_path)
+
+ self.score_artists()
+ self.score_albums()
+
+ self.stat_generic()
+ self.stat_unresolved_tracks()
+ self.stat_unresolved_albums()
+
+ self.save_stats(output_path)
-def log_stats(mapping_path : Path, output_path : Path, local_lib: Library, remote_lib: Library):
- find_add_local_paths(mapping_path, local_lib, remote_lib)
- generate_stats(local_lib, remote_lib, output_path)
+def log_stats(mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
+ stat_gen = StatGenerator()
+ stat_gen.log_stats(mapping_path, output_path, local_lib, remote_lib)
From 013e8b139a915686307c876ca7130d3d512d60e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Mon, 5 Jan 2026 17:42:56 +0300
Subject: [PATCH 15/30] stable stat
---
Flow.py | 6 +-
main.py | 6 +-
src/Library.py | 29 ++++++
src/LibrarySerializer.py | 4 +
src/MapLocalToSpotify.py | 140 ---------------------------
src/StatGenerator.py | 10 +-
src/local/ParseLocal.py | 16 ++-
src/mappers/FuzzyMapper.py | 68 +++++++++++++
src/mappers/RemoteLibraryResolver.py | 9 ++
src/mappers/SpotifySearchMapping.py | 56 +++++++++++
src/spotify/ParseSpotify.py | 11 +--
11 files changed, 181 insertions(+), 174 deletions(-)
delete mode 100644 src/MapLocalToSpotify.py
create mode 100644 src/mappers/FuzzyMapper.py
create mode 100644 src/mappers/RemoteLibraryResolver.py
create mode 100644 src/mappers/SpotifySearchMapping.py
diff --git a/Flow.py b/Flow.py
index e982d39..79c1759 100644
--- a/Flow.py
+++ b/Flow.py
@@ -10,7 +10,7 @@ from src.spotify.ParseSpotify import Parser
from src.local.LocalLibraryIndexer import update_local_library
from src.local.ParseLocal import LocalParser
-from src.MapLocalToSpotify import map_local_to_spotify
+from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
@@ -50,10 +50,10 @@ class Flow:
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
def map_local_to_spotify(self):
- map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
+ resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
def log_stats(self):
- log_stats(work_dir / "local_to_spotify_id_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
+ log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
def run(self, arg):
diff --git a/main.py b/main.py
index 1069171..f811d33 100644
--- a/main.py
+++ b/main.py
@@ -7,12 +7,12 @@ if __name__ == '__main__':
# flow.fetch_spotify()
# flow.fetch_local()
- # flow.parse_spotify_library()
- # flow.parse_local_library()
+ flow.parse_spotify_library()
+ flow.parse_local_library()
flow.load_libraries()
- # flow.map_local_to_spotify()
+ flow.map_local_to_spotify()
flow.log_stats()
diff --git a/src/Library.py b/src/Library.py
index c3287b8..68e2c63 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -4,11 +4,13 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+
class Artist:
def __init__(self):
self.id : str = None
self.title : str = None
self.tracks : List[Track] = []
+ self.albums : List[Album] = []
class Track:
@@ -49,3 +51,30 @@ class Library:
self.top_tracks : List[Track]= []
self.top_artists : List[Artist] = []
+ self.artist_map : dict[str, Artist] = None
+ self.album_map : dict[str, Album] = None
+ self.track_map : dict[str, Track] = None
+
+
+ def update_cache(self):
+ self.update_id_maps()
+ self.create_reverse_links()
+
+ def update_id_maps(self):
+ self.artist_map : dict[str, Artist] = { artist.id : artist for artist in self.artists }
+ self.album_map : dict[str, Album] = { album.id : album for album in self.albums }
+ self.track_map : dict[str, Track] = { track.id : track for track in self.tracks }
+
+ def create_reverse_links(self):
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if track not in artist.tracks:
+ artist.tracks.append(track)
+
+ if track not in track.album.tracks:
+ track.album.tracks.append(track)
+
+ for album_id, album in self.album_map.items():
+ for artist in album.artists:
+ if album not in artist.albums:
+ artist.albums.append(album)
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index cbccd9f..e94a0f7 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -32,6 +32,7 @@ class LibrarySaver:
"id": artist.id,
"title": artist.title,
"tracks": [track.id for track in artist.tracks],
+ "albums": [album.id for album in artist.albums],
}
@staticmethod
@@ -127,6 +128,7 @@ class LibraryLoader:
artist.id = data.get("id")
artist.title = data.get("title")
artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
+ artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else []
self.artist_map[artist.id] = artist
return artist
@@ -155,4 +157,6 @@ class LibraryLoader:
library.top_tracks = [self.track_map[track] for track in data.get("top_tracks")]
library.liked_tracks = [self.track_map[track] for track in data.get("liked_tracks")]
+ library.update_cache()
+
return library
diff --git a/src/MapLocalToSpotify.py b/src/MapLocalToSpotify.py
deleted file mode 100644
index 84e37c5..0000000
--- a/src/MapLocalToSpotify.py
+++ /dev/null
@@ -1,140 +0,0 @@
-
-from rapidfuzz import fuzz
-from tqdm import tqdm
-import requests
-
-from src.Library import *
-from src.helpers import *
-from src.spotify.SpotifyWebAPI import find_song, update_access_token
-
-
-def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
-
- set_workdir(output_dir)
-
- update_access_token(client_id, client_secret)
-
- def find_local_songs_on_spotify():
- search_log = []
-
- total = len(local_lib.tracks)
- found_tracks_map = {}
- with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
- for track in local_lib.tracks:
-
- parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title]
- search_pattern = " ".join(p for p in parts if p)
-
- if search_pattern == "":
- print(f"cannot generate search pattern for the track - {track.local_path}")
- pbar.update(1)
- continue
-
- found_tracks = find_song(search_pattern)[0:10]
- found_tracks_map[track.id] = found_tracks
- search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
- pbar.update(1)
-
- save_data(search_log, "spotify_search_log")
- save_data(found_tracks_map, "spotify_found_map")
-
- return found_tracks_map
-
- spotify_found_tracks = find_local_songs_on_spotify()
-
- spotify_user_track = {track.id for track in spotify_lib.tracks}
-
- spotify_pattern = {}
- for local_id, found_items in spotify_found_tracks.items():
- spotify_pattern[local_id] = {"score": 1.0, "items": []}
-
- if not len(found_items):
- continue
-
- for found_item in found_items:
- spotify_track_id = found_item['id']
- if spotify_track_id in spotify_user_track:
- spotify_pattern[local_id]['items'].append(spotify_track_id)
-
- save_data(spotify_pattern, "local_to_spotify_id_map")
-
-
-def fuzzy_pattern_generator():
- local_library = get_data("local_library")
- track_descriptions = get_data("tracks")
-
- def get_matched(src_pattern, patterns):
- result = {}
- for target_pattern in patterns:
- similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern)
- result[target_pattern] = similarity_score / 100
-
- sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True)
- return sorted_results
-
- def resolve_tracks(s_patterns, t_patterns):
-
- pattern_map = {}
-
- total = len(s_patterns)
- with tqdm(total=total, desc='Progress') as pbar:
- for s_pattern in s_patterns:
- res = get_matched(s_pattern, t_patterns)
- matched = {"score": 0, "items": []}
- if len(res):
- matched['items'] = res[0:min(len(res), 4)]
- matched['score'] = res[0][1]
- pattern_map[s_pattern] = matched
- pbar.update(1)
-
- return pattern_map
-
- source_patterns = []
- for track in track_descriptions:
- artists = get_artists_str(track['track']['artists'])
- pattern = f"{track['track']['name']} {artists}"
- source_patterns.append(pattern)
-
- target_patterns = [local['path'] for _, local in local_library.items()]
-
- fuzzy_pattern = resolve_tracks(source_patterns, target_patterns)
- save_data(fuzzy_pattern, "link_pattern_fuzzy")
-
-
-def fuzzy_tag_pattern_generator():
- local_library = get_data("local_library")
- spotify_library = get_data("tracks")
- mappings = {}
-
- def fuzzy_tags_ratio(first: SongTags, second: SongTags):
- title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100
- artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100
- album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100
- return title_ratio, artist_ratio, album_ratio
-
- with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar:
- for local_track_id, local_track in local_library.items():
- mapping = mappings.get(local_track_id, {"score": 1.0, "items": []})
- ratios = []
- for remote_track in spotify_library:
- track = remote_track['track']
-
- tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album'])
- tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name'])
-
- ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id']))
-
- filtered_ratios = []
- threshold = 0.81
- for ratio in ratios:
- if all([val > threshold for val in ratio[0]]):
- filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1]))
-
- filtered_ratios.sort(reverse=True, key=lambda x: x[0])
- if len(filtered_ratios):
- mapping['items'] = [filtered_ratios[0][1]]
-
- mappings[local_track_id] = mapping
- pbar.update(1)
-
- save_data(mappings, "link_pattern_tags")
\ No newline at end of file
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index b6dcaee..f597e17 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -14,13 +14,7 @@ def get_mapping(mapping_path) -> dict[str, Any]:
with Path(mapping_path).open("r", encoding="utf-8") as f:
data = json.load(f)
- out = {}
-
- for local, remotes in data.items():
- if len(remotes["items"]):
- out[local] = remotes["items"][0]
-
- return out
+ return data
class AlbumStat:
@@ -99,7 +93,7 @@ class StatGenerator:
self.remote_id_map = {track.id: track for track in self.remote_lib.tracks}
self.local_id_map = {track.id: track for track in self.local_lib.tracks}
- for local, remote in self.mapping.items():
+ for remote, local in self.mapping.items():
self.remote_id_map[remote].local_path = self.local_id_map[local].local_path
def score_artists(self):
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
index 602e89e..084bbce 100644
--- a/src/local/ParseLocal.py
+++ b/src/local/ParseLocal.py
@@ -104,15 +104,6 @@ class LocalParser:
return album
- def add_tracks_to_albums_and_artists(self):
- for track_id, track in self.track_map.items():
- for artist in track.artists:
- if track not in artist.tracks:
- artist.tracks.append(track)
-
- if track not in track.album.tracks:
- track.album.tracks.append(track)
-
def parse(self, tracks_file: Path) -> Library:
tracks_content = json.loads(tracks_file.read_text())
@@ -129,7 +120,12 @@ class LocalParser:
self.track_map[track.id] = track
self.library.tracks.append(track)
- self.add_tracks_to_albums_and_artists()
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if artist not in track.album.artists:
+ track.album.artists.append(artist)
+
+ self.library.update_cache()
return self.library
diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py
new file mode 100644
index 0000000..ca78db9
--- /dev/null
+++ b/src/mappers/FuzzyMapper.py
@@ -0,0 +1,68 @@
+
+from rapidfuzz import fuzz
+from tqdm import tqdm
+
+from src.Library import *
+from src.helpers import *
+
+
+def get_score(first: str, second: str) -> float:
+ return fuzz.token_set_ratio(first, second)
+
+def is_close_enough(first: float) -> bool:
+ return first > 50
+
+def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
+
+ artist_map : dict[str, tuple[str, float]] = {}
+ album_map : dict[str, tuple[str, float]] = {}
+ track_map : dict[str, tuple[str, float]] = {}
+
+ with (tqdm(total=len(remote_lib.artists), desc='searching remote artists in local library') as process_bar):
+ for remote_artist in remote_lib.artists:
+ for local_artist in local_lib.artists:
+ score = get_score(remote_artist.title, local_artist.title)
+ if not is_close_enough(score):
+ continue
+
+ if remote_artist.id not in artist_map or score > artist_map[remote_artist.id][1]:
+ artist_map[remote_artist.id] = (local_artist.id, score)
+
+ process_bar.update(1)
+
+
+ with tqdm(total=len(artist_map), desc='searching remote albums for each artist in local library') as process_bar:
+ for remote_artist_id, (local_artist_id, _) in artist_map.items():
+ remote_artist = remote_lib.artist_map[remote_artist_id]
+ local_artist = local_lib.artist_map[local_artist_id]
+
+ for remote_album in remote_artist.albums:
+ for local_album in local_artist.albums:
+ score = get_score(remote_album.title, local_album.title)
+ if not is_close_enough(score):
+ continue
+
+ if remote_album.id not in artist_map or score > album_map[remote_album.id][1]:
+ album_map[remote_album.id] = (local_album.id, score)
+
+ process_bar.update(1)
+
+ with tqdm(total=len(album_map), desc='searching remote tracks for each album in local library') as process_bar:
+ for remote_album_id, (local_album_id, _) in album_map.items():
+ remote_album = remote_lib.album_map[remote_album_id]
+ local_album = local_lib.album_map[local_album_id]
+
+ for remote_track in remote_album.tracks:
+ for local_track in local_album.tracks:
+ score = get_score(remote_track.title, local_track.title)
+ if not is_close_enough(score):
+ continue
+
+ if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
+ track_map[remote_track.id] = (local_track.id, score)
+
+ process_bar.update(1)
+
+ out = { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() }
+ set_workdir(output_dir)
+ save_data(out, "remote_to_local_map")
diff --git a/src/mappers/RemoteLibraryResolver.py b/src/mappers/RemoteLibraryResolver.py
new file mode 100644
index 0000000..7342848
--- /dev/null
+++ b/src/mappers/RemoteLibraryResolver.py
@@ -0,0 +1,9 @@
+
+import src.mappers.FuzzyMapper as Fuzzy
+import src.mappers.SpotifySearchMapping as Spotify
+
+from src.Library import *
+
+def resolve_remote_tracks(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
+ Fuzzy.generate_map(local_lib, spotify_lib, output_dir)
+ # Spotify.generate_map(client_id, client_secret, local_lib, spotify_lib, output_dir)
\ No newline at end of file
diff --git a/src/mappers/SpotifySearchMapping.py b/src/mappers/SpotifySearchMapping.py
new file mode 100644
index 0000000..41c5bbb
--- /dev/null
+++ b/src/mappers/SpotifySearchMapping.py
@@ -0,0 +1,56 @@
+
+from tqdm import tqdm
+
+from src.Library import *
+from src.helpers import *
+from src.spotify.SpotifyWebAPI import find_song, update_access_token
+
+def generate_map(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
+
+ set_workdir(output_dir)
+
+ update_access_token(client_id, client_secret)
+
+ def find_local_songs_on_spotify():
+ search_log = []
+
+ total = len(local_lib.tracks)
+ found_tracks_map = {}
+ with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
+ for track in local_lib.tracks:
+
+ parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title]
+ search_pattern = " ".join(p for p in parts if p)
+
+ if search_pattern == "":
+ print(f"cannot generate search pattern for the track - {track.local_path}")
+ pbar.update(1)
+ continue
+
+ found_tracks = find_song(search_pattern)[0:10]
+ found_tracks_map[track.id] = found_tracks
+ search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
+ pbar.update(1)
+
+ save_data(search_log, "spotify_search_log")
+ save_data(found_tracks_map, "spotify_found_map")
+
+ return found_tracks_map
+
+ spotify_found_tracks = find_local_songs_on_spotify()
+
+ spotify_user_track = {track.id for track in spotify_lib.tracks}
+
+ spotify_pattern = {}
+ for local_id, found_items in spotify_found_tracks.items():
+ spotify_pattern[local_id] = {"score": 1.0, "items": []}
+
+ if not len(found_items):
+ continue
+
+ for found_item in found_items:
+ spotify_track_id = found_item['id']
+ if spotify_track_id in spotify_user_track:
+ spotify_pattern[local_id]['items'].append(spotify_track_id)
+
+ save_data(spotify_pattern, "local_to_spotify_id_map")
\ No newline at end of file
diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py
index 49882cb..50806bf 100644
--- a/src/spotify/ParseSpotify.py
+++ b/src/spotify/ParseSpotify.py
@@ -29,19 +29,10 @@ class Parser:
self.parse_top_artists(json.loads(top_artists.read_text()))
self.parse_top_tracks(json.loads(top_tracks.read_text()))
- self.add_tracks_to_albums_and_artists()
+ self.library.update_cache()
return self.library
- def add_tracks_to_albums_and_artists(self):
- for track_id, track in self.track_map.items():
- for artist in track.artists:
- if track not in artist.tracks:
- artist.tracks.append(track)
-
- if track not in track.album.tracks:
- track.album.tracks.append(track)
-
def parse_playlists(self, data):
for name, tracks in data.items():
From 11fb2816e34f5cbe3dec1f5dab4f41deb909b41c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Mon, 5 Jan 2026 17:42:56 +0300
Subject: [PATCH 16/30] stable stat
---
Flow.py | 6 +-
main.py | 6 +-
src/Library.py | 29 ++++++
src/LibrarySerializer.py | 4 +
src/MapLocalToSpotify.py | 140 ---------------------------
src/StatGenerator.py | 10 +-
src/local/ParseLocal.py | 16 ++-
src/mappers/FuzzyMapper.py | 68 +++++++++++++
src/mappers/RemoteLibraryResolver.py | 9 ++
src/mappers/SpotifySearchMapping.py | 56 +++++++++++
src/spotify/ParseSpotify.py | 11 +--
11 files changed, 181 insertions(+), 174 deletions(-)
delete mode 100644 src/MapLocalToSpotify.py
create mode 100644 src/mappers/FuzzyMapper.py
create mode 100644 src/mappers/RemoteLibraryResolver.py
create mode 100644 src/mappers/SpotifySearchMapping.py
diff --git a/Flow.py b/Flow.py
index e982d39..79c1759 100644
--- a/Flow.py
+++ b/Flow.py
@@ -10,7 +10,7 @@ from src.spotify.ParseSpotify import Parser
from src.local.LocalLibraryIndexer import update_local_library
from src.local.ParseLocal import LocalParser
-from src.MapLocalToSpotify import map_local_to_spotify
+from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
@@ -50,10 +50,10 @@ class Flow:
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
def map_local_to_spotify(self):
- map_local_to_spotify(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
+ resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
def log_stats(self):
- log_stats(work_dir / "local_to_spotify_id_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
+ log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
def run(self, arg):
diff --git a/main.py b/main.py
index 1069171..f811d33 100644
--- a/main.py
+++ b/main.py
@@ -7,12 +7,12 @@ if __name__ == '__main__':
# flow.fetch_spotify()
# flow.fetch_local()
- # flow.parse_spotify_library()
- # flow.parse_local_library()
+ flow.parse_spotify_library()
+ flow.parse_local_library()
flow.load_libraries()
- # flow.map_local_to_spotify()
+ flow.map_local_to_spotify()
flow.log_stats()
diff --git a/src/Library.py b/src/Library.py
index c3287b8..68e2c63 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -4,11 +4,13 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+
class Artist:
def __init__(self):
self.id : str = None
self.title : str = None
self.tracks : List[Track] = []
+ self.albums : List[Album] = []
class Track:
@@ -49,3 +51,30 @@ class Library:
self.top_tracks : List[Track]= []
self.top_artists : List[Artist] = []
+ self.artist_map : dict[str, Artist] = None
+ self.album_map : dict[str, Album] = None
+ self.track_map : dict[str, Track] = None
+
+
+ def update_cache(self):
+ self.update_id_maps()
+ self.create_reverse_links()
+
+ def update_id_maps(self):
+ self.artist_map : dict[str, Artist] = { artist.id : artist for artist in self.artists }
+ self.album_map : dict[str, Album] = { album.id : album for album in self.albums }
+ self.track_map : dict[str, Track] = { track.id : track for track in self.tracks }
+
+ def create_reverse_links(self):
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if track not in artist.tracks:
+ artist.tracks.append(track)
+
+ if track not in track.album.tracks:
+ track.album.tracks.append(track)
+
+ for album_id, album in self.album_map.items():
+ for artist in album.artists:
+ if album not in artist.albums:
+ artist.albums.append(album)
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index cbccd9f..e94a0f7 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -32,6 +32,7 @@ class LibrarySaver:
"id": artist.id,
"title": artist.title,
"tracks": [track.id for track in artist.tracks],
+ "albums": [album.id for album in artist.albums],
}
@staticmethod
@@ -127,6 +128,7 @@ class LibraryLoader:
artist.id = data.get("id")
artist.title = data.get("title")
artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
+ artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else []
self.artist_map[artist.id] = artist
return artist
@@ -155,4 +157,6 @@ class LibraryLoader:
library.top_tracks = [self.track_map[track] for track in data.get("top_tracks")]
library.liked_tracks = [self.track_map[track] for track in data.get("liked_tracks")]
+ library.update_cache()
+
return library
diff --git a/src/MapLocalToSpotify.py b/src/MapLocalToSpotify.py
deleted file mode 100644
index 84e37c5..0000000
--- a/src/MapLocalToSpotify.py
+++ /dev/null
@@ -1,140 +0,0 @@
-
-from rapidfuzz import fuzz
-from tqdm import tqdm
-import requests
-
-from src.Library import *
-from src.helpers import *
-from src.spotify.SpotifyWebAPI import find_song, update_access_token
-
-
-def map_local_to_spotify(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
-
- set_workdir(output_dir)
-
- update_access_token(client_id, client_secret)
-
- def find_local_songs_on_spotify():
- search_log = []
-
- total = len(local_lib.tracks)
- found_tracks_map = {}
- with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
- for track in local_lib.tracks:
-
- parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title]
- search_pattern = " ".join(p for p in parts if p)
-
- if search_pattern == "":
- print(f"cannot generate search pattern for the track - {track.local_path}")
- pbar.update(1)
- continue
-
- found_tracks = find_song(search_pattern)[0:10]
- found_tracks_map[track.id] = found_tracks
- search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
- pbar.update(1)
-
- save_data(search_log, "spotify_search_log")
- save_data(found_tracks_map, "spotify_found_map")
-
- return found_tracks_map
-
- spotify_found_tracks = find_local_songs_on_spotify()
-
- spotify_user_track = {track.id for track in spotify_lib.tracks}
-
- spotify_pattern = {}
- for local_id, found_items in spotify_found_tracks.items():
- spotify_pattern[local_id] = {"score": 1.0, "items": []}
-
- if not len(found_items):
- continue
-
- for found_item in found_items:
- spotify_track_id = found_item['id']
- if spotify_track_id in spotify_user_track:
- spotify_pattern[local_id]['items'].append(spotify_track_id)
-
- save_data(spotify_pattern, "local_to_spotify_id_map")
-
-
-def fuzzy_pattern_generator():
- local_library = get_data("local_library")
- track_descriptions = get_data("tracks")
-
- def get_matched(src_pattern, patterns):
- result = {}
- for target_pattern in patterns:
- similarity_score = fuzz.token_set_ratio(src_pattern, target_pattern)
- result[target_pattern] = similarity_score / 100
-
- sorted_results = sorted(result.items(), key=lambda item: item[1], reverse=True)
- return sorted_results
-
- def resolve_tracks(s_patterns, t_patterns):
-
- pattern_map = {}
-
- total = len(s_patterns)
- with tqdm(total=total, desc='Progress') as pbar:
- for s_pattern in s_patterns:
- res = get_matched(s_pattern, t_patterns)
- matched = {"score": 0, "items": []}
- if len(res):
- matched['items'] = res[0:min(len(res), 4)]
- matched['score'] = res[0][1]
- pattern_map[s_pattern] = matched
- pbar.update(1)
-
- return pattern_map
-
- source_patterns = []
- for track in track_descriptions:
- artists = get_artists_str(track['track']['artists'])
- pattern = f"{track['track']['name']} {artists}"
- source_patterns.append(pattern)
-
- target_patterns = [local['path'] for _, local in local_library.items()]
-
- fuzzy_pattern = resolve_tracks(source_patterns, target_patterns)
- save_data(fuzzy_pattern, "link_pattern_fuzzy")
-
-
-def fuzzy_tag_pattern_generator():
- local_library = get_data("local_library")
- spotify_library = get_data("tracks")
- mappings = {}
-
- def fuzzy_tags_ratio(first: SongTags, second: SongTags):
- title_ratio = fuzz.token_set_ratio(first.title, second.title) / 100
- artist_ratio = fuzz.token_set_ratio(first.artist, second.artist) / 100
- album_ratio = fuzz.token_set_ratio(first.album, second.album) / 100
- return title_ratio, artist_ratio, album_ratio
-
- with tqdm(total=len(local_library), desc='fuzzy_tag_pattern_generator') as pbar:
- for local_track_id, local_track in local_library.items():
- mapping = mappings.get(local_track_id, {"score": 1.0, "items": []})
- ratios = []
- for remote_track in spotify_library:
- track = remote_track['track']
-
- tag1 = SongTags(local_track['name'], local_track['artist'], local_track['album'])
- tag2 = SongTags(track['name'], get_artists_str(track['artists']), track['album']['name'])
-
- ratios.append((fuzzy_tags_ratio(tag1, tag2), track['id']))
-
- filtered_ratios = []
- threshold = 0.81
- for ratio in ratios:
- if all([val > threshold for val in ratio[0]]):
- filtered_ratios.append((sum(i for i in ratio[0]) / len(ratio[0]), ratio[1]))
-
- filtered_ratios.sort(reverse=True, key=lambda x: x[0])
- if len(filtered_ratios):
- mapping['items'] = [filtered_ratios[0][1]]
-
- mappings[local_track_id] = mapping
- pbar.update(1)
-
- save_data(mappings, "link_pattern_tags")
\ No newline at end of file
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index b6dcaee..f597e17 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -14,13 +14,7 @@ def get_mapping(mapping_path) -> dict[str, Any]:
with Path(mapping_path).open("r", encoding="utf-8") as f:
data = json.load(f)
- out = {}
-
- for local, remotes in data.items():
- if len(remotes["items"]):
- out[local] = remotes["items"][0]
-
- return out
+ return data
class AlbumStat:
@@ -99,7 +93,7 @@ class StatGenerator:
self.remote_id_map = {track.id: track for track in self.remote_lib.tracks}
self.local_id_map = {track.id: track for track in self.local_lib.tracks}
- for local, remote in self.mapping.items():
+ for remote, local in self.mapping.items():
self.remote_id_map[remote].local_path = self.local_id_map[local].local_path
def score_artists(self):
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
index 602e89e..084bbce 100644
--- a/src/local/ParseLocal.py
+++ b/src/local/ParseLocal.py
@@ -104,15 +104,6 @@ class LocalParser:
return album
- def add_tracks_to_albums_and_artists(self):
- for track_id, track in self.track_map.items():
- for artist in track.artists:
- if track not in artist.tracks:
- artist.tracks.append(track)
-
- if track not in track.album.tracks:
- track.album.tracks.append(track)
-
def parse(self, tracks_file: Path) -> Library:
tracks_content = json.loads(tracks_file.read_text())
@@ -129,7 +120,12 @@ class LocalParser:
self.track_map[track.id] = track
self.library.tracks.append(track)
- self.add_tracks_to_albums_and_artists()
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if artist not in track.album.artists:
+ track.album.artists.append(artist)
+
+ self.library.update_cache()
return self.library
diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py
new file mode 100644
index 0000000..ca78db9
--- /dev/null
+++ b/src/mappers/FuzzyMapper.py
@@ -0,0 +1,68 @@
+
+from rapidfuzz import fuzz
+from tqdm import tqdm
+
+from src.Library import *
+from src.helpers import *
+
+
+def get_score(first: str, second: str) -> float:
+ return fuzz.token_set_ratio(first, second)
+
+def is_close_enough(first: float) -> bool:
+ return first > 50
+
+def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
+
+ artist_map : dict[str, tuple[str, float]] = {}
+ album_map : dict[str, tuple[str, float]] = {}
+ track_map : dict[str, tuple[str, float]] = {}
+
+ with (tqdm(total=len(remote_lib.artists), desc='searching remote artists in local library') as process_bar):
+ for remote_artist in remote_lib.artists:
+ for local_artist in local_lib.artists:
+ score = get_score(remote_artist.title, local_artist.title)
+ if not is_close_enough(score):
+ continue
+
+ if remote_artist.id not in artist_map or score > artist_map[remote_artist.id][1]:
+ artist_map[remote_artist.id] = (local_artist.id, score)
+
+ process_bar.update(1)
+
+
+ with tqdm(total=len(artist_map), desc='searching remote albums for each artist in local library') as process_bar:
+ for remote_artist_id, (local_artist_id, _) in artist_map.items():
+ remote_artist = remote_lib.artist_map[remote_artist_id]
+ local_artist = local_lib.artist_map[local_artist_id]
+
+ for remote_album in remote_artist.albums:
+ for local_album in local_artist.albums:
+ score = get_score(remote_album.title, local_album.title)
+ if not is_close_enough(score):
+ continue
+
+ if remote_album.id not in artist_map or score > album_map[remote_album.id][1]:
+ album_map[remote_album.id] = (local_album.id, score)
+
+ process_bar.update(1)
+
+ with tqdm(total=len(album_map), desc='searching remote tracks for each album in local library') as process_bar:
+ for remote_album_id, (local_album_id, _) in album_map.items():
+ remote_album = remote_lib.album_map[remote_album_id]
+ local_album = local_lib.album_map[local_album_id]
+
+ for remote_track in remote_album.tracks:
+ for local_track in local_album.tracks:
+ score = get_score(remote_track.title, local_track.title)
+ if not is_close_enough(score):
+ continue
+
+ if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
+ track_map[remote_track.id] = (local_track.id, score)
+
+ process_bar.update(1)
+
+ out = { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() }
+ set_workdir(output_dir)
+ save_data(out, "remote_to_local_map")
diff --git a/src/mappers/RemoteLibraryResolver.py b/src/mappers/RemoteLibraryResolver.py
new file mode 100644
index 0000000..7342848
--- /dev/null
+++ b/src/mappers/RemoteLibraryResolver.py
@@ -0,0 +1,9 @@
+
+import src.mappers.FuzzyMapper as Fuzzy
+import src.mappers.SpotifySearchMapping as Spotify
+
+from src.Library import *
+
+def resolve_remote_tracks(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
+ Fuzzy.generate_map(local_lib, spotify_lib, output_dir)
+ # Spotify.generate_map(client_id, client_secret, local_lib, spotify_lib, output_dir)
\ No newline at end of file
diff --git a/src/mappers/SpotifySearchMapping.py b/src/mappers/SpotifySearchMapping.py
new file mode 100644
index 0000000..41c5bbb
--- /dev/null
+++ b/src/mappers/SpotifySearchMapping.py
@@ -0,0 +1,56 @@
+
+from tqdm import tqdm
+
+from src.Library import *
+from src.helpers import *
+from src.spotify.SpotifyWebAPI import find_song, update_access_token
+
+def generate_map(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
+
+ set_workdir(output_dir)
+
+ update_access_token(client_id, client_secret)
+
+ def find_local_songs_on_spotify():
+ search_log = []
+
+ total = len(local_lib.tracks)
+ found_tracks_map = {}
+ with tqdm(total=total, desc='Searching local songs on spotify') as pbar:
+ for track in local_lib.tracks:
+
+ parts = [track.title, track.artists[0].title if len(track.artists) else "", track.album.title]
+ search_pattern = " ".join(p for p in parts if p)
+
+ if search_pattern == "":
+ print(f"cannot generate search pattern for the track - {track.local_path}")
+ pbar.update(1)
+ continue
+
+ found_tracks = find_song(search_pattern)[0:10]
+ found_tracks_map[track.id] = found_tracks
+ search_log.append({"path" : track.local_path, "search_pattern": search_pattern, "result" : found_tracks_map[track.id]})
+ pbar.update(1)
+
+ save_data(search_log, "spotify_search_log")
+ save_data(found_tracks_map, "spotify_found_map")
+
+ return found_tracks_map
+
+ spotify_found_tracks = find_local_songs_on_spotify()
+
+ spotify_user_track = {track.id for track in spotify_lib.tracks}
+
+ spotify_pattern = {}
+ for local_id, found_items in spotify_found_tracks.items():
+ spotify_pattern[local_id] = {"score": 1.0, "items": []}
+
+ if not len(found_items):
+ continue
+
+ for found_item in found_items:
+ spotify_track_id = found_item['id']
+ if spotify_track_id in spotify_user_track:
+ spotify_pattern[local_id]['items'].append(spotify_track_id)
+
+ save_data(spotify_pattern, "local_to_spotify_id_map")
\ No newline at end of file
diff --git a/src/spotify/ParseSpotify.py b/src/spotify/ParseSpotify.py
index 49882cb..50806bf 100644
--- a/src/spotify/ParseSpotify.py
+++ b/src/spotify/ParseSpotify.py
@@ -29,19 +29,10 @@ class Parser:
self.parse_top_artists(json.loads(top_artists.read_text()))
self.parse_top_tracks(json.loads(top_tracks.read_text()))
- self.add_tracks_to_albums_and_artists()
+ self.library.update_cache()
return self.library
- def add_tracks_to_albums_and_artists(self):
- for track_id, track in self.track_map.items():
- for artist in track.artists:
- if track not in artist.tracks:
- artist.tracks.append(track)
-
- if track not in track.album.tracks:
- track.album.tracks.append(track)
-
def parse_playlists(self, data):
for name, tracks in data.items():
From d5cb3a48a40fe77f9dec99610c55321dd2468b84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Tue, 6 Jan 2026 02:34:04 +0300
Subject: [PATCH 17/30] nice stats and stable
---
Flow.py | 2 +-
main.py | 2 +-
src/StatGenerator.py | 155 ++++++++++++++++++++++---------------------
src/helpers.py | 2 +-
4 files changed, 81 insertions(+), 80 deletions(-)
diff --git a/Flow.py b/Flow.py
index 79c1759..bcbdd16 100644
--- a/Flow.py
+++ b/Flow.py
@@ -53,7 +53,7 @@ class Flow:
resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
def log_stats(self):
- log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
+ log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats", self.local_library, self.spotify_library)
def run(self, arg):
diff --git a/main.py b/main.py
index f811d33..5e395a4 100644
--- a/main.py
+++ b/main.py
@@ -5,7 +5,7 @@ if __name__ == '__main__':
flow = Flow.Flow()
# flow.fetch_spotify()
- # flow.fetch_local()
+ flow.fetch_local()
flow.parse_spotify_library()
flow.parse_local_library()
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index f597e17..ac16622 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -1,4 +1,5 @@
from src.Library import *
+from src.helpers import *
from pathlib import Path
import json
@@ -52,20 +53,8 @@ class ArtistStat:
return (f"{str(int(self.resolved_precent))} % - "
f"{self.artist.title}")
-
-class StatGenerator:
- def __init__(self):
- self.out = {
- "local_count": 0,
- "resolved_count": 0,
-
- "popular_artists": [],
- "popular_albums": [],
-
- "unresolved_albums": [],
- "unresolved_tracks": [],
- }
-
+class LibraryStats:
+ def __init__(self, mapping_path : Path, local_lib: Library, remote_lib: Library):
self.max_popular_albums = 100
self.max_popular_artists = 100
@@ -79,17 +68,19 @@ class StatGenerator:
self.albums_map: dict[str, AlbumStat] = {}
self.artists_map: dict[str, ArtistStat] = {}
- self.local_lib: Library = Library()
- self.remote_lib: Library = Library()
+ self.local_lib: Library = local_lib
+ self.remote_lib: Library = remote_lib
- self.mapping: dict[str, Any] = Any
+ self.mapping: dict[str, Any] = get_mapping(mapping_path)
self.remote_id_map: dict[str, Track] = {}
self.local_id_map: dict[str, Track] = {}
- def find_add_local_paths(self, mapping_path: Path):
- self.mapping = get_mapping(mapping_path)
+ self.find_add_local_paths()
+ self.score_artists()
+ self.score_albums()
+ def find_add_local_paths(self):
self.remote_id_map = {track.id: track for track in self.remote_lib.tracks}
self.local_id_map = {track.id: track for track in self.local_lib.tracks}
@@ -134,75 +125,85 @@ class StatGenerator:
for artist in album_stat.album.artists:
self.artists_map[artist.id].albums_stats.append(album_stat)
- def stat_generic(self):
- self.out["resolved_count"] = len(self.mapping)
- def gen_album_songs(album_stat):
- tracks = []
+ def gen_album(self, album_stat, condition):
+ tracks = []
- for track in album_stat.album.tracks:
+ for track in album_stat.album.tracks:
+ if condition.should_add_track(track):
tracks.append(f"{"✔" if is_resolved(track) else "𐄂"} {track.title} -> {track.local_path}")
- return {
- "album" : album_stat.get_str(),
- "tracks" : tracks
- }
+ return {
+ "album" : album_stat.get_str(),
+ "tracks" : tracks
+ }
+
+ def gen_artist(self, artist_stat, condition):
+ albums = []
+
+ for album_stat in artist_stat.albums_stats:
+ if condition.should_add_album(album_stat):
+ albums.append(self.gen_album(album_stat, condition))
+
+ return {
+ "artist" : artist_stat.get_str(),
+ "albums" : albums,
+ }
+
+ def generate_stat_tree(self, condition):
+ artists = []
for artist_stat in self.artists:
- artist_log = {
- "artist": artist_stat.get_str(),
- "albums": [gen_album_songs(album) for album in artist_stat.albums_stats]
- }
+ if condition.should_add_artist(artist_stat):
+ artists.append(self.gen_artist(artist_stat, condition))
- if len(self.out["popular_artists"]) < self.max_popular_artists:
- self.out["popular_artists"].append(artist_log)
+ return {
+ "artists" : artists
+ }
- for album_data in self.albums:
- log = album_data.get_str()
+ def save_tree(self, output_path, condition):
+ save_data(self.generate_stat_tree(condition), output_path)
- if len(self.out["popular_albums"]) < self.max_popular_albums:
- self.out["popular_albums"].append(log)
+ def save_all(self, output_path):
- def stat_unresolved_albums(self):
- for album_data in self.albums:
- if album_data.resolved_precent > self.resolved_percentage_threshold:
- continue
+ class Condition:
+ def should_add_artist(self, item):
+ return True
- log = album_data.get_str()
+ def should_add_album(self, item):
+ return True
- if len(self.out["unresolved_albums"]) < self.max_unresolved_albums:
- self.out["unresolved_albums"].append(log)
-
- def stat_unresolved_tracks(self):
- self.out["local_count"] = len(self.local_lib.tracks)
-
- for track in self.remote_lib.top_tracks:
- if not track.local_path:
-
- log = f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}"
- if len(self.out["unresolved_tracks"]) < self.max_unresolved_tracks:
- self.out["unresolved_tracks"].append(log)
-
- def save_stats(self, output_path: Path):
- with output_path.open("w", encoding="utf-8") as f:
- json.dump(self.out, f, indent=2, ensure_ascii=False)
-
- def log_stats(self, mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
- self.remote_lib = remote_lib
- self.local_lib = local_lib
-
- self.find_add_local_paths(mapping_path)
-
- self.score_artists()
- self.score_albums()
-
- self.stat_generic()
- self.stat_unresolved_tracks()
- self.stat_unresolved_albums()
-
- self.save_stats(output_path)
+ def should_add_track(self, item):
+ return True
-def log_stats(mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
- stat_gen = StatGenerator()
- stat_gen.log_stats(mapping_path, output_path, local_lib, remote_lib)
+ self.save_tree(output_path, Condition())
+
+
+ def save_threshold(self, output_path, threshold = 80):
+
+ class Condition:
+ def should_add_artist(self, item : ArtistStat):
+ for album in item.albums_stats:
+ if self.should_add_album(album):
+ return True
+
+ return False
+
+ def should_add_album(self, item: AlbumStat):
+ return item.resolved_precent < threshold and len(item.album.tracks) > 3
+
+ def should_add_track(self, item: Track):
+ return not is_resolved(item)
+
+ self.save_tree(output_path, Condition())
+
+
+
+def log_stats(mapping_path: Path, output_dir: Path, local_lib: Library, remote_lib: Library):
+ set_workdir(output_dir)
+
+ stat = LibraryStats(mapping_path, local_lib, remote_lib)
+
+ stat.save_all("stats_all")
+ stat.save_threshold("stats_missing")
diff --git a/src/helpers.py b/src/helpers.py
index 8f81bbf..3c7db3e 100644
--- a/src/helpers.py
+++ b/src/helpers.py
@@ -46,7 +46,7 @@ def save_data(data, name):
file_path = os.path.join(data_dir, f"{name}.json")
with open(file_path, 'w') as file:
- json.dump(data, file, indent=4)
+ json.dump(data, file, indent=4, ensure_ascii=False)
print(f"Data saved to {file_path}.")
From 9c9ddb7a9ed278ac06e9858a9bf786c6f4fd0736 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Tue, 6 Jan 2026 02:34:04 +0300
Subject: [PATCH 18/30] nice stats and stable
---
Flow.py | 2 +-
main.py | 2 +-
src/StatGenerator.py | 155 ++++++++++++++++++++++---------------------
src/helpers.py | 2 +-
4 files changed, 81 insertions(+), 80 deletions(-)
diff --git a/Flow.py b/Flow.py
index 79c1759..bcbdd16 100644
--- a/Flow.py
+++ b/Flow.py
@@ -53,7 +53,7 @@ class Flow:
resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
def log_stats(self):
- log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats.json", self.local_library, self.spotify_library)
+ log_stats(work_dir / "remote_to_local_map.json", work_dir / "stats", self.local_library, self.spotify_library)
def run(self, arg):
diff --git a/main.py b/main.py
index f811d33..5e395a4 100644
--- a/main.py
+++ b/main.py
@@ -5,7 +5,7 @@ if __name__ == '__main__':
flow = Flow.Flow()
# flow.fetch_spotify()
- # flow.fetch_local()
+ flow.fetch_local()
flow.parse_spotify_library()
flow.parse_local_library()
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index f597e17..ac16622 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -1,4 +1,5 @@
from src.Library import *
+from src.helpers import *
from pathlib import Path
import json
@@ -52,20 +53,8 @@ class ArtistStat:
return (f"{str(int(self.resolved_precent))} % - "
f"{self.artist.title}")
-
-class StatGenerator:
- def __init__(self):
- self.out = {
- "local_count": 0,
- "resolved_count": 0,
-
- "popular_artists": [],
- "popular_albums": [],
-
- "unresolved_albums": [],
- "unresolved_tracks": [],
- }
-
+class LibraryStats:
+ def __init__(self, mapping_path : Path, local_lib: Library, remote_lib: Library):
self.max_popular_albums = 100
self.max_popular_artists = 100
@@ -79,17 +68,19 @@ class StatGenerator:
self.albums_map: dict[str, AlbumStat] = {}
self.artists_map: dict[str, ArtistStat] = {}
- self.local_lib: Library = Library()
- self.remote_lib: Library = Library()
+ self.local_lib: Library = local_lib
+ self.remote_lib: Library = remote_lib
- self.mapping: dict[str, Any] = Any
+ self.mapping: dict[str, Any] = get_mapping(mapping_path)
self.remote_id_map: dict[str, Track] = {}
self.local_id_map: dict[str, Track] = {}
- def find_add_local_paths(self, mapping_path: Path):
- self.mapping = get_mapping(mapping_path)
+ self.find_add_local_paths()
+ self.score_artists()
+ self.score_albums()
+ def find_add_local_paths(self):
self.remote_id_map = {track.id: track for track in self.remote_lib.tracks}
self.local_id_map = {track.id: track for track in self.local_lib.tracks}
@@ -134,75 +125,85 @@ class StatGenerator:
for artist in album_stat.album.artists:
self.artists_map[artist.id].albums_stats.append(album_stat)
- def stat_generic(self):
- self.out["resolved_count"] = len(self.mapping)
- def gen_album_songs(album_stat):
- tracks = []
+ def gen_album(self, album_stat, condition):
+ tracks = []
- for track in album_stat.album.tracks:
+ for track in album_stat.album.tracks:
+ if condition.should_add_track(track):
tracks.append(f"{"✔" if is_resolved(track) else "𐄂"} {track.title} -> {track.local_path}")
- return {
- "album" : album_stat.get_str(),
- "tracks" : tracks
- }
+ return {
+ "album" : album_stat.get_str(),
+ "tracks" : tracks
+ }
+
+ def gen_artist(self, artist_stat, condition):
+ albums = []
+
+ for album_stat in artist_stat.albums_stats:
+ if condition.should_add_album(album_stat):
+ albums.append(self.gen_album(album_stat, condition))
+
+ return {
+ "artist" : artist_stat.get_str(),
+ "albums" : albums,
+ }
+
+ def generate_stat_tree(self, condition):
+ artists = []
for artist_stat in self.artists:
- artist_log = {
- "artist": artist_stat.get_str(),
- "albums": [gen_album_songs(album) for album in artist_stat.albums_stats]
- }
+ if condition.should_add_artist(artist_stat):
+ artists.append(self.gen_artist(artist_stat, condition))
- if len(self.out["popular_artists"]) < self.max_popular_artists:
- self.out["popular_artists"].append(artist_log)
+ return {
+ "artists" : artists
+ }
- for album_data in self.albums:
- log = album_data.get_str()
+ def save_tree(self, output_path, condition):
+ save_data(self.generate_stat_tree(condition), output_path)
- if len(self.out["popular_albums"]) < self.max_popular_albums:
- self.out["popular_albums"].append(log)
+ def save_all(self, output_path):
- def stat_unresolved_albums(self):
- for album_data in self.albums:
- if album_data.resolved_precent > self.resolved_percentage_threshold:
- continue
+ class Condition:
+ def should_add_artist(self, item):
+ return True
- log = album_data.get_str()
+ def should_add_album(self, item):
+ return True
- if len(self.out["unresolved_albums"]) < self.max_unresolved_albums:
- self.out["unresolved_albums"].append(log)
-
- def stat_unresolved_tracks(self):
- self.out["local_count"] = len(self.local_lib.tracks)
-
- for track in self.remote_lib.top_tracks:
- if not track.local_path:
-
- log = f"{track.title} - {track.artists[0].title if len(track.artists) else ""} - {track.album.title}"
- if len(self.out["unresolved_tracks"]) < self.max_unresolved_tracks:
- self.out["unresolved_tracks"].append(log)
-
- def save_stats(self, output_path: Path):
- with output_path.open("w", encoding="utf-8") as f:
- json.dump(self.out, f, indent=2, ensure_ascii=False)
-
- def log_stats(self, mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
- self.remote_lib = remote_lib
- self.local_lib = local_lib
-
- self.find_add_local_paths(mapping_path)
-
- self.score_artists()
- self.score_albums()
-
- self.stat_generic()
- self.stat_unresolved_tracks()
- self.stat_unresolved_albums()
-
- self.save_stats(output_path)
+ def should_add_track(self, item):
+ return True
-def log_stats(mapping_path: Path, output_path: Path, local_lib: Library, remote_lib: Library):
- stat_gen = StatGenerator()
- stat_gen.log_stats(mapping_path, output_path, local_lib, remote_lib)
+ self.save_tree(output_path, Condition())
+
+
+ def save_threshold(self, output_path, threshold = 80):
+
+ class Condition:
+ def should_add_artist(self, item : ArtistStat):
+ for album in item.albums_stats:
+ if self.should_add_album(album):
+ return True
+
+ return False
+
+ def should_add_album(self, item: AlbumStat):
+ return item.resolved_precent < threshold and len(item.album.tracks) > 3
+
+ def should_add_track(self, item: Track):
+ return not is_resolved(item)
+
+ self.save_tree(output_path, Condition())
+
+
+
+def log_stats(mapping_path: Path, output_dir: Path, local_lib: Library, remote_lib: Library):
+ set_workdir(output_dir)
+
+ stat = LibraryStats(mapping_path, local_lib, remote_lib)
+
+ stat.save_all("stats_all")
+ stat.save_threshold("stats_missing")
diff --git a/src/helpers.py b/src/helpers.py
index 8f81bbf..3c7db3e 100644
--- a/src/helpers.py
+++ b/src/helpers.py
@@ -46,7 +46,7 @@ def save_data(data, name):
file_path = os.path.join(data_dir, f"{name}.json")
with open(file_path, 'w') as file:
- json.dump(data, file, indent=4)
+ json.dump(data, file, indent=4, ensure_ascii=False)
print(f"Data saved to {file_path}.")
From eb745301e542acb4abeb484303a4d6da70c57add Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 17 Jan 2026 14:24:05 +0300
Subject: [PATCH 19/30] lyrics fetcher
---
Flow.py | 51 ++++++++++++-
main.py | 20 ++++-
src/Library.py | 1 +
src/LibrarySerializer.py | 2 +
src/local/LocalLibraryIndexer.py | 111 +++++++++++++++++++++++-----
src/local/LocalPLaylistGenerator.py | 2 +-
src/local/ParseLocal.py | 2 +
src/lyrics/lyrics.py | 111 ++++++++++++++++++++++++++++
8 files changed, 277 insertions(+), 23 deletions(-)
create mode 100644 src/lyrics/lyrics.py
diff --git a/Flow.py b/Flow.py
index bcbdd16..4731f1c 100644
--- a/Flow.py
+++ b/Flow.py
@@ -1,7 +1,11 @@
+from tqdm import tqdm
+import src.lyrics.lyrics as Lyrics
+
from Env import client_id, client_secret, local_library_path
from Env import work_dir
+from src.Library import *
from src.LibrarySerializer import LibrarySaver, LibraryLoader
from src.spotify.SpotifyWebAPI import fetch_all
@@ -15,8 +19,8 @@ from src.StatGenerator import log_stats
class Flow:
- spotify_library = None
- local_library = None
+ spotify_library : Library = None
+ local_library : Library = None
@staticmethod
def fetch_spotify(self):
@@ -45,6 +49,49 @@ class Flow:
library_parsed = parser.parse(work_dir / "local" / "local_library.json")
LibrarySaver(library_parsed).save(work_dir / "local_library.json")
+
+ def fetch_and_save_lyrics(self):
+ """
+ Fetches and writes lyrics for tracks that don't have lyrics yet.
+ Displays stats and uses a progress bar.
+ """
+
+ # ---------- Step 1: Collect tracks without lyrics ----------
+ tracks_missing_lyrics = [
+ track for track in self.local_library.tracks
+ if len(track.artists) > 0 and not track.has_lyrics
+ ]
+
+ total_tracks = len(self.local_library.tracks)
+ missing_count = len(tracks_missing_lyrics)
+
+ print(f"[INFO] Total tracks in library: {total_tracks}")
+ print(f"[INFO] Tracks missing lyrics: {missing_count}")
+
+ if missing_count == 0:
+ print("[INFO] No tracks need lyrics. Exiting.")
+ return
+
+ # ---------- Step 2: Fetch and write lyrics with progress bar ----------
+ for track in tqdm(tracks_missing_lyrics, desc="Fetching lyrics", unit="track"):
+ artist_name = track.artists[0].title
+ album_title = track.album.title if track.album else "Unknown Album"
+ print(f"\n[INFO] Processing: '{track.title}' by '{artist_name}' on '{album_title}'")
+
+ lyrics_text = Lyrics.fetch_lyrics(artist_name, album_title, track.title, track.duration_ms)
+
+ if lyrics_text:
+ print("[SUCCESS] Lyrics found, writing to file...")
+ file_path = local_library_path / track.local_path
+ try:
+ Lyrics.write_lyrics(file_path, lyrics_text)
+ print(f"[SUCCESS] Lyrics written for '{track.title}'")
+ except Exception as e:
+ print(f"[ERROR] Failed to write lyrics for '{track.title}': {e}")
+ else:
+ print(f"[WARNING] Lyrics not found for '{track.title}'")
+
+
def load_libraries(self):
self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
diff --git a/main.py b/main.py
index 5e395a4..8499c07 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,21 @@
+
import Flow
-if __name__ == '__main__':
+def test1():
+ flow = Flow.Flow()
+
+ # flow.fetch_local()
+ flow.parse_local_library()
+ flow.load_libraries()
+
+ # flow.map_local_to_spotify()
+ # flow.log_stats()
+
+ flow.fetch_and_save_lyrics()
+
+
+def main():
flow = Flow.Flow()
# flow.fetch_spotify()
@@ -17,3 +31,7 @@ if __name__ == '__main__':
flow.log_stats()
print("asd")
+
+
+test1()
+# main()
diff --git a/src/Library.py b/src/Library.py
index 68e2c63..0de72ce 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -22,6 +22,7 @@ class Track:
self.duration_ms : int = None
self.added_at : datetime = None
self.local_path : Path = None
+ self.has_lyrics : bool = False
class Album:
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index e94a0f7..84f6a46 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -45,6 +45,7 @@ class LibrarySaver:
"duration_ms": track.duration_ms,
"added_at": _dt_to_str(track.added_at),
"local_path": track.local_path,
+ "has_lyrics": track.has_lyrics,
}
@staticmethod
@@ -119,6 +120,7 @@ class LibraryLoader:
track.local_path = data.get("local_path")
track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
track.album = self.get_album(data.get("album"))
+ track.has_lyrics = data.get("has_lyrics")
self.track_map[track.id] = track
return track
diff --git a/src/local/LocalLibraryIndexer.py b/src/local/LocalLibraryIndexer.py
index d3a7ce2..1a8eaab 100644
--- a/src/local/LocalLibraryIndexer.py
+++ b/src/local/LocalLibraryIndexer.py
@@ -1,8 +1,8 @@
-import os
import fnmatch
import mutagen
import eyed3
from mutagen.flac import FLAC
+from mutagen import File as MutagenFile
from src.helpers import *
@@ -11,6 +11,52 @@ music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
exclude_directories = ("*mary--*", "*example-word*")
+from mutagen.id3 import ID3
+from mutagen.id3 import ID3NoHeaderError
+
+
+def detect_lyrics(abs_path, ext):
+ ext = ext.lower()
+
+ # ---------- MP3 ----------
+ if ext == ".mp3":
+ try:
+ tags = ID3(abs_path)
+ except ID3NoHeaderError:
+ return False
+
+ return bool(
+ tags.getall("SYLT") or
+ tags.getall("USLT")
+ )
+
+ # ---------- FLAC / OGG / OPUS ----------
+ if ext in {".flac", ".ogg", ".opus"}:
+ try:
+ audio = MutagenFile(abs_path)
+ if not audio or not audio.tags:
+ return False
+
+ for key in audio.tags.keys():
+ k = key.upper()
+ if k in {"LYRICS", "LRC", "SYNCEDLYRICS"}:
+ if audio.tags[key]:
+ return True
+ except Exception:
+ return False
+
+ # ---------- Fallback ----------
+ try:
+ audio = MutagenFile(abs_path)
+ if audio and audio.tags:
+ for key in audio.tags.keys():
+ if "LYRIC" in key.upper():
+ return True
+ except Exception:
+ pass
+
+ return False
+
def update_local_library(root_dir, output_dir):
set_workdir(output_dir / "local")
@@ -29,50 +75,77 @@ def update_local_library(root_dir, output_dir):
return prev_id
for dir_path, dir_names, filenames in os.walk(root_dir):
- dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)]
- filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)]
+ dir_names[:] = [
+ d for d in dir_names
+ if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)
+ ]
+
+ filtered_filenames = [
+ filename for filename in filenames
+ if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)
+ ]
for filename in filtered_filenames:
song_name, track_type = os.path.splitext(filename)
relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir)
song_id = get_new_song_id(song_id, relative_path)
+
new_library[song_id] = {
"name": song_name,
"path": relative_path,
"type": track_type,
"artist": "",
- "album": ""
+ "album": "",
+ "duration": None
}
for track_id, track in new_library.items():
abs_path = os.path.join(root_dir, track['path'])
- match track['type']:
+
+ track_abs_path = os.path.join(root_dir, track["path"])
+ track["has_lyrics"] = detect_lyrics(track_abs_path, track['type'].lower())
+
+ match track['type'].lower():
case ".mp3":
mp3track = eyed3.load(abs_path)
- if not mp3track:
- print(f"Failed to load mp3 {abs_path} using just name of the file as track name")
+ if not mp3track or not mp3track.info:
+ print(f"Failed to load mp3 {abs_path}")
track['name'] = track['path']
else:
+ track['duration'] = mp3track.info.time_secs
+
tag = mp3track.tag
- if not tag:
- print(f"Invalid mp3 header {abs_path} using just name of the file as track name")
- track['name'] = track['path']
- else:
+ if tag:
track['artist'] = tag.artist
track['name'] = tag.title
track['album'] = tag.album
case ".flac":
try:
- metadata = FLAC(abs_path).tags
- if 'artist' in metadata:
- track['artist'] = " ".join(metadata['artist'])
- if 'TITLE' in metadata:
- track['name'] = " ".join(metadata['TITLE'])
- if 'album' in metadata:
- track['album'] = " ".join(metadata['album'])
+ audio = FLAC(abs_path)
+
+ if audio.info:
+ track['duration'] = audio.info.length
+
+ metadata = audio.tags
+ if metadata:
+ if 'artist' in metadata:
+ track['artist'] = " ".join(metadata['artist'])
+ if 'TITLE' in metadata:
+ track['name'] = " ".join(metadata['TITLE'])
+ if 'album' in metadata:
+ track['album'] = " ".join(metadata['album'])
+
except mutagen.MutagenError:
- print(f"Invalid flac header {abs_path} using just name of the file as track name")
+ print(f"Invalid flac header {abs_path}")
track['name'] = track['path']
+ case _:
+ try:
+ audio = MutagenFile(abs_path)
+ if audio and audio.info:
+ track['duration'] = audio.info.length
+ except Exception:
+ pass
+
save_data(new_library, "local_library")
diff --git a/src/local/LocalPLaylistGenerator.py b/src/local/LocalPLaylistGenerator.py
index 0299498..1c0f1e1 100644
--- a/src/local/LocalPLaylistGenerator.py
+++ b/src/local/LocalPLaylistGenerator.py
@@ -1,7 +1,7 @@
from pathlib import Path
-def generate_playlists(local_library, library_path, remote_playlists, link_pattern, output_dir):
+def ():
output_dir = Path(output_dir)
output_playlists = []
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
index 084bbce..49b3f55 100644
--- a/src/local/ParseLocal.py
+++ b/src/local/ParseLocal.py
@@ -116,6 +116,8 @@ class LocalParser:
track.artists = self.parse_artists(data.get("artist"))
track.album = self.parse_album(data.get("album"))
track.local_path = data.get("path")
+ track.duration_ms = data.get("duration")
+ track.has_lyrics = data.get("has_lyrics")
self.track_map[track.id] = track
self.library.tracks.append(track)
diff --git a/src/lyrics/lyrics.py b/src/lyrics/lyrics.py
new file mode 100644
index 0000000..fb427de
--- /dev/null
+++ b/src/lyrics/lyrics.py
@@ -0,0 +1,111 @@
+import requests
+from mutagen import File
+from mutagen.id3 import ID3, USLT, ID3NoHeaderError
+import os
+
+def fetch_lyrics(artist: str, album: str, track: str, duration: int = 0) -> str:
+
+ skip_artists = [
+ "shurupis",
+ "unknown"
+ ]
+
+ skip_album = [
+ "unknown"
+ ]
+
+ if artist in skip_artists or album in skip_album:
+ return ""
+
+
+ url = "https://lrclib.net/api/get"
+ params = {
+ "artist_name": artist,
+ "track_name": track,
+ "album_name": album,
+ "duration": duration,
+ }
+ try:
+ resp = requests.get(url, params=params, timeout=10)
+ if resp.status_code != 200:
+ try:
+ err = resp.json()
+ print(err.get("error"), "-", err.get("message"))
+ except ValueError:
+ print("HTTPError", resp.status_code)
+ return ""
+ data = resp.json()
+ if "error" in data:
+ print(data.get("error"), "-", data.get("message"))
+ return ""
+ return data.get("syncedLyrics", "")
+ except requests.RequestException as e:
+ print(type(e).__name__, "-", str(e))
+ return ""
+ except ValueError as e:
+ print(type(e).__name__, "-", str(e))
+ return ""
+
+
+def write_lyrics(file_path, lyrics_text, lang="eng"):
+ """
+ Writes lyrics text into tags recognized by Navidrome.
+ Provides detailed logs and success/failure info.
+ """
+
+ ext = os.path.splitext(file_path)[1].lower()
+ print(f"[INFO] Processing file: {file_path} (extension: {ext})")
+
+ try:
+ # ---------- MP3 ----------
+ if ext == ".mp3":
+ print("[INFO] Detected MP3 file")
+ try:
+ tags = ID3(file_path)
+ print("[INFO] Existing ID3 tags loaded")
+ except ID3NoHeaderError:
+ tags = ID3()
+ print("[INFO] No ID3 header found, creating new tags")
+
+ tags.delall("USLT")
+ print("[INFO] Removed existing USLT frames (lyrics)")
+
+ tags.add(
+ USLT(
+ encoding=3, # UTF-8
+ lang=lang,
+ desc="",
+ text=lyrics_text
+ )
+ )
+ tags.save(file_path)
+ print("[SUCCESS] Lyrics written to MP3 tags")
+ return
+
+ # ---------- FLAC / OGG / OPUS ----------
+ audio = File(file_path)
+ if audio is None:
+ print(f"[ERROR] Unsupported or unrecognized file format: {file_path}")
+ return
+
+ if ext in {".flac", ".ogg", ".opus"}:
+ print(f"[INFO] Detected {ext.upper()} file")
+ audio["LRC"] = lyrics_text
+ audio["SYNCEDLYRICS"] = lyrics_text
+ audio.save()
+ print(f"[SUCCESS] Lyrics written to {ext.upper()} tags")
+ return
+
+ # ---------- Fallback ----------
+ if audio.tags is not None:
+ print("[INFO] Using fallback tag writing")
+ audio["LYRICS"] = lyrics_text
+ audio.save()
+ print("[SUCCESS] Lyrics written using fallback tags")
+ return
+
+ print(f"[WARNING] No suitable tag frame found for {file_path}")
+
+ except Exception as e:
+ print(f"[ERROR] Failed to write lyrics to {file_path}: {e}")
+
From 5d38a146a1232cbe34325bb2cb5509b4867c8a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 17 Jan 2026 14:24:05 +0300
Subject: [PATCH 20/30] lyrics fetcher
---
Flow.py | 51 ++++++++++++-
main.py | 20 ++++-
src/Library.py | 1 +
src/LibrarySerializer.py | 2 +
src/local/LocalLibraryIndexer.py | 111 +++++++++++++++++++++++-----
src/local/LocalPLaylistGenerator.py | 2 +-
src/local/ParseLocal.py | 2 +
src/lyrics/lyrics.py | 111 ++++++++++++++++++++++++++++
8 files changed, 277 insertions(+), 23 deletions(-)
create mode 100644 src/lyrics/lyrics.py
diff --git a/Flow.py b/Flow.py
index bcbdd16..4731f1c 100644
--- a/Flow.py
+++ b/Flow.py
@@ -1,7 +1,11 @@
+from tqdm import tqdm
+import src.lyrics.lyrics as Lyrics
+
from Env import client_id, client_secret, local_library_path
from Env import work_dir
+from src.Library import *
from src.LibrarySerializer import LibrarySaver, LibraryLoader
from src.spotify.SpotifyWebAPI import fetch_all
@@ -15,8 +19,8 @@ from src.StatGenerator import log_stats
class Flow:
- spotify_library = None
- local_library = None
+ spotify_library : Library = None
+ local_library : Library = None
@staticmethod
def fetch_spotify(self):
@@ -45,6 +49,49 @@ class Flow:
library_parsed = parser.parse(work_dir / "local" / "local_library.json")
LibrarySaver(library_parsed).save(work_dir / "local_library.json")
+
+ def fetch_and_save_lyrics(self):
+ """
+ Fetches and writes lyrics for tracks that don't have lyrics yet.
+ Displays stats and uses a progress bar.
+ """
+
+ # ---------- Step 1: Collect tracks without lyrics ----------
+ tracks_missing_lyrics = [
+ track for track in self.local_library.tracks
+ if len(track.artists) > 0 and not track.has_lyrics
+ ]
+
+ total_tracks = len(self.local_library.tracks)
+ missing_count = len(tracks_missing_lyrics)
+
+ print(f"[INFO] Total tracks in library: {total_tracks}")
+ print(f"[INFO] Tracks missing lyrics: {missing_count}")
+
+ if missing_count == 0:
+ print("[INFO] No tracks need lyrics. Exiting.")
+ return
+
+ # ---------- Step 2: Fetch and write lyrics with progress bar ----------
+ for track in tqdm(tracks_missing_lyrics, desc="Fetching lyrics", unit="track"):
+ artist_name = track.artists[0].title
+ album_title = track.album.title if track.album else "Unknown Album"
+ print(f"\n[INFO] Processing: '{track.title}' by '{artist_name}' on '{album_title}'")
+
+ lyrics_text = Lyrics.fetch_lyrics(artist_name, album_title, track.title, track.duration_ms)
+
+ if lyrics_text:
+ print("[SUCCESS] Lyrics found, writing to file...")
+ file_path = local_library_path / track.local_path
+ try:
+ Lyrics.write_lyrics(file_path, lyrics_text)
+ print(f"[SUCCESS] Lyrics written for '{track.title}'")
+ except Exception as e:
+ print(f"[ERROR] Failed to write lyrics for '{track.title}': {e}")
+ else:
+ print(f"[WARNING] Lyrics not found for '{track.title}'")
+
+
def load_libraries(self):
self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
diff --git a/main.py b/main.py
index 5e395a4..8499c07 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,21 @@
+
import Flow
-if __name__ == '__main__':
+def test1():
+ flow = Flow.Flow()
+
+ # flow.fetch_local()
+ flow.parse_local_library()
+ flow.load_libraries()
+
+ # flow.map_local_to_spotify()
+ # flow.log_stats()
+
+ flow.fetch_and_save_lyrics()
+
+
+def main():
flow = Flow.Flow()
# flow.fetch_spotify()
@@ -17,3 +31,7 @@ if __name__ == '__main__':
flow.log_stats()
print("asd")
+
+
+test1()
+# main()
diff --git a/src/Library.py b/src/Library.py
index 68e2c63..0de72ce 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -22,6 +22,7 @@ class Track:
self.duration_ms : int = None
self.added_at : datetime = None
self.local_path : Path = None
+ self.has_lyrics : bool = False
class Album:
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index e94a0f7..84f6a46 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -45,6 +45,7 @@ class LibrarySaver:
"duration_ms": track.duration_ms,
"added_at": _dt_to_str(track.added_at),
"local_path": track.local_path,
+ "has_lyrics": track.has_lyrics,
}
@staticmethod
@@ -119,6 +120,7 @@ class LibraryLoader:
track.local_path = data.get("local_path")
track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
track.album = self.get_album(data.get("album"))
+ track.has_lyrics = data.get("has_lyrics")
self.track_map[track.id] = track
return track
diff --git a/src/local/LocalLibraryIndexer.py b/src/local/LocalLibraryIndexer.py
index d3a7ce2..1a8eaab 100644
--- a/src/local/LocalLibraryIndexer.py
+++ b/src/local/LocalLibraryIndexer.py
@@ -1,8 +1,8 @@
-import os
import fnmatch
import mutagen
import eyed3
from mutagen.flac import FLAC
+from mutagen import File as MutagenFile
from src.helpers import *
@@ -11,6 +11,52 @@ music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
exclude_directories = ("*mary--*", "*example-word*")
+from mutagen.id3 import ID3
+from mutagen.id3 import ID3NoHeaderError
+
+
+def detect_lyrics(abs_path, ext):
+ ext = ext.lower()
+
+ # ---------- MP3 ----------
+ if ext == ".mp3":
+ try:
+ tags = ID3(abs_path)
+ except ID3NoHeaderError:
+ return False
+
+ return bool(
+ tags.getall("SYLT") or
+ tags.getall("USLT")
+ )
+
+ # ---------- FLAC / OGG / OPUS ----------
+ if ext in {".flac", ".ogg", ".opus"}:
+ try:
+ audio = MutagenFile(abs_path)
+ if not audio or not audio.tags:
+ return False
+
+ for key in audio.tags.keys():
+ k = key.upper()
+ if k in {"LYRICS", "LRC", "SYNCEDLYRICS"}:
+ if audio.tags[key]:
+ return True
+ except Exception:
+ return False
+
+ # ---------- Fallback ----------
+ try:
+ audio = MutagenFile(abs_path)
+ if audio and audio.tags:
+ for key in audio.tags.keys():
+ if "LYRIC" in key.upper():
+ return True
+ except Exception:
+ pass
+
+ return False
+
def update_local_library(root_dir, output_dir):
set_workdir(output_dir / "local")
@@ -29,50 +75,77 @@ def update_local_library(root_dir, output_dir):
return prev_id
for dir_path, dir_names, filenames in os.walk(root_dir):
- dir_names[:] = [d for d in dir_names if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)]
- filtered_filenames = [filename for filename in filenames if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)]
+ dir_names[:] = [
+ d for d in dir_names
+ if not any(fnmatch.fnmatch(d, exclude) for exclude in exclude_directories)
+ ]
+
+ filtered_filenames = [
+ filename for filename in filenames
+ if any(fnmatch.fnmatch(filename, ext) for ext in music_extensions)
+ ]
for filename in filtered_filenames:
song_name, track_type = os.path.splitext(filename)
relative_path = os.path.relpath(os.path.join(dir_path, filename), root_dir)
song_id = get_new_song_id(song_id, relative_path)
+
new_library[song_id] = {
"name": song_name,
"path": relative_path,
"type": track_type,
"artist": "",
- "album": ""
+ "album": "",
+ "duration": None
}
for track_id, track in new_library.items():
abs_path = os.path.join(root_dir, track['path'])
- match track['type']:
+
+ track_abs_path = os.path.join(root_dir, track["path"])
+ track["has_lyrics"] = detect_lyrics(track_abs_path, track['type'].lower())
+
+ match track['type'].lower():
case ".mp3":
mp3track = eyed3.load(abs_path)
- if not mp3track:
- print(f"Failed to load mp3 {abs_path} using just name of the file as track name")
+ if not mp3track or not mp3track.info:
+ print(f"Failed to load mp3 {abs_path}")
track['name'] = track['path']
else:
+ track['duration'] = mp3track.info.time_secs
+
tag = mp3track.tag
- if not tag:
- print(f"Invalid mp3 header {abs_path} using just name of the file as track name")
- track['name'] = track['path']
- else:
+ if tag:
track['artist'] = tag.artist
track['name'] = tag.title
track['album'] = tag.album
case ".flac":
try:
- metadata = FLAC(abs_path).tags
- if 'artist' in metadata:
- track['artist'] = " ".join(metadata['artist'])
- if 'TITLE' in metadata:
- track['name'] = " ".join(metadata['TITLE'])
- if 'album' in metadata:
- track['album'] = " ".join(metadata['album'])
+ audio = FLAC(abs_path)
+
+ if audio.info:
+ track['duration'] = audio.info.length
+
+ metadata = audio.tags
+ if metadata:
+ if 'artist' in metadata:
+ track['artist'] = " ".join(metadata['artist'])
+ if 'TITLE' in metadata:
+ track['name'] = " ".join(metadata['TITLE'])
+ if 'album' in metadata:
+ track['album'] = " ".join(metadata['album'])
+
except mutagen.MutagenError:
- print(f"Invalid flac header {abs_path} using just name of the file as track name")
+ print(f"Invalid flac header {abs_path}")
track['name'] = track['path']
+ case _:
+ try:
+ audio = MutagenFile(abs_path)
+ if audio and audio.info:
+ track['duration'] = audio.info.length
+ except Exception:
+ pass
+
save_data(new_library, "local_library")
diff --git a/src/local/LocalPLaylistGenerator.py b/src/local/LocalPLaylistGenerator.py
index 0299498..1c0f1e1 100644
--- a/src/local/LocalPLaylistGenerator.py
+++ b/src/local/LocalPLaylistGenerator.py
@@ -1,7 +1,7 @@
from pathlib import Path
-def generate_playlists(local_library, library_path, remote_playlists, link_pattern, output_dir):
+def ():
output_dir = Path(output_dir)
output_playlists = []
diff --git a/src/local/ParseLocal.py b/src/local/ParseLocal.py
index 084bbce..49b3f55 100644
--- a/src/local/ParseLocal.py
+++ b/src/local/ParseLocal.py
@@ -116,6 +116,8 @@ class LocalParser:
track.artists = self.parse_artists(data.get("artist"))
track.album = self.parse_album(data.get("album"))
track.local_path = data.get("path")
+ track.duration_ms = data.get("duration")
+ track.has_lyrics = data.get("has_lyrics")
self.track_map[track.id] = track
self.library.tracks.append(track)
diff --git a/src/lyrics/lyrics.py b/src/lyrics/lyrics.py
new file mode 100644
index 0000000..fb427de
--- /dev/null
+++ b/src/lyrics/lyrics.py
@@ -0,0 +1,111 @@
+import requests
+from mutagen import File
+from mutagen.id3 import ID3, USLT, ID3NoHeaderError
+import os
+
+def fetch_lyrics(artist: str, album: str, track: str, duration: int = 0) -> str:
+
+ skip_artists = [
+ "shurupis",
+ "unknown"
+ ]
+
+ skip_album = [
+ "unknown"
+ ]
+
+ if artist in skip_artists or album in skip_album:
+ return ""
+
+
+ url = "https://lrclib.net/api/get"
+ params = {
+ "artist_name": artist,
+ "track_name": track,
+ "album_name": album,
+ "duration": duration,
+ }
+ try:
+ resp = requests.get(url, params=params, timeout=10)
+ if resp.status_code != 200:
+ try:
+ err = resp.json()
+ print(err.get("error"), "-", err.get("message"))
+ except ValueError:
+ print("HTTPError", resp.status_code)
+ return ""
+ data = resp.json()
+ if "error" in data:
+ print(data.get("error"), "-", data.get("message"))
+ return ""
+ return data.get("syncedLyrics", "")
+ except requests.RequestException as e:
+ print(type(e).__name__, "-", str(e))
+ return ""
+ except ValueError as e:
+ print(type(e).__name__, "-", str(e))
+ return ""
+
+
+def write_lyrics(file_path, lyrics_text, lang="eng"):
+ """
+ Writes lyrics text into tags recognized by Navidrome.
+ Provides detailed logs and success/failure info.
+ """
+
+ ext = os.path.splitext(file_path)[1].lower()
+ print(f"[INFO] Processing file: {file_path} (extension: {ext})")
+
+ try:
+ # ---------- MP3 ----------
+ if ext == ".mp3":
+ print("[INFO] Detected MP3 file")
+ try:
+ tags = ID3(file_path)
+ print("[INFO] Existing ID3 tags loaded")
+ except ID3NoHeaderError:
+ tags = ID3()
+ print("[INFO] No ID3 header found, creating new tags")
+
+ tags.delall("USLT")
+ print("[INFO] Removed existing USLT frames (lyrics)")
+
+ tags.add(
+ USLT(
+ encoding=3, # UTF-8
+ lang=lang,
+ desc="",
+ text=lyrics_text
+ )
+ )
+ tags.save(file_path)
+ print("[SUCCESS] Lyrics written to MP3 tags")
+ return
+
+ # ---------- FLAC / OGG / OPUS ----------
+ audio = File(file_path)
+ if audio is None:
+ print(f"[ERROR] Unsupported or unrecognized file format: {file_path}")
+ return
+
+ if ext in {".flac", ".ogg", ".opus"}:
+ print(f"[INFO] Detected {ext.upper()} file")
+ audio["LRC"] = lyrics_text
+ audio["SYNCEDLYRICS"] = lyrics_text
+ audio.save()
+ print(f"[SUCCESS] Lyrics written to {ext.upper()} tags")
+ return
+
+ # ---------- Fallback ----------
+ if audio.tags is not None:
+ print("[INFO] Using fallback tag writing")
+ audio["LYRICS"] = lyrics_text
+ audio.save()
+ print("[SUCCESS] Lyrics written using fallback tags")
+ return
+
+ print(f"[WARNING] No suitable tag frame found for {file_path}")
+
+ except Exception as e:
+ print(f"[ERROR] Failed to write lyrics to {file_path}: {e}")
+
From e5a6723cd004696413fd889f96daf8803e87ba7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Wed, 4 Feb 2026 21:21:19 +0300
Subject: [PATCH 21/30] stable
---
Flow.py | 8 ++++----
main.py | 2 +-
src/{ => backends}/itunes/ParseItunesToJson.py | 0
src/{ => backends}/local/LocalLibraryIndexer.py | 0
src/{ => backends}/local/LocalPLaylistGenerator.py | 0
src/{ => backends}/local/ParseLocal.py | 0
src/{ => backends}/spotify/ParseSpotify.py | 0
src/{ => backends}/spotify/SpotifyAuthenticator.py | 0
src/{ => backends}/spotify/SpotifyWebAPI.py | 2 +-
src/lyrics/lyrics.py | 6 ++++--
src/mappers/SpotifySearchMapping.py | 2 +-
11 files changed, 11 insertions(+), 9 deletions(-)
rename src/{ => backends}/itunes/ParseItunesToJson.py (100%)
rename src/{ => backends}/local/LocalLibraryIndexer.py (100%)
rename src/{ => backends}/local/LocalPLaylistGenerator.py (100%)
rename src/{ => backends}/local/ParseLocal.py (100%)
rename src/{ => backends}/spotify/ParseSpotify.py (100%)
rename src/{ => backends}/spotify/SpotifyAuthenticator.py (100%)
rename src/{ => backends}/spotify/SpotifyWebAPI.py (98%)
diff --git a/Flow.py b/Flow.py
index 4731f1c..e1ac33d 100644
--- a/Flow.py
+++ b/Flow.py
@@ -8,11 +8,11 @@ from Env import work_dir
from src.Library import *
from src.LibrarySerializer import LibrarySaver, LibraryLoader
-from src.spotify.SpotifyWebAPI import fetch_all
-from src.spotify.ParseSpotify import Parser
+from src.backends.spotify.SpotifyWebAPI import fetch_all
+from src.backends.spotify.ParseSpotify import Parser
-from src.local.LocalLibraryIndexer import update_local_library
-from src.local.ParseLocal import LocalParser
+from src.backends.local.LocalLibraryIndexer import update_local_library
+from src.backends.local.ParseLocal import LocalParser
from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
diff --git a/main.py b/main.py
index 8499c07..af9ed31 100644
--- a/main.py
+++ b/main.py
@@ -5,7 +5,7 @@ import Flow
def test1():
flow = Flow.Flow()
- # flow.fetch_local()
+ flow.fetch_local()
flow.parse_local_library()
flow.load_libraries()
diff --git a/src/itunes/ParseItunesToJson.py b/src/backends/itunes/ParseItunesToJson.py
similarity index 100%
rename from src/itunes/ParseItunesToJson.py
rename to src/backends/itunes/ParseItunesToJson.py
diff --git a/src/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py
similarity index 100%
rename from src/local/LocalLibraryIndexer.py
rename to src/backends/local/LocalLibraryIndexer.py
diff --git a/src/local/LocalPLaylistGenerator.py b/src/backends/local/LocalPLaylistGenerator.py
similarity index 100%
rename from src/local/LocalPLaylistGenerator.py
rename to src/backends/local/LocalPLaylistGenerator.py
diff --git a/src/local/ParseLocal.py b/src/backends/local/ParseLocal.py
similarity index 100%
rename from src/local/ParseLocal.py
rename to src/backends/local/ParseLocal.py
diff --git a/src/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
similarity index 100%
rename from src/spotify/ParseSpotify.py
rename to src/backends/spotify/ParseSpotify.py
diff --git a/src/spotify/SpotifyAuthenticator.py b/src/backends/spotify/SpotifyAuthenticator.py
similarity index 100%
rename from src/spotify/SpotifyAuthenticator.py
rename to src/backends/spotify/SpotifyAuthenticator.py
diff --git a/src/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py
similarity index 98%
rename from src/spotify/SpotifyWebAPI.py
rename to src/backends/spotify/SpotifyWebAPI.py
index 5ddc12f..8f4807f 100644
--- a/src/spotify/SpotifyWebAPI.py
+++ b/src/backends/spotify/SpotifyWebAPI.py
@@ -2,7 +2,7 @@ import requests
from PIL import Image
from io import BytesIO
-from src.spotify.SpotifyAuthenticator import authenticate
+from src.backends.spotify.SpotifyAuthenticator import authenticate
from src.helpers import *
diff --git a/src/lyrics/lyrics.py b/src/lyrics/lyrics.py
index fb427de..ae6ed6d 100644
--- a/src/lyrics/lyrics.py
+++ b/src/lyrics/lyrics.py
@@ -7,11 +7,13 @@ def fetch_lyrics(artist: str, album: str, track: str, duration: int = 0) -> str:
skip_artists = [
"shurupis",
- "unknown"
+ "unknown",
+ "Unknown"
]
skip_album = [
- "unknown"
+ "unknown",
+ "Unknown"
]
if artist in skip_artists or album in skip_album:
diff --git a/src/mappers/SpotifySearchMapping.py b/src/mappers/SpotifySearchMapping.py
index 41c5bbb..e8878e9 100644
--- a/src/mappers/SpotifySearchMapping.py
+++ b/src/mappers/SpotifySearchMapping.py
@@ -3,7 +3,7 @@ from tqdm import tqdm
from src.Library import *
from src.helpers import *
-from src.spotify.SpotifyWebAPI import find_song, update_access_token
+from src.backends.spotify.SpotifyWebAPI import find_song, update_access_token
def generate_map(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
From 35905ea9428cc78d575551daa6a00b3701aa8728 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Wed, 4 Feb 2026 21:21:19 +0300
Subject: [PATCH 22/30] stable
---
Flow.py | 8 ++++----
main.py | 2 +-
src/{ => backends}/itunes/ParseItunesToJson.py | 0
src/{ => backends}/local/LocalLibraryIndexer.py | 0
src/{ => backends}/local/LocalPLaylistGenerator.py | 0
src/{ => backends}/local/ParseLocal.py | 0
src/{ => backends}/spotify/ParseSpotify.py | 0
src/{ => backends}/spotify/SpotifyAuthenticator.py | 0
src/{ => backends}/spotify/SpotifyWebAPI.py | 2 +-
src/lyrics/lyrics.py | 6 ++++--
src/mappers/SpotifySearchMapping.py | 2 +-
11 files changed, 11 insertions(+), 9 deletions(-)
rename src/{ => backends}/itunes/ParseItunesToJson.py (100%)
rename src/{ => backends}/local/LocalLibraryIndexer.py (100%)
rename src/{ => backends}/local/LocalPLaylistGenerator.py (100%)
rename src/{ => backends}/local/ParseLocal.py (100%)
rename src/{ => backends}/spotify/ParseSpotify.py (100%)
rename src/{ => backends}/spotify/SpotifyAuthenticator.py (100%)
rename src/{ => backends}/spotify/SpotifyWebAPI.py (98%)
diff --git a/Flow.py b/Flow.py
index 4731f1c..e1ac33d 100644
--- a/Flow.py
+++ b/Flow.py
@@ -8,11 +8,11 @@ from Env import work_dir
from src.Library import *
from src.LibrarySerializer import LibrarySaver, LibraryLoader
-from src.spotify.SpotifyWebAPI import fetch_all
-from src.spotify.ParseSpotify import Parser
+from src.backends.spotify.SpotifyWebAPI import fetch_all
+from src.backends.spotify.ParseSpotify import Parser
-from src.local.LocalLibraryIndexer import update_local_library
-from src.local.ParseLocal import LocalParser
+from src.backends.local.LocalLibraryIndexer import update_local_library
+from src.backends.local.ParseLocal import LocalParser
from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
diff --git a/main.py b/main.py
index 8499c07..af9ed31 100644
--- a/main.py
+++ b/main.py
@@ -5,7 +5,7 @@ import Flow
def test1():
flow = Flow.Flow()
- # flow.fetch_local()
+ flow.fetch_local()
flow.parse_local_library()
flow.load_libraries()
diff --git a/src/itunes/ParseItunesToJson.py b/src/backends/itunes/ParseItunesToJson.py
similarity index 100%
rename from src/itunes/ParseItunesToJson.py
rename to src/backends/itunes/ParseItunesToJson.py
diff --git a/src/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py
similarity index 100%
rename from src/local/LocalLibraryIndexer.py
rename to src/backends/local/LocalLibraryIndexer.py
diff --git a/src/local/LocalPLaylistGenerator.py b/src/backends/local/LocalPLaylistGenerator.py
similarity index 100%
rename from src/local/LocalPLaylistGenerator.py
rename to src/backends/local/LocalPLaylistGenerator.py
diff --git a/src/local/ParseLocal.py b/src/backends/local/ParseLocal.py
similarity index 100%
rename from src/local/ParseLocal.py
rename to src/backends/local/ParseLocal.py
diff --git a/src/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
similarity index 100%
rename from src/spotify/ParseSpotify.py
rename to src/backends/spotify/ParseSpotify.py
diff --git a/src/spotify/SpotifyAuthenticator.py b/src/backends/spotify/SpotifyAuthenticator.py
similarity index 100%
rename from src/spotify/SpotifyAuthenticator.py
rename to src/backends/spotify/SpotifyAuthenticator.py
diff --git a/src/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py
similarity index 98%
rename from src/spotify/SpotifyWebAPI.py
rename to src/backends/spotify/SpotifyWebAPI.py
index 5ddc12f..8f4807f 100644
--- a/src/spotify/SpotifyWebAPI.py
+++ b/src/backends/spotify/SpotifyWebAPI.py
@@ -2,7 +2,7 @@ import requests
from PIL import Image
from io import BytesIO
-from src.spotify.SpotifyAuthenticator import authenticate
+from src.backends.spotify.SpotifyAuthenticator import authenticate
from src.helpers import *
diff --git a/src/lyrics/lyrics.py b/src/lyrics/lyrics.py
index fb427de..ae6ed6d 100644
--- a/src/lyrics/lyrics.py
+++ b/src/lyrics/lyrics.py
@@ -7,11 +7,13 @@ def fetch_lyrics(artist: str, album: str, track: str, duration: int = 0) -> str:
skip_artists = [
"shurupis",
- "unknown"
+ "unknown",
+ "Unknown"
]
skip_album = [
- "unknown"
+ "unknown",
+ "Unknown"
]
if artist in skip_artists or album in skip_album:
diff --git a/src/mappers/SpotifySearchMapping.py b/src/mappers/SpotifySearchMapping.py
index 41c5bbb..e8878e9 100644
--- a/src/mappers/SpotifySearchMapping.py
+++ b/src/mappers/SpotifySearchMapping.py
@@ -3,7 +3,7 @@ from tqdm import tqdm
from src.Library import *
from src.helpers import *
-from src.spotify.SpotifyWebAPI import find_song, update_access_token
+from src.backends.spotify.SpotifyWebAPI import find_song, update_access_token
def generate_map(client_id, client_secret, local_lib : Library, spotify_lib : Library, output_dir):
From 9390ead5bb4ee2a545433522f3fdd3534e2fda41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 7 Feb 2026 00:48:46 +0300
Subject: [PATCH 23/30] nice
---
Flow.py | 4 +
Gui.py | 206 ++++++++++++++++++++++
error.log | 2 +
gui.sh | 1 +
main.py | 14 +-
src/Library.py | 123 +++++++++----
src/LibrarySerializer.py | 31 +++-
src/StatGenerator.py | 21 +--
src/backends/local/LocalLibraryIndexer.py | 194 +++++++++++---------
src/backends/local/ParseLocal.py | 53 +-----
src/backends/navidrome/navidrome.py | 110 ++++++++++++
src/backends/spotify/ParseSpotify.py | 7 +-
src/mappers/FuzzyMapper.py | 54 +++++-
13 files changed, 630 insertions(+), 190 deletions(-)
create mode 100644 Gui.py
create mode 100644 error.log
create mode 100755 gui.sh
create mode 100644 src/backends/navidrome/navidrome.py
diff --git a/Flow.py b/Flow.py
index e1ac33d..5e813e0 100644
--- a/Flow.py
+++ b/Flow.py
@@ -96,6 +96,10 @@ class Flow:
self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
+ def save_libraries(self):
+ LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json")
+ LibrarySaver(self.local_library).save(work_dir / "local_library.json")
+
def map_local_to_spotify(self):
resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
diff --git a/Gui.py b/Gui.py
new file mode 100644
index 0000000..85fd6e3
--- /dev/null
+++ b/Gui.py
@@ -0,0 +1,206 @@
+import time
+
+import streamlit as st
+from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
+import pandas as pd
+
+import Flow
+
+from src.Library import Library, Artist, Album, Track, Playlist
+
+def track_to_row(track: Track):
+ return {
+ "Title": track.title,
+ "Album": track.album.title if track.album else "",
+ "Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists),
+ "Lyrics": track.has_lyrics,
+ "AutoScore": track.auto_score,
+ "Added": track.added_at.strftime("%Y-%m-%d") if track.added_at else "",
+ "LocalId": str(track.local_id) if track.local_id else "",
+ "Path": str(track.local_path) if track.local_path else "",
+ }
+
+def artist_to_row(artist: Artist):
+ return {
+ "Title": artist.title,
+ "AutoScore": artist.auto_score,
+ "Resolved": int(artist.resolved_percentage * 100),
+ "LocalId": str(artist.local_id) if artist.local_id else "",
+ }
+
+def album_to_row(album: Album):
+ return {
+ "Title": album.title,
+ "AutoScore": album.auto_score,
+ "Resolved": int(album.resolved_percentage * 100),
+ "LocalId": str(album.local_id) if album.local_id else "",
+ }
+
+
+class LibraryView:
+ def __init__(self):
+ self.lib : Library = None
+ self.libs : list[Library] = None
+ self.view = None
+ self.lyrics_only = False
+ self.query = None
+
+ def add_libraries(self, libs : list[Library]):
+ self.libs = libs
+
+ def render(self):
+ st.set_page_config(layout="wide")
+
+ with st.sidebar:
+ lib = st.selectbox(
+ "Library",
+ options=[lib.name for lib in self.libs],
+ )
+
+ for iter in self.libs:
+ if iter.name == lib:
+ self.lib = iter
+
+ st.write(f"Showing library '{self.lib.name}'")
+
+ self.view = st.radio(
+ "View",
+ ["Tree"]
+ )
+
+ if self.view == "Tree":
+ self.render_artists_albums_tracks()
+ # elif self.view == "Tracks":
+ # self.render_tracks()
+ # elif self.view == "Playlists":
+ # self.render_playlists()
+
+ def track_visible(
+ self,
+ track: Track,
+ artist: Artist | None,
+ album: Album | None,
+ ):
+ if self.lyrics_only and not track.has_lyrics:
+ return False
+
+ if self.query and self.query.lower() not in track.title.lower():
+ return False
+
+ if artist and all(a.id != artist.id for a in track.artists):
+ return False
+
+ if album and (not track.album or track.album.id != album.id):
+ return False
+
+ return True
+
+ def render_artists_albums_tracks(self):
+ # --- ARTISTS TABLE ---
+ st.subheader("Artists")
+ artist_rows = [artist_to_row(a) for a in self.lib.artists]
+ df_artists = pd.DataFrame(artist_rows)
+
+ gb_artists = GridOptionsBuilder.from_dataframe(df_artists)
+ gb_artists.configure_selection(selection_mode="multiple", use_checkbox=True)
+ gb_artists.configure_column("Title", filter="agTextColumnFilter")
+ gb_artists.configure_column("AutoScore", sort="desc")
+ grid_options_artists = gb_artists.build()
+
+ grid_response_artists = AgGrid(
+ df_artists,
+ gridOptions=grid_options_artists,
+ update_mode=GridUpdateMode.SELECTION_CHANGED,
+ allow_unsafe_jscode=True,
+ enable_enterprise_modules=False,
+ fit_columns_on_grid_load=True,
+ )
+
+ selected_rows_df = grid_response_artists.get("selected_rows")
+ selected_artist_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
+ selected_artists = [a for a in self.lib.artists if a.title in selected_artist_titles]
+
+ # --- ALBUMS TABLE ---
+ st.subheader("Albums")
+ if selected_artists:
+ albums_list = []
+ for artist in selected_artists:
+ albums_list.extend(artist.albums)
+ albums_list = list({al.id: al for al in albums_list}.values())
+ else:
+ albums_list = self.lib.albums
+
+ album_rows = [album_to_row(a) for a in albums_list]
+ df_albums = pd.DataFrame(album_rows)
+
+ gb_albums = GridOptionsBuilder.from_dataframe(df_albums)
+ gb_albums.configure_selection(selection_mode="multiple", use_checkbox=True)
+ gb_albums.configure_column("Title", filter="agTextColumnFilter")
+ gb_albums.configure_column("AutoScore", sort="desc")
+ grid_options_albums = gb_albums.build()
+
+ grid_response_albums = AgGrid(
+ df_albums,
+ gridOptions=grid_options_albums,
+ update_mode=GridUpdateMode.SELECTION_CHANGED,
+ allow_unsafe_jscode=True,
+ enable_enterprise_modules=False,
+ fit_columns_on_grid_load=True,
+ )
+
+ selected_rows_df = grid_response_albums.get("selected_rows")
+ selected_album_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
+ selected_albums = [al for al in albums_list if al.title in selected_album_titles]
+
+ # --- TRACKS TABLE ---
+ st.subheader("Tracks")
+ if selected_albums:
+ tracks_list = []
+ for album in selected_albums:
+ tracks_list.extend(album.tracks)
+ elif selected_artists:
+ tracks_list = []
+ for artist in selected_artists:
+ tracks_list.extend([t for t in self.lib.tracks if artist in t.artists])
+ else:
+ tracks_list = self.lib.tracks
+
+ track_rows = [track_to_row(t) for t in tracks_list]
+ df_tracks = pd.DataFrame(track_rows)
+
+ gb_tracks = GridOptionsBuilder.from_dataframe(df_tracks)
+ gb_tracks.configure_selection(selection_mode="multiple", use_checkbox=True)
+ gb_tracks.configure_column("Title", filter="agTextColumnFilter")
+ gb_tracks.configure_column("AutoScore", sort="desc")
+ grid_options_tracks = gb_tracks.build()
+
+ AgGrid(
+ df_tracks,
+ gridOptions=grid_options_tracks,
+ update_mode=GridUpdateMode.SELECTION_CHANGED,
+ allow_unsafe_jscode=True,
+ enable_enterprise_modules=False,
+ fit_columns_on_grid_load=True,
+ )
+
+
+def run():
+ flow = Flow.Flow()
+
+ # flow.fetch_spotify()
+ # flow.fetch_local()
+
+ # flow.parse_spotify_library()
+ # flow.parse_local_library()
+
+ flow.load_libraries()
+
+ # flow.map_local_to_spotify()
+
+ # flow.log_stats()
+
+ gui = LibraryView()
+ gui.add_libraries([flow.spotify_library, flow.local_library])
+ gui.render()
+
+run()
\ No newline at end of file
diff --git a/error.log b/error.log
new file mode 100644
index 0000000..7c830a2
--- /dev/null
+++ b/error.log
@@ -0,0 +1,2 @@
+
+(process:38049): GLib-GIO-CRITICAL **: 14:06:16.561: g_dbus_connection_emit_signal: assertion 'G_IS_DBUS_CONNECTION (connection)' failed
diff --git a/gui.sh b/gui.sh
new file mode 100755
index 0000000..eb55739
--- /dev/null
+++ b/gui.sh
@@ -0,0 +1 @@
+streamlit run Gui.py
diff --git a/main.py b/main.py
index af9ed31..646539a 100644
--- a/main.py
+++ b/main.py
@@ -5,15 +5,14 @@ import Flow
def test1():
flow = Flow.Flow()
- flow.fetch_local()
+ # flow.fetch_local()
flow.parse_local_library()
flow.load_libraries()
# flow.map_local_to_spotify()
# flow.log_stats()
- flow.fetch_and_save_lyrics()
-
+ # flow.fetch_and_save_lyrics()
def main():
flow = Flow.Flow()
@@ -27,11 +26,12 @@ def main():
flow.load_libraries()
flow.map_local_to_spotify()
+ flow.save_libraries()
- flow.log_stats()
+ # flow.log_stats()
- print("asd")
+ # print("asd")
-test1()
-# main()
+# test1()
+main()
diff --git a/src/Library.py b/src/Library.py
index 0de72ce..46d7566 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -7,64 +7,86 @@ from typing import List, Optional, Any
class Artist:
def __init__(self):
- self.id : str = None
- self.title : str = None
- self.tracks : List[Track] = []
- self.albums : List[Album] = []
+ self.id: str = None
+ self.title: str = None
+ self.tracks: List[Track] = []
+ self.albums: List[Album] = []
+ self.local_id: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
+ self.resolved_percentage : float = 0
class Track:
def __init__(self):
- self.id : str = None
- self.title : str = None
- self.artists : List[Artist] = []
- self.album : Album = None
- self.duration_ms : int = None
- self.added_at : datetime = None
- self.local_path : Path = None
- self.has_lyrics : bool = False
+ self.id: str = None
+ self.title: str = None
+ self.artists: List[Artist] = []
+ self.album: Album = None
+ self.duration_ms: int = None
+ self.added_at: datetime = None
+ self.local_path: Path = None
+ self.has_lyrics: bool = False
+ self.local_id: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
class Album:
def __init__(self):
- self.id : str = None
- self.title : str = None
- self.release_date : datetime = None
- self.tracks : List[Track] = []
- self.artists : List[Artist] = []
+ self.id: str = None
+ self.title: str = None
+ self.release_date: datetime = None
+ self.tracks: List[Track] = []
+ self.artists: List[Artist] = []
+ self.local_id: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
+ self.resolved_percentage : float = 0
class Playlist:
def __init__(self):
- self.title : str = None
- self.tracks : List[Track] = []
- self.created_at : datetime = None
- self.description : str = None
+ self.title: str = None
+ self.tracks: List[Track] = []
+ self.created_at: datetime = None
+ self.description: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
+ self.resolved_percentage : float = 0
class Library:
def __init__(self):
- self.artists : List[Artist] = []
- self.playlists : List[Playlist] = []
- self.tracks : List[Track] = []
- self.albums : List[Album] = []
- self.liked_tracks : List[Track] = []
- self.top_tracks : List[Track]= []
- self.top_artists : List[Artist] = []
+ self.artists: List[Artist] = []
+ self.playlists: List[Playlist] = []
+ self.tracks: List[Track] = []
+ self.albums: List[Album] = []
- self.artist_map : dict[str, Artist] = None
- self.album_map : dict[str, Album] = None
- self.track_map : dict[str, Track] = None
+ self.liked_tracks: List[Track] = []
+ self.top_tracks: List[Track] = []
+ self.top_artists: List[Artist] = []
+ self.artist_map: dict[str, Artist] = None
+ self.album_map: dict[str, Album] = None
+ self.track_map: dict[str, Track] = None
+
+ self.name = "Unnamed"
def update_cache(self):
self.update_id_maps()
self.create_reverse_links()
+ self.score_albums()
+ self.calc_resolved_percentage()
def update_id_maps(self):
- self.artist_map : dict[str, Artist] = { artist.id : artist for artist in self.artists }
- self.album_map : dict[str, Album] = { album.id : album for album in self.albums }
- self.track_map : dict[str, Track] = { track.id : track for track in self.tracks }
+ self.artist_map: dict[str, Artist] = {artist.id: artist for artist in self.artists}
+ self.album_map: dict[str, Album] = {album.id: album for album in self.albums}
+ self.track_map: dict[str, Track] = {track.id: track for track in self.tracks}
def create_reverse_links(self):
for track_id, track in self.track_map.items():
@@ -79,3 +101,36 @@ class Library:
for artist in album.artists:
if album not in artist.albums:
artist.albums.append(album)
+
+ def auto_score(self):
+ self.score_tracks()
+ self.score_artists()
+ self.score_albums()
+
+ def score_tracks(self):
+ for idx, track in enumerate(self.top_tracks):
+ score = len(self.top_tracks) - idx
+ track.auto_score = score
+
+ def score_artists(self):
+ for track in self.tracks:
+ for artist in track.artists:
+ artist.auto_score += track.auto_score
+
+ def score_albums(self):
+ for track in self.tracks:
+ track.album.auto_score += track.auto_score
+
+ def calc_resolved_percentage(self):
+ for artist in self.artists:
+ artist_count_resolved = 0
+ artist_count = 0
+ for album in artist.albums:
+ album_count_resolved = 0
+ for track in album.tracks:
+ if track.local_id:
+ album_count_resolved += 1
+ album.resolved_percentage = album_count_resolved / len(album.tracks)
+ artist_count_resolved += album_count_resolved
+ artist_count += len(album.tracks)
+ artist.resolved_percentage = artist_count_resolved / artist_count
\ No newline at end of file
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index 84f6a46..f331824 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -33,6 +33,10 @@ class LibrarySaver:
"title": artist.title,
"tracks": [track.id for track in artist.tracks],
"albums": [album.id for album in artist.albums],
+ "local_id": artist.local_id,
+ "auto_score": artist.auto_score,
+ "fav" : artist.fav,
+ "rating" : artist.rating,
}
@staticmethod
@@ -46,6 +50,10 @@ class LibrarySaver:
"added_at": _dt_to_str(track.added_at),
"local_path": track.local_path,
"has_lyrics": track.has_lyrics,
+ "local_id": track.local_id,
+ "auto_score": track.auto_score,
+ "fav" : track.fav,
+ "rating" : track.rating,
}
@staticmethod
@@ -55,6 +63,9 @@ class LibrarySaver:
"tracks": [track.id for track in playlist.tracks],
"created_at": _dt_to_str(playlist.created_at),
"description": playlist.description,
+ "auto_score": playlist.auto_score,
+ "fav" : playlist.fav,
+ "rating" : playlist.rating,
}
@staticmethod
@@ -64,12 +75,17 @@ class LibrarySaver:
"title": album.title,
"release_date": album.release_date,
"tracks": [track.id for track in album.tracks],
- "artists": [artist.id for artist in album.artists]
+ "artists": [artist.id for artist in album.artists],
+ "local_id": album.local_id,
+ "auto_score": album.auto_score,
+ "fav" : album.fav,
+ "rating" : album.rating,
}
def save(self, path: Path) -> None:
data = {
+ "name": self.library.name,
"tracks": [self._track_to_dict(track) for track in self.library.tracks],
"albums": [self._album_to_dict(album) for album in self.library.albums],
"artists": [self._artist_to_dict(artist) for artist in self.library.artists],
@@ -121,6 +137,10 @@ class LibraryLoader:
track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
track.album = self.get_album(data.get("album"))
track.has_lyrics = data.get("has_lyrics")
+ track.local_id = data.get("local_id")
+ track.auto_score = data.get("auto_score")
+ track.fav = data.get("fav")
+ track.rating = data.get("rating")
self.track_map[track.id] = track
return track
@@ -131,6 +151,10 @@ class LibraryLoader:
artist.title = data.get("title")
artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else []
+ artist.local_id = data.get("local_id")
+ artist.auto_score = data.get("auto_score")
+ artist.fav = data.get("fav")
+ artist.rating = data.get("rating")
self.artist_map[artist.id] = artist
return artist
@@ -141,6 +165,10 @@ class LibraryLoader:
album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
album.title = data.get("title")
album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
+ album.local_id = data.get("local_id")
+ album.auto_score = data.get("auto_score")
+ album.fav = data.get("fav")
+ album.rating = data.get("rating")
self.album_map[album.id] = album
return album
@@ -151,6 +179,7 @@ class LibraryLoader:
library = Library()
+ library.name = data.get("name")
library.tracks = [self.load_track(track) for track in data.get("tracks")]
library.albums = [self.load_album(album) for album in data.get("albums")]
library.artists = [self.load_artist(artist) for artist in data.get("artists")]
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index ac16622..7c17ab5 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -15,13 +15,12 @@ def get_mapping(mapping_path) -> dict[str, Any]:
with Path(mapping_path).open("r", encoding="utf-8") as f:
data = json.load(f)
- return data
+ return data["track_map"]
class AlbumStat:
def __init__(self):
self.album: Album = Album()
- self.score: int = 0
self.resolved_precent: float = 0
def calc_resolved_precent(self):
@@ -37,7 +36,6 @@ class AlbumStat:
class ArtistStat:
def __init__(self):
self.artist: Artist = Artist()
- self.score: int = 0
self.resolved_precent: float = 0
self.albums_stats: list[AlbumStat] = []
@@ -87,18 +85,22 @@ class LibraryStats:
for remote, local in self.mapping.items():
self.remote_id_map[remote].local_path = self.local_id_map[local].local_path
+ def score_tracks(self):
+ for idx, track in enumerate(self.remote_lib.top_tracks):
+ score = len(self.remote_lib.top_tracks) - idx
+ track.auto_score = score
+
def score_artists(self):
for artist in self.remote_lib.artists:
artist_stat = ArtistStat()
artist_stat.artist = artist
- artist_stat.score = 0
+ artist_stat.artist.auto_score = 0
self.artists_map[artist_stat.artist.id] = artist_stat
self.artists.append(artist_stat)
- for idx, track in enumerate(self.remote_lib.top_tracks):
- score = len(self.remote_lib.top_tracks) - idx
+ for track in self.remote_lib.tracks:
for artist in track.artists:
- self.artists_map[artist.id].score += score
+ self.artists_map[artist.id].artist.auto_score += track.auto_score
for artist_stat in self.artists:
artist_stat.calc_resolved_precent()
@@ -111,9 +113,8 @@ class LibraryStats:
stat.album = album
self.albums_map[album.id] = stat
- for idx, track in enumerate(self.remote_lib.top_tracks):
- score = len(self.remote_lib.top_tracks) - idx
- self.albums_map[track.album.id].score += score
+ for track in self.remote_lib.tracks:
+ self.albums_map[track.album.id].album.auto_score += track.auto_score
for _, album in self.albums_map.items():
album.calc_resolved_precent()
diff --git a/src/backends/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py
index 1a8eaab..6f132d4 100644
--- a/src/backends/local/LocalLibraryIndexer.py
+++ b/src/backends/local/LocalLibraryIndexer.py
@@ -1,64 +1,97 @@
+import os
import fnmatch
-import mutagen
-import eyed3
-from mutagen.flac import FLAC
from mutagen import File as MutagenFile
+from mutagen.id3 import ID3, ID3NoHeaderError
+from mutagen.flac import FLAC
+from mutagen.mp4 import MP4
+from mutagen.oggvorbis import OggVorbis
+from tqdm import tqdm
from src.helpers import *
-
music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
exclude_directories = ("*mary--*", "*example-word*")
-from mutagen.id3 import ID3
-from mutagen.id3 import ID3NoHeaderError
-
-
def detect_lyrics(abs_path, ext):
ext = ext.lower()
- # ---------- MP3 ----------
- if ext == ".mp3":
- try:
- tags = ID3(abs_path)
- except ID3NoHeaderError:
- return False
-
- return bool(
- tags.getall("SYLT") or
- tags.getall("USLT")
- )
-
- # ---------- FLAC / OGG / OPUS ----------
- if ext in {".flac", ".ogg", ".opus"}:
- try:
- audio = MutagenFile(abs_path)
- if not audio or not audio.tags:
- return False
-
- for key in audio.tags.keys():
- k = key.upper()
- if k in {"LYRICS", "LRC", "SYNCEDLYRICS"}:
- if audio.tags[key]:
- return True
- except Exception:
- return False
-
- # ---------- Fallback ----------
try:
audio = MutagenFile(abs_path)
- if audio and audio.tags:
- for key in audio.tags.keys():
- if "LYRIC" in key.upper():
- return True
+ if not audio or not audio.tags:
+ return False
+
+ # MP3-specific SYLT/USLT detection
+ if ext == ".mp3" and isinstance(audio, ID3):
+ return bool(audio.getall("SYLT") or audio.getall("USLT"))
+
+ # FLAC/OGG/OPUS lyrics detection
+ for key, value in audio.tags.items():
+ k = key.upper()
+ if k in {"LYRICS", "LRC", "SYNCEDLYRICS"} and value:
+ return True
+
+ # Generic fallback
+ for key in audio.tags.keys():
+ if "LYRIC" in key.upper():
+ return True
+
except Exception:
- pass
+ return False
return False
-def update_local_library(root_dir, output_dir):
+def get_artist(tags, fallback=None) -> list :
+ # FLAC / OGG / general multiple artist fields
+ artist_fields = ["ARTISTS", "TXXX:ARTISTS", "©ART", "artist"]
+ for field in artist_fields:
+ if field in tags and tags[field]:
+ values = tags.get(field)
+ if not isinstance(values, list):
+ return values.text
+ return values
+
+ # M4A
+ if '\xa9ART' in tags and tags['\xa9ART']:
+ values = tags['\xa9ART']
+ return values
+
+ if tags.get("TPE1"):
+ text = getattr(tags.get("TPE1"), "text", None)
+ if text:
+ return text
+
+ return fallback
+
+def safe_tag_get(tags, key, index=0):
+ try:
+ value = tags.get(key, None)
+ if isinstance(value, list):
+ # filter out None/empty, convert to str
+ value = [str(v) for v in value if v]
+ return value[index] if len(value) > index else fallback
+ return str(value)
+ except (KeyError, ValueError, TypeError):
+ return None
+
+def get_title(tags):
+ ids = ["TIT2", "TITLE", "title", "\xa9nam"]
+ for key in ids:
+ val = safe_tag_get(tags, key)
+ if val and val != "None":
+ return val
+ return None
+
+def get_album(tags):
+ ids = ["TALB", "album", "\xa9alb"]
+ for key in ids:
+ val = safe_tag_get(tags, key)
+ if val and val != "None":
+ return val
+ return None
+
+def update_local_library(root_dir, output_dir):
set_workdir(output_dir / "local")
old_library = get_data("local_library")
@@ -91,61 +124,48 @@ def update_local_library(root_dir, output_dir):
song_id = get_new_song_id(song_id, relative_path)
new_library[song_id] = {
- "name": song_name,
+ "name": None,
"path": relative_path,
"type": track_type,
- "artist": "",
- "album": "",
+ "artists": None,
+ "album": None,
"duration": None
}
- for track_id, track in new_library.items():
- abs_path = os.path.join(root_dir, track['path'])
+ to_remove = []
- track_abs_path = os.path.join(root_dir, track["path"])
- track["has_lyrics"] = detect_lyrics(track_abs_path, track['type'].lower())
+ with tqdm(total=len(new_library.items()), desc='Indexing local library') as process_bar:
+ for track_id, track in new_library.items():
+ process_bar.update(1)
+
+ abs_path = os.path.join(root_dir, track['path'])
+ track["has_lyrics"] = detect_lyrics(abs_path, track['type'].lower())
+
+ try:
+ audio = MutagenFile(abs_path)
+ if audio and audio.tags:
+ tags = audio.tags
+
+ track['artists'] = get_artist(tags)
+ track['name'] = get_title(tags)
+ track['album'] = get_album(tags)
+
+ if not len(track['artists']) or not track['name'] or not track['album']:
+ print(track)
+ raise "Cannot index track"
- match track['type'].lower():
- case ".mp3":
- mp3track = eyed3.load(abs_path)
- if not mp3track or not mp3track.info:
- print(f"Failed to load mp3 {abs_path}")
- track['name'] = track['path']
else:
- track['duration'] = mp3track.info.time_secs
+ print(track)
+ to_remove.append(track_id)
+ # raise "Cannot index track"
- tag = mp3track.tag
- if tag:
- track['artist'] = tag.artist
- track['name'] = tag.title
- track['album'] = tag.album
+ except Exception as e:
+ print(e)
+ print(abs_path)
+ raise e
- case ".flac":
- try:
- audio = FLAC(abs_path)
- if audio.info:
- track['duration'] = audio.info.length
-
- metadata = audio.tags
- if metadata:
- if 'artist' in metadata:
- track['artist'] = " ".join(metadata['artist'])
- if 'TITLE' in metadata:
- track['name'] = " ".join(metadata['TITLE'])
- if 'album' in metadata:
- track['album'] = " ".join(metadata['album'])
-
- except mutagen.MutagenError:
- print(f"Invalid flac header {abs_path}")
- track['name'] = track['path']
-
- case _:
- try:
- audio = MutagenFile(abs_path)
- if audio and audio.info:
- track['duration'] = audio.info.length
- except Exception:
- pass
+ for rem in to_remove:
+ new_library.pop(rem)
save_data(new_library, "local_library")
diff --git a/src/backends/local/ParseLocal.py b/src/backends/local/ParseLocal.py
index 49b3f55..b1fbba3 100644
--- a/src/backends/local/ParseLocal.py
+++ b/src/backends/local/ParseLocal.py
@@ -10,46 +10,6 @@ PROTECTED_ARTISTS = {
"The Good, The Bad & The Queen",
}
-def split_artists(artist_tag: str) -> list[str]:
- if not artist_tag:
- return []
-
- placeholder_comma = "§COMMA§"
- placeholder_amp = "§AMP§"
-
- protected_map = {}
-
- # Protect known artist names
- for artist in PROTECTED_ARTISTS:
- protected = (
- artist
- .replace(",", placeholder_comma)
- .replace("&", placeholder_amp)
- )
- protected_map[protected] = artist
- artist_tag = artist_tag.replace(artist, protected)
-
- # Split remaining separators
- parts = re.split(r"[,&]", artist_tag)
-
- # Restore protected names and trim spaces
- result = []
- for part in parts:
- part = part.strip()
- if not part:
- continue
-
- part = (
- part
- .replace(placeholder_comma, ",")
- .replace(placeholder_amp, "&")
- )
-
- result.append(part)
-
- return result
-
-
class LocalParser:
def __init__(self):
self.track_map: dict[str, Track] = {}
@@ -60,9 +20,7 @@ class LocalParser:
def parse_artists(self, data: str) -> List[Artist]:
if not data:
- data = "unknown"
-
- data = split_artists(data)
+ raise "No artist found"
artists = []
@@ -87,7 +45,7 @@ class LocalParser:
def parse_album(self, data: str) -> Album:
if not data:
- data = "unknown"
+ raise "album is none"
name = data
album_id = name
@@ -113,12 +71,15 @@ class LocalParser:
track.id = track_id
track.title = data.get("name")
- track.artists = self.parse_artists(data.get("artist"))
+ track.artists = self.parse_artists(data.get("artists"))
track.album = self.parse_album(data.get("album"))
track.local_path = data.get("path")
track.duration_ms = data.get("duration")
track.has_lyrics = data.get("has_lyrics")
+ if not track.album or not len(track.artists) or not track.title:
+ raise "error parsing"
+
self.track_map[track.id] = track
self.library.tracks.append(track)
@@ -129,6 +90,8 @@ class LocalParser:
self.library.update_cache()
+ self.library.name = "Local"
+
return self.library
diff --git a/src/backends/navidrome/navidrome.py b/src/backends/navidrome/navidrome.py
new file mode 100644
index 0000000..c7d04f4
--- /dev/null
+++ b/src/backends/navidrome/navidrome.py
@@ -0,0 +1,110 @@
+import sqlite3
+import os
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+@dataclass
+class Track:
+ id: int
+ title: str
+ path: str
+ favorite: bool
+ rating: Optional[int]
+ play_count: int
+ last_played: Optional[str]
+ playlists: List[str] = field(default_factory=list)
+
+@dataclass
+class Album:
+ id: int
+ title: str
+ tracks: List[Track] = field(default_factory=list)
+
+@dataclass
+class Artist:
+ id: str
+ name: str
+ albums: List[Album] = field(default_factory=list)
+
+class NavidromeDB:
+ def __init__(self, db_path: str):
+ if not os.path.isfile(db_path):
+ raise FileNotFoundError(f"Navidrome DB not found: {db_path}")
+ self.conn = sqlite3.connect(db_path)
+ self.conn.row_factory = sqlite3.Row
+
+ def close(self):
+ self.conn.close()
+
+ def get_artists(self) -> List[Artist]:
+ artists = []
+ c = self.conn.cursor()
+ c.execute("SELECT id, name FROM artist ORDER BY name")
+ for row in c.fetchall():
+ artist = Artist(id=row["id"], name=row["name"])
+ artist.albums = self.get_albums(artist.id)
+ artists.append(artist)
+ return artists
+
+ def get_albums(self, artist_id: str) -> List[Album]:
+ albums = []
+ c = self.conn.cursor()
+ c.execute(
+ "SELECT id, name FROM album WHERE album_artist_id=? ORDER BY name",
+ (artist_id,)
+ )
+ for row in c.fetchall():
+ album = Album(id=row["id"], title=row["name"])
+ album.tracks = self.get_tracks(album.id)
+ albums.append(album)
+ return albums
+
+ def get_tracks(self, album_id: int) -> List[Track]:
+ tracks = []
+ c = self.conn.cursor()
+ c.execute(
+ "SELECT t.id, t.name AS Title, t.path, ut.favorite, ut.rating, ut.play_count AS PlayCount, ut.last_played AS LastPlayed "
+ "FROM track t "
+ "LEFT JOIN usertrack ut ON ut.track_id = t.id "
+ "WHERE t.album_id=? ORDER BY t.track_number",
+ (album_id,)
+ )
+ for row in c.fetchall():
+ track = Track(
+ id=row["id"],
+ title=row["Title"],
+ path=row["path"],
+ favorite=bool(row["favorite"]),
+ rating=row["rating"],
+ play_count=row["PlayCount"] or 0,
+ last_played=row["LastPlayed"],
+ playlists=self.get_playlists_for_track(row["id"])
+ )
+ tracks.append(track)
+ return tracks
+
+ def get_playlists_for_track(self, track_id: int) -> List[str]:
+ c = self.conn.cursor()
+ c.execute(
+ "SELECT p.name FROM playlist p "
+ "JOIN playlisttrack pt ON pt.playlist_id=p.id "
+ "WHERE pt.track_id=?",
+ (track_id,)
+ )
+ return [row["name"] for row in c.fetchall()]
+
+# Example usage
+if __name__ == "__main__":
+ db_path = "/mnt/main/data/music/navidrome.db"
+ navidb = NavidromeDB(db_path)
+
+ try:
+ artists = navidb.get_artists()
+ for artist in artists:
+ print(f"Artist: {artist.name}")
+ for album in artist.albums:
+ print(f" Album: {album.title}")
+ for track in album.tracks:
+ print(f" Track: {track.title} | Path: {track.path} | Fav: {track.favorite} | Plays: {track.play_count}")
+ finally:
+ navidb.close()
diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
index 50806bf..29c24e7 100644
--- a/src/backends/spotify/ParseSpotify.py
+++ b/src/backends/spotify/ParseSpotify.py
@@ -30,6 +30,8 @@ class Parser:
self.parse_top_tracks(json.loads(top_tracks.read_text()))
self.library.update_cache()
+ self.library.name = "Spotify"
+ self.library.auto_score()
return self.library
@@ -75,7 +77,7 @@ class Parser:
track.id = track_id
track.title = data.get("name")
- track.artists = [self.parse_artist(artist) for artist in data.get("artists")]
+ track.artists = [] + self.parse_album(data.get("album")).artists
track.album = self.parse_album(data.get("album"))
track.duration_ms = data.get("duration_ms")
@@ -114,6 +116,9 @@ class Parser:
artist.id = data.get("id")
artist.title = data.get("name")
+ if artist.title is None:
+ artist.title = "err_no_title"
+
self.library.artists.append(artist)
return artist
diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py
index ca78db9..c6413d0 100644
--- a/src/mappers/FuzzyMapper.py
+++ b/src/mappers/FuzzyMapper.py
@@ -5,12 +5,46 @@ from tqdm import tqdm
from src.Library import *
from src.helpers import *
+import unicodedata
+
+user_def_artist_map = {
+ "donda" : "kanye west",
+}
+
+def normalize(s):
+ return unicodedata.normalize("NFKC", s).replace("‐", "-")
def get_score(first: str, second: str) -> float:
+ first_norm = normalize(first).lower()
+ second_norm = normalize(second).lower()
+ return float(first_norm == second_norm) * 100
return fuzz.token_set_ratio(first, second)
+def get_album_score(first: str, second: str) -> float:
+ return get_score(first, second)
+
+def get_track_score(first: str, second: str) -> float:
+ return get_score(first, second)
+
+def get_artist_score(first: str, second: str) -> float:
+ if not first or not second:
+ return 0.0
+
+ first_lower = first.lower()
+ second_lower = second.lower()
+
+ for key, value in user_def_artist_map.items():
+ key_lower = key.lower()
+ value_lower = value.lower()
+ if (first_lower == key_lower and second_lower == value_lower) or \
+ (first_lower == value_lower and second_lower == key_lower):
+ return 100.0
+
+ return get_score(first, second)
+
+
def is_close_enough(first: float) -> bool:
- return first > 50
+ return first > 90
def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
@@ -21,12 +55,13 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
with (tqdm(total=len(remote_lib.artists), desc='searching remote artists in local library') as process_bar):
for remote_artist in remote_lib.artists:
for local_artist in local_lib.artists:
- score = get_score(remote_artist.title, local_artist.title)
+ score = get_artist_score(remote_artist.title, local_artist.title)
if not is_close_enough(score):
continue
if remote_artist.id not in artist_map or score > artist_map[remote_artist.id][1]:
artist_map[remote_artist.id] = (local_artist.id, score)
+ remote_artist.local_id = local_artist.id
process_bar.update(1)
@@ -38,12 +73,13 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
for remote_album in remote_artist.albums:
for local_album in local_artist.albums:
- score = get_score(remote_album.title, local_album.title)
+ score = get_album_score(remote_album.title, local_album.title)
if not is_close_enough(score):
continue
if remote_album.id not in artist_map or score > album_map[remote_album.id][1]:
album_map[remote_album.id] = (local_album.id, score)
+ remote_album.local_id = local_album.id
process_bar.update(1)
@@ -54,15 +90,23 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
for remote_track in remote_album.tracks:
for local_track in local_album.tracks:
- score = get_score(remote_track.title, local_track.title)
+ score = get_track_score(remote_track.title, local_track.title)
if not is_close_enough(score):
continue
if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
track_map[remote_track.id] = (local_track.id, score)
+ remote_track.local_id = local_track.id
process_bar.update(1)
- out = { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() }
+ remote_lib.calc_resolved_percentage()
+
+ out = {
+ "artist_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in artist_map.items() },
+ "album_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in album_map.items() },
+ "track_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() },
+ }
+
set_workdir(output_dir)
save_data(out, "remote_to_local_map")
From 99cd872227fa01bd237a5994ef10b3712e489141 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 7 Feb 2026 00:48:46 +0300
Subject: [PATCH 24/30] nice
---
Flow.py | 4 +
Gui.py | 206 ++++++++++++++++++++++
error.log | 2 +
gui.sh | 1 +
main.py | 14 +-
src/Library.py | 123 +++++++++----
src/LibrarySerializer.py | 31 +++-
src/StatGenerator.py | 21 +--
src/backends/local/LocalLibraryIndexer.py | 194 +++++++++++---------
src/backends/local/ParseLocal.py | 53 +-----
src/backends/navidrome/navidrome.py | 110 ++++++++++++
src/backends/spotify/ParseSpotify.py | 7 +-
src/mappers/FuzzyMapper.py | 54 +++++-
13 files changed, 630 insertions(+), 190 deletions(-)
create mode 100644 Gui.py
create mode 100644 error.log
create mode 100755 gui.sh
create mode 100644 src/backends/navidrome/navidrome.py
diff --git a/Flow.py b/Flow.py
index e1ac33d..5e813e0 100644
--- a/Flow.py
+++ b/Flow.py
@@ -96,6 +96,10 @@ class Flow:
self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
+ def save_libraries(self):
+ LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json")
+ LibrarySaver(self.local_library).save(work_dir / "local_library.json")
+
def map_local_to_spotify(self):
resolve_remote_tracks(client_id, client_secret, self.local_library, self.spotify_library, work_dir)
diff --git a/Gui.py b/Gui.py
new file mode 100644
index 0000000..85fd6e3
--- /dev/null
+++ b/Gui.py
@@ -0,0 +1,206 @@
+import time
+
+import streamlit as st
+from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
+import pandas as pd
+
+import Flow
+
+from src.Library import Library, Artist, Album, Track, Playlist
+
+def track_to_row(track: Track):
+ return {
+ "Title": track.title,
+ "Album": track.album.title if track.album else "",
+ "Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists),
+ "Lyrics": track.has_lyrics,
+ "AutoScore": track.auto_score,
+ "Added": track.added_at.strftime("%Y-%m-%d") if track.added_at else "",
+ "LocalId": str(track.local_id) if track.local_id else "",
+ "Path": str(track.local_path) if track.local_path else "",
+ }
+
+def artist_to_row(artist: Artist):
+ return {
+ "Title": artist.title,
+ "AutoScore": artist.auto_score,
+ "Resolved": int(artist.resolved_percentage * 100),
+ "LocalId": str(artist.local_id) if artist.local_id else "",
+ }
+
+def album_to_row(album: Album):
+ return {
+ "Title": album.title,
+ "AutoScore": album.auto_score,
+ "Resolved": int(album.resolved_percentage * 100),
+ "LocalId": str(album.local_id) if album.local_id else "",
+ }
+
+
+class LibraryView:
+ def __init__(self):
+ self.lib : Library = None
+ self.libs : list[Library] = None
+ self.view = None
+ self.lyrics_only = False
+ self.query = None
+
+ def add_libraries(self, libs : list[Library]):
+ self.libs = libs
+
+ def render(self):
+ st.set_page_config(layout="wide")
+
+ with st.sidebar:
+ lib = st.selectbox(
+ "Library",
+ options=[lib.name for lib in self.libs],
+ )
+
+ for iter in self.libs:
+ if iter.name == lib:
+ self.lib = iter
+
+ st.write(f"Showing library '{self.lib.name}'")
+
+ self.view = st.radio(
+ "View",
+ ["Tree"]
+ )
+
+ if self.view == "Tree":
+ self.render_artists_albums_tracks()
+ # elif self.view == "Tracks":
+ # self.render_tracks()
+ # elif self.view == "Playlists":
+ # self.render_playlists()
+
+ def track_visible(
+ self,
+ track: Track,
+ artist: Artist | None,
+ album: Album | None,
+ ):
+ if self.lyrics_only and not track.has_lyrics:
+ return False
+
+ if self.query and self.query.lower() not in track.title.lower():
+ return False
+
+ if artist and all(a.id != artist.id for a in track.artists):
+ return False
+
+ if album and (not track.album or track.album.id != album.id):
+ return False
+
+ return True
+
+ def render_artists_albums_tracks(self):
+ # --- ARTISTS TABLE ---
+ st.subheader("Artists")
+ artist_rows = [artist_to_row(a) for a in self.lib.artists]
+ df_artists = pd.DataFrame(artist_rows)
+
+ gb_artists = GridOptionsBuilder.from_dataframe(df_artists)
+ gb_artists.configure_selection(selection_mode="multiple", use_checkbox=True)
+ gb_artists.configure_column("Title", filter="agTextColumnFilter")
+ gb_artists.configure_column("AutoScore", sort="desc")
+ grid_options_artists = gb_artists.build()
+
+ grid_response_artists = AgGrid(
+ df_artists,
+ gridOptions=grid_options_artists,
+ update_mode=GridUpdateMode.SELECTION_CHANGED,
+ allow_unsafe_jscode=True,
+ enable_enterprise_modules=False,
+ fit_columns_on_grid_load=True,
+ )
+
+ selected_rows_df = grid_response_artists.get("selected_rows")
+ selected_artist_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
+ selected_artists = [a for a in self.lib.artists if a.title in selected_artist_titles]
+
+ # --- ALBUMS TABLE ---
+ st.subheader("Albums")
+ if selected_artists:
+ albums_list = []
+ for artist in selected_artists:
+ albums_list.extend(artist.albums)
+ albums_list = list({al.id: al for al in albums_list}.values())
+ else:
+ albums_list = self.lib.albums
+
+ album_rows = [album_to_row(a) for a in albums_list]
+ df_albums = pd.DataFrame(album_rows)
+
+ gb_albums = GridOptionsBuilder.from_dataframe(df_albums)
+ gb_albums.configure_selection(selection_mode="multiple", use_checkbox=True)
+ gb_albums.configure_column("Title", filter="agTextColumnFilter")
+ gb_albums.configure_column("AutoScore", sort="desc")
+ grid_options_albums = gb_albums.build()
+
+ grid_response_albums = AgGrid(
+ df_albums,
+ gridOptions=grid_options_albums,
+ update_mode=GridUpdateMode.SELECTION_CHANGED,
+ allow_unsafe_jscode=True,
+ enable_enterprise_modules=False,
+ fit_columns_on_grid_load=True,
+ )
+
+ selected_rows_df = grid_response_albums.get("selected_rows")
+ selected_album_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
+ selected_albums = [al for al in albums_list if al.title in selected_album_titles]
+
+ # --- TRACKS TABLE ---
+ st.subheader("Tracks")
+ if selected_albums:
+ tracks_list = []
+ for album in selected_albums:
+ tracks_list.extend(album.tracks)
+ elif selected_artists:
+ tracks_list = []
+ for artist in selected_artists:
+ tracks_list.extend([t for t in self.lib.tracks if artist in t.artists])
+ else:
+ tracks_list = self.lib.tracks
+
+ track_rows = [track_to_row(t) for t in tracks_list]
+ df_tracks = pd.DataFrame(track_rows)
+
+ gb_tracks = GridOptionsBuilder.from_dataframe(df_tracks)
+ gb_tracks.configure_selection(selection_mode="multiple", use_checkbox=True)
+ gb_tracks.configure_column("Title", filter="agTextColumnFilter")
+ gb_tracks.configure_column("AutoScore", sort="desc")
+ grid_options_tracks = gb_tracks.build()
+
+ AgGrid(
+ df_tracks,
+ gridOptions=grid_options_tracks,
+ update_mode=GridUpdateMode.SELECTION_CHANGED,
+ allow_unsafe_jscode=True,
+ enable_enterprise_modules=False,
+ fit_columns_on_grid_load=True,
+ )
+
+
+def run():
+ flow = Flow.Flow()
+
+ # flow.fetch_spotify()
+ # flow.fetch_local()
+
+ # flow.parse_spotify_library()
+ # flow.parse_local_library()
+
+ flow.load_libraries()
+
+ # flow.map_local_to_spotify()
+
+ # flow.log_stats()
+
+ gui = LibraryView()
+ gui.add_libraries([flow.spotify_library, flow.local_library])
+ gui.render()
+
+run()
\ No newline at end of file
diff --git a/error.log b/error.log
new file mode 100644
index 0000000..7c830a2
--- /dev/null
+++ b/error.log
@@ -0,0 +1,2 @@
+
+(process:38049): GLib-GIO-CRITICAL **: 14:06:16.561: g_dbus_connection_emit_signal: assertion 'G_IS_DBUS_CONNECTION (connection)' failed
diff --git a/gui.sh b/gui.sh
new file mode 100755
index 0000000..eb55739
--- /dev/null
+++ b/gui.sh
@@ -0,0 +1 @@
+streamlit run Gui.py
diff --git a/main.py b/main.py
index af9ed31..646539a 100644
--- a/main.py
+++ b/main.py
@@ -5,15 +5,14 @@ import Flow
def test1():
flow = Flow.Flow()
- flow.fetch_local()
+ # flow.fetch_local()
flow.parse_local_library()
flow.load_libraries()
# flow.map_local_to_spotify()
# flow.log_stats()
- flow.fetch_and_save_lyrics()
-
+ # flow.fetch_and_save_lyrics()
def main():
flow = Flow.Flow()
@@ -27,11 +26,12 @@ def main():
flow.load_libraries()
flow.map_local_to_spotify()
+ flow.save_libraries()
- flow.log_stats()
+ # flow.log_stats()
- print("asd")
+ # print("asd")
-test1()
-# main()
+# test1()
+main()
diff --git a/src/Library.py b/src/Library.py
index 0de72ce..46d7566 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -7,64 +7,86 @@ from typing import List, Optional, Any
class Artist:
def __init__(self):
- self.id : str = None
- self.title : str = None
- self.tracks : List[Track] = []
- self.albums : List[Album] = []
+ self.id: str = None
+ self.title: str = None
+ self.tracks: List[Track] = []
+ self.albums: List[Album] = []
+ self.local_id: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
+ self.resolved_percentage : float = 0
class Track:
def __init__(self):
- self.id : str = None
- self.title : str = None
- self.artists : List[Artist] = []
- self.album : Album = None
- self.duration_ms : int = None
- self.added_at : datetime = None
- self.local_path : Path = None
- self.has_lyrics : bool = False
+ self.id: str = None
+ self.title: str = None
+ self.artists: List[Artist] = []
+ self.album: Album = None
+ self.duration_ms: int = None
+ self.added_at: datetime = None
+ self.local_path: Path = None
+ self.has_lyrics: bool = False
+ self.local_id: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
class Album:
def __init__(self):
- self.id : str = None
- self.title : str = None
- self.release_date : datetime = None
- self.tracks : List[Track] = []
- self.artists : List[Artist] = []
+ self.id: str = None
+ self.title: str = None
+ self.release_date: datetime = None
+ self.tracks: List[Track] = []
+ self.artists: List[Artist] = []
+ self.local_id: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
+ self.resolved_percentage : float = 0
class Playlist:
def __init__(self):
- self.title : str = None
- self.tracks : List[Track] = []
- self.created_at : datetime = None
- self.description : str = None
+ self.title: str = None
+ self.tracks: List[Track] = []
+ self.created_at: datetime = None
+ self.description: str = None
+ self.auto_score: float = 0
+ self.fav: bool = False
+ self.rating: int = 0
+ self.resolved_percentage : float = 0
class Library:
def __init__(self):
- self.artists : List[Artist] = []
- self.playlists : List[Playlist] = []
- self.tracks : List[Track] = []
- self.albums : List[Album] = []
- self.liked_tracks : List[Track] = []
- self.top_tracks : List[Track]= []
- self.top_artists : List[Artist] = []
+ self.artists: List[Artist] = []
+ self.playlists: List[Playlist] = []
+ self.tracks: List[Track] = []
+ self.albums: List[Album] = []
- self.artist_map : dict[str, Artist] = None
- self.album_map : dict[str, Album] = None
- self.track_map : dict[str, Track] = None
+ self.liked_tracks: List[Track] = []
+ self.top_tracks: List[Track] = []
+ self.top_artists: List[Artist] = []
+ self.artist_map: dict[str, Artist] = None
+ self.album_map: dict[str, Album] = None
+ self.track_map: dict[str, Track] = None
+
+ self.name = "Unnamed"
def update_cache(self):
self.update_id_maps()
self.create_reverse_links()
+ self.score_albums()
+ self.calc_resolved_percentage()
def update_id_maps(self):
- self.artist_map : dict[str, Artist] = { artist.id : artist for artist in self.artists }
- self.album_map : dict[str, Album] = { album.id : album for album in self.albums }
- self.track_map : dict[str, Track] = { track.id : track for track in self.tracks }
+ self.artist_map: dict[str, Artist] = {artist.id: artist for artist in self.artists}
+ self.album_map: dict[str, Album] = {album.id: album for album in self.albums}
+ self.track_map: dict[str, Track] = {track.id: track for track in self.tracks}
def create_reverse_links(self):
for track_id, track in self.track_map.items():
@@ -79,3 +101,36 @@ class Library:
for artist in album.artists:
if album not in artist.albums:
artist.albums.append(album)
+
+ def auto_score(self):
+ self.score_tracks()
+ self.score_artists()
+ self.score_albums()
+
+ def score_tracks(self):
+ for idx, track in enumerate(self.top_tracks):
+ score = len(self.top_tracks) - idx
+ track.auto_score = score
+
+ def score_artists(self):
+ for track in self.tracks:
+ for artist in track.artists:
+ artist.auto_score += track.auto_score
+
+ def score_albums(self):
+ for track in self.tracks:
+ track.album.auto_score += track.auto_score
+
+ def calc_resolved_percentage(self):
+ for artist in self.artists:
+ artist_count_resolved = 0
+ artist_count = 0
+ for album in artist.albums:
+ album_count_resolved = 0
+ for track in album.tracks:
+ if track.local_id:
+ album_count_resolved += 1
+ album.resolved_percentage = album_count_resolved / len(album.tracks)
+ artist_count_resolved += album_count_resolved
+ artist_count += len(album.tracks)
+ artist.resolved_percentage = artist_count_resolved / artist_count
\ No newline at end of file
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index 84f6a46..f331824 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -33,6 +33,10 @@ class LibrarySaver:
"title": artist.title,
"tracks": [track.id for track in artist.tracks],
"albums": [album.id for album in artist.albums],
+ "local_id": artist.local_id,
+ "auto_score": artist.auto_score,
+ "fav" : artist.fav,
+ "rating" : artist.rating,
}
@staticmethod
@@ -46,6 +50,10 @@ class LibrarySaver:
"added_at": _dt_to_str(track.added_at),
"local_path": track.local_path,
"has_lyrics": track.has_lyrics,
+ "local_id": track.local_id,
+ "auto_score": track.auto_score,
+ "fav" : track.fav,
+ "rating" : track.rating,
}
@staticmethod
@@ -55,6 +63,9 @@ class LibrarySaver:
"tracks": [track.id for track in playlist.tracks],
"created_at": _dt_to_str(playlist.created_at),
"description": playlist.description,
+ "auto_score": playlist.auto_score,
+ "fav" : playlist.fav,
+ "rating" : playlist.rating,
}
@staticmethod
@@ -64,12 +75,17 @@ class LibrarySaver:
"title": album.title,
"release_date": album.release_date,
"tracks": [track.id for track in album.tracks],
- "artists": [artist.id for artist in album.artists]
+ "artists": [artist.id for artist in album.artists],
+ "local_id": album.local_id,
+ "auto_score": album.auto_score,
+ "fav" : album.fav,
+ "rating" : album.rating,
}
def save(self, path: Path) -> None:
data = {
+ "name": self.library.name,
"tracks": [self._track_to_dict(track) for track in self.library.tracks],
"albums": [self._album_to_dict(album) for album in self.library.albums],
"artists": [self._artist_to_dict(artist) for artist in self.library.artists],
@@ -121,6 +137,10 @@ class LibraryLoader:
track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
track.album = self.get_album(data.get("album"))
track.has_lyrics = data.get("has_lyrics")
+ track.local_id = data.get("local_id")
+ track.auto_score = data.get("auto_score")
+ track.fav = data.get("fav")
+ track.rating = data.get("rating")
self.track_map[track.id] = track
return track
@@ -131,6 +151,10 @@ class LibraryLoader:
artist.title = data.get("title")
artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else []
+ artist.local_id = data.get("local_id")
+ artist.auto_score = data.get("auto_score")
+ artist.fav = data.get("fav")
+ artist.rating = data.get("rating")
self.artist_map[artist.id] = artist
return artist
@@ -141,6 +165,10 @@ class LibraryLoader:
album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
album.title = data.get("title")
album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
+ album.local_id = data.get("local_id")
+ album.auto_score = data.get("auto_score")
+ album.fav = data.get("fav")
+ album.rating = data.get("rating")
self.album_map[album.id] = album
return album
@@ -151,6 +179,7 @@ class LibraryLoader:
library = Library()
+ library.name = data.get("name")
library.tracks = [self.load_track(track) for track in data.get("tracks")]
library.albums = [self.load_album(album) for album in data.get("albums")]
library.artists = [self.load_artist(artist) for artist in data.get("artists")]
diff --git a/src/StatGenerator.py b/src/StatGenerator.py
index ac16622..7c17ab5 100644
--- a/src/StatGenerator.py
+++ b/src/StatGenerator.py
@@ -15,13 +15,12 @@ def get_mapping(mapping_path) -> dict[str, Any]:
with Path(mapping_path).open("r", encoding="utf-8") as f:
data = json.load(f)
- return data
+ return data["track_map"]
class AlbumStat:
def __init__(self):
self.album: Album = Album()
- self.score: int = 0
self.resolved_precent: float = 0
def calc_resolved_precent(self):
@@ -37,7 +36,6 @@ class AlbumStat:
class ArtistStat:
def __init__(self):
self.artist: Artist = Artist()
- self.score: int = 0
self.resolved_precent: float = 0
self.albums_stats: list[AlbumStat] = []
@@ -87,18 +85,22 @@ class LibraryStats:
for remote, local in self.mapping.items():
self.remote_id_map[remote].local_path = self.local_id_map[local].local_path
+ def score_tracks(self):
+ for idx, track in enumerate(self.remote_lib.top_tracks):
+ score = len(self.remote_lib.top_tracks) - idx
+ track.auto_score = score
+
def score_artists(self):
for artist in self.remote_lib.artists:
artist_stat = ArtistStat()
artist_stat.artist = artist
- artist_stat.score = 0
+ artist_stat.artist.auto_score = 0
self.artists_map[artist_stat.artist.id] = artist_stat
self.artists.append(artist_stat)
- for idx, track in enumerate(self.remote_lib.top_tracks):
- score = len(self.remote_lib.top_tracks) - idx
+ for track in self.remote_lib.tracks:
for artist in track.artists:
- self.artists_map[artist.id].score += score
+ self.artists_map[artist.id].artist.auto_score += track.auto_score
for artist_stat in self.artists:
artist_stat.calc_resolved_precent()
@@ -111,9 +113,8 @@ class LibraryStats:
stat.album = album
self.albums_map[album.id] = stat
- for idx, track in enumerate(self.remote_lib.top_tracks):
- score = len(self.remote_lib.top_tracks) - idx
- self.albums_map[track.album.id].score += score
+ for track in self.remote_lib.tracks:
+ self.albums_map[track.album.id].album.auto_score += track.auto_score
for _, album in self.albums_map.items():
album.calc_resolved_precent()
diff --git a/src/backends/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py
index 1a8eaab..6f132d4 100644
--- a/src/backends/local/LocalLibraryIndexer.py
+++ b/src/backends/local/LocalLibraryIndexer.py
@@ -1,64 +1,97 @@
+import os
import fnmatch
-import mutagen
-import eyed3
-from mutagen.flac import FLAC
from mutagen import File as MutagenFile
+from mutagen.id3 import ID3, ID3NoHeaderError
+from mutagen.flac import FLAC
+from mutagen.mp4 import MP4
+from mutagen.oggvorbis import OggVorbis
+from tqdm import tqdm
from src.helpers import *
-
music_extensions = ('*.mp3', '*.flac', '*.wav', '*.aac', '*.ogg', '*.m4a')
exclude_directories = ("*mary--*", "*example-word*")
-from mutagen.id3 import ID3
-from mutagen.id3 import ID3NoHeaderError
-
-
def detect_lyrics(abs_path, ext):
ext = ext.lower()
- # ---------- MP3 ----------
- if ext == ".mp3":
- try:
- tags = ID3(abs_path)
- except ID3NoHeaderError:
- return False
-
- return bool(
- tags.getall("SYLT") or
- tags.getall("USLT")
- )
-
- # ---------- FLAC / OGG / OPUS ----------
- if ext in {".flac", ".ogg", ".opus"}:
- try:
- audio = MutagenFile(abs_path)
- if not audio or not audio.tags:
- return False
-
- for key in audio.tags.keys():
- k = key.upper()
- if k in {"LYRICS", "LRC", "SYNCEDLYRICS"}:
- if audio.tags[key]:
- return True
- except Exception:
- return False
-
- # ---------- Fallback ----------
try:
audio = MutagenFile(abs_path)
- if audio and audio.tags:
- for key in audio.tags.keys():
- if "LYRIC" in key.upper():
- return True
+ if not audio or not audio.tags:
+ return False
+
+ # MP3-specific SYLT/USLT detection
+ if ext == ".mp3" and isinstance(audio, ID3):
+ return bool(audio.getall("SYLT") or audio.getall("USLT"))
+
+ # FLAC/OGG/OPUS lyrics detection
+ for key, value in audio.tags.items():
+ k = key.upper()
+ if k in {"LYRICS", "LRC", "SYNCEDLYRICS"} and value:
+ return True
+
+ # Generic fallback
+ for key in audio.tags.keys():
+ if "LYRIC" in key.upper():
+ return True
+
except Exception:
- pass
+ return False
return False
-def update_local_library(root_dir, output_dir):
+def get_artist(tags, fallback=None) -> list :
+ # FLAC / OGG / general multiple artist fields
+ artist_fields = ["ARTISTS", "TXXX:ARTISTS", "©ART", "artist"]
+ for field in artist_fields:
+ if field in tags and tags[field]:
+ values = tags.get(field)
+ if not isinstance(values, list):
+ return values.text
+ return values
+
+ # M4A
+ if '\xa9ART' in tags and tags['\xa9ART']:
+ values = tags['\xa9ART']
+ return values
+
+ if tags.get("TPE1"):
+ text = getattr(tags.get("TPE1"), "text", None)
+ if text:
+ return text
+
+ return fallback
+
+def safe_tag_get(tags, key, index=0):
+ try:
+ value = tags.get(key, None)
+ if isinstance(value, list):
+ # filter out None/empty, convert to str
+ value = [str(v) for v in value if v]
+ return value[index] if len(value) > index else fallback
+ return str(value)
+ except (KeyError, ValueError, TypeError):
+ return None
+
+def get_title(tags):
+ ids = ["TIT2", "TITLE", "title", "\xa9nam"]
+ for key in ids:
+ val = safe_tag_get(tags, key)
+ if val and val != "None":
+ return val
+ return None
+
+def get_album(tags):
+ ids = ["TALB", "album", "\xa9alb"]
+ for key in ids:
+ val = safe_tag_get(tags, key)
+ if val and val != "None":
+ return val
+ return None
+
+def update_local_library(root_dir, output_dir):
set_workdir(output_dir / "local")
old_library = get_data("local_library")
@@ -91,61 +124,48 @@ def update_local_library(root_dir, output_dir):
song_id = get_new_song_id(song_id, relative_path)
new_library[song_id] = {
- "name": song_name,
+ "name": None,
"path": relative_path,
"type": track_type,
- "artist": "",
- "album": "",
+ "artists": None,
+ "album": None,
"duration": None
}
- for track_id, track in new_library.items():
- abs_path = os.path.join(root_dir, track['path'])
+ to_remove = []
- track_abs_path = os.path.join(root_dir, track["path"])
- track["has_lyrics"] = detect_lyrics(track_abs_path, track['type'].lower())
+ with tqdm(total=len(new_library.items()), desc='Indexing local library') as process_bar:
+ for track_id, track in new_library.items():
+ process_bar.update(1)
+
+ abs_path = os.path.join(root_dir, track['path'])
+ track["has_lyrics"] = detect_lyrics(abs_path, track['type'].lower())
+
+ try:
+ audio = MutagenFile(abs_path)
+ if audio and audio.tags:
+ tags = audio.tags
+
+ track['artists'] = get_artist(tags)
+ track['name'] = get_title(tags)
+ track['album'] = get_album(tags)
+
+ if not len(track['artists']) or not track['name'] or not track['album']:
+ print(track)
+ raise "Cannot index track"
- match track['type'].lower():
- case ".mp3":
- mp3track = eyed3.load(abs_path)
- if not mp3track or not mp3track.info:
- print(f"Failed to load mp3 {abs_path}")
- track['name'] = track['path']
else:
- track['duration'] = mp3track.info.time_secs
+ print(track)
+ to_remove.append(track_id)
+ # raise "Cannot index track"
- tag = mp3track.tag
- if tag:
- track['artist'] = tag.artist
- track['name'] = tag.title
- track['album'] = tag.album
+ except Exception as e:
+ print(e)
+ print(abs_path)
+ raise e
- case ".flac":
- try:
- audio = FLAC(abs_path)
- if audio.info:
- track['duration'] = audio.info.length
-
- metadata = audio.tags
- if metadata:
- if 'artist' in metadata:
- track['artist'] = " ".join(metadata['artist'])
- if 'TITLE' in metadata:
- track['name'] = " ".join(metadata['TITLE'])
- if 'album' in metadata:
- track['album'] = " ".join(metadata['album'])
-
- except mutagen.MutagenError:
- print(f"Invalid flac header {abs_path}")
- track['name'] = track['path']
-
- case _:
- try:
- audio = MutagenFile(abs_path)
- if audio and audio.info:
- track['duration'] = audio.info.length
- except Exception:
- pass
+ for rem in to_remove:
+ new_library.pop(rem)
save_data(new_library, "local_library")
diff --git a/src/backends/local/ParseLocal.py b/src/backends/local/ParseLocal.py
index 49b3f55..b1fbba3 100644
--- a/src/backends/local/ParseLocal.py
+++ b/src/backends/local/ParseLocal.py
@@ -10,46 +10,6 @@ PROTECTED_ARTISTS = {
"The Good, The Bad & The Queen",
}
-def split_artists(artist_tag: str) -> list[str]:
- if not artist_tag:
- return []
-
- placeholder_comma = "§COMMA§"
- placeholder_amp = "§AMP§"
-
- protected_map = {}
-
- # Protect known artist names
- for artist in PROTECTED_ARTISTS:
- protected = (
- artist
- .replace(",", placeholder_comma)
- .replace("&", placeholder_amp)
- )
- protected_map[protected] = artist
- artist_tag = artist_tag.replace(artist, protected)
-
- # Split remaining separators
- parts = re.split(r"[,&]", artist_tag)
-
- # Restore protected names and trim spaces
- result = []
- for part in parts:
- part = part.strip()
- if not part:
- continue
-
- part = (
- part
- .replace(placeholder_comma, ",")
- .replace(placeholder_amp, "&")
- )
-
- result.append(part)
-
- return result
-
-
class LocalParser:
def __init__(self):
self.track_map: dict[str, Track] = {}
@@ -60,9 +20,7 @@ class LocalParser:
def parse_artists(self, data: str) -> List[Artist]:
if not data:
- data = "unknown"
-
- data = split_artists(data)
+ raise "No artist found"
artists = []
@@ -87,7 +45,7 @@ class LocalParser:
def parse_album(self, data: str) -> Album:
if not data:
- data = "unknown"
+ raise "album is none"
name = data
album_id = name
@@ -113,12 +71,15 @@ class LocalParser:
track.id = track_id
track.title = data.get("name")
- track.artists = self.parse_artists(data.get("artist"))
+ track.artists = self.parse_artists(data.get("artists"))
track.album = self.parse_album(data.get("album"))
track.local_path = data.get("path")
track.duration_ms = data.get("duration")
track.has_lyrics = data.get("has_lyrics")
+ if not track.album or not len(track.artists) or not track.title:
+ raise "error parsing"
+
self.track_map[track.id] = track
self.library.tracks.append(track)
@@ -129,6 +90,8 @@ class LocalParser:
self.library.update_cache()
+ self.library.name = "Local"
+
return self.library
diff --git a/src/backends/navidrome/navidrome.py b/src/backends/navidrome/navidrome.py
new file mode 100644
index 0000000..c7d04f4
--- /dev/null
+++ b/src/backends/navidrome/navidrome.py
@@ -0,0 +1,110 @@
+import sqlite3
+import os
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+@dataclass
+class Track:
+ id: int
+ title: str
+ path: str
+ favorite: bool
+ rating: Optional[int]
+ play_count: int
+ last_played: Optional[str]
+ playlists: List[str] = field(default_factory=list)
+
+@dataclass
+class Album:
+ id: int
+ title: str
+ tracks: List[Track] = field(default_factory=list)
+
+@dataclass
+class Artist:
+ id: str
+ name: str
+ albums: List[Album] = field(default_factory=list)
+
+class NavidromeDB:
+ def __init__(self, db_path: str):
+ if not os.path.isfile(db_path):
+ raise FileNotFoundError(f"Navidrome DB not found: {db_path}")
+ self.conn = sqlite3.connect(db_path)
+ self.conn.row_factory = sqlite3.Row
+
+ def close(self):
+ self.conn.close()
+
+ def get_artists(self) -> List[Artist]:
+ artists = []
+ c = self.conn.cursor()
+ c.execute("SELECT id, name FROM artist ORDER BY name")
+ for row in c.fetchall():
+ artist = Artist(id=row["id"], name=row["name"])
+ artist.albums = self.get_albums(artist.id)
+ artists.append(artist)
+ return artists
+
+ def get_albums(self, artist_id: str) -> List[Album]:
+ albums = []
+ c = self.conn.cursor()
+ c.execute(
+ "SELECT id, name FROM album WHERE album_artist_id=? ORDER BY name",
+ (artist_id,)
+ )
+ for row in c.fetchall():
+ album = Album(id=row["id"], title=row["name"])
+ album.tracks = self.get_tracks(album.id)
+ albums.append(album)
+ return albums
+
+ def get_tracks(self, album_id: int) -> List[Track]:
+ tracks = []
+ c = self.conn.cursor()
+ c.execute(
+ "SELECT t.id, t.name AS Title, t.path, ut.favorite, ut.rating, ut.play_count AS PlayCount, ut.last_played AS LastPlayed "
+ "FROM track t "
+ "LEFT JOIN usertrack ut ON ut.track_id = t.id "
+ "WHERE t.album_id=? ORDER BY t.track_number",
+ (album_id,)
+ )
+ for row in c.fetchall():
+ track = Track(
+ id=row["id"],
+ title=row["Title"],
+ path=row["path"],
+ favorite=bool(row["favorite"]),
+ rating=row["rating"],
+ play_count=row["PlayCount"] or 0,
+ last_played=row["LastPlayed"],
+ playlists=self.get_playlists_for_track(row["id"])
+ )
+ tracks.append(track)
+ return tracks
+
+ def get_playlists_for_track(self, track_id: int) -> List[str]:
+ c = self.conn.cursor()
+ c.execute(
+ "SELECT p.name FROM playlist p "
+ "JOIN playlisttrack pt ON pt.playlist_id=p.id "
+ "WHERE pt.track_id=?",
+ (track_id,)
+ )
+ return [row["name"] for row in c.fetchall()]
+
+# Example usage
+if __name__ == "__main__":
+ db_path = "/mnt/main/data/music/navidrome.db"
+ navidb = NavidromeDB(db_path)
+
+ try:
+ artists = navidb.get_artists()
+ for artist in artists:
+ print(f"Artist: {artist.name}")
+ for album in artist.albums:
+ print(f" Album: {album.title}")
+ for track in album.tracks:
+ print(f" Track: {track.title} | Path: {track.path} | Fav: {track.favorite} | Plays: {track.play_count}")
+ finally:
+ navidb.close()
diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
index 50806bf..29c24e7 100644
--- a/src/backends/spotify/ParseSpotify.py
+++ b/src/backends/spotify/ParseSpotify.py
@@ -30,6 +30,8 @@ class Parser:
self.parse_top_tracks(json.loads(top_tracks.read_text()))
self.library.update_cache()
+ self.library.name = "Spotify"
+ self.library.auto_score()
return self.library
@@ -75,7 +77,7 @@ class Parser:
track.id = track_id
track.title = data.get("name")
- track.artists = [self.parse_artist(artist) for artist in data.get("artists")]
+ track.artists = [] + self.parse_album(data.get("album")).artists
track.album = self.parse_album(data.get("album"))
track.duration_ms = data.get("duration_ms")
@@ -114,6 +116,9 @@ class Parser:
artist.id = data.get("id")
artist.title = data.get("name")
+ if artist.title is None:
+ artist.title = "err_no_title"
+
self.library.artists.append(artist)
return artist
diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py
index ca78db9..c6413d0 100644
--- a/src/mappers/FuzzyMapper.py
+++ b/src/mappers/FuzzyMapper.py
@@ -5,12 +5,46 @@ from tqdm import tqdm
from src.Library import *
from src.helpers import *
+import unicodedata
+
+user_def_artist_map = {
+ "donda" : "kanye west",
+}
+
+def normalize(s):
+ return unicodedata.normalize("NFKC", s).replace("‐", "-")
def get_score(first: str, second: str) -> float:
+ first_norm = normalize(first).lower()
+ second_norm = normalize(second).lower()
+ return float(first_norm == second_norm) * 100
return fuzz.token_set_ratio(first, second)
+def get_album_score(first: str, second: str) -> float:
+ return get_score(first, second)
+
+def get_track_score(first: str, second: str) -> float:
+ return get_score(first, second)
+
+def get_artist_score(first: str, second: str) -> float:
+ if not first or not second:
+ return 0.0
+
+ first_lower = first.lower()
+ second_lower = second.lower()
+
+ for key, value in user_def_artist_map.items():
+ key_lower = key.lower()
+ value_lower = value.lower()
+ if (first_lower == key_lower and second_lower == value_lower) or \
+ (first_lower == value_lower and second_lower == key_lower):
+ return 100.0
+
+ return get_score(first, second)
+
+
def is_close_enough(first: float) -> bool:
- return first > 50
+ return first > 90
def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
@@ -21,12 +55,13 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
with (tqdm(total=len(remote_lib.artists), desc='searching remote artists in local library') as process_bar):
for remote_artist in remote_lib.artists:
for local_artist in local_lib.artists:
- score = get_score(remote_artist.title, local_artist.title)
+ score = get_artist_score(remote_artist.title, local_artist.title)
if not is_close_enough(score):
continue
if remote_artist.id not in artist_map or score > artist_map[remote_artist.id][1]:
artist_map[remote_artist.id] = (local_artist.id, score)
+ remote_artist.local_id = local_artist.id
process_bar.update(1)
@@ -38,12 +73,13 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
for remote_album in remote_artist.albums:
for local_album in local_artist.albums:
- score = get_score(remote_album.title, local_album.title)
+ score = get_album_score(remote_album.title, local_album.title)
if not is_close_enough(score):
continue
if remote_album.id not in artist_map or score > album_map[remote_album.id][1]:
album_map[remote_album.id] = (local_album.id, score)
+ remote_album.local_id = local_album.id
process_bar.update(1)
@@ -54,15 +90,23 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
for remote_track in remote_album.tracks:
for local_track in local_album.tracks:
- score = get_score(remote_track.title, local_track.title)
+ score = get_track_score(remote_track.title, local_track.title)
if not is_close_enough(score):
continue
if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
track_map[remote_track.id] = (local_track.id, score)
+ remote_track.local_id = local_track.id
process_bar.update(1)
- out = { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() }
+ remote_lib.calc_resolved_percentage()
+
+ out = {
+ "artist_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in artist_map.items() },
+ "album_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in album_map.items() },
+ "track_map" : { remote_track_id : local_track_id for remote_track_id, (local_track_id, _) in track_map.items() },
+ }
+
set_workdir(output_dir)
save_data(out, "remote_to_local_map")
From 426ad381de39667ed3e168fdbe2193adbb87de00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 7 Feb 2026 11:24:59 +0300
Subject: [PATCH 25/30] tmp
---
Flow.py | 13 ++
main.py | 7 +-
src/Library.py | 17 ++
src/backends/itunes/ParseItunesToJson.py | 101 ------------
src/backends/itunes/ParseItunesXml.py | 192 +++++++++++++++++++++++
5 files changed, 226 insertions(+), 104 deletions(-)
delete mode 100755 src/backends/itunes/ParseItunesToJson.py
create mode 100755 src/backends/itunes/ParseItunesXml.py
diff --git a/Flow.py b/Flow.py
index 5e813e0..0031017 100644
--- a/Flow.py
+++ b/Flow.py
@@ -13,6 +13,8 @@ from src.backends.spotify.ParseSpotify import Parser
from src.backends.local.LocalLibraryIndexer import update_local_library
from src.backends.local.ParseLocal import LocalParser
+from src.backends.itunes.ParseItunesXml import ParseItunesXml
+
from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
@@ -43,6 +45,17 @@ class Flow:
LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
+ @staticmethod
+ def parse_itunes_library():
+ parser = ParseItunesXml()
+
+ library_parsed = parser.parse(
+ work_dir / "itunes" / "Library.xml",
+ work_dir / "itunes" / "Library.json"
+ )
+
+ LibrarySaver(library_parsed).save(work_dir / "itunes_library.json")
+
@staticmethod
def parse_local_library():
parser = LocalParser()
diff --git a/main.py b/main.py
index 646539a..26588d1 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,6 @@
-
-
import Flow
+
def test1():
flow = Flow.Flow()
@@ -14,13 +13,15 @@ def test1():
# flow.fetch_and_save_lyrics()
+
def main():
flow = Flow.Flow()
# flow.fetch_spotify()
- flow.fetch_local()
+ # flow.fetch_local()
flow.parse_spotify_library()
+ flow.parse_itunes_library()
flow.parse_local_library()
flow.load_libraries()
diff --git a/src/Library.py b/src/Library.py
index 46d7566..7bcefac 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -4,6 +4,23 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+class Entity:
+ def __init__(self):
+ self.id: str = None
+ self.mbid : str = None
+
+ self.title: str = None
+
+ self.fav: bool = False
+ self.rating: int = 0
+ self.play_count : int = 0
+ self.date_added : datetime = None
+ self.date_released : datetime = None
+ self.date_last_play : datetime = None
+ self.auto_score: float = 0
+
+ self.local_id: str = None
+ self.resolved_percentage : float = 0
class Artist:
def __init__(self):
diff --git a/src/backends/itunes/ParseItunesToJson.py b/src/backends/itunes/ParseItunesToJson.py
deleted file mode 100755
index 1505a36..0000000
--- a/src/backends/itunes/ParseItunesToJson.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import xmltodict
-import json
-
-
-def flatten_dict(d):
-
- out = []
- for song in d:
- newSong = {}
-
- strCount = 0
- intCount = 0
- dateCount = 0
-
- def getStr(id):
- nonlocal strCount
- if id not in song["key"]:
- return "undef"
- if len(song["string"]) <= strCount:
- return "error"
- strCount += 1
- return song["string"][strCount - 1]
-
- def getInt(id):
- nonlocal intCount
- if id not in song["key"]:
- return -1
- if len(song["integer"]) <= intCount:
- return -1
- intCount += 1
- return song["integer"][intCount - 1]
-
- def getDate(id):
- nonlocal dateCount
- if id not in song["key"]:
- return "undef"
- if len(song["date"]) <= dateCount:
- return "error"
- dateCount += 1
- return song["date"][dateCount - 1]
-
- newSong["Track ID"] = getInt("Track ID")
- newSong["Name"] = getStr("Name")
- newSong["Artist"] = getStr("Artist")
- newSong["Album Artist"] = getStr("Album Artist")
- newSong["Composer"] = getStr("Composer")
- newSong["Album"] = getStr("Album")
- newSong["Genre"] = getStr("Genre")
- newSong["Kind"] = getStr("Kind")
- newSong["Size"] = getInt("Size")
- newSong["Total Time"] = getInt("Total Time")
- newSong["Disc Number"] = getInt("Disc Number")
- newSong["Disc Count"] = getInt("Disc Count")
- newSong["Track Number"] = getInt("Track Number")
- newSong["Track Count"] = getInt("Track Count")
- newSong["Year"] = getInt("Year")
- newSong["Date Modified"] = getDate("Date Modified")
- newSong["Date Added"] = getDate("Date Added")
- newSong["Bit Rate"] = getInt("Bit Rate")
- newSong["Sample Rate"] = getInt("Sample Rate")
- newSong["Play Count"] = getInt("Play Count")
- newSong["Play Date"] = getInt("Play Date")
- newSong["Play Date UTC"] = getDate("Play Date UTC")
- newSong["Skip Count"] = getInt("Skip Count")
- newSong["Skip Date"] = getDate("Skip Date")
- newSong["Release Date"] = getDate("Release Date")
- newSong["Album Rating"] = getInt("Album Rating")
- newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"]
- newSong["Loved"] = "Loved" in song["key"]
- newSong["Album Loved"] = "Album Loved" in song["key"]
- newSong["Explicit"] = "Explicit" in song["key"]
- newSong["Compilation"] = "Compilation" in song["key"]
- newSong["Artwork Count"] = getInt("Artwork Count")
- newSong["Sort Album"] = getStr("Sort Album")
- newSong["Sort Artist"] = getStr("Sort Artist")
- newSong["Sort Name"] = getStr("Sort Name")
- newSong["Persistent ID"] = getStr("Persistent ID")
- newSong["Track Type"] = getStr("Track Type")
-
- out.append(newSong)
-
- return out
-
-
-def convert(filename, out_path):
- with open(filename, 'r', encoding='utf-8') as xml_file:
- data_dict = xmltodict.parse(xml_file.read())
-
- # Extract the "Tracks" dictionary to be flattened
- tracks_dict = data_dict['plist']['dict']['dict']
-
- flat_tracks_dict = flatten_dict(tracks_dict["dict"])
-
- json_data = json.dumps(flat_tracks_dict, indent=2)
-
- with open(out_path, 'w', encoding='utf-8') as json_file:
- json_file.write(json_data)
-
-
-if __name__ == "__main__":
- convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json')
diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py
new file mode 100755
index 0000000..0af465f
--- /dev/null
+++ b/src/backends/itunes/ParseItunesXml.py
@@ -0,0 +1,192 @@
+import xmltodict
+import json
+import xmltodict
+import json
+import re
+
+from src.Library import *
+
+class XmlToJson:
+ def flatten_dict(self, d):
+
+ out = []
+ for song in d:
+ newSong = {}
+
+ strCount = 0
+ intCount = 0
+ dateCount = 0
+
+ def getStr(id):
+ nonlocal strCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["string"]) <= strCount:
+ return "error"
+ strCount += 1
+ return song["string"][strCount - 1]
+
+ def getInt(id):
+ nonlocal intCount
+ if id not in song["key"]:
+ return -1
+ if len(song["integer"]) <= intCount:
+ return -1
+ intCount += 1
+ return song["integer"][intCount - 1]
+
+ def getDate(id):
+ nonlocal dateCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["date"]) <= dateCount:
+ return "error"
+ dateCount += 1
+ return song["date"][dateCount - 1]
+
+ newSong["Track ID"] = getInt("Track ID")
+ newSong["Name"] = getStr("Name")
+ newSong["Artist"] = getStr("Artist")
+ newSong["Album Artist"] = getStr("Album Artist")
+ newSong["Composer"] = getStr("Composer")
+ newSong["Album"] = getStr("Album")
+ newSong["Genre"] = getStr("Genre")
+ newSong["Kind"] = getStr("Kind")
+ newSong["Size"] = getInt("Size")
+ newSong["Total Time"] = getInt("Total Time")
+ newSong["Disc Number"] = getInt("Disc Number")
+ newSong["Disc Count"] = getInt("Disc Count")
+ newSong["Track Number"] = getInt("Track Number")
+ newSong["Track Count"] = getInt("Track Count")
+ newSong["Year"] = getInt("Year")
+ newSong["Date Modified"] = getDate("Date Modified")
+ newSong["Date Added"] = getDate("Date Added")
+ newSong["Bit Rate"] = getInt("Bit Rate")
+ newSong["Sample Rate"] = getInt("Sample Rate")
+ newSong["Play Count"] = getInt("Play Count")
+ newSong["Play Date"] = getInt("Play Date")
+ newSong["Play Date UTC"] = getDate("Play Date UTC")
+ newSong["Skip Count"] = getInt("Skip Count")
+ newSong["Skip Date"] = getDate("Skip Date")
+ newSong["Release Date"] = getDate("Release Date")
+ newSong["Album Rating"] = getInt("Album Rating")
+ newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"]
+ newSong["Loved"] = "Loved" in song["key"]
+ newSong["Album Loved"] = "Album Loved" in song["key"]
+ newSong["Explicit"] = "Explicit" in song["key"]
+ newSong["Compilation"] = "Compilation" in song["key"]
+ newSong["Artwork Count"] = getInt("Artwork Count")
+ newSong["Sort Album"] = getStr("Sort Album")
+ newSong["Sort Artist"] = getStr("Sort Artist")
+ newSong["Sort Name"] = getStr("Sort Name")
+ newSong["Persistent ID"] = getStr("Persistent ID")
+ newSong["Track Type"] = getStr("Track Type")
+
+ out.append(newSong)
+
+ return out
+
+
+ def convert(self, filename, out_path):
+ with open(filename, 'r', encoding='utf-8') as xml_file:
+ data_dict = xmltodict.parse(xml_file.read())
+
+ # Extract the "Tracks" dictionary to be flattened
+ tracks_dict = data_dict['plist']['dict']['dict']
+
+ flat_tracks_dict = self.flatten_dict(tracks_dict["dict"])
+
+ json_data = json.dumps(flat_tracks_dict, indent=2)
+
+ with open(out_path, 'w', encoding='utf-8') as json_file:
+ json_file.write(json_data)
+
+
+class ParseItunesXml:
+ def __init__(self):
+ self.track_map: dict[str, Track] = {}
+ self.artist_map: dict[str, Artist] = {}
+ self.album_map: dict[str, Album] = {}
+ self.library = Library()
+
+
+ def parse_artists(self, data: list) -> List[Artist]:
+ if not data:
+ raise "No artist found"
+
+ artists = []
+
+ for artist_data in data:
+ name = artist_data
+ artist_id = name
+
+ if artist_id in self.artist_map:
+ artists.append(self.artist_map[artist_id])
+ continue
+
+ artist = Artist()
+ artist.id = artist_id
+ artist.title = name
+
+ self.artist_map[artist_id] = artist
+ self.library.artists.append(artist)
+
+ artists.append(artist)
+
+ return artists
+
+ def parse_album(self, data: str) -> Album:
+ if not data:
+ raise "album is none"
+
+ name = data
+ album_id = name
+
+ if album_id in self.album_map:
+ return self.album_map[album_id]
+
+ album = Album()
+ album.id = album_id
+ album.title = name
+
+ self.album_map[album_id] = album
+ self.library.albums.append(album)
+
+ return album
+
+
+ def parse(self, tracks_file_xml: Path, tracks_file_json: Path) -> Library:
+ XmlToJson().convert(tracks_file_xml, tracks_file_json)
+
+ tracks_content = json.loads(tracks_file_json.read_text())
+
+ for data in tracks_content:
+ track = Track()
+
+ track.id = data.get("Track ID")
+ track.title = data.get("Name")
+ track.artists = self.parse_artists([data.get("Artist")])
+ track.album = self.parse_album(data.get("Album"))
+
+ if not track.album or not len(track.artists) or not track.title:
+ raise "error parsing"
+
+ self.track_map[track.id] = track
+ self.library.tracks.append(track)
+
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if artist not in track.album.artists:
+ track.album.artists.append(artist)
+
+ self.library.update_cache()
+
+ self.library.name = "Itunes"
+
+ return self.library
+
+
+ def parse_track_item(self, id, track: dict[str, Any]) -> Track:
+
+ return track
+
From b6c59e81adfdf30c9cbcb94908b70245d10a4b32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 7 Feb 2026 11:24:59 +0300
Subject: [PATCH 26/30] tmp
---
Flow.py | 13 ++
main.py | 7 +-
src/Library.py | 17 ++
src/backends/itunes/ParseItunesToJson.py | 101 ------------
src/backends/itunes/ParseItunesXml.py | 192 +++++++++++++++++++++++
5 files changed, 226 insertions(+), 104 deletions(-)
delete mode 100755 src/backends/itunes/ParseItunesToJson.py
create mode 100755 src/backends/itunes/ParseItunesXml.py
diff --git a/Flow.py b/Flow.py
index 5e813e0..0031017 100644
--- a/Flow.py
+++ b/Flow.py
@@ -13,6 +13,8 @@ from src.backends.spotify.ParseSpotify import Parser
from src.backends.local.LocalLibraryIndexer import update_local_library
from src.backends.local.ParseLocal import LocalParser
+from src.backends.itunes.ParseItunesXml import ParseItunesXml
+
from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
@@ -43,6 +45,17 @@ class Flow:
LibrarySaver(library_parsed).save(work_dir / "spotify_library.json")
+ @staticmethod
+ def parse_itunes_library():
+ parser = ParseItunesXml()
+
+ library_parsed = parser.parse(
+ work_dir / "itunes" / "Library.xml",
+ work_dir / "itunes" / "Library.json"
+ )
+
+ LibrarySaver(library_parsed).save(work_dir / "itunes_library.json")
+
@staticmethod
def parse_local_library():
parser = LocalParser()
diff --git a/main.py b/main.py
index 646539a..26588d1 100644
--- a/main.py
+++ b/main.py
@@ -1,7 +1,6 @@
-
-
import Flow
+
def test1():
flow = Flow.Flow()
@@ -14,13 +13,15 @@ def test1():
# flow.fetch_and_save_lyrics()
+
def main():
flow = Flow.Flow()
# flow.fetch_spotify()
- flow.fetch_local()
+ # flow.fetch_local()
flow.parse_spotify_library()
+ flow.parse_itunes_library()
flow.parse_local_library()
flow.load_libraries()
diff --git a/src/Library.py b/src/Library.py
index 46d7566..7bcefac 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -4,6 +4,23 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+class Entity:
+ def __init__(self):
+ self.id: str = None
+ self.mbid : str = None
+
+ self.title: str = None
+
+ self.fav: bool = False
+ self.rating: int = 0
+ self.play_count : int = 0
+ self.date_added : datetime = None
+ self.date_released : datetime = None
+ self.date_last_play : datetime = None
+ self.auto_score: float = 0
+
+ self.local_id: str = None
+ self.resolved_percentage : float = 0
class Artist:
def __init__(self):
diff --git a/src/backends/itunes/ParseItunesToJson.py b/src/backends/itunes/ParseItunesToJson.py
deleted file mode 100755
index 1505a36..0000000
--- a/src/backends/itunes/ParseItunesToJson.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import xmltodict
-import json
-
-
-def flatten_dict(d):
-
- out = []
- for song in d:
- newSong = {}
-
- strCount = 0
- intCount = 0
- dateCount = 0
-
- def getStr(id):
- nonlocal strCount
- if id not in song["key"]:
- return "undef"
- if len(song["string"]) <= strCount:
- return "error"
- strCount += 1
- return song["string"][strCount - 1]
-
- def getInt(id):
- nonlocal intCount
- if id not in song["key"]:
- return -1
- if len(song["integer"]) <= intCount:
- return -1
- intCount += 1
- return song["integer"][intCount - 1]
-
- def getDate(id):
- nonlocal dateCount
- if id not in song["key"]:
- return "undef"
- if len(song["date"]) <= dateCount:
- return "error"
- dateCount += 1
- return song["date"][dateCount - 1]
-
- newSong["Track ID"] = getInt("Track ID")
- newSong["Name"] = getStr("Name")
- newSong["Artist"] = getStr("Artist")
- newSong["Album Artist"] = getStr("Album Artist")
- newSong["Composer"] = getStr("Composer")
- newSong["Album"] = getStr("Album")
- newSong["Genre"] = getStr("Genre")
- newSong["Kind"] = getStr("Kind")
- newSong["Size"] = getInt("Size")
- newSong["Total Time"] = getInt("Total Time")
- newSong["Disc Number"] = getInt("Disc Number")
- newSong["Disc Count"] = getInt("Disc Count")
- newSong["Track Number"] = getInt("Track Number")
- newSong["Track Count"] = getInt("Track Count")
- newSong["Year"] = getInt("Year")
- newSong["Date Modified"] = getDate("Date Modified")
- newSong["Date Added"] = getDate("Date Added")
- newSong["Bit Rate"] = getInt("Bit Rate")
- newSong["Sample Rate"] = getInt("Sample Rate")
- newSong["Play Count"] = getInt("Play Count")
- newSong["Play Date"] = getInt("Play Date")
- newSong["Play Date UTC"] = getDate("Play Date UTC")
- newSong["Skip Count"] = getInt("Skip Count")
- newSong["Skip Date"] = getDate("Skip Date")
- newSong["Release Date"] = getDate("Release Date")
- newSong["Album Rating"] = getInt("Album Rating")
- newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"]
- newSong["Loved"] = "Loved" in song["key"]
- newSong["Album Loved"] = "Album Loved" in song["key"]
- newSong["Explicit"] = "Explicit" in song["key"]
- newSong["Compilation"] = "Compilation" in song["key"]
- newSong["Artwork Count"] = getInt("Artwork Count")
- newSong["Sort Album"] = getStr("Sort Album")
- newSong["Sort Artist"] = getStr("Sort Artist")
- newSong["Sort Name"] = getStr("Sort Name")
- newSong["Persistent ID"] = getStr("Persistent ID")
- newSong["Track Type"] = getStr("Track Type")
-
- out.append(newSong)
-
- return out
-
-
-def convert(filename, out_path):
- with open(filename, 'r', encoding='utf-8') as xml_file:
- data_dict = xmltodict.parse(xml_file.read())
-
- # Extract the "Tracks" dictionary to be flattened
- tracks_dict = data_dict['plist']['dict']['dict']
-
- flat_tracks_dict = flatten_dict(tracks_dict["dict"])
-
- json_data = json.dumps(flat_tracks_dict, indent=2)
-
- with open(out_path, 'w', encoding='utf-8') as json_file:
- json_file.write(json_data)
-
-
-if __name__ == "__main__":
- convert('./prefetched/ItunesLibrary.xml', './prefetched/ItunesLibrary.json')
diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py
new file mode 100755
index 0000000..0af465f
--- /dev/null
+++ b/src/backends/itunes/ParseItunesXml.py
@@ -0,0 +1,192 @@
+import xmltodict
+import json
+import xmltodict
+import json
+import re
+
+from src.Library import *
+
+class XmlToJson:
+ def flatten_dict(self, d):
+
+ out = []
+ for song in d:
+ newSong = {}
+
+ strCount = 0
+ intCount = 0
+ dateCount = 0
+
+ def getStr(id):
+ nonlocal strCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["string"]) <= strCount:
+ return "error"
+ strCount += 1
+ return song["string"][strCount - 1]
+
+ def getInt(id):
+ nonlocal intCount
+ if id not in song["key"]:
+ return -1
+ if len(song["integer"]) <= intCount:
+ return -1
+ intCount += 1
+ return song["integer"][intCount - 1]
+
+ def getDate(id):
+ nonlocal dateCount
+ if id not in song["key"]:
+ return "undef"
+ if len(song["date"]) <= dateCount:
+ return "error"
+ dateCount += 1
+ return song["date"][dateCount - 1]
+
+ newSong["Track ID"] = getInt("Track ID")
+ newSong["Name"] = getStr("Name")
+ newSong["Artist"] = getStr("Artist")
+ newSong["Album Artist"] = getStr("Album Artist")
+ newSong["Composer"] = getStr("Composer")
+ newSong["Album"] = getStr("Album")
+ newSong["Genre"] = getStr("Genre")
+ newSong["Kind"] = getStr("Kind")
+ newSong["Size"] = getInt("Size")
+ newSong["Total Time"] = getInt("Total Time")
+ newSong["Disc Number"] = getInt("Disc Number")
+ newSong["Disc Count"] = getInt("Disc Count")
+ newSong["Track Number"] = getInt("Track Number")
+ newSong["Track Count"] = getInt("Track Count")
+ newSong["Year"] = getInt("Year")
+ newSong["Date Modified"] = getDate("Date Modified")
+ newSong["Date Added"] = getDate("Date Added")
+ newSong["Bit Rate"] = getInt("Bit Rate")
+ newSong["Sample Rate"] = getInt("Sample Rate")
+ newSong["Play Count"] = getInt("Play Count")
+ newSong["Play Date"] = getInt("Play Date")
+ newSong["Play Date UTC"] = getDate("Play Date UTC")
+ newSong["Skip Count"] = getInt("Skip Count")
+ newSong["Skip Date"] = getDate("Skip Date")
+ newSong["Release Date"] = getDate("Release Date")
+ newSong["Album Rating"] = getInt("Album Rating")
+ newSong["Album Rating Computed"] = "Album Rating Computed" in song["key"]
+ newSong["Loved"] = "Loved" in song["key"]
+ newSong["Album Loved"] = "Album Loved" in song["key"]
+ newSong["Explicit"] = "Explicit" in song["key"]
+ newSong["Compilation"] = "Compilation" in song["key"]
+ newSong["Artwork Count"] = getInt("Artwork Count")
+ newSong["Sort Album"] = getStr("Sort Album")
+ newSong["Sort Artist"] = getStr("Sort Artist")
+ newSong["Sort Name"] = getStr("Sort Name")
+ newSong["Persistent ID"] = getStr("Persistent ID")
+ newSong["Track Type"] = getStr("Track Type")
+
+ out.append(newSong)
+
+ return out
+
+
+ def convert(self, filename, out_path):
+ with open(filename, 'r', encoding='utf-8') as xml_file:
+ data_dict = xmltodict.parse(xml_file.read())
+
+ # Extract the "Tracks" dictionary to be flattened
+ tracks_dict = data_dict['plist']['dict']['dict']
+
+ flat_tracks_dict = self.flatten_dict(tracks_dict["dict"])
+
+ json_data = json.dumps(flat_tracks_dict, indent=2)
+
+ with open(out_path, 'w', encoding='utf-8') as json_file:
+ json_file.write(json_data)
+
+
+class ParseItunesXml:
+ def __init__(self):
+ self.track_map: dict[str, Track] = {}
+ self.artist_map: dict[str, Artist] = {}
+ self.album_map: dict[str, Album] = {}
+ self.library = Library()
+
+
+ def parse_artists(self, data: list) -> List[Artist]:
+ if not data:
+ raise "No artist found"
+
+ artists = []
+
+ for artist_data in data:
+ name = artist_data
+ artist_id = name
+
+ if artist_id in self.artist_map:
+ artists.append(self.artist_map[artist_id])
+ continue
+
+ artist = Artist()
+ artist.id = artist_id
+ artist.title = name
+
+ self.artist_map[artist_id] = artist
+ self.library.artists.append(artist)
+
+ artists.append(artist)
+
+ return artists
+
+ def parse_album(self, data: str) -> Album:
+ if not data:
+ raise "album is none"
+
+ name = data
+ album_id = name
+
+ if album_id in self.album_map:
+ return self.album_map[album_id]
+
+ album = Album()
+ album.id = album_id
+ album.title = name
+
+ self.album_map[album_id] = album
+ self.library.albums.append(album)
+
+ return album
+
+
+ def parse(self, tracks_file_xml: Path, tracks_file_json: Path) -> Library:
+ XmlToJson().convert(tracks_file_xml, tracks_file_json)
+
+ tracks_content = json.loads(tracks_file_json.read_text())
+
+ for data in tracks_content:
+ track = Track()
+
+ track.id = data.get("Track ID")
+ track.title = data.get("Name")
+ track.artists = self.parse_artists([data.get("Artist")])
+ track.album = self.parse_album(data.get("Album"))
+
+ if not track.album or not len(track.artists) or not track.title:
+ raise "error parsing"
+
+ self.track_map[track.id] = track
+ self.library.tracks.append(track)
+
+ for track_id, track in self.track_map.items():
+ for artist in track.artists:
+ if artist not in track.album.artists:
+ track.album.artists.append(artist)
+
+ self.library.update_cache()
+
+ self.library.name = "Itunes"
+
+ return self.library
+
+
+ def parse_track_item(self, id, track: dict[str, Any]) -> Track:
+
+ return track
+
From b6d88d435a5d5c8496f3d9799d42db073cd2dd25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 7 Feb 2026 15:17:59 +0300
Subject: [PATCH 27/30] nice
---
Flow.py | 2 +
Gui.py | 38 +++++----
src/Library.py | 69 +++++----------
src/LibrarySerializer.py | 116 +++++++++++++-------------
src/backends/itunes/ParseItunesXml.py | 8 +-
src/backends/spotify/ParseSpotify.py | 7 ++
6 files changed, 112 insertions(+), 128 deletions(-)
diff --git a/Flow.py b/Flow.py
index 0031017..9cd7a04 100644
--- a/Flow.py
+++ b/Flow.py
@@ -23,6 +23,7 @@ from src.StatGenerator import log_stats
class Flow:
spotify_library : Library = None
local_library : Library = None
+ itunes_library : Library = None
@staticmethod
def fetch_spotify(self):
@@ -107,6 +108,7 @@ class Flow:
def load_libraries(self):
self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
+ self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
def save_libraries(self):
diff --git a/Gui.py b/Gui.py
index 85fd6e3..3c0ecc8 100644
--- a/Gui.py
+++ b/Gui.py
@@ -6,34 +6,36 @@ import pandas as pd
import Flow
-from src.Library import Library, Artist, Album, Track, Playlist
+from src.Library import *
+
+def entity_to_row(entity: Entity):
+ return {
+ "id": entity.id,
+ "mbid": entity.mbid,
+ "Title": entity.title,
+ "PLay Count": entity.play_count,
+ "Resolved": int(entity.resolved_percentage * 100),
+ "AutoScore": entity.auto_score,
+ "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
+ "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
+ "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
+ "LocalId": str(entity.local_id) if entity.local_id else None,
+ "Path": str(entity.local_path) if entity.local_path else None,
+ }
def track_to_row(track: Track):
- return {
- "Title": track.title,
+ return entity_to_row(track) | {
"Album": track.album.title if track.album else "",
"Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists),
"Lyrics": track.has_lyrics,
- "AutoScore": track.auto_score,
- "Added": track.added_at.strftime("%Y-%m-%d") if track.added_at else "",
- "LocalId": str(track.local_id) if track.local_id else "",
- "Path": str(track.local_path) if track.local_path else "",
}
def artist_to_row(artist: Artist):
- return {
- "Title": artist.title,
- "AutoScore": artist.auto_score,
- "Resolved": int(artist.resolved_percentage * 100),
- "LocalId": str(artist.local_id) if artist.local_id else "",
+ return entity_to_row(artist) | {
}
def album_to_row(album: Album):
- return {
- "Title": album.title,
- "AutoScore": album.auto_score,
- "Resolved": int(album.resolved_percentage * 100),
- "LocalId": str(album.local_id) if album.local_id else "",
+ return entity_to_row(album) | {
}
@@ -200,7 +202,7 @@ def run():
# flow.log_stats()
gui = LibraryView()
- gui.add_libraries([flow.spotify_library, flow.local_library])
+ gui.add_libraries([flow.spotify_library, flow.itunes_library, flow.local_library])
gui.render()
run()
\ No newline at end of file
diff --git a/src/Library.py b/src/Library.py
index 7bcefac..246d581 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -4,77 +4,58 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+
class Entity:
def __init__(self):
self.id: str = None
- self.mbid : str = None
+ self.mbid: str = None
self.title: str = None
+ self.duration_ms: int = None
self.fav: bool = False
self.rating: int = 0
- self.play_count : int = 0
- self.date_added : datetime = None
- self.date_released : datetime = None
- self.date_last_play : datetime = None
+ self.play_count: int = 0
+ self.date_added: datetime = None
+ self.date_released: datetime = None
+ self.date_last_play: datetime = None
self.auto_score: float = 0
self.local_id: str = None
- self.resolved_percentage : float = 0
+ self.local_path: Path = None
+ self.resolved_percentage: float = 0
-class Artist:
+
+class Artist(Entity):
def __init__(self):
- self.id: str = None
- self.title: str = None
+ super().__init__()
self.tracks: List[Track] = []
self.albums: List[Album] = []
- self.local_id: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
- self.resolved_percentage : float = 0
-class Track:
+class Track(Entity):
def __init__(self):
- self.id: str = None
- self.title: str = None
+ super().__init__()
+
self.artists: List[Artist] = []
self.album: Album = None
- self.duration_ms: int = None
- self.added_at: datetime = None
- self.local_path: Path = None
+
self.has_lyrics: bool = False
- self.local_id: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
-class Album:
+class Album(Entity):
def __init__(self):
- self.id: str = None
- self.title: str = None
- self.release_date: datetime = None
+ super().__init__()
self.tracks: List[Track] = []
self.artists: List[Artist] = []
- self.local_id: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
- self.resolved_percentage : float = 0
-class Playlist:
+class Playlist(Entity):
def __init__(self):
- self.title: str = None
+ super().__init__()
self.tracks: List[Track] = []
self.created_at: datetime = None
self.description: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
- self.resolved_percentage : float = 0
class Library:
@@ -120,14 +101,8 @@ class Library:
artist.albums.append(album)
def auto_score(self):
- self.score_tracks()
- self.score_artists()
self.score_albums()
-
- def score_tracks(self):
- for idx, track in enumerate(self.top_tracks):
- score = len(self.top_tracks) - idx
- track.auto_score = score
+ self.score_artists()
def score_artists(self):
for track in self.tracks:
@@ -150,4 +125,4 @@ class Library:
album.resolved_percentage = album_count_resolved / len(album.tracks)
artist_count_resolved += album_count_resolved
artist_count += len(album.tracks)
- artist.resolved_percentage = artist_count_resolved / artist_count
\ No newline at end of file
+ artist.resolved_percentage = artist_count_resolved / artist_count
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index f331824..278776a 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -26,60 +26,51 @@ class LibrarySaver:
def __init__(self, library: Library):
self.library = library
- @staticmethod
- def _artist_to_dict(artist: Artist) -> dict[str, Any]:
+ def _entity_to_dict(self, entity: Entity) -> dict[str, Any]:
return {
- "id": artist.id,
- "title": artist.title,
- "tracks": [track.id for track in artist.tracks],
- "albums": [album.id for album in artist.albums],
- "local_id": artist.local_id,
- "auto_score": artist.auto_score,
- "fav" : artist.fav,
- "rating" : artist.rating,
+ "id": entity.id,
+ "mbid": entity.mbid,
+
+ "title": entity.title,
+ "duration": entity.duration_ms,
+
+ "fav" : entity.fav,
+ "rating" : entity.rating,
+ "play_count": entity.play_count,
+ "date_added": entity.date_added,
+ "date_released": entity.date_released,
+ "date_last_play": entity.date_last_play,
+ "auto_score": entity.auto_score,
+
+ "local_id": entity.local_id,
+ "local_path": entity.local_path,
+ "resolved_percentage": entity.resolved_percentage,
}
- @staticmethod
- def _track_to_dict(track: Track) -> dict[str, Any]:
- return {
- "id": track.id,
- "title": track.title,
+ def _artist_to_dict(self, artist: Artist) -> dict[str, Any]:
+ return self._entity_to_dict(artist) | {
+ "tracks": [track.id for track in artist.tracks],
+ "albums": [album.id for album in artist.albums],
+ }
+
+ def _track_to_dict(self, track: Track) -> dict[str, Any]:
+ return self._entity_to_dict(track) | {
"artists": [artist.id for artist in track.artists],
"album": track.album.id,
"duration_ms": track.duration_ms,
- "added_at": _dt_to_str(track.added_at),
- "local_path": track.local_path,
"has_lyrics": track.has_lyrics,
- "local_id": track.local_id,
- "auto_score": track.auto_score,
- "fav" : track.fav,
- "rating" : track.rating,
}
- @staticmethod
- def _playlist_to_dict(playlist: Playlist) -> dict[str, Any]:
- return {
- "title": playlist.title,
+ def _playlist_to_dict(self, playlist: Playlist) -> dict[str, Any]:
+ return self._entity_to_dict(playlist) | {
"tracks": [track.id for track in playlist.tracks],
- "created_at": _dt_to_str(playlist.created_at),
"description": playlist.description,
- "auto_score": playlist.auto_score,
- "fav" : playlist.fav,
- "rating" : playlist.rating,
}
- @staticmethod
- def _album_to_dict(album: Album) -> dict[str, Any]:
- return {
- "id": album.id,
- "title": album.title,
- "release_date": album.release_date,
+ def _album_to_dict(self ,album: Album) -> dict[str, Any]:
+ return self._entity_to_dict(album) | {
"tracks": [track.id for track in album.tracks],
"artists": [artist.id for artist in album.artists],
- "local_id": album.local_id,
- "auto_score": album.auto_score,
- "fav" : album.fav,
- "rating" : album.rating,
}
def save(self, path: Path) -> None:
@@ -100,6 +91,7 @@ class LibrarySaver:
json.dump(data, f, indent=2, ensure_ascii=False)
+
class LibraryLoader:
def __init__(self):
self.track_map: dict[str, Track] = {}
@@ -127,48 +119,54 @@ class LibraryLoader:
self.track_map[id] = item
return item
+ def load_entity(self, entity : Entity, data: dict[str, Any]):
+ entity.id = data.get("id")
+ entity.mbid = data.get("mbid")
+
+ entity.title = data.get("title")
+ entity.duration_ms = data.get("duration_ms")
+
+ entity.fav = data.get("fav")
+ entity.rating = data.get("rating")
+ entity.play_count = data.get("play_count")
+ entity.date_added = _str_to_dt(data.get("date_added"))
+ entity.date_released = _str_to_dt(data.get("date_released"))
+ entity.date_last_play = _str_to_dt(data.get("date_last_play"))
+ entity.auto_score = data.get("auto_score")
+
+ entity.local_id = data.get("local_id")
+ entity.local_path = data.get("local_path")
+ entity.resolved_percentage = data.get("resolved_percentage")
+
+
def load_track(self, data: dict[str, Any]) -> Track:
track = self.get_track(data.get("id"))
- track.id = data.get("id")
- track.title = data.get("title")
+ self.load_entity(track, data)
+
track.duration_ms = data.get("duration_ms")
- track.added_at = _str_to_dt(data.get("added_at"))
- track.local_path = data.get("local_path")
track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
track.album = self.get_album(data.get("album"))
track.has_lyrics = data.get("has_lyrics")
- track.local_id = data.get("local_id")
- track.auto_score = data.get("auto_score")
- track.fav = data.get("fav")
- track.rating = data.get("rating")
self.track_map[track.id] = track
return track
def load_artist(self, data: dict[str, Any]):
artist = self.get_artist(data.get("id"))
- artist.id = data.get("id")
- artist.title = data.get("title")
+ self.load_entity(artist, data)
+
artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else []
- artist.local_id = data.get("local_id")
- artist.auto_score = data.get("auto_score")
- artist.fav = data.get("fav")
- artist.rating = data.get("rating")
self.artist_map[artist.id] = artist
return artist
def load_album(self, data: dict[str, Any]):
album = self.get_album(data.get("id"))
- album.id = data.get("id")
+ self.load_entity(album, data)
+
album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
- album.title = data.get("title")
album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
- album.local_id = data.get("local_id")
- album.auto_score = data.get("auto_score")
- album.fav = data.get("fav")
- album.rating = data.get("rating")
self.album_map[album.id] = album
return album
diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py
index 0af465f..71eb2d7 100755
--- a/src/backends/itunes/ParseItunesXml.py
+++ b/src/backends/itunes/ParseItunesXml.py
@@ -167,6 +167,7 @@ class ParseItunesXml:
track.title = data.get("Name")
track.artists = self.parse_artists([data.get("Artist")])
track.album = self.parse_album(data.get("Album"))
+ track.play_count = int(data.get("Play Count"))
if not track.album or not len(track.artists) or not track.title:
raise "error parsing"
@@ -183,10 +184,9 @@ class ParseItunesXml:
self.library.name = "Itunes"
+ for track in self.library.tracks:
+ track.auto_score = track.play_count
+
return self.library
- def parse_track_item(self, id, track: dict[str, Any]) -> Track:
-
- return track
-
diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
index 29c24e7..cfca39d 100644
--- a/src/backends/spotify/ParseSpotify.py
+++ b/src/backends/spotify/ParseSpotify.py
@@ -33,6 +33,8 @@ class Parser:
self.library.name = "Spotify"
self.library.auto_score()
+ self.score_tracks()
+
return self.library
@@ -122,3 +124,8 @@ class Parser:
self.library.artists.append(artist)
return artist
+
+ def score_tracks(self):
+ for idx, track in enumerate(self.library.top_tracks):
+ score = len(self.library.top_tracks) - idx
+ track.auto_score = score
\ No newline at end of file
From 9cd2c9189b706b0360da20c7be4e76264148009b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sat, 7 Feb 2026 15:17:59 +0300
Subject: [PATCH 28/30] nice
---
Flow.py | 2 +
Gui.py | 38 +++++----
src/Library.py | 69 +++++----------
src/LibrarySerializer.py | 116 +++++++++++++-------------
src/backends/itunes/ParseItunesXml.py | 8 +-
src/backends/spotify/ParseSpotify.py | 7 ++
6 files changed, 112 insertions(+), 128 deletions(-)
diff --git a/Flow.py b/Flow.py
index 0031017..9cd7a04 100644
--- a/Flow.py
+++ b/Flow.py
@@ -23,6 +23,7 @@ from src.StatGenerator import log_stats
class Flow:
spotify_library : Library = None
local_library : Library = None
+ itunes_library : Library = None
@staticmethod
def fetch_spotify(self):
@@ -107,6 +108,7 @@ class Flow:
def load_libraries(self):
self.spotify_library = LibraryLoader().load(work_dir / "spotify_library.json")
+ self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
def save_libraries(self):
diff --git a/Gui.py b/Gui.py
index 85fd6e3..3c0ecc8 100644
--- a/Gui.py
+++ b/Gui.py
@@ -6,34 +6,36 @@ import pandas as pd
import Flow
-from src.Library import Library, Artist, Album, Track, Playlist
+from src.Library import *
+
+def entity_to_row(entity: Entity):
+ return {
+ "id": entity.id,
+ "mbid": entity.mbid,
+ "Title": entity.title,
+ "PLay Count": entity.play_count,
+ "Resolved": int(entity.resolved_percentage * 100),
+ "AutoScore": entity.auto_score,
+ "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
+ "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
+ "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
+ "LocalId": str(entity.local_id) if entity.local_id else None,
+ "Path": str(entity.local_path) if entity.local_path else None,
+ }
def track_to_row(track: Track):
- return {
- "Title": track.title,
+ return entity_to_row(track) | {
"Album": track.album.title if track.album else "",
"Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists),
"Lyrics": track.has_lyrics,
- "AutoScore": track.auto_score,
- "Added": track.added_at.strftime("%Y-%m-%d") if track.added_at else "",
- "LocalId": str(track.local_id) if track.local_id else "",
- "Path": str(track.local_path) if track.local_path else "",
}
def artist_to_row(artist: Artist):
- return {
- "Title": artist.title,
- "AutoScore": artist.auto_score,
- "Resolved": int(artist.resolved_percentage * 100),
- "LocalId": str(artist.local_id) if artist.local_id else "",
+ return entity_to_row(artist) | {
}
def album_to_row(album: Album):
- return {
- "Title": album.title,
- "AutoScore": album.auto_score,
- "Resolved": int(album.resolved_percentage * 100),
- "LocalId": str(album.local_id) if album.local_id else "",
+ return entity_to_row(album) | {
}
@@ -200,7 +202,7 @@ def run():
# flow.log_stats()
gui = LibraryView()
- gui.add_libraries([flow.spotify_library, flow.local_library])
+ gui.add_libraries([flow.spotify_library, flow.itunes_library, flow.local_library])
gui.render()
run()
\ No newline at end of file
diff --git a/src/Library.py b/src/Library.py
index 7bcefac..246d581 100644
--- a/src/Library.py
+++ b/src/Library.py
@@ -4,77 +4,58 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional, Any
+
class Entity:
def __init__(self):
self.id: str = None
- self.mbid : str = None
+ self.mbid: str = None
self.title: str = None
+ self.duration_ms: int = None
self.fav: bool = False
self.rating: int = 0
- self.play_count : int = 0
- self.date_added : datetime = None
- self.date_released : datetime = None
- self.date_last_play : datetime = None
+ self.play_count: int = 0
+ self.date_added: datetime = None
+ self.date_released: datetime = None
+ self.date_last_play: datetime = None
self.auto_score: float = 0
self.local_id: str = None
- self.resolved_percentage : float = 0
+ self.local_path: Path = None
+ self.resolved_percentage: float = 0
-class Artist:
+
+class Artist(Entity):
def __init__(self):
- self.id: str = None
- self.title: str = None
+ super().__init__()
self.tracks: List[Track] = []
self.albums: List[Album] = []
- self.local_id: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
- self.resolved_percentage : float = 0
-class Track:
+class Track(Entity):
def __init__(self):
- self.id: str = None
- self.title: str = None
+ super().__init__()
+
self.artists: List[Artist] = []
self.album: Album = None
- self.duration_ms: int = None
- self.added_at: datetime = None
- self.local_path: Path = None
+
self.has_lyrics: bool = False
- self.local_id: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
-class Album:
+class Album(Entity):
def __init__(self):
- self.id: str = None
- self.title: str = None
- self.release_date: datetime = None
+ super().__init__()
self.tracks: List[Track] = []
self.artists: List[Artist] = []
- self.local_id: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
- self.resolved_percentage : float = 0
-class Playlist:
+class Playlist(Entity):
def __init__(self):
- self.title: str = None
+ super().__init__()
self.tracks: List[Track] = []
self.created_at: datetime = None
self.description: str = None
- self.auto_score: float = 0
- self.fav: bool = False
- self.rating: int = 0
- self.resolved_percentage : float = 0
class Library:
@@ -120,14 +101,8 @@ class Library:
artist.albums.append(album)
def auto_score(self):
- self.score_tracks()
- self.score_artists()
self.score_albums()
-
- def score_tracks(self):
- for idx, track in enumerate(self.top_tracks):
- score = len(self.top_tracks) - idx
- track.auto_score = score
+ self.score_artists()
def score_artists(self):
for track in self.tracks:
@@ -150,4 +125,4 @@ class Library:
album.resolved_percentage = album_count_resolved / len(album.tracks)
artist_count_resolved += album_count_resolved
artist_count += len(album.tracks)
- artist.resolved_percentage = artist_count_resolved / artist_count
\ No newline at end of file
+ artist.resolved_percentage = artist_count_resolved / artist_count
diff --git a/src/LibrarySerializer.py b/src/LibrarySerializer.py
index f331824..278776a 100644
--- a/src/LibrarySerializer.py
+++ b/src/LibrarySerializer.py
@@ -26,60 +26,51 @@ class LibrarySaver:
def __init__(self, library: Library):
self.library = library
- @staticmethod
- def _artist_to_dict(artist: Artist) -> dict[str, Any]:
+ def _entity_to_dict(self, entity: Entity) -> dict[str, Any]:
return {
- "id": artist.id,
- "title": artist.title,
- "tracks": [track.id for track in artist.tracks],
- "albums": [album.id for album in artist.albums],
- "local_id": artist.local_id,
- "auto_score": artist.auto_score,
- "fav" : artist.fav,
- "rating" : artist.rating,
+ "id": entity.id,
+ "mbid": entity.mbid,
+
+ "title": entity.title,
+ "duration": entity.duration_ms,
+
+ "fav" : entity.fav,
+ "rating" : entity.rating,
+ "play_count": entity.play_count,
+ "date_added": entity.date_added,
+ "date_released": entity.date_released,
+ "date_last_play": entity.date_last_play,
+ "auto_score": entity.auto_score,
+
+ "local_id": entity.local_id,
+ "local_path": entity.local_path,
+ "resolved_percentage": entity.resolved_percentage,
}
- @staticmethod
- def _track_to_dict(track: Track) -> dict[str, Any]:
- return {
- "id": track.id,
- "title": track.title,
+ def _artist_to_dict(self, artist: Artist) -> dict[str, Any]:
+ return self._entity_to_dict(artist) | {
+ "tracks": [track.id for track in artist.tracks],
+ "albums": [album.id for album in artist.albums],
+ }
+
+ def _track_to_dict(self, track: Track) -> dict[str, Any]:
+ return self._entity_to_dict(track) | {
"artists": [artist.id for artist in track.artists],
"album": track.album.id,
"duration_ms": track.duration_ms,
- "added_at": _dt_to_str(track.added_at),
- "local_path": track.local_path,
"has_lyrics": track.has_lyrics,
- "local_id": track.local_id,
- "auto_score": track.auto_score,
- "fav" : track.fav,
- "rating" : track.rating,
}
- @staticmethod
- def _playlist_to_dict(playlist: Playlist) -> dict[str, Any]:
- return {
- "title": playlist.title,
+ def _playlist_to_dict(self, playlist: Playlist) -> dict[str, Any]:
+ return self._entity_to_dict(playlist) | {
"tracks": [track.id for track in playlist.tracks],
- "created_at": _dt_to_str(playlist.created_at),
"description": playlist.description,
- "auto_score": playlist.auto_score,
- "fav" : playlist.fav,
- "rating" : playlist.rating,
}
- @staticmethod
- def _album_to_dict(album: Album) -> dict[str, Any]:
- return {
- "id": album.id,
- "title": album.title,
- "release_date": album.release_date,
+ def _album_to_dict(self ,album: Album) -> dict[str, Any]:
+ return self._entity_to_dict(album) | {
"tracks": [track.id for track in album.tracks],
"artists": [artist.id for artist in album.artists],
- "local_id": album.local_id,
- "auto_score": album.auto_score,
- "fav" : album.fav,
- "rating" : album.rating,
}
def save(self, path: Path) -> None:
@@ -100,6 +91,7 @@ class LibrarySaver:
json.dump(data, f, indent=2, ensure_ascii=False)
+
class LibraryLoader:
def __init__(self):
self.track_map: dict[str, Track] = {}
@@ -127,48 +119,54 @@ class LibraryLoader:
self.track_map[id] = item
return item
+ def load_entity(self, entity : Entity, data: dict[str, Any]):
+ entity.id = data.get("id")
+ entity.mbid = data.get("mbid")
+
+ entity.title = data.get("title")
+ entity.duration_ms = data.get("duration_ms")
+
+ entity.fav = data.get("fav")
+ entity.rating = data.get("rating")
+ entity.play_count = data.get("play_count")
+ entity.date_added = _str_to_dt(data.get("date_added"))
+ entity.date_released = _str_to_dt(data.get("date_released"))
+ entity.date_last_play = _str_to_dt(data.get("date_last_play"))
+ entity.auto_score = data.get("auto_score")
+
+ entity.local_id = data.get("local_id")
+ entity.local_path = data.get("local_path")
+ entity.resolved_percentage = data.get("resolved_percentage")
+
+
def load_track(self, data: dict[str, Any]) -> Track:
track = self.get_track(data.get("id"))
- track.id = data.get("id")
- track.title = data.get("title")
+ self.load_entity(track, data)
+
track.duration_ms = data.get("duration_ms")
- track.added_at = _str_to_dt(data.get("added_at"))
- track.local_path = data.get("local_path")
track.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
track.album = self.get_album(data.get("album"))
track.has_lyrics = data.get("has_lyrics")
- track.local_id = data.get("local_id")
- track.auto_score = data.get("auto_score")
- track.fav = data.get("fav")
- track.rating = data.get("rating")
self.track_map[track.id] = track
return track
def load_artist(self, data: dict[str, Any]):
artist = self.get_artist(data.get("id"))
- artist.id = data.get("id")
- artist.title = data.get("title")
+ self.load_entity(artist, data)
+
artist.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
artist.albums = [self.get_album(album_id) for album_id in data.get("albums")] if data.get("albums") else []
- artist.local_id = data.get("local_id")
- artist.auto_score = data.get("auto_score")
- artist.fav = data.get("fav")
- artist.rating = data.get("rating")
self.artist_map[artist.id] = artist
return artist
def load_album(self, data: dict[str, Any]):
album = self.get_album(data.get("id"))
- album.id = data.get("id")
+ self.load_entity(album, data)
+
album.tracks = [self.get_track(track_id) for track_id in data.get("tracks")]
- album.title = data.get("title")
album.artists = [self.get_artist(artist_id) for artist_id in data.get("artists")]
- album.local_id = data.get("local_id")
- album.auto_score = data.get("auto_score")
- album.fav = data.get("fav")
- album.rating = data.get("rating")
self.album_map[album.id] = album
return album
diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py
index 0af465f..71eb2d7 100755
--- a/src/backends/itunes/ParseItunesXml.py
+++ b/src/backends/itunes/ParseItunesXml.py
@@ -167,6 +167,7 @@ class ParseItunesXml:
track.title = data.get("Name")
track.artists = self.parse_artists([data.get("Artist")])
track.album = self.parse_album(data.get("Album"))
+ track.play_count = int(data.get("Play Count"))
if not track.album or not len(track.artists) or not track.title:
raise "error parsing"
@@ -183,10 +184,9 @@ class ParseItunesXml:
self.library.name = "Itunes"
+ for track in self.library.tracks:
+ track.auto_score = track.play_count
+
return self.library
- def parse_track_item(self, id, track: dict[str, Any]) -> Track:
-
- return track
-
diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
index 29c24e7..cfca39d 100644
--- a/src/backends/spotify/ParseSpotify.py
+++ b/src/backends/spotify/ParseSpotify.py
@@ -33,6 +33,8 @@ class Parser:
self.library.name = "Spotify"
self.library.auto_score()
+ self.score_tracks()
+
return self.library
@@ -122,3 +124,8 @@ class Parser:
self.library.artists.append(artist)
return artist
+
+ def score_tracks(self):
+ for idx, track in enumerate(self.library.top_tracks):
+ score = len(self.library.top_tracks) - idx
+ track.auto_score = score
\ No newline at end of file
From cf90ce43fe0e9bc9ef728428da106d75740c2192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 14 Jun 2026 23:46:09 +0300
Subject: [PATCH 29/30] untested
---
Flow.py | 11 +++++
Gui.py | 6 +--
main.py | 3 ++
src/backends/itunes/ParseItunesXml.py | 2 +
src/backends/local/LocalLibraryIndexer.py | 12 +++++
src/backends/mb/mbidFinder.py | 54 +++++++++++++++++++++++
src/backends/spotify/ParseSpotify.py | 2 +-
src/mappers/DummyLibGenerator.py | 46 +++++++++++++++++++
src/mappers/FuzzyMapper.py | 54 +++++++++++++++--------
9 files changed, 168 insertions(+), 22 deletions(-)
create mode 100644 src/backends/mb/mbidFinder.py
create mode 100644 src/mappers/DummyLibGenerator.py
diff --git a/Flow.py b/Flow.py
index 9cd7a04..fd10266 100644
--- a/Flow.py
+++ b/Flow.py
@@ -19,6 +19,8 @@ from src.backends.itunes.ParseItunesXml import ParseItunesXml
from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
+import src.backends.mb.mbidFinder as mb
+import src.mappers.DummyLibGenerator as libgen
class Flow:
spotify_library : Library = None
@@ -63,6 +65,10 @@ class Flow:
library_parsed = parser.parse(work_dir / "local" / "local_library.json")
LibrarySaver(library_parsed).save(work_dir / "local_library.json")
+ def find_mbids(self):
+ mb.find_mbids(self.spotify_library)
+ mb.find_mbids(self.itunes_library)
+ self.save_libraries()
def fetch_and_save_lyrics(self):
"""
@@ -111,6 +117,11 @@ class Flow:
self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
+ def gen_fake_libs(self):
+ libgen.generate_dummy_library(self.spotify_library, work_dir / "spotify" / "dummylib")
+ libgen.generate_dummy_library(self.itunes_library, work_dir / "itunes" / "dummylib")
+
+
def save_libraries(self):
LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json")
LibrarySaver(self.local_library).save(work_dir / "local_library.json")
diff --git a/Gui.py b/Gui.py
index 3c0ecc8..12d1144 100644
--- a/Gui.py
+++ b/Gui.py
@@ -16,9 +16,9 @@ def entity_to_row(entity: Entity):
"PLay Count": entity.play_count,
"Resolved": int(entity.resolved_percentage * 100),
"AutoScore": entity.auto_score,
- "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
- "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
- "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
+ # "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
+ # "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
+ # "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
"LocalId": str(entity.local_id) if entity.local_id else None,
"Path": str(entity.local_path) if entity.local_path else None,
}
diff --git a/main.py b/main.py
index 26588d1..05a781d 100644
--- a/main.py
+++ b/main.py
@@ -26,6 +26,9 @@ def main():
flow.load_libraries()
+ # flow.gen_fake_libs()
+ # flow.find_mbids()
+
flow.map_local_to_spotify()
flow.save_libraries()
diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py
index 71eb2d7..4b0f417 100755
--- a/src/backends/itunes/ParseItunesXml.py
+++ b/src/backends/itunes/ParseItunesXml.py
@@ -187,6 +187,8 @@ class ParseItunesXml:
for track in self.library.tracks:
track.auto_score = track.play_count
+ self.library.auto_score()
+
return self.library
diff --git a/src/backends/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py
index 6f132d4..08629f7 100644
--- a/src/backends/local/LocalLibraryIndexer.py
+++ b/src/backends/local/LocalLibraryIndexer.py
@@ -64,6 +64,17 @@ def get_artist(tags, fallback=None) -> list :
return fallback
+def get_mb_artist(tags, fallback=None) -> list :
+ artist_fields = ["TXXX:MusicBrainz Artist Id", "----:com.apple.iTunes:MusicBrainz Artist Id", "MUSICBRAINZ_ARTISTID"]
+ for field in artist_fields:
+ if field in tags and tags[field]:
+ values = tags.get(field)
+ if not isinstance(values, list):
+ return values.text
+ return values
+
+ return fallback
+
def safe_tag_get(tags, key, index=0):
try:
value = tags.get(key, None)
@@ -147,6 +158,7 @@ def update_local_library(root_dir, output_dir):
tags = audio.tags
track['artists'] = get_artist(tags)
+ # track['artists_mbid'] = get_mb_artist(tags)
track['name'] = get_title(tags)
track['album'] = get_album(tags)
diff --git a/src/backends/mb/mbidFinder.py b/src/backends/mb/mbidFinder.py
new file mode 100644
index 0000000..6ffcd09
--- /dev/null
+++ b/src/backends/mb/mbidFinder.py
@@ -0,0 +1,54 @@
+from src.Library import *
+
+import musicbrainzngs as mb
+from tqdm import tqdm
+
+mb.set_useragent(
+ "resolve_spotify",
+ "1.0",
+ "ilusha.main@gmail.com"
+)
+
+def find_mbid_by_tags(track: Track) -> str | None:
+ if not track.title or not track.artists:
+ return None
+
+ artist_names = [a.title for a in track.artists if a.title]
+ if not artist_names:
+ return None
+
+ query_parts = [
+ f'recording:"{track.title}"',
+ f'artist:"{artist_names[0]}"'
+ ]
+
+ query = " AND ".join(query_parts)
+
+ try:
+ result = mb.search_recordings(query=query, limit=5)
+ except Exception:
+ return None
+
+ recordings = result.get("recording-list", [])
+ if not recordings:
+ return None
+
+ # Pick best-scored result
+ best = max(
+ recordings,
+ key=lambda r: int(r.get("score", 0))
+ )
+
+ return best.get("id")
+
+
+def find_mbids(library: Library):
+ tracks = library.tracks
+
+ for track in tqdm(tracks, desc="Resolving MBIDs of " + library.name + " library", unit="track"):
+ if track.mbid:
+ continue
+
+ track.mbid = find_mbid_by_tags(track)
+ if not track.mbid:
+ print(f"not found for track {track.title} {track.artists[0].title}")
diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
index cfca39d..fa5d80c 100644
--- a/src/backends/spotify/ParseSpotify.py
+++ b/src/backends/spotify/ParseSpotify.py
@@ -31,9 +31,9 @@ class Parser:
self.library.update_cache()
self.library.name = "Spotify"
- self.library.auto_score()
self.score_tracks()
+ self.library.auto_score()
return self.library
diff --git a/src/mappers/DummyLibGenerator.py b/src/mappers/DummyLibGenerator.py
new file mode 100644
index 0000000..7aaa544
--- /dev/null
+++ b/src/mappers/DummyLibGenerator.py
@@ -0,0 +1,46 @@
+from mutagen.easyid3 import EasyID3
+from pydub import AudioSegment
+from mutagen.easyid3 import EasyID3
+
+from src.Library import *
+
+def safe_name(s: str) -> str:
+ return (
+ s.replace("/", "_")
+ .replace("\\", "_")
+ .replace(":", "_")
+ .replace("*", "_")
+ .replace("?", "_")
+ .replace('"', "_")
+ .replace("<", "_")
+ .replace(">", "_")
+ .replace("|", "_")
+ .strip()
+ )
+
+def generate_dummy_library(library: Library, root_path: Path):
+ root_path.mkdir(parents=True, exist_ok=True)
+
+ for artist in library.artists:
+ artist_path = root_path / safe_name(artist.title or f"artist_{artist.id}")
+ artist_path.mkdir(exist_ok=True)
+
+ for album in artist.albums:
+ album_path = artist_path / safe_name(album.title or f"album_{album.id}")
+ album_path.mkdir(exist_ok=True)
+
+ for track in album.tracks:
+ track_title = track.title or f"track_{track.id}"
+ track_file = album_path / f"{safe_name(track_title)}.mp3"
+
+ # duration_ms = max(track.duration_ms or 100, 100)
+ duration_ms = 10
+ silence = AudioSegment.silent(duration=duration_ms)
+ silence.export(track_file, format="mp3")
+
+ audio = EasyID3(track_file)
+ audio["title"] = track.title or ""
+ audio["artist"] = ", ".join([a.title or "" for a in track.artists])
+ audio["album"] = album.title or ""
+ audio["tracknumber"] = str(album.tracks.index(track) + 1)
+ audio.save()
diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py
index c6413d0..1dbbf75 100644
--- a/src/mappers/FuzzyMapper.py
+++ b/src/mappers/FuzzyMapper.py
@@ -4,6 +4,8 @@ from tqdm import tqdm
from src.Library import *
from src.helpers import *
+import re
+import string
import unicodedata
@@ -11,34 +13,49 @@ user_def_artist_map = {
"donda" : "kanye west",
}
-def normalize(s):
- return unicodedata.normalize("NFKC", s).replace("‐", "-")
+user_def_album_map = {
+ "music" : "music sorry 4 da wait",
+}
def get_score(first: str, second: str) -> float:
- first_norm = normalize(first).lower()
- second_norm = normalize(second).lower()
- return float(first_norm == second_norm) * 100
- return fuzz.token_set_ratio(first, second)
+ return float(first == second) * 100
+
+def clean_title(title: str) -> str:
+ title = unicodedata.normalize("NFKC", title).replace("‐", "-")
+ title = unicodedata.normalize("NFKD", title)
+ title = re.sub(r"\([^)]*\)", "", title)
+ title = "".join(
+ c for c in title
+ if not unicodedata.category(c).startswith("P")
+ )
+ title = title.translate(str.maketrans("", "", string.punctuation))
+ title = title.lower()
+ return " ".join(title.split())
+
+def find_mapping(first, second, mapping):
+ for key, value in mapping.items():
+ if (first == key and second == value) or (first == value and second == key):
+ return True
+ return False
def get_album_score(first: str, second: str) -> float:
+ first = clean_title(first)
+ second = clean_title(second)
+
+ if find_mapping(first, second, user_def_album_map):
+ return 100
+
return get_score(first, second)
def get_track_score(first: str, second: str) -> float:
- return get_score(first, second)
+ return get_score(clean_title(first), clean_title(second))
def get_artist_score(first: str, second: str) -> float:
- if not first or not second:
- return 0.0
+ first = clean_title(first)
+ second = clean_title(second)
- first_lower = first.lower()
- second_lower = second.lower()
-
- for key, value in user_def_artist_map.items():
- key_lower = key.lower()
- value_lower = value.lower()
- if (first_lower == key_lower and second_lower == value_lower) or \
- (first_lower == value_lower and second_lower == key_lower):
- return 100.0
+ if find_mapping(first, second, user_def_artist_map):
+ return 100
return get_score(first, second)
@@ -97,6 +114,7 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
track_map[remote_track.id] = (local_track.id, score)
remote_track.local_id = local_track.id
+ remote_track.local_path = local_track.local_path
process_bar.update(1)
From 4fd68572871318585fbf4f9522915da2815b4730 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?=
=?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?=
=?UTF-8?q?=D0=B8=D1=87?=
Date: Sun, 14 Jun 2026 23:46:09 +0300
Subject: [PATCH 30/30] untested
---
Flow.py | 11 +++++
Gui.py | 6 +--
main.py | 3 ++
src/backends/itunes/ParseItunesXml.py | 2 +
src/backends/local/LocalLibraryIndexer.py | 12 +++++
src/backends/mb/mbidFinder.py | 54 +++++++++++++++++++++++
src/backends/spotify/ParseSpotify.py | 2 +-
src/mappers/DummyLibGenerator.py | 46 +++++++++++++++++++
src/mappers/FuzzyMapper.py | 54 +++++++++++++++--------
9 files changed, 168 insertions(+), 22 deletions(-)
create mode 100644 src/backends/mb/mbidFinder.py
create mode 100644 src/mappers/DummyLibGenerator.py
diff --git a/Flow.py b/Flow.py
index 9cd7a04..fd10266 100644
--- a/Flow.py
+++ b/Flow.py
@@ -19,6 +19,8 @@ from src.backends.itunes.ParseItunesXml import ParseItunesXml
from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
from src.StatGenerator import log_stats
+import src.backends.mb.mbidFinder as mb
+import src.mappers.DummyLibGenerator as libgen
class Flow:
spotify_library : Library = None
@@ -63,6 +65,10 @@ class Flow:
library_parsed = parser.parse(work_dir / "local" / "local_library.json")
LibrarySaver(library_parsed).save(work_dir / "local_library.json")
+ def find_mbids(self):
+ mb.find_mbids(self.spotify_library)
+ mb.find_mbids(self.itunes_library)
+ self.save_libraries()
def fetch_and_save_lyrics(self):
"""
@@ -111,6 +117,11 @@ class Flow:
self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json")
self.local_library = LibraryLoader().load(work_dir / "local_library.json")
+ def gen_fake_libs(self):
+ libgen.generate_dummy_library(self.spotify_library, work_dir / "spotify" / "dummylib")
+ libgen.generate_dummy_library(self.itunes_library, work_dir / "itunes" / "dummylib")
+
+
def save_libraries(self):
LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json")
LibrarySaver(self.local_library).save(work_dir / "local_library.json")
diff --git a/Gui.py b/Gui.py
index 3c0ecc8..12d1144 100644
--- a/Gui.py
+++ b/Gui.py
@@ -16,9 +16,9 @@ def entity_to_row(entity: Entity):
"PLay Count": entity.play_count,
"Resolved": int(entity.resolved_percentage * 100),
"AutoScore": entity.auto_score,
- "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
- "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
- "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
+ # "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
+ # "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
+ # "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
"LocalId": str(entity.local_id) if entity.local_id else None,
"Path": str(entity.local_path) if entity.local_path else None,
}
diff --git a/main.py b/main.py
index 26588d1..05a781d 100644
--- a/main.py
+++ b/main.py
@@ -26,6 +26,9 @@ def main():
flow.load_libraries()
+ # flow.gen_fake_libs()
+ # flow.find_mbids()
+
flow.map_local_to_spotify()
flow.save_libraries()
diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py
index 71eb2d7..4b0f417 100755
--- a/src/backends/itunes/ParseItunesXml.py
+++ b/src/backends/itunes/ParseItunesXml.py
@@ -187,6 +187,8 @@ class ParseItunesXml:
for track in self.library.tracks:
track.auto_score = track.play_count
+ self.library.auto_score()
+
return self.library
diff --git a/src/backends/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py
index 6f132d4..08629f7 100644
--- a/src/backends/local/LocalLibraryIndexer.py
+++ b/src/backends/local/LocalLibraryIndexer.py
@@ -64,6 +64,17 @@ def get_artist(tags, fallback=None) -> list :
return fallback
+def get_mb_artist(tags, fallback=None) -> list :
+ artist_fields = ["TXXX:MusicBrainz Artist Id", "----:com.apple.iTunes:MusicBrainz Artist Id", "MUSICBRAINZ_ARTISTID"]
+ for field in artist_fields:
+ if field in tags and tags[field]:
+ values = tags.get(field)
+ if not isinstance(values, list):
+ return values.text
+ return values
+
+ return fallback
+
def safe_tag_get(tags, key, index=0):
try:
value = tags.get(key, None)
@@ -147,6 +158,7 @@ def update_local_library(root_dir, output_dir):
tags = audio.tags
track['artists'] = get_artist(tags)
+ # track['artists_mbid'] = get_mb_artist(tags)
track['name'] = get_title(tags)
track['album'] = get_album(tags)
diff --git a/src/backends/mb/mbidFinder.py b/src/backends/mb/mbidFinder.py
new file mode 100644
index 0000000..6ffcd09
--- /dev/null
+++ b/src/backends/mb/mbidFinder.py
@@ -0,0 +1,54 @@
+from src.Library import *
+
+import musicbrainzngs as mb
+from tqdm import tqdm
+
+mb.set_useragent(
+ "resolve_spotify",
+ "1.0",
+ "ilusha.main@gmail.com"
+)
+
+def find_mbid_by_tags(track: Track) -> str | None:
+ if not track.title or not track.artists:
+ return None
+
+ artist_names = [a.title for a in track.artists if a.title]
+ if not artist_names:
+ return None
+
+ query_parts = [
+ f'recording:"{track.title}"',
+ f'artist:"{artist_names[0]}"'
+ ]
+
+ query = " AND ".join(query_parts)
+
+ try:
+ result = mb.search_recordings(query=query, limit=5)
+ except Exception:
+ return None
+
+ recordings = result.get("recording-list", [])
+ if not recordings:
+ return None
+
+ # Pick best-scored result
+ best = max(
+ recordings,
+ key=lambda r: int(r.get("score", 0))
+ )
+
+ return best.get("id")
+
+
+def find_mbids(library: Library):
+ tracks = library.tracks
+
+ for track in tqdm(tracks, desc="Resolving MBIDs of " + library.name + " library", unit="track"):
+ if track.mbid:
+ continue
+
+ track.mbid = find_mbid_by_tags(track)
+ if not track.mbid:
+ print(f"not found for track {track.title} {track.artists[0].title}")
diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py
index cfca39d..fa5d80c 100644
--- a/src/backends/spotify/ParseSpotify.py
+++ b/src/backends/spotify/ParseSpotify.py
@@ -31,9 +31,9 @@ class Parser:
self.library.update_cache()
self.library.name = "Spotify"
- self.library.auto_score()
self.score_tracks()
+ self.library.auto_score()
return self.library
diff --git a/src/mappers/DummyLibGenerator.py b/src/mappers/DummyLibGenerator.py
new file mode 100644
index 0000000..7aaa544
--- /dev/null
+++ b/src/mappers/DummyLibGenerator.py
@@ -0,0 +1,46 @@
+from mutagen.easyid3 import EasyID3
+from pydub import AudioSegment
+from mutagen.easyid3 import EasyID3
+
+from src.Library import *
+
+def safe_name(s: str) -> str:
+ return (
+ s.replace("/", "_")
+ .replace("\\", "_")
+ .replace(":", "_")
+ .replace("*", "_")
+ .replace("?", "_")
+ .replace('"', "_")
+ .replace("<", "_")
+ .replace(">", "_")
+ .replace("|", "_")
+ .strip()
+ )
+
+def generate_dummy_library(library: Library, root_path: Path):
+ root_path.mkdir(parents=True, exist_ok=True)
+
+ for artist in library.artists:
+ artist_path = root_path / safe_name(artist.title or f"artist_{artist.id}")
+ artist_path.mkdir(exist_ok=True)
+
+ for album in artist.albums:
+ album_path = artist_path / safe_name(album.title or f"album_{album.id}")
+ album_path.mkdir(exist_ok=True)
+
+ for track in album.tracks:
+ track_title = track.title or f"track_{track.id}"
+ track_file = album_path / f"{safe_name(track_title)}.mp3"
+
+ # duration_ms = max(track.duration_ms or 100, 100)
+ duration_ms = 10
+ silence = AudioSegment.silent(duration=duration_ms)
+ silence.export(track_file, format="mp3")
+
+ audio = EasyID3(track_file)
+ audio["title"] = track.title or ""
+ audio["artist"] = ", ".join([a.title or "" for a in track.artists])
+ audio["album"] = album.title or ""
+ audio["tracknumber"] = str(album.tracks.index(track) + 1)
+ audio.save()
diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py
index c6413d0..1dbbf75 100644
--- a/src/mappers/FuzzyMapper.py
+++ b/src/mappers/FuzzyMapper.py
@@ -4,6 +4,8 @@ from tqdm import tqdm
from src.Library import *
from src.helpers import *
+import re
+import string
import unicodedata
@@ -11,34 +13,49 @@ user_def_artist_map = {
"donda" : "kanye west",
}
-def normalize(s):
- return unicodedata.normalize("NFKC", s).replace("‐", "-")
+user_def_album_map = {
+ "music" : "music sorry 4 da wait",
+}
def get_score(first: str, second: str) -> float:
- first_norm = normalize(first).lower()
- second_norm = normalize(second).lower()
- return float(first_norm == second_norm) * 100
- return fuzz.token_set_ratio(first, second)
+ return float(first == second) * 100
+
+def clean_title(title: str) -> str:
+ title = unicodedata.normalize("NFKC", title).replace("‐", "-")
+ title = unicodedata.normalize("NFKD", title)
+ title = re.sub(r"\([^)]*\)", "", title)
+ title = "".join(
+ c for c in title
+ if not unicodedata.category(c).startswith("P")
+ )
+ title = title.translate(str.maketrans("", "", string.punctuation))
+ title = title.lower()
+ return " ".join(title.split())
+
+def find_mapping(first, second, mapping):
+ for key, value in mapping.items():
+ if (first == key and second == value) or (first == value and second == key):
+ return True
+ return False
def get_album_score(first: str, second: str) -> float:
+ first = clean_title(first)
+ second = clean_title(second)
+
+ if find_mapping(first, second, user_def_album_map):
+ return 100
+
return get_score(first, second)
def get_track_score(first: str, second: str) -> float:
- return get_score(first, second)
+ return get_score(clean_title(first), clean_title(second))
def get_artist_score(first: str, second: str) -> float:
- if not first or not second:
- return 0.0
+ first = clean_title(first)
+ second = clean_title(second)
- first_lower = first.lower()
- second_lower = second.lower()
-
- for key, value in user_def_artist_map.items():
- key_lower = key.lower()
- value_lower = value.lower()
- if (first_lower == key_lower and second_lower == value_lower) or \
- (first_lower == value_lower and second_lower == key_lower):
- return 100.0
+ if find_mapping(first, second, user_def_artist_map):
+ return 100
return get_score(first, second)
@@ -97,6 +114,7 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
track_map[remote_track.id] = (local_track.id, score)
remote_track.local_id = local_track.id
+ remote_track.local_path = local_track.local_path
process_bar.update(1)