From 4fd68572871318585fbf4f9522915da2815b4730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Sun, 14 Jun 2026 23:46:09 +0300 Subject: [PATCH 1/4] untested --- Flow.py | 11 +++++ Gui.py | 6 +-- main.py | 3 ++ src/backends/itunes/ParseItunesXml.py | 2 + src/backends/local/LocalLibraryIndexer.py | 12 +++++ src/backends/mb/mbidFinder.py | 54 +++++++++++++++++++++++ src/backends/spotify/ParseSpotify.py | 2 +- src/mappers/DummyLibGenerator.py | 46 +++++++++++++++++++ src/mappers/FuzzyMapper.py | 54 +++++++++++++++-------- 9 files changed, 168 insertions(+), 22 deletions(-) create mode 100644 src/backends/mb/mbidFinder.py create mode 100644 src/mappers/DummyLibGenerator.py diff --git a/Flow.py b/Flow.py index 9cd7a04..fd10266 100644 --- a/Flow.py +++ b/Flow.py @@ -19,6 +19,8 @@ from src.backends.itunes.ParseItunesXml import ParseItunesXml from src.mappers.RemoteLibraryResolver import resolve_remote_tracks from src.StatGenerator import log_stats +import src.backends.mb.mbidFinder as mb +import src.mappers.DummyLibGenerator as libgen class Flow: spotify_library : Library = None @@ -63,6 +65,10 @@ class Flow: library_parsed = parser.parse(work_dir / "local" / "local_library.json") LibrarySaver(library_parsed).save(work_dir / "local_library.json") + def find_mbids(self): + mb.find_mbids(self.spotify_library) + mb.find_mbids(self.itunes_library) + self.save_libraries() def fetch_and_save_lyrics(self): """ @@ -111,6 +117,11 @@ class Flow: self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json") self.local_library = LibraryLoader().load(work_dir / "local_library.json") + def gen_fake_libs(self): + libgen.generate_dummy_library(self.spotify_library, work_dir / "spotify" / "dummylib") + libgen.generate_dummy_library(self.itunes_library, work_dir / "itunes" / "dummylib") + + def save_libraries(self): LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json") LibrarySaver(self.local_library).save(work_dir / "local_library.json") diff --git a/Gui.py b/Gui.py index 3c0ecc8..12d1144 100644 --- a/Gui.py +++ b/Gui.py @@ -16,9 +16,9 @@ def entity_to_row(entity: Entity): "PLay Count": entity.play_count, "Resolved": int(entity.resolved_percentage * 100), "AutoScore": entity.auto_score, - "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None, - "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None, - "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None, + # "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None, + # "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None, + # "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None, "LocalId": str(entity.local_id) if entity.local_id else None, "Path": str(entity.local_path) if entity.local_path else None, } diff --git a/main.py b/main.py index 26588d1..05a781d 100644 --- a/main.py +++ b/main.py @@ -26,6 +26,9 @@ def main(): flow.load_libraries() + # flow.gen_fake_libs() + # flow.find_mbids() + flow.map_local_to_spotify() flow.save_libraries() diff --git a/src/backends/itunes/ParseItunesXml.py b/src/backends/itunes/ParseItunesXml.py index 71eb2d7..4b0f417 100755 --- a/src/backends/itunes/ParseItunesXml.py +++ b/src/backends/itunes/ParseItunesXml.py @@ -187,6 +187,8 @@ class ParseItunesXml: for track in self.library.tracks: track.auto_score = track.play_count + self.library.auto_score() + return self.library diff --git a/src/backends/local/LocalLibraryIndexer.py b/src/backends/local/LocalLibraryIndexer.py index 6f132d4..08629f7 100644 --- a/src/backends/local/LocalLibraryIndexer.py +++ b/src/backends/local/LocalLibraryIndexer.py @@ -64,6 +64,17 @@ def get_artist(tags, fallback=None) -> list : return fallback +def get_mb_artist(tags, fallback=None) -> list : + artist_fields = ["TXXX:MusicBrainz Artist Id", "----:com.apple.iTunes:MusicBrainz Artist Id", "MUSICBRAINZ_ARTISTID"] + for field in artist_fields: + if field in tags and tags[field]: + values = tags.get(field) + if not isinstance(values, list): + return values.text + return values + + return fallback + def safe_tag_get(tags, key, index=0): try: value = tags.get(key, None) @@ -147,6 +158,7 @@ def update_local_library(root_dir, output_dir): tags = audio.tags track['artists'] = get_artist(tags) + # track['artists_mbid'] = get_mb_artist(tags) track['name'] = get_title(tags) track['album'] = get_album(tags) diff --git a/src/backends/mb/mbidFinder.py b/src/backends/mb/mbidFinder.py new file mode 100644 index 0000000..6ffcd09 --- /dev/null +++ b/src/backends/mb/mbidFinder.py @@ -0,0 +1,54 @@ +from src.Library import * + +import musicbrainzngs as mb +from tqdm import tqdm + +mb.set_useragent( + "resolve_spotify", + "1.0", + "ilusha.main@gmail.com" +) + +def find_mbid_by_tags(track: Track) -> str | None: + if not track.title or not track.artists: + return None + + artist_names = [a.title for a in track.artists if a.title] + if not artist_names: + return None + + query_parts = [ + f'recording:"{track.title}"', + f'artist:"{artist_names[0]}"' + ] + + query = " AND ".join(query_parts) + + try: + result = mb.search_recordings(query=query, limit=5) + except Exception: + return None + + recordings = result.get("recording-list", []) + if not recordings: + return None + + # Pick best-scored result + best = max( + recordings, + key=lambda r: int(r.get("score", 0)) + ) + + return best.get("id") + + +def find_mbids(library: Library): + tracks = library.tracks + + for track in tqdm(tracks, desc="Resolving MBIDs of " + library.name + " library", unit="track"): + if track.mbid: + continue + + track.mbid = find_mbid_by_tags(track) + if not track.mbid: + print(f"not found for track {track.title} {track.artists[0].title}") diff --git a/src/backends/spotify/ParseSpotify.py b/src/backends/spotify/ParseSpotify.py index cfca39d..fa5d80c 100644 --- a/src/backends/spotify/ParseSpotify.py +++ b/src/backends/spotify/ParseSpotify.py @@ -31,9 +31,9 @@ class Parser: self.library.update_cache() self.library.name = "Spotify" - self.library.auto_score() self.score_tracks() + self.library.auto_score() return self.library diff --git a/src/mappers/DummyLibGenerator.py b/src/mappers/DummyLibGenerator.py new file mode 100644 index 0000000..7aaa544 --- /dev/null +++ b/src/mappers/DummyLibGenerator.py @@ -0,0 +1,46 @@ +from mutagen.easyid3 import EasyID3 +from pydub import AudioSegment +from mutagen.easyid3 import EasyID3 + +from src.Library import * + +def safe_name(s: str) -> str: + return ( + s.replace("/", "_") + .replace("\\", "_") + .replace(":", "_") + .replace("*", "_") + .replace("?", "_") + .replace('"', "_") + .replace("<", "_") + .replace(">", "_") + .replace("|", "_") + .strip() + ) + +def generate_dummy_library(library: Library, root_path: Path): + root_path.mkdir(parents=True, exist_ok=True) + + for artist in library.artists: + artist_path = root_path / safe_name(artist.title or f"artist_{artist.id}") + artist_path.mkdir(exist_ok=True) + + for album in artist.albums: + album_path = artist_path / safe_name(album.title or f"album_{album.id}") + album_path.mkdir(exist_ok=True) + + for track in album.tracks: + track_title = track.title or f"track_{track.id}" + track_file = album_path / f"{safe_name(track_title)}.mp3" + + # duration_ms = max(track.duration_ms or 100, 100) + duration_ms = 10 + silence = AudioSegment.silent(duration=duration_ms) + silence.export(track_file, format="mp3") + + audio = EasyID3(track_file) + audio["title"] = track.title or "" + audio["artist"] = ", ".join([a.title or "" for a in track.artists]) + audio["album"] = album.title or "" + audio["tracknumber"] = str(album.tracks.index(track) + 1) + audio.save() diff --git a/src/mappers/FuzzyMapper.py b/src/mappers/FuzzyMapper.py index c6413d0..1dbbf75 100644 --- a/src/mappers/FuzzyMapper.py +++ b/src/mappers/FuzzyMapper.py @@ -4,6 +4,8 @@ from tqdm import tqdm from src.Library import * from src.helpers import * +import re +import string import unicodedata @@ -11,34 +13,49 @@ user_def_artist_map = { "donda" : "kanye west", } -def normalize(s): - return unicodedata.normalize("NFKC", s).replace("‐", "-") +user_def_album_map = { + "music" : "music sorry 4 da wait", +} def get_score(first: str, second: str) -> float: - first_norm = normalize(first).lower() - second_norm = normalize(second).lower() - return float(first_norm == second_norm) * 100 - return fuzz.token_set_ratio(first, second) + return float(first == second) * 100 + +def clean_title(title: str) -> str: + title = unicodedata.normalize("NFKC", title).replace("‐", "-") + title = unicodedata.normalize("NFKD", title) + title = re.sub(r"\([^)]*\)", "", title) + title = "".join( + c for c in title + if not unicodedata.category(c).startswith("P") + ) + title = title.translate(str.maketrans("", "", string.punctuation)) + title = title.lower() + return " ".join(title.split()) + +def find_mapping(first, second, mapping): + for key, value in mapping.items(): + if (first == key and second == value) or (first == value and second == key): + return True + return False def get_album_score(first: str, second: str) -> float: + first = clean_title(first) + second = clean_title(second) + + if find_mapping(first, second, user_def_album_map): + return 100 + return get_score(first, second) def get_track_score(first: str, second: str) -> float: - return get_score(first, second) + return get_score(clean_title(first), clean_title(second)) def get_artist_score(first: str, second: str) -> float: - if not first or not second: - return 0.0 + first = clean_title(first) + second = clean_title(second) - first_lower = first.lower() - second_lower = second.lower() - - for key, value in user_def_artist_map.items(): - key_lower = key.lower() - value_lower = value.lower() - if (first_lower == key_lower and second_lower == value_lower) or \ - (first_lower == value_lower and second_lower == key_lower): - return 100.0 + if find_mapping(first, second, user_def_artist_map): + return 100 return get_score(first, second) @@ -97,6 +114,7 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path): if remote_track.id not in artist_map or score > track_map[remote_track.id][1]: track_map[remote_track.id] = (local_track.id, score) remote_track.local_id = local_track.id + remote_track.local_path = local_track.local_path process_bar.update(1) From 16b55b0674d3cb8cc77b1759144079ea2cc439f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 00:03:10 +0300 Subject: [PATCH 2/4] spotify: update fetching to current Web API - OAuth: URL-encode authorize params (scope spaces were unencoded), bind loopback redirect server to 127.0.0.1 to match the redirect_uri per Spotify's 2025 loopback rules, capture refresh_token. - Search: URL-encode the query and drop the bogus track=1 param. - Pagination: follow the paging object's `next` field instead of manual offset/total bookkeeping (Spotify's recommended approach). - Fix invalid `raise "string"` in save_image_from_url. Co-Authored-By: Claude Opus 4.8 --- src/backends/spotify/SpotifyAuthenticator.py | 11 ++- src/backends/spotify/SpotifyWebAPI.py | 80 ++++++-------------- 2 files changed, 33 insertions(+), 58 deletions(-) diff --git a/src/backends/spotify/SpotifyAuthenticator.py b/src/backends/spotify/SpotifyAuthenticator.py index 5e27f7d..28cce0a 100644 --- a/src/backends/spotify/SpotifyAuthenticator.py +++ b/src/backends/spotify/SpotifyAuthenticator.py @@ -11,6 +11,7 @@ import os.path client_id = None client_secret = None token = None +refresh_token = None def retrieve_web_api_token(authorization_code): @@ -44,7 +45,10 @@ def retrieve_web_api_token(authorization_code): print(token_response.json()) raise ValueError - token = token_response.json()['access_token'] + global refresh_token + response_json = token_response.json() + token = response_json['access_token'] + refresh_token = response_json.get('refresh_token') class RequestHandler(BaseHTTPRequestHandler): @@ -73,7 +77,7 @@ class RequestHandler(BaseHTTPRequestHandler): def run_redirect_server(): - httpd = HTTPServer(('localhost', 8888), RequestHandler) + httpd = HTTPServer(('127.0.0.1', 8888), RequestHandler) print('Starting HTTP redirection server') httpd.serve_forever() @@ -88,8 +92,9 @@ def open_authenticate_page(): "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private", } - auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}" + auth_url = f"{authorize_url}?{urllib.parse.urlencode(params)}" webbrowser.open(auth_url) + print(f"If the browser did not open, visit:\n{auth_url}") def authenticate(id, secret): diff --git a/src/backends/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py index 8f4807f..73fe61b 100644 --- a/src/backends/spotify/SpotifyWebAPI.py +++ b/src/backends/spotify/SpotifyWebAPI.py @@ -1,4 +1,5 @@ import requests +import urllib.parse from PIL import Image from io import BytesIO @@ -11,7 +12,7 @@ token = '' def fetch(endpoint, method='GET', body=None): - url = f'https://api.spotify.com/{endpoint}' + url = endpoint if endpoint.startswith('http') else f'https://api.spotify.com/{endpoint}' headers = { 'Authorization': f'Bearer {token}', } @@ -21,83 +22,52 @@ def fetch(endpoint, method='GET', body=None): return response.json() +def fetch_paginated(endpoint): + """Collect every item of a paging object by following its `next` field, + which is Spotify's recommended way to page (more robust than tracking + offset/total manually).""" + items = [] + page = fetch(endpoint) + while True: + items += page.get('items', []) + next_url = page.get('next') + if not next_url: + break + page = fetch(next_url) + return items + + def find_song(song_pattern): - return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items'] + query = urllib.parse.quote(song_pattern) + return fetch(f"v1/search?type=track&limit=10&q={query}")['tracks']['items'] def fetch_playlists(user_id): print("Fetching playlists") - endpoint = f"v1/me/playlists" - pl_total = fetch(f'{endpoint}?limit=1')['total'] - pl_count = 0 - playlists = [] - - while pl_count < pl_total: - res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}') - playlists += res['items'] - pl_count += FETCH_STEP - - return playlists + return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}") def fetch_playlist_items(playlist_id): - endpoint = f"v1/playlists/{playlist_id}/tracks" - items = fetch(f'{endpoint}?fields=total')['total'] - loaded = 0 - tracks = [] - - while loaded < items: - res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}') - - tracks += res['items'] - loaded += FETCH_STEP - - return tracks + return fetch_paginated(f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}") def fetch_tracks(user_id): print("Fetching tracks") - endpoint = f"v1/me/tracks" - total = fetch(f"{endpoint}?limit=1&market=ES")['total'] - current = 0 - tracks = [] - - while total > current: - res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES") - tracks += res['items'] - current += FETCH_STEP - + tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES") save_data(tracks, "tracks") return tracks def fetch_tracks_top(): print("Fetching top tracks") - total = fetch(f'v1/me/top/tracks?fields=total')['total'] - current = 0 - tracks = [] - - while current < total: - res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term') - tracks += res['items'] - current += FETCH_STEP - + tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term") save_data(tracks, "top_tracks") return tracks def fetch_artists_top(): print("Fetching top artists") - - total = fetch(f'v1/me/top/artists?fields=total')['total'] - current = 0 - artists = [] - - while current < total: - res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term') - artists += res['items'] - current += FETCH_STEP - + artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term") save_data(artists, "top_artists") return artists @@ -140,7 +110,7 @@ def save_image_from_url(url, name, image_dir="playlist_covers"): response = requests.get(url) if response.status_code != 200: - raise "Cannot fetch the playlist cover" + raise RuntimeError("Cannot fetch the playlist cover") image = Image.open(BytesIO(response.content)) From 36058c318a6c3556855723aa57eccb1278af9bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 00:39:59 +0300 Subject: [PATCH 3/4] spotify: resilient fetching, progress bars, venv launcher - fetch() uses a retrying session (connection resets, 5xx, 429 Retry-After) with backoff, so a transient drop mid-fetch no longer aborts the run. - tqdm progress bars on all paginated fetches (driven by paging `total`). - Flow.fetch_spotify(): drop stray `self` from the @staticmethod. - gui.sh runs the venv's streamlit from the script dir; add requirements.txt. Co-Authored-By: Claude Opus 4.8 --- Flow.py | 2 +- gui.sh | 6 ++- main.py | 12 ++--- requirements.txt | 9 ++++ src/backends/spotify/SpotifyWebAPI.py | 68 ++++++++++++++++++++------- 5 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 requirements.txt diff --git a/Flow.py b/Flow.py index 9cd7a04..9ced6ae 100644 --- a/Flow.py +++ b/Flow.py @@ -26,7 +26,7 @@ class Flow: itunes_library : Library = None @staticmethod - def fetch_spotify(self): + def fetch_spotify(): fetch_all(client_id, client_secret, work_dir) @staticmethod diff --git a/gui.sh b/gui.sh index eb55739..9be5f9d 100755 --- a/gui.sh +++ b/gui.sh @@ -1 +1,5 @@ -streamlit run Gui.py +#!/usr/bin/env bash +# Run from the script's directory using the project virtualenv, +# so it works regardless of whether the venv is activated / on PATH. +cd "$(dirname "$0")" +exec .venv/bin/streamlit run Gui.py "$@" diff --git a/main.py b/main.py index 26588d1..68707c7 100644 --- a/main.py +++ b/main.py @@ -17,17 +17,17 @@ def test1(): def main(): flow = Flow.Flow() - # flow.fetch_spotify() + flow.fetch_spotify() # flow.fetch_local() flow.parse_spotify_library() - flow.parse_itunes_library() - flow.parse_local_library() + #flow.parse_itunes_library() + #flow.parse_local_library() - flow.load_libraries() + #flow.load_libraries() - flow.map_local_to_spotify() - flow.save_libraries() + #flow.map_local_to_spotify() + #flow.save_libraries() # flow.log_stats() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9a1d950 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +streamlit==1.58.0 +streamlit-aggrid==1.2.1.post2 +pandas==3.0.3 +Pillow==12.2.0 +mutagen==1.47.0 +rapidfuzz==3.14.5 +requests==2.34.2 +tqdm==4.68.2 +xmltodict==1.0.4 diff --git a/src/backends/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py index 73fe61b..e795d8a 100644 --- a/src/backends/spotify/SpotifyWebAPI.py +++ b/src/backends/spotify/SpotifyWebAPI.py @@ -1,7 +1,10 @@ import requests import urllib.parse +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from PIL import Image from io import BytesIO +from tqdm import tqdm from src.backends.spotify.SpotifyAuthenticator import authenticate @@ -9,6 +12,31 @@ from src.helpers import * FETCH_STEP = 50 token = '' +REQUEST_TIMEOUT = 30 + + +def _build_session(): + """Session that retries transient failures (dropped connections, 5xx) and + honours Spotify's 429 Retry-After, so a single reset mid-fetch doesn't + abort the whole run.""" + session = requests.Session() + retry = Retry( + total=6, + connect=6, + read=6, + backoff_factor=1.5, # sleeps ~0, 1.5, 3, 6, 12, 24s between attempts + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET", "POST"}), + respect_retry_after_header=True, + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +_session = _build_session() def fetch(endpoint, method='GET', body=None): @@ -17,23 +45,27 @@ def fetch(endpoint, method='GET', body=None): 'Authorization': f'Bearer {token}', } - response = requests.request(method, url, headers=headers, json=body) + response = _session.request(method, url, headers=headers, json=body, timeout=REQUEST_TIMEOUT) response.raise_for_status() return response.json() -def fetch_paginated(endpoint): +def fetch_paginated(endpoint, desc="Fetching", leave=True): """Collect every item of a paging object by following its `next` field, which is Spotify's recommended way to page (more robust than tracking - offset/total manually).""" + offset/total manually). Shows a tqdm progress bar driven by the paging + object's `total`.""" items = [] page = fetch(endpoint) - while True: - items += page.get('items', []) - next_url = page.get('next') - if not next_url: - break - page = fetch(next_url) + with tqdm(total=page.get('total'), desc=desc, unit="item", leave=leave) as pbar: + while True: + batch = page.get('items', []) + items += batch + pbar.update(len(batch)) + next_url = page.get('next') + if not next_url: + break + page = fetch(next_url) return items @@ -43,31 +75,31 @@ def find_song(song_pattern): def fetch_playlists(user_id): - print("Fetching playlists") - return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}") + return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}", desc="Fetching playlists") def fetch_playlist_items(playlist_id): - return fetch_paginated(f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}") + return fetch_paginated( + f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}", + desc="Fetching playlist tracks", + leave=False, + ) def fetch_tracks(user_id): - print("Fetching tracks") - tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES") + tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES", desc="Fetching liked tracks") save_data(tracks, "tracks") return tracks def fetch_tracks_top(): - print("Fetching top tracks") - tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term") + tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top tracks") save_data(tracks, "top_tracks") return tracks def fetch_artists_top(): - print("Fetching top artists") - artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term") + artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top artists") save_data(artists, "top_artists") return artists From 83f782d125b2c3801e783b4c44b295b8510ee7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 00:39:59 +0300 Subject: [PATCH 4/4] Remove hardcoded credentials; read them from the environment - Env.py now reads SPOTIFY_CLIENT_ID/SPOTIFY_CLIENT_SECRET (and optional path overrides) from the environment instead of hardcoding them. - Add .env.example and ignore .env / .venv. - Rewrite README with setup, usage and layout (drops the leaked secret). Note: secrets are also purged from prior history in this push. Co-Authored-By: Claude Opus 4.8 --- .env.example | 9 +++++ .gitignore | 2 + Env.py | 14 +++++-- README.md | 103 +++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..27f5a90 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +# Copy to `.env` and fill in, then `source .env` before running. +# Get these from https://developer.spotify.com/dashboard (your app's settings). +# Register `http://127.0.0.1:8888` as a Redirect URI in that app. +export SPOTIFY_CLIENT_ID=your_client_id_here +export SPOTIFY_CLIENT_SECRET=your_client_secret_here + +# Optional overrides: +# export MUSIC_LIBRARY_PATH=~/Music +# export WORK_DIR=./work_dir/new diff --git a/.gitignore b/.gitignore index 880f9c3..7b75b45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ prefetched* __pycache__ .idea secret +.env +.venv work_dir old tmp \ No newline at end of file diff --git a/Env.py b/Env.py index 6549c6b..c609347 100644 --- a/Env.py +++ b/Env.py @@ -1,7 +1,13 @@ +import os from pathlib import Path -local_library_path = Path("/home/auser/Music/") -work_dir = Path("./work_dir/new") +# Local config. Paths can be overridden via environment variables. +local_library_path = Path(os.environ.get("MUSIC_LIBRARY_PATH", "~/Music")).expanduser() +work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new")) -client_id = '***REDACTED***' -client_secret = "***REDACTED***" +# Spotify app credentials. NEVER hardcode these here — set them in the +# environment (see .env.example) so they don't end up in version control. +# export SPOTIFY_CLIENT_ID=... +# export SPOTIFY_CLIENT_SECRET=... +client_id = os.environ.get("SPOTIFY_CLIENT_ID") +client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET") diff --git a/README.md b/README.md index 68b9ba5..62d8bd7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,100 @@ -# spotifyFetcher -loads all data through web api
-client secret - ***REDACTED*** +# MusicIndexer + +Pull your Spotify library and listening stats via the Spotify Web API, index a +local music library (audio tags + iTunes XML), and match the two so you can see +which tracks you own, what's missing, and your top artists/tracks/playlists — in +a Streamlit GUI. + +## Features + +- **Spotify backend** — OAuth (Authorization Code) login, then fetch the user + profile, liked tracks, all playlists and their tracks, long-term top tracks + and top artists, and playlist/profile cover art. Resilient paginated fetching + (follows `next`, retries dropped connections, honours `429` rate limits) with + progress bars. +- **Local backend** — index a folder of audio files via `mutagen` tags. +- **iTunes backend** — parse an iTunes Library XML export. +- **Matching** — map local tracks to Spotify tracks (fuzzy matcher; ISRC/MBID + matching planned). +- **Stats & lyrics** — generate listening stats and fetch lyrics. +- **GUI** — browse libraries as artist → album → track tables. + +## Setup + +Requires Python 3.13+. + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +### Credentials + +Create an app at https://developer.spotify.com/dashboard and add +`http://127.0.0.1:8888` as a **Redirect URI**. Credentials are read from the +environment — never commit them. + +```bash +cp .env.example .env +# edit .env with your client id/secret +source .env +``` + +`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (plus optional +`MUSIC_LIBRARY_PATH` and `WORK_DIR`) from the environment. + +## Usage + +Edit `main.py` to uncomment the steps you want, then run: + +```bash +python main.py +``` + +The flow steps (see `Flow.py`) include: + +| Step | What it does | +|------|--------------| +| `fetch_spotify()` | OAuth login + download all Spotify data into `work_dir/spotify/` | +| `fetch_local()` | index local audio files into `work_dir/local/` | +| `parse_spotify_library()` / `parse_itunes_library()` / `parse_local_library()` | build normalized libraries | +| `load_libraries()` / `save_libraries()` | (de)serialize libraries to `work_dir` | +| `map_local_to_spotify()` | match local tracks against the Spotify library | +| `fetch_and_save_lyrics()` | fetch lyrics for tracks | +| `log_stats()` | generate stats | + +On first `fetch_spotify()` a browser opens for the Spotify login; a local server +on `127.0.0.1:8888` catches the redirect and exchanges the code for a token. + +### GUI + +```bash +./gui.sh # runs: .venv/bin/streamlit run Gui.py +``` + +## Layout + +``` +main.py entry point (CLI flow) +Gui.py / gui.sh Streamlit GUI +Flow.py orchestrates fetch / parse / map / stats +Env.py config (reads credentials from the environment) +src/ + Library.py data model (Library / Artist / Album / Track) + LibrarySerializer.py load/save libraries + backends/ + spotify/ Web API client, OAuth, parser + local/ local-file indexer, parser, playlist generator + itunes/ iTunes XML parser + navidrome/ Navidrome backend + mappers/ local↔Spotify matching (fuzzy / search) + lyrics/ lyrics fetching +work_dir/ fetched data and serialized libraries (gitignored) +``` + +## Notes + +- Data is cached as JSON under `work_dir/` (gitignored); re-running reuses it. +- The deprecated Spotify endpoints (audio-features, recommendations, etc.) are + not used — only currently-supported endpoints.