diff --git a/Flow.py b/Flow.py index 9cd7a04..9ced6ae 100644 --- a/Flow.py +++ b/Flow.py @@ -26,7 +26,7 @@ class Flow: itunes_library : Library = None @staticmethod - def fetch_spotify(self): + def fetch_spotify(): fetch_all(client_id, client_secret, work_dir) @staticmethod diff --git a/gui.sh b/gui.sh index eb55739..9be5f9d 100755 --- a/gui.sh +++ b/gui.sh @@ -1 +1,5 @@ -streamlit run Gui.py +#!/usr/bin/env bash +# Run from the script's directory using the project virtualenv, +# so it works regardless of whether the venv is activated / on PATH. +cd "$(dirname "$0")" +exec .venv/bin/streamlit run Gui.py "$@" diff --git a/main.py b/main.py index 26588d1..68707c7 100644 --- a/main.py +++ b/main.py @@ -17,17 +17,17 @@ def test1(): def main(): flow = Flow.Flow() - # flow.fetch_spotify() + flow.fetch_spotify() # flow.fetch_local() flow.parse_spotify_library() - flow.parse_itunes_library() - flow.parse_local_library() + #flow.parse_itunes_library() + #flow.parse_local_library() - flow.load_libraries() + #flow.load_libraries() - flow.map_local_to_spotify() - flow.save_libraries() + #flow.map_local_to_spotify() + #flow.save_libraries() # flow.log_stats() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9a1d950 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +streamlit==1.58.0 +streamlit-aggrid==1.2.1.post2 +pandas==3.0.3 +Pillow==12.2.0 +mutagen==1.47.0 +rapidfuzz==3.14.5 +requests==2.34.2 +tqdm==4.68.2 +xmltodict==1.0.4 diff --git a/src/backends/spotify/SpotifyWebAPI.py b/src/backends/spotify/SpotifyWebAPI.py index 73fe61b..e795d8a 100644 --- a/src/backends/spotify/SpotifyWebAPI.py +++ b/src/backends/spotify/SpotifyWebAPI.py @@ -1,7 +1,10 @@ import requests import urllib.parse +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from PIL import Image from io import BytesIO +from tqdm import tqdm from src.backends.spotify.SpotifyAuthenticator import authenticate @@ -9,6 +12,31 @@ from src.helpers import * FETCH_STEP = 50 token = '' +REQUEST_TIMEOUT = 30 + + +def _build_session(): + """Session that retries transient failures (dropped connections, 5xx) and + honours Spotify's 429 Retry-After, so a single reset mid-fetch doesn't + abort the whole run.""" + session = requests.Session() + retry = Retry( + total=6, + connect=6, + read=6, + backoff_factor=1.5, # sleeps ~0, 1.5, 3, 6, 12, 24s between attempts + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET", "POST"}), + respect_retry_after_header=True, + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +_session = _build_session() def fetch(endpoint, method='GET', body=None): @@ -17,23 +45,27 @@ def fetch(endpoint, method='GET', body=None): 'Authorization': f'Bearer {token}', } - response = requests.request(method, url, headers=headers, json=body) + response = _session.request(method, url, headers=headers, json=body, timeout=REQUEST_TIMEOUT) response.raise_for_status() return response.json() -def fetch_paginated(endpoint): +def fetch_paginated(endpoint, desc="Fetching", leave=True): """Collect every item of a paging object by following its `next` field, which is Spotify's recommended way to page (more robust than tracking - offset/total manually).""" + offset/total manually). Shows a tqdm progress bar driven by the paging + object's `total`.""" items = [] page = fetch(endpoint) - while True: - items += page.get('items', []) - next_url = page.get('next') - if not next_url: - break - page = fetch(next_url) + with tqdm(total=page.get('total'), desc=desc, unit="item", leave=leave) as pbar: + while True: + batch = page.get('items', []) + items += batch + pbar.update(len(batch)) + next_url = page.get('next') + if not next_url: + break + page = fetch(next_url) return items @@ -43,31 +75,31 @@ def find_song(song_pattern): def fetch_playlists(user_id): - print("Fetching playlists") - return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}") + return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}", desc="Fetching playlists") def fetch_playlist_items(playlist_id): - return fetch_paginated(f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}") + return fetch_paginated( + f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}", + desc="Fetching playlist tracks", + leave=False, + ) def fetch_tracks(user_id): - print("Fetching tracks") - tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES") + tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES", desc="Fetching liked tracks") save_data(tracks, "tracks") return tracks def fetch_tracks_top(): - print("Fetching top tracks") - tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term") + tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top tracks") save_data(tracks, "top_tracks") return tracks def fetch_artists_top(): - print("Fetching top artists") - artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term") + artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top artists") save_data(artists, "top_artists") return artists