spotify: resilient fetching, progress bars, venv launcher

- fetch() uses a retrying session (connection resets, 5xx, 429 Retry-After)
  with backoff, so a transient drop mid-fetch no longer aborts the run.
- tqdm progress bars on all paginated fetches (driven by paging `total`).
- Flow.fetch_spotify(): drop stray `self` from the @staticmethod.
- gui.sh runs the venv's streamlit from the script dir; add requirements.txt.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Flow.py

@@ -26,7 +26,7 @@ class Flow:
     itunes_library : Library = None
 
     @staticmethod
-    def fetch_spotify(self):
+    def fetch_spotify():
         fetch_all(client_id, client_secret, work_dir)
 
     @staticmethod

gui.sh

@@ -1 +1,5 @@
-streamlit run Gui.py
+#!/usr/bin/env bash
+# Run from the script's directory using the project virtualenv,
+# so it works regardless of whether the venv is activated / on PATH.
+cd "$(dirname "$0")"
+exec .venv/bin/streamlit run Gui.py "$@"

main.py

@@ -17,17 +17,17 @@ def test1():
 def main():
     flow = Flow.Flow()
 
-    # flow.fetch_spotify()
+    flow.fetch_spotify()
     # flow.fetch_local()
 
     flow.parse_spotify_library()
-    flow.parse_itunes_library()
-    flow.parse_local_library()
+    #flow.parse_itunes_library()
+    #flow.parse_local_library()
 
-    flow.load_libraries()
+    #flow.load_libraries()
 
-    flow.map_local_to_spotify()
-    flow.save_libraries()
+    #flow.map_local_to_spotify()
+    #flow.save_libraries()
 
     # flow.log_stats()
 

requirements.txt

@@ -0,0 +1,9 @@
+streamlit==1.58.0
+streamlit-aggrid==1.2.1.post2
+pandas==3.0.3
+Pillow==12.2.0
+mutagen==1.47.0
+rapidfuzz==3.14.5
+requests==2.34.2
+tqdm==4.68.2
+xmltodict==1.0.4

src/backends/spotify/SpotifyWebAPI.py

@@ -1,7 +1,10 @@
 import requests
 import urllib.parse
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 from PIL import Image
 from io import BytesIO
+from tqdm import tqdm
 
 from src.backends.spotify.SpotifyAuthenticator import authenticate
 
@@ -9,6 +12,31 @@ from src.helpers import *
 
 FETCH_STEP = 50
 token = ''
+REQUEST_TIMEOUT = 30
+
+
+def _build_session():
+    """Session that retries transient failures (dropped connections, 5xx) and
+    honours Spotify's 429 Retry-After, so a single reset mid-fetch doesn't
+    abort the whole run."""
+    session = requests.Session()
+    retry = Retry(
+        total=6,
+        connect=6,
+        read=6,
+        backoff_factor=1.5,  # sleeps ~0, 1.5, 3, 6, 12, 24s between attempts
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=frozenset({"GET", "POST"}),
+        respect_retry_after_header=True,
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
+
+_session = _build_session()
 
 
 def fetch(endpoint, method='GET', body=None):
@@ -17,19 +45,23 @@ def fetch(endpoint, method='GET', body=None):
         'Authorization': f'Bearer {token}',
     }
 
-    response = requests.request(method, url, headers=headers, json=body)
+    response = _session.request(method, url, headers=headers, json=body, timeout=REQUEST_TIMEOUT)
     response.raise_for_status()
     return response.json()
 
 
-def fetch_paginated(endpoint):
+def fetch_paginated(endpoint, desc="Fetching", leave=True):
     """Collect every item of a paging object by following its `next` field,
     which is Spotify's recommended way to page (more robust than tracking
-    offset/total manually)."""
+    offset/total manually). Shows a tqdm progress bar driven by the paging
+    object's `total`."""
     items = []
     page = fetch(endpoint)
+    with tqdm(total=page.get('total'), desc=desc, unit="item", leave=leave) as pbar:
         while True:
-        items += page.get('items', [])
+            batch = page.get('items', [])
+            items += batch
+            pbar.update(len(batch))
             next_url = page.get('next')
             if not next_url:
                 break
@@ -43,31 +75,31 @@ def find_song(song_pattern):
 
 
 def fetch_playlists(user_id):
-    print("Fetching playlists")
-    return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}")
+    return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}", desc="Fetching playlists")
 
 
 def fetch_playlist_items(playlist_id):
-    return fetch_paginated(f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}")
+    return fetch_paginated(
+        f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}",
+        desc="Fetching playlist tracks",
+        leave=False,
+    )
 
 
 def fetch_tracks(user_id):
-    print("Fetching tracks")
-    tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES")
+    tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES", desc="Fetching liked tracks")
     save_data(tracks, "tracks")
     return tracks
 
 
 def fetch_tracks_top():
-    print("Fetching top tracks")
-    tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term")
+    tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top tracks")
     save_data(tracks, "top_tracks")
     return tracks
 
 
 def fetch_artists_top():
-    print("Fetching top artists")
-    artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term")
+    artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top artists")
     save_data(artists, "top_artists")
     return artists