Remove hardcoded credentials; read them from the environment

- Env.py now reads SPOTIFY_CLIENT_ID/SPOTIFY_CLIENT_SECRET (and optional
  path overrides) from the environment instead of hardcoding them.
- Add .env.example and ignore .env / .venv.
- Rewrite README with setup, usage and layout (drops the leaked secret).

Note: secrets are also purged from prior history in this push.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -0,0 +1,9 @@
+# Copy to `.env` and fill in, then `source .env` before running.
+# Get these from https://developer.spotify.com/dashboard (your app's settings).
+# Register `http://127.0.0.1:8888` as a Redirect URI in that app.
+export SPOTIFY_CLIENT_ID=your_client_id_here
+export SPOTIFY_CLIENT_SECRET=your_client_secret_here
+
+# Optional overrides:
+# export MUSIC_LIBRARY_PATH=~/Music
+# export WORK_DIR=./work_dir/new

.gitignore

@@ -2,6 +2,8 @@ prefetched*
 __pycache__
 .idea
 secret
+.env
+.venv
 work_dir
 old
 tmp

Env.py

@@ -1,7 +1,13 @@
+import os
 from pathlib import Path
 
-local_library_path = Path("/home/auser/Music/")
-work_dir = Path("./work_dir/new")
+# Local config. Paths can be overridden via environment variables.
+local_library_path = Path(os.environ.get("MUSIC_LIBRARY_PATH", "~/Music")).expanduser()
+work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new"))
 
-client_id = '***REDACTED***'
-client_secret = "***REDACTED***"
+# Spotify app credentials. NEVER hardcode these here — set them in the
+# environment (see .env.example) so they don't end up in version control.
+#   export SPOTIFY_CLIENT_ID=...
+#   export SPOTIFY_CLIENT_SECRET=...
+client_id = os.environ.get("SPOTIFY_CLIENT_ID")
+client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET")

Flow.py

@@ -19,8 +19,6 @@ from src.backends.itunes.ParseItunesXml import ParseItunesXml
 from src.mappers.RemoteLibraryResolver import resolve_remote_tracks
 from src.StatGenerator import log_stats
 
-import src.backends.mb.mbidFinder as mb
-import src.mappers.DummyLibGenerator as libgen
 
 class Flow:
     spotify_library : Library = None
@@ -28,7 +26,7 @@ class Flow:
     itunes_library : Library = None
 
     @staticmethod
-    def fetch_spotify(self):
+    def fetch_spotify():
         fetch_all(client_id, client_secret, work_dir)
 
     @staticmethod
@@ -65,10 +63,6 @@ class Flow:
         library_parsed = parser.parse(work_dir / "local" / "local_library.json")
         LibrarySaver(library_parsed).save(work_dir / "local_library.json")
 
-    def find_mbids(self):
-        mb.find_mbids(self.spotify_library)
-        mb.find_mbids(self.itunes_library)
-        self.save_libraries()
 
     def fetch_and_save_lyrics(self):
         """
@@ -117,11 +111,6 @@ class Flow:
         self.itunes_library = LibraryLoader().load(work_dir / "itunes_library.json")
         self.local_library = LibraryLoader().load(work_dir / "local_library.json")
 
-    def gen_fake_libs(self):
-        libgen.generate_dummy_library(self.spotify_library, work_dir / "spotify" / "dummylib")
-        libgen.generate_dummy_library(self.itunes_library, work_dir / "itunes" / "dummylib")
-
-
     def save_libraries(self):
         LibrarySaver(self.spotify_library).save(work_dir / "spotify_library.json")
         LibrarySaver(self.local_library).save(work_dir / "local_library.json")

Gui.py

@@ -16,9 +16,9 @@ def entity_to_row(entity: Entity):
         "PLay Count": entity.play_count,
         "Resolved": int(entity.resolved_percentage * 100),
         "AutoScore": entity.auto_score,
-        # "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
-        # "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
-        # "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
+        "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
+        "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
+        "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
         "LocalId": str(entity.local_id) if entity.local_id else None,
         "Path": str(entity.local_path) if entity.local_path else None,
     }

README.md

@@ -1,3 +1,100 @@
-# spotifyFetcher
-loads all data through web api<br>
-client secret - ***REDACTED***
+# MusicIndexer
+
+Pull your Spotify library and listening stats via the Spotify Web API, index a
+local music library (audio tags + iTunes XML), and match the two so you can see
+which tracks you own, what's missing, and your top artists/tracks/playlists — in
+a Streamlit GUI.
+
+## Features
+
+- **Spotify backend** — OAuth (Authorization Code) login, then fetch the user
+  profile, liked tracks, all playlists and their tracks, long-term top tracks
+  and top artists, and playlist/profile cover art. Resilient paginated fetching
+  (follows `next`, retries dropped connections, honours `429` rate limits) with
+  progress bars.
+- **Local backend** — index a folder of audio files via `mutagen` tags.
+- **iTunes backend** — parse an iTunes Library XML export.
+- **Matching** — map local tracks to Spotify tracks (fuzzy matcher; ISRC/MBID
+  matching planned).
+- **Stats & lyrics** — generate listening stats and fetch lyrics.
+- **GUI** — browse libraries as artist → album → track tables.
+
+## Setup
+
+Requires Python 3.13+.
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Credentials
+
+Create an app at https://developer.spotify.com/dashboard and add
+`http://127.0.0.1:8888` as a **Redirect URI**. Credentials are read from the
+environment — never commit them.
+
+```bash
+cp .env.example .env
+# edit .env with your client id/secret
+source .env
+```
+
+`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (plus optional
+`MUSIC_LIBRARY_PATH` and `WORK_DIR`) from the environment.
+
+## Usage
+
+Edit `main.py` to uncomment the steps you want, then run:
+
+```bash
+python main.py
+```
+
+The flow steps (see `Flow.py`) include:
+
+| Step | What it does |
+|------|--------------|
+| `fetch_spotify()` | OAuth login + download all Spotify data into `work_dir/spotify/` |
+| `fetch_local()` | index local audio files into `work_dir/local/` |
+| `parse_spotify_library()` / `parse_itunes_library()` / `parse_local_library()` | build normalized libraries |
+| `load_libraries()` / `save_libraries()` | (de)serialize libraries to `work_dir` |
+| `map_local_to_spotify()` | match local tracks against the Spotify library |
+| `fetch_and_save_lyrics()` | fetch lyrics for tracks |
+| `log_stats()` | generate stats |
+
+On first `fetch_spotify()` a browser opens for the Spotify login; a local server
+on `127.0.0.1:8888` catches the redirect and exchanges the code for a token.
+
+### GUI
+
+```bash
+./gui.sh        # runs: .venv/bin/streamlit run Gui.py
+```
+
+## Layout
+
+```
+main.py                 entry point (CLI flow)
+Gui.py / gui.sh         Streamlit GUI
+Flow.py                 orchestrates fetch / parse / map / stats
+Env.py                  config (reads credentials from the environment)
+src/
+  Library.py            data model (Library / Artist / Album / Track)
+  LibrarySerializer.py  load/save libraries
+  backends/
+    spotify/            Web API client, OAuth, parser
+    local/              local-file indexer, parser, playlist generator
+    itunes/             iTunes XML parser
+    navidrome/          Navidrome backend
+  mappers/              local↔Spotify matching (fuzzy / search)
+  lyrics/               lyrics fetching
+work_dir/               fetched data and serialized libraries (gitignored)
+```
+
+## Notes
+
+- Data is cached as JSON under `work_dir/` (gitignored); re-running reuses it.
+- The deprecated Spotify endpoints (audio-features, recommendations, etc.) are
+  not used — only currently-supported endpoints.

gui.sh

@@ -1 +1,5 @@
-streamlit run Gui.py
+#!/usr/bin/env bash
+# Run from the script's directory using the project virtualenv,
+# so it works regardless of whether the venv is activated / on PATH.
+cd "$(dirname "$0")"
+exec .venv/bin/streamlit run Gui.py "$@"

main.py

@@ -17,20 +17,17 @@ def test1():
 def main():
     flow = Flow.Flow()
 
-    # flow.fetch_spotify()
+    flow.fetch_spotify()
     # flow.fetch_local()
 
     flow.parse_spotify_library()
-    flow.parse_itunes_library()
-    flow.parse_local_library()
+    #flow.parse_itunes_library()
+    #flow.parse_local_library()
 
-    flow.load_libraries()
+    #flow.load_libraries()
 
-    # flow.gen_fake_libs()
-    # flow.find_mbids()
-
-    flow.map_local_to_spotify()
-    flow.save_libraries()
+    #flow.map_local_to_spotify()
+    #flow.save_libraries()
 
     # flow.log_stats()
 

requirements.txt

@@ -0,0 +1,9 @@
+streamlit==1.58.0
+streamlit-aggrid==1.2.1.post2
+pandas==3.0.3
+Pillow==12.2.0
+mutagen==1.47.0
+rapidfuzz==3.14.5
+requests==2.34.2
+tqdm==4.68.2
+xmltodict==1.0.4

src/backends/itunes/ParseItunesXml.py

@@ -187,8 +187,6 @@ class ParseItunesXml:
         for track in self.library.tracks:
             track.auto_score = track.play_count
 
-        self.library.auto_score()
-
         return self.library
 
 

src/backends/local/LocalLibraryIndexer.py

@@ -64,17 +64,6 @@ def get_artist(tags, fallback=None) -> list :
 
     return fallback
 
-def get_mb_artist(tags, fallback=None) -> list :
-    artist_fields = ["TXXX:MusicBrainz Artist Id", "----:com.apple.iTunes:MusicBrainz Artist Id", "MUSICBRAINZ_ARTISTID"]
-    for field in artist_fields:
-        if field in tags and tags[field]:
-            values = tags.get(field)
-            if not isinstance(values, list):
-                return values.text
-            return values
-
-    return fallback
-
 def safe_tag_get(tags, key, index=0):
     try:
         value = tags.get(key, None)
@@ -158,7 +147,6 @@ def update_local_library(root_dir, output_dir):
                     tags = audio.tags
 
                     track['artists'] = get_artist(tags)
-                    # track['artists_mbid'] = get_mb_artist(tags)
                     track['name'] = get_title(tags)
                     track['album'] = get_album(tags)
 

src/backends/mb/mbidFinder.py

@@ -1,54 +0,0 @@
-from src.Library import *
-
-import musicbrainzngs as mb
-from tqdm import tqdm
-
-mb.set_useragent(
-    "resolve_spotify",
-    "1.0",
-    "ilusha.main@gmail.com"
-)
-
-def find_mbid_by_tags(track: Track) -> str | None:
-    if not track.title or not track.artists:
-        return None
-
-    artist_names = [a.title for a in track.artists if a.title]
-    if not artist_names:
-        return None
-
-    query_parts = [
-        f'recording:"{track.title}"',
-        f'artist:"{artist_names[0]}"'
-    ]
-
-    query = " AND ".join(query_parts)
-
-    try:
-        result = mb.search_recordings(query=query, limit=5)
-    except Exception:
-        return None
-
-    recordings = result.get("recording-list", [])
-    if not recordings:
-        return None
-
-    # Pick best-scored result
-    best = max(
-        recordings,
-        key=lambda r: int(r.get("score", 0))
-    )
-
-    return best.get("id")
-
-
-def find_mbids(library: Library):
-    tracks = library.tracks
-
-    for track in tqdm(tracks, desc="Resolving MBIDs of " + library.name + " library", unit="track"):
-        if track.mbid:
-            continue
-
-        track.mbid = find_mbid_by_tags(track)
-        if not track.mbid:
-            print(f"not found for track {track.title} {track.artists[0].title}")

src/backends/spotify/ParseSpotify.py

@@ -31,9 +31,9 @@ class Parser:
 
         self.library.update_cache()
         self.library.name = "Spotify"
+        self.library.auto_score()
 
         self.score_tracks()
-        self.library.auto_score()
 
         return self.library
 

src/backends/spotify/SpotifyAuthenticator.py

@@ -11,6 +11,7 @@ import os.path
 client_id = None
 client_secret = None
 token = None
+refresh_token = None
 
 
 def retrieve_web_api_token(authorization_code):
@@ -44,7 +45,10 @@ def retrieve_web_api_token(authorization_code):
         print(token_response.json())
         raise ValueError
 
-    token = token_response.json()['access_token']
+    global refresh_token
+    response_json = token_response.json()
+    token = response_json['access_token']
+    refresh_token = response_json.get('refresh_token')
 
 
 class RequestHandler(BaseHTTPRequestHandler):
@@ -73,7 +77,7 @@ class RequestHandler(BaseHTTPRequestHandler):
 
 
 def run_redirect_server():
-    httpd = HTTPServer(('localhost', 8888), RequestHandler)
+    httpd = HTTPServer(('127.0.0.1', 8888), RequestHandler)
     print('Starting HTTP redirection server')
     httpd.serve_forever()
 
@@ -88,8 +92,9 @@ def open_authenticate_page():
         "scope": "user-top-read playlist-read-collaborative user-read-email user-library-read user-read-private playlist-read-private",
     }
 
-    auth_url = f"{authorize_url}?client_id={params['client_id']}&response_type={params['response_type']}&redirect_uri={params['redirect_uri']}&scope={params['scope']}"
+    auth_url = f"{authorize_url}?{urllib.parse.urlencode(params)}"
     webbrowser.open(auth_url)
+    print(f"If the browser did not open, visit:\n{auth_url}")
 
 
 def authenticate(id, secret):

src/backends/spotify/SpotifyWebAPI.py

@@ -1,6 +1,10 @@
 import requests
+import urllib.parse
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 from PIL import Image
 from io import BytesIO
+from tqdm import tqdm
 
 from src.backends.spotify.SpotifyAuthenticator import authenticate
 
@@ -8,96 +12,94 @@ from src.helpers import *
 
 FETCH_STEP = 50
 token = ''
+REQUEST_TIMEOUT = 30
+
+
+def _build_session():
+    """Session that retries transient failures (dropped connections, 5xx) and
+    honours Spotify's 429 Retry-After, so a single reset mid-fetch doesn't
+    abort the whole run."""
+    session = requests.Session()
+    retry = Retry(
+        total=6,
+        connect=6,
+        read=6,
+        backoff_factor=1.5,  # sleeps ~0, 1.5, 3, 6, 12, 24s between attempts
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=frozenset({"GET", "POST"}),
+        respect_retry_after_header=True,
+        raise_on_status=False,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
+
+_session = _build_session()
 
 
 def fetch(endpoint, method='GET', body=None):
-    url = f'https://api.spotify.com/{endpoint}'
+    url = endpoint if endpoint.startswith('http') else f'https://api.spotify.com/{endpoint}'
     headers = {
         'Authorization': f'Bearer {token}',
     }
 
-    response = requests.request(method, url, headers=headers, json=body)
+    response = _session.request(method, url, headers=headers, json=body, timeout=REQUEST_TIMEOUT)
     response.raise_for_status()
     return response.json()
 
 
+def fetch_paginated(endpoint, desc="Fetching", leave=True):
+    """Collect every item of a paging object by following its `next` field,
+    which is Spotify's recommended way to page (more robust than tracking
+    offset/total manually). Shows a tqdm progress bar driven by the paging
+    object's `total`."""
+    items = []
+    page = fetch(endpoint)
+    with tqdm(total=page.get('total'), desc=desc, unit="item", leave=leave) as pbar:
+        while True:
+            batch = page.get('items', [])
+            items += batch
+            pbar.update(len(batch))
+            next_url = page.get('next')
+            if not next_url:
+                break
+            page = fetch(next_url)
+    return items
+
+
 def find_song(song_pattern):
-    return fetch(f"v1/search/?type=track&track=1&q={song_pattern}")['tracks']['items']
+    query = urllib.parse.quote(song_pattern)
+    return fetch(f"v1/search?type=track&limit=10&q={query}")['tracks']['items']
 
 
 def fetch_playlists(user_id):
-    print("Fetching playlists")
-    endpoint = f"v1/me/playlists"
-    pl_total = fetch(f'{endpoint}?limit=1')['total']
-    pl_count = 0
-    playlists = []
-
-    while pl_count < pl_total:
-        res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={pl_count}')
-        playlists += res['items']
-        pl_count += FETCH_STEP
-
-    return playlists
+    return fetch_paginated(f"v1/me/playlists?limit={FETCH_STEP}", desc="Fetching playlists")
 
 
 def fetch_playlist_items(playlist_id):
-    endpoint = f"v1/playlists/{playlist_id}/tracks"
-    items = fetch(f'{endpoint}?fields=total')['total']
-    loaded = 0
-    tracks = []
-
-    while loaded < items:
-        res = fetch(f'{endpoint}?limit={FETCH_STEP}&offset={loaded}')
-        
-        tracks += res['items']
-        loaded += FETCH_STEP
-
-    return tracks
+    return fetch_paginated(
+        f"v1/playlists/{playlist_id}/tracks?limit={FETCH_STEP}",
+        desc="Fetching playlist tracks",
+        leave=False,
+    )
 
 
 def fetch_tracks(user_id):
-    print("Fetching tracks")
-    endpoint = f"v1/me/tracks"
-    total = fetch(f"{endpoint}?limit=1&market=ES")['total']
-    current = 0
-    tracks = []
-
-    while total > current:
-        res = fetch(f"{endpoint}?limit={FETCH_STEP}&offset={current}&market=ES")
-        tracks += res['items']
-        current += FETCH_STEP
-
+    tracks = fetch_paginated(f"v1/me/tracks?limit={FETCH_STEP}&market=ES", desc="Fetching liked tracks")
     save_data(tracks, "tracks")
     return tracks
 
 
 def fetch_tracks_top():
-    print("Fetching top tracks")
-    total = fetch(f'v1/me/top/tracks?fields=total')['total']
-    current = 0
-    tracks = []
-
-    while current < total:
-        res = fetch(f'v1/me/top/tracks?limit={FETCH_STEP}&offset={current}&time_range=long_term')
-        tracks += res['items']
-        current += FETCH_STEP
-
+    tracks = fetch_paginated(f"v1/me/top/tracks?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top tracks")
     save_data(tracks, "top_tracks")
     return tracks
 
 
 def fetch_artists_top():
-    print("Fetching top artists")
-
-    total = fetch(f'v1/me/top/artists?fields=total')['total']
-    current = 0
-    artists = []
-
-    while current < total:
-        res = fetch(f'v1/me/top/artists?limit={FETCH_STEP}&offset={current}&time_range=long_term')
-        artists += res['items']
-        current += FETCH_STEP
-
+    artists = fetch_paginated(f"v1/me/top/artists?limit={FETCH_STEP}&time_range=long_term", desc="Fetching top artists")
     save_data(artists, "top_artists")
     return artists
 
@@ -140,7 +142,7 @@ def save_image_from_url(url, name, image_dir="playlist_covers"):
     response = requests.get(url)
 
     if response.status_code != 200:
-        raise "Cannot fetch the playlist cover"
+        raise RuntimeError("Cannot fetch the playlist cover")
 
     image = Image.open(BytesIO(response.content))
 

src/mappers/DummyLibGenerator.py

@@ -1,46 +0,0 @@
-from mutagen.easyid3 import EasyID3
-from pydub import AudioSegment
-from mutagen.easyid3 import EasyID3
-
-from src.Library import *
-
-def safe_name(s: str) -> str:
-    return (
-        s.replace("/", "_")
-         .replace("\\", "_")
-         .replace(":", "_")
-         .replace("*", "_")
-         .replace("?", "_")
-         .replace('"', "_")
-         .replace("<", "_")
-         .replace(">", "_")
-         .replace("|", "_")
-         .strip()
-    )
-
-def generate_dummy_library(library: Library, root_path: Path):
-    root_path.mkdir(parents=True, exist_ok=True)
-
-    for artist in library.artists:
-        artist_path = root_path / safe_name(artist.title or f"artist_{artist.id}")
-        artist_path.mkdir(exist_ok=True)
-
-        for album in artist.albums:
-            album_path = artist_path / safe_name(album.title or f"album_{album.id}")
-            album_path.mkdir(exist_ok=True)
-
-            for track in album.tracks:
-                track_title = track.title or f"track_{track.id}"
-                track_file = album_path / f"{safe_name(track_title)}.mp3"
-
-                # duration_ms = max(track.duration_ms or 100, 100)
-                duration_ms = 10
-                silence = AudioSegment.silent(duration=duration_ms)
-                silence.export(track_file, format="mp3")
-
-                audio = EasyID3(track_file)
-                audio["title"] = track.title or ""
-                audio["artist"] = ", ".join([a.title or "" for a in track.artists])
-                audio["album"] = album.title or ""
-                audio["tracknumber"] = str(album.tracks.index(track) + 1)
-                audio.save()

src/mappers/FuzzyMapper.py

@@ -4,8 +4,6 @@ from tqdm import tqdm
 
 from src.Library import *
 from src.helpers import *
-import re
-import string
 
 import unicodedata
 
@@ -13,49 +11,34 @@ user_def_artist_map = {
     "donda" : "kanye west",
 }
 
-user_def_album_map = {
-    "music" : "music sorry 4 da wait",
-}
+def normalize(s):
+    return unicodedata.normalize("NFKC", s).replace("‐", "-")
 
 def get_score(first: str, second: str) -> float:
-    return float(first == second) * 100
-
-def clean_title(title: str) -> str:
-    title = unicodedata.normalize("NFKC", title).replace("‐", "-")
-    title = unicodedata.normalize("NFKD", title)
-    title = re.sub(r"\([^)]*\)", "", title)
-    title = "".join(
-        c for c in title
-        if not unicodedata.category(c).startswith("P")
-    )
-    title = title.translate(str.maketrans("", "", string.punctuation))
-    title = title.lower()
-    return " ".join(title.split())
-
-def find_mapping(first, second, mapping):
-    for key, value in mapping.items():
-        if (first == key and second == value) or (first == value and second == key):
-            return True
-    return False
+    first_norm = normalize(first).lower()
+    second_norm = normalize(second).lower()
+    return float(first_norm == second_norm) * 100
+    return fuzz.token_set_ratio(first, second)
 
 def get_album_score(first: str, second: str) -> float:
-    first = clean_title(first)
-    second = clean_title(second)
-
-    if find_mapping(first, second, user_def_album_map):
-        return 100
-
     return get_score(first, second)
 
 def get_track_score(first: str, second: str) -> float:
-    return get_score(clean_title(first), clean_title(second))
+    return get_score(first, second)
 
 def get_artist_score(first: str, second: str) -> float:
-    first = clean_title(first)
-    second = clean_title(second)
+    if not first or not second:
+        return 0.0
 
-    if find_mapping(first, second, user_def_artist_map):
-        return 100
+    first_lower = first.lower()
+    second_lower = second.lower()
+
+    for key, value in user_def_artist_map.items():
+        key_lower = key.lower()
+        value_lower = value.lower()
+        if (first_lower == key_lower and second_lower == value_lower) or \
+           (first_lower == value_lower and second_lower == key_lower):
+            return 100.0
 
     return get_score(first, second)
 
@@ -114,7 +97,6 @@ def generate_map(local_lib : Library, remote_lib : Library, output_dir : Path):
                     if remote_track.id not in artist_map or score > track_map[remote_track.id][1]:
                         track_map[remote_track.id] = (local_track.id, score)
                         remote_track.local_id = local_track.id
-                        remote_track.local_path = local_track.local_path
 
             process_bar.update(1)