Auto-load .env; make --spotify-full force the cached path too

The live fetch was being skipped because creds never reached the process:
Env.py only read os.environ, and .env uses `export ...`, so without a prior
`source .env` the run fell to the cached branch — which then skipped on an
unchanged cache signature ("Spotify cache unchanged — skipping").

- Env.py now auto-loads a project-root .env (setdefault, so an exported value
  still wins; tolerates `export ` and quotes).
- --spotify-full now also passes force=True to ingest_spotify, so the cached
  fallback re-ingests instead of skipping.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -1,9 +1,18 @@
-# Copy to `.env` and fill in, then `source .env` before running.
-# Get these from https://developer.spotify.com/dashboard (your app's settings).
-# Register `http://127.0.0.1:8888` as a Redirect URI in that app.
+# Copy to `.env`, fill in, then `source .env` before running.
+
+# --- the three things you point at your own data ---------------------------
+# Folder of audio files to scan (tags incl. ISRC / MusicBrainz id are read).
+export LOCAL_LIBRARY_PATH=~/Music
+# iTunes/Music "Library.xml" you export manually (File > Library > Export Library…).
+export ITUNES_XML_PATH=~/Music/iTunes\ Library.xml
+# Storage root: the database and cached Spotify responses live here.
+export DATA_DIR=./data
+
+# --- Spotify credentials (only needed for live fetching) -------------------
+# https://developer.spotify.com/dashboard — register http://127.0.0.1:8888 as a Redirect URI.
 export SPOTIFY_CLIENT_ID=your_client_id_here
 export SPOTIFY_CLIENT_SECRET=your_client_secret_here
 
-# Optional overrides:
-# export MUSIC_LIBRARY_PATH=~/Music
-# export WORK_DIR=./work_dir/new
+# --- optional --------------------------------------------------------------
+# export MB_CONTACT=you@example.com                         # MusicBrainz User-Agent contact
+# export DATABASE_URL=postgresql+psycopg://u:p@localhost/mi # use Postgres instead of SQLite

.gitignore

@@ -4,6 +4,8 @@ __pycache__
 secret
 .env
 .venv
+*.db
+/data/
 work_dir
 old
 tmp

Env.py

@@ -1,13 +1,62 @@
+"""Configuration. The three things you point at your own data:
+
+  LOCAL_LIBRARY_PATH  folder of audio files to scan
+  ITUNES_XML_PATH     iTunes/Music "Library.xml" you exported manually
+  DATA_DIR            storage root: the database + cached fetches live here
+
+Everything can be overridden via environment variables (see .env.example).
+"""
 import os
 from pathlib import Path
 
-# Local config. Paths can be overridden via environment variables.
-local_library_path = Path(os.environ.get("MUSIC_LIBRARY_PATH", "~/Music")).expanduser()
-work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new"))
 
-# Spotify app credentials. NEVER hardcode these here — set them in the
-# environment (see .env.example) so they don't end up in version control.
-#   export SPOTIFY_CLIENT_ID=...
-#   export SPOTIFY_CLIENT_SECRET=...
+def _load_dotenv():
+    """Load a project-root .env into os.environ if present, so `python main.py`
+    works without remembering to `source .env`. Already-set environment
+    variables win (setdefault), so an explicit `source .env`/export still
+    overrides. Supports an optional leading `export ` and quoted values."""
+    path = Path(__file__).with_name(".env")
+    if not path.exists():
+        return
+    for raw in path.read_text().splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("export "):
+            line = line[len("export "):]
+        key, sep, value = line.partition("=")
+        if not sep:
+            continue
+        os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
+
+
+_load_dotenv()
+
+
+def _path(env, default):
+    return Path(os.environ.get(env, default)).expanduser()
+
+
+# --- where YOUR data is read from / written to -------------------------------
+# Local music library to scan (tags incl. ISRC/MusicBrainz id are read).
+local_library_path = _path("LOCAL_LIBRARY_PATH", "~/Music")
+
+# iTunes / Apple Music library export. In the Music app:
+# File -> Library -> Export Library…  (produces an XML/plist file).
+itunes_xml_path = _path("ITUNES_XML_PATH", "~/Music/iTunes Library.xml")
+
+# Storage root — the database and cached Spotify responses go here.
+data_dir = _path("DATA_DIR", "./data")
+
+# --- derived storage locations -----------------------------------------------
+spotify_cache_dir = data_dir / "spotify"   # raw Spotify API responses
+database_url = os.environ.get(
+    "DATABASE_URL", f"sqlite:///{(data_dir / 'musicindexer.db').as_posix()}"
+)
+
+# --- Spotify app credentials (only needed for live fetching) -----------------
 client_id = os.environ.get("SPOTIFY_CLIENT_ID")
 client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET")
+
+# Contact string sent to MusicBrainz in the User-Agent.
+mb_contact = os.environ.get("MB_CONTACT", "music-indexer@example.com")

Gui.py

@@ -1,208 +1,96 @@
-import time
+"""Streamlit GUI: what's in your Spotify / iTunes libraries that you do NOT
+have locally, decided by real ID matching (recording MBID / ISRC) with a
+normalized-text fallback.
 
-import streamlit as st
-from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
+Reads only the database (run `python main.py ingest` and `enrich` first).
+"""
 import pandas as pd
+import streamlit as st
+from st_aggrid import AgGrid, GridOptionsBuilder
 
-import Flow
+from src.db.database import init_db
+from src.match.matcher import classify_all, summary
 
-from src.Library import *
-
-def entity_to_row(entity: Entity):
-    return {
-        "id": entity.id,
-        "mbid": entity.mbid,
-        "Title": entity.title,
-        "PLay Count": entity.play_count,
-        "Resolved": int(entity.resolved_percentage * 100),
-        "AutoScore": entity.auto_score,
-        "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
-        "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
-        "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
-        "LocalId": str(entity.local_id) if entity.local_id else None,
-        "Path": str(entity.local_path) if entity.local_path else None,
-    }
-
-def track_to_row(track: Track):
-    return entity_to_row(track) | {
-        "Album": track.album.title if track.album else "",
-        "Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists),
-        "Lyrics": track.has_lyrics,
-    }
-
-def artist_to_row(artist: Artist):
-    return entity_to_row(artist) | {
-    }
-
-def album_to_row(album: Album):
-    return entity_to_row(album) | {
-    }
+st.set_page_config(layout="wide", page_title="MusicIndexer — Missing tracks")
 
 
-class LibraryView:
-    def __init__(self):
-        self.lib : Library = None
-        self.libs : list[Library] = None
-        self.view = None
-        self.lyrics_only = False
-        self.query = None
+@st.cache_data(ttl=30)
+def load_data():
+    init_db()
+    missing, present = classify_all()
+    return missing, present, summary()
 
-    def add_libraries(self, libs : list[Library]):
-        self.libs = libs
 
-    def render(self):
-        st.set_page_config(layout="wide")
+def grid(df, key):
+    gb = GridOptionsBuilder.from_dataframe(df)
+    gb.configure_default_column(filter=True, sortable=True, resizable=True)
+    gb.configure_pagination(paginationAutoPageSize=False, paginationPageSize=50)
+    AgGrid(df, gridOptions=gb.build(), height=620, fit_columns_on_grid_load=True, key=key)
 
-        with st.sidebar:
-            lib = st.selectbox(
-                "Library",
-                options=[lib.name for lib in self.libs],
+
+def main():
+    st.title("🎧 Tracks you're missing locally")
+
+    if st.sidebar.button("🔄 Refresh from DB"):
+        load_data.clear()
+
+    missing, present, summ = load_data()
+
+    # --- summary metrics ---
+    for source in ("spotify", "itunes"):
+        s = summ.get(source, {})
+        if not s:
+            continue
+        st.subheader(source.capitalize())
+        c1, c2, c3, c4, c5 = st.columns(5)
+        c1.metric("Total", s["total"])
+        c2.metric("Missing locally", s["missing"])
+        c3.metric("Have (ID)", s["present_mbid"])
+        c4.metric("Have (text)", s["present_text"])
+        coverage = (s["has_mbid"] / s["total"] * 100) if s["total"] else 0
+        c5.metric("MBID coverage", f"{coverage:.0f}%")
+        if source == "spotify":
+            st.caption(
+                f"Origins — liked: {s.get('liked', 0)} · "
+                f"in a playlist: {s.get('from_playlist', 0)} · "
+                f"top tracks: {s.get('from_top', 0)}"
             )
 
-            for iter in self.libs:
-                if iter.name == lib:
-                    self.lib = iter
+    local = summ.get("local", {})
+    st.caption(
+        f"Local library: {local.get('total', 0)} tracks, "
+        f"{local.get('has_mbid', 0)} with a resolved MBID. "
+        "‘Have (ID)’ = certain ISRC/MBID match; ‘Have (text)’ = normalized "
+        "artist/title match (run `fetch`/`enrich` to upgrade text matches to ID)."
+    )
 
-            st.write(f"Showing library '{self.lib.name}'")
+    st.divider()
 
-            self.view = st.radio(
-                "View",
-                ["Tree"]
-            )
+    # --- controls ---
+    view = st.sidebar.radio("View", ["Missing", "Present", "All"])
+    sources = st.sidebar.multiselect("Sources", ["spotify", "itunes"],
+                                     default=["spotify", "itunes"])
 
-        if self.view == "Tree":
-            self.render_artists_albums_tracks()
-        # elif self.view == "Tracks":
-            # self.render_tracks()
-        # elif self.view == "Playlists":
-        #     self.render_playlists()
+    if view == "Missing":
+        rows = [dict(r, status="missing") for r in missing]
+    elif view == "Present":
+        rows = [dict(r, status="present") for r in present]
+    else:
+        rows = ([dict(r, status="missing") for r in missing]
+                + [dict(r, status="present") for r in present])
 
-    def track_visible(
-        self,
-        track: Track,
-        artist: Artist | None,
-        album: Album | None,
-    ):
-        if self.lyrics_only and not track.has_lyrics:
-            return False
+    rows = [r for r in rows if r["source"] in sources]
 
-        if self.query and self.query.lower() not in track.title.lower():
-            return False
+    if not rows:
+        st.info("Nothing to show — run `python main.py ingest` (and `enrich`).")
+        return
 
-        if artist and all(a.id != artist.id for a in track.artists):
-            return False
-
-        if album and (not track.album or track.album.id != album.id):
-            return False
-
-        return True
-
-    def render_artists_albums_tracks(self):
-        # --- ARTISTS TABLE ---
-        st.subheader("Artists")
-        artist_rows = [artist_to_row(a) for a in self.lib.artists]
-        df_artists = pd.DataFrame(artist_rows)
-
-        gb_artists = GridOptionsBuilder.from_dataframe(df_artists)
-        gb_artists.configure_selection(selection_mode="multiple", use_checkbox=True)
-        gb_artists.configure_column("Title", filter="agTextColumnFilter")
-        gb_artists.configure_column("AutoScore", sort="desc")
-        grid_options_artists = gb_artists.build()
-
-        grid_response_artists = AgGrid(
-            df_artists,
-            gridOptions=grid_options_artists,
-            update_mode=GridUpdateMode.SELECTION_CHANGED,
-            allow_unsafe_jscode=True,
-            enable_enterprise_modules=False,
-            fit_columns_on_grid_load=True,
-        )
-
-        selected_rows_df = grid_response_artists.get("selected_rows")
-        selected_artist_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
-        selected_artists = [a for a in self.lib.artists if a.title in selected_artist_titles]
-
-        # --- ALBUMS TABLE ---
-        st.subheader("Albums")
-        if selected_artists:
-            albums_list = []
-            for artist in selected_artists:
-                albums_list.extend(artist.albums)
-            albums_list = list({al.id: al for al in albums_list}.values())
-        else:
-            albums_list = self.lib.albums
-
-        album_rows = [album_to_row(a) for a in albums_list]
-        df_albums = pd.DataFrame(album_rows)
-
-        gb_albums = GridOptionsBuilder.from_dataframe(df_albums)
-        gb_albums.configure_selection(selection_mode="multiple", use_checkbox=True)
-        gb_albums.configure_column("Title", filter="agTextColumnFilter")
-        gb_albums.configure_column("AutoScore", sort="desc")
-        grid_options_albums = gb_albums.build()
-
-        grid_response_albums = AgGrid(
-            df_albums,
-            gridOptions=grid_options_albums,
-            update_mode=GridUpdateMode.SELECTION_CHANGED,
-            allow_unsafe_jscode=True,
-            enable_enterprise_modules=False,
-            fit_columns_on_grid_load=True,
-        )
-
-        selected_rows_df = grid_response_albums.get("selected_rows")
-        selected_album_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
-        selected_albums = [al for al in albums_list if al.title in selected_album_titles]
-
-        # --- TRACKS TABLE ---
-        st.subheader("Tracks")
-        if selected_albums:
-            tracks_list = []
-            for album in selected_albums:
-                tracks_list.extend(album.tracks)
-        elif selected_artists:
-            tracks_list = []
-            for artist in selected_artists:
-                tracks_list.extend([t for t in self.lib.tracks if artist in t.artists])
-        else:
-            tracks_list = self.lib.tracks
-
-        track_rows = [track_to_row(t) for t in tracks_list]
-        df_tracks = pd.DataFrame(track_rows)
-
-        gb_tracks = GridOptionsBuilder.from_dataframe(df_tracks)
-        gb_tracks.configure_selection(selection_mode="multiple", use_checkbox=True)
-        gb_tracks.configure_column("Title", filter="agTextColumnFilter")
-        gb_tracks.configure_column("AutoScore", sort="desc")
-        grid_options_tracks = gb_tracks.build()
-
-        AgGrid(
-            df_tracks,
-            gridOptions=grid_options_tracks,
-            update_mode=GridUpdateMode.SELECTION_CHANGED,
-            allow_unsafe_jscode=True,
-            enable_enterprise_modules=False,
-            fit_columns_on_grid_load=True,
-        )
+    df = pd.DataFrame(rows)[
+        ["status", "source", "origin", "artist", "title", "album", "matched_by",
+         "isrc", "mbid", "play_count", "spotify_id"]
+    ]
+    st.write(f"**{len(df)}** tracks")
+    grid(df, key=f"{view}-{'-'.join(sorted(sources))}")
 
 
-def run():
-    flow = Flow.Flow()
-
-    # flow.fetch_spotify()
-    # flow.fetch_local()
-
-    # flow.parse_spotify_library()
-    # flow.parse_local_library()
-
-    flow.load_libraries()
-
-    # flow.map_local_to_spotify()
-
-    # flow.log_stats()
-
-    gui = LibraryView()
-    gui.add_libraries([flow.spotify_library, flow.itunes_library, flow.local_library])
-    gui.render()
-
-run()
+main()

README.md

@@ -5,19 +5,38 @@ local music library (audio tags + iTunes XML), and match the two so you can see
 which tracks you own, what's missing, and your top artists/tracks/playlists — in
 a Streamlit GUI.
 
-## Features
+It is **database-backed and incremental**: data lives in a DB (SQLite by
+default, Postgres optional) and every step UPSERTs, so re-running only updates
+what changed and nothing already fetched is lost. The headline use case is
+seeing **which Spotify/iTunes tracks you don't have locally**, decided by real
+recording identifiers (ISRC / MusicBrainz MBID) with a normalized-text fallback.
 
-- **Spotify backend** — OAuth (Authorization Code) login, then fetch the user
-  profile, liked tracks, all playlists and their tracks, long-term top tracks
-  and top artists, and playlist/profile cover art. Resilient paginated fetching
-  (follows `next`, retries dropped connections, honours `429` rate limits) with
-  progress bars.
-- **Local backend** — index a folder of audio files via `mutagen` tags.
-- **iTunes backend** — parse an iTunes Library XML export.
-- **Matching** — map local tracks to Spotify tracks (fuzzy matcher; ISRC/MBID
-  matching planned).
-- **Stats & lyrics** — generate listening stats and fetch lyrics.
-- **GUI** — browse libraries as artist → album → track tables.
+## Pipeline
+
+```
+ingest  ->  enrich (MBIDs)  ->  match  ->  GUI
+```
+
+| Step | Command | Incremental rule |
+|------|---------|------------------|
+| **ingest** | `python main.py ingest` | upsert tracks by `(source, source_id)`; never drops enrichment |
+| **enrich** | `python main.py enrich` | query MusicBrainz only for tracks with no MBID yet; every lookup (incl. "not found") cached; rate-limited & resumable |
+| **match** | `python main.py match` | classify each Spotify/iTunes track: present locally (ISRC > MBID > text) or missing |
+| **GUI** | `./gui.sh` | browse missing/present with the match method shown |
+
+### How matching works
+
+Each track is resolved to a MusicBrainz **recording MBID** by searching with the
+*normalized* artist+title, so the same song from different sources lands on the
+same MBID (and shares one cached lookup). A Spotify/iTunes track counts as
+"have" if its ISRC, its MBID, or its normalized artist|title matches a local
+track — otherwise it's **missing**. The GUI labels each match `isrc`/`mbid`
+(ID-certain) or `text` (fuzzy).
+
+> Note: the local library here has no ISRC/MBID tags, so before `enrich` runs,
+> matching is normalized-text only (still useful). `enrich` upgrades matches to
+> ID-certain. Tagging local files with Picard/AcoustID (ISRC) would make the ID
+> tiers exact end-to-end.
 
 ## Setup
 
@@ -29,72 +48,94 @@ source .venv/bin/activate
 pip install -r requirements.txt
 ```
 
-### Credentials
-
-Create an app at https://developer.spotify.com/dashboard and add
-`http://127.0.0.1:8888` as a **Redirect URI**. Credentials are read from the
-environment — never commit them.
+### Configure (`.env`)
 
 ```bash
-cp .env.example .env
-# edit .env with your client id/secret
-source .env
+cp .env.example .env       # then edit — it's auto-loaded (no `source` needed)
 ```
 
-`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (plus optional
-`MUSIC_LIBRARY_PATH` and `WORK_DIR`) from the environment.
+The project root `.env` is read automatically on startup; any variable already
+exported in your shell still takes precedence.
 
-## Usage
+Three things you point at your own data:
 
-Edit `main.py` to uncomment the steps you want, then run:
+| Variable | What |
+|----------|------|
+| `LOCAL_LIBRARY_PATH` | folder of audio files to scan (ISRC/MusicBrainz tags are read) |
+| `ITUNES_XML_PATH` | the `Library.xml` you export from Music (File → Library → Export Library…) |
+| `DATA_DIR` | storage root — the database and cached Spotify responses live here |
+
+`SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` are only needed for live Spotify
+fetching (register `http://127.0.0.1:8888` as a Redirect URI in your
+[Spotify app](https://developer.spotify.com/dashboard)). Set
+`DATABASE_URL=postgresql+psycopg://…` to use Postgres instead of SQLite — no
+code change.
+
+## Two workflows
 
 ```bash
-python main.py
+python main.py fetch      # do all the work, incrementally, and report
+python main.py display    # open the GUI to see what you're missing
 ```
 
-The flow steps (see `Flow.py`) include:
+**`fetch`** pulls Spotify, scans the local library, parses the iTunes export,
+resolves MBIDs, and matches — logging progress at each step. The **first run is
+slow** (full scan + MusicBrainz lookups at ~1 req/s); **later runs are quick**
+because each source only touches what changed:
 
-| Step | What it does |
-|------|--------------|
-| `fetch_spotify()` | OAuth login + download all Spotify data into `work_dir/spotify/` |
-| `fetch_local()` | index local audio files into `work_dir/local/` |
-| `parse_spotify_library()` / `parse_itunes_library()` / `parse_local_library()` | build normalized libraries |
-| `load_libraries()` / `save_libraries()` | (de)serialize libraries to `work_dir` |
-| `map_local_to_spotify()` | match local tracks against the Spotify library |
-| `fetch_and_save_lyrics()` | fetch lyrics for tracks |
-| `log_stats()` | generate stats |
+- **Spotify** — playlists are re-fetched only when their `snapshot_id` changes;
+  liked songs are paged newest-first and stop at the newest one already stored;
+  cover art downloads only when missing or its URL changed (`--spotify-full`
+  forces a complete resync).
+  - Your library is **liked songs + playlist tracks**. The *top-tracks* endpoint
+    is **not** pulled by default (those are just listening stats, not saved
+    music, and would inflate "missing"). Add `--include-top` to pull them too —
+    they're fetched as full Spotify objects, so they carry an ISRC for real ID
+    matching like everything else.
+  - Each Spotify track records its **origin** (`liked` / `playlist` / `top`,
+    combinable). On a `--spotify-full` run, rows that are *only* there because
+    the top endpoint listed them (neither liked nor in a playlist) are pruned —
+    run `python main.py fetch --spotify-full` once to clean up an existing DB.
+- **Local** — only files whose mtime/size changed; deleted files are pruned.
+- **iTunes** — skipped entirely if the export's mtime is unchanged.
+- **MBID enrichment** — only still-unresolved tracks.
 
-On first `fetch_spotify()` a browser opens for the Spotify login; a local server
-on `127.0.0.1:8888` catches the redirect and exchanges the code for a token.
-
-### GUI
+Useful flags/helpers:
 
 ```bash
-./gui.sh        # runs: .venv/bin/streamlit run Gui.py
+python main.py fetch --enrich-limit 500   # cap MBID lookups this run
+python main.py enrich                      # resume MBID resolution only
+python main.py match                       # print the present/missing summary
 ```
 
+> ISRC/MBID matching works as soon as `fetch` runs (local tags + Spotify ISRC).
+> MusicBrainz enrichment fills in the rest (iTunes, untagged local files); it's
+> cached and resumable, so you can stop/continue any time.
+
 ## Layout
 
 ```
-main.py                 entry point (CLI flow)
-Gui.py / gui.sh         Streamlit GUI
-Flow.py                 orchestrates fetch / parse / map / stats
-Env.py                  config (reads credentials from the environment)
+main.py                 CLI: fetch / display / enrich / match
+Gui.py / gui.sh         Streamlit GUI (missing-tracks viewer)
+Env.py                  config: LOCAL_LIBRARY_PATH / ITUNES_XML_PATH / DATA_DIR
 src/
-  Library.py            data model (Library / Artist / Album / Track)
-  LibrarySerializer.py  load/save libraries
-  backends/
-    spotify/            Web API client, OAuth, parser
-    local/              local-file indexer, parser, playlist generator
-    itunes/             iTunes XML parser
-    navidrome/          Navidrome backend
-  mappers/              local↔Spotify matching (fuzzy / search)
-  lyrics/               lyrics fetching
-work_dir/               fetched data and serialized libraries (gitignored)
+  workflows.py          run_fetch (the whole pipeline) + run_display
+  db/                   SQLAlchemy models + engine/session
+  ingest/
+    spotify_raw.py      live fetch + ingest raw Spotify JSON (ISRC)
+    local_scan.py       incremental file scan (reads ISRC/MBID tags)
+    itunes_xml.py       parse the iTunes Library.xml (plist)
+    tags.py             read ISRC / recording MBID / duration from a file
+  enrich/               MusicBrainz MBID resolver (cached, resumable)
+  match/                normalization + cross-source matcher
+  backends/             lower-level Spotify Web API client / OAuth
+$DATA_DIR/              database + cached Spotify responses (gitignored)
 ```
 
 ## Notes
 
-- Data is cached as JSON under `work_dir/` (gitignored); re-running reuses it.
-- The deprecated Spotify endpoints (audio-features, recommendations, etc.) are
-  not used — only currently-supported endpoints.
+- The DB is the source of truth; everything is incremental and re-runnable.
+- Local matching is real-ID where files are tagged (ISRC/MusicBrainz, e.g. via
+  Picard); untagged files fall back to MusicBrainz text lookup, then text.
+- Only currently-supported Spotify endpoints are used (no deprecated
+  audio-features/recommendations/etc.).

main.py

@@ -1,38 +1,95 @@
-import Flow
+"""MusicIndexer CLI. Two main workflows plus granular helpers.
+
+    python main.py fetch       # do all the work: pull + scan + parse + enrich + match
+    python main.py display     # open the GUI to see what you're missing
+
+`fetch` is incremental and DB-backed (see Env.py): the first run is slow
+(initial scan + MusicBrainz lookups), later runs only touch what changed.
+Helpers: `enrich` (resume MBID resolution), `match` (print the summary).
+"""
+import argparse
+import logging
+import sys
 
 
-def test1():
-    flow = Flow.Flow()
+def _setup_logging():
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
 
-    # flow.fetch_local()
-    flow.parse_local_library()
-    flow.load_libraries()
 
-    # flow.map_local_to_spotify()
-    # flow.log_stats()
+def cmd_fetch(args):
+    _setup_logging()
+    from src.workflows import run_fetch
 
-    # flow.fetch_and_save_lyrics()
+    run_fetch(enrich_limit=args.enrich_limit, enrich=not args.no_enrich,
+              spotify_full=args.spotify_full, include_top=args.include_top)
+
+
+def cmd_display(args):
+    from src.workflows import run_display
+
+    run_display()
+
+
+def cmd_enrich(args):
+    _setup_logging()
+    from src.db.database import init_db
+    from src.enrich.mbid import resolve_mbids, unresolved_count
+
+    init_db()
+    print(f"unresolved before: {unresolved_count(args.source)}")
+    print(f"resolved this run: {resolve_mbids(limit=args.limit, source=args.source)}")
+    print(f"unresolved after:  {unresolved_count(args.source)}")
+
+
+def cmd_match(args):
+    import json
+
+    from src.match.matcher import summary
+
+    print(json.dumps(summary(), indent=2))
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__)
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    fetch = sub.add_parser("fetch", help="incrementally fetch + scan + enrich + match")
+    fetch.add_argument("--enrich-limit", type=int, default=None,
+                       help="cap MBID lookups this run (default: resolve all)")
+    fetch.add_argument("--no-enrich", action="store_true",
+                       help="skip MusicBrainz enrichment; use MBIDs already on tracks")
+    fetch.add_argument("--spotify-full", action="store_true",
+                       help="force a full Spotify resync (ignore snapshot/watermark)")
+    fetch.add_argument("--include-top", action="store_true",
+                       help="also pull the top-tracks endpoint (off by default; "
+                            "with --spotify-full, top-only tracks are otherwise pruned)")
+
+    sub.add_parser("display", help="open the Streamlit GUI")
+
+    enrich = sub.add_parser("enrich", help="resume MBID resolution only")
+    enrich.add_argument("--limit", type=int, default=None)
+    enrich.add_argument("--source", choices=["spotify", "itunes", "local"], default=None)
+
+    sub.add_parser("match", help="print present/missing summary")
+    return parser
 
 
 def main():
-    flow = Flow.Flow()
-
-    flow.fetch_spotify()
-    # flow.fetch_local()
-
-    flow.parse_spotify_library()
-    #flow.parse_itunes_library()
-    #flow.parse_local_library()
-
-    #flow.load_libraries()
-
-    #flow.map_local_to_spotify()
-    #flow.save_libraries()
-
-    # flow.log_stats()
-
-    # print("asd")
+    args = build_parser().parse_args()
+    try:
+        {
+            "fetch": cmd_fetch,
+            "display": cmd_display,
+            "enrich": cmd_enrich,
+            "match": cmd_match,
+        }[args.cmd](args)
+    except KeyboardInterrupt:
+        # Ctrl+C: stop cleanly, no traceback. Work already committed is kept
+        # (each step writes incrementally), so just re-run to continue.
+        print("\ninterrupted — partial progress saved; re-run to continue.",
+              file=sys.stderr)
+        sys.exit(130)
 
 
-# test1()
-main()
+if __name__ == "__main__":
+    main()

old-work-dir/first-fetch/user_data.json

@@ -0,0 +1,26 @@
+{
+    "display_name": "ilusha",
+    "external_urls": {
+        "spotify": "https://open.spotify.com/user/31uszk7zb6zdzrzst4q3rvkfrfz4"
+    },
+    "followers": {
+        "href": null,
+        "total": 8
+    },
+    "href": "https://api.spotify.com/v1/users/31uszk7zb6zdzrzst4q3rvkfrfz4",
+    "id": "31uszk7zb6zdzrzst4q3rvkfrfz4",
+    "images": [
+        {
+            "url": "https://i.scdn.co/image/ab6775700000ee85b4a21a7f0c35258b3fa4b79d",
+            "height": 300,
+            "width": 300
+        },
+        {
+            "url": "https://i.scdn.co/image/ab67757000003b82b4a21a7f0c35258b3fa4b79d",
+            "height": 64,
+            "width": 64
+        }
+    ],
+    "type": "user",
+    "uri": "spotify:user:31uszk7zb6zdzrzst4q3rvkfrfz4"
+}

old-work-dir/first-fetch/user_profile.json

@@ -0,0 +1,33 @@
+{
+    "country": "NL",
+    "display_name": "ilusha",
+    "email": "ilusha.basic@gmail.com",
+    "explicit_content": {
+        "filter_enabled": false,
+        "filter_locked": false
+    },
+    "external_urls": {
+        "spotify": "https://open.spotify.com/user/31uszk7zb6zdzrzst4q3rvkfrfz4"
+    },
+    "followers": {
+        "href": null,
+        "total": 8
+    },
+    "href": "https://api.spotify.com/v1/users/31uszk7zb6zdzrzst4q3rvkfrfz4",
+    "id": "31uszk7zb6zdzrzst4q3rvkfrfz4",
+    "images": [
+        {
+            "height": 300,
+            "url": "https://i.scdn.co/image/ab6775700000ee85b4a21a7f0c35258b3fa4b79d",
+            "width": 300
+        },
+        {
+            "height": 64,
+            "url": "https://i.scdn.co/image/ab67757000003b82b4a21a7f0c35258b3fa4b79d",
+            "width": 64
+        }
+    ],
+    "product": "free",
+    "type": "user",
+    "uri": "spotify:user:31uszk7zb6zdzrzst4q3rvkfrfz4"
+}