From 0e64d439c423a0be2e3784d72ce08943fac14535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A8=D1=83=D1=80=D1=83=D0=BF=D0=BE=D0=B2=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F=20=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=87?= Date: Mon, 15 Jun 2026 01:25:21 +0300 Subject: [PATCH] DB-backed incremental redesign: find tracks missing locally Replace the JSON-file pipeline with a SQLite/SQLAlchemy store (Postgres via DATABASE_URL) and an incremental, ID-based matcher whose goal is "which Spotify/iTunes tracks am I missing locally". - src/db: models (Track/MbCache/FileState/IngestState) + engine/session. - src/ingest: upsert prefetched spotify/itunes/local (work_dir/latest-1) by (source, source_id); pulls ISRC from raw Spotify; preserves enrichment. - src/enrich: MusicBrainz recording-MBID resolver. Resolves via NORMALIZED text search (so the same song across sources lands on one MBID + one cached lookup), ISRC as exact fallback. Only touches unresolved tracks, caches positive AND negative results, rate-limited and resumable. - src/match: normalization + cross-source matcher with tiers ISRC > MBID > normalized text; reports present/missing and the match method. - main.py: CLI (ingest / enrich / match). Gui.py: missing-tracks viewer. - README + requirements (SQLAlchemy, musicbrainzngs); ignore *.db. Incremental everywhere: re-running updates only what changed; enrich continues where it stopped. Live Spotify/local backends are left intact. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + Env.py | 12 ++ Gui.py | 268 ++++++++++++----------------------------- README.md | 121 +++++++++++-------- main.py | 96 ++++++++++----- requirements.txt | 2 + src/db/__init__.py | 0 src/db/database.py | 35 ++++++ src/db/models.py | 109 +++++++++++++++++ src/enrich/__init__.py | 0 src/enrich/mbid.py | 143 ++++++++++++++++++++++ src/ingest/__init__.py | 0 src/ingest/ingest.py | 130 ++++++++++++++++++++ src/match/__init__.py | 0 src/match/matcher.py | 93 ++++++++++++++ src/match/normalize.py | 48 ++++++++ 16 files changed, 784 insertions(+), 274 deletions(-) create mode 100644 src/db/__init__.py create mode 100644 src/db/database.py create mode 100644 src/db/models.py create mode 100644 src/enrich/__init__.py create mode 100644 src/enrich/mbid.py create mode 100644 src/ingest/__init__.py create mode 100644 src/ingest/ingest.py create mode 100644 src/match/__init__.py create mode 100644 src/match/matcher.py create mode 100644 src/match/normalize.py diff --git a/.gitignore b/.gitignore index 7b75b45..11a26b7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__ secret .env .venv +*.db work_dir old tmp \ No newline at end of file diff --git a/Env.py b/Env.py index c609347..304d2a5 100644 --- a/Env.py +++ b/Env.py @@ -11,3 +11,15 @@ work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new")) # export SPOTIFY_CLIENT_SECRET=... client_id = os.environ.get("SPOTIFY_CLIENT_ID") client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET") + +# Persistent store. Defaults to a local SQLite file; point DATABASE_URL at a +# Postgres instance (postgresql+psycopg://user:pass@host/db) to switch with no +# code changes. +database_url = os.environ.get("DATABASE_URL", "sqlite:///./musicindexer.db") + +# Folder of prefetched JSON used to (re)build the DB without hitting the network. +prefetched_dir = Path(os.environ.get("PREFETCHED_DIR", "./work_dir/latest-1")) + +# Contact email sent to MusicBrainz in the User-Agent (be a good API citizen). +mb_contact = os.environ.get("MB_CONTACT", "music-indexer@example.com") + diff --git a/Gui.py b/Gui.py index 3c0ecc8..9003e57 100644 --- a/Gui.py +++ b/Gui.py @@ -1,208 +1,90 @@ -import time +"""Streamlit GUI: what's in your Spotify / iTunes libraries that you do NOT +have locally, decided by real ID matching (recording MBID / ISRC) with a +normalized-text fallback. -import streamlit as st -from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode +Reads only the database (run `python main.py ingest` and `enrich` first). +""" import pandas as pd +import streamlit as st +from st_aggrid import AgGrid, GridOptionsBuilder -import Flow +from src.db.database import init_db +from src.match.matcher import classify_all, summary -from src.Library import * - -def entity_to_row(entity: Entity): - return { - "id": entity.id, - "mbid": entity.mbid, - "Title": entity.title, - "PLay Count": entity.play_count, - "Resolved": int(entity.resolved_percentage * 100), - "AutoScore": entity.auto_score, - "Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None, - "Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None, - "Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None, - "LocalId": str(entity.local_id) if entity.local_id else None, - "Path": str(entity.local_path) if entity.local_path else None, - } - -def track_to_row(track: Track): - return entity_to_row(track) | { - "Album": track.album.title if track.album else "", - "Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists), - "Lyrics": track.has_lyrics, - } - -def artist_to_row(artist: Artist): - return entity_to_row(artist) | { - } - -def album_to_row(album: Album): - return entity_to_row(album) | { - } +st.set_page_config(layout="wide", page_title="MusicIndexer β€” Missing tracks") -class LibraryView: - def __init__(self): - self.lib : Library = None - self.libs : list[Library] = None - self.view = None - self.lyrics_only = False - self.query = None - - def add_libraries(self, libs : list[Library]): - self.libs = libs - - def render(self): - st.set_page_config(layout="wide") - - with st.sidebar: - lib = st.selectbox( - "Library", - options=[lib.name for lib in self.libs], - ) - - for iter in self.libs: - if iter.name == lib: - self.lib = iter - - st.write(f"Showing library '{self.lib.name}'") - - self.view = st.radio( - "View", - ["Tree"] - ) - - if self.view == "Tree": - self.render_artists_albums_tracks() - # elif self.view == "Tracks": - # self.render_tracks() - # elif self.view == "Playlists": - # self.render_playlists() - - def track_visible( - self, - track: Track, - artist: Artist | None, - album: Album | None, - ): - if self.lyrics_only and not track.has_lyrics: - return False - - if self.query and self.query.lower() not in track.title.lower(): - return False - - if artist and all(a.id != artist.id for a in track.artists): - return False - - if album and (not track.album or track.album.id != album.id): - return False - - return True - - def render_artists_albums_tracks(self): - # --- ARTISTS TABLE --- - st.subheader("Artists") - artist_rows = [artist_to_row(a) for a in self.lib.artists] - df_artists = pd.DataFrame(artist_rows) - - gb_artists = GridOptionsBuilder.from_dataframe(df_artists) - gb_artists.configure_selection(selection_mode="multiple", use_checkbox=True) - gb_artists.configure_column("Title", filter="agTextColumnFilter") - gb_artists.configure_column("AutoScore", sort="desc") - grid_options_artists = gb_artists.build() - - grid_response_artists = AgGrid( - df_artists, - gridOptions=grid_options_artists, - update_mode=GridUpdateMode.SELECTION_CHANGED, - allow_unsafe_jscode=True, - enable_enterprise_modules=False, - fit_columns_on_grid_load=True, - ) - - selected_rows_df = grid_response_artists.get("selected_rows") - selected_artist_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else [] - selected_artists = [a for a in self.lib.artists if a.title in selected_artist_titles] - - # --- ALBUMS TABLE --- - st.subheader("Albums") - if selected_artists: - albums_list = [] - for artist in selected_artists: - albums_list.extend(artist.albums) - albums_list = list({al.id: al for al in albums_list}.values()) - else: - albums_list = self.lib.albums - - album_rows = [album_to_row(a) for a in albums_list] - df_albums = pd.DataFrame(album_rows) - - gb_albums = GridOptionsBuilder.from_dataframe(df_albums) - gb_albums.configure_selection(selection_mode="multiple", use_checkbox=True) - gb_albums.configure_column("Title", filter="agTextColumnFilter") - gb_albums.configure_column("AutoScore", sort="desc") - grid_options_albums = gb_albums.build() - - grid_response_albums = AgGrid( - df_albums, - gridOptions=grid_options_albums, - update_mode=GridUpdateMode.SELECTION_CHANGED, - allow_unsafe_jscode=True, - enable_enterprise_modules=False, - fit_columns_on_grid_load=True, - ) - - selected_rows_df = grid_response_albums.get("selected_rows") - selected_album_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else [] - selected_albums = [al for al in albums_list if al.title in selected_album_titles] - - # --- TRACKS TABLE --- - st.subheader("Tracks") - if selected_albums: - tracks_list = [] - for album in selected_albums: - tracks_list.extend(album.tracks) - elif selected_artists: - tracks_list = [] - for artist in selected_artists: - tracks_list.extend([t for t in self.lib.tracks if artist in t.artists]) - else: - tracks_list = self.lib.tracks - - track_rows = [track_to_row(t) for t in tracks_list] - df_tracks = pd.DataFrame(track_rows) - - gb_tracks = GridOptionsBuilder.from_dataframe(df_tracks) - gb_tracks.configure_selection(selection_mode="multiple", use_checkbox=True) - gb_tracks.configure_column("Title", filter="agTextColumnFilter") - gb_tracks.configure_column("AutoScore", sort="desc") - grid_options_tracks = gb_tracks.build() - - AgGrid( - df_tracks, - gridOptions=grid_options_tracks, - update_mode=GridUpdateMode.SELECTION_CHANGED, - allow_unsafe_jscode=True, - enable_enterprise_modules=False, - fit_columns_on_grid_load=True, - ) +@st.cache_data(ttl=30) +def load_data(): + init_db() + missing, present = classify_all() + return missing, present, summary() -def run(): - flow = Flow.Flow() +def grid(df, key): + gb = GridOptionsBuilder.from_dataframe(df) + gb.configure_default_column(filter=True, sortable=True, resizable=True) + gb.configure_pagination(paginationAutoPageSize=False, paginationPageSize=50) + AgGrid(df, gridOptions=gb.build(), height=620, fit_columns_on_grid_load=True, key=key) - # flow.fetch_spotify() - # flow.fetch_local() - # flow.parse_spotify_library() - # flow.parse_local_library() +def main(): + st.title("🎧 Tracks you're missing locally") - flow.load_libraries() + if st.sidebar.button("πŸ”„ Refresh from DB"): + load_data.clear() - # flow.map_local_to_spotify() + missing, present, summ = load_data() - # flow.log_stats() + # --- summary metrics --- + for source in ("spotify", "itunes"): + s = summ.get(source, {}) + if not s: + continue + st.subheader(source.capitalize()) + c1, c2, c3, c4, c5 = st.columns(5) + c1.metric("Total", s["total"]) + c2.metric("Missing locally", s["missing"]) + c3.metric("Have (MBID)", s["present_mbid"]) + c4.metric("Have (text)", s["present_text"]) + coverage = (s["has_mbid"] / s["total"] * 100) if s["total"] else 0 + c5.metric("MBID coverage", f"{coverage:.0f}%") - gui = LibraryView() - gui.add_libraries([flow.spotify_library, flow.itunes_library, flow.local_library]) - gui.render() + local = summ.get("local", {}) + st.caption( + f"Local library: {local.get('total', 0)} tracks, " + f"{local.get('has_mbid', 0)} with a resolved MBID. " + "β€˜Have (MBID)’ = certain ID match; β€˜Have (text)’ = normalized " + "artist/title match (run `enrich` to upgrade text matches to ID)." + ) -run() \ No newline at end of file + st.divider() + + # --- controls --- + view = st.sidebar.radio("View", ["Missing", "Present", "All"]) + sources = st.sidebar.multiselect("Sources", ["spotify", "itunes"], + default=["spotify", "itunes"]) + + if view == "Missing": + rows = [dict(r, status="missing") for r in missing] + elif view == "Present": + rows = [dict(r, status="present") for r in present] + else: + rows = ([dict(r, status="missing") for r in missing] + + [dict(r, status="present") for r in present]) + + rows = [r for r in rows if r["source"] in sources] + + if not rows: + st.info("Nothing to show β€” run `python main.py ingest` (and `enrich`).") + return + + df = pd.DataFrame(rows)[ + ["status", "source", "artist", "title", "album", "matched_by", + "isrc", "mbid", "play_count", "spotify_id"] + ] + st.write(f"**{len(df)}** tracks") + grid(df, key=f"{view}-{'-'.join(sorted(sources))}") + + +main() diff --git a/README.md b/README.md index 62d8bd7..6e087c4 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,38 @@ local music library (audio tags + iTunes XML), and match the two so you can see which tracks you own, what's missing, and your top artists/tracks/playlists β€” in a Streamlit GUI. -## Features +It is **database-backed and incremental**: data lives in a DB (SQLite by +default, Postgres optional) and every step UPSERTs, so re-running only updates +what changed and nothing already fetched is lost. The headline use case is +seeing **which Spotify/iTunes tracks you don't have locally**, decided by real +recording identifiers (ISRC / MusicBrainz MBID) with a normalized-text fallback. -- **Spotify backend** β€” OAuth (Authorization Code) login, then fetch the user - profile, liked tracks, all playlists and their tracks, long-term top tracks - and top artists, and playlist/profile cover art. Resilient paginated fetching - (follows `next`, retries dropped connections, honours `429` rate limits) with - progress bars. -- **Local backend** β€” index a folder of audio files via `mutagen` tags. -- **iTunes backend** β€” parse an iTunes Library XML export. -- **Matching** β€” map local tracks to Spotify tracks (fuzzy matcher; ISRC/MBID - matching planned). -- **Stats & lyrics** β€” generate listening stats and fetch lyrics. -- **GUI** β€” browse libraries as artist β†’ album β†’ track tables. +## Pipeline + +``` +ingest -> enrich (MBIDs) -> match -> GUI +``` + +| Step | Command | Incremental rule | +|------|---------|------------------| +| **ingest** | `python main.py ingest` | upsert tracks by `(source, source_id)`; never drops enrichment | +| **enrich** | `python main.py enrich` | query MusicBrainz only for tracks with no MBID yet; every lookup (incl. "not found") cached; rate-limited & resumable | +| **match** | `python main.py match` | classify each Spotify/iTunes track: present locally (ISRC > MBID > text) or missing | +| **GUI** | `./gui.sh` | browse missing/present with the match method shown | + +### How matching works + +Each track is resolved to a MusicBrainz **recording MBID** by searching with the +*normalized* artist+title, so the same song from different sources lands on the +same MBID (and shares one cached lookup). A Spotify/iTunes track counts as +"have" if its ISRC, its MBID, or its normalized artist|title matches a local +track β€” otherwise it's **missing**. The GUI labels each match `isrc`/`mbid` +(ID-certain) or `text` (fuzzy). + +> Note: the local library here has no ISRC/MBID tags, so before `enrich` runs, +> matching is normalized-text only (still useful). `enrich` upgrades matches to +> ID-certain. Tagging local files with Picard/AcoustID (ISRC) would make the ID +> tiers exact end-to-end. ## Setup @@ -41,60 +60,56 @@ cp .env.example .env source .env ``` -`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (plus optional -`MUSIC_LIBRARY_PATH` and `WORK_DIR`) from the environment. +`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (only needed for +live fetching), plus optional `DATABASE_URL`, `PREFETCHED_DIR` and `MB_CONTACT`. + +### Database + +Defaults to a local SQLite file (`musicindexer.db`). To use Postgres instead, +set one env var β€” no code changes: + +```bash +export DATABASE_URL=postgresql+psycopg://user:pass@localhost:5432/musicindexer +``` ## Usage -Edit `main.py` to uncomment the steps you want, then run: - ```bash -python main.py +python main.py ingest # load prefetched data (work_dir/latest-1) into the DB +python main.py enrich # resolve MusicBrainz MBIDs (resumable; ~1 req/s) +python main.py match # print present/missing summary +./gui.sh # browse what you're missing ``` -The flow steps (see `Flow.py`) include: +`ingest` and the GUI work immediately (text-based matching). `enrich` queries +MusicBrainz only for tracks that don't have an MBID yet, caches every result, +and is **resumable** β€” if it's interrupted, just run it again to continue. Run +`python main.py enrich --limit 300` to do a quick partial pass, or +`--source local` to restrict to one source. -| Step | What it does | -|------|--------------| -| `fetch_spotify()` | OAuth login + download all Spotify data into `work_dir/spotify/` | -| `fetch_local()` | index local audio files into `work_dir/local/` | -| `parse_spotify_library()` / `parse_itunes_library()` / `parse_local_library()` | build normalized libraries | -| `load_libraries()` / `save_libraries()` | (de)serialize libraries to `work_dir` | -| `map_local_to_spotify()` | match local tracks against the Spotify library | -| `fetch_and_save_lyrics()` | fetch lyrics for tracks | -| `log_stats()` | generate stats | - -On first `fetch_spotify()` a browser opens for the Spotify login; a local server -on `127.0.0.1:8888` catches the redirect and exchanges the code for a token. - -### GUI - -```bash -./gui.sh # runs: .venv/bin/streamlit run Gui.py -``` +> Live fetching (`SpotifyWebAPI`, the local indexer) is unchanged and still +> available; this redesign reads prefetched JSON so the pipeline can be built +> and demoed without network access. ## Layout ``` -main.py entry point (CLI flow) -Gui.py / gui.sh Streamlit GUI -Flow.py orchestrates fetch / parse / map / stats -Env.py config (reads credentials from the environment) +main.py CLI: ingest / enrich / match +Gui.py / gui.sh Streamlit GUI (missing-tracks viewer) +Env.py config (DB URL, paths, credentials from env) src/ - Library.py data model (Library / Artist / Album / Track) - LibrarySerializer.py load/save libraries - backends/ - spotify/ Web API client, OAuth, parser - local/ local-file indexer, parser, playlist generator - itunes/ iTunes XML parser - navidrome/ Navidrome backend - mappers/ local↔Spotify matching (fuzzy / search) - lyrics/ lyrics fetching -work_dir/ fetched data and serialized libraries (gitignored) + db/ SQLAlchemy models + engine/session + ingest/ load prefetched JSON into the DB (upsert) + enrich/ MusicBrainz MBID resolver (cached, resumable) + match/ normalization + cross-source matcher + backends/ live fetchers/parsers (spotify, local, itunes, ...) + Library.py in-memory data model used by the backends +work_dir/latest-1/ prefetched JSON used by `ingest` (gitignored) +musicindexer.db the database (gitignored) ``` ## Notes -- Data is cached as JSON under `work_dir/` (gitignored); re-running reuses it. -- The deprecated Spotify endpoints (audio-features, recommendations, etc.) are - not used β€” only currently-supported endpoints. +- The DB is the source of truth; `work_dir/` JSON is only an ingest source. +- Only currently-supported Spotify endpoints are used (no deprecated + audio-features/recommendations/etc.). diff --git a/main.py b/main.py index 68707c7..4d2e5ec 100644 --- a/main.py +++ b/main.py @@ -1,38 +1,78 @@ -import Flow +"""MusicIndexer pipeline CLI. + +Everything is incremental and DB-backed (see Env.database_url). Typical flow: + + python main.py ingest # load prefetched data into the DB (upsert) + python main.py enrich # resolve recording MBIDs (resumable, cached) + python main.py match # print present/missing summary + ./gui.sh # browse what you're missing + +`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it +just continues where it stopped. +""" +import argparse +import json + +from src.db.database import init_db -def test1(): - flow = Flow.Flow() +def cmd_init(args): + init_db() + print("Database initialized.") - # flow.fetch_local() - flow.parse_local_library() - flow.load_libraries() - # flow.map_local_to_spotify() - # flow.log_stats() +def cmd_ingest(args): + init_db() + from src.ingest.ingest import ingest_all - # flow.fetch_and_save_lyrics() + results = ingest_all() + for source, (new, updated) in results.items(): + print(f"{source:8} +{new} new, {updated} updated") + + +def cmd_enrich(args): + init_db() + from src.enrich.mbid import resolve_mbids, unresolved_count + + print(f"unresolved before: {unresolved_count(args.source)}") + stats = resolve_mbids(limit=args.limit, source=args.source) + print(f"resolved this run: {stats}") + print(f"unresolved after: {unresolved_count(args.source)}") + + +def cmd_match(args): + from src.match.matcher import summary + + print(json.dumps(summary(), indent=2)) + + +def build_parser(): + parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + + sub.add_parser("init", help="create database tables") + sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)") + + enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks") + enrich.add_argument("--limit", type=int, default=None, + help="cap how many tracks to resolve this run") + enrich.add_argument("--source", choices=["spotify", "itunes", "local"], + default=None, help="restrict to one source") + + sub.add_parser("match", help="print present/missing summary") + return parser def main(): - flow = Flow.Flow() - - flow.fetch_spotify() - # flow.fetch_local() - - flow.parse_spotify_library() - #flow.parse_itunes_library() - #flow.parse_local_library() - - #flow.load_libraries() - - #flow.map_local_to_spotify() - #flow.save_libraries() - - # flow.log_stats() - - # print("asd") + args = build_parser().parse_args() + handlers = { + "init": cmd_init, + "ingest": cmd_ingest, + "enrich": cmd_enrich, + "match": cmd_match, + } + handlers[args.cmd](args) -# test1() -main() +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index 9a1d950..f83072d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ rapidfuzz==3.14.5 requests==2.34.2 tqdm==4.68.2 xmltodict==1.0.4 +SQLAlchemy==2.0.50 +musicbrainzngs==0.7.1 diff --git a/src/db/__init__.py b/src/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/db/database.py b/src/db/database.py new file mode 100644 index 0000000..2016900 --- /dev/null +++ b/src/db/database.py @@ -0,0 +1,35 @@ +"""Engine/session helpers. The DB URL comes from Env.database_url so the same +code runs on SQLite (default) or Postgres by changing one env var.""" +from contextlib import contextmanager + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from Env import database_url +from src.db.models import Base + +_engine = create_engine(database_url, future=True) +SessionLocal = sessionmaker(bind=_engine, future=True, expire_on_commit=False) + + +def get_engine(): + return _engine + + +def init_db(): + """Create tables if they don't exist (safe to call every run).""" + Base.metadata.create_all(_engine) + + +@contextmanager +def session_scope(): + """Transactional session: commit on success, roll back on error.""" + session = SessionLocal() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() diff --git a/src/db/models.py b/src/db/models.py new file mode 100644 index 0000000..6e0164d --- /dev/null +++ b/src/db/models.py @@ -0,0 +1,109 @@ +"""SQLAlchemy models for the music index. + +Storage is source-of-truth and incremental: every ingest/enrich step UPSERTs +into these tables, so re-running only updates what changed and nothing already +fetched is lost. +""" +from datetime import datetime + +from sqlalchemy import ( + Column, + DateTime, + Float, + Integer, + String, + UniqueConstraint, +) +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + +SOURCE_SPOTIFY = "spotify" +SOURCE_ITUNES = "itunes" +SOURCE_LOCAL = "local" + +# mbid_status values +MB_UNRESOLVED = None # never looked up +MB_FOUND = "found" +MB_NOTFOUND = "notfound" +MB_ERROR = "error" + + +class Track(Base): + """One track as seen in one source (spotify / itunes / local). + + The same song appears as several rows (one per source); the matcher links + them by ``match_key`` (recording MBID when known, else a normalized text + key). ``(source, source_id)`` is the natural key used for upserts. + """ + + __tablename__ = "track" + + id = Column(Integer, primary_key=True) + source = Column(String, nullable=False, index=True) + source_id = Column(String, nullable=False) # spotify id / itunes track id / local path + + title = Column(String) + artist = Column(String) # primary artist (display + matching) + artists = Column(String) # all artists, joined with ", " + album = Column(String) + duration_ms = Column(Integer) + play_count = Column(Integer, default=0) + local_path = Column(String) + + # Stable identifiers used for ID-based matching. + isrc = Column(String, index=True) + mbid = Column(String, index=True) # recording MBID + norm_key = Column(String, index=True) # normalized "artist|title" fallback + + # Enrichment bookkeeping so the resolver only touches unresolved rows. + mbid_status = Column(String, index=True) # None | found | notfound | error + mbid_source = Column(String) # isrc | text + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + __table_args__ = ( + UniqueConstraint("source", "source_id", name="uq_track_source_id"), + ) + + @property + def match_key(self) -> str: + """Key used to decide identity across sources: prefer the recording + MBID, fall back to the normalized text key.""" + return self.mbid or self.norm_key + + +class MbCache(Base): + """Cache of MusicBrainz lookups (positive AND negative) so the resolver is + incremental and never repeats a query β€” keyed by ISRC or by a normalized + text query.""" + + __tablename__ = "mb_cache" + + query_key = Column(String, primary_key=True) # "isrc:XXX" or "txt:artist|title|album" + mbid = Column(String) + score = Column(Integer) + status = Column(String) # found | notfound | error + fetched_at = Column(DateTime, default=datetime.utcnow) + + +class FileState(Base): + """Per-file index state for incremental local scanning: a file is only + re-read when its mtime/size change.""" + + __tablename__ = "file_state" + + path = Column(String, primary_key=True) + mtime = Column(Float) + size = Column(Integer) + indexed_at = Column(DateTime, default=datetime.utcnow) + + +class IngestState(Base): + """Small key/value table for incremental-fetch bookkeeping, e.g. playlist + snapshot ids or the newest liked-track id seen on the last run.""" + + __tablename__ = "ingest_state" + + key = Column(String, primary_key=True) + value = Column(String) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) diff --git a/src/enrich/__init__.py b/src/enrich/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/enrich/mbid.py b/src/enrich/mbid.py new file mode 100644 index 0000000..51bca6d --- /dev/null +++ b/src/enrich/mbid.py @@ -0,0 +1,143 @@ +"""Resolve a MusicBrainz recording MBID for each track WITHOUT touching files. + +Strategy per the matching design: + 1. If the track has an ISRC (Spotify), resolve it directly via MB's ISRC + endpoint β€” this is exact. + 2. Otherwise (iTunes / local), text-search MB by artist + title and accept + the top hit only above a score threshold. + +Incremental + cheap: + - Only tracks with mbid_status NULL (or 'error', to retry) are processed. + - Every distinct lookup (ISRC or normalized text) is cached in mb_cache, + including negative results, so duplicates across sources cost no network + and re-runs never repeat a query. + - MB's ~1 req/s limit is respected by musicbrainzngs; commits happen + periodically so an interrupted run resumes where it left off. +""" +from datetime import datetime + +import musicbrainzngs as mb +from sqlalchemy import or_ +from tqdm import tqdm + +from Env import mb_contact +from src.db.database import session_scope +from src.db.models import MbCache, Track, MB_ERROR, MB_FOUND, MB_NOTFOUND +from src.match.normalize import normalize_artist, normalize_title + +TEXT_SCORE_THRESHOLD = 90 +_COMMIT_EVERY = 20 +_agent_ready = False + + +def _ensure_agent(): + global _agent_ready + if not _agent_ready: + mb.set_useragent("MusicIndexer", "0.1", mb_contact) + mb.set_rate_limit(1.0, 1) # 1 request / second + _agent_ready = True + + +def _score(rec): + try: + return int(rec.get("ext:score", rec.get("score", 0))) + except (TypeError, ValueError): + return 0 + + +def _lookup_isrc(isrc): + try: + res = mb.get_recordings_by_isrc(isrc) + except mb.ResponseError as exc: + if getattr(getattr(exc, "cause", None), "code", None) == 404: + return None, None, MB_NOTFOUND + raise + recs = res.get("isrc", {}).get("recording-list", []) + return (recs[0]["id"], None, MB_FOUND) if recs else (None, None, MB_NOTFOUND) + + +def _lookup_text(artist, title): + res = mb.search_recordings(artist=artist, recording=title, limit=5) + recs = res.get("recording-list", []) + if not recs: + return None, None, MB_NOTFOUND + best = max(recs, key=_score) + score = _score(best) + if score >= TEXT_SCORE_THRESHOLD: + return best["id"], score, MB_FOUND + return None, score, MB_NOTFOUND + + +def _cached(session, key, lookup): + """Return (mbid, score, status) from cache or by calling `lookup` once.""" + row = session.get(MbCache, key) + if row is not None: + return row.mbid, row.score, row.status + mbid, score, status = lookup() + session.add(MbCache(query_key=key, mbid=mbid, score=score, + status=status, fetched_at=datetime.utcnow())) + return mbid, score, status + + +def _resolve_one(session, track): + """(mbid, score, status, source). + + Text search on NORMALIZED artist/title is the primary lookup so that the + same song from different sources issues an identical query -> identical + recording MBID (and a shared cache entry). ISRC is an exact fallback when + the text search finds nothing. + """ + na, nt = normalize_artist(track.artist or ""), normalize_title(track.title or "") + if na and nt: + key = f"txt:{na}|{nt}" + mbid, score, status = _cached(session, key, lambda: _lookup_text(na, nt)) + if status == MB_FOUND: + return mbid, score, MB_FOUND, "text" + + if track.isrc: + mbid, score, status = _cached( + session, f"isrc:{track.isrc}", lambda: _lookup_isrc(track.isrc) + ) + if status == MB_FOUND: + return mbid, score, MB_FOUND, "isrc" + + return None, None, MB_NOTFOUND, ("text" if na and nt else None) + + +def resolve_mbids(limit=None, source=None, retry_errors=True): + """Resolve MBIDs for unresolved (and optionally errored) tracks.""" + _ensure_agent() + stats = {MB_FOUND: 0, MB_NOTFOUND: 0, MB_ERROR: 0} + with session_scope() as session: + status_filter = ( + or_(Track.mbid_status.is_(None), Track.mbid_status == MB_ERROR) + if retry_errors else Track.mbid_status.is_(None) + ) + query = session.query(Track).filter(status_filter) + if source: + query = query.filter(Track.source == source) + tracks = query.order_by(Track.id).all() + if limit: + tracks = tracks[:limit] + + for i, track in enumerate(tqdm(tracks, desc="resolve MBID", unit="track")): + try: + mbid, score, status, src = _resolve_one(session, track) + track.mbid = mbid + track.mbid_status = status + track.mbid_source = src + stats[status] = stats.get(status, 0) + 1 + except mb.WebServiceError: + track.mbid_status = MB_ERROR + stats[MB_ERROR] += 1 + if (i + 1) % _COMMIT_EVERY == 0: + session.commit() + return stats + + +def unresolved_count(source=None): + with session_scope() as session: + q = session.query(Track).filter(Track.mbid_status.is_(None)) + if source: + q = q.filter(Track.source == source) + return q.count() diff --git a/src/ingest/__init__.py b/src/ingest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ingest/ingest.py b/src/ingest/ingest.py new file mode 100644 index 0000000..86ea3f2 --- /dev/null +++ b/src/ingest/ingest.py @@ -0,0 +1,130 @@ +"""Load prefetched JSON (work_dir/latest-1) into the DB. + +This stands in for the live backends while keeping the same contract they will +use: an idempotent UPSERT keyed by (source, source_id). Re-running updates only +changed descriptive fields and NEVER discards enrichment (mbid/mbid_status) or +rows that disappeared from a single export. +""" +import json + +from tqdm import tqdm + +from Env import prefetched_dir +from src.db.database import session_scope +from src.db.models import ( + Track, + SOURCE_ITUNES, + SOURCE_LOCAL, + SOURCE_SPOTIFY, +) +from src.match.normalize import norm_key + + +def _load(path): + with open(path, encoding="utf-8") as fh: + return json.load(fh) + + +def _id_name_maps(lib): + artists = {a["id"]: a.get("title") for a in lib.get("artists", [])} + albums = {al["id"]: al.get("title") for al in lib.get("albums", [])} + return artists, albums + + +def _spotify_isrc_map(spotify_dir): + """spotify_track_id -> ISRC, gathered from every raw payload that carries + full track objects (liked, top, playlist tracks).""" + out = {} + for fname in ("playlistTracks.json", "top_tracks.json", "tracks.json"): + path = spotify_dir / fname + if not path.exists(): + continue + data = _load(path) + items = data if isinstance(data, list) else [v for lst in data.values() for v in lst] + for item in items: + track = item.get("track") if isinstance(item, dict) and "track" in item else item + if track and track.get("id"): + isrc = (track.get("external_ids") or {}).get("isrc") + if isrc: + out[track["id"]] = isrc + return out + + +class _Upserter: + """Preloads existing rows for a source so upserts are O(1) lookups.""" + + def __init__(self, session, source): + self.session = session + self.source = source + self.existing = { + t.source_id: t + for t in session.query(Track).filter_by(source=source).all() + } + self.n_new = 0 + self.n_updated = 0 + + def upsert(self, source_id, *, isrc=None, **fields): + track = self.existing.get(source_id) + if track is None: + track = Track(source=self.source, source_id=source_id, isrc=isrc, **fields) + self.session.add(track) + self.existing[source_id] = track + self.n_new += 1 + return + # Existing: refresh descriptive fields, preserve enrichment unless the + # ISRC actually changed (then the cached MBID is stale). + for key, value in fields.items(): + setattr(track, key, value) + if isrc is not None and isrc != track.isrc: + track.isrc = isrc + track.mbid = None + track.mbid_status = None + track.mbid_source = None + elif isrc is not None: + track.isrc = isrc + self.n_updated += 1 + + +def _ingest_library(session, source, lib, *, source_id_field, isrc_map=None): + artists, albums = _id_name_maps(lib) + up = _Upserter(session, source) + for t in tqdm(lib.get("tracks", []), desc=f"ingest {source}", unit="track"): + source_id = t.get(source_id_field) or t.get("id") + if not source_id: + continue + names = [artists.get(aid, aid) for aid in t.get("artists", []) if aid] + names = [n for n in names if n] + primary = names[0] if names else None + title = t.get("title") + up.upsert( + source_id, + title=title, + artist=primary, + artists=", ".join(names), + album=albums.get(t.get("album"), t.get("album")), + duration_ms=t.get("duration_ms"), + play_count=t.get("play_count") or 0, + local_path=t.get("local_path"), + norm_key=norm_key(primary or "", title or ""), + isrc=(isrc_map or {}).get(source_id), + ) + return up.n_new, up.n_updated + + +def ingest_all(): + base = prefetched_dir + results = {} + with session_scope() as session: + results[SOURCE_SPOTIFY] = _ingest_library( + session, SOURCE_SPOTIFY, _load(base / "spotify_library.json"), + source_id_field="id", isrc_map=_spotify_isrc_map(base / "spotify"), + ) + results[SOURCE_ITUNES] = _ingest_library( + session, SOURCE_ITUNES, _load(base / "itunes_library.json"), + source_id_field="id", + ) + results[SOURCE_LOCAL] = _ingest_library( + session, SOURCE_LOCAL, _load(base / "local_library.json"), + source_id_field="local_path", + ) + return results diff --git a/src/match/__init__.py b/src/match/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/match/matcher.py b/src/match/matcher.py new file mode 100644 index 0000000..f7b9909 --- /dev/null +++ b/src/match/matcher.py @@ -0,0 +1,93 @@ +"""Cross-source matching: decide which Spotify/iTunes tracks you do NOT have +locally, using real identifiers first and normalized text as a fallback. + +A "wanted" track (Spotify or iTunes) counts as present locally if either: + - its recording MBID equals a local track's MBID -> method "mbid" (certain) + - its normalized artist|title equals a local one -> method "text" (fuzzy) +Otherwise it is reported as missing. +""" +from src.db.database import session_scope +from src.db.models import ( + SOURCE_ITUNES, + SOURCE_LOCAL, + SOURCE_SPOTIFY, + Track, +) + +WANTED_SOURCES = (SOURCE_SPOTIFY, SOURCE_ITUNES) + + +def _local_index(session): + local = session.query(Track).filter_by(source=SOURCE_LOCAL).all() + isrcs = {t.isrc for t in local if t.isrc} + mbids = {t.mbid for t in local if t.mbid} + norms = {t.norm_key for t in local if t.norm_key} + return isrcs, mbids, norms + + +def _classify(track, local_isrcs, local_mbids, local_norms): + """Return (has_local: bool, method). Tiers, most certain first: + ISRC > recording MBID > normalized text.""" + if track.isrc and track.isrc in local_isrcs: + return True, "isrc" + if track.mbid and track.mbid in local_mbids: + return True, "mbid" + if track.norm_key and track.norm_key in local_norms: + return True, "text" + return False, None + + +def _row(track, method): + return { + "source": track.source, + "artist": track.artist, + "title": track.title, + "album": track.album, + "isrc": track.isrc, + "mbid": track.mbid, + "matched_by": method, + "spotify_id": track.source_id if track.source == SOURCE_SPOTIFY else None, + "play_count": track.play_count, + } + + +def classify_all(sources=WANTED_SOURCES): + """Return (missing_rows, present_rows) for the given wanted sources.""" + with session_scope() as session: + local_isrcs, local_mbids, local_norms = _local_index(session) + wanted = session.query(Track).filter(Track.source.in_(sources)).all() + missing, present = [], [] + for track in wanted: + has_local, method = _classify(track, local_isrcs, local_mbids, local_norms) + (present if has_local else missing).append(_row(track, method)) + return missing, present + + +def missing_tracks(sources=WANTED_SOURCES): + return classify_all(sources)[0] + + +def summary(): + """Per-source counts: total / present-by-mbid / present-by-text / missing, + plus how many tracks carry an MBID at all (enrichment coverage).""" + with session_scope() as session: + local_isrcs, local_mbids, local_norms = _local_index(session) + out = {} + for source in WANTED_SOURCES: + tracks = session.query(Track).filter_by(source=source).all() + counts = {"total": len(tracks), "present_mbid": 0, + "present_text": 0, "missing": 0, "has_mbid": 0} + for track in tracks: + if track.mbid: + counts["has_mbid"] += 1 + has_local, method = _classify(track, local_isrcs, local_mbids, local_norms) + if not has_local: + counts["missing"] += 1 + elif method == "text": + counts["present_text"] += 1 + else: # isrc or mbid -> ID-certain + counts["present_mbid"] += 1 + out[source] = counts + local_total = session.query(Track).filter_by(source=SOURCE_LOCAL).count() + out[SOURCE_LOCAL] = {"total": local_total, "has_mbid": len(local_mbids)} + return out diff --git a/src/match/normalize.py b/src/match/normalize.py new file mode 100644 index 0000000..ebb50a8 --- /dev/null +++ b/src/match/normalize.py @@ -0,0 +1,48 @@ +"""Text normalization for the fuzzy/text-fallback match key. + +Goal: collapse cosmetic differences between sources (accents, case, "feat." +clauses, "- Remastered 2011" / "(Live)" qualifiers, punctuation) so the same +song lands on the same key, while keeping genuinely different versions +(e.g. Remix) distinct. +""" +import re +import unicodedata + +_FEAT = re.compile(r"\s*[\(\[]?\s*(feat|ft|featuring|with)\.?\s.*$", re.IGNORECASE) +_BRACKETS = re.compile(r"\s*[\(\[][^\)\]]*[\)\]]") +# trailing "- Remastered 2011", "- Live", "- Mono", "- 2009 Remaster", etc. +_DASH_QUALIFIER = re.compile( + r"\s*-\s*.*?\b(remaster(ed)?|remastered|live|mono|stereo|version|edit|" + r"anniversary|deluxe|bonus|radio|single)\b.*$", + re.IGNORECASE, +) +_NON_ALNUM = re.compile(r"[^\w\s]", re.UNICODE) +_WS = re.compile(r"\s+") + + +def strip_accents(s: str) -> str: + s = unicodedata.normalize("NFKD", s) + return "".join(c for c in s if not unicodedata.combining(c)) + + +def _base(s: str) -> str: + s = strip_accents(s or "").lower() + s = _NON_ALNUM.sub(" ", s) + return _WS.sub(" ", s).strip() + + +def normalize_artist(name: str) -> str: + return _base(name) + + +def normalize_title(title: str) -> str: + s = title or "" + s = _FEAT.sub("", s) + s = _DASH_QUALIFIER.sub("", s) + s = _BRACKETS.sub("", s) + return _base(s) + + +def norm_key(artist: str, title: str) -> str: + """Identity key when no recording MBID is available.""" + return f"{normalize_artist(artist)}|{normalize_title(title)}"