diff --git a/.env.example b/.env.example index 27f5a90..b5fecaa 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,18 @@ -# Copy to `.env` and fill in, then `source .env` before running. -# Get these from https://developer.spotify.com/dashboard (your app's settings). -# Register `http://127.0.0.1:8888` as a Redirect URI in that app. +# Copy to `.env`, fill in, then `source .env` before running. + +# --- the three things you point at your own data --------------------------- +# Folder of audio files to scan (tags incl. ISRC / MusicBrainz id are read). +export LOCAL_LIBRARY_PATH=~/Music +# iTunes/Music "Library.xml" you export manually (File > Library > Export Library…). +export ITUNES_XML_PATH=~/Music/iTunes\ Library.xml +# Storage root: the database and cached Spotify responses live here. +export DATA_DIR=./data + +# --- Spotify credentials (only needed for live fetching) ------------------- +# https://developer.spotify.com/dashboard — register http://127.0.0.1:8888 as a Redirect URI. export SPOTIFY_CLIENT_ID=your_client_id_here export SPOTIFY_CLIENT_SECRET=your_client_secret_here -# Optional overrides: -# export MUSIC_LIBRARY_PATH=~/Music -# export WORK_DIR=./work_dir/new +# --- optional -------------------------------------------------------------- +# export MB_CONTACT=you@example.com # MusicBrainz User-Agent contact +# export DATABASE_URL=postgresql+psycopg://u:p@localhost/mi # use Postgres instead of SQLite diff --git a/.gitignore b/.gitignore index 11a26b7..855bc79 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ secret .env .venv *.db +/data/ work_dir old tmp \ No newline at end of file diff --git a/Env.py b/Env.py index 304d2a5..ef038ad 100644 --- a/Env.py +++ b/Env.py @@ -1,25 +1,39 @@ +"""Configuration. The three things you point at your own data: + + LOCAL_LIBRARY_PATH folder of audio files to scan + ITUNES_XML_PATH iTunes/Music "Library.xml" you exported manually + DATA_DIR storage root: the database + cached fetches live here + +Everything can be overridden via environment variables (see .env.example). +""" import os from pathlib import Path -# Local config. Paths can be overridden via environment variables. -local_library_path = Path(os.environ.get("MUSIC_LIBRARY_PATH", "~/Music")).expanduser() -work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new")) -# Spotify app credentials. NEVER hardcode these here — set them in the -# environment (see .env.example) so they don't end up in version control. -# export SPOTIFY_CLIENT_ID=... -# export SPOTIFY_CLIENT_SECRET=... +def _path(env, default): + return Path(os.environ.get(env, default)).expanduser() + + +# --- where YOUR data is read from / written to ------------------------------- +# Local music library to scan (tags incl. ISRC/MusicBrainz id are read). +local_library_path = _path("LOCAL_LIBRARY_PATH", "~/Music") + +# iTunes / Apple Music library export. In the Music app: +# File -> Library -> Export Library… (produces an XML/plist file). +itunes_xml_path = _path("ITUNES_XML_PATH", "~/Music/iTunes Library.xml") + +# Storage root — the database and cached Spotify responses go here. +data_dir = _path("DATA_DIR", "./data") + +# --- derived storage locations ----------------------------------------------- +spotify_cache_dir = data_dir / "spotify" # raw Spotify API responses +database_url = os.environ.get( + "DATABASE_URL", f"sqlite:///{(data_dir / 'musicindexer.db').as_posix()}" +) + +# --- Spotify app credentials (only needed for live fetching) ----------------- client_id = os.environ.get("SPOTIFY_CLIENT_ID") client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET") -# Persistent store. Defaults to a local SQLite file; point DATABASE_URL at a -# Postgres instance (postgresql+psycopg://user:pass@host/db) to switch with no -# code changes. -database_url = os.environ.get("DATABASE_URL", "sqlite:///./musicindexer.db") - -# Folder of prefetched JSON used to (re)build the DB without hitting the network. -prefetched_dir = Path(os.environ.get("PREFETCHED_DIR", "./work_dir/latest-1")) - -# Contact email sent to MusicBrainz in the User-Agent (be a good API citizen). +# Contact string sent to MusicBrainz in the User-Agent. mb_contact = os.environ.get("MB_CONTACT", "music-indexer@example.com") - diff --git a/Gui.py b/Gui.py index 9003e57..25dd763 100644 --- a/Gui.py +++ b/Gui.py @@ -45,7 +45,7 @@ def main(): c1, c2, c3, c4, c5 = st.columns(5) c1.metric("Total", s["total"]) c2.metric("Missing locally", s["missing"]) - c3.metric("Have (MBID)", s["present_mbid"]) + c3.metric("Have (ID)", s["present_mbid"]) c4.metric("Have (text)", s["present_text"]) coverage = (s["has_mbid"] / s["total"] * 100) if s["total"] else 0 c5.metric("MBID coverage", f"{coverage:.0f}%") @@ -54,8 +54,8 @@ def main(): st.caption( f"Local library: {local.get('total', 0)} tracks, " f"{local.get('has_mbid', 0)} with a resolved MBID. " - "‘Have (MBID)’ = certain ID match; ‘Have (text)’ = normalized " - "artist/title match (run `enrich` to upgrade text matches to ID)." + "‘Have (ID)’ = certain ISRC/MBID match; ‘Have (text)’ = normalized " + "artist/title match (run `fetch`/`enrich` to upgrade text matches to ID)." ) st.divider() diff --git a/README.md b/README.md index 6e087c4..89f0472 100644 --- a/README.md +++ b/README.md @@ -48,68 +48,73 @@ source .venv/bin/activate pip install -r requirements.txt ``` -### Credentials - -Create an app at https://developer.spotify.com/dashboard and add -`http://127.0.0.1:8888` as a **Redirect URI**. Credentials are read from the -environment — never commit them. +### Configure (`.env`) ```bash -cp .env.example .env -# edit .env with your client id/secret -source .env +cp .env.example .env # then edit and `source .env` ``` -`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (only needed for -live fetching), plus optional `DATABASE_URL`, `PREFETCHED_DIR` and `MB_CONTACT`. +Three things you point at your own data: -### Database +| Variable | What | +|----------|------| +| `LOCAL_LIBRARY_PATH` | folder of audio files to scan (ISRC/MusicBrainz tags are read) | +| `ITUNES_XML_PATH` | the `Library.xml` you export from Music (File → Library → Export Library…) | +| `DATA_DIR` | storage root — the database and cached Spotify responses live here | -Defaults to a local SQLite file (`musicindexer.db`). To use Postgres instead, -set one env var — no code changes: +`SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` are only needed for live Spotify +fetching (register `http://127.0.0.1:8888` as a Redirect URI in your +[Spotify app](https://developer.spotify.com/dashboard)). Set +`DATABASE_URL=postgresql+psycopg://…` to use Postgres instead of SQLite — no +code change. + +## Two workflows ```bash -export DATABASE_URL=postgresql+psycopg://user:pass@localhost:5432/musicindexer +python main.py fetch # do all the work, incrementally, and report +python main.py display # open the GUI to see what you're missing ``` -## Usage +**`fetch`** pulls Spotify, scans the local library, parses the iTunes export, +resolves MBIDs, and matches — logging progress at each step. The **first run is +slow** (full scan + MusicBrainz lookups at ~1 req/s); **later runs are quick** +because only changed files, the iTunes export if it changed, and still-unresolved +MBIDs are touched. Useful flags/helpers: ```bash -python main.py ingest # load prefetched data (work_dir/latest-1) into the DB -python main.py enrich # resolve MusicBrainz MBIDs (resumable; ~1 req/s) -python main.py match # print present/missing summary -./gui.sh # browse what you're missing +python main.py fetch --enrich-limit 500 # cap MBID lookups this run +python main.py enrich # resume MBID resolution only +python main.py match # print the present/missing summary ``` -`ingest` and the GUI work immediately (text-based matching). `enrich` queries -MusicBrainz only for tracks that don't have an MBID yet, caches every result, -and is **resumable** — if it's interrupted, just run it again to continue. Run -`python main.py enrich --limit 300` to do a quick partial pass, or -`--source local` to restrict to one source. - -> Live fetching (`SpotifyWebAPI`, the local indexer) is unchanged and still -> available; this redesign reads prefetched JSON so the pipeline can be built -> and demoed without network access. +> ISRC/MBID matching works as soon as `fetch` runs (local tags + Spotify ISRC). +> MusicBrainz enrichment fills in the rest (iTunes, untagged local files); it's +> cached and resumable, so you can stop/continue any time. ## Layout ``` -main.py CLI: ingest / enrich / match +main.py CLI: fetch / display / enrich / match Gui.py / gui.sh Streamlit GUI (missing-tracks viewer) -Env.py config (DB URL, paths, credentials from env) +Env.py config: LOCAL_LIBRARY_PATH / ITUNES_XML_PATH / DATA_DIR src/ + workflows.py run_fetch (the whole pipeline) + run_display db/ SQLAlchemy models + engine/session - ingest/ load prefetched JSON into the DB (upsert) + ingest/ + spotify_raw.py live fetch + ingest raw Spotify JSON (ISRC) + local_scan.py incremental file scan (reads ISRC/MBID tags) + itunes_xml.py parse the iTunes Library.xml (plist) + tags.py read ISRC / recording MBID / duration from a file enrich/ MusicBrainz MBID resolver (cached, resumable) match/ normalization + cross-source matcher - backends/ live fetchers/parsers (spotify, local, itunes, ...) - Library.py in-memory data model used by the backends -work_dir/latest-1/ prefetched JSON used by `ingest` (gitignored) -musicindexer.db the database (gitignored) + backends/ lower-level Spotify Web API client / OAuth +$DATA_DIR/ database + cached Spotify responses (gitignored) ``` ## Notes -- The DB is the source of truth; `work_dir/` JSON is only an ingest source. +- The DB is the source of truth; everything is incremental and re-runnable. +- Local matching is real-ID where files are tagged (ISRC/MusicBrainz, e.g. via + Picard); untagged files fall back to MusicBrainz text lookup, then text. - Only currently-supported Spotify endpoints are used (no deprecated audio-features/recommendations/etc.). diff --git a/main.py b/main.py index 4d2e5ec..a9f4f93 100644 --- a/main.py +++ b/main.py @@ -1,46 +1,47 @@ -"""MusicIndexer pipeline CLI. +"""MusicIndexer CLI. Two main workflows plus granular helpers. -Everything is incremental and DB-backed (see Env.database_url). Typical flow: + python main.py fetch # do all the work: pull + scan + parse + enrich + match + python main.py display # open the GUI to see what you're missing - python main.py ingest # load prefetched data into the DB (upsert) - python main.py enrich # resolve recording MBIDs (resumable, cached) - python main.py match # print present/missing summary - ./gui.sh # browse what you're missing - -`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it -just continues where it stopped. +`fetch` is incremental and DB-backed (see Env.py): the first run is slow +(initial scan + MusicBrainz lookups), later runs only touch what changed. +Helpers: `enrich` (resume MBID resolution), `match` (print the summary). """ import argparse -import json - -from src.db.database import init_db +import logging -def cmd_init(args): - init_db() - print("Database initialized.") +def _setup_logging(): + logging.basicConfig(level=logging.INFO, format="%(message)s") -def cmd_ingest(args): - init_db() - from src.ingest.ingest import ingest_all +def cmd_fetch(args): + _setup_logging() + from src.workflows import run_fetch - results = ingest_all() - for source, (new, updated) in results.items(): - print(f"{source:8} +{new} new, {updated} updated") + run_fetch(enrich_limit=args.enrich_limit) + + +def cmd_display(args): + from src.workflows import run_display + + run_display() def cmd_enrich(args): - init_db() + _setup_logging() + from src.db.database import init_db from src.enrich.mbid import resolve_mbids, unresolved_count + init_db() print(f"unresolved before: {unresolved_count(args.source)}") - stats = resolve_mbids(limit=args.limit, source=args.source) - print(f"resolved this run: {stats}") + print(f"resolved this run: {resolve_mbids(limit=args.limit, source=args.source)}") print(f"unresolved after: {unresolved_count(args.source)}") def cmd_match(args): + import json + from src.match.matcher import summary print(json.dumps(summary(), indent=2)) @@ -50,14 +51,15 @@ def build_parser(): parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__) sub = parser.add_subparsers(dest="cmd", required=True) - sub.add_parser("init", help="create database tables") - sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)") + fetch = sub.add_parser("fetch", help="incrementally fetch + scan + enrich + match") + fetch.add_argument("--enrich-limit", type=int, default=None, + help="cap MBID lookups this run (default: resolve all)") - enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks") - enrich.add_argument("--limit", type=int, default=None, - help="cap how many tracks to resolve this run") - enrich.add_argument("--source", choices=["spotify", "itunes", "local"], - default=None, help="restrict to one source") + sub.add_parser("display", help="open the Streamlit GUI") + + enrich = sub.add_parser("enrich", help="resume MBID resolution only") + enrich.add_argument("--limit", type=int, default=None) + enrich.add_argument("--source", choices=["spotify", "itunes", "local"], default=None) sub.add_parser("match", help="print present/missing summary") return parser @@ -65,13 +67,12 @@ def build_parser(): def main(): args = build_parser().parse_args() - handlers = { - "init": cmd_init, - "ingest": cmd_ingest, + { + "fetch": cmd_fetch, + "display": cmd_display, "enrich": cmd_enrich, "match": cmd_match, - } - handlers[args.cmd](args) + }[args.cmd](args) if __name__ == "__main__": diff --git a/src/db/database.py b/src/db/database.py index 2016900..90a41c6 100644 --- a/src/db/database.py +++ b/src/db/database.py @@ -17,7 +17,10 @@ def get_engine(): def init_db(): - """Create tables if they don't exist (safe to call every run).""" + """Create the storage dir + tables if missing (safe to call every run).""" + from Env import data_dir + + data_dir.mkdir(parents=True, exist_ok=True) Base.metadata.create_all(_engine) diff --git a/src/enrich/mbid.py b/src/enrich/mbid.py index 51bca6d..f2a64cf 100644 --- a/src/enrich/mbid.py +++ b/src/enrich/mbid.py @@ -14,6 +14,7 @@ Incremental + cheap: - MB's ~1 req/s limit is respected by musicbrainzngs; commits happen periodically so an interrupted run resumes where it left off. """ +import logging from datetime import datetime import musicbrainzngs as mb @@ -35,6 +36,8 @@ def _ensure_agent(): if not _agent_ready: mb.set_useragent("MusicIndexer", "0.1", mb_contact) mb.set_rate_limit(1.0, 1) # 1 request / second + # musicbrainzngs logs a warning per unparsed XML element; quiet it. + logging.getLogger("musicbrainzngs").setLevel(logging.ERROR) _agent_ready = True @@ -80,20 +83,13 @@ def _cached(session, key, lookup): def _resolve_one(session, track): - """(mbid, score, status, source). + """(mbid, score, source). - Text search on NORMALIZED artist/title is the primary lookup so that the - same song from different sources issues an identical query -> identical - recording MBID (and a shared cache entry). ISRC is an exact fallback when - the text search finds nothing. + ISRC first (exact recording), then text search on NORMALIZED artist/title + (so the same song across sources shares one query + one cached lookup). + Used to enrich Spotify / iTunes / untagged-local; tagged local files + already carry their MBID from the file and skip this. """ - na, nt = normalize_artist(track.artist or ""), normalize_title(track.title or "") - if na and nt: - key = f"txt:{na}|{nt}" - mbid, score, status = _cached(session, key, lambda: _lookup_text(na, nt)) - if status == MB_FOUND: - return mbid, score, MB_FOUND, "text" - if track.isrc: mbid, score, status = _cached( session, f"isrc:{track.isrc}", lambda: _lookup_isrc(track.isrc) @@ -101,7 +97,13 @@ def _resolve_one(session, track): if status == MB_FOUND: return mbid, score, MB_FOUND, "isrc" - return None, None, MB_NOTFOUND, ("text" if na and nt else None) + na, nt = normalize_artist(track.artist or ""), normalize_title(track.title or "") + if na and nt: + key = f"txt:{na}|{nt}" + mbid, score, status = _cached(session, key, lambda: _lookup_text(na, nt)) + return mbid, score, status, "text" + + return None, None, MB_NOTFOUND, None def resolve_mbids(limit=None, source=None, retry_errors=True): @@ -117,7 +119,7 @@ def resolve_mbids(limit=None, source=None, retry_errors=True): if source: query = query.filter(Track.source == source) tracks = query.order_by(Track.id).all() - if limit: + if limit is not None: tracks = tracks[:limit] for i, track in enumerate(tqdm(tracks, desc="resolve MBID", unit="track")): diff --git a/src/ingest/ingest.py b/src/ingest/ingest.py deleted file mode 100644 index 86ea3f2..0000000 --- a/src/ingest/ingest.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Load prefetched JSON (work_dir/latest-1) into the DB. - -This stands in for the live backends while keeping the same contract they will -use: an idempotent UPSERT keyed by (source, source_id). Re-running updates only -changed descriptive fields and NEVER discards enrichment (mbid/mbid_status) or -rows that disappeared from a single export. -""" -import json - -from tqdm import tqdm - -from Env import prefetched_dir -from src.db.database import session_scope -from src.db.models import ( - Track, - SOURCE_ITUNES, - SOURCE_LOCAL, - SOURCE_SPOTIFY, -) -from src.match.normalize import norm_key - - -def _load(path): - with open(path, encoding="utf-8") as fh: - return json.load(fh) - - -def _id_name_maps(lib): - artists = {a["id"]: a.get("title") for a in lib.get("artists", [])} - albums = {al["id"]: al.get("title") for al in lib.get("albums", [])} - return artists, albums - - -def _spotify_isrc_map(spotify_dir): - """spotify_track_id -> ISRC, gathered from every raw payload that carries - full track objects (liked, top, playlist tracks).""" - out = {} - for fname in ("playlistTracks.json", "top_tracks.json", "tracks.json"): - path = spotify_dir / fname - if not path.exists(): - continue - data = _load(path) - items = data if isinstance(data, list) else [v for lst in data.values() for v in lst] - for item in items: - track = item.get("track") if isinstance(item, dict) and "track" in item else item - if track and track.get("id"): - isrc = (track.get("external_ids") or {}).get("isrc") - if isrc: - out[track["id"]] = isrc - return out - - -class _Upserter: - """Preloads existing rows for a source so upserts are O(1) lookups.""" - - def __init__(self, session, source): - self.session = session - self.source = source - self.existing = { - t.source_id: t - for t in session.query(Track).filter_by(source=source).all() - } - self.n_new = 0 - self.n_updated = 0 - - def upsert(self, source_id, *, isrc=None, **fields): - track = self.existing.get(source_id) - if track is None: - track = Track(source=self.source, source_id=source_id, isrc=isrc, **fields) - self.session.add(track) - self.existing[source_id] = track - self.n_new += 1 - return - # Existing: refresh descriptive fields, preserve enrichment unless the - # ISRC actually changed (then the cached MBID is stale). - for key, value in fields.items(): - setattr(track, key, value) - if isrc is not None and isrc != track.isrc: - track.isrc = isrc - track.mbid = None - track.mbid_status = None - track.mbid_source = None - elif isrc is not None: - track.isrc = isrc - self.n_updated += 1 - - -def _ingest_library(session, source, lib, *, source_id_field, isrc_map=None): - artists, albums = _id_name_maps(lib) - up = _Upserter(session, source) - for t in tqdm(lib.get("tracks", []), desc=f"ingest {source}", unit="track"): - source_id = t.get(source_id_field) or t.get("id") - if not source_id: - continue - names = [artists.get(aid, aid) for aid in t.get("artists", []) if aid] - names = [n for n in names if n] - primary = names[0] if names else None - title = t.get("title") - up.upsert( - source_id, - title=title, - artist=primary, - artists=", ".join(names), - album=albums.get(t.get("album"), t.get("album")), - duration_ms=t.get("duration_ms"), - play_count=t.get("play_count") or 0, - local_path=t.get("local_path"), - norm_key=norm_key(primary or "", title or ""), - isrc=(isrc_map or {}).get(source_id), - ) - return up.n_new, up.n_updated - - -def ingest_all(): - base = prefetched_dir - results = {} - with session_scope() as session: - results[SOURCE_SPOTIFY] = _ingest_library( - session, SOURCE_SPOTIFY, _load(base / "spotify_library.json"), - source_id_field="id", isrc_map=_spotify_isrc_map(base / "spotify"), - ) - results[SOURCE_ITUNES] = _ingest_library( - session, SOURCE_ITUNES, _load(base / "itunes_library.json"), - source_id_field="id", - ) - results[SOURCE_LOCAL] = _ingest_library( - session, SOURCE_LOCAL, _load(base / "local_library.json"), - source_id_field="local_path", - ) - return results diff --git a/src/ingest/itunes_xml.py b/src/ingest/itunes_xml.py new file mode 100644 index 0000000..03d57e3 --- /dev/null +++ b/src/ingest/itunes_xml.py @@ -0,0 +1,68 @@ +"""Ingest a manually-exported iTunes/Music Library.xml into the DB. + +The file is an Apple property list, so plistlib parses it natively. Upsert by +the iTunes persistent/numeric Track ID; skipped entirely if the file's mtime +hasn't changed since the last run. +""" +import logging +import os +import plistlib + +from tqdm import tqdm + +from Env import itunes_xml_path +from src.db.database import session_scope +from src.db.models import IngestState, SOURCE_ITUNES, Track +from src.match.normalize import norm_key + +log = logging.getLogger("musicindexer.itunes") +_STATE_KEY = "itunes_xml_mtime" + + +def ingest_itunes(xml_path=None, force=False): + xml_path = str(xml_path or itunes_xml_path) + if not os.path.isfile(xml_path): + log.warning("iTunes XML not found: %s — skipping", xml_path) + return {"new": 0, "updated": 0, "skipped": True} + + mtime = str(os.path.getmtime(xml_path)) + with session_scope() as session: + state = session.get(IngestState, _STATE_KEY) + if state and state.value == mtime and not force: + log.info("iTunes XML unchanged — skipping") + return {"new": 0, "updated": 0, "skipped": True} + + with open(xml_path, "rb") as fh: + plist = plistlib.load(fh) + raw_tracks = plist.get("Tracks", {}) + + counts = {"new": 0, "updated": 0, "skipped": False} + with session_scope() as session: + existing = {t.source_id: t for t in + session.query(Track).filter_by(source=SOURCE_ITUNES).all()} + for tid, tr in tqdm(raw_tracks.items(), desc="ingest itunes", unit="track"): + name = tr.get("Name") + artist = tr.get("Artist") or tr.get("Album Artist") + if not name: + continue + fields = dict( + title=name, artist=artist, artists=artist, album=tr.get("Album"), + duration_ms=tr.get("Total Time"), play_count=tr.get("Play Count") or 0, + norm_key=norm_key(artist or "", name), + ) + source_id = str(tid) + track = existing.get(source_id) + if track is None: + session.add(Track(source=SOURCE_ITUNES, source_id=source_id, **fields)) + counts["new"] += 1 + else: + for key, value in fields.items(): + setattr(track, key, value) + counts["updated"] += 1 + + state = session.get(IngestState, _STATE_KEY) or IngestState(key=_STATE_KEY) + state.value = mtime + session.merge(state) + + log.info("itunes: %(new)d new, %(updated)d updated", counts) + return counts diff --git a/src/ingest/local_scan.py b/src/ingest/local_scan.py new file mode 100644 index 0000000..beb1f08 --- /dev/null +++ b/src/ingest/local_scan.py @@ -0,0 +1,99 @@ +"""Incrementally scan the local music library into the DB. + +Only files whose mtime/size changed are re-read (tracked in file_state); files +that disappeared are pruned. Embedded ISRC and MusicBrainz recording MBID are +read from tags, so tagged tracks get real identifiers with no network. +""" +import logging +import os +from datetime import datetime + +from tqdm import tqdm + +from Env import local_library_path +from src.db.database import session_scope +from src.db.models import FileState, MB_FOUND, SOURCE_LOCAL, Track +from src.ingest.tags import read_track_tags +from src.match.normalize import norm_key + +log = logging.getLogger("musicindexer.local") + +MUSIC_EXTS = (".mp3", ".flac", ".m4a", ".ogg", ".opus", ".wav", ".aac") + + +def _iter_files(root): + for dirpath, _dirs, names in os.walk(root): + for name in names: + if os.path.splitext(name)[1].lower() in MUSIC_EXTS: + yield os.path.join(dirpath, name) + + +def scan_local(root=None): + root = str(root or local_library_path) + if not os.path.isdir(root): + log.warning("local library path not found: %s — skipping", root) + return {"new": 0, "updated": 0, "unchanged": 0, "removed": 0} + + files = list(_iter_files(root)) + counts = {"new": 0, "updated": 0, "unchanged": 0, "removed": 0} + + with session_scope() as session: + states = {fs.path: fs for fs in session.query(FileState).all()} + tracks = {t.source_id: t for t in + session.query(Track).filter_by(source=SOURCE_LOCAL).all()} + seen = set() + + for path in tqdm(files, desc="scan local", unit="file"): + seen.add(path) + stat = os.stat(path) + state = states.get(path) + if state and state.mtime == stat.st_mtime and state.size == stat.st_size: + counts["unchanged"] += 1 + continue + + tags = read_track_tags(path) + if not tags or not (tags["title"] or tags["artist"]): + continue + + fields = dict( + title=tags["title"], artist=tags["artist"], artists=tags["artists"], + album=tags["album"], duration_ms=tags["duration_ms"], + isrc=tags["isrc"], local_path=path, + norm_key=norm_key(tags["artist"] or "", tags["title"] or ""), + ) + track = tracks.get(path) + if track is None: + track = Track(source=SOURCE_LOCAL, source_id=path, **fields) + session.add(track) + tracks[path] = track + counts["new"] += 1 + else: + for key, value in fields.items(): + setattr(track, key, value) + counts["updated"] += 1 + # Tagged files come with their recording MBID -> resolved for free. + if tags["mbid"]: + track.mbid = tags["mbid"] + track.mbid_status = MB_FOUND + track.mbid_source = "tag" + + if state is None: + session.add(FileState(path=path, mtime=stat.st_mtime, + size=stat.st_size, indexed_at=datetime.utcnow())) + else: + state.mtime, state.size, state.indexed_at = ( + stat.st_mtime, stat.st_size, datetime.utcnow()) + + # Prune rows whose file no longer exists. + for path, track in tracks.items(): + if path not in seen and os.path.isabs(path): + session.delete(track) + counts["removed"] += 1 + for path, state in states.items(): + if path not in seen: + session.delete(state) + + log.info("local: %(new)d new, %(updated)d updated, %(unchanged)d unchanged, " + "%(removed)d removed (%(total)d files scanned)", + {**counts, "total": len(files)}) + return counts diff --git a/src/ingest/spotify_raw.py b/src/ingest/spotify_raw.py new file mode 100644 index 0000000..11c6dd5 --- /dev/null +++ b/src/ingest/spotify_raw.py @@ -0,0 +1,85 @@ +"""Ingest raw Spotify API responses (the JSON written by SpotifyWebAPI) into the +DB. Unions liked + top + playlist tracks, upserting by Spotify track id and +pulling the ISRC from external_ids — so Spotify tracks get a real identifier +for matching with no extra calls. +""" +import json +import logging +import os + +from tqdm import tqdm + +from Env import spotify_cache_dir +from src.db.database import session_scope +from src.db.models import SOURCE_SPOTIFY, Track +from src.match.normalize import norm_key + +log = logging.getLogger("musicindexer.spotify") + +# Files written by SpotifyWebAPI.fetch_data(), each holding full track objects. +_SOURCES = ("tracks.json", "top_tracks.json", "playlistTracks.json") + + +def _track_objects(spotify_dir): + by_id = {} + for fname in _SOURCES: + path = os.path.join(spotify_dir, fname) + if not os.path.isfile(path): + continue + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + items = data if isinstance(data, list) else [v for lst in data.values() for v in lst] + for item in items: + track = item.get("track") if isinstance(item, dict) and "track" in item else item + if track and track.get("id"): + by_id[track["id"]] = track + return by_id + + +def ingest_spotify(spotify_dir=None): + spotify_dir = str(spotify_dir or spotify_cache_dir) + if not os.path.isdir(spotify_dir): + log.warning("no Spotify cache at %s — skipping", spotify_dir) + return {"new": 0, "updated": 0} + + objects = _track_objects(spotify_dir) + counts = {"new": 0, "updated": 0} + with session_scope() as session: + existing = {t.source_id: t for t in + session.query(Track).filter_by(source=SOURCE_SPOTIFY).all()} + for spotify_id, t in tqdm(objects.items(), desc="ingest spotify", unit="track"): + names = [a.get("name") for a in t.get("artists", []) if a.get("name")] + primary = names[0] if names else None + title = t.get("name") + fields = dict( + title=title, artist=primary, artists=", ".join(names), + album=(t.get("album") or {}).get("name"), + duration_ms=t.get("duration_ms"), + isrc=(t.get("external_ids") or {}).get("isrc"), + norm_key=norm_key(primary or "", title or ""), + ) + track = existing.get(spotify_id) + if track is None: + session.add(Track(source=SOURCE_SPOTIFY, source_id=spotify_id, **fields)) + counts["new"] += 1 + else: + new_isrc = fields.pop("isrc") + for key, value in fields.items(): + setattr(track, key, value) + if new_isrc and new_isrc != track.isrc: # stale enrichment + track.isrc, track.mbid, track.mbid_status, track.mbid_source = ( + new_isrc, None, None, None) + elif new_isrc: + track.isrc = new_isrc + counts["updated"] += 1 + + log.info("spotify: %(new)d new, %(updated)d updated", counts) + return counts + + +def fetch_spotify_live(): + """Run the real OAuth Spotify fetch into the cache dir (requires creds).""" + from Env import client_id, client_secret, data_dir + from src.backends.spotify.SpotifyWebAPI import fetch_all + + fetch_all(client_id, client_secret, data_dir) # writes data_dir/spotify/*.json diff --git a/src/ingest/tags.py b/src/ingest/tags.py new file mode 100644 index 0000000..4da44d3 --- /dev/null +++ b/src/ingest/tags.py @@ -0,0 +1,84 @@ +"""Read matching-relevant metadata from an audio file, including the embedded +MusicBrainz recording MBID and ISRC when present (e.g. Picard-tagged files). + +Returns a flat dict so the local scanner can upsert without knowing tag formats. +""" +from mutagen import File as MutagenFile + +_MB_OWNER = "http://musicbrainz.org" + + +def _first(value): + if value is None: + return None + if isinstance(value, list): + return str(value[0]) if value else None + return str(value) + + +def _mp4_freeform(raw, key): + if key in raw: + try: + return _first([bytes(x).decode("utf-8", "ignore") for x in raw[key]]) + except Exception: + return None + return None + + +def read_track_tags(path): + try: + audio = MutagenFile(path) + except Exception: + return None + if audio is None: + return None + + duration_ms = None + if audio.info and getattr(audio.info, "length", None): + duration_ms = int(audio.info.length * 1000) + + # Easy view normalizes common keys across ID3/Vorbis/MP4. + etags = {} + try: + easy = MutagenFile(path, easy=True) + etags = easy.tags or {} + except Exception: + etags = {} + + def eget(key): + try: + return _first(etags.get(key)) + except Exception: + return None + + title = eget("title") + artist = eget("artist") + album = eget("album") + artist_values = etags.get("artist") if isinstance(etags.get("artist"), list) else None + artists = ", ".join(str(a) for a in artist_values) if artist_values else artist + isrc = eget("isrc") + mbid = eget("musicbrainz_trackid") # recording MBID + + raw = audio.tags + if raw is not None: + if mbid is None: + try: # ID3 stores the recording MBID in a UFID frame + for frame in raw.getall("UFID"): + if getattr(frame, "owner", None) == _MB_OWNER: + mbid = frame.data.decode("ascii", "ignore") + break + except Exception: + pass + mbid = mbid or _mp4_freeform(raw, "----:com.apple.iTunes:MusicBrainz Track Id") + if isrc is None: + isrc = _mp4_freeform(raw, "----:com.apple.iTunes:ISRC") + + return { + "title": title, + "artist": artist, + "artists": artists or artist, + "album": album, + "duration_ms": duration_ms, + "isrc": isrc or None, + "mbid": mbid or None, + } diff --git a/src/workflows.py b/src/workflows.py new file mode 100644 index 0000000..d96a37c --- /dev/null +++ b/src/workflows.py @@ -0,0 +1,86 @@ +"""Top-level workflows. + +`run_fetch` does *all* the work incrementally and reports: pull Spotify, scan +the local library, parse the iTunes export, resolve MBIDs, then match. The first +run is slow (initial scan + MusicBrainz lookups); later runs only touch what +changed, so they're quick. +""" +import logging +import subprocess +import sys +import time + +from Env import ( + client_id, + client_secret, + itunes_xml_path, + local_library_path, + spotify_cache_dir, +) +from src.db.database import init_db +from src.enrich.mbid import resolve_mbids, unresolved_count +from src.ingest.itunes_xml import ingest_itunes +from src.ingest.local_scan import scan_local +from src.ingest.spotify_raw import fetch_spotify_live, ingest_spotify +from src.match.matcher import summary + +log = logging.getLogger("musicindexer") + + +def _step(title): + log.info("") + log.info("== %s", title) + + +def run_fetch(enrich_limit=None): + started = time.time() + init_db() + log.info("MusicIndexer — incremental fetch") + log.info(" local library : %s", local_library_path) + log.info(" itunes xml : %s", itunes_xml_path) + log.info(" spotify cache : %s", spotify_cache_dir) + + _step("Spotify") + if client_id and client_secret: + log.info("credentials present -> live fetch") + try: + fetch_spotify_live() + except Exception as exc: # keep going on cached data + log.error("live fetch failed (%s); using cached responses", exc) + else: + log.info("no SPOTIFY_CLIENT_ID/SECRET -> using cached responses") + ingest_spotify() + + _step("iTunes") + ingest_itunes() + + _step("Local library") + scan_local() + + _step("MusicBrainz enrichment (incremental)") + todo = unresolved_count() + log.info("%d tracks still need an MBID", todo) + if todo: + stats = resolve_mbids(limit=enrich_limit) + log.info("resolved this run: %s", stats) + log.info("remaining unresolved: %d", unresolved_count()) + + _step("Matching") + summ = summary() + for source in ("spotify", "itunes"): + s = summ.get(source) + if s: + log.info("%-7s total=%-5d MISSING=%-5d have(id)=%-5d have(text)=%-5d", + source, s["total"], s["missing"], s["present_mbid"], s["present_text"]) + local = summ.get("local", {}) + log.info("local total=%-5d with_mbid=%d", local.get("total", 0), local.get("has_mbid", 0)) + + log.info("") + log.info("done in %.1fs — run `python main.py display` to browse what's missing", + time.time() - started) + return summ + + +def run_display(): + """Launch the Streamlit GUI.""" + subprocess.run([sys.executable, "-m", "streamlit", "run", "Gui.py"])