ilusha/MusicIndexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Real backends + fetch/display workflows; config paths

Browse source 
Wire the incremental pipeline to real sources and expose two workflows.

Config (Env.py): three explicit paths — LOCAL_LIBRARY_PATH (scan source),
ITUNES_XML_PATH (manual export), DATA_DIR (storage root: DB + Spotify cache).

Real, incremental ingesters:
- local_scan: walks the library, skips files whose mtime/size are unchanged,
  prunes deleted ones, and reads embedded ISRC + recording MBID from tags
  (tagged files get real IDs with no network), via tags.py.
- itunes_xml: parse the iTunes Library.xml as a plist; skip if mtime unchanged.
- spotify_raw: ingest raw Spotify JSON (ISRC from external_ids); live fetch
  wrapper for when credentials are set.

Workflows (src/workflows.py + main.py):
- `fetch`  — pull + scan + parse + enrich + match, with readable step logs.
  First run is slow (full scan + MB lookups); later runs only touch changes
  (verified: ~71s first, ~0.6s second).
- `display` — launch the GUI.

Also: enrich uses ISRC-first again (local MBIDs now come from tags), fix
--enrich-limit 0, quiet musicbrainzngs XML warnings, ignore /data/.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Шурупов Илья Викторович 2026-06-15 01:46:56 +03:00
parent 0e64d439c4
commit 03b98d9a1d
14 changed files with 570 additions and 243 deletions
Download patch file Download diff file Expand all files Collapse all files

21
.env.example
View file

@ -1,9 +1,18 @@
# Copy to `.env` and fill in, then `source .env` before running. # Copy to `.env`, fill in, then `source .env` before running.
# Get these from https://developer.spotify.com/dashboard (your app's settings).
# Register `http://127.0.0.1:8888` as a Redirect URI in that app. # --- the three things you point at your own data ---------------------------
# Folder of audio files to scan (tags incl. ISRC / MusicBrainz id are read).
export LOCAL_LIBRARY_PATH=~/Music
# iTunes/Music "Library.xml" you export manually (File > Library > Export Library…).
export ITUNES_XML_PATH=~/Music/iTunes\ Library.xml
# Storage root: the database and cached Spotify responses live here.
export DATA_DIR=./data
# --- Spotify credentials (only needed for live fetching) -------------------
# https://developer.spotify.com/dashboard — register http://127.0.0.1:8888 as a Redirect URI.
export SPOTIFY_CLIENT_ID=your_client_id_here export SPOTIFY_CLIENT_ID=your_client_id_here
export SPOTIFY_CLIENT_SECRET=your_client_secret_here export SPOTIFY_CLIENT_SECRET=your_client_secret_here
# Optional overrides: # --- optional --------------------------------------------------------------
# export MUSIC_LIBRARY_PATH=~/Music # export MB_CONTACT=you@example.com # MusicBrainz User-Agent contact
# export WORK_DIR=./work_dir/new # export DATABASE_URL=postgresql+psycopg://u:p@localhost/mi # use Postgres instead of SQLite

1
.gitignore vendored
View file

@ -5,6 +5,7 @@ secret
.env .env
.venv .venv
*.db *.db
/data/
work_dir work_dir
old old
tmp tmp

48
Env.py
View file

@ -1,25 +1,39 @@
"""Configuration. The three things you point at your own data:
LOCAL_LIBRARY_PATH folder of audio files to scan
ITUNES_XML_PATH iTunes/Music "Library.xml" you exported manually
DATA_DIR storage root: the database + cached fetches live here
Everything can be overridden via environment variables (see .env.example).
"""
import os import os
from pathlib import Path from pathlib import Path
# Local config. Paths can be overridden via environment variables.
local_library_path = Path(os.environ.get("MUSIC_LIBRARY_PATH", "~/Music")).expanduser()
work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new"))
# Spotify app credentials. NEVER hardcode these here — set them in the def _path(env, default):
# environment (see .env.example) so they don't end up in version control. return Path(os.environ.get(env, default)).expanduser()
# export SPOTIFY_CLIENT_ID=...
# export SPOTIFY_CLIENT_SECRET=...
# --- where YOUR data is read from / written to -------------------------------
# Local music library to scan (tags incl. ISRC/MusicBrainz id are read).
local_library_path = _path("LOCAL_LIBRARY_PATH", "~/Music")
# iTunes / Apple Music library export. In the Music app:
# File -> Library -> Export Library… (produces an XML/plist file).
itunes_xml_path = _path("ITUNES_XML_PATH", "~/Music/iTunes Library.xml")
# Storage root — the database and cached Spotify responses go here.
data_dir = _path("DATA_DIR", "./data")
# --- derived storage locations -----------------------------------------------
spotify_cache_dir = data_dir / "spotify" # raw Spotify API responses
database_url = os.environ.get(
"DATABASE_URL", f"sqlite:///{(data_dir / 'musicindexer.db').as_posix()}"
)
# --- Spotify app credentials (only needed for live fetching) -----------------
client_id = os.environ.get("SPOTIFY_CLIENT_ID") client_id = os.environ.get("SPOTIFY_CLIENT_ID")
client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET") client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET")
# Persistent store. Defaults to a local SQLite file; point DATABASE_URL at a # Contact string sent to MusicBrainz in the User-Agent.
# Postgres instance (postgresql+psycopg://user:pass@host/db) to switch with no
# code changes.
database_url = os.environ.get("DATABASE_URL", "sqlite:///./musicindexer.db")
# Folder of prefetched JSON used to (re)build the DB without hitting the network.
prefetched_dir = Path(os.environ.get("PREFETCHED_DIR", "./work_dir/latest-1"))
# Contact email sent to MusicBrainz in the User-Agent (be a good API citizen).
mb_contact = os.environ.get("MB_CONTACT", "music-indexer@example.com") mb_contact = os.environ.get("MB_CONTACT", "music-indexer@example.com")

6
Gui.py
View file

@ -45,7 +45,7 @@ def main():
c1, c2, c3, c4, c5 = st.columns(5) c1, c2, c3, c4, c5 = st.columns(5)
c1.metric("Total", s["total"]) c1.metric("Total", s["total"])
c2.metric("Missing locally", s["missing"]) c2.metric("Missing locally", s["missing"])
c3.metric("Have (MBID)", s["present_mbid"]) c3.metric("Have (ID)", s["present_mbid"])
c4.metric("Have (text)", s["present_text"]) c4.metric("Have (text)", s["present_text"])
coverage = (s["has_mbid"] / s["total"] * 100) if s["total"] else 0 coverage = (s["has_mbid"] / s["total"] * 100) if s["total"] else 0
c5.metric("MBID coverage", f"{coverage:.0f}%") c5.metric("MBID coverage", f"{coverage:.0f}%")
@ -54,8 +54,8 @@ def main():
st.caption( st.caption(
f"Local library: {local.get('total', 0)} tracks, " f"Local library: {local.get('total', 0)} tracks, "
f"{local.get('has_mbid', 0)} with a resolved MBID. " f"{local.get('has_mbid', 0)} with a resolved MBID. "
"‘Have (MBID)’ = certain ID match; ‘Have (text)’ = normalized " "‘Have (ID)’ = certain ISRC/MBID match; ‘Have (text)’ = normalized "
"artist/title match (run `enrich` to upgrade text matches to ID)." "artist/title match (run `fetch`/`enrich` to upgrade text matches to ID)."
) )
st.divider() st.divider()

77
README.md
View file

@ -48,68 +48,73 @@ source .venv/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
``` ```
### Credentials ### Configure (`.env`)
Create an app at https://developer.spotify.com/dashboard and add
`http://127.0.0.1:8888` as a **Redirect URI**. Credentials are read from the
environment — never commit them.
```bash ```bash
cp .env.example .env cp .env.example .env # then edit and `source .env`
# edit .env with your client id/secret
source .env
``` ```
`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (only needed for Three things you point at your own data:
live fetching), plus optional `DATABASE_URL`, `PREFETCHED_DIR` and `MB_CONTACT`.
### Database | Variable | What |
|----------|------|
| `LOCAL_LIBRARY_PATH` | folder of audio files to scan (ISRC/MusicBrainz tags are read) |
| `ITUNES_XML_PATH` | the `Library.xml` you export from Music (File → Library → Export Library…) |
| `DATA_DIR` | storage root — the database and cached Spotify responses live here |
Defaults to a local SQLite file (`musicindexer.db`). To use Postgres instead, `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` are only needed for live Spotify
set one env var — no code changes: fetching (register `http://127.0.0.1:8888` as a Redirect URI in your
[Spotify app](https://developer.spotify.com/dashboard)). Set
`DATABASE_URL=postgresql+psycopg://…` to use Postgres instead of SQLite — no
code change.
## Two workflows
```bash ```bash
export DATABASE_URL=postgresql+psycopg://user:pass@localhost:5432/musicindexer python main.py fetch # do all the work, incrementally, and report
python main.py display # open the GUI to see what you're missing
``` ```
## Usage **`fetch`** pulls Spotify, scans the local library, parses the iTunes export,
resolves MBIDs, and matches — logging progress at each step. The **first run is
slow** (full scan + MusicBrainz lookups at ~1 req/s); **later runs are quick**
because only changed files, the iTunes export if it changed, and still-unresolved
MBIDs are touched. Useful flags/helpers:
```bash ```bash
python main.py ingest # load prefetched data (work_dir/latest-1) into the DB python main.py fetch --enrich-limit 500 # cap MBID lookups this run
python main.py enrich # resolve MusicBrainz MBIDs (resumable; ~1 req/s) python main.py enrich # resume MBID resolution only
python main.py match # print present/missing summary python main.py match # print the present/missing summary
./gui.sh # browse what you're missing
``` ```
`ingest` and the GUI work immediately (text-based matching). `enrich` queries > ISRC/MBID matching works as soon as `fetch` runs (local tags + Spotify ISRC).
MusicBrainz only for tracks that don't have an MBID yet, caches every result, > MusicBrainz enrichment fills in the rest (iTunes, untagged local files); it's
and is **resumable** — if it's interrupted, just run it again to continue. Run > cached and resumable, so you can stop/continue any time.
`python main.py enrich --limit 300` to do a quick partial pass, or
`--source local` to restrict to one source.
> Live fetching (`SpotifyWebAPI`, the local indexer) is unchanged and still
> available; this redesign reads prefetched JSON so the pipeline can be built
> and demoed without network access.
## Layout ## Layout
``` ```
main.py CLI: ingest / enrich / match main.py CLI: fetch / display / enrich / match
Gui.py / gui.sh Streamlit GUI (missing-tracks viewer) Gui.py / gui.sh Streamlit GUI (missing-tracks viewer)
Env.py config (DB URL, paths, credentials from env) Env.py config: LOCAL_LIBRARY_PATH / ITUNES_XML_PATH / DATA_DIR
src/ src/
workflows.py run_fetch (the whole pipeline) + run_display
db/ SQLAlchemy models + engine/session db/ SQLAlchemy models + engine/session
ingest/ load prefetched JSON into the DB (upsert) ingest/
spotify_raw.py live fetch + ingest raw Spotify JSON (ISRC)
local_scan.py incremental file scan (reads ISRC/MBID tags)
itunes_xml.py parse the iTunes Library.xml (plist)
tags.py read ISRC / recording MBID / duration from a file
enrich/ MusicBrainz MBID resolver (cached, resumable) enrich/ MusicBrainz MBID resolver (cached, resumable)
match/ normalization + cross-source matcher match/ normalization + cross-source matcher
backends/ live fetchers/parsers (spotify, local, itunes, ...) backends/ lower-level Spotify Web API client / OAuth
Library.py in-memory data model used by the backends $DATA_DIR/ database + cached Spotify responses (gitignored)
work_dir/latest-1/ prefetched JSON used by `ingest` (gitignored)
musicindexer.db the database (gitignored)
``` ```
## Notes ## Notes
- The DB is the source of truth; `work_dir/` JSON is only an ingest source. - The DB is the source of truth; everything is incremental and re-runnable.
- Local matching is real-ID where files are tagged (ISRC/MusicBrainz, e.g. via
Picard); untagged files fall back to MusicBrainz text lookup, then text.
- Only currently-supported Spotify endpoints are used (no deprecated - Only currently-supported Spotify endpoints are used (no deprecated
audio-features/recommendations/etc.). audio-features/recommendations/etc.).

73
main.py
View file

@ -1,46 +1,47 @@
"""MusicIndexer pipeline CLI. """MusicIndexer CLI. Two main workflows plus granular helpers.
Everything is incremental and DB-backed (see Env.database_url). Typical flow: python main.py fetch # do all the work: pull + scan + parse + enrich + match
python main.py display # open the GUI to see what you're missing
python main.py ingest # load prefetched data into the DB (upsert) `fetch` is incremental and DB-backed (see Env.py): the first run is slow
python main.py enrich # resolve recording MBIDs (resumable, cached) (initial scan + MusicBrainz lookups), later runs only touch what changed.
python main.py match # print present/missing summary Helpers: `enrich` (resume MBID resolution), `match` (print the summary).
./gui.sh # browse what you're missing
`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it
just continues where it stopped.
""" """
import argparse import argparse
import json import logging
from src.db.database import init_db
def cmd_init(args): def _setup_logging():
init_db() logging.basicConfig(level=logging.INFO, format="%(message)s")
print("Database initialized.")
def cmd_ingest(args): def cmd_fetch(args):
init_db() _setup_logging()
from src.ingest.ingest import ingest_all from src.workflows import run_fetch
results = ingest_all() run_fetch(enrich_limit=args.enrich_limit)
for source, (new, updated) in results.items():
print(f"{source:8} +{new} new, {updated} updated")
def cmd_display(args):
from src.workflows import run_display
run_display()
def cmd_enrich(args): def cmd_enrich(args):
init_db() _setup_logging()
from src.db.database import init_db
from src.enrich.mbid import resolve_mbids, unresolved_count from src.enrich.mbid import resolve_mbids, unresolved_count
init_db()
print(f"unresolved before: {unresolved_count(args.source)}") print(f"unresolved before: {unresolved_count(args.source)}")
stats = resolve_mbids(limit=args.limit, source=args.source) print(f"resolved this run: {resolve_mbids(limit=args.limit, source=args.source)}")
print(f"resolved this run: {stats}")
print(f"unresolved after: {unresolved_count(args.source)}") print(f"unresolved after: {unresolved_count(args.source)}")
def cmd_match(args): def cmd_match(args):
import json
from src.match.matcher import summary from src.match.matcher import summary
print(json.dumps(summary(), indent=2)) print(json.dumps(summary(), indent=2))
@ -50,14 +51,15 @@ def build_parser():
parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__) parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__)
sub = parser.add_subparsers(dest="cmd", required=True) sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("init", help="create database tables") fetch = sub.add_parser("fetch", help="incrementally fetch + scan + enrich + match")
sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)") fetch.add_argument("--enrich-limit", type=int, default=None,
help="cap MBID lookups this run (default: resolve all)")
enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks") sub.add_parser("display", help="open the Streamlit GUI")
enrich.add_argument("--limit", type=int, default=None,
help="cap how many tracks to resolve this run") enrich = sub.add_parser("enrich", help="resume MBID resolution only")
enrich.add_argument("--source", choices=["spotify", "itunes", "local"], enrich.add_argument("--limit", type=int, default=None)
default=None, help="restrict to one source") enrich.add_argument("--source", choices=["spotify", "itunes", "local"], default=None)
sub.add_parser("match", help="print present/missing summary") sub.add_parser("match", help="print present/missing summary")
return parser return parser
@ -65,13 +67,12 @@ def build_parser():
def main(): def main():
args = build_parser().parse_args() args = build_parser().parse_args()
handlers = { {
"init": cmd_init, "fetch": cmd_fetch,
"ingest": cmd_ingest, "display": cmd_display,
"enrich": cmd_enrich, "enrich": cmd_enrich,
"match": cmd_match, "match": cmd_match,
} }[args.cmd](args)
handlers[args.cmd](args)
if __name__ == "__main__": if __name__ == "__main__":

5
src/db/database.py
View file

@ -17,7 +17,10 @@ def get_engine():
def init_db(): def init_db():
"""Create tables if they don't exist (safe to call every run).""" """Create the storage dir + tables if missing (safe to call every run)."""
from Env import data_dir
data_dir.mkdir(parents=True, exist_ok=True)
Base.metadata.create_all(_engine) Base.metadata.create_all(_engine)

30
src/enrich/mbid.py
View file

@ -14,6 +14,7 @@ Incremental + cheap:
- MB's ~1 req/s limit is respected by musicbrainzngs; commits happen - MB's ~1 req/s limit is respected by musicbrainzngs; commits happen
periodically so an interrupted run resumes where it left off. periodically so an interrupted run resumes where it left off.
""" """
import logging
from datetime import datetime from datetime import datetime
import musicbrainzngs as mb import musicbrainzngs as mb
@ -35,6 +36,8 @@ def _ensure_agent():
if not _agent_ready: if not _agent_ready:
mb.set_useragent("MusicIndexer", "0.1", mb_contact) mb.set_useragent("MusicIndexer", "0.1", mb_contact)
mb.set_rate_limit(1.0, 1) # 1 request / second mb.set_rate_limit(1.0, 1) # 1 request / second
# musicbrainzngs logs a warning per unparsed XML element; quiet it.
logging.getLogger("musicbrainzngs").setLevel(logging.ERROR)
_agent_ready = True _agent_ready = True
@ -80,20 +83,13 @@ def _cached(session, key, lookup):
def _resolve_one(session, track): def _resolve_one(session, track):
"""(mbid, score, status, source). """(mbid, score, source).
Text search on NORMALIZED artist/title is the primary lookup so that the ISRC first (exact recording), then text search on NORMALIZED artist/title
same song from different sources issues an identical query -> identical (so the same song across sources shares one query + one cached lookup).
recording MBID (and a shared cache entry). ISRC is an exact fallback when Used to enrich Spotify / iTunes / untagged-local; tagged local files
the text search finds nothing. already carry their MBID from the file and skip this.
""" """
na, nt = normalize_artist(track.artist or ""), normalize_title(track.title or "")
if na and nt:
key = f"txt:{na}|{nt}"
mbid, score, status = _cached(session, key, lambda: _lookup_text(na, nt))
if status == MB_FOUND:
return mbid, score, MB_FOUND, "text"
if track.isrc: if track.isrc:
mbid, score, status = _cached( mbid, score, status = _cached(
session, f"isrc:{track.isrc}", lambda: _lookup_isrc(track.isrc) session, f"isrc:{track.isrc}", lambda: _lookup_isrc(track.isrc)
@ -101,7 +97,13 @@ def _resolve_one(session, track):
if status == MB_FOUND: if status == MB_FOUND:
return mbid, score, MB_FOUND, "isrc" return mbid, score, MB_FOUND, "isrc"
return None, None, MB_NOTFOUND, ("text" if na and nt else None) na, nt = normalize_artist(track.artist or ""), normalize_title(track.title or "")
if na and nt:
key = f"txt:{na}|{nt}"
mbid, score, status = _cached(session, key, lambda: _lookup_text(na, nt))
return mbid, score, status, "text"
return None, None, MB_NOTFOUND, None
def resolve_mbids(limit=None, source=None, retry_errors=True): def resolve_mbids(limit=None, source=None, retry_errors=True):
@ -117,7 +119,7 @@ def resolve_mbids(limit=None, source=None, retry_errors=True):
if source: if source:
query = query.filter(Track.source == source) query = query.filter(Track.source == source)
tracks = query.order_by(Track.id).all() tracks = query.order_by(Track.id).all()
if limit: if limit is not None:
tracks = tracks[:limit] tracks = tracks[:limit]
for i, track in enumerate(tqdm(tracks, desc="resolve MBID", unit="track")): for i, track in enumerate(tqdm(tracks, desc="resolve MBID", unit="track")):

130
src/ingest/ingest.py
View file

@ -1,130 +0,0 @@
"""Load prefetched JSON (work_dir/latest-1) into the DB.
This stands in for the live backends while keeping the same contract they will
use: an idempotent UPSERT keyed by (source, source_id). Re-running updates only
changed descriptive fields and NEVER discards enrichment (mbid/mbid_status) or
rows that disappeared from a single export.
"""
import json
from tqdm import tqdm
from Env import prefetched_dir
from src.db.database import session_scope
from src.db.models import (
Track,
SOURCE_ITUNES,
SOURCE_LOCAL,
SOURCE_SPOTIFY,
)
from src.match.normalize import norm_key
def _load(path):
with open(path, encoding="utf-8") as fh:
return json.load(fh)
def _id_name_maps(lib):
artists = {a["id"]: a.get("title") for a in lib.get("artists", [])}
albums = {al["id"]: al.get("title") for al in lib.get("albums", [])}
return artists, albums
def _spotify_isrc_map(spotify_dir):
"""spotify_track_id -> ISRC, gathered from every raw payload that carries
full track objects (liked, top, playlist tracks)."""
out = {}
for fname in ("playlistTracks.json", "top_tracks.json", "tracks.json"):
path = spotify_dir / fname
if not path.exists():
continue
data = _load(path)
items = data if isinstance(data, list) else [v for lst in data.values() for v in lst]
for item in items:
track = item.get("track") if isinstance(item, dict) and "track" in item else item
if track and track.get("id"):
isrc = (track.get("external_ids") or {}).get("isrc")
if isrc:
out[track["id"]] = isrc
return out
class _Upserter:
"""Preloads existing rows for a source so upserts are O(1) lookups."""
def __init__(self, session, source):
self.session = session
self.source = source
self.existing = {
t.source_id: t
for t in session.query(Track).filter_by(source=source).all()
}
self.n_new = 0
self.n_updated = 0
def upsert(self, source_id, *, isrc=None, **fields):
track = self.existing.get(source_id)
if track is None:
track = Track(source=self.source, source_id=source_id, isrc=isrc, **fields)
self.session.add(track)
self.existing[source_id] = track
self.n_new += 1
return
# Existing: refresh descriptive fields, preserve enrichment unless the
# ISRC actually changed (then the cached MBID is stale).
for key, value in fields.items():
setattr(track, key, value)
if isrc is not None and isrc != track.isrc:
track.isrc = isrc
track.mbid = None
track.mbid_status = None
track.mbid_source = None
elif isrc is not None:
track.isrc = isrc
self.n_updated += 1
def _ingest_library(session, source, lib, *, source_id_field, isrc_map=None):
artists, albums = _id_name_maps(lib)
up = _Upserter(session, source)
for t in tqdm(lib.get("tracks", []), desc=f"ingest {source}", unit="track"):
source_id = t.get(source_id_field) or t.get("id")
if not source_id:
continue
names = [artists.get(aid, aid) for aid in t.get("artists", []) if aid]
names = [n for n in names if n]
primary = names[0] if names else None
title = t.get("title")
up.upsert(
source_id,
title=title,
artist=primary,
artists=", ".join(names),
album=albums.get(t.get("album"), t.get("album")),
duration_ms=t.get("duration_ms"),
play_count=t.get("play_count") or 0,
local_path=t.get("local_path"),
norm_key=norm_key(primary or "", title or ""),
isrc=(isrc_map or {}).get(source_id),
)
return up.n_new, up.n_updated
def ingest_all():
base = prefetched_dir
results = {}
with session_scope() as session:
results[SOURCE_SPOTIFY] = _ingest_library(
session, SOURCE_SPOTIFY, _load(base / "spotify_library.json"),
source_id_field="id", isrc_map=_spotify_isrc_map(base / "spotify"),
)
results[SOURCE_ITUNES] = _ingest_library(
session, SOURCE_ITUNES, _load(base / "itunes_library.json"),
source_id_field="id",
)
results[SOURCE_LOCAL] = _ingest_library(
session, SOURCE_LOCAL, _load(base / "local_library.json"),
source_id_field="local_path",
)
return results

68
src/ingest/itunes_xml.py Normal file
View file

@ -0,0 +1,68 @@
"""Ingest a manually-exported iTunes/Music Library.xml into the DB.
The file is an Apple property list, so plistlib parses it natively. Upsert by
the iTunes persistent/numeric Track ID; skipped entirely if the file's mtime
hasn't changed since the last run.
"""
import logging
import os
import plistlib
from tqdm import tqdm
from Env import itunes_xml_path
from src.db.database import session_scope
from src.db.models import IngestState, SOURCE_ITUNES, Track
from src.match.normalize import norm_key
log = logging.getLogger("musicindexer.itunes")
_STATE_KEY = "itunes_xml_mtime"
def ingest_itunes(xml_path=None, force=False):
xml_path = str(xml_path or itunes_xml_path)
if not os.path.isfile(xml_path):
log.warning("iTunes XML not found: %s — skipping", xml_path)
return {"new": 0, "updated": 0, "skipped": True}
mtime = str(os.path.getmtime(xml_path))
with session_scope() as session:
state = session.get(IngestState, _STATE_KEY)
if state and state.value == mtime and not force:
log.info("iTunes XML unchanged — skipping")
return {"new": 0, "updated": 0, "skipped": True}
with open(xml_path, "rb") as fh:
plist = plistlib.load(fh)
raw_tracks = plist.get("Tracks", {})
counts = {"new": 0, "updated": 0, "skipped": False}
with session_scope() as session:
existing = {t.source_id: t for t in
session.query(Track).filter_by(source=SOURCE_ITUNES).all()}
for tid, tr in tqdm(raw_tracks.items(), desc="ingest itunes", unit="track"):
name = tr.get("Name")
artist = tr.get("Artist") or tr.get("Album Artist")
if not name:
continue
fields = dict(
title=name, artist=artist, artists=artist, album=tr.get("Album"),
duration_ms=tr.get("Total Time"), play_count=tr.get("Play Count") or 0,
norm_key=norm_key(artist or "", name),
)
source_id = str(tid)
track = existing.get(source_id)
if track is None:
session.add(Track(source=SOURCE_ITUNES, source_id=source_id, **fields))
counts["new"] += 1
else:
for key, value in fields.items():
setattr(track, key, value)
counts["updated"] += 1
state = session.get(IngestState, _STATE_KEY) or IngestState(key=_STATE_KEY)
state.value = mtime
session.merge(state)
log.info("itunes: %(new)d new, %(updated)d updated", counts)
return counts

99
src/ingest/local_scan.py Normal file
View file

@ -0,0 +1,99 @@
"""Incrementally scan the local music library into the DB.
Only files whose mtime/size changed are re-read (tracked in file_state); files
that disappeared are pruned. Embedded ISRC and MusicBrainz recording MBID are
read from tags, so tagged tracks get real identifiers with no network.
"""
import logging
import os
from datetime import datetime
from tqdm import tqdm
from Env import local_library_path
from src.db.database import session_scope
from src.db.models import FileState, MB_FOUND, SOURCE_LOCAL, Track
from src.ingest.tags import read_track_tags
from src.match.normalize import norm_key
log = logging.getLogger("musicindexer.local")
MUSIC_EXTS = (".mp3", ".flac", ".m4a", ".ogg", ".opus", ".wav", ".aac")
def _iter_files(root):
for dirpath, _dirs, names in os.walk(root):
for name in names:
if os.path.splitext(name)[1].lower() in MUSIC_EXTS:
yield os.path.join(dirpath, name)
def scan_local(root=None):
root = str(root or local_library_path)
if not os.path.isdir(root):
log.warning("local library path not found: %s — skipping", root)
return {"new": 0, "updated": 0, "unchanged": 0, "removed": 0}
files = list(_iter_files(root))
counts = {"new": 0, "updated": 0, "unchanged": 0, "removed": 0}
with session_scope() as session:
states = {fs.path: fs for fs in session.query(FileState).all()}
tracks = {t.source_id: t for t in
session.query(Track).filter_by(source=SOURCE_LOCAL).all()}
seen = set()
for path in tqdm(files, desc="scan local", unit="file"):
seen.add(path)
stat = os.stat(path)
state = states.get(path)
if state and state.mtime == stat.st_mtime and state.size == stat.st_size:
counts["unchanged"] += 1
continue
tags = read_track_tags(path)
if not tags or not (tags["title"] or tags["artist"]):
continue
fields = dict(
title=tags["title"], artist=tags["artist"], artists=tags["artists"],
album=tags["album"], duration_ms=tags["duration_ms"],
isrc=tags["isrc"], local_path=path,
norm_key=norm_key(tags["artist"] or "", tags["title"] or ""),
)
track = tracks.get(path)
if track is None:
track = Track(source=SOURCE_LOCAL, source_id=path, **fields)
session.add(track)
tracks[path] = track
counts["new"] += 1
else:
for key, value in fields.items():
setattr(track, key, value)
counts["updated"] += 1
# Tagged files come with their recording MBID -> resolved for free.
if tags["mbid"]:
track.mbid = tags["mbid"]
track.mbid_status = MB_FOUND
track.mbid_source = "tag"
if state is None:
session.add(FileState(path=path, mtime=stat.st_mtime,
size=stat.st_size, indexed_at=datetime.utcnow()))
else:
state.mtime, state.size, state.indexed_at = (
stat.st_mtime, stat.st_size, datetime.utcnow())
# Prune rows whose file no longer exists.
for path, track in tracks.items():
if path not in seen and os.path.isabs(path):
session.delete(track)
counts["removed"] += 1
for path, state in states.items():
if path not in seen:
session.delete(state)
log.info("local: %(new)d new, %(updated)d updated, %(unchanged)d unchanged, "
"%(removed)d removed (%(total)d files scanned)",
{**counts, "total": len(files)})
return counts

85
src/ingest/spotify_raw.py Normal file
View file

@ -0,0 +1,85 @@
"""Ingest raw Spotify API responses (the JSON written by SpotifyWebAPI) into the
DB. Unions liked + top + playlist tracks, upserting by Spotify track id and
pulling the ISRC from external_ids — so Spotify tracks get a real identifier
for matching with no extra calls.
"""
import json
import logging
import os
from tqdm import tqdm
from Env import spotify_cache_dir
from src.db.database import session_scope
from src.db.models import SOURCE_SPOTIFY, Track
from src.match.normalize import norm_key
log = logging.getLogger("musicindexer.spotify")
# Files written by SpotifyWebAPI.fetch_data(), each holding full track objects.
_SOURCES = ("tracks.json", "top_tracks.json", "playlistTracks.json")
def _track_objects(spotify_dir):
by_id = {}
for fname in _SOURCES:
path = os.path.join(spotify_dir, fname)
if not os.path.isfile(path):
continue
with open(path, encoding="utf-8") as fh:
data = json.load(fh)
items = data if isinstance(data, list) else [v for lst in data.values() for v in lst]
for item in items:
track = item.get("track") if isinstance(item, dict) and "track" in item else item
if track and track.get("id"):
by_id[track["id"]] = track
return by_id
def ingest_spotify(spotify_dir=None):
spotify_dir = str(spotify_dir or spotify_cache_dir)
if not os.path.isdir(spotify_dir):
log.warning("no Spotify cache at %s — skipping", spotify_dir)
return {"new": 0, "updated": 0}
objects = _track_objects(spotify_dir)
counts = {"new": 0, "updated": 0}
with session_scope() as session:
existing = {t.source_id: t for t in
session.query(Track).filter_by(source=SOURCE_SPOTIFY).all()}
for spotify_id, t in tqdm(objects.items(), desc="ingest spotify", unit="track"):
names = [a.get("name") for a in t.get("artists", []) if a.get("name")]
primary = names[0] if names else None
title = t.get("name")
fields = dict(
title=title, artist=primary, artists=", ".join(names),
album=(t.get("album") or {}).get("name"),
duration_ms=t.get("duration_ms"),
isrc=(t.get("external_ids") or {}).get("isrc"),
norm_key=norm_key(primary or "", title or ""),
)
track = existing.get(spotify_id)
if track is None:
session.add(Track(source=SOURCE_SPOTIFY, source_id=spotify_id, **fields))
counts["new"] += 1
else:
new_isrc = fields.pop("isrc")
for key, value in fields.items():
setattr(track, key, value)
if new_isrc and new_isrc != track.isrc: # stale enrichment
track.isrc, track.mbid, track.mbid_status, track.mbid_source = (
new_isrc, None, None, None)
elif new_isrc:
track.isrc = new_isrc
counts["updated"] += 1
log.info("spotify: %(new)d new, %(updated)d updated", counts)
return counts
def fetch_spotify_live():
"""Run the real OAuth Spotify fetch into the cache dir (requires creds)."""
from Env import client_id, client_secret, data_dir
from src.backends.spotify.SpotifyWebAPI import fetch_all
fetch_all(client_id, client_secret, data_dir) # writes data_dir/spotify/*.json

84
src/ingest/tags.py Normal file
View file

@ -0,0 +1,84 @@
"""Read matching-relevant metadata from an audio file, including the embedded
MusicBrainz recording MBID and ISRC when present (e.g. Picard-tagged files).
Returns a flat dict so the local scanner can upsert without knowing tag formats.
"""
from mutagen import File as MutagenFile
_MB_OWNER = "http://musicbrainz.org"
def _first(value):
if value is None:
return None
if isinstance(value, list):
return str(value[0]) if value else None
return str(value)
def _mp4_freeform(raw, key):
if key in raw:
try:
return _first([bytes(x).decode("utf-8", "ignore") for x in raw[key]])
except Exception:
return None
return None
def read_track_tags(path):
try:
audio = MutagenFile(path)
except Exception:
return None
if audio is None:
return None
duration_ms = None
if audio.info and getattr(audio.info, "length", None):
duration_ms = int(audio.info.length * 1000)
# Easy view normalizes common keys across ID3/Vorbis/MP4.
etags = {}
try:
easy = MutagenFile(path, easy=True)
etags = easy.tags or {}
except Exception:
etags = {}
def eget(key):
try:
return _first(etags.get(key))
except Exception:
return None
title = eget("title")
artist = eget("artist")
album = eget("album")
artist_values = etags.get("artist") if isinstance(etags.get("artist"), list) else None
artists = ", ".join(str(a) for a in artist_values) if artist_values else artist
isrc = eget("isrc")
mbid = eget("musicbrainz_trackid") # recording MBID
raw = audio.tags
if raw is not None:
if mbid is None:
try: # ID3 stores the recording MBID in a UFID frame
for frame in raw.getall("UFID"):
if getattr(frame, "owner", None) == _MB_OWNER:
mbid = frame.data.decode("ascii", "ignore")
break
except Exception:
pass
mbid = mbid or _mp4_freeform(raw, "----:com.apple.iTunes:MusicBrainz Track Id")
if isrc is None:
isrc = _mp4_freeform(raw, "----:com.apple.iTunes:ISRC")
return {
"title": title,
"artist": artist,
"artists": artists or artist,
"album": album,
"duration_ms": duration_ms,
"isrc": isrc or None,
"mbid": mbid or None,
}

86
src/workflows.py Normal file
View file

@ -0,0 +1,86 @@
"""Top-level workflows.
`run_fetch` does *all* the work incrementally and reports: pull Spotify, scan
the local library, parse the iTunes export, resolve MBIDs, then match. The first
run is slow (initial scan + MusicBrainz lookups); later runs only touch what
changed, so they're quick.
"""
import logging
import subprocess
import sys
import time
from Env import (
client_id,
client_secret,
itunes_xml_path,
local_library_path,
spotify_cache_dir,
)
from src.db.database import init_db
from src.enrich.mbid import resolve_mbids, unresolved_count
from src.ingest.itunes_xml import ingest_itunes
from src.ingest.local_scan import scan_local
from src.ingest.spotify_raw import fetch_spotify_live, ingest_spotify
from src.match.matcher import summary
log = logging.getLogger("musicindexer")
def _step(title):
log.info("")
log.info("== %s", title)
def run_fetch(enrich_limit=None):
started = time.time()
init_db()
log.info("MusicIndexer — incremental fetch")
log.info(" local library : %s", local_library_path)
log.info(" itunes xml : %s", itunes_xml_path)
log.info(" spotify cache : %s", spotify_cache_dir)
_step("Spotify")
if client_id and client_secret:
log.info("credentials present -> live fetch")
try:
fetch_spotify_live()
except Exception as exc: # keep going on cached data
log.error("live fetch failed (%s); using cached responses", exc)
else:
log.info("no SPOTIFY_CLIENT_ID/SECRET -> using cached responses")
ingest_spotify()
_step("iTunes")
ingest_itunes()
_step("Local library")
scan_local()
_step("MusicBrainz enrichment (incremental)")
todo = unresolved_count()
log.info("%d tracks still need an MBID", todo)
if todo:
stats = resolve_mbids(limit=enrich_limit)
log.info("resolved this run: %s", stats)
log.info("remaining unresolved: %d", unresolved_count())
_step("Matching")
summ = summary()
for source in ("spotify", "itunes"):
s = summ.get(source)
if s:
log.info("%-7s total=%-5d MISSING=%-5d have(id)=%-5d have(text)=%-5d",
source, s["total"], s["missing"], s["present_mbid"], s["present_text"])
local = summ.get("local", {})
log.info("local total=%-5d with_mbid=%d", local.get("total", 0), local.get("has_mbid", 0))
log.info("")
log.info("done in %.1fs — run `python main.py display` to browse what's missing",
time.time() - started)
return summ
def run_display():
"""Launch the Streamlit GUI."""
subprocess.run([sys.executable, "-m", "streamlit", "run", "Gui.py"])