ilusha/MusicIndexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

DB-backed incremental redesign: find tracks missing locally

Browse source 
Replace the JSON-file pipeline with a SQLite/SQLAlchemy store (Postgres via
DATABASE_URL) and an incremental, ID-based matcher whose goal is "which
Spotify/iTunes tracks am I missing locally".

- src/db: models (Track/MbCache/FileState/IngestState) + engine/session.
- src/ingest: upsert prefetched spotify/itunes/local (work_dir/latest-1) by
  (source, source_id); pulls ISRC from raw Spotify; preserves enrichment.
- src/enrich: MusicBrainz recording-MBID resolver. Resolves via NORMALIZED
  text search (so the same song across sources lands on one MBID + one cached
  lookup), ISRC as exact fallback. Only touches unresolved tracks, caches
  positive AND negative results, rate-limited and resumable.
- src/match: normalization + cross-source matcher with tiers ISRC > MBID >
  normalized text; reports present/missing and the match method.
- main.py: CLI (ingest / enrich / match). Gui.py: missing-tracks viewer.
- README + requirements (SQLAlchemy, musicbrainzngs); ignore *.db.

Incremental everywhere: re-running updates only what changed; enrich continues
where it stopped. Live Spotify/local backends are left intact.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Шурупов Илья Викторович 2026-06-15 01:25:21 +03:00
parent 83f782d125
commit 0e64d439c4
16 changed files with 784 additions and 274 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -4,6 +4,7 @@ __pycache__
secret
.env
.venv
*.db
work_dir
old
tmp

12
Env.py
View file

@ -11,3 +11,15 @@ work_dir = Path(os.environ.get("WORK_DIR", "./work_dir/new"))
# export SPOTIFY_CLIENT_SECRET=...
client_id = os.environ.get("SPOTIFY_CLIENT_ID")
client_secret = os.environ.get("SPOTIFY_CLIENT_SECRET")
# Persistent store. Defaults to a local SQLite file; point DATABASE_URL at a
# Postgres instance (postgresql+psycopg://user:pass@host/db) to switch with no
# code changes.
database_url = os.environ.get("DATABASE_URL", "sqlite:///./musicindexer.db")
# Folder of prefetched JSON used to (re)build the DB without hitting the network.
prefetched_dir = Path(os.environ.get("PREFETCHED_DIR", "./work_dir/latest-1"))
# Contact email sent to MusicBrainz in the User-Agent (be a good API citizen).
mb_contact = os.environ.get("MB_CONTACT", "music-indexer@example.com")

268
Gui.py
View file

@ -1,208 +1,90 @@
import time
"""Streamlit GUI: what's in your Spotify / iTunes libraries that you do NOT
have locally, decided by real ID matching (recording MBID / ISRC) with a
normalized-text fallback.
import streamlit as st
from st_aggrid import AgGrid, GridOptionsBuilder, GridUpdateMode
Reads only the database (run `python main.py ingest` and `enrich` first).
"""
import pandas as pd
import streamlit as st
from st_aggrid import AgGrid, GridOptionsBuilder
import Flow
from src.db.database import init_db
from src.match.matcher import classify_all, summary
from src.Library import *
def entity_to_row(entity: Entity):
return {
"id": entity.id,
"mbid": entity.mbid,
"Title": entity.title,
"PLay Count": entity.play_count,
"Resolved": int(entity.resolved_percentage * 100),
"AutoScore": entity.auto_score,
"Added": entity.date_added.strftime("%Y-%m-%d") if entity.date_added else None,
"Played": entity.date_last_play.strftime("%Y-%m-%d") if entity.date_last_play else None,
"Released": entity.date_released.strftime("%Y-%m-%d") if entity.date_released else None,
"LocalId": str(entity.local_id) if entity.local_id else None,
"Path": str(entity.local_path) if entity.local_path else None,
}
def track_to_row(track: Track):
return entity_to_row(track) | {
"Album": track.album.title if track.album else "",
"Artists": ", ".join(a.title if a.title else "no_arist_name" for a in track.artists),
"Lyrics": track.has_lyrics,
}
def artist_to_row(artist: Artist):
return entity_to_row(artist) | {
}
def album_to_row(album: Album):
return entity_to_row(album) | {
}
st.set_page_config(layout="wide", page_title="MusicIndexer — Missing tracks")
class LibraryView:
def __init__(self):
self.lib : Library = None
self.libs : list[Library] = None
self.view = None
self.lyrics_only = False
self.query = None
def add_libraries(self, libs : list[Library]):
self.libs = libs
def render(self):
st.set_page_config(layout="wide")
with st.sidebar:
lib = st.selectbox(
"Library",
options=[lib.name for lib in self.libs],
)
for iter in self.libs:
if iter.name == lib:
self.lib = iter
st.write(f"Showing library '{self.lib.name}'")
self.view = st.radio(
"View",
["Tree"]
)
if self.view == "Tree":
self.render_artists_albums_tracks()
# elif self.view == "Tracks":
# self.render_tracks()
# elif self.view == "Playlists":
# self.render_playlists()
def track_visible(
self,
track: Track,
artist: Artist | None,
album: Album | None,
):
if self.lyrics_only and not track.has_lyrics:
return False
if self.query and self.query.lower() not in track.title.lower():
return False
if artist and all(a.id != artist.id for a in track.artists):
return False
if album and (not track.album or track.album.id != album.id):
return False
return True
def render_artists_albums_tracks(self):
# --- ARTISTS TABLE ---
st.subheader("Artists")
artist_rows = [artist_to_row(a) for a in self.lib.artists]
df_artists = pd.DataFrame(artist_rows)
gb_artists = GridOptionsBuilder.from_dataframe(df_artists)
gb_artists.configure_selection(selection_mode="multiple", use_checkbox=True)
gb_artists.configure_column("Title", filter="agTextColumnFilter")
gb_artists.configure_column("AutoScore", sort="desc")
grid_options_artists = gb_artists.build()
grid_response_artists = AgGrid(
df_artists,
gridOptions=grid_options_artists,
update_mode=GridUpdateMode.SELECTION_CHANGED,
allow_unsafe_jscode=True,
enable_enterprise_modules=False,
fit_columns_on_grid_load=True,
)
selected_rows_df = grid_response_artists.get("selected_rows")
selected_artist_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
selected_artists = [a for a in self.lib.artists if a.title in selected_artist_titles]
# --- ALBUMS TABLE ---
st.subheader("Albums")
if selected_artists:
albums_list = []
for artist in selected_artists:
albums_list.extend(artist.albums)
albums_list = list({al.id: al for al in albums_list}.values())
else:
albums_list = self.lib.albums
album_rows = [album_to_row(a) for a in albums_list]
df_albums = pd.DataFrame(album_rows)
gb_albums = GridOptionsBuilder.from_dataframe(df_albums)
gb_albums.configure_selection(selection_mode="multiple", use_checkbox=True)
gb_albums.configure_column("Title", filter="agTextColumnFilter")
gb_albums.configure_column("AutoScore", sort="desc")
grid_options_albums = gb_albums.build()
grid_response_albums = AgGrid(
df_albums,
gridOptions=grid_options_albums,
update_mode=GridUpdateMode.SELECTION_CHANGED,
allow_unsafe_jscode=True,
enable_enterprise_modules=False,
fit_columns_on_grid_load=True,
)
selected_rows_df = grid_response_albums.get("selected_rows")
selected_album_titles = selected_rows_df["Title"].tolist() if selected_rows_df is not None and not selected_rows_df.empty else []
selected_albums = [al for al in albums_list if al.title in selected_album_titles]
# --- TRACKS TABLE ---
st.subheader("Tracks")
if selected_albums:
tracks_list = []
for album in selected_albums:
tracks_list.extend(album.tracks)
elif selected_artists:
tracks_list = []
for artist in selected_artists:
tracks_list.extend([t for t in self.lib.tracks if artist in t.artists])
else:
tracks_list = self.lib.tracks
track_rows = [track_to_row(t) for t in tracks_list]
df_tracks = pd.DataFrame(track_rows)
gb_tracks = GridOptionsBuilder.from_dataframe(df_tracks)
gb_tracks.configure_selection(selection_mode="multiple", use_checkbox=True)
gb_tracks.configure_column("Title", filter="agTextColumnFilter")
gb_tracks.configure_column("AutoScore", sort="desc")
grid_options_tracks = gb_tracks.build()
AgGrid(
df_tracks,
gridOptions=grid_options_tracks,
update_mode=GridUpdateMode.SELECTION_CHANGED,
allow_unsafe_jscode=True,
enable_enterprise_modules=False,
fit_columns_on_grid_load=True,
)
@st.cache_data(ttl=30)
def load_data():
init_db()
missing, present = classify_all()
return missing, present, summary()
def run():
flow = Flow.Flow()
def grid(df, key):
gb = GridOptionsBuilder.from_dataframe(df)
gb.configure_default_column(filter=True, sortable=True, resizable=True)
gb.configure_pagination(paginationAutoPageSize=False, paginationPageSize=50)
AgGrid(df, gridOptions=gb.build(), height=620, fit_columns_on_grid_load=True, key=key)
# flow.fetch_spotify()
# flow.fetch_local()
# flow.parse_spotify_library()
# flow.parse_local_library()
def main():
st.title("🎧 Tracks you're missing locally")
flow.load_libraries()
if st.sidebar.button("🔄 Refresh from DB"):
load_data.clear()
# flow.map_local_to_spotify()
missing, present, summ = load_data()
# flow.log_stats()
# --- summary metrics ---
for source in ("spotify", "itunes"):
s = summ.get(source, {})
if not s:
continue
st.subheader(source.capitalize())
c1, c2, c3, c4, c5 = st.columns(5)
c1.metric("Total", s["total"])
c2.metric("Missing locally", s["missing"])
c3.metric("Have (MBID)", s["present_mbid"])
c4.metric("Have (text)", s["present_text"])
coverage = (s["has_mbid"] / s["total"] * 100) if s["total"] else 0
c5.metric("MBID coverage", f"{coverage:.0f}%")
gui = LibraryView()
gui.add_libraries([flow.spotify_library, flow.itunes_library, flow.local_library])
gui.render()
local = summ.get("local", {})
st.caption(
f"Local library: {local.get('total', 0)} tracks, "
f"{local.get('has_mbid', 0)} with a resolved MBID. "
"‘Have (MBID)’ = certain ID match; ‘Have (text)’ = normalized "
"artist/title match (run `enrich` to upgrade text matches to ID)."
)
run()
st.divider()
# --- controls ---
view = st.sidebar.radio("View", ["Missing", "Present", "All"])
sources = st.sidebar.multiselect("Sources", ["spotify", "itunes"],
default=["spotify", "itunes"])
if view == "Missing":
rows = [dict(r, status="missing") for r in missing]
elif view == "Present":
rows = [dict(r, status="present") for r in present]
else:
rows = ([dict(r, status="missing") for r in missing]
+ [dict(r, status="present") for r in present])
rows = [r for r in rows if r["source"] in sources]
if not rows:
st.info("Nothing to show — run `python main.py ingest` (and `enrich`).")
return
df = pd.DataFrame(rows)[
["status", "source", "artist", "title", "album", "matched_by",
"isrc", "mbid", "play_count", "spotify_id"]
]
st.write(f"**{len(df)}** tracks")
grid(df, key=f"{view}-{'-'.join(sorted(sources))}")
main()

121
README.md
View file

@ -5,19 +5,38 @@ local music library (audio tags + iTunes XML), and match the two so you can see
which tracks you own, what's missing, and your top artists/tracks/playlists — in
a Streamlit GUI.
## Features
It is **database-backed and incremental**: data lives in a DB (SQLite by
default, Postgres optional) and every step UPSERTs, so re-running only updates
what changed and nothing already fetched is lost. The headline use case is
seeing **which Spotify/iTunes tracks you don't have locally**, decided by real
recording identifiers (ISRC / MusicBrainz MBID) with a normalized-text fallback.
- **Spotify backend** — OAuth (Authorization Code) login, then fetch the user
profile, liked tracks, all playlists and their tracks, long-term top tracks
and top artists, and playlist/profile cover art. Resilient paginated fetching
(follows `next`, retries dropped connections, honours `429` rate limits) with
progress bars.
- **Local backend** — index a folder of audio files via `mutagen` tags.
- **iTunes backend** — parse an iTunes Library XML export.
- **Matching** — map local tracks to Spotify tracks (fuzzy matcher; ISRC/MBID
matching planned).
- **Stats & lyrics** — generate listening stats and fetch lyrics.
- **GUI** — browse libraries as artist → album → track tables.
## Pipeline
```
ingest -> enrich (MBIDs) -> match -> GUI
```
| Step | Command | Incremental rule |
|------|---------|------------------|
| **ingest** | `python main.py ingest` | upsert tracks by `(source, source_id)`; never drops enrichment |
| **enrich** | `python main.py enrich` | query MusicBrainz only for tracks with no MBID yet; every lookup (incl. "not found") cached; rate-limited & resumable |
| **match** | `python main.py match` | classify each Spotify/iTunes track: present locally (ISRC > MBID > text) or missing |
| **GUI** | `./gui.sh` | browse missing/present with the match method shown |
### How matching works
Each track is resolved to a MusicBrainz **recording MBID** by searching with the
*normalized* artist+title, so the same song from different sources lands on the
same MBID (and shares one cached lookup). A Spotify/iTunes track counts as
"have" if its ISRC, its MBID, or its normalized artist|title matches a local
track — otherwise it's **missing**. The GUI labels each match `isrc`/`mbid`
(ID-certain) or `text` (fuzzy).
> Note: the local library here has no ISRC/MBID tags, so before `enrich` runs,
> matching is normalized-text only (still useful). `enrich` upgrades matches to
> ID-certain. Tagging local files with Picard/AcoustID (ISRC) would make the ID
> tiers exact end-to-end.
## Setup
@ -41,60 +60,56 @@ cp .env.example .env
source .env
```
`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (plus optional
`MUSIC_LIBRARY_PATH` and `WORK_DIR`) from the environment.
`Env.py` reads `SPOTIFY_CLIENT_ID` / `SPOTIFY_CLIENT_SECRET` (only needed for
live fetching), plus optional `DATABASE_URL`, `PREFETCHED_DIR` and `MB_CONTACT`.
### Database
Defaults to a local SQLite file (`musicindexer.db`). To use Postgres instead,
set one env var — no code changes:
```bash
export DATABASE_URL=postgresql+psycopg://user:pass@localhost:5432/musicindexer
```
## Usage
Edit `main.py` to uncomment the steps you want, then run:
```bash
python main.py
python main.py ingest # load prefetched data (work_dir/latest-1) into the DB
python main.py enrich # resolve MusicBrainz MBIDs (resumable; ~1 req/s)
python main.py match # print present/missing summary
./gui.sh # browse what you're missing
```
The flow steps (see `Flow.py`) include:
`ingest` and the GUI work immediately (text-based matching). `enrich` queries
MusicBrainz only for tracks that don't have an MBID yet, caches every result,
and is **resumable** — if it's interrupted, just run it again to continue. Run
`python main.py enrich --limit 300` to do a quick partial pass, or
`--source local` to restrict to one source.
| Step | What it does |
|------|--------------|
| `fetch_spotify()` | OAuth login + download all Spotify data into `work_dir/spotify/` |
| `fetch_local()` | index local audio files into `work_dir/local/` |
| `parse_spotify_library()` / `parse_itunes_library()` / `parse_local_library()` | build normalized libraries |
| `load_libraries()` / `save_libraries()` | (de)serialize libraries to `work_dir` |
| `map_local_to_spotify()` | match local tracks against the Spotify library |
| `fetch_and_save_lyrics()` | fetch lyrics for tracks |
| `log_stats()` | generate stats |
On first `fetch_spotify()` a browser opens for the Spotify login; a local server
on `127.0.0.1:8888` catches the redirect and exchanges the code for a token.
### GUI
```bash
./gui.sh # runs: .venv/bin/streamlit run Gui.py
```
> Live fetching (`SpotifyWebAPI`, the local indexer) is unchanged and still
> available; this redesign reads prefetched JSON so the pipeline can be built
> and demoed without network access.
## Layout
```
main.py entry point (CLI flow)
Gui.py / gui.sh Streamlit GUI
Flow.py orchestrates fetch / parse / map / stats
Env.py config (reads credentials from the environment)
main.py CLI: ingest / enrich / match
Gui.py / gui.sh Streamlit GUI (missing-tracks viewer)
Env.py config (DB URL, paths, credentials from env)
src/
Library.py data model (Library / Artist / Album / Track)
LibrarySerializer.py load/save libraries
backends/
spotify/ Web API client, OAuth, parser
local/ local-file indexer, parser, playlist generator
itunes/ iTunes XML parser
navidrome/ Navidrome backend
mappers/ local↔Spotify matching (fuzzy / search)
lyrics/ lyrics fetching
work_dir/ fetched data and serialized libraries (gitignored)
db/ SQLAlchemy models + engine/session
ingest/ load prefetched JSON into the DB (upsert)
enrich/ MusicBrainz MBID resolver (cached, resumable)
match/ normalization + cross-source matcher
backends/ live fetchers/parsers (spotify, local, itunes, ...)
Library.py in-memory data model used by the backends
work_dir/latest-1/ prefetched JSON used by `ingest` (gitignored)
musicindexer.db the database (gitignored)
```
## Notes
- Data is cached as JSON under `work_dir/` (gitignored); re-running reuses it.
- The deprecated Spotify endpoints (audio-features, recommendations, etc.) are
not used — only currently-supported endpoints.
- The DB is the source of truth; `work_dir/` JSON is only an ingest source.
- Only currently-supported Spotify endpoints are used (no deprecated
audio-features/recommendations/etc.).

96
main.py
View file

@ -1,38 +1,78 @@
import Flow
"""MusicIndexer pipeline CLI.
Everything is incremental and DB-backed (see Env.database_url). Typical flow:
python main.py ingest # load prefetched data into the DB (upsert)
python main.py enrich # resolve recording MBIDs (resumable, cached)
python main.py match # print present/missing summary
./gui.sh # browse what you're missing
`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it
just continues where it stopped.
"""
import argparse
import json
from src.db.database import init_db
def test1():
flow = Flow.Flow()
def cmd_init(args):
init_db()
print("Database initialized.")
# flow.fetch_local()
flow.parse_local_library()
flow.load_libraries()
# flow.map_local_to_spotify()
# flow.log_stats()
def cmd_ingest(args):
init_db()
from src.ingest.ingest import ingest_all
# flow.fetch_and_save_lyrics()
results = ingest_all()
for source, (new, updated) in results.items():
print(f"{source:8} +{new} new, {updated} updated")
def cmd_enrich(args):
init_db()
from src.enrich.mbid import resolve_mbids, unresolved_count
print(f"unresolved before: {unresolved_count(args.source)}")
stats = resolve_mbids(limit=args.limit, source=args.source)
print(f"resolved this run: {stats}")
print(f"unresolved after: {unresolved_count(args.source)}")
def cmd_match(args):
from src.match.matcher import summary
print(json.dumps(summary(), indent=2))
def build_parser():
parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__)
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("init", help="create database tables")
sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)")
enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks")
enrich.add_argument("--limit", type=int, default=None,
help="cap how many tracks to resolve this run")
enrich.add_argument("--source", choices=["spotify", "itunes", "local"],
default=None, help="restrict to one source")
sub.add_parser("match", help="print present/missing summary")
return parser
def main():
flow = Flow.Flow()
flow.fetch_spotify()
# flow.fetch_local()
flow.parse_spotify_library()
#flow.parse_itunes_library()
#flow.parse_local_library()
#flow.load_libraries()
#flow.map_local_to_spotify()
#flow.save_libraries()
# flow.log_stats()
# print("asd")
args = build_parser().parse_args()
handlers = {
"init": cmd_init,
"ingest": cmd_ingest,
"enrich": cmd_enrich,
"match": cmd_match,
}
handlers[args.cmd](args)
# test1()
main()
if __name__ == "__main__":
main()

2
requirements.txt
View file

@ -7,3 +7,5 @@ rapidfuzz==3.14.5
requests==2.34.2
tqdm==4.68.2
xmltodict==1.0.4
SQLAlchemy==2.0.50
musicbrainzngs==0.7.1

0
src/db/__init__.py Normal file
View file

35
src/db/database.py Normal file
View file

@ -0,0 +1,35 @@
"""Engine/session helpers. The DB URL comes from Env.database_url so the same
code runs on SQLite (default) or Postgres by changing one env var."""
from contextlib import contextmanager
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from Env import database_url
from src.db.models import Base
_engine = create_engine(database_url, future=True)
SessionLocal = sessionmaker(bind=_engine, future=True, expire_on_commit=False)
def get_engine():
return _engine
def init_db():
"""Create tables if they don't exist (safe to call every run)."""
Base.metadata.create_all(_engine)
@contextmanager
def session_scope():
"""Transactional session: commit on success, roll back on error."""
session = SessionLocal()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()

109
src/db/models.py Normal file
View file

@ -0,0 +1,109 @@
"""SQLAlchemy models for the music index.
Storage is source-of-truth and incremental: every ingest/enrich step UPSERTs
into these tables, so re-running only updates what changed and nothing already
fetched is lost.
"""
from datetime import datetime
from sqlalchemy import (
Column,
DateTime,
Float,
Integer,
String,
UniqueConstraint,
)
from sqlalchemy.orm import declarative_base
Base = declarative_base()
SOURCE_SPOTIFY = "spotify"
SOURCE_ITUNES = "itunes"
SOURCE_LOCAL = "local"
# mbid_status values
MB_UNRESOLVED = None # never looked up
MB_FOUND = "found"
MB_NOTFOUND = "notfound"
MB_ERROR = "error"
class Track(Base):
"""One track as seen in one source (spotify / itunes / local).
The same song appears as several rows (one per source); the matcher links
them by ``match_key`` (recording MBID when known, else a normalized text
key). ``(source, source_id)`` is the natural key used for upserts.
"""
__tablename__ = "track"
id = Column(Integer, primary_key=True)
source = Column(String, nullable=False, index=True)
source_id = Column(String, nullable=False) # spotify id / itunes track id / local path
title = Column(String)
artist = Column(String) # primary artist (display + matching)
artists = Column(String) # all artists, joined with ", "
album = Column(String)
duration_ms = Column(Integer)
play_count = Column(Integer, default=0)
local_path = Column(String)
# Stable identifiers used for ID-based matching.
isrc = Column(String, index=True)
mbid = Column(String, index=True) # recording MBID
norm_key = Column(String, index=True) # normalized "artist|title" fallback
# Enrichment bookkeeping so the resolver only touches unresolved rows.
mbid_status = Column(String, index=True) # None | found | notfound | error
mbid_source = Column(String) # isrc | text
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
__table_args__ = (
UniqueConstraint("source", "source_id", name="uq_track_source_id"),
)
@property
def match_key(self) -> str:
"""Key used to decide identity across sources: prefer the recording
MBID, fall back to the normalized text key."""
return self.mbid or self.norm_key
class MbCache(Base):
"""Cache of MusicBrainz lookups (positive AND negative) so the resolver is
incremental and never repeats a query — keyed by ISRC or by a normalized
text query."""
__tablename__ = "mb_cache"
query_key = Column(String, primary_key=True) # "isrc:XXX" or "txt:artist|title|album"
mbid = Column(String)
score = Column(Integer)
status = Column(String) # found | notfound | error
fetched_at = Column(DateTime, default=datetime.utcnow)
class FileState(Base):
"""Per-file index state for incremental local scanning: a file is only
re-read when its mtime/size change."""
__tablename__ = "file_state"
path = Column(String, primary_key=True)
mtime = Column(Float)
size = Column(Integer)
indexed_at = Column(DateTime, default=datetime.utcnow)
class IngestState(Base):
"""Small key/value table for incremental-fetch bookkeeping, e.g. playlist
snapshot ids or the newest liked-track id seen on the last run."""
__tablename__ = "ingest_state"
key = Column(String, primary_key=True)
value = Column(String)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)

0
src/enrich/__init__.py Normal file
View file

143
src/enrich/mbid.py Normal file
View file

@ -0,0 +1,143 @@
"""Resolve a MusicBrainz recording MBID for each track WITHOUT touching files.
Strategy per the matching design:
1. If the track has an ISRC (Spotify), resolve it directly via MB's ISRC
endpoint — this is exact.
2. Otherwise (iTunes / local), text-search MB by artist + title and accept
the top hit only above a score threshold.
Incremental + cheap:
- Only tracks with mbid_status NULL (or 'error', to retry) are processed.
- Every distinct lookup (ISRC or normalized text) is cached in mb_cache,
including negative results, so duplicates across sources cost no network
and re-runs never repeat a query.
- MB's ~1 req/s limit is respected by musicbrainzngs; commits happen
periodically so an interrupted run resumes where it left off.
"""
from datetime import datetime
import musicbrainzngs as mb
from sqlalchemy import or_
from tqdm import tqdm
from Env import mb_contact
from src.db.database import session_scope
from src.db.models import MbCache, Track, MB_ERROR, MB_FOUND, MB_NOTFOUND
from src.match.normalize import normalize_artist, normalize_title
TEXT_SCORE_THRESHOLD = 90
_COMMIT_EVERY = 20
_agent_ready = False
def _ensure_agent():
global _agent_ready
if not _agent_ready:
mb.set_useragent("MusicIndexer", "0.1", mb_contact)
mb.set_rate_limit(1.0, 1) # 1 request / second
_agent_ready = True
def _score(rec):
try:
return int(rec.get("ext:score", rec.get("score", 0)))
except (TypeError, ValueError):
return 0
def _lookup_isrc(isrc):
try:
res = mb.get_recordings_by_isrc(isrc)
except mb.ResponseError as exc:
if getattr(getattr(exc, "cause", None), "code", None) == 404:
return None, None, MB_NOTFOUND
raise
recs = res.get("isrc", {}).get("recording-list", [])
return (recs[0]["id"], None, MB_FOUND) if recs else (None, None, MB_NOTFOUND)
def _lookup_text(artist, title):
res = mb.search_recordings(artist=artist, recording=title, limit=5)
recs = res.get("recording-list", [])
if not recs:
return None, None, MB_NOTFOUND
best = max(recs, key=_score)
score = _score(best)
if score >= TEXT_SCORE_THRESHOLD:
return best["id"], score, MB_FOUND
return None, score, MB_NOTFOUND
def _cached(session, key, lookup):
"""Return (mbid, score, status) from cache or by calling `lookup` once."""
row = session.get(MbCache, key)
if row is not None:
return row.mbid, row.score, row.status
mbid, score, status = lookup()
session.add(MbCache(query_key=key, mbid=mbid, score=score,
status=status, fetched_at=datetime.utcnow()))
return mbid, score, status
def _resolve_one(session, track):
"""(mbid, score, status, source).
Text search on NORMALIZED artist/title is the primary lookup so that the
same song from different sources issues an identical query -> identical
recording MBID (and a shared cache entry). ISRC is an exact fallback when
the text search finds nothing.
"""
na, nt = normalize_artist(track.artist or ""), normalize_title(track.title or "")
if na and nt:
key = f"txt:{na}|{nt}"
mbid, score, status = _cached(session, key, lambda: _lookup_text(na, nt))
if status == MB_FOUND:
return mbid, score, MB_FOUND, "text"
if track.isrc:
mbid, score, status = _cached(
session, f"isrc:{track.isrc}", lambda: _lookup_isrc(track.isrc)
)
if status == MB_FOUND:
return mbid, score, MB_FOUND, "isrc"
return None, None, MB_NOTFOUND, ("text" if na and nt else None)
def resolve_mbids(limit=None, source=None, retry_errors=True):
"""Resolve MBIDs for unresolved (and optionally errored) tracks."""
_ensure_agent()
stats = {MB_FOUND: 0, MB_NOTFOUND: 0, MB_ERROR: 0}
with session_scope() as session:
status_filter = (
or_(Track.mbid_status.is_(None), Track.mbid_status == MB_ERROR)
if retry_errors else Track.mbid_status.is_(None)
)
query = session.query(Track).filter(status_filter)
if source:
query = query.filter(Track.source == source)
tracks = query.order_by(Track.id).all()
if limit:
tracks = tracks[:limit]
for i, track in enumerate(tqdm(tracks, desc="resolve MBID", unit="track")):
try:
mbid, score, status, src = _resolve_one(session, track)
track.mbid = mbid
track.mbid_status = status
track.mbid_source = src
stats[status] = stats.get(status, 0) + 1
except mb.WebServiceError:
track.mbid_status = MB_ERROR
stats[MB_ERROR] += 1
if (i + 1) % _COMMIT_EVERY == 0:
session.commit()
return stats
def unresolved_count(source=None):
with session_scope() as session:
q = session.query(Track).filter(Track.mbid_status.is_(None))
if source:
q = q.filter(Track.source == source)
return q.count()

0
src/ingest/__init__.py Normal file
View file

130
src/ingest/ingest.py Normal file
View file

@ -0,0 +1,130 @@
"""Load prefetched JSON (work_dir/latest-1) into the DB.
This stands in for the live backends while keeping the same contract they will
use: an idempotent UPSERT keyed by (source, source_id). Re-running updates only
changed descriptive fields and NEVER discards enrichment (mbid/mbid_status) or
rows that disappeared from a single export.
"""
import json
from tqdm import tqdm
from Env import prefetched_dir
from src.db.database import session_scope
from src.db.models import (
Track,
SOURCE_ITUNES,
SOURCE_LOCAL,
SOURCE_SPOTIFY,
)
from src.match.normalize import norm_key
def _load(path):
with open(path, encoding="utf-8") as fh:
return json.load(fh)
def _id_name_maps(lib):
artists = {a["id"]: a.get("title") for a in lib.get("artists", [])}
albums = {al["id"]: al.get("title") for al in lib.get("albums", [])}
return artists, albums
def _spotify_isrc_map(spotify_dir):
"""spotify_track_id -> ISRC, gathered from every raw payload that carries
full track objects (liked, top, playlist tracks)."""
out = {}
for fname in ("playlistTracks.json", "top_tracks.json", "tracks.json"):
path = spotify_dir / fname
if not path.exists():
continue
data = _load(path)
items = data if isinstance(data, list) else [v for lst in data.values() for v in lst]
for item in items:
track = item.get("track") if isinstance(item, dict) and "track" in item else item
if track and track.get("id"):
isrc = (track.get("external_ids") or {}).get("isrc")
if isrc:
out[track["id"]] = isrc
return out
class _Upserter:
"""Preloads existing rows for a source so upserts are O(1) lookups."""
def __init__(self, session, source):
self.session = session
self.source = source
self.existing = {
t.source_id: t
for t in session.query(Track).filter_by(source=source).all()
}
self.n_new = 0
self.n_updated = 0
def upsert(self, source_id, *, isrc=None, **fields):
track = self.existing.get(source_id)
if track is None:
track = Track(source=self.source, source_id=source_id, isrc=isrc, **fields)
self.session.add(track)
self.existing[source_id] = track
self.n_new += 1
return
# Existing: refresh descriptive fields, preserve enrichment unless the
# ISRC actually changed (then the cached MBID is stale).
for key, value in fields.items():
setattr(track, key, value)
if isrc is not None and isrc != track.isrc:
track.isrc = isrc
track.mbid = None
track.mbid_status = None
track.mbid_source = None
elif isrc is not None:
track.isrc = isrc
self.n_updated += 1
def _ingest_library(session, source, lib, *, source_id_field, isrc_map=None):
artists, albums = _id_name_maps(lib)
up = _Upserter(session, source)
for t in tqdm(lib.get("tracks", []), desc=f"ingest {source}", unit="track"):
source_id = t.get(source_id_field) or t.get("id")
if not source_id:
continue
names = [artists.get(aid, aid) for aid in t.get("artists", []) if aid]
names = [n for n in names if n]
primary = names[0] if names else None
title = t.get("title")
up.upsert(
source_id,
title=title,
artist=primary,
artists=", ".join(names),
album=albums.get(t.get("album"), t.get("album")),
duration_ms=t.get("duration_ms"),
play_count=t.get("play_count") or 0,
local_path=t.get("local_path"),
norm_key=norm_key(primary or "", title or ""),
isrc=(isrc_map or {}).get(source_id),
)
return up.n_new, up.n_updated
def ingest_all():
base = prefetched_dir
results = {}
with session_scope() as session:
results[SOURCE_SPOTIFY] = _ingest_library(
session, SOURCE_SPOTIFY, _load(base / "spotify_library.json"),
source_id_field="id", isrc_map=_spotify_isrc_map(base / "spotify"),
)
results[SOURCE_ITUNES] = _ingest_library(
session, SOURCE_ITUNES, _load(base / "itunes_library.json"),
source_id_field="id",
)
results[SOURCE_LOCAL] = _ingest_library(
session, SOURCE_LOCAL, _load(base / "local_library.json"),
source_id_field="local_path",
)
return results

0
src/match/__init__.py Normal file
View file

93
src/match/matcher.py Normal file
View file

@ -0,0 +1,93 @@
"""Cross-source matching: decide which Spotify/iTunes tracks you do NOT have
locally, using real identifiers first and normalized text as a fallback.
A "wanted" track (Spotify or iTunes) counts as present locally if either:
- its recording MBID equals a local track's MBID -> method "mbid" (certain)
- its normalized artist|title equals a local one -> method "text" (fuzzy)
Otherwise it is reported as missing.
"""
from src.db.database import session_scope
from src.db.models import (
SOURCE_ITUNES,
SOURCE_LOCAL,
SOURCE_SPOTIFY,
Track,
)
WANTED_SOURCES = (SOURCE_SPOTIFY, SOURCE_ITUNES)
def _local_index(session):
local = session.query(Track).filter_by(source=SOURCE_LOCAL).all()
isrcs = {t.isrc for t in local if t.isrc}
mbids = {t.mbid for t in local if t.mbid}
norms = {t.norm_key for t in local if t.norm_key}
return isrcs, mbids, norms
def _classify(track, local_isrcs, local_mbids, local_norms):
"""Return (has_local: bool, method). Tiers, most certain first:
ISRC > recording MBID > normalized text."""
if track.isrc and track.isrc in local_isrcs:
return True, "isrc"
if track.mbid and track.mbid in local_mbids:
return True, "mbid"
if track.norm_key and track.norm_key in local_norms:
return True, "text"
return False, None
def _row(track, method):
return {
"source": track.source,
"artist": track.artist,
"title": track.title,
"album": track.album,
"isrc": track.isrc,
"mbid": track.mbid,
"matched_by": method,
"spotify_id": track.source_id if track.source == SOURCE_SPOTIFY else None,
"play_count": track.play_count,
}
def classify_all(sources=WANTED_SOURCES):
"""Return (missing_rows, present_rows) for the given wanted sources."""
with session_scope() as session:
local_isrcs, local_mbids, local_norms = _local_index(session)
wanted = session.query(Track).filter(Track.source.in_(sources)).all()
missing, present = [], []
for track in wanted:
has_local, method = _classify(track, local_isrcs, local_mbids, local_norms)
(present if has_local else missing).append(_row(track, method))
return missing, present
def missing_tracks(sources=WANTED_SOURCES):
return classify_all(sources)[0]
def summary():
"""Per-source counts: total / present-by-mbid / present-by-text / missing,
plus how many tracks carry an MBID at all (enrichment coverage)."""
with session_scope() as session:
local_isrcs, local_mbids, local_norms = _local_index(session)
out = {}
for source in WANTED_SOURCES:
tracks = session.query(Track).filter_by(source=source).all()
counts = {"total": len(tracks), "present_mbid": 0,
"present_text": 0, "missing": 0, "has_mbid": 0}
for track in tracks:
if track.mbid:
counts["has_mbid"] += 1
has_local, method = _classify(track, local_isrcs, local_mbids, local_norms)
if not has_local:
counts["missing"] += 1
elif method == "text":
counts["present_text"] += 1
else: # isrc or mbid -> ID-certain
counts["present_mbid"] += 1
out[source] = counts
local_total = session.query(Track).filter_by(source=SOURCE_LOCAL).count()
out[SOURCE_LOCAL] = {"total": local_total, "has_mbid": len(local_mbids)}
return out

48
src/match/normalize.py Normal file
View file

@ -0,0 +1,48 @@
"""Text normalization for the fuzzy/text-fallback match key.
Goal: collapse cosmetic differences between sources (accents, case, "feat."
clauses, "- Remastered 2011" / "(Live)" qualifiers, punctuation) so the same
song lands on the same key, while keeping genuinely different versions
(e.g. Remix) distinct.
"""
import re
import unicodedata
_FEAT = re.compile(r"\s*[\(\[]?\s*(feat|ft|featuring|with)\.?\s.*$", re.IGNORECASE)
_BRACKETS = re.compile(r"\s*[\(\[][^\)\]]*[\)\]]")
# trailing "- Remastered 2011", "- Live", "- Mono", "- 2009 Remaster", etc.
_DASH_QUALIFIER = re.compile(
r"\s*-\s*.*?\b(remaster(ed)?|remastered|live|mono|stereo|version|edit|"
r"anniversary|deluxe|bonus|radio|single)\b.*$",
re.IGNORECASE,
)
_NON_ALNUM = re.compile(r"[^\w\s]", re.UNICODE)
_WS = re.compile(r"\s+")
def strip_accents(s: str) -> str:
s = unicodedata.normalize("NFKD", s)
return "".join(c for c in s if not unicodedata.combining(c))
def _base(s: str) -> str:
s = strip_accents(s or "").lower()
s = _NON_ALNUM.sub(" ", s)
return _WS.sub(" ", s).strip()
def normalize_artist(name: str) -> str:
return _base(name)
def normalize_title(title: str) -> str:
s = title or ""
s = _FEAT.sub("", s)
s = _DASH_QUALIFIER.sub("", s)
s = _BRACKETS.sub("", s)
return _base(s)
def norm_key(artist: str, title: str) -> str:
"""Identity key when no recording MBID is available."""
return f"{normalize_artist(artist)}|{normalize_title(title)}"