DB-backed incremental redesign: find tracks missing locally
Replace the JSON-file pipeline with a SQLite/SQLAlchemy store (Postgres via DATABASE_URL) and an incremental, ID-based matcher whose goal is "which Spotify/iTunes tracks am I missing locally". - src/db: models (Track/MbCache/FileState/IngestState) + engine/session. - src/ingest: upsert prefetched spotify/itunes/local (work_dir/latest-1) by (source, source_id); pulls ISRC from raw Spotify; preserves enrichment. - src/enrich: MusicBrainz recording-MBID resolver. Resolves via NORMALIZED text search (so the same song across sources lands on one MBID + one cached lookup), ISRC as exact fallback. Only touches unresolved tracks, caches positive AND negative results, rate-limited and resumable. - src/match: normalization + cross-source matcher with tiers ISRC > MBID > normalized text; reports present/missing and the match method. - main.py: CLI (ingest / enrich / match). Gui.py: missing-tracks viewer. - README + requirements (SQLAlchemy, musicbrainzngs); ignore *.db. Incremental everywhere: re-running updates only what changed; enrich continues where it stopped. Live Spotify/local backends are left intact. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
83f782d125
commit
0e64d439c4
16 changed files with 784 additions and 274 deletions
96
main.py
96
main.py
|
|
@ -1,38 +1,78 @@
|
|||
import Flow
|
||||
"""MusicIndexer pipeline CLI.
|
||||
|
||||
Everything is incremental and DB-backed (see Env.database_url). Typical flow:
|
||||
|
||||
python main.py ingest # load prefetched data into the DB (upsert)
|
||||
python main.py enrich # resolve recording MBIDs (resumable, cached)
|
||||
python main.py match # print present/missing summary
|
||||
./gui.sh # browse what you're missing
|
||||
|
||||
`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it
|
||||
just continues where it stopped.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from src.db.database import init_db
|
||||
|
||||
|
||||
def test1():
|
||||
flow = Flow.Flow()
|
||||
def cmd_init(args):
|
||||
init_db()
|
||||
print("Database initialized.")
|
||||
|
||||
# flow.fetch_local()
|
||||
flow.parse_local_library()
|
||||
flow.load_libraries()
|
||||
|
||||
# flow.map_local_to_spotify()
|
||||
# flow.log_stats()
|
||||
def cmd_ingest(args):
|
||||
init_db()
|
||||
from src.ingest.ingest import ingest_all
|
||||
|
||||
# flow.fetch_and_save_lyrics()
|
||||
results = ingest_all()
|
||||
for source, (new, updated) in results.items():
|
||||
print(f"{source:8} +{new} new, {updated} updated")
|
||||
|
||||
|
||||
def cmd_enrich(args):
|
||||
init_db()
|
||||
from src.enrich.mbid import resolve_mbids, unresolved_count
|
||||
|
||||
print(f"unresolved before: {unresolved_count(args.source)}")
|
||||
stats = resolve_mbids(limit=args.limit, source=args.source)
|
||||
print(f"resolved this run: {stats}")
|
||||
print(f"unresolved after: {unresolved_count(args.source)}")
|
||||
|
||||
|
||||
def cmd_match(args):
|
||||
from src.match.matcher import summary
|
||||
|
||||
print(json.dumps(summary(), indent=2))
|
||||
|
||||
|
||||
def build_parser():
|
||||
parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("init", help="create database tables")
|
||||
sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)")
|
||||
|
||||
enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks")
|
||||
enrich.add_argument("--limit", type=int, default=None,
|
||||
help="cap how many tracks to resolve this run")
|
||||
enrich.add_argument("--source", choices=["spotify", "itunes", "local"],
|
||||
default=None, help="restrict to one source")
|
||||
|
||||
sub.add_parser("match", help="print present/missing summary")
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
flow = Flow.Flow()
|
||||
|
||||
flow.fetch_spotify()
|
||||
# flow.fetch_local()
|
||||
|
||||
flow.parse_spotify_library()
|
||||
#flow.parse_itunes_library()
|
||||
#flow.parse_local_library()
|
||||
|
||||
#flow.load_libraries()
|
||||
|
||||
#flow.map_local_to_spotify()
|
||||
#flow.save_libraries()
|
||||
|
||||
# flow.log_stats()
|
||||
|
||||
# print("asd")
|
||||
args = build_parser().parse_args()
|
||||
handlers = {
|
||||
"init": cmd_init,
|
||||
"ingest": cmd_ingest,
|
||||
"enrich": cmd_enrich,
|
||||
"match": cmd_match,
|
||||
}
|
||||
handlers[args.cmd](args)
|
||||
|
||||
|
||||
# test1()
|
||||
main()
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue