Real backends + fetch/display workflows; config paths

Wire the incremental pipeline to real sources and expose two workflows.

Config (Env.py): three explicit paths — LOCAL_LIBRARY_PATH (scan source),
ITUNES_XML_PATH (manual export), DATA_DIR (storage root: DB + Spotify cache).

Real, incremental ingesters:
- local_scan: walks the library, skips files whose mtime/size are unchanged,
  prunes deleted ones, and reads embedded ISRC + recording MBID from tags
  (tagged files get real IDs with no network), via tags.py.
- itunes_xml: parse the iTunes Library.xml as a plist; skip if mtime unchanged.
- spotify_raw: ingest raw Spotify JSON (ISRC from external_ids); live fetch
  wrapper for when credentials are set.

Workflows (src/workflows.py + main.py):
- `fetch`  — pull + scan + parse + enrich + match, with readable step logs.
  First run is slow (full scan + MB lookups); later runs only touch changes
  (verified: ~71s first, ~0.6s second).
- `display` — launch the GUI.

Also: enrich uses ISRC-first again (local MBIDs now come from tags), fix
--enrich-limit 0, quiet musicbrainzngs XML warnings, ignore /data/.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Шурупов Илья Викторович 2026-06-15 01:46:56 +03:00
parent 0e64d439c4
commit 03b98d9a1d
14 changed files with 570 additions and 243 deletions

73
main.py
View file

@ -1,46 +1,47 @@
"""MusicIndexer pipeline CLI.
"""MusicIndexer CLI. Two main workflows plus granular helpers.
Everything is incremental and DB-backed (see Env.database_url). Typical flow:
python main.py fetch # do all the work: pull + scan + parse + enrich + match
python main.py display # open the GUI to see what you're missing
python main.py ingest # load prefetched data into the DB (upsert)
python main.py enrich # resolve recording MBIDs (resumable, cached)
python main.py match # print present/missing summary
./gui.sh # browse what you're missing
`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it
just continues where it stopped.
`fetch` is incremental and DB-backed (see Env.py): the first run is slow
(initial scan + MusicBrainz lookups), later runs only touch what changed.
Helpers: `enrich` (resume MBID resolution), `match` (print the summary).
"""
import argparse
import json
from src.db.database import init_db
import logging
def cmd_init(args):
init_db()
print("Database initialized.")
def _setup_logging():
logging.basicConfig(level=logging.INFO, format="%(message)s")
def cmd_ingest(args):
init_db()
from src.ingest.ingest import ingest_all
def cmd_fetch(args):
_setup_logging()
from src.workflows import run_fetch
results = ingest_all()
for source, (new, updated) in results.items():
print(f"{source:8} +{new} new, {updated} updated")
run_fetch(enrich_limit=args.enrich_limit)
def cmd_display(args):
from src.workflows import run_display
run_display()
def cmd_enrich(args):
init_db()
_setup_logging()
from src.db.database import init_db
from src.enrich.mbid import resolve_mbids, unresolved_count
init_db()
print(f"unresolved before: {unresolved_count(args.source)}")
stats = resolve_mbids(limit=args.limit, source=args.source)
print(f"resolved this run: {stats}")
print(f"resolved this run: {resolve_mbids(limit=args.limit, source=args.source)}")
print(f"unresolved after: {unresolved_count(args.source)}")
def cmd_match(args):
import json
from src.match.matcher import summary
print(json.dumps(summary(), indent=2))
@ -50,14 +51,15 @@ def build_parser():
parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__)
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("init", help="create database tables")
sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)")
fetch = sub.add_parser("fetch", help="incrementally fetch + scan + enrich + match")
fetch.add_argument("--enrich-limit", type=int, default=None,
help="cap MBID lookups this run (default: resolve all)")
enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks")
enrich.add_argument("--limit", type=int, default=None,
help="cap how many tracks to resolve this run")
enrich.add_argument("--source", choices=["spotify", "itunes", "local"],
default=None, help="restrict to one source")
sub.add_parser("display", help="open the Streamlit GUI")
enrich = sub.add_parser("enrich", help="resume MBID resolution only")
enrich.add_argument("--limit", type=int, default=None)
enrich.add_argument("--source", choices=["spotify", "itunes", "local"], default=None)
sub.add_parser("match", help="print present/missing summary")
return parser
@ -65,13 +67,12 @@ def build_parser():
def main():
args = build_parser().parse_args()
handlers = {
"init": cmd_init,
"ingest": cmd_ingest,
{
"fetch": cmd_fetch,
"display": cmd_display,
"enrich": cmd_enrich,
"match": cmd_match,
}
handlers[args.cmd](args)
}[args.cmd](args)
if __name__ == "__main__":