Real backends + fetch/display workflows; config paths
Wire the incremental pipeline to real sources and expose two workflows. Config (Env.py): three explicit paths — LOCAL_LIBRARY_PATH (scan source), ITUNES_XML_PATH (manual export), DATA_DIR (storage root: DB + Spotify cache). Real, incremental ingesters: - local_scan: walks the library, skips files whose mtime/size are unchanged, prunes deleted ones, and reads embedded ISRC + recording MBID from tags (tagged files get real IDs with no network), via tags.py. - itunes_xml: parse the iTunes Library.xml as a plist; skip if mtime unchanged. - spotify_raw: ingest raw Spotify JSON (ISRC from external_ids); live fetch wrapper for when credentials are set. Workflows (src/workflows.py + main.py): - `fetch` — pull + scan + parse + enrich + match, with readable step logs. First run is slow (full scan + MB lookups); later runs only touch changes (verified: ~71s first, ~0.6s second). - `display` — launch the GUI. Also: enrich uses ISRC-first again (local MBIDs now come from tags), fix --enrich-limit 0, quiet musicbrainzngs XML warnings, ignore /data/. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
0e64d439c4
commit
03b98d9a1d
14 changed files with 570 additions and 243 deletions
73
main.py
73
main.py
|
|
@ -1,46 +1,47 @@
|
|||
"""MusicIndexer pipeline CLI.
|
||||
"""MusicIndexer CLI. Two main workflows plus granular helpers.
|
||||
|
||||
Everything is incremental and DB-backed (see Env.database_url). Typical flow:
|
||||
python main.py fetch # do all the work: pull + scan + parse + enrich + match
|
||||
python main.py display # open the GUI to see what you're missing
|
||||
|
||||
python main.py ingest # load prefetched data into the DB (upsert)
|
||||
python main.py enrich # resolve recording MBIDs (resumable, cached)
|
||||
python main.py match # print present/missing summary
|
||||
./gui.sh # browse what you're missing
|
||||
|
||||
`enrich` only queries MusicBrainz for still-unresolved tracks, so re-running it
|
||||
just continues where it stopped.
|
||||
`fetch` is incremental and DB-backed (see Env.py): the first run is slow
|
||||
(initial scan + MusicBrainz lookups), later runs only touch what changed.
|
||||
Helpers: `enrich` (resume MBID resolution), `match` (print the summary).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from src.db.database import init_db
|
||||
import logging
|
||||
|
||||
|
||||
def cmd_init(args):
|
||||
init_db()
|
||||
print("Database initialized.")
|
||||
def _setup_logging():
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
|
||||
def cmd_ingest(args):
|
||||
init_db()
|
||||
from src.ingest.ingest import ingest_all
|
||||
def cmd_fetch(args):
|
||||
_setup_logging()
|
||||
from src.workflows import run_fetch
|
||||
|
||||
results = ingest_all()
|
||||
for source, (new, updated) in results.items():
|
||||
print(f"{source:8} +{new} new, {updated} updated")
|
||||
run_fetch(enrich_limit=args.enrich_limit)
|
||||
|
||||
|
||||
def cmd_display(args):
|
||||
from src.workflows import run_display
|
||||
|
||||
run_display()
|
||||
|
||||
|
||||
def cmd_enrich(args):
|
||||
init_db()
|
||||
_setup_logging()
|
||||
from src.db.database import init_db
|
||||
from src.enrich.mbid import resolve_mbids, unresolved_count
|
||||
|
||||
init_db()
|
||||
print(f"unresolved before: {unresolved_count(args.source)}")
|
||||
stats = resolve_mbids(limit=args.limit, source=args.source)
|
||||
print(f"resolved this run: {stats}")
|
||||
print(f"resolved this run: {resolve_mbids(limit=args.limit, source=args.source)}")
|
||||
print(f"unresolved after: {unresolved_count(args.source)}")
|
||||
|
||||
|
||||
def cmd_match(args):
|
||||
import json
|
||||
|
||||
from src.match.matcher import summary
|
||||
|
||||
print(json.dumps(summary(), indent=2))
|
||||
|
|
@ -50,14 +51,15 @@ def build_parser():
|
|||
parser = argparse.ArgumentParser(prog="musicindexer", description=__doc__)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("init", help="create database tables")
|
||||
sub.add_parser("ingest", help="load prefetched JSON into the DB (upsert)")
|
||||
fetch = sub.add_parser("fetch", help="incrementally fetch + scan + enrich + match")
|
||||
fetch.add_argument("--enrich-limit", type=int, default=None,
|
||||
help="cap MBID lookups this run (default: resolve all)")
|
||||
|
||||
enrich = sub.add_parser("enrich", help="resolve MBIDs for unresolved tracks")
|
||||
enrich.add_argument("--limit", type=int, default=None,
|
||||
help="cap how many tracks to resolve this run")
|
||||
enrich.add_argument("--source", choices=["spotify", "itunes", "local"],
|
||||
default=None, help="restrict to one source")
|
||||
sub.add_parser("display", help="open the Streamlit GUI")
|
||||
|
||||
enrich = sub.add_parser("enrich", help="resume MBID resolution only")
|
||||
enrich.add_argument("--limit", type=int, default=None)
|
||||
enrich.add_argument("--source", choices=["spotify", "itunes", "local"], default=None)
|
||||
|
||||
sub.add_parser("match", help="print present/missing summary")
|
||||
return parser
|
||||
|
|
@ -65,13 +67,12 @@ def build_parser():
|
|||
|
||||
def main():
|
||||
args = build_parser().parse_args()
|
||||
handlers = {
|
||||
"init": cmd_init,
|
||||
"ingest": cmd_ingest,
|
||||
{
|
||||
"fetch": cmd_fetch,
|
||||
"display": cmd_display,
|
||||
"enrich": cmd_enrich,
|
||||
"match": cmd_match,
|
||||
}
|
||||
handlers[args.cmd](args)
|
||||
}[args.cmd](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue