mirror of
https://github.com/holub/mame
synced 2025-04-19 23:12:11 +03:00
minimaws: load ROMs and disks, and add a romident subcommand
This commit is contained in:
Vas Crabb
parent
1077396473
commit
8e31f22bcd
@ -5,7 +5,150 @@
|
||||
|
||||
from . import dbaccess
|
||||
|
||||
import codecs
|
||||
import hashlib
|
||||
import os
|
||||
import os.path
|
||||
import struct
|
||||
import sys
|
||||
import zlib
|
||||
|
||||
|
||||
class _Identifier(object):
|
||||
def __init__(self, dbcurs, **kwargs):
|
||||
super(_Identifier, self).__init__(**kwargs)
|
||||
self.dbcurs = dbcurs
|
||||
self.pathwidth = 0
|
||||
self.labelwidth = 0
|
||||
self.matches = { }
|
||||
self.unmatched = [ ]
|
||||
|
||||
def processRomFile(self, path, f):
|
||||
crc, sha1 = self.digestRom(f)
|
||||
matched = False
|
||||
for shortname, description, label, bad in self.dbcurs.get_rom_dumps(crc, sha1):
|
||||
matched = True
|
||||
self.labelwidth = max(len(label), self.labelwidth)
|
||||
romset = self.matches.get(shortname)
|
||||
if romset is None:
|
||||
romset = (description, [])
|
||||
self.matches[shortname] = romset
|
||||
romset[1].append((path, label, bad))
|
||||
if not matched:
|
||||
self.unmatched.append((path, crc, sha1))
|
||||
self.pathwidth = max(len(path), self.pathwidth)
|
||||
|
||||
def processChd(self, path, sha1):
|
||||
matched = False
|
||||
for shortname, description, label, bad in self.dbcurs.get_disk_dumps(sha1):
|
||||
matched = True
|
||||
self.labelwidth = max(len(label), self.labelwidth)
|
||||
romset = self.matches.get(shortname)
|
||||
if romset is None:
|
||||
romset = (description, [])
|
||||
self.matches[shortname] = romset
|
||||
romset[1].append((path, label, bad))
|
||||
if not matched:
|
||||
self.unmatched.append((path, None, sha1))
|
||||
self.pathwidth = max(len(path), self.pathwidth)
|
||||
|
||||
def processFile(self, path):
|
||||
if os.path.splitext(path)[1].lower() != '.chd':
|
||||
with open(path, mode='rb', buffering=0) as f:
|
||||
self.processRomFile(path, f)
|
||||
else:
|
||||
with open(path, mode='rb') as f:
|
||||
sha1 = self.probeChd(f)
|
||||
if sha1 is None:
|
||||
f.seek(0)
|
||||
self.processRomFile(path, f)
|
||||
else:
|
||||
self.processChd(path, sha1)
|
||||
|
||||
def processPath(self, path, depth=0):
|
||||
try:
|
||||
if not os.path.isdir(path):
|
||||
self.processFile(path)
|
||||
elif depth > 5:
|
||||
sys.stderr.write('Not examining \'%s\' - maximum depth exceeded\n')
|
||||
else:
|
||||
for name in os.listdir(path):
|
||||
self.processPath(os.path.join(path, name), depth + 1)
|
||||
except BaseException as e:
|
||||
sys.stderr.write('Error identifying \'%s\': %s\n' % (path, e))
|
||||
|
||||
def printResults(self):
|
||||
pw = self.pathwidth - (self.pathwidth % 4) + 4
|
||||
lw = self.labelwidth - (self.labelwidth % 4) + 4
|
||||
first = True
|
||||
for shortname, romset in sorted(self.matches.items()):
|
||||
if first:
|
||||
first = False
|
||||
else:
|
||||
sys.stdout.write('\n')
|
||||
sys.stdout.write('%-20s%s\n' % (shortname, romset[0]))
|
||||
for path, label, bad in romset[1]:
|
||||
if bad:
|
||||
sys.stdout.write(' %-*s= %-*s(BAD)\n' % (pw, path, lw, label))
|
||||
else:
|
||||
sys.stdout.write(' %-*s= %s\n' % (pw, path, label))
|
||||
if self.unmatched:
|
||||
if first:
|
||||
first = False
|
||||
else:
|
||||
sys.stdout.write('\n')
|
||||
sys.stdout.write('Unmatched\n')
|
||||
for path, crc, sha1 in self.unmatched:
|
||||
if crc is not None:
|
||||
sys.stdout.write(' %-*sCRC(%08x) SHA1(%s)\n' % (pw, path, crc, sha1))
|
||||
else:
|
||||
sys.stdout.write(' %-*sSHA1(%s)\n' % (pw, path, sha1))
|
||||
|
||||
@staticmethod
|
||||
def iterateBlocks(f, s=65536):
|
||||
while True:
|
||||
buf = f.read(s)
|
||||
if buf:
|
||||
yield buf
|
||||
else:
|
||||
break
|
||||
|
||||
@staticmethod
|
||||
def digestRom(f):
|
||||
crc = zlib.crc32(bytes())
|
||||
sha = hashlib.sha1()
|
||||
for block in _Identifier.iterateBlocks(f):
|
||||
crc = zlib.crc32(block, crc)
|
||||
sha.update(block)
|
||||
return crc & 0xffffffff, sha.hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def probeChd(f):
|
||||
buf = f.read(16)
|
||||
if (len(buf) != 16) or (buf[:8] != b'MComprHD'):
|
||||
return None
|
||||
headerlen, version = struct.unpack('>II', buf[8:])
|
||||
if version == 3:
|
||||
if headerlen != 120:
|
||||
return None
|
||||
sha1offs = 80
|
||||
elif version == 4:
|
||||
if headerlen != 108:
|
||||
return None
|
||||
sha1offs = 48
|
||||
elif version == 5:
|
||||
if headerlen != 124:
|
||||
return None
|
||||
sha1offs = 84
|
||||
else:
|
||||
return None
|
||||
f.seek(sha1offs)
|
||||
if f.tell() != sha1offs:
|
||||
return None
|
||||
buf = f.read(20)
|
||||
if len(buf) != 20:
|
||||
return None
|
||||
return codecs.getencoder('hex_codec')(buf)[0].decode('ascii')
|
||||
|
||||
|
||||
def do_listfull(options):
|
||||
@ -68,6 +211,7 @@ def do_listbrothers(options):
|
||||
dbcurs.close()
|
||||
dbconn.close()
|
||||
|
||||
|
||||
def do_listaffected(options):
|
||||
dbconn = dbaccess.QueryConnection(options.database)
|
||||
dbcurs = dbconn.cursor()
|
||||
@ -81,3 +225,14 @@ def do_listaffected(options):
|
||||
sys.stderr.write('No matching systems found for \'%s\'\n' % (options.pattern, ))
|
||||
dbcurs.close()
|
||||
dbconn.close()
|
||||
|
||||
|
||||
def do_romident(options):
|
||||
dbconn = dbaccess.QueryConnection(options.database)
|
||||
dbcurs = dbconn.cursor()
|
||||
ident = _Identifier(dbcurs)
|
||||
for path in options.path:
|
||||
ident.processPath(path)
|
||||
ident.printResults()
|
||||
dbcurs.close()
|
||||
dbconn.close()
|
||||
|
||||
@ -141,6 +141,35 @@ class SchemaQueries(object):
|
||||
' size INTEGER NOT NULL,\n' \
|
||||
' FOREIGN KEY (machine) REFERENCES machine (id),\n' \
|
||||
' FOREIGN KEY (machine, size) REFERENCES ramoption (machine, size))'
|
||||
CREATE_ROM = \
|
||||
'CREATE TABLE rom (\n' \
|
||||
' id INTEGER PRIMARY KEY,\n' \
|
||||
' crc INTEGER NOT NULL,\n' \
|
||||
' sha1 TEXT NOT NULL,\n' \
|
||||
' UNIQUE (crc ASC, sha1 ASC))'
|
||||
CREATE_ROMDUMP = \
|
||||
'CREATE TABLE romdump (\n' \
|
||||
' machine INTEGER NOT NULL,\n' \
|
||||
' rom INTEGER NOT NULL,\n' \
|
||||
' name TEXT NOT NULL,\n' \
|
||||
' bad INTEGER NOT NULL,\n' \
|
||||
' FOREIGN KEY (machine) REFERENCES machine (id),\n' \
|
||||
' FOREIGN KEY (rom) REFERENCES rom (id),\n' \
|
||||
' UNIQUE (machine, rom, name))'
|
||||
CREATE_DISK = \
|
||||
'CREATE TABLE disk (\n' \
|
||||
' id INTEGER PRIMARY KEY,\n' \
|
||||
' sha1 TEXT NOT NULL,\n' \
|
||||
' UNIQUE (sha1 ASC))'
|
||||
CREATE_DISKDUMP = \
|
||||
'CREATE TABLE diskdump (\n' \
|
||||
' machine INTEGER NOT NULL,\n' \
|
||||
' disk INTEGER NOT NULL,\n' \
|
||||
' name TEXT NOT NULL,\n' \
|
||||
' bad INTEGER NOT NULL,\n' \
|
||||
' FOREIGN KEY (machine) REFERENCES machine (id),\n' \
|
||||
' FOREIGN KEY (disk) REFERENCES disk (id),\n' \
|
||||
' UNIQUE (machine, disk, name))'
|
||||
|
||||
CREATE_TEMPORARY_DEVICEREFERENCE = 'CREATE TEMPORARY TABLE temp_devicereference (id INTEGER PRIMARY KEY, machine INTEGER NOT NULL, device TEXT NOT NULL, UNIQUE (machine, device))'
|
||||
CREATE_TEMPORARY_SLOTOPTION = 'CREATE TEMPORARY TABLE temp_slotoption (id INTEGER PRIMARY KEY, slot INTEGER NOT NULL, device TEXT NOT NULL, name TEXT NOT NULL)'
|
||||
@ -164,6 +193,10 @@ class SchemaQueries(object):
|
||||
|
||||
INDEX_DIPSWITCH_MACHINE_ISCONFIG = 'CREATE INDEX dipswitch_machine_isconfig ON dipswitch (machine ASC, isconfig ASC)'
|
||||
|
||||
INDEX_ROMDUMP_ROM = 'CREATE INDEX romdump_rom ON romdump (rom ASC)'
|
||||
|
||||
INDEX_DISKDUMP_DISK = 'CREATE INDEX diskdump_disk ON diskdump (disk ASC)'
|
||||
|
||||
DROP_MACHINE_ISDEVICE_SHORTNAME = 'DROP INDEX IF EXISTS machine_isdevice_shortname'
|
||||
DROP_MACHINE_ISDEVICE_DESCRIPTION = 'DROP INDEX IF EXISTS machine_isdevice_description'
|
||||
DROP_MACHINE_RUNNABLE_SHORTNAME = 'DROP INDEX IF EXISTS machine_runnable_shortname'
|
||||
@ -178,6 +211,10 @@ class SchemaQueries(object):
|
||||
|
||||
DROP_DIPSWITCH_MACHINE_ISCONFIG = 'DROP INDEX IF EXISTS dipswitch_machine_isconfig'
|
||||
|
||||
DROP_ROMDUMP_ROM = 'DROP INDEX IF EXISTS romdump_rom'
|
||||
|
||||
DROP_DISKDUMP_DISK = 'DROP INDEX IF EXISTS diskdump_disk'
|
||||
|
||||
CREATE_TABLES = (
|
||||
CREATE_FEATURETYPE,
|
||||
CREATE_SOURCEFILE,
|
||||
@ -196,7 +233,11 @@ class SchemaQueries(object):
|
||||
CREATE_SLOTOPTION,
|
||||
CREATE_SLOTDEFAULT,
|
||||
CREATE_RAMOPTION,
|
||||
CREATE_RAMDEFAULT)
|
||||
CREATE_RAMDEFAULT,
|
||||
CREATE_ROM,
|
||||
CREATE_ROMDUMP,
|
||||
CREATE_DISK,
|
||||
CREATE_DISKDUMP)
|
||||
|
||||
CREATE_TEMPORARY_TABLES = (
|
||||
CREATE_TEMPORARY_DEVICEREFERENCE,
|
||||
@ -212,7 +253,9 @@ class SchemaQueries(object):
|
||||
INDEX_SYSTEM_MANUFACTURER,
|
||||
INDEX_ROMOF_PARENT,
|
||||
INDEX_CLONEOF_PARENT,
|
||||
INDEX_DIPSWITCH_MACHINE_ISCONFIG)
|
||||
INDEX_DIPSWITCH_MACHINE_ISCONFIG,
|
||||
INDEX_ROMDUMP_ROM,
|
||||
INDEX_DISKDUMP_DISK)
|
||||
|
||||
DROP_INDEXES = (
|
||||
DROP_MACHINE_ISDEVICE_SHORTNAME,
|
||||
@ -223,7 +266,9 @@ class SchemaQueries(object):
|
||||
DROP_SYSTEM_MANUFACTURER,
|
||||
DROP_ROMOF_PARENT,
|
||||
DROP_CLONEOF_PARENT,
|
||||
DROP_DIPSWITCH_MACHINE_ISCONFIG)
|
||||
DROP_DIPSWITCH_MACHINE_ISCONFIG,
|
||||
DROP_ROMDUMP_ROM,
|
||||
DROP_DISKDUMP_DISK)
|
||||
|
||||
|
||||
class UpdateQueries(object):
|
||||
@ -242,6 +287,10 @@ class UpdateQueries(object):
|
||||
ADD_SLOT = 'INSERT INTO slot (machine, name) VALUES (?, ?)'
|
||||
ADD_RAMOPTION = 'INSERT INTO ramoption (machine, size, name) VALUES (?, ?, ?)'
|
||||
ADD_RAMDEFAULT = 'INSERT INTO ramdefault (machine, size) VALUES (?, ?)'
|
||||
ADD_ROM = 'INSERT OR IGNORE INTO rom (crc, sha1) VALUES (?, ?)'
|
||||
ADD_ROMDUMP = 'INSERT OR IGNORE INTO romdump (machine, rom, name, bad) SELECT ?, id, ?, ? FROM rom WHERE crc = ? AND sha1 = ?'
|
||||
ADD_DISK = 'INSERT OR IGNORE INTO disk (sha1) VALUES (?)'
|
||||
ADD_DISKDUMP = 'INSERT OR IGNORE INTO diskdump (machine, disk, name, bad) SELECT ?, id, ?, ? FROM disk WHERE sha1 = ?'
|
||||
|
||||
ADD_TEMPORARY_DEVICEREFERENCE = 'INSERT OR IGNORE INTO temp_devicereference (machine, device) VALUES (?, ?)'
|
||||
ADD_TEMPORARY_SLOTOPTION = 'INSERT INTO temp_slotoption (slot, device, name) VALUES (?, ?, ?)'
|
||||
@ -458,6 +507,20 @@ class QueryCursor(object):
|
||||
'ORDER BY ramoption.size',
|
||||
(machine, ))
|
||||
|
||||
def get_rom_dumps(self, crc, sha1):
|
||||
return self.dbcurs.execute(
|
||||
'SELECT machine.shortname AS shortname, machine.description AS description, romdump.name AS label, romdump.bad AS bad ' \
|
||||
'FROM romdump LEFT JOIN machine ON romdump.machine = machine.id ' \
|
||||
'WHERE romdump.rom = (SELECT id FROM rom WHERE crc = ? AND sha1 = ?)',
|
||||
(crc, sha1))
|
||||
|
||||
def get_disk_dumps(self, sha1):
|
||||
return self.dbcurs.execute(
|
||||
'SELECT machine.shortname AS shortname, machine.description AS description, diskdump.name AS label, diskdump.bad AS bad ' \
|
||||
'FROM diskdump LEFT JOIN machine ON diskdump.machine = machine.id ' \
|
||||
'WHERE diskdump.disk = (SELECT id FROM disk WHERE sha1 = ?)',
|
||||
(sha1, ))
|
||||
|
||||
|
||||
class UpdateCursor(object):
|
||||
def __init__(self, dbconn, **kwargs):
|
||||
@ -536,6 +599,22 @@ class UpdateCursor(object):
|
||||
self.dbcurs.execute(UpdateQueries.ADD_RAMDEFAULT, (machine, size))
|
||||
return self.dbcurs.lastrowid
|
||||
|
||||
def add_rom(self, crc, sha1):
|
||||
self.dbcurs.execute(UpdateQueries.ADD_ROM, (crc, sha1))
|
||||
return self.dbcurs.lastrowid
|
||||
|
||||
def add_romdump(self, machine, name, crc, sha1, bad):
|
||||
self.dbcurs.execute(UpdateQueries.ADD_ROMDUMP, (machine, name, 1 if bad else 0, crc, sha1))
|
||||
return self.dbcurs.lastrowid
|
||||
|
||||
def add_disk(self, sha1):
|
||||
self.dbcurs.execute(UpdateQueries.ADD_DISK, (sha1, ))
|
||||
return self.dbcurs.lastrowid
|
||||
|
||||
def add_diskdump(self, machine, name, sha1, bad):
|
||||
self.dbcurs.execute(UpdateQueries.ADD_DISKDUMP, (machine, name, 1 if bad else 0, sha1))
|
||||
return self.dbcurs.lastrowid
|
||||
|
||||
|
||||
class QueryConnection(object):
|
||||
def __init__(self, database, **kwargs):
|
||||
|
||||
@ -206,6 +206,22 @@ class MachineHandler(ElementHandler):
|
||||
status = 0 if 'status' not in attrs else 2 if attrs['status'] == 'unemulated' else 1
|
||||
overall = status if 'overall' not in attrs else 2 if attrs['overall'] == 'unemulated' else 1
|
||||
self.dbcurs.add_feature(self.id, attrs['type'], status, overall)
|
||||
elif name == 'rom':
|
||||
crc = attrs.get('crc')
|
||||
sha1 = attrs.get('sha1')
|
||||
if (crc is not None) and (sha1 is not None):
|
||||
crc = int(crc, 16)
|
||||
sha1 = sha1.lower()
|
||||
self.dbcurs.add_rom(crc, sha1)
|
||||
status = attrs.get('status', 'good')
|
||||
self.dbcurs.add_romdump(self.id, attrs['name'], crc, sha1, status != 'good')
|
||||
elif name == 'disk':
|
||||
sha1 = attrs.get('sha1')
|
||||
if sha1 is not None:
|
||||
sha1 = sha1.lower()
|
||||
self.dbcurs.add_disk(sha1)
|
||||
status = attrs.get('status', 'good')
|
||||
self.dbcurs.add_diskdump(self.id, attrs['name'], sha1, status != 'good')
|
||||
self.setChildHandler(name, attrs, self.IGNORE)
|
||||
|
||||
def endChildHandler(self, name, handler):
|
||||
|
||||
@ -26,6 +26,11 @@
|
||||
## $ python minimaws.py listclones "unkch*"
|
||||
## $ python minimaws.py listbrothers superx
|
||||
##
|
||||
## The romident command does not support archives, but it's far faster
|
||||
## than using MAME as it has optimised indexes:
|
||||
##
|
||||
## $ python minimaws.py romident 27c64.bin dump-dir
|
||||
##
|
||||
## One more sophisticated query command is provided that MAME has no
|
||||
## equivalent for. The listaffected command shows all runnable machines
|
||||
## that reference devices defined in specified source files:
|
||||
@ -100,6 +105,9 @@ if __name__ == '__main__':
|
||||
subparser = subparsers.add_parser('listaffected', help='show drivers affected by source change(s)')
|
||||
subparser.add_argument('pattern', nargs='+', metavar='<pat>', help='source file glob pattern')
|
||||
|
||||
subparser = subparsers.add_parser('romident', help='identify ROM dump(s)')
|
||||
subparser.add_argument('path', nargs='+', metavar='<path>', help='ROM dump file/directory path')
|
||||
|
||||
subparser = subparsers.add_parser('serve', help='serve over HTTP')
|
||||
subparser.add_argument('--port', metavar='<port>', default=8080, type=int, help='server TCP port')
|
||||
subparser.add_argument('--host', metavar='<host>', default='', help='server TCP hostname')
|
||||
@ -120,6 +128,8 @@ if __name__ == '__main__':
|
||||
lib.auxverbs.do_listbrothers(options)
|
||||
elif options.command == 'listaffected':
|
||||
lib.auxverbs.do_listaffected(options)
|
||||
elif options.command == 'romident':
|
||||
lib.auxverbs.do_romident(options)
|
||||
elif options.command == 'serve':
|
||||
lib.wsgiserve.run_server(options)
|
||||
elif options.command == 'load':
|
||||
|
||||
Loading…
Reference in New Issue
Block a user