diff --git a/scripts/minimaws/lib/auxverbs.py b/scripts/minimaws/lib/auxverbs.py index c68c528af6f..b67a2fafb99 100644 --- a/scripts/minimaws/lib/auxverbs.py +++ b/scripts/minimaws/lib/auxverbs.py @@ -5,7 +5,150 @@ from . import dbaccess +import codecs +import hashlib +import os +import os.path +import struct import sys +import zlib + + +class _Identifier(object): + def __init__(self, dbcurs, **kwargs): + super(_Identifier, self).__init__(**kwargs) + self.dbcurs = dbcurs + self.pathwidth = 0 + self.labelwidth = 0 + self.matches = { } + self.unmatched = [ ] + + def processRomFile(self, path, f): + crc, sha1 = self.digestRom(f) + matched = False + for shortname, description, label, bad in self.dbcurs.get_rom_dumps(crc, sha1): + matched = True + self.labelwidth = max(len(label), self.labelwidth) + romset = self.matches.get(shortname) + if romset is None: + romset = (description, []) + self.matches[shortname] = romset + romset[1].append((path, label, bad)) + if not matched: + self.unmatched.append((path, crc, sha1)) + self.pathwidth = max(len(path), self.pathwidth) + + def processChd(self, path, sha1): + matched = False + for shortname, description, label, bad in self.dbcurs.get_disk_dumps(sha1): + matched = True + self.labelwidth = max(len(label), self.labelwidth) + romset = self.matches.get(shortname) + if romset is None: + romset = (description, []) + self.matches[shortname] = romset + romset[1].append((path, label, bad)) + if not matched: + self.unmatched.append((path, None, sha1)) + self.pathwidth = max(len(path), self.pathwidth) + + def processFile(self, path): + if os.path.splitext(path)[1].lower() != '.chd': + with open(path, mode='rb', buffering=0) as f: + self.processRomFile(path, f) + else: + with open(path, mode='rb') as f: + sha1 = self.probeChd(f) + if sha1 is None: + f.seek(0) + self.processRomFile(path, f) + else: + self.processChd(path, sha1) + + def processPath(self, path, depth=0): + try: + if not os.path.isdir(path): + self.processFile(path) + elif depth > 5: + sys.stderr.write('Not examining \'%s\' - maximum depth exceeded\n') + else: + for name in os.listdir(path): + self.processPath(os.path.join(path, name), depth + 1) + except BaseException as e: + sys.stderr.write('Error identifying \'%s\': %s\n' % (path, e)) + + def printResults(self): + pw = self.pathwidth - (self.pathwidth % 4) + 4 + lw = self.labelwidth - (self.labelwidth % 4) + 4 + first = True + for shortname, romset in sorted(self.matches.items()): + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('%-20s%s\n' % (shortname, romset[0])) + for path, label, bad in romset[1]: + if bad: + sys.stdout.write(' %-*s= %-*s(BAD)\n' % (pw, path, lw, label)) + else: + sys.stdout.write(' %-*s= %s\n' % (pw, path, label)) + if self.unmatched: + if first: + first = False + else: + sys.stdout.write('\n') + sys.stdout.write('Unmatched\n') + for path, crc, sha1 in self.unmatched: + if crc is not None: + sys.stdout.write(' %-*sCRC(%08x) SHA1(%s)\n' % (pw, path, crc, sha1)) + else: + sys.stdout.write(' %-*sSHA1(%s)\n' % (pw, path, sha1)) + + @staticmethod + def iterateBlocks(f, s=65536): + while True: + buf = f.read(s) + if buf: + yield buf + else: + break + + @staticmethod + def digestRom(f): + crc = zlib.crc32(bytes()) + sha = hashlib.sha1() + for block in _Identifier.iterateBlocks(f): + crc = zlib.crc32(block, crc) + sha.update(block) + return crc & 0xffffffff, sha.hexdigest() + + @staticmethod + def probeChd(f): + buf = f.read(16) + if (len(buf) != 16) or (buf[:8] != b'MComprHD'): + return None + headerlen, version = struct.unpack('>II', buf[8:]) + if version == 3: + if headerlen != 120: + return None + sha1offs = 80 + elif version == 4: + if headerlen != 108: + return None + sha1offs = 48 + elif version == 5: + if headerlen != 124: + return None + sha1offs = 84 + else: + return None + f.seek(sha1offs) + if f.tell() != sha1offs: + return None + buf = f.read(20) + if len(buf) != 20: + return None + return codecs.getencoder('hex_codec')(buf)[0].decode('ascii') def do_listfull(options): @@ -68,6 +211,7 @@ def do_listbrothers(options): dbcurs.close() dbconn.close() + def do_listaffected(options): dbconn = dbaccess.QueryConnection(options.database) dbcurs = dbconn.cursor() @@ -81,3 +225,14 @@ def do_listaffected(options): sys.stderr.write('No matching systems found for \'%s\'\n' % (options.pattern, )) dbcurs.close() dbconn.close() + + +def do_romident(options): + dbconn = dbaccess.QueryConnection(options.database) + dbcurs = dbconn.cursor() + ident = _Identifier(dbcurs) + for path in options.path: + ident.processPath(path) + ident.printResults() + dbcurs.close() + dbconn.close() diff --git a/scripts/minimaws/lib/dbaccess.py b/scripts/minimaws/lib/dbaccess.py index 57de6aeb236..d577c1831bd 100644 --- a/scripts/minimaws/lib/dbaccess.py +++ b/scripts/minimaws/lib/dbaccess.py @@ -141,6 +141,35 @@ class SchemaQueries(object): ' size INTEGER NOT NULL,\n' \ ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ ' FOREIGN KEY (machine, size) REFERENCES ramoption (machine, size))' + CREATE_ROM = \ + 'CREATE TABLE rom (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' crc INTEGER NOT NULL,\n' \ + ' sha1 TEXT NOT NULL,\n' \ + ' UNIQUE (crc ASC, sha1 ASC))' + CREATE_ROMDUMP = \ + 'CREATE TABLE romdump (\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' rom INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (rom) REFERENCES rom (id),\n' \ + ' UNIQUE (machine, rom, name))' + CREATE_DISK = \ + 'CREATE TABLE disk (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' sha1 TEXT NOT NULL,\n' \ + ' UNIQUE (sha1 ASC))' + CREATE_DISKDUMP = \ + 'CREATE TABLE diskdump (\n' \ + ' machine INTEGER NOT NULL,\n' \ + ' disk INTEGER NOT NULL,\n' \ + ' name TEXT NOT NULL,\n' \ + ' bad INTEGER NOT NULL,\n' \ + ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ + ' FOREIGN KEY (disk) REFERENCES disk (id),\n' \ + ' UNIQUE (machine, disk, name))' CREATE_TEMPORARY_DEVICEREFERENCE = 'CREATE TEMPORARY TABLE temp_devicereference (id INTEGER PRIMARY KEY, machine INTEGER NOT NULL, device TEXT NOT NULL, UNIQUE (machine, device))' CREATE_TEMPORARY_SLOTOPTION = 'CREATE TEMPORARY TABLE temp_slotoption (id INTEGER PRIMARY KEY, slot INTEGER NOT NULL, device TEXT NOT NULL, name TEXT NOT NULL)' @@ -164,6 +193,10 @@ class SchemaQueries(object): INDEX_DIPSWITCH_MACHINE_ISCONFIG = 'CREATE INDEX dipswitch_machine_isconfig ON dipswitch (machine ASC, isconfig ASC)' + INDEX_ROMDUMP_ROM = 'CREATE INDEX romdump_rom ON romdump (rom ASC)' + + INDEX_DISKDUMP_DISK = 'CREATE INDEX diskdump_disk ON diskdump (disk ASC)' + DROP_MACHINE_ISDEVICE_SHORTNAME = 'DROP INDEX IF EXISTS machine_isdevice_shortname' DROP_MACHINE_ISDEVICE_DESCRIPTION = 'DROP INDEX IF EXISTS machine_isdevice_description' DROP_MACHINE_RUNNABLE_SHORTNAME = 'DROP INDEX IF EXISTS machine_runnable_shortname' @@ -178,6 +211,10 @@ class SchemaQueries(object): DROP_DIPSWITCH_MACHINE_ISCONFIG = 'DROP INDEX IF EXISTS dipswitch_machine_isconfig' + DROP_ROMDUMP_ROM = 'DROP INDEX IF EXISTS romdump_rom' + + DROP_DISKDUMP_DISK = 'DROP INDEX IF EXISTS diskdump_disk' + CREATE_TABLES = ( CREATE_FEATURETYPE, CREATE_SOURCEFILE, @@ -196,7 +233,11 @@ class SchemaQueries(object): CREATE_SLOTOPTION, CREATE_SLOTDEFAULT, CREATE_RAMOPTION, - CREATE_RAMDEFAULT) + CREATE_RAMDEFAULT, + CREATE_ROM, + CREATE_ROMDUMP, + CREATE_DISK, + CREATE_DISKDUMP) CREATE_TEMPORARY_TABLES = ( CREATE_TEMPORARY_DEVICEREFERENCE, @@ -212,7 +253,9 @@ class SchemaQueries(object): INDEX_SYSTEM_MANUFACTURER, INDEX_ROMOF_PARENT, INDEX_CLONEOF_PARENT, - INDEX_DIPSWITCH_MACHINE_ISCONFIG) + INDEX_DIPSWITCH_MACHINE_ISCONFIG, + INDEX_ROMDUMP_ROM, + INDEX_DISKDUMP_DISK) DROP_INDEXES = ( DROP_MACHINE_ISDEVICE_SHORTNAME, @@ -223,7 +266,9 @@ class SchemaQueries(object): DROP_SYSTEM_MANUFACTURER, DROP_ROMOF_PARENT, DROP_CLONEOF_PARENT, - DROP_DIPSWITCH_MACHINE_ISCONFIG) + DROP_DIPSWITCH_MACHINE_ISCONFIG, + DROP_ROMDUMP_ROM, + DROP_DISKDUMP_DISK) class UpdateQueries(object): @@ -242,6 +287,10 @@ class UpdateQueries(object): ADD_SLOT = 'INSERT INTO slot (machine, name) VALUES (?, ?)' ADD_RAMOPTION = 'INSERT INTO ramoption (machine, size, name) VALUES (?, ?, ?)' ADD_RAMDEFAULT = 'INSERT INTO ramdefault (machine, size) VALUES (?, ?)' + ADD_ROM = 'INSERT OR IGNORE INTO rom (crc, sha1) VALUES (?, ?)' + ADD_ROMDUMP = 'INSERT OR IGNORE INTO romdump (machine, rom, name, bad) SELECT ?, id, ?, ? FROM rom WHERE crc = ? AND sha1 = ?' + ADD_DISK = 'INSERT OR IGNORE INTO disk (sha1) VALUES (?)' + ADD_DISKDUMP = 'INSERT OR IGNORE INTO diskdump (machine, disk, name, bad) SELECT ?, id, ?, ? FROM disk WHERE sha1 = ?' ADD_TEMPORARY_DEVICEREFERENCE = 'INSERT OR IGNORE INTO temp_devicereference (machine, device) VALUES (?, ?)' ADD_TEMPORARY_SLOTOPTION = 'INSERT INTO temp_slotoption (slot, device, name) VALUES (?, ?, ?)' @@ -458,6 +507,20 @@ class QueryCursor(object): 'ORDER BY ramoption.size', (machine, )) + def get_rom_dumps(self, crc, sha1): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, romdump.name AS label, romdump.bad AS bad ' \ + 'FROM romdump LEFT JOIN machine ON romdump.machine = machine.id ' \ + 'WHERE romdump.rom = (SELECT id FROM rom WHERE crc = ? AND sha1 = ?)', + (crc, sha1)) + + def get_disk_dumps(self, sha1): + return self.dbcurs.execute( + 'SELECT machine.shortname AS shortname, machine.description AS description, diskdump.name AS label, diskdump.bad AS bad ' \ + 'FROM diskdump LEFT JOIN machine ON diskdump.machine = machine.id ' \ + 'WHERE diskdump.disk = (SELECT id FROM disk WHERE sha1 = ?)', + (sha1, )) + class UpdateCursor(object): def __init__(self, dbconn, **kwargs): @@ -536,6 +599,22 @@ class UpdateCursor(object): self.dbcurs.execute(UpdateQueries.ADD_RAMDEFAULT, (machine, size)) return self.dbcurs.lastrowid + def add_rom(self, crc, sha1): + self.dbcurs.execute(UpdateQueries.ADD_ROM, (crc, sha1)) + return self.dbcurs.lastrowid + + def add_romdump(self, machine, name, crc, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_ROMDUMP, (machine, name, 1 if bad else 0, crc, sha1)) + return self.dbcurs.lastrowid + + def add_disk(self, sha1): + self.dbcurs.execute(UpdateQueries.ADD_DISK, (sha1, )) + return self.dbcurs.lastrowid + + def add_diskdump(self, machine, name, sha1, bad): + self.dbcurs.execute(UpdateQueries.ADD_DISKDUMP, (machine, name, 1 if bad else 0, sha1)) + return self.dbcurs.lastrowid + class QueryConnection(object): def __init__(self, database, **kwargs): diff --git a/scripts/minimaws/lib/lxparse.py b/scripts/minimaws/lib/lxparse.py index dfd8cbae45e..96669ea99eb 100644 --- a/scripts/minimaws/lib/lxparse.py +++ b/scripts/minimaws/lib/lxparse.py @@ -206,6 +206,22 @@ class MachineHandler(ElementHandler): status = 0 if 'status' not in attrs else 2 if attrs['status'] == 'unemulated' else 1 overall = status if 'overall' not in attrs else 2 if attrs['overall'] == 'unemulated' else 1 self.dbcurs.add_feature(self.id, attrs['type'], status, overall) + elif name == 'rom': + crc = attrs.get('crc') + sha1 = attrs.get('sha1') + if (crc is not None) and (sha1 is not None): + crc = int(crc, 16) + sha1 = sha1.lower() + self.dbcurs.add_rom(crc, sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_romdump(self.id, attrs['name'], crc, sha1, status != 'good') + elif name == 'disk': + sha1 = attrs.get('sha1') + if sha1 is not None: + sha1 = sha1.lower() + self.dbcurs.add_disk(sha1) + status = attrs.get('status', 'good') + self.dbcurs.add_diskdump(self.id, attrs['name'], sha1, status != 'good') self.setChildHandler(name, attrs, self.IGNORE) def endChildHandler(self, name, handler): diff --git a/scripts/minimaws/minimaws.py b/scripts/minimaws/minimaws.py index 656956a0275..399efa1784a 100755 --- a/scripts/minimaws/minimaws.py +++ b/scripts/minimaws/minimaws.py @@ -26,6 +26,11 @@ ## $ python minimaws.py listclones "unkch*" ## $ python minimaws.py listbrothers superx ## +## The romident command does not support archives, but it's far faster +## than using MAME as it has optimised indexes: +## +## $ python minimaws.py romident 27c64.bin dump-dir +## ## One more sophisticated query command is provided that MAME has no ## equivalent for. The listaffected command shows all runnable machines ## that reference devices defined in specified source files: @@ -100,6 +105,9 @@ if __name__ == '__main__': subparser = subparsers.add_parser('listaffected', help='show drivers affected by source change(s)') subparser.add_argument('pattern', nargs='+', metavar='', help='source file glob pattern') + subparser = subparsers.add_parser('romident', help='identify ROM dump(s)') + subparser.add_argument('path', nargs='+', metavar='', help='ROM dump file/directory path') + subparser = subparsers.add_parser('serve', help='serve over HTTP') subparser.add_argument('--port', metavar='', default=8080, type=int, help='server TCP port') subparser.add_argument('--host', metavar='', default='', help='server TCP hostname') @@ -120,6 +128,8 @@ if __name__ == '__main__': lib.auxverbs.do_listbrothers(options) elif options.command == 'listaffected': lib.auxverbs.do_listaffected(options) + elif options.command == 'romident': + lib.auxverbs.do_romident(options) elif options.command == 'serve': lib.wsgiserve.run_server(options) elif options.command == 'load':