diff --git a/hash/gameboy.xml b/hash/gameboy.xml index baed31824ff..4e32bdf05f8 100644 --- a/hash/gameboy.xml +++ b/hash/gameboy.xml @@ -17128,7 +17128,7 @@ - + @@ -17739,8 +17739,7 @@ Nintendo - - + @@ -17763,8 +17762,7 @@ Nintendo - - + @@ -17786,8 +17784,7 @@ Nintendo - - + diff --git a/hash/saturn.xml b/hash/saturn.xml index 1a2dca87fa3..efe068137c2 100644 --- a/hash/saturn.xml +++ b/hash/saturn.xml @@ -21034,9 +21034,9 @@ Olympic Soccer (Fra) T-7904H-09 1998 SNK + - diff --git a/hash/snes.xml b/hash/snes.xml index c01f6cbbd56..001268c90b0 100644 --- a/hash/snes.xml +++ b/hash/snes.xml @@ -2751,9 +2751,7 @@ Beyond that last category are the roms waiting to be classified. - - - + @@ -4589,8 +4587,7 @@ more investigation needed... 1994 Nintendo - - + diff --git a/hash/vsmile_cart.xml b/hash/vsmile_cart.xml index 2366856b370..3c39c5b2b31 100644 --- a/hash/vsmile_cart.xml +++ b/hash/vsmile_cart.xml @@ -3809,7 +3809,6 @@ V.Smile Smartbook Smartidges (need a Smartbook touch tablet connected to a regul - diff --git a/scripts/minimaws/lib/dbaccess.py b/scripts/minimaws/lib/dbaccess.py index 24dbb163171..f81dad66b83 100644 --- a/scripts/minimaws/lib/dbaccess.py +++ b/scripts/minimaws/lib/dbaccess.py @@ -141,6 +141,71 @@ class SchemaQueries(object): ' size INTEGER NOT NULL,\n' \ ' FOREIGN KEY (machine) REFERENCES machine (id),\n' \ ' FOREIGN KEY (machine, size) REFERENCES ramoption (machine, size))' + CREATE_SOFTWAREINFOTYPE = \ + 'CREATE TABLE softwareinfotype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' name TEXT NOT NULL,\n' \ + ' UNIQUE (name ASC))' + CREATE_SOFTWARESHAREDFEATTYPE = \ + 'CREATE TABLE softwaresharedfeattype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' name TEXT NOT NULL,\n' \ + ' UNIQUE (name ASC))' + CREATE_SOFTWAREPARTFEATURETYPE = \ + 'CREATE TABLE softwarepartfeaturetype (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' name TEXT NOT NULL,\n' \ + ' UNIQUE (name ASC))' + CREATE_SOFTWARELIST = \ + 'CREATE TABLE softwarelist (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' shortname TEXT NOT NULL,\n' \ + ' description TEXT NOT NULL,\n' \ + ' UNIQUE (shortname ASC))' + CREATE_SOFTWARE = \ + 'CREATE TABLE software (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' softwarelist INTEGER NOT NULL,\n' \ + ' shortname TEXT NOT NULL,\n' \ + ' supported INTEGER NOT NULL,\n' \ + ' description TEXT NOT NULL,\n' \ + ' year TEXT NOT NULL,\n' \ + ' publisher TEXT NOT NULL,\n' \ + ' UNIQUE (softwarelist ASC, shortname ASC),\n' \ + ' FOREIGN KEY (softwarelist) REFERENCES softwarelist (id))' + CREATE_SOFTWAREINFO = \ + 'CREATE TABLE softwareinfo (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' software INTEGER NOT NULL,\n' \ + ' infotype INTEGER NOT NULL,\n' \ + ' value TEXT NOT NULL,\n' \ + ' FOREIGN KEY (software) REFERENCES software (id),\n' \ + ' FOREIGN KEY (infotype) REFERENCES softwareinfotype (id))' + CREATE_SOFTWARESHAREDFEAT = \ + 'CREATE TABLE softwaresharedfeat (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' software INTEGER NOT NULL,\n' \ + ' sharedfeattype INTEGER NOT NULL,\n' \ + ' value TEXT NOT NULL,\n' \ + ' UNIQUE (software ASC, sharedfeattype ASC),\n' \ + ' FOREIGN KEY (software) REFERENCES software (id),\n' \ + ' FOREIGN KEY (sharedfeattype) REFERENCES softwaresharedfeattype (id))' + CREATE_SOFTWAREPART = \ + 'CREATE TABLE softwarepart (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' software INTEGER NOT NULL,\n' \ + ' shortname TEXT NOT NULL,\n' \ + ' interface TEXT NOT NULL,\n' \ + ' UNIQUE (software ASC, shortname ASC))' + CREATE_SOFTWAREPARTFEATURE = \ + 'CREATE TABLE softwarepartfeature (\n' \ + ' id INTEGER PRIMARY KEY,\n' \ + ' part INTEGER NOT NULL,\n' \ + ' featuretype INTEGER NOT NULL,\n' \ + ' value TEXT NOT NULL,\n' \ + ' UNIQUE (part ASC, featuretype ASC),\n' \ + ' FOREIGN KEY (part) REFERENCES softwarepart (id),\n' \ + ' FOREIGN KEY (featuretype) REFERENCES softwarepartfeaturetype (id))' CREATE_ROM = \ 'CREATE TABLE rom (\n' \ ' id INTEGER PRIMARY KEY,\n' \ @@ -193,6 +258,22 @@ class SchemaQueries(object): INDEX_DIPSWITCH_MACHINE_ISCONFIG = 'CREATE INDEX dipswitch_machine_isconfig ON dipswitch (machine ASC, isconfig ASC)' + INDEX_SOFTWARE_SOFTWARELIST_SUPPORTED = 'CREATE INDEX software_softwarelist_supported ON software (softwarelist ASC, supported ASC)' + INDEX_SOFTWARE_SOFTWARELIST_YEAR = 'CREATE INDEX software_softwarelist_year ON software (softwarelist ASC, year ASC)' + INDEX_SOFTWARE_SOFTWARELIST_PUBLISHER = 'CREATE INDEX software_softwarelist_publisher ON software (softwarelist ASC, publisher ASC)' + INDEX_SOFTWARE_SHORTNAME_SOFTWARELIST = 'CREATE INDEX software_shortname_softwarelist ON software (shortname ASC, softwarelist ASC)' + INDEX_SOFTWARE_YEAR_SOFTWARELIST = 'CREATE INDEX software_year_softwarelist ON software (year ASC, softwarelist ASC)' + INDEX_SOFTWARE_PUBLISHER_SOFTWARELIST = 'CREATE INDEX software_publisher_softwarelist ON software (publisher ASC, softwarelist ASC)' + + INDEX_SOFTWAREINFO_SOFTWARE_INFOTYPE = 'CREATE INDEX softwareinfo_software_infotype ON softwareinfo (software ASC, infotype ASC)' + INDEX_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE = 'CREATE INDEX softwareinfo_infotype_value_software ON softwareinfo (infotype ASC, value ASC, software ASC)' + + INDEX_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE = 'CREATE INDEX softwaresharedfeat_sharedfeattype_value_software ON softwaresharedfeat (sharedfeattype ASC, value ASC, software ASC)' + + INDEX_SOFTWAREPART_INTERFACE_SOFTWARE = 'CREATE INDEX softwarepart_interface_software ON softwarepart (interface ASC, software ASC)' + + INDEX_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART = 'CREATE INDEX softwarepartfeature_featuretype_value_part ON softwarepartfeature (featuretype ASC, value ASC, part ASC)' + INDEX_ROMDUMP_ROM = 'CREATE INDEX romdump_rom ON romdump (rom ASC)' INDEX_DISKDUMP_DISK = 'CREATE INDEX diskdump_disk ON diskdump (disk ASC)' @@ -211,6 +292,22 @@ class SchemaQueries(object): DROP_DIPSWITCH_MACHINE_ISCONFIG = 'DROP INDEX IF EXISTS dipswitch_machine_isconfig' + DROP_SOFTWARE_SOFTWARELIST_SUPPORTED = 'DROP INDEX IF EXISTS software_softwarelist_supported' + DROP_SOFTWARE_SOFTWARELIST_YEAR = 'DROP INDEX IF EXISTS software_softwarelist_year' + DROP_SOFTWARE_SOFTWARELIST_PUBLISHER = 'DROP INDEX IF EXISTS software_softwarelist_publisher' + DROP_SOFTWARE_SHORTNAME_SOFTWARELIST = 'DROP INDEX IF EXISTS software_shortname_softwarelist' + DROP_SOFTWARE_YEAR_SOFTWARELIST = 'DROP INDEX IF EXISTS software_year_softwarelist' + DROP_SOFTWARE_PUBLISHER_SOFTWARELIST = 'DROP INDEX IF EXISTS software_publisher_softwarelist' + + DROP_SOFTWAREINFO_SOFTWARE_INFOTYPE = 'DROP INDEX IF EXISTS softwareinfo_software_infotype' + DROP_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE = 'DROP INDEX IF EXISTS softwareinfo_infotype_value_software' + + DROP_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE = 'DROP INDEX IF EXISTS softwaresharedfeat_sharedfeattype_value_software' + + DROP_SOFTWAREPART_INTERFACE_SOFTWARE = 'DROP INDEX IF EXISTS softwarepart_interface_software' + + DROP_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART = 'DROP INDEX IF EXISTS softwarepartfeature_featuretype_value_part' + DROP_ROMDUMP_ROM = 'DROP INDEX IF EXISTS romdump_rom' DROP_DISKDUMP_DISK = 'DROP INDEX IF EXISTS diskdump_disk' @@ -234,6 +331,15 @@ class SchemaQueries(object): CREATE_SLOTDEFAULT, CREATE_RAMOPTION, CREATE_RAMDEFAULT, + CREATE_SOFTWAREINFOTYPE, + CREATE_SOFTWARESHAREDFEATTYPE, + CREATE_SOFTWAREPARTFEATURETYPE, + CREATE_SOFTWARELIST, + CREATE_SOFTWARE, + CREATE_SOFTWAREINFO, + CREATE_SOFTWARESHAREDFEAT, + CREATE_SOFTWAREPART, + CREATE_SOFTWAREPARTFEATURE, CREATE_ROM, CREATE_ROMDUMP, CREATE_DISK, @@ -254,6 +360,17 @@ class SchemaQueries(object): INDEX_ROMOF_PARENT, INDEX_CLONEOF_PARENT, INDEX_DIPSWITCH_MACHINE_ISCONFIG, + INDEX_SOFTWARE_SOFTWARELIST_SUPPORTED, + INDEX_SOFTWARE_SOFTWARELIST_YEAR, + INDEX_SOFTWARE_SOFTWARELIST_PUBLISHER, + INDEX_SOFTWARE_SHORTNAME_SOFTWARELIST, + INDEX_SOFTWARE_YEAR_SOFTWARELIST, + INDEX_SOFTWARE_PUBLISHER_SOFTWARELIST, + INDEX_SOFTWAREINFO_SOFTWARE_INFOTYPE, + INDEX_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE, + INDEX_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE, + INDEX_SOFTWAREPART_INTERFACE_SOFTWARE, + INDEX_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART, INDEX_ROMDUMP_ROM, INDEX_DISKDUMP_DISK) @@ -267,6 +384,17 @@ class SchemaQueries(object): DROP_ROMOF_PARENT, DROP_CLONEOF_PARENT, DROP_DIPSWITCH_MACHINE_ISCONFIG, + DROP_SOFTWARE_SOFTWARELIST_SUPPORTED, + DROP_SOFTWARE_SOFTWARELIST_YEAR, + DROP_SOFTWARE_SOFTWARELIST_PUBLISHER, + DROP_SOFTWARE_SHORTNAME_SOFTWARELIST, + DROP_SOFTWARE_YEAR_SOFTWARELIST, + DROP_SOFTWARE_PUBLISHER_SOFTWARELIST, + DROP_SOFTWAREINFO_SOFTWARE_INFOTYPE, + DROP_SOFTWAREINFO_INFOTYPE_VALUE_SOFTWARE, + DROP_SOFTWARESHAREDFEAT_SHAREDFEATTYPE_VALUE_SOFTWARE, + DROP_SOFTWAREPART_INTERFACE_SOFTWARE, + DROP_SOFTWAREPARTFEATURE_FEATURETYPE_VALUE_PART, DROP_ROMDUMP_ROM, DROP_DISKDUMP_DISK) @@ -287,6 +415,15 @@ class UpdateQueries(object): ADD_SLOT = 'INSERT INTO slot (machine, name) VALUES (?, ?)' ADD_RAMOPTION = 'INSERT INTO ramoption (machine, size, name) VALUES (?, ?, ?)' ADD_RAMDEFAULT = 'INSERT INTO ramdefault (machine, size) VALUES (?, ?)' + ADD_SOFTWAREINFOTYPE = 'INSERT OR IGNORE INTO softwareinfotype (name) VALUES (?)' + ADD_SOFTWARESHAREDFEATTYPE = 'INSERT OR IGNORE INTO softwaresharedfeattype (name) VALUES (?)' + ADD_SOFTWAREPARTFEATURETYPE = 'INSERT OR IGNORE INTO softwarepartfeaturetype (name) VALUES(?)' + ADD_SOFTWARELIST = 'INSERT INTO softwarelist (shortname, description) VALUES (?, ?)' + ADD_SOFTWARE = 'INSERT INTO software (softwarelist, shortname, supported, description, year, publisher) VALUES (?, ?, ?, ?, ?, ?)' + ADD_SOFTWAREINFO = 'INSERT INTO softwareinfo (software, infotype, value) SELECT ?, id, ? FROM softwareinfotype WHERE name = ?' + ADD_SOFTWARESHAREDFEAT = 'INSERT INTO softwaresharedfeat (software, sharedfeattype, value) SELECT ?, id, ? FROM softwaresharedfeattype WHERE name = ?' + ADD_SOFTWAREPART = 'INSERT INTO softwarepart (software, shortname, interface) VALUES (?, ?, ?)' + ADD_SOFTWAREPARTFEATURE = 'INSERT INTO softwarepartfeature (part, featuretype, value) SELECT ?, id, ? FROM softwarepartfeaturetype WHERE name = ?' ADD_ROM = 'INSERT OR IGNORE INTO rom (crc, sha1) VALUES (?, ?)' ADD_ROMDUMP = 'INSERT OR IGNORE INTO romdump (machine, rom, name, bad) SELECT ?, id, ?, ? FROM rom WHERE crc = ? AND sha1 = ?' ADD_DISK = 'INSERT OR IGNORE INTO disk (sha1) VALUES (?)' @@ -599,6 +736,39 @@ class UpdateCursor(object): self.dbcurs.execute(UpdateQueries.ADD_RAMDEFAULT, (machine, size)) return self.dbcurs.lastrowid + def add_softwarelist(self, shortname, description): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARELIST, (shortname, description)) + return self.dbcurs.lastrowid + + def add_softwareinfotype(self, name): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREINFOTYPE, (name, )) + + def add_softwaresharedfeattype(self, name): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARESHAREDFEATTYPE, (name, )) + + def add_softwarepartfeaturetype(self, name): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREPARTFEATURETYPE, (name, )) + + def add_software(self, softwarelist, shortname, supported, description, year, publisher): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARE, (softwarelist, shortname, supported, description, year, publisher)) + return self.dbcurs.lastrowid + + def add_softwareinfo(self, software, infotype, value): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREINFO, (software, value, infotype)) + return self.dbcurs.lastrowid + + def add_softwaresharedfeat(self, software, sharedfeattype, value): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWARESHAREDFEAT, (software, value, sharedfeattype)) + return self.dbcurs.lastrowid + + def add_softwarepart(self, software, shortname, interface): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREPART, (software, shortname, interface)) + return self.dbcurs.lastrowid + + def add_softwarepartfeature(self, part, featuretype, value): + self.dbcurs.execute(UpdateQueries.ADD_SOFTWAREPARTFEATURE, (part, value, featuretype)) + return self.dbcurs.lastrowid + def add_rom(self, crc, sha1): self.dbcurs.execute(UpdateQueries.ADD_ROM, (crc, sha1)) return self.dbcurs.lastrowid diff --git a/scripts/minimaws/lib/lxparse.py b/scripts/minimaws/lib/lxparse.py index 96669ea99eb..746cba33fe8 100644 --- a/scripts/minimaws/lib/lxparse.py +++ b/scripts/minimaws/lib/lxparse.py @@ -5,6 +5,8 @@ from . import dbaccess +import os +import os.path import subprocess import xml.sax import xml.sax.saxutils @@ -191,6 +193,9 @@ class MachineHandler(ElementHandler): self.romof = attrs.get('romof') self.dbcurs.add_sourcefile(self.sourcefile) + def endMainElement(self, name): + self.dbcurs.close() + def startChildElement(self, name, attrs): if name in self.CHILD_HANDLERS: self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) @@ -238,9 +243,6 @@ class MachineHandler(ElementHandler): self.manufacturer = handler.text self.dbcurs.add_system(self.id, self.year, self.manufacturer) - def endMainElement(self, name): - self.dbcurs.close() - class ListXmlHandler(ElementHandler): def __init__(self, dbconn, **kwargs): @@ -259,12 +261,10 @@ class ListXmlHandler(ElementHandler): msg=('Expected "mame" element but found "%s"' % (name, )), exception=None, locator=self.locator) - self.dbconn.prepare_for_load() self.machines = 0 def endMainElement(self, name): - # TODO: build index by first letter or whatever - self.dbconn.finalise_load() + self.dbconn.commit() def startChildElement(self, name, attrs): if name != 'machine': @@ -286,11 +286,127 @@ class ListXmlHandler(ElementHandler): pass +class SoftwarePartHandler(ElementHandler): + def __init__(self, parent, **kwargs): + super(SoftwarePartHandler, self).__init__(parent=parent, **kwargs) + self.dbcurs = parent.dbcurs + self.software = parent.id + + def startMainElement(self, name, attrs): + self.id = self.dbcurs.add_softwarepart(self.software, attrs['name'], attrs['interface']) + + def startChildElement(self, name, attrs): + if name == 'feature': + self.dbcurs.add_softwarepartfeaturetype(attrs['name']) + self.dbcurs.add_softwarepartfeature(self.id, attrs['name'], attrs['value']) + self.setChildHandler(name, attrs, self.IGNORE) + + +class SoftwareHandler(ElementHandler): + CHILD_HANDLERS = { + 'description': TextAccumulator, + 'year': TextAccumulator, + 'publisher': TextAccumulator, + 'part': SoftwarePartHandler } + + def __init__(self, parent, **kwargs): + super(SoftwareHandler, self).__init__(parent=parent, **kwargs) + self.dbcurs = self.dbconn.cursor() + self.softwarelist = parent.id + + def startMainElement(self, name, attrs): + self.shortname = attrs['name'] + self.cloneof = attrs.get('cloneof') + self.supported = 0 if (attrs.get('supported', 'yes') == 'yes') else 1 if (attrs.get('supported', 'yes') == 'partial') else 2 + + def endMainElement(self, name): + self.dbcurs.close() + + def startChildElement(self, name, attrs): + if name in self.CHILD_HANDLERS: + self.setChildHandler(name, attrs, self.CHILD_HANDLERS[name](self)) + else: + if name == 'info': + self.dbcurs.add_softwareinfotype(attrs['name']) + self.dbcurs.add_softwareinfo(self.id, attrs['name'], attrs['value']) + elif name == 'sharedfeat': + self.dbcurs.add_softwaresharedfeattype(attrs['name']) + self.dbcurs.add_softwaresharedfeat(self.id, attrs['name'], attrs['value']) + self.setChildHandler(name, attrs, self.IGNORE) + + def endChildHandler(self, name, handler): + if name == 'description': + self.description = handler.text + elif name == 'year': + self.year = handler.text + elif name == 'publisher': + self.publisher = handler.text + self.id = self.dbcurs.add_software(self.softwarelist, self.shortname, self.supported, self.description, self.year, self.publisher) + + +class SoftwareListHandler(ElementHandler): + def __init__(self, dbconn, **kwargs): + super(SoftwareListHandler, self).__init__(parent=None, **kwargs) + self.dbconn = dbconn + + def startDocument(self): + pass + + def endDocument(self): + pass + + def startMainElement(self, name, attrs): + if name != 'softwarelist': + raise xml.sax.SAXParseException( + msg=('Expected "softwarelist" element but found "%s"' % (name, )), + exception=None, + locator=self.locator) + self.shortname = attrs['name'] + self.description = attrs['description'] + self.entries = 0 + dbcurs = self.dbconn.cursor() + self.id = dbcurs.add_softwarelist(self.shortname, self.description) + dbcurs.close() + + def endMainElement(self, name): + self.dbconn.commit() + + def startChildElement(self, name, attrs): + if name != 'software': + raise xml.sax.SAXParseException( + msg=('Expected "software" element but found "%s"' % (name, )), + exception=None, + locator=self.locator) + self.setChildHandler(name, attrs, SoftwareHandler(self)) + + def endChildHandler(self, name, handler): + if name == 'software': + if self.entries >= 1023: + self.dbconn.commit() + self.entries = 0 + else: + self.entries += 1 + + def processingInstruction(self, target, data): + pass + + def load_info(options): + dbconn = dbaccess.UpdateConnection(options.database) + dbconn.prepare_for_load() parser = xml.sax.make_parser() - parser.setContentHandler(ListXmlHandler(dbaccess.UpdateConnection(options.database))) + + parser.setContentHandler(ListXmlHandler(dbconn)) if options.executable is not None: task = subprocess.Popen([options.executable, '-listxml'], stdout=subprocess.PIPE) parser.parse(task.stdout) else: parser.parse(options.file) + + if options.softwarepath: + parser.setContentHandler(SoftwareListHandler(dbconn)) + for path in options.softwarepath: + for filename in [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.xml')]: + parser.parse(filename) + + dbconn.finalise_load() diff --git a/scripts/minimaws/minimaws.py b/scripts/minimaws/minimaws.py index 5b657af5524..58ca456159c 100755 --- a/scripts/minimaws/minimaws.py +++ b/scripts/minimaws/minimaws.py @@ -111,6 +111,7 @@ if __name__ == '__main__': group = subparser.add_mutually_exclusive_group(required=True) group.add_argument('--executable', metavar='', help='emulator executable') group.add_argument('--file', metavar='', help='XML machine information file') + subparser.add_argument('--softwarepath', action='append', metavar='', help='Software list directory path') options = parser.parse_args()