plugins/data: store data in database (nw)

plugins/data/data_command.lua

@@ -1,8 +1,10 @@
 local dat = {}
 local info, ver
-
 local datread = require("data/load_dat")
-datread, ver = datread.open("command.dat", nil)
+do
+	local convert = require("data/button_char")
+	datread, ver = datread.open("command.dat", "# Command List%-Shorthand", convert)
+end
 
 function dat.check(set, softlist)
 	if softlist or not datread then
@@ -13,8 +15,7 @@ function dat.check(set, softlist)
 	if not status or not info then
 		return nil
 	end
-	local convert = require("data/button_char")
+	info = "#jf\n" .. info
-	info = "#jf\n" .. convert(info)
 	return "Command"
 end
 

plugins/data/data_mameinfo.lua

@@ -1,7 +1,7 @@
 local dat = {}
 local ver, info
 local datread = require("data/load_dat")
-datread, ver = datread.open("mameinfo.dat", "# MAMEINFO.DAT")
+datread, ver = datread.open("mameinfo.dat", "# MAMEINFO.DAT", function(str) return str:gsub("\n\n", "\n") end)
 
 function dat.check(set, softlist)
 	if softlist or not datread then
@@ -17,7 +17,6 @@ function dat.check(set, softlist)
 	if drvinfo then
 		info = info .. "\n\n--- DRIVER INFO ---\nDriver: " .. sourcefile .. "\n\n" .. drvinfo
 	end
-	info = info:gsub("\n\n", "\n")
 	return "Mameinfo"
 end
 

plugins/data/data_messinfo.lua

@@ -2,7 +2,7 @@ local dat = {}
 local ver, info
 
 local datread = require("data/load_dat")
-datread, ver = datread.open("messinfo.dat", "#     MESSINFO.DAT")
+datread, ver = datread.open("messinfo.dat", "#     MESSINFO.DAT", function(str) return str:gsub("\n\n", "\n") end)
 
 function dat.check(set, softlist)
 	if softlist or not datread then
@@ -18,7 +18,6 @@ function dat.check(set, softlist)
 	if drvinfo then
 		info = info .. "\n\n--- DRIVER INFO ---\nDriver: " .. sourcefile .. "\n\n" .. drvinfo
 	end
-	info = info:gsub("\n\n", "\n")
 	return "Messinfo"
 end
 

plugins/data/load_dat.lua

@@ -1,95 +1,163 @@
+local sql = require("lsqlite3")
 local datfile = {}
+local db = sql.open(mame_manager:ui():options().entries.historypath:value():match("([^;]+)") .. "/history.db")
+if db then
+	local found = false
+	db:exec("select * from sqllite_master where name = version", function() found = true return 0 end)
+	if not found then
+		db:exec([[
+			CREATE TABLE version (
+				version VARCHAR NOT NULL,
+				datfile VARCHAR UNIQUE NOT NULL)]])
+	end
+end
 
-function datfile.open(file, vertag)
+function datfile.open(file, vertag, fixupcb)
-	local data = {}
+	if not db then
-	local ver
+		return nil
+	end
+	local function read(tag1, tag2, set)
+		local data
+		local stmt = db:prepare("SELECT f.data FROM \"" .. file .. "_idx\" AS fi, \"" .. file .. [["
+		 AS f WHERE fi.type = ? AND fi.val = ? AND fi.romset = ? AND f.rowid = fi.data]])
+		stmt:bind_values(tag1, tag2, set)
+		if stmt:step() == sql.ROW then
+			data = stmt:get_value(0)
+		end
+		stmt:finalize()
+		return data
+	end
+
+	local ver, dbver
 	local filepath
 	local fh
+
 	for path in mame_manager:ui():options().entries.historypath:value():gmatch("([^;]+)") do
 		filepath = lfs.env_replace(path) .. "/" .. file
-		fh = io.open(filepath, "rb")
+		fh = io.open(filepath, "r")
 		if fh then
 			break
 		end
+		return
 	end
-	if not fh then
+	-- remove unsafe chars from file for use in sql statement
+	file = file:gsub("[^%w%._]", "")
+
+	local stmt = db:prepare("SELECT version FROM version WHERE datfile = ?")
+	stmt:bind_values(file)
+	if stmt:step() == sql.ROW then
+		dbver = stmt:get_value(0)
+	end
+	stmt:finalize()
+	
+	if not dbver then
+		db:exec("CREATE TABLE \"" .. file .. [[_idx" (
+				type VARCHAR NOT NULL,
+				val VARCHAR NOT NULL,
+				romset VARCHAR NOT NULL,
+				data INTEGER NOT NULL)]])
+		db:exec("CREATE TABLE \"" .. file .. "\" (data CLOB NOT NULL)")
+		db:exec("CREATE INDEX typeval ON \"" .. file .. "_idx\"(type, val)")
+	elseif not fh then
+		-- data in database but missing file, just use what we have
+		return read, dbver
+	elseif not fh and not dbver then
 		return nil
 	end
-	do
+
-		local inblock = false
-		local buffer = fh:read("a")
 	if vertag then
-			local match = buffer:match(vertag .. "%s*([^%s]+)")
+		for line in fh:lines() do
+			local match = line:match(vertag .. "%s*([^%s]+)")
 			if match then
 				ver = match
+				break
 			end
 		end
+	else
+		-- use file ctime for version
+		ver = tostring(lfs.attributes(filepath, "change"))
+	end
+	if ver == dbver then
+		return read, dbver
+	end
+
+	if dbver then
+		db:exec("DELETE FROM \"" .. file .. "\"")
+		db:exec("DELETE FROM \"" .. file .. "_idx\"")
+		stmt = db:prepare("UPDATE version SET version = ? WHERE datfile = ?")
+	else
+		stmt = db:prepare("INSERT INTO version VALUES (?, ?)")
+	end
+	stmt:bind_values(ver, file)
+	stmt:step()
+	stmt:finalize()
+
+	do
+		local inblock = false
+		fh:seek("set")
+		local buffer = fh:read("a")
+		db:exec("BEGIN TRANSACTION")
 		local function gmatchpos()
 			local pos = 1
 			local function iter()
-				local spos, epos = buffer:find("\n$", pos, true)
+				local tag1, tag2, data, start, inblock = false
+				while not data do
+					local spos, epos, match = buffer:find("\n($[^\n]*)", pos)
 					if not spos then
 						return nil
 					end
-				spos = spos + 1
+					if match ~= "$end" and not inblock then
-				local spos, epos, match = buffer:find("([^\n]+)", spos)
+						if not tag1 then
-				pos = epos + 1
+							tag1 = match
-				return match, pos, iter
+						else
+							tag2 = match
+							start = epos + 1
+							inblock = true
+						end
+					elseif inblock == true then
+						data = buffer:sub(start, spos)
+						inblock = false
+					end
+					pos = epos
+				end
+				return tag1, tag2, data
 			end
 			return iter
 		end
-		for line, epos, iter in gmatchpos() do
+		for info1, info2, data in gmatchpos() do
-
+			local tag, set = info1:match("^%$([^%s=]+)=?([^%s]*)")
-			local flag = line:sub(1, 1)
-			if flag ~= "#" then
-				if flag == "$" then
-					if line:sub(1, 4) == "$end" then
-						inblock = false
-					elseif not inblock then
-						local tag, set = line:match("^%$([^%s=]+)=?([^%s]*)")
 			if set and set ~= "" then
 				local tags = {}
 				local sets = {}
-							local tag1 = ""
 				tag:gsub("([^,]+)", function(s) tags[#tags + 1] = s end)
 				set:gsub("([^,]+)", function(s) sets[#sets + 1] = s end)
-							repeat
+				if #tags > 0 and #sets > 0 then
-								tag1, epos = iter()
+					local tag1 = info2:match("^$([^%s]*)")
-							until tag1:sub(1, 1) == "$"
+					if fixupcb then
-							tag1 = tag1:match("^$([^%s]*)")
+						data = fixupcb(data)
-							if not data[tag1] then
-								data[tag1] = {}
 					end
+					stmt = db:prepare("INSERT INTO \"" .. file .. "\" VALUES (?)")
+					stmt:bind_values(data)
+					stmt:step()
+					local row = stmt:last_insert_rowid()
+					stmt:finalize()
 					for num1, tag2 in pairs(tags) do
-								if not data[tag1][tag2] then
-									data[tag1][tag2] = {}
-								end
 						for num2, set in pairs(sets) do
-									data[tag1][tag2][set] = epos
+							if fixupcb then
+								fixupcb(data)
 							end
-							end
+							stmt = db:prepare("INSERT INTO \"" .. file .. "_idx\" VALUES (?, ?, ?, ?)")
-						end
+							stmt:bind_values(tag1, tag2, set, row)
-						inblock = true
+							stmt:step()
+							stmt:finalize()
 						end
 					end
 				end
 			end
 		end
+		db:exec("END TRANSACTION")
+	end
 	fh:close()
-	fh = io.open(filepath, "r")
-	local function read(tag1, tag2, set)
-		local output = {}
-		if not data[tag1][tag2][set] then
-			return nil
-		end
-		fh:seek("set", data[tag1][tag2][set])
-		for line in fh:lines() do
-			if line:sub(1, 4) == "$end" then
-				return table.concat(output, "\n")
-			end
-			output[#output + 1] = line
-		end
-	end
 
 	return read, ver
 end