Commit a9fce5b2 authored by Ricardo Cosme's avatar Ricardo Cosme

Merge branch 'OPENBUS-2970' into '02_00_01'

[OPENBUS-2970] Migração da base de dados do barramento para SQLite3

+ Suporte ao SQLite3 através do LuaSQLite3 (módulo database.lua)
 - Suporte ao formato anterior (módulo database_legacy.lua)
 - Módulo responsável por converter um banco no formato anterior
   em uma base SQLite (módulo database_converter.lua)

See merge request !1
parents 2ef870ff ec03019c
......@@ -84,6 +84,13 @@ if ! $(luascs)
}
use-project luascs : $(luascs)/bbuild ;
local luasqlite3 = [ os.environ LUASQLITE3 ] ;
if ! $(luasqlite3)
{
luasqlite3 = "$(deps)/luasqlite3" ;
}
use-project luasqlite3 : $(luasqlite3)/bbuild ;
scs-idl = [ os.environ SCS_IDL ] ;
if ! $(scs-idl)
{
......@@ -141,7 +148,9 @@ make luaopenbus.c
$(root)/lua/openbus/idl.lua
$(root)/lua/openbus/util/argcheck.lua
$(root)/lua/openbus/util/autotable.lua
$(root)/lua/openbus/util/database_converter.lua
$(root)/lua/openbus/util/database.lua
$(root)/lua/openbus/util/database_legacy.lua
$(root)/lua/openbus/util/except.lua
$(root)/lua/openbus/util/logger.lua
$(root)/lua/openbus/util/messages.lua
......@@ -211,6 +220,8 @@ local common-requirements =
<library>/oil//oil
<library>/oil//luaidl
<library>/luascs//luascs
<library>/sqlite//sqlite3
<library>/luasqlite3//lsqlite3
<dependency>/lce//lce
<dependency>/loop//loop
<dependency>/loop//luacothread
......@@ -221,6 +232,8 @@ local common-requirements =
<dependency>/luasocket//luasocket
<dependency>/luastruct//luastruct
<dependency>/luavararg//luavararg
<dependency>/sqlite//sqlite3
<dependency>/luasqlite3//lsqlite3
;
lib luaopenbus
......
local _G = require "_G"
local assert = _G.assert
local ipairs = _G.ipairs
local loadfile = _G.loadfile
local pairs = _G.pairs
local pcall = _G.pcall
local select = _G.select
local setmetatable = _G.setmetatable
local os = require "os"
local removefile = os.remove
local renamefile = os.rename
local tmpname = os.tmpname
local io = require "io"
local open = io.open
local lsqlite = require "lsqlite3"
local lfs = require "lfs"
local listdir = lfs.dir
local getattribute = lfs.attributes
local makedir = lfs.mkdir
local removedir = lfs.rmdir
local Viewer = require "loop.debug.Viewer"
local log = require "openbus.util.logger"
local oo = require "openbus.util.oo"
local class = oo.class
local DataBase = class()
local Serializer = Viewer{ nolabels = true }
local function createpath(path)
local result, errmsg = getattribute(path, "mode")
if result == "directory" then
result, errmsg = true, nil
elseif result ~= nil then
result, errmsg = false, "'"..path..
"' expected to be directory (got "..result..")"
elseif errmsg:match("^cannot obtain information") then
result, errmsg = makedir(path)
if not result then
errmsg = "unable to create directory '"..path.."' ("..errmsg..")"
end
end
return result, errmsg
end
local function removepath(path)
for filename in listdir(path) do
local filepath = path..filename
if filename:find("%.lua$")
and getattribute(filepath, "mode") == "file" then
local ok, errmsg = removefile(filepath)
if ok == nil then
return nil, "unable to remove file '"..filepath.."' ("..errmsg..")"
end
local SQL_create_tables = [[
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS category (
id TEXT PRIMARY KEY,
name TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS certificate (
entity TEXT PRIMARY KEY,
certificate BLOB NOT NULL
);
CREATE TABLE IF NOT EXISTS entity (
id TEXT PRIMARY KEY,
name TEXT,
category TEXT NOT NULL
REFERENCES category(id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS login (
id TEXT PRIMARY KEY,
entity TEXT NOT NULL,
encodedkey BLOB NOT NULL,
allowLegacyDelegate INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS loginObserver (
id TEXT PRIMARY KEY,
ior TEXT NOT NULL,
legacy INTEGER,
login TEXT NOT NULL
REFERENCES login(id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS watchedLogin (
login_observer TEXT NOT NULL
REFERENCES loginObserver(id)
ON DELETE CASCADE,
login TEXT NOT NULL
REFERENCES login(id)
ON DELETE CASCADE,
CONSTRAINT pkey PRIMARY KEY (
login_observer,
login)
);
CREATE TABLE IF NOT EXISTS interface (
repid TEXT PRIMARY KEY
);
CREATE TABLE IF NOT EXISTS entityInterface (
entity TEXT
REFERENCES entity(id)
ON DELETE CASCADE,
interface TEXT
REFERENCES interface(repid)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS facet (
name TEXT,
interface_name TEXT,
offer TEXT
REFERENCES offer(id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS propertyOffer (
name TEXT NOT NULL,
value TEXT NOT NULL,
offer TEXT NOT NULL
REFERENCES offer(id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS propertyOfferRegistryObserver (
name TEXT NOT NULL,
value TEXT NOT NULL,
offer_registry_observer TEXT NOT NULL
REFERENCES offerRegistryObserver(id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS offer (
id TEXT PRIMARY KEY,
service_ref TEXT NOT NULL,
entity TEXT NOT NULL,
login TEXT NOT NULL,
timestamp TEXT NOT NULL,
day TEXT NOT NULL,
month TEXT NOT NULL,
year TEXT NOT NULL,
hour TEXT NOT NULL,
minute TEXT NOT NULL,
second TEXT NOT NULL,
component_name TEXT NOT NULL,
component_major_version TEXT NOT NULL,
component_minor_version TEXT NOT NULL,
component_patch_version TEXT NOT NULL,
component_platform_spec TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS offerObserver (
id TEXT PRIMARY KEY,
login TEXT NOT NULL,
observer TEXT NOT NULL,
offer TEXT NOT NULL
REFERENCES offer(id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS offerRegistryObserver (
id TEXT PRIMARY KEY,
login TEXT NOT NULL,
observer TEXT NOT NULL
);
]]
local actions = {
-- INSERT
{ name = "addCategory",
values = { "id", "name" } },
{ name="addEntity",
values = { "id", "name", "category" } },
{ name="addInterface",
values = { "repid" } },
{ name="addEntityInterface",
values = { "entity", "interface" } },
{ name="addOffer",
values = { "id", "service_ref", "entity", "login", "timestamp",
"day", "month", "year", "hour", "minute",
"second", "component_name", "component_major_version",
"component_minor_version", "component_patch_version",
"component_platform_spec" } },
{ name="addPropertyOffer",
values = { "name", "value", "offer" } },
{ name="addPropertyOfferRegistryObserver",
values = { "name", "value", "offer_registry_observer" } },
{ name="addFacet",
values = { "name", "interface_name", "offer" } },
{ name="addOfferObserver",
values = { "id", "login", "observer", "offer" } },
{ name="addOfferRegistryObserver",
values = { "id", "login", "observer" } },
{ name="addSettings",
values = { "key", "value" } },
{ name="addLogin",
values = { "id", "entity", "encodedKey", "allowLegacyDelegate" } },
{ name="addLoginObserver",
values = { "id", "ior", "legacy", "login" } },
{ name="addWatchedLogin",
values = { "login_observer", "login" } },
{ name="addCertificate",
values = { "certificate", "entity" } },
-- DELETE
{ name="delCategory",
where = { "id" } },
{ name="delEntity",
where = { "id" } },
{ name="delEntityInterface",
where = { "entity", "interface" } },
{ name="delInterface",
where = { "repid" } },
{ name="delOffer",
where = { "id" } },
{ name="delPropertyOffer",
where = { "offer" } },
{ name="delOfferObserver",
where = { "id" } },
{ name="delOfferRegistryObserver",
where = { "id" } },
{ name="delLogin",
where = { "id" } },
{ name="delLoginObserver",
where = { "id" } },
{ name="delWatchedLogin",
where = { "login_observer", "login" } },
{ name="delCertificate",
where = { "entity" } },
-- UPDATE
{ name="setCategory",
set="name",
where="id" },
{ name="setEntity",
set="name",
where="id" },
{ name="setCertificate",
set="certificate",
where="entity" },
-- SELECT
{ name="getCategory",
select = { "id", "name" } },
{ name="getCertificate",
select = { "entity, certificate" } },
{ name="getEntity",
select = { "id", "name", "category" },
from = { "entity" } },
{ name="getEntityById",
select = { "id" },
from = { "entity" },
where = { "id" } },
{ name="getEntityWithCerts",
select = { "entity" },
from = { "certificate" } },
{ name="getInterface",
select = { "repid" },
from = { "interface" } },
{ name="getAuthorizedInterface",
select = { "interface.repid" },
from = { "entityInterface", "interface" },
where_hc = { "interface.repid = entityInterface.interface" },
where = { "entityInterface.entity" } },
{ name="getOffer",
select = { "*" },
from = { "offer" } },
{ name="getPropertyOffer",
select = { "*" },
from = { "propertyOffer" },
where = { "offer" } },
{ name="getFacet",
select = { "*" },
from = { "facet" },
where = { "offer" } },
{ name="getOfferObserver",
select = { "*" },
from = { "offerObserver" },
where = { "offer" } },
{ name="getOfferRegistryObserver",
select = { "*" },
from = { "offerRegistryObserver" },
},
{ name="getSettings",
select = { "value" },
from = { "settings" },
where = { "key" }
},
{ name="getLogin",
select = { "*" },
from = { "login" }
},
{ name="getLoginObserver",
select = { "*" },
from = { "loginObserver" }
},
{ name="getWatchedLoginByObserver",
select = { "login" },
from = { "watchedLogin" },
where = { "login_observer" },
},
{ name="getAuthorizedInterfaces",
select = { "interface.repid" },
from = { "entityInterface", "interface" },
where_hc = { "interface.repid = entityInterface.interface" },
where = { "entityInterface.entity" },
},
}
local emsgprefix = "SQLite error with code="
local function herror(code, extmsg)
if code and code ~= lsqlite.OK then
local msg = emsgprefix..tostring(code)
if extmsg then
msg = msg.."; "..extmsg
end
end
local ok, errmsg = removedir(path)
if ok == nil then
return nil, "unable to remove directory '"..path.."' ("..errmsg..")"
return nil, msg
end
return true
end
local function loadfrom_cont(path, ok, ...)
if ok then return ... end
local errmsg = ...
return nil, "corrupted file '"..path.."' ("..errmsg..")"
end
local function loadfrom(path)
local result, errmsg = loadfile(path, "t", {})
if result == nil then
if errmsg:find("No such file or directory") then
return
end
return nil, "unable to load file '"..path.."' ("..errmsg..")"
end
return loadfrom_cont(path, pcall(result))
local function gsubSQL(sql)
return "SQL: [["..string.gsub(sql, '%s+', ' ').."]]"
end
local function saveto(path, ...)
local temp = path..".tmp" -- must be in the same path
-- of the final file because
-- 'os.rename' can only
-- rename files in the same
-- file system.
local result, errmsg = open(temp, "w")
if result == nil then
errmsg = "unable to create temporary file '"..temp.."' ("..errmsg..")"
else
local file = result
result, errmsg = file:write("return ", Serializer:tostring(...))
file:close()
if result == nil then
errmsg = "unable to write temporary file '"..temp.."' ("..errmsg..")"
else
local code
result, errmsg, code = renamefile(temp, path)
if result == nil then
if errmsg:find("File exists", 1, true) and code == 17 then
---[[
assert(removefile(path))
--[=[--]]
result, errmsg = assert(renamefile(path, path .. ".delete.me"))
assert(removefile(path .. ".delete.me"))
--]=]
result, errmsg = renamefile(temp, path)
else
errmsg = "unable to replace file '"..path.."' (with file "..errmsg..")"
end
end
end
removefile(temp)
local function iClause(sql, clause, entries, sep, suf)
if not entries then
return sql
end
return result, errmsg
end
local function closeobject(obj)
obj.path = nil
setmetatable(obj, nil)
end
local Table = class()
function Table:__init()
function self.iterator(next)
local file = next()
while file ~= nil do
local path = self.path..file
if getattribute(path, "mode") == "file" then
return file:match("^(.+)%.lua$"), assert(loadfrom(path))
end
file = next()
end
if sql then
sql = sql.." "
else
sql = ""
end
end
function Table:getentry(key)
return loadfrom(self.path..key..".lua")
end
function Table:setentry(key, ...)
return saveto(self.path..key..".lua", ...)
end
function Table:setentryfield(key, field, ...)
local path = self.path..key..".lua"
local result, errmsg = loadfrom(path)
if result == nil then
if errmsg ~= nil then
return result, errmsg
end
result = {}
if not string.find(sql, clause) then
sql = sql..clause.." "
else
sql = sql.." AND "
end
local count = select("#", ...)
local value = select(count, ...)
local place = result
for i = 1, count-1 do
local value = place[field]
if value == nil then
value = {}
place[field] = value
end
place = value
field = select(i, ...)
for i, col in ipairs(entries) do
if i > 1 then sql = sql..sep.." " end
sql = sql..col
if suf then sql = sql.." "..suf.." " end
end
place[field] = value
result, errmsg = saveto(path, result)
return result, errmsg
return sql
end
function Table:removeentry(key)
local path = self.path..key..".lua"
local ok, errmsg = removefile(path)
if ok == nil then
return nil, "unable to remove file '"..path.."' ("..errmsg..")"
local function buildSQL(action)
local name = action.name
local verb = string.sub(name, 1, 3)
local sql
local stable = string.lower(string.sub(name, 4, 4))
..string.sub(name, 5, -1)
if "add" == verb then
sql = "INSERT INTO "..stable.." ("
local values = action.values
sql = sql..table.concat(values, ",")
sql = sql..") VALUES ("
sql = sql..string.rep("?", #values, ",")
sql = sql..")"
elseif "del" == verb then
sql = "DELETE FROM "..stable.." "
sql = iClause(sql, "WHERE", action.where, "AND", "= ?")
elseif "set" == verb then
local stable = action.table or stable
sql = "UPDATE "..stable.. " "
sql = sql.."SET "..action.set.. " = ? "
sql = sql.."WHERE "..action.where.." = ?"
elseif "get" == verb then
sql = iClause(nil, "SELECT", action.select, ",")
local from = action.from
if from then
sql = iClause(sql, "FROM", action.from, ",")
else
sql = sql.." FROM "..stable
end
sql = iClause(sql, "WHERE", action.where, "AND", "= ?")
sql = iClause(sql, "WHERE", action.where_hc, "AND")
end
return true
return sql
end
function Table:ientries()
return self.iterator, listdir(self.path)
function DataBase:aexec(sql)
local gsql = gsubSQL(sql)
log:database(sql)
assert(herror(self.conn:exec(sql), gsql))
end
function Table:remove()
local result, errmsg = removepath(self.path)
if result then
self.base.tables[self.name] = nil
self.base, self.name = nil, nil
closeobject(self)
end
return result, errmsg
end
local TableNamePat = "[^.][^/\\?*]*"
local DataBase = class()
local stmts = {}
function DataBase:__init()
local path = self.path
local tables = {}
for name in listdir(path) do
local tablepath = path..name.."/"
if name:match(TableNamePat)
and getattribute(tablepath, "mode") == "directory" then
tables[name] = Table{ base = self, name = name, path = tablepath }
local conn = self.conn
self:aexec("BEGIN;")
self:aexec(SQL_create_tables)
self:aexec("PRAGMA foreign_keys=ON;")
local pstmts = {}
for _, action in ipairs(actions) do
local sql = buildSQL(action)
local res, errcode = conn:prepare(sql)
if not res then
assert(herror(errcode, gsubSQL(sql)))
end
local key = action.name
pstmts[key] = res
stmts[key] = sql
end
self.tables = tables
self:aexec("COMMIT;")
self.pstmts = pstmts
end
function DataBase:gettable(name)
local table = self.tables[name]
if table == nil then
local path = self.path..name
local result, errmsg = createpath(path)
if not result then
return nil, errmsg
end
table = Table{ base = self, name = name, path = path.."/" }
self.tables[name] = table
function DataBase:exec(stmt)
local gsql = gsubSQL(stmt)
log:database(stmt)
local res, errmsg = herror(self.conn:exec(stmt))
errmsg = gsql.." "..emsgprefix..tostring(errcode)
if errcode == lsqlite.DONE then
return true
elseif errcode == lsqlite.ERROR then
return nil, errmsg.."; "..self.conn:errmsg()
end
return table
end
function DataBase:itables()
return pairs(self.tables)
return nil, errmsg
end
function DataBase:close()
for name, table in pairs(self.tables) do
closeobject(table)
function DataBase:pexec(action, ...)
local pstmt = self.pstmts[action]
local sql = gsubSQL(stmts[action]).." "
local res, errmsg = herror(pstmt:bind_values(...))
if not res then
return nil, sql..errmsg
end
closeobject(self)
return true
local errcode = pstmt:step()
log:database(sql.." with values {"
..table.concat({...}, ", ").."}")
errmsg = sql..emsgprefix..tostring(errcode)
if errcode == lsqlite.DONE then
pstmt:reset()
return true, errcode
elseif errcode == lsqlite.ROW then
return true, errcode
elseif errcode == lsqlite.ERROR then
errmsg = errmsg.."; "..self.conn:errmsg()
end
return nil, errmsg
end
local module = {}
function module.open(path)
local result, errmsg = createpath(path)
if not result then return nil, errmsg end
return DataBase{ path = path.."/" }
local conn, errcode, errmsg = lsqlite.open(path)
if not conn then return herror(errcode, errmsg) end
return DataBase{ conn = conn }
end
return module
local db = require "openbus.util.database"
local dblegacy = require "openbus.util.database_legacy"
local module = {}
local function bool2int(val)
return (val and 1) or 0
end
function module.convert(dblegacy, db)
db.conn:exec("BEGIN;")
local certificateDB = dblegacy:gettable("Certificates")
for entity in certificateDB:ientries() do
local certificate = certificateDB:getentry(entity)
assert(db:pexec("addCertificate", certificate, entity))
end
local autosets = dblegacy:gettable("AutoSetttings")
local busid = autosets:getentry("BusId")
if busid then
assert(db:pexec("addSettings", "BusId", busid))
end
local loginsDB = dblegacy:gettable("Logins")
for id, data in loginsDB:ientries() do
assert(db:pexec("addLogin", id, data.entity, data.encodedkey,
bool2int(data.allowLegacyDelegate)))
end
local loginobsDB = dblegacy:gettable("LoginObservers")
for id, data in loginobsDB:ientries() do
assert(db:pexec("addLoginObserver", id, data.ior, bool2int(data.legacy),
data.login))
for login in pairs(data.watched) do
assert(db:pexec("addWatchedLogin", id, login))
end
end