diff --git a/libs/calculate-deps.lua b/libs/calculate-deps.lua index 23346b9..f7961c9 100644 --- a/libs/calculate-deps.lua +++ b/libs/calculate-deps.lua @@ -16,81 +16,175 @@ limitations under the License. --]] -local normalize = require('semver').normalize local gte = require('semver').gte local log = require('log').log +local exec = require('exec') local queryDb = require('pkg').queryDb local colorize = require('pretty-print').colorize +local queryGit = require('pkg').queryGit +local normalize = require('semver').normalize -return function (db, deps, newDeps) - - local addDep, processDeps - - function processDeps(dependencies) - if not dependencies then return end - for alias, dep in pairs(dependencies) do - local name, version = dep:match("^([^@]+)@?(.*)$") - if #version == 0 then - version = nil - end - if type(alias) == "number" then - alias = name:match("([^/]+)$") - end - if not name:find("/") then - error("Package names must include owner/name at a minimum") - end - if version then - local ok - ok, version = pcall(normalize, version) - if not ok then - error("Invalid dependency version: " .. dep) - end - end - addDep(alias, name, version) +local processDeps +local db, deps, configs + +local GIT_SCHEMES = { + "^https?://", -- over http/s + "^ssh://", -- over ssh + "^git://", -- over git + "^ftps?://", -- over ftp/s + "^[^:]+:", -- over ssh +} + +local function isGit(dep) + for i = 1, #GIT_SCHEMES do + if dep:match(GIT_SCHEMES[i]) then + return true end end + return false +end + +local function resolveDep(alias, dep) + -- match for author/name@version + local name, version = dep:match("^([^@]+)@?(.*)$") + -- resolve alias name, in case it's a number (an array index) + if type(alias) == "number" then + alias = name:match("([^/]+)$") + end + + -- make sure owner is provided + if not name:find("/") then -- FIXME: this does match on `author/` or `/package` + error("Package names must include owner/name at a minimum") + end - function addDep(alias, name, version) - local meta = deps[alias] - if meta then - if name ~= meta.name then - local message = string.format("%s %s ~= %s", - alias, meta.name, name) - log("alias conflict", message, "failure") - return - end - if version then - if not gte(meta.version, version) then - local message = string.format("%s %s ~= %s", - alias, meta.version, version) - log("version conflict", message, "failure") - return - end - end + -- resolve version + if #version ~= 0 then + local ok + ok, version = pcall(normalize, version) + if not ok then + error("Invalid dependency version: " .. dep) + end + else + version = nil + end + + -- check for already installed packages + local meta = deps[alias] + if meta then + -- is there an alias conflict? + if name ~= meta.name then + local message = string.format("%s %s ~= %s", + alias, meta.name, name) + log("alias conflict", message, "failure") + -- is there a version conflict? + elseif version and not gte(meta.version, version) then + local message = string.format("%s %s ~= %s", + alias, meta.version, version) + log("version conflict", message, "failure") + -- re-process package dependencies if everything is ok else - local author, pname = name:match("^([^/]+)/(.*)$") - local match, hash = db.match(author, pname, version) - - if not match then - error("No such " - .. (version and "version" or "package") .. ": " - .. name - .. (version and '@' .. version or '')) - end - local kind - meta, kind, hash = assert(queryDb(db, hash)) - meta.db = db - meta.hash = hash - meta.kind = kind - deps[alias] = meta + processDeps(meta.dependencies) end + return + end - processDeps(meta.dependencies) + -- extract author and package names from "author/package" + -- and match against the local db for the resources + -- if not available locally, and an upstream is set, match the upstream db + local author, pname = name:match("^([^/]+)/(.*)$") + local match, hash = db.match(author, pname, version) + + -- no such package has been found locally nor upstream + if not match then + error("No such " + .. (version and "version" or "package") .. ": " + .. name + .. (version and '@' .. version or '')) + end + + -- query package metadata, and mark it for installation + local kind + meta, kind, hash = assert(queryDb(db, hash)) + meta.db = db + meta.hash = hash + meta.kind = kind + deps[alias] = meta + + -- handle the dependencies of the module + processDeps(meta.dependencies) +end +-- TODO: implement git protocol over https, to be used in case `git` cli isn't available +-- TODO: implement someway to specify a branch/tag when fetching +-- TODO: implement handling git submodules, or shall we not? +local function resolveGitDep(url) + -- fetch the repo tree, don't include any tags + log("fetching", colorize("highlight", url)) + local _, stderr, code = exec("git", "--git-dir=" .. configs.database, + "fetch", "--no-tags", "--depth=1", url) + + -- was the fetch successful? + if code ~= 0 then + if stderr:match("^ENOENT") then + error("Cannot find git. Please make sure git is installed and available.") + else + error((stderr:gsub("\n$", ""))) + end + end + + -- load the fetched module tree + local raw = db.storage.read("FETCH_HEAD") + local hash = raw:match("^(.-)\t\t.-\n$") + assert(hash and #hash ~= 0, "Unable to retrieve FETCH_HEAD\n" .. raw) + hash = db.loadAs("commit", hash).tree + + -- query module's metadata, and match author/name + local meta, kind + meta, kind, hash = queryGit(db, hash) + assert(meta, "Unable to find a valid package") + local author, name = meta.name:match("^([^/]+)/(.*)$") + + -- check for installed packages and their version + local oldMeta = deps[name] + if oldMeta and not gte(oldMeta.version, meta.version) then + local message = string.format("%s %s ~= %s", + name, oldMeta.version, meta.version) + log("version conflict", message, "failure") + return + end + + -- create a ref/tags/author/name/version pointing to module's tree + db.write(author, name, meta.version, hash) + + -- mark the dep for installation + meta.db = db + meta.hash = hash + meta.kind = kind + deps[name] = meta + + -- handle the dependencies of the module + processDeps(meta.dependencies) +end + +function processDeps(dependencies) + if not dependencies then return end + -- iterate through dependencies and resolve each entry + for alias, dep in pairs(dependencies) do + if isGit(dep) then + resolveGitDep(dep) + else + resolveDep(alias, dep) + end end +end +return function (core, depsMap, newDeps) + -- assign gitDb and depsMap as upvalues to be visible everywhere + -- then start processing newDeps + db, deps, configs = core.db, depsMap, core.config processDeps(newDeps) + -- collect all deps names and log them local names = {} for k in pairs(deps) do names[#names + 1] = k @@ -103,6 +197,5 @@ return function (db, deps, newDeps) colorize("highlight", name), meta.path or meta.version)) end - return deps end diff --git a/libs/core.lua b/libs/core.lua index 73ccba1..c8337eb 100644 --- a/libs/core.lua +++ b/libs/core.lua @@ -156,7 +156,7 @@ local function makeCore(config) end if meta.dependencies and kind == "tree" then local deps = {} - calculateDeps(core.db, deps, meta.dependencies) + calculateDeps(core, deps, meta.dependencies) meta.snapshot = installDeps(core.db, hash, deps, false) log("snapshot hash", meta.snapshot) end @@ -437,7 +437,7 @@ local function makeCore(config) local kind, hash = assert(import(core.db, zfs, source, rules, true)) assert(kind == "tree", "Only tree packages are supported for now") local deps = getInstalled(zfs, source) - calculateDeps(core.db, deps, meta.dependencies) + calculateDeps(core, deps, meta.dependencies) hash = installDeps(core.db, hash, deps, true) return makeZip(hash, target, luvi_source) end @@ -504,7 +504,7 @@ local function makeCore(config) end local deps = {} - calculateDeps(core.db, deps, meta.dependencies) + calculateDeps(core, deps, meta.dependencies) local tagObj = db.loadAs("tag", hash) if tagObj.type ~= "tree" then error("Only tags pointing to trees are currently supported for make") @@ -546,7 +546,7 @@ local function makeCore(config) function core.installList(path, newDeps) local deps = getInstalled(gfs, path) - calculateDeps(core.db, deps, newDeps) + calculateDeps(core, deps, newDeps) installDepsFs(core.db, gfs, path, deps, true) return deps end diff --git a/libs/pkg.lua b/libs/pkg.lua index 894586c..3e6d0ed 100644 --- a/libs/pkg.lua +++ b/libs/pkg.lua @@ -22,9 +22,10 @@ Package Metadata Commands These commands work with packages metadata. -pkg.query(fs, path) -> meta, path - Query an on-disk path for package info. -pkg.queryDb(db, path) -> meta, kind - Query an in-db hash for package info. -pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info +pkg.query(fs, path) -> meta, path - Query an on-disk path for package info. +pkg.queryDb(db, path) -> meta, kind, hash - Query an in-db hash for package info. +plg.queryGit(db, path) -> meta, kind, hash - Query an in-db hash fetched with `git fetch` for package info. +pky.normalize(meta) -> author, tag, version - Extract and normalize pkg info ]] local isFile = require('git').modes.isFile @@ -174,6 +175,46 @@ local function queryDb(db, hash) return meta, kind, hash end +local function queryGit(db, hash) + local method = db.offlineLoadAny or db.load -- is rdb loaded? + local kind, value = method(hash) + if not kind then + error("Attempt to load the fetched tree") + elseif kind ~= "tree" then + error("Illegal kind: " .. kind) + end + + local tree = listToMap(value) + local path = "tree:" .. hash + local entry = tree["package.lua"] + if entry then + path = path .. "/package.lua" + elseif tree["init.lua"] then + entry = tree["init.lua"] + path = path .. "/init.lua" + else + -- check if the tree only contains a single lua file, and treat it as a package. + -- since in most git hosting services you won't have blob-pointing tag, + -- this has to make some assumption (or otherwise not support it) + -- in this case, it makes the assumption that a single-file package's repo + -- only has a single lua file + for name, meta in pairs(tree) do + if name:sub(-4) == ".lua" and isFile(meta.mode) then + if entry then -- it contains more than a single lua file + return nil, "ENOENT: No package.lua or init.lua in tree:" .. hash + end + entry = tree[name] + path = "blob:" .. entry.hash + kind = "blob" + hash = entry.hash + end + end + end + + local meta = evalModule(db.loadAs("blob", entry.hash), path) + return meta, kind, hash +end + local function normalize(meta) local author, tag = meta.name:match("^([^/]+)/(.*)$") return author, tag, semver.normalize(meta.version) @@ -183,5 +224,6 @@ end return { query = query, queryDb = queryDb, + queryGit = queryGit, normalize = normalize, } diff --git a/libs/rdb.lua b/libs/rdb.lua index d390e24..40c2215 100644 --- a/libs/rdb.lua +++ b/libs/rdb.lua @@ -24,7 +24,8 @@ local httpCodec = require('http-codec') local websocketCodec = require('websocket-codec') local makeRemote = require('codec').makeRemote local deframe = require('git').deframe -local decodeTag = require('git').decoders.tag +local decoders = require('git').decoders +local decodeTag = decoders.tag local verifySignature = require('verify-signature') local function connectRemote(url, timeout) @@ -141,6 +142,12 @@ return function(db, url, timeout) return assert(db.offlineLoad(hash)) end + function db.offlineLoadAny(hash) + local raw = assert(db.offlineLoad(hash), "no such hash") + local kind, value = deframe(raw) + return kind, decoders[kind](value) + end + function db.fetch(list) local refs = {} repeat