From d803f79406e347c715418f55a41a82324a528b1b Mon Sep 17 00:00:00 2001 From: Drew Short Date: Sat, 13 Jan 2018 03:22:02 -0600 Subject: [PATCH] Massive cleanup of SPM * Added bootstrap.lua to download and install SPM on a fresh machine * Added support for downloading minified sources for packages --- .gitlab-ci.yml | 7 +- gruntfile.js | 16 +- src/bootstrap.lua | 35 +++ src/programs.cfg | 3 +- src/repos.cfg | 5 +- src/spm/spm.lua | 398 +++++++++++++++++++--------------- src/warricksothr/programs.cfg | 16 ++ 7 files changed, 297 insertions(+), 183 deletions(-) create mode 100644 src/bootstrap.lua create mode 100644 src/warricksothr/programs.cfg diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b533082..a5f6227 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,9 @@ compile: artifacts: expire_in: 6 hours paths: - - dist/**/*.lua - dist/*.cfg + - dist/**/*.cfg + - dist/**/*.lua only: - master - develop @@ -41,6 +42,7 @@ development: ENVIRONMENT: "Development" TARGET_DIRECTORY: "~/download_dir/oc/development" DOWNLOAD_URL_BASE: "https://sothr.com/download/oc/development" + MINIFIED_SOURCE: "false" script: - echo "Deployment To Development Starting..." - eval `ssh-agent -s` @@ -48,6 +50,7 @@ development: - pushd ./dist - find * -type f -exec sed -i -e 's@{ENV}@'"$ENVIRONMENT"'@g' {} \; - find * -type f -exec sed -i -e 's@{URL_BASE}@'"$DOWNLOAD_URL_BASE"'@g' {} \; + - find * -type f -exec sed -i -e 's@{MINIFIED}@'"MINIFIED_SOURCE"'@g' {} \; - popd - rsync -avz --delete-delay -e "ssh -p $DEPLOY_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress dist/* $DEPLOY_USER@$DEPLOY_HOST:$TARGET_DIRECTORY only: @@ -66,6 +69,7 @@ production: ENVIRONMENT: "Release" TARGET_DIRECTORY: "~/download_dir/oc/release" DOWNLOAD_URL_BASE: "https://sothr.com/download/oc/release" + MINIFIED_SOURCE: "true" script: - echo "Deployment To Production Starting..." - eval `ssh-agent -s` @@ -73,6 +77,7 @@ production: - pushd ./dist - find * -type f -exec sed -i -e 's@{ENV}@'"$ENVIRONMENT"'@g' {} \; - find * -type f -exec sed -i -e 's@{URL_BASE}@'"$DOWNLOAD_URL_BASE"'@g' {} \; + - find * -type f -exec sed -i -e 's@{MINIFIED}@'"MINIFIED_SOURCE"'@g' {} \; - popd - rsync -avz --delete-delay -e "ssh -p $DEPLOY_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --progress dist/* $DEPLOY_USER@$DEPLOY_HOST:$TARGET_DIRECTORY only: diff --git a/gruntfile.js b/gruntfile.js index 6916298..159669a 100644 --- a/gruntfile.js +++ b/gruntfile.js @@ -24,7 +24,7 @@ module.exports = function (grunt) { const fs = require('fs'); const luamin = require('luamin'); const util = require('util'); - grunt.file.expand({filter: 'isDirectory'}, "dist/**/*").forEach(function (dir) { + grunt.file.expand({filter: 'isDirectory'}, "dist/**").forEach(function (dir) { grunt.file.expand({filter: 'isFile'}, dir + "/*.lua").forEach(function (file) { const directory = file.substr(0, file.lastIndexOf('/')); const file_name = file.substr(file.lastIndexOf('/') + 1); @@ -45,9 +45,21 @@ module.exports = function (grunt) { }); }); + grunt.registerTask("build-spm-bootstrap", "Make sure a single bootstrap file exists for SPM", function () { + const fs = require('fs'); + grunt.file.expand({filter: 'isFile'}, "dist/**/bootstrap.min.lua").forEach(function (minified_file) { + const directory = minified_file.substr(0, minified_file.lastIndexOf('/')); + const minified_contents = fs.readFileSync(minified_file, 'utf8'); + fs.writeFileSync(directory + '/bootstrap.lua', minified_contents); + fs.unlinkSync(minified_file); + console.log("Removed unminified bootstrap file: " + directory + '/bootstrap.lua') + }); + }); + grunt.registerTask("default", [ "clean", "copy", - "minify-lua-modules" + "minify-lua-modules", + "build-spm-bootstrap" ]); }; diff --git a/src/bootstrap.lua b/src/bootstrap.lua new file mode 100644 index 0000000..bb08489 --- /dev/null +++ b/src/bootstrap.lua @@ -0,0 +1,35 @@ +--[[ + SPM Bootstrap Installer + Minimal functionality to install pull SPM and install itself. + ]] + +local component = require("component") + +local wget = loadfile("/bin/wget.lua") + +if not component.isAvailable("internet") then + io.stderr:write("This program requires an internet card to run.") + return +end + +local downloadSrc + +if "{MINIFIED}" == "true" then + downloadSrc = "{URL_BASE}/spm/spm.min.lua" +else + downloadSrc = "{URL_BASE}/spm/spm.lua" +end + +local saveLocation = "/tmp/spm.lua" + +term.write("Downloading Temporary SPM\n") +wget("-q", downloadSrc, saveLocation) + +local spm = require(saveLocation) + +term.write("Installing SPM Locally\n") +spm("install", "spm") + +term.write("Removing Temporary SPM\n") +fs.remove(saveLocation) +term.write("Done\n") \ No newline at end of file diff --git a/src/programs.cfg b/src/programs.cfg index 601ece6..14364a5 100644 --- a/src/programs.cfg +++ b/src/programs.cfg @@ -10,6 +10,7 @@ note = "SPM operates similarly to OPPM, with slight modifications to referential repositories", hidden = false, repo = "", - version = "0.2" + version = "0.2.1", + minified = {MINIFIED} } } \ No newline at end of file diff --git a/src/repos.cfg b/src/repos.cfg index a577435..053b276 100644 --- a/src/repos.cfg +++ b/src/repos.cfg @@ -1,5 +1,8 @@ { - ["Sothr's Software"] = { + ["Sothr Package Manager"] = { src = "{URL_BASE}" + }, + ["Warrick's Software"] = { + src = "{URL_BASE}/warricksothr" } } \ No newline at end of file diff --git a/src/spm/spm.lua b/src/spm/spm.lua index deb8319..5405cf3 100644 --- a/src/spm/spm.lua +++ b/src/spm/spm.lua @@ -21,8 +21,10 @@ if not component.isAvailable("internet") then end local internet = require("internet") ---local repoLocationConfig="https://raw.githubusercontent.com/OpenPrograms/openprograms.github.io/master/repos.cfg" -local repoLocationConfig = "https://sothr.com/download/oc/release/repos.cfg" +local githubContentRoot = "https://raw.githubusercontent.com/" +local repoLocationConfig = "{URL_BASE}/repos.cfg" +local localInstalledPackagesFile = "/etc/spdata.svd" +local localConfigFile = "/etc/spm.cfg" local args, options = shell.parse(...) @@ -34,48 +36,54 @@ local function printUsage() print("'spm list [-i] ' to get a list of available packages containing the specified substring") print(" -i: Only list already installed packages") print("'spm info ' to get further information about a program package") - print("'spm install [-f] [path]' to download a package to a directory on your system (or /usr by default)") + print("'spm install [-f] [-m] [path]' to download a package to a directory on your system (or /usr by default)") print("'spm update ' to update an already installed package") print("'spm update all' to update every already installed package") print("'spm uninstall ' to remove a package from your system") print(" -f: Force creation of directories and overwriting of existing files.") end +-- Make an internet http/s request for data local function getContent(url) local sContent = "" local result, response = pcall(internet.request, url) + if not result then return nil end + for chunk in response do sContent = sContent .. chunk end + return sContent end +-- Request repository information local function getRepos() local success, sRepos = pcall(getContent, repoLocationConfig) + if not success then io.stderr:write("Could not connect to the Internet. Please ensure you have an Internet connection.") return -1 end + return serial.unserialize(sRepos) end +-- Get Packages for src local function getPackages(src) local success, sPackages = pcall(getContent, src .. "/programs.cfg") + if not success or not sPackages then return -1 end + return serial.unserialize(sPackages) end local function getGithubPackages(repo) - local success, sPackages = pcall(getContent, "https://raw.githubusercontent.com/" .. repo .. "/master/programs.cfg") - if not success or not sPackages then - return -1 - end - return serial.unserialize(sPackages) + return getPackages(githubContentRoot .. repo .. "/master") end --For sorting table values by alphabet @@ -85,6 +93,7 @@ local function compare(a, b) return a:sub(i, i) < b:sub(i, i) end end + return #a < #b end @@ -96,127 +105,134 @@ local function downloadFile(url, path, force) end end -local function readFromFile(fNum) - local path - if fNum == 1 then - path = "/etc/spdata.svd" - elseif fNum == 2 then - path = "/etc/spm.cfg" - if not fs.exists(path) then - local tProcess = process.running() - path = fs.concat(fs.path(shell.resolve(tProcess)), "/etc/spm.cfg") - end - end +local function readFromFile(path) if not fs.exists(fs.path(path)) then fs.makeDirectory(fs.path(path)) end + if not fs.exists(path) then return { -1 } end + local file, msg = io.open(path, "rb") if not file then io.stderr:write("Error while trying to read file at " .. path .. ": " .. msg) return end - local sPacks = file:read("*a") + + local serializedTableData = file:read("*a") file:close() - return serial.unserialize(sPacks) or { -1 } + return serial.unserialize(serializedTableData) or { -1 } +end + +local function readFromLocalInstallFile() + readFromFile(localInstalledPackagesFile) +end + +local function readLocalConfigFile() + local path = localConfigFile + if not fs.exists(path) then + return { -1 } + else + return readFromFile(path) + end end -local function saveToFile(tPacks) - local file, msg = io.open("/etc/spdata.svd", "wb") +local function saveToInstalledPackagesFile(installedPackages) + + local file, msg = io.open(localInstalledPackagesFile, "wb") + if not file then io.stderr:write("Error while trying to save package names: " .. msg) return end - local sPacks = serial.serialize(tPacks) + + local sPacks = serial.serialize(installedPackages) file:write(sPacks) file:close() end local function listPackages(filter) filter = filter or false + if filter then filter = string.lower(filter) end - local packages = {} + + local packageTable = {} print("Receiving Package list...") if not options.i then - local success, repos = pcall(getRepos) - if not success or repos == -1 then + local success, repositories = pcall(getRepos) + + if not success or repositories == -1 then io.stderr:write("Unable to connect to the Internet.\n") return - elseif repos == nil then + elseif repositories == nil then print("Error while trying to receive repository list") return end - for _, j in pairs(repos) do - local lPacks + + for _, repository in pairs(repositories) do + local packages local target - if j.src then - print("Checking Repository " .. j.src) - lPacks = getPackages(j.src) - target = j.src - elseif j.repo then - print("Checking Repository " .. j.repo) - lPacks = getGithubPackages(j.repo) - target = j.repo + + if repository.src then + print("Checking Repository " .. repository.src) + packages = getPackages(repository.src) + target = repository.src + elseif repository.repo then + print("Checking Repository " .. repository.repo) + packages = getGithubPackages(repository.repo) + target = repository.repo end - if lPacks == nil then + + if packages == nil then io.stderr:write("Error while trying to receive package list for " .. target .. "\n") return - elseif type(lPacks) == "table" then - for k in pairs(lPacks) do - if not k.hidden then - table.insert(packages, k) - end - end - end - end - --[[ - local lRepos = readFromFile(2) - if lRepos then - for _, j in pairs(lRepos.repos) do - for k in pairs(j) do - if not k.hidden then - table.insert(packages, k) + elseif type(packages) == "table" then + for package in pairs(packages) do + if not package.hidden then + table.insert(packageTable, package) end end end end - ]] else - local lPacks = {} - local tPacks = readFromFile(1) - for i in pairs(tPacks) do - table.insert(lPacks, i) + local localPackageTable = {} + local localPackages = readFromLocalInstallFile() + for localPackage in pairs(localPackages) do + table.insert(localPackageTable, localPackage) end - packages = lPacks + packageTable = localPackageTable end + if filter then - local lPacks = {} - for i, j in ipairs(packages) do - if (#j >= #filter) and string.find(j, filter, 1, true) ~= nil then - table.insert(lPacks, j) + local packages = {} + for _, package in ipairs(packageTable) do + if (#package >= #filter) and string.find(package, filter, 1, true) ~= nil then + table.insert(packages, package) end end - packages = lPacks + packageTable = packages end - table.sort(packages, compare) - return packages + + table.sort(packageTable, compare) + return packageTable end -local function printPackages(tPacks) - if tPacks == nil or not tPacks[1] then +local function printPackages(packages) + + if packages == nil or not packages[1] then print("No package matching specified filter found.") return end + term.clear() local xRes, yRes = gpu.getResolution() print("--SPM {ENV} Package list--") local xCur, yCur = term.getCursor() - for _, j in ipairs(tPacks) do - term.write(j .. "\n") + for _, package in ipairs(packages) do + term.write(package .. "\n") yCur = yCur + 1 if yCur > yRes - 1 then term.write("[Press any key to continue]") @@ -230,44 +246,36 @@ local function printPackages(tPacks) end end -local function getInformation(pack) - local success, repos = pcall(getRepos) - if not success or repos == -1 then +local function getInformation(requestedPackage) + local success, repositories = pcall(getRepos) + + if not success or repositories == -1 then io.stderr:write("Unable to connect to the Internet.\n") return end - for _, j in pairs(repos) do - local lPacks + + for _, repository in pairs(repositories) do + local packages local target - if j.src then - lPacks = getPackages(j.src) - target = j.src - elseif j.repo then - lPacks = getGithubPackages(j.repo) - target = "https://raw.githubusercontent.com/" .. j.repo + + if repository.src then + packages = getPackages(repository.src) + target = repository.src + elseif repository.repo then + packages = getGithubPackages(repository.repo) + target = githubContentRoot .. repository.repo end - if lPacks == nil then + + if packages == nil then io.stderr:write("Error while trying to receive package list for " .. target .. "\n") - elseif type(lPacks) == "table" then - for k in pairs(lPacks) do - if k == pack then - return lPacks[k], target + elseif type(packages) == "table" then + for package in pairs(packages) do + if package == requestedPackage then + return packages[package], target end end end end - --[[ - local lRepos = readFromFile(2) - if lRepos then - for i, j in pairs(lRepos.repos) do - for k in pairs(j) do - if k == pack then - return j[k], i - end - end - end - end - ]] return nil end @@ -276,196 +284,230 @@ local function provideInfo(pack) printUsage() return end + pack = string.lower(pack) local info = getInformation(pack) if not info then print("Package does not exist") return end + local done = false print("--Information about package '" .. pack .. "'--") if info.name then print("Name: " .. info.name) done = true end + + if info.version then + print("Version: " .. info.version) + done = true + end + if info.description then print("Description: " .. info.description) done = true end + if info.authors then print("Authors: " .. info.authors) done = true end + if info.note then print("Note: " .. info.note) done = true end + if not done then print("No information provided.") end end -local function installPackage(pack, path, update) - update = update or false - if not pack then +local function installPackage(requestedPackage, installPath, doUpdate) + doUpdate = doUpdate or false + + if not requestedPackage then printUsage() return end - if not path and not update then - local lConfig = readFromFile(2) - path = lConfig.path or "/usr" - print("Installing package to " .. path .. "...") - elseif not update then - path = shell.resolve(path) - print("Installing package to " .. path .. "...") + + if not installPath and not doUpdate then + local localConfig = readLocalConfigFile() + installPath = localConfig.path or "/usr" + print("Installing package to " .. installPath .. "...") + elseif not doUpdate then + installPath = shell.resolve(installPath) + print("Installing package to " .. installPath .. "...") end - pack = string.lower(pack) - local tPacks = readFromFile(1) - if not tPacks then + requestedPackage = string.lower(requestedPackage) + + local locallyInstalledPackages = readFromLocalInstallFile() + + if not locallyInstalledPackages then io.stderr:write("Error while trying to read local package names") return - elseif tPacks[1] == -1 then - table.remove(tPacks, 1) + elseif locallyInstalledPackages[1] == -1 then + table.remove(locallyInstalledPackages, 1) end - local info, repo = getInformation(pack) - if not info then + local packageInfo, repositoryPath = getInformation(requestedPackage) + if not packageInfo then print("Package does not exist") return end - if update then - print("Updating package " .. pack) - path = nil - for i, j in pairs(info.files) do - for k, v in pairs(tPacks[pack]) do - if k == i then - path = string.gsub(fs.path(v), j .. "/?$", "/") + + if doUpdate then + print("Updating package " .. requestedPackage) + installPath = nil + for filePath, fileInstallPath in pairs(packageInfo.files) do + for installedFilePath, fileLocalInstallPath in pairs(locallyInstalledPackages[requestedPackage]) do + if installedFilePath == filePath then + installPath = string.gsub(fs.path(fileLocalInstallPath), fileInstallPath .. "/?$", "/") break end end - if path then + if installPath then break end end - path = shell.resolve(string.gsub(path, "^/?", "/"), nil) + installPath = shell.resolve(string.gsub(installPath, "^/?", "/"), nil) end - if not update and fs.exists(path) then - if not fs.isDirectory(path) then + + if not doUpdate and fs.exists(installPath) then + if not fs.isDirectory(installPath) then if options.f then - path = fs.concat(fs.path(path), pack) - fs.makeDirectory(path) + installPath = fs.concat(fs.path(installPath), requestedPackage) + fs.makeDirectory(installPath) else print("Path points to a file, needs to be a directory.") return end end - elseif not update then + elseif not doUpdate then if options.f then - fs.makeDirectory(path) + fs.makeDirectory(installPath) else print("Directory does not exist.") return end end - if tPacks[pack] and (not update) then + + if locallyInstalledPackages[requestedPackage] and (not doUpdate) then print("Package has already been installed") return - elseif not tPacks[pack] and update then + elseif not locallyInstalledPackages[requestedPackage] and doUpdate then print("Package has not been installed.") print("If it has, uninstall it manually and reinstall it.") return end - if update then + + if doUpdate then term.write("Removing old files...") - for i, j in pairs(tPacks[pack]) do - fs.remove(j) + for installedFilePath, fileLocalInstallPath in pairs(locallyInstalledPackages[requestedPackage]) do + fs.remove(fileLocalInstallPath) end term.write("Done.\n") end - tPacks[pack] = {} + + locallyInstalledPackages[requestedPackage] = {} term.write("Installing Files...") - for i, j in pairs(info.files) do - local nPath - if string.find(j, "^//") then - local lPath = string.sub(j, 2) - if not fs.exists(lPath) then - fs.makeDirectory(lPath) + for filePath, requestedInstallPath in pairs(packageInfo.files) do + local installedPath + + if string.find(requestedInstallPath, "^//") then + local localInstalledPath = string.sub(requestedInstallPath, 2) + if not fs.exists(localInstalledPath) then + fs.makeDirectory(localInstalledPath) end - nPath = fs.concat(lPath, string.gsub(i, ".+(/.-)$", "%1"), nil) + installedPath = fs.concat(localInstalledPath, string.gsub(filePath, ".+(/.-)$", "%1"), nil) else - local lPath = fs.concat(path, j) - if not fs.exists(lPath) then - fs.makeDirectory(lPath) + local localInstalledPath = fs.concat(installPath, requestedInstallPath) + if not fs.exists(localInstalledPath) then + fs.makeDirectory(localInstalledPath) end - nPath = fs.concat(path, j, string.gsub(i, ".+(/.-)$", "%1"), nil) + installedPath = fs.concat(installPath, requestedInstallPath, string.gsub(filePath, ".+(/.-)$", "%1"), nil) + end + + if packageInfo.minified or options.m then + filePath = string.gsub(filePath, "(.+)\.(.+)$", "%1\.min\.%2") end - local success = pcall(downloadFile, repo .. "/" .. i, nPath) + + local success = pcall(downloadFile, repositoryPath .. "/" .. filePath, installedPath) if success then - tPacks[pack][i] = nPath + locallyInstalledPackages[requestedPackage][filePath] = installedPath end end - if info.dependencies then + + if packageInfo.dependencies then term.write("Done.\nInstalling Dependencies...") - for i, j in pairs(info.dependencies) do - local nPath - if string.find(j, "^//") then - nPath = string.sub(j, 2) + for packageDependency, dependencyPath in pairs(packageInfo.dependencies) do + local installedDependencyPath + + if string.find(dependencyPath, "^//") then + installedDependencyPath = string.sub(dependencyPath, 2) else - nPath = fs.concat(path, j, string.gsub(i, ".+(/.-)$", "%1"), nil) + installedDependencyPath = fs.concat(installPath, dependencyPath, string.gsub(packageDependency, ".+(/.-)$", "%1"), nil) end - if string.lower(string.sub(i, 1, 4)) == "http" then - local success = pcall(downloadFile, i, nPath) + + if string.lower(string.sub(packageDependency, 1, 4)) == "http" then + local success = pcall(downloadFile, packageDependency, installedDependencyPath) if success then - tPacks[pack][i] = nPath + locallyInstalledPackages[requestedPackage][packageDependency] = installedDependencyPath end else - local depInfo = getInformation(string.lower(i)) - if not depInfo then - term.write("\nDependency package " .. i .. " does not exist.") + local dependencyInfo = getInformation(string.lower(packageDependency)) + if not dependencyInfo then + term.write("\nDependency package " .. packageDependency .. " does not exist.") end - installPackage(string.lower(i), fs.concat(path, j)) + installPackage(string.lower(packageDependency), fs.concat(installPath, dependencyPath)) end end end + term.write("Done.\n") - saveToFile(tPacks) - print("Successfully installed package " .. pack) + saveToInstalledPackagesFile(locallyInstalledPackages) + print("Successfully installed package " .. requestedPackage) end -local function uninstallPackage(pack) - local info, repo = getInformation(pack) - if not info then +local function uninstallPackage(requestedPackage) + local packageInformation, repositoryLocation = getInformation(requestedPackage) + if not packageInformation then print("Package does not exist") return end - local tFiles = readFromFile(1) - if not tFiles then + + local locallyInstalledFiles = readFromLocalInstallFile() + if not locallyInstalledFiles then io.stderr:write("Error while trying to read package names") return - elseif tFiles[1] == -1 then - table.remove(tFiles, 1) + elseif locallyInstalledFiles[1] == -1 then + table.remove(locallyInstalledFiles, 1) end - if not tFiles[pack] then + + if not locallyInstalledFiles[requestedPackage] then print("Package has not been installed.") print("If it has, you have to remove it manually.") return end + term.write("Removing package files...") - for i, j in pairs(tFiles[pack]) do + for i, j in pairs(locallyInstalledFiles[requestedPackage]) do fs.remove(j) end + term.write("Done\nRemoving references...") - tFiles[pack] = nil - saveToFile(tFiles) + locallyInstalledFiles[requestedPackage] = nil + saveToInstalledPackagesFile(locallyInstalledFiles) term.write("Done.\n") - print("Successfully uninstalled package " .. pack) + print("Successfully uninstalled package " .. requestedPackage) end local function updatePackage(pack) if pack == "all" then print("Updating everything...") - local tFiles = readFromFile(1) + local tFiles = readFromLocalInstallFile() if not tFiles then io.stderr:write("Error while trying to read package names") return diff --git a/src/warricksothr/programs.cfg b/src/warricksothr/programs.cfg new file mode 100644 index 0000000..f0a5b84 --- /dev/null +++ b/src/warricksothr/programs.cfg @@ -0,0 +1,16 @@ +{ + ["reactor_control"] = { + files = { + ["reactor_control/reactor_control.lua"] = "/" + }, + dependencies = {}, + name = "Reactor Control Module", + description = "A Simple Package For Managing A Nuclear Reactor", + authors = "WarrickSothr", + note = "Maintains a Nuclear Reactor", + hidden = false, + repo = "", + version = "0.2.4", + minified = {MINIFIED} + } +} \ No newline at end of file