Skip to content

GitLab

B
baas
Commit 987c17de authored by Efthymia Bika's avatar Efthymia Bika
Browse files
Options

Use child_process.spawn instead of exec 

This allows better activity report by streaming output data on screen
and preventing buffer issues.
Also refactored in order to call duplicity from one place only
parent 617a8ba8
Hide whitespace changes
Inline Side-by-side
Showing with 207 additions and 251 deletions
baas/backup-list.html
View file @ 987c17de
...@@ -473,13 +473,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. ...@@ -473,13 +473,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
<div class="small-4 columns"> <div class="small-4 columns">
<a id="remove-all-button" class="button radius right small" <a id="remove-all-button" class="button radius right small"
onclick="if(check_remove_all()) { onclick="if(check_remove_all()) {
remove_all($('#remove-all').val(), false); remove_all(false);
$(this).hide()}"> $(this).hide()}">
<i class="fa fa-trash-o"></i>&nbsp;Remove All</a> <i class="fa fa-trash-o"></i>&nbsp;Remove All</a>
</div> </div>
<div class="small-4 columns hide" id="force-delete"> <div class="small-4 columns hide" id="force-delete">
<a id="remove-all-button-force" class="button radius right small alert" <a id="remove-all-button-force" class="button radius right small alert"
onclick="if(check_remove_all()) remove_all($('#remove-all').val(), true)"> onclick="if(check_remove_all()) remove_all(true)">
<i class="fa fa-trash-o"></i>&nbsp;Force Delete</a> <i class="fa fa-trash-o"></i>&nbsp;Force Delete</a>
</div> </div>
</div> </div>
......
baas/config.js
View file @ 987c17de
...@@ -19,6 +19,7 @@ var mkdirp = require("mkdirp"); ...@@ -19,6 +19,7 @@ var mkdirp = require("mkdirp");
var exec = require('child_process').exec; var exec = require('child_process').exec;
var execFileSync = require('child_process').execFileSync; var execFileSync = require('child_process').execFileSync;
var execFile = require('child_process').execFile; var execFile = require('child_process').execFile;
var spawn = require('child_process').spawn;
var BAAS_HOME_DIR = '.baas'; var BAAS_HOME_DIR = '.baas';
var CLOUDS_CONF_FILE = 'clouds.rc'; var CLOUDS_CONF_FILE = 'clouds.rc';
......
baas/dup_calls.js
View file @ 987c17de
...@@ -13,18 +13,27 @@ ...@@ -13,18 +13,27 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>. // along with this program. If not, see <http://www.gnu.org/licenses/>.
var dup_verbosity = " -v8 "; function get_backup_set() {
var cloud = $("#cloud").val().replace(/^\s+|\s+$/gm,'');
var backup_name = $("#backup-name").val().replace(/^\s+|\s+$/gm,'');
return backups[cloud + "/" + backup_name];
}
function backup(restore) { function backup(restore) {
$("#loader").show(); $("#loader").show();
if($("#error-alert")) $("#error-alert").hide(); if($("#error-alert")) $("#error-alert").hide();
$("#msg").html("");
if(!restore) { if(!restore) {
save_backup_set(false); save_backup_set(false);
disable_form(true); disable_form(true);
disable_actions(true); disable_actions(true);
disable_buttons(true); disable_buttons(true);
call_duplicity("backup", get_backup_set(), false);
} else {
$('#res-passphrase-error small').hide();
call_duplicity("restore", null, false);
} }
run_duplicity(restore, false);
} }
function get_env_values() { function get_env_values() {
...@@ -55,225 +64,82 @@ function build_win_commands() { ...@@ -55,225 +64,82 @@ function build_win_commands() {
"export SWIFT_PREAUTHURL=" + env_values[1] + ";" + "export SWIFT_PREAUTHURL=" + env_values[1] + ";" +
"export SWIFT_PREAUTHTOKEN=" + env_values[2] + ";"; "export SWIFT_PREAUTHTOKEN=" + env_values[2] + ";";
} }
function parse_cloud_error(restore, stderr, backup) { function parse_cloud_error(data) {
var cloud_error = var cloud_error =
new RegExp("AttributeError 'NoneType' object has no attribute 'find'") new RegExp("AttributeError 'NoneType' object has no attribute 'find'")
.exec(stderr); .exec(data);
if(cloud_error) { if(cloud_error) {
toggle_error(false, ""); toggle_error(false, "");
show_alert_box("A problem occured.<br>" + show_alert_box("A problem occured.<br>" +
"Please check your <a href='#'" + "Please check your <a href='#' " +
" onclick=$('#cloud-settings-link').trigger('click')>cloud settings</a>", "onclick=$('#cloud-settings-link').trigger('click')>" +
"error", false); "cloud settings</a>", "error", false);
if(!restore && backup) {
backups[backup].last_status = "Failed";
write_conf_file(BACKUP_CONF_FILE, backups);
disable_buttons(false);
}
return true; return true;
} }
return false; return false;
} }
function build_extra_args(field_value, type) { function build_extra_args(field_value, type, params) {
var args = field_value.split(","); var args = field_value.split(",");
var result = "";
$.each(args, function(i, value) { $.each(args, function(i, value) {
value = value.trim(); value = value.trim();
if(!value) return true; if(!value) return true;
if(process.platform == 'win32') { if(process.platform == 'win32') {
win_value = value.replace(/\\/g, "/"); value = value.replace(/\\/g, "/");
result += " --" + type + " " + escape_quote_str(win_value) + " ";
} else {
result += " --" + type + " " + escape_quote_str(value) + " ";
} }
var arg1 = "--" + type;
params.push(arg1);
params.push(value);
}); });
return result;
} }
function run_duplicity(restore, force) { function check_restore_errors(data) {
var exist_error =
var container_name = ""; new RegExp("Restore destination directory.* " +
var cloud = ""; "already exists.\nWill not overwrite.")
var time_arg = ""; .exec(data);
var file_arg = ""; var gpg_error = new RegExp("GPGError: GPG Failed").
var file_to_restore = $("#res-file").val(); exec(data);
var exclude_arg = ""; if(exist_error) {
var include_arg = ""; toggle_error(false, "");
var type_arg = ""; $("#modal-confirm").foundation("reveal", "open");
var i = 0;
if(restore) { $("#modal-confirm").on('close.fndtn.reveal',
container_name = $("#res-backup-name").val(); function(e) {
cloud = $("#res-cloud").val(); if(e.namespace !== "fndtn.reveal") return;
var timestamp = $("#timestamp").val(); i++;
if(timestamp) { $(this).click(function(event) {
time_arg = " --time " + timestamp; // event is fired more than once
} // so have to check
if(file_to_restore) { if(event.target.id == "modal-accept" &&
file_arg = " --file-to-restore " + i == 1) {
escape_quote_str(file_to_restore) + " "; $("#loader").show();
} call_duplicity("restore", null, true);
} else { }
container_name = $("#backup-name").val(); });
cloud = $("#cloud").val();
var exclude = $("#exclude").val();
if(exclude) {
exclude_arg = build_extra_args(exclude, "exclude");
}
var include = $("#include").val();
if(include) {
include_arg = build_extra_args(include, "include");
}
var backup_type = $("input[name=backup-type]:checked").val();
type_arg = " " + backup_type + " ";
backups[cloud + "/" + container_name].last_status = "Running";
}
var directory = "";
if(restore) {
if(file_to_restore) {
directory = path.join($("#res-directory").html(),
file_to_restore);
fs.stat(directory, function (err, stats) {
if(err) {
try {
mkdirp.sync(directory);
} catch(e) {
toggle_error(e, e);
}
} }
}); );
} else { } else if(gpg_error) {
directory = $("#res-directory").html(); toggle_error(false, "");
} $('#res-passphrase-error small').
text(errors.passphrase_wrong);
$('#res-passphrase-error small').show();
} else { } else {
directory = $("#directory").html(); $('#res-passphrase-error small').hide();
show_alert_box("A problem occured during restoring",
"error", false);
toggle_error(true, data.toString());
} }
}
var log_file = path.join(BAAS_LOG_DIR, "dup_" + function toggle_msgs(data, msgDiv) {
new Date().toISOString() + ".log"); if(data) {
var log_arg = " --log-file '" + log_file + "' "; $("#" + msgDiv).addClass("panel");
$("#" + msgDiv).append(data.toString());
var force_arg = (force) ? " --force " : ""; $("#" + msgDiv).animate({scrollTop: "+=300px"}, "slow");
var exclude_device_files_arg = (restore) ? " " :
" --exclude-device-files ";
cacert_arg = " --ssl-cacert-file " + clouds[cloud].cert + " ";
function dup_output(error, stdout, stderr) {
toggle_error(error, stderr);
if(error) {
$("#loader").hide();
if(parse_cloud_error(restore, stderr,
cloud + "/" + container_name)) {
return;
} else {
if(!restore) {
show_alert_box("There was a problem uploading backup set",
"error", false);
disable_form(false);
disable_actions(true);
backups[cloud + "/" + container_name].last_status =
"Failed";
} else {
var exist_error =
new RegExp("Restore destination directory.* " +
"already exists.\nWill not overwrite.")
.exec(stderr);
var gpg_error = new RegExp("GPGError: GPG Failed").
exec(stderr);
if(exist_error) {
toggle_error(false, "");
$("#modal-confirm").foundation("reveal", "open");
var i = 0;
$("#modal-confirm").on('close.fndtn.reveal',
function(e) {
if(e.namespace !== "fndtn.reveal") return;
i++;
$(this).click(function(event) {
// event is fired more than once
// so have to check
if(event.target.id == "modal-accept" &&
i == 1) {
$("#loader").show();
run_duplicity(true, true);
}
});
}
);
} else if(gpg_error) {
toggle_error(false, "");
$('#res-passphrase-error small').
text(errors.passphrase_wrong);
$('#res-passphrase-error small').show();
} else {
$('#res-passphrase-error small').hide();
show_alert_box("A problem occured during restoring",
"error", false);
toggle_error(true, stderr);
}
}
}
} else {
$("#loader").hide();
show_alert_box("Successfully completed", "success", true);
if(!restore) {
backups[cloud + "/" + container_name].last_status =
"Completed";
backups[cloud + "/" + container_name].last_backup =
new Date();
if(typeof backups[cloud + "/" + container_name].first_backup
== 'undefined') {
backups[cloud + "/" + container_name].first_backup =
new Date();
}
disable_actions(false);
$("#inc").prop("disabled", false);
$("#inc").prop("checked", true);
}
}
if(!restore) {
disable_buttons(false);
write_conf_file(BACKUP_CONF_FILE, backups);
}
}
if(process.platform == 'win32') {
directory = directory.replace(/\\/g, "\\\\");
var args = ["-c", "/usr/bin/cygpath " + escape_quote_str(directory)];
execFile(CYGWIN_BASH, args,
function(error, stdout, stderr) {
directory = String(stdout).replace(/(\r\n|\n|\r)/gm, "");
toggle_error(error, stderr);
var dirs = escape_quote_str(directory) + " swift://" +
container_name;
if(restore) {
dirs = " swift://" + container_name + " " +
escape_quote_str(directory);
}
var cmd = build_win_commands();
var dup_cmd = DUPLICITY_PATH + " " + type_arg + cacert_arg +
force_arg + exclude_device_files_arg + include_arg +
exclude_arg + file_arg + time_arg + dirs + ";";
var args = ["-c", cmd + dup_cmd];
execFile(CYGWIN_BASH, args, dup_output);
}
);
} else { } else {
set_envs(); $("#" + msgDiv).removeClass("panel");
$("#" + msgDiv).html("");
var dirs = escape_quote_str(directory) + " swift://" + container_name;
if(restore) {
dirs = " swift://" + container_name + " " +
escape_quote_str(directory);
}
var dup_cmd = DUPLICITY_PATH + " " + type_arg + cacert_arg +
force_arg + exclude_device_files_arg + dup_verbosity + log_arg +
include_arg + exclude_arg + file_arg + time_arg + dirs + ";";
exec(dup_cmd , dup_output);
} }
} }
...@@ -282,73 +148,160 @@ function load_status() { ...@@ -282,73 +148,160 @@ function load_status() {
$("#loader").show(); $("#loader").show();
$("#status_contents").html(""); $("#status_contents").html("");
function puts(error, stdout, stderr) { call_duplicity("status", get_backup_set(), false);
toggle_error(error, stderr); }
if(!error) {
$("#status_contents").html(stdout); function remove_all(force) {
} else { $("#loader").show();
parse_cloud_error(false, stderr, false); call_duplicity("remove", get_backup_set(), force);
}
$("#loader").hide(); }
}
var cacert_arg = " --ssl-cacert-file " + function call_duplicity(mode, backup_set, force) {
clouds[$("#cloud").val()].cert; var win_cmd = "";
var dup_cmd = DUPLICITY_PATH + cacert_arg +
" collection-status swift://" + container;
if(process.platform == 'win32') { if(process.platform == 'win32') {
var cmd = build_win_commands(); win_cmd = build_win_commands();
var args = ["-c", cmd + dup_cmd];
execFile(CYGWIN_BASH, args, puts);
} else { } else {
set_envs(); set_envs();
exec(dup_cmd, {maxBuffer: 1000*1024} , puts);
} }
}
function remove_all(time, force) { var args = [];
$("#loader").show(); switch(mode) {
case "backup":
args = [backup_set.backup_type, // full | incremental
backup_set.local_dir,
"swift://" + backup_set.container];
function puts(error, stdout, stderr) { args.push("--exclude-device-files");
toggle_error(false, ""); args.push("--num-retries", "2");
if(error) { build_extra_args(backup_set.include, "include", args);
if(!parse_cloud_error(false, stderr, false)) { build_extra_args(backup_set.exclude, "exclude", args);
$("#cleanup-msg").html(stderr); break;
$("#cleanup-msg").addClass("panel");
$("#force-delete").hide(); case "restore":
} var container_name = $("#res-backup-name").val();
} else { var timestamp = $("#timestamp").val();
$("#cleanup-msg").addClass("panel"); var file_to_restore = $("#res-file").val();
$("#cleanup-msg").html(stdout); var local_dir = "";
if(!force) { if(file_to_restore) {
$("#force-delete").show(); local_dir = path.join($("#res-directory").html(),
file_to_restore);
fs.stat(local_dir, function (err, stats) {
if(err) {
try { mkdirp.sync(local_dir); }
catch(e) { toggle_error(e, e); }
}
});
} else { } else {
$("#force-delete").hide(); local_dir = $("#res-directory").html();
} }
args = ["swift://" + container_name, local_dir];
if(timestamp) args.push("--time", timestamp);
if(file_to_restore) args.push("--file-to-restore",
file_to_restore);
args.push("-v8");
break;
case "status":
args = ["collection-status", "swift://" + backup_set.container];
break;
case "remove":
var time = $("#remove-all").val();
args = ["remove-older-than", time,
"swift://" + backup_set.container];
break;
default:
break;
}
var log_file = path.join(BAAS_LOG_DIR, "dup_" +
new Date().toISOString() + ".log");
args.push("--log-file", log_file);
if(backup_set) {
args.push("--ssl-cacert-file", clouds[backup_set.cloud].cert);
} else {
var res_cloud = $("#res-cloud").val();
args.push("--ssl-cacert-file", clouds[res_cloud].cert);
}
if(force) args.push("--force");
// call duplicity
var wProcess = spawn(DUPLICITY_PATH, args);
function dup_call_out(data) {
if(mode == "status") {
toggle_msgs(data, "status_contents");
} else if(mode == "remove") {
toggle_msgs(data, "cleanup-msg");
var nothing_to_del = new RegExp( var nothing_to_del = new RegExp(
"No old backup sets found, nothing deleted").exec(stdout); "No old backup sets found, nothing deleted").exec(data);
if(nothing_to_del) { if(nothing_to_del) {
$("#force-delete").hide();
$("#remove-all-button").show(); $("#remove-all-button").show();
$("#force-delete").hide();
} else {
if(!force) $("#force-delete").show();
else {
$("#remove-all-button").show();
$("#force-delete").hide();
}
}
} else {
toggle_msgs(data, "msg");
if(mode == "backup") {
backup_set.last_status = "Running";
} }
} }
$("#loader").hide();
} }
var force_arg = ""; function dup_call_err(data) {
if(force) { var cloud_error = false;
force_arg = " --force "; if(parse_cloud_error(data)) cloud_error = true;
if(mode == "backup") {
toggle_msgs(data, "msg");
disable_form(false);
disable_actions(true);
backup_set.last_status = "Failed";
if(!cloud_error) {
show_alert_box("There was a problem uploading backup set",
"error", false);
}
} else if(mode == "restore") {
if(!cloud_error) {
check_restore_errors(data);
}
} else if(mode == "remove") {
toggle_msgs(data, "cleanup-msg");
$("#force-delete").hide();
}
} }
var cacert_arg = " --ssl-cacert-file " + function dup_call_exit(code) {
clouds[$("#cloud").val()].cert; $("#loader").hide();
var dup_cmd = DUPLICITY_PATH + " remove-older-than " + if(mode == "backup") {
time + force_arg + cacert_arg + " swift://" + container; if(code == 0) {
if(process.platform == 'win32') { show_alert_box("Successfully completed", "success", true);
var cmd = build_win_commands(); backup_set.last_status = "Completed";
var args = ["-c", cmd + dup_cmd]; backup_set.last_backup = new Date();
execFile(CYGWIN_BASH, args, puts); if(typeof backup_set.first_backup == 'undefined') {
} else { backup_set.first_backup = new Date();
set_envs(); }
exec(dup_cmd, {maxBuffer: 1000*1024} , puts); $("#inc").prop("disabled", false);
$("#inc").prop("checked", true);
disable_actions(false);
}
disable_buttons(false);
write_conf_file(BACKUP_CONF_FILE, backups);
} else if(mode == "restore") {
if(code == 0) {
show_alert_box("Successfully completed", "success", true);
}
}
} }
// bind listeners
wProcess.stdout.on('data', dup_call_out);
wProcess.stderr.on('data', dup_call_err);
wProcess.on('exit', dup_call_exit);
} }
baas/static/stylesheets/custom.css
View file @ 987c17de
...@@ -88,6 +88,8 @@ body legend, .template-list-title { ...@@ -88,6 +88,8 @@ body legend, .template-list-title {
#msg, #cleanup-msg { #msg, #cleanup-msg {
word-break: break-word; word-break: break-word;
margin-right: 15px; margin-right: 15px;
overflow: auto;
max-height: 200px;
} }
.dir-field { .dir-field {
overflow-x: auto; overflow-x: auto;
......
baas/timeview.js
View file @ 987c17de
...@@ -177,7 +177,7 @@ function load_timeview() { ...@@ -177,7 +177,7 @@ function load_timeview() {
function puts(error, stdout, stderr) { function puts(error, stdout, stderr) {
$("#loader").hide(); $("#loader").hide();
toggle_error(error, stderr); toggle_error(error, stderr);
if(!error) { if(!parse_cloud_error(stderr) && !error) {
var datetime_reg = /\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}\s+\d+/g; var datetime_reg = /\d{4}-\d{2}-\d{2}.\d{2}:\d{2}:\d{2}\s+\d+/g;
var dates = stdout.match(datetime_reg); var dates = stdout.match(datetime_reg);
var dates_list = ""; var dates_list = "";
......