Skip to content

Commandline-Stats #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
github-isgee wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Commandline-Stats #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c020337
bangadm for cmdline reporting, additional system status, small improv…
Apr 29, 2025
798fdbf
added d-phys patch for reporting mail sender addr
Jul 24, 2025
1603c08
added BKP_SKIP_FILESONLY_JOB
Aug 11, 2025
82349ff
clarify config docs
upekkha Feb 7, 2025
f775b6b
clean trailing white-space
upekkha Feb 7, 2025
07903ed
Fix log message
frukto Feb 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions BaNG
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,32 @@ if ( $serverconfig{bkpmode} eq "rsync") {

reorder_queue_by_priority( $taskid, $host_arg, $group_arg );

# add all queued jobs to db, in order of their priority.
# this allows distinction between still queued, running
# and finished jobs.
foreach my $bkpjob (@queue) {
my $exclsubfolders = $bkpjob->{'exclsubfolders'} || 0;

# append custom string to path written to DB
# if we are excluding subfolders
my $bkpfrompath = $exclsubfolders
? $bkpjob->{path}."/FILESONLY"
: $bkpjob->{path};

$bkpfrompath =~ s/'//g;
$bkpfrompath =~ s/\/\//\//g;

bangstat_report_queue_backupjob(
$bkpjob->{taskid},
$bkpjob->{jobid},
$bkpjob->{host},
$bkpjob->{group},
$bkpfrompath,
$bkpjob->{srcfolder},
targetpath($bkpjob->{host}, $bkpjob->{group})
);
}

run_rsync_threads($host_arg, $group_arg, $nthreads_arg, $dryrun_arg, $cron);

logit( $taskid, $host_arg, $group_arg, "Task $taskid finished!" );
Expand Down
157 changes: 155 additions & 2 deletions BaNGadm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#!/usr/bin/env perl
use strict;
use warnings;

use Cwd qw( abs_path );
use Data::Dumper;
use File::Basename;
use Getopt::Long qw( :config no_auto_abbrev );
use POSIX qw( strftime );
Expand All @@ -12,6 +14,7 @@ use BaNG::Config;
use BaNG::BackupServer;
use BaNG::Wipe;
use BaNG::Reporting;
use BaNG::Statistics;

my $version = '0.1';
my $add_arg;
Expand All @@ -33,6 +36,15 @@ my $prefix_arg;
my $showgroups;
my $get_oob_snapshots_arg;

my $showrep_recentbackups_arg;
my $showrep_recenttasks_arg;
my $showrep_jobstatus_arg;
# host_arg and group_arg already defined above!
my $taskid_arg;
my $jobid_arg;
my $jobstatus_arg;
my $lastxhours_arg;

#################################
# Main
#
Expand Down Expand Up @@ -276,6 +288,122 @@ if ($tidy_arg) {

}

if ($showrep_recentbackups_arg) {
printf("Recent Backups\n");
printf("\n");

my %RecentBackups = bangstat_recentbackups($host_arg);

my $unified_backups = [];

foreach my $hostgroupkey (sort(keys(%RecentBackups))) {
foreach my $backup (@{$RecentBackups{$hostgroupkey}}) {
if (!defined($backup->{'JobID'})) {
$backup->{'JobID'} = '-';
}

push(@$unified_backups, $backup);
}
}

my $formats = [
{ 'colkey' => 'BkpHost', 'title' => 'Host', 'align' => 'l', 'maxlen' => 10 },
{ 'colkey' => 'BkpGroup', 'title' => 'Group', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'TaskID', 'title' => 'TaskID', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'JobID', 'title' => 'JobID', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'Cron', 'title' => 'Cron', 'align' => 'r', 'maxlen' => 4 },
{ 'colkey' => 'isThread', 'title' => 'Thr', 'align' => 'r', 'maxlen' => 3 },
{ 'colkey' => 'ErrStatus', 'title' => 'ErrSt', 'align' => 'r', 'maxlen' => 5 },
{ 'colkey' => 'JobStatus', 'title' => 'JobSt', 'align' => 'r', 'maxlen' => 5 },
{ 'colkey' => 'Jobs', 'title' => 'Jobs', 'align' => 'r', 'maxlen' => 4 },
{ 'colkey' => 'Starttime', 'title' => 'Start', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'Runtime', 'title' => 'Runtime', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesCreated', 'title' => 'FilesCr', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesDel', 'title' => 'FilesDel', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesTrans', 'title' => 'FilesTrans', 'align' => 'r', 'maxlen' => 10 },
{ 'colkey' => 'SizeTrans', 'title' => 'SizeTrans', 'align' => 'r', 'maxlen' => 9 },
{ 'colkey' => 'TotFileSize', 'title' => 'TotFileSize', 'align' => 'r', 'maxlen' => 11 },
{ 'colkey' => 'NumOfFiles', 'title' => 'NumFiles', 'align' => 'r', 'maxlen' => 8 }
];

print_formatted_table($unified_backups, $formats);
printf("\n");
}

if ($showrep_recenttasks_arg) {
printf("Recent Tasks\n");
printf("\n");

my $RecentTasks = bangstat_recent_tasks();

my $formats = [
{ 'colkey' => 'TaskID', 'title' => 'TaskID', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'Taskname', 'title' => 'Taskname', 'align' => 'l', 'maxlen' => 30 },
{ 'colkey' => 'Description', 'title' => 'Desc', 'align' => 'l', 'maxlen' => 10 },
{ 'colkey' => 'Cron', 'title' => 'Cron', 'align' => 'r', 'maxlen' => 4 },
{ 'colkey' => 'isThread', 'title' => 'Thr', 'align' => 'r', 'maxlen' => 3 },
{ 'colkey' => 'ErrStatus', 'title' => 'ErrSt', 'align' => 'r', 'maxlen' => 5 },
{ 'colkey' => 'JobStatus', 'title' => 'JobSt', 'align' => 'r', 'maxlen' => 5 },
{ 'colkey' => 'Jobs', 'title' => 'Jobs', 'align' => 'r', 'maxlen' => 4 },
{ 'colkey' => 'BkpToHost', 'title' => 'Host', 'align' => 'l', 'maxlen' => 10 },
{ 'colkey' => 'Starttime', 'title' => 'Start', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'Runtime', 'title' => 'Runtime', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesCreated', 'title' => 'FilesCr', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesDel', 'title' => 'FilesDel', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesTrans', 'title' => 'FilesTrans', 'align' => 'r', 'maxlen' => 10 },
{ 'colkey' => 'SizeTrans', 'title' => 'SizeTrans', 'align' => 'r', 'maxlen' => 9 },
{ 'colkey' => 'TotFileSize', 'title' => 'TotFileSize', 'align' => 'r', 'maxlen' => 11 },
{ 'colkey' => 'NumOfFiles', 'title' => 'NumFiles', 'align' => 'r', 'maxlen' => 8 }
];

print_formatted_table($RecentTasks->{'Data'}, $formats);
printf("\n");
}

if ($showrep_jobstatus_arg) {

printf("Job Status Report (last %s hours)\n", defined($lastxhours_arg) ? $lastxhours_arg : 24);
printf("\n");

my $jobstatus = string_to_jobstatus($jobstatus_arg);

my $backups_found = bangstat_jobs_by_jobstatus(
$taskid_arg,
$host_arg,
$group_arg,
$jobid_arg,
$jobstatus,
$lastxhours_arg
);

foreach my $backup (@$backups_found) {
$backup->{'JobStatus'} = jobstatus_to_string($backup->{'JobStatus'});
}

my $formats = [
{ 'colkey' => 'TaskID', 'title' => 'TaskID', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'BkpHost', 'title' => 'Host', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'BkpGroup', 'title' => 'Group', 'align' => 'r', 'maxlen' => 20 },
{ 'colkey' => 'JobID', 'title' => 'JobID', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'JobStatus', 'title' => 'JobStatus', 'align' => 'r', 'maxlen' => 12 },
{ 'colkey' => 'ErrStatus', 'title' => 'ErrSt', 'align' => 'r', 'maxlen' => 10 },
{ 'colkey' => 'Starttime', 'title' => 'Start', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'Stoptime', 'title' => 'Stop', 'align' => 'l', 'maxlen' => 20 },
{ 'colkey' => 'Runtime', 'title' => 'Runtime', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'isThread', 'title' => 'Thr', 'align' => 'r', 'maxlen' => 3 },
{ 'colkey' => 'NumOfFiles', 'title' => 'NumFiles', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesTrans', 'title' => 'FilesTrans', 'align' => 'r', 'maxlen' => 10 },
{ 'colkey' => 'FilesCreated', 'title' => 'FilesCr', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'FilesDel', 'title' => 'FilesDel', 'align' => 'r', 'maxlen' => 8 },
{ 'colkey' => 'TotFileSize', 'title' => 'TotFileSize', 'align' => 'r', 'maxlen' => 11 },
{ 'colkey' => 'SizeTrans', 'title' => 'SizeTrans', 'align' => 'r', 'maxlen' => 9 },
{ 'colkey' => 'BkpFromPath', 'title' => 'BkpFromPath', 'align' => 'l', 'maxlen' => 30 }
];

print_formatted_table($backups_found, $formats);
printf("\n");
}

#################################
# Command line arguments
#
Expand All @@ -301,15 +429,28 @@ sub parse_command_options {
'p|prefix=s' => \$prefix_arg,
'showgroups' => \$showgroups,
'setprop=s' => \$setprop,
'get_oob_snapshots:i' => \$get_oob_snapshots_arg,
'taskid=s' => \$taskid_arg,
'jobid=s' => \$jobid_arg,
'jobstatus=s' => \$jobstatus_arg,
'lastxhours=s'=> \$lastxhours_arg,
'get_oob_snapshots:i' => \$get_oob_snapshots_arg,
'showrep-recentbackups' => \$showrep_recentbackups_arg,
'showrep-recenttasks' => \$showrep_recenttasks_arg,
'showrep-jobstatus' => \$showrep_jobstatus_arg

) or usage('Invalid commmand line options.');

usage('Do not use together!') if ( $croncreate && $showgroups );
usage('Check the arguments!') unless (
( $add_arg || $delete_arg ) && (( $host_arg && $group_arg ) || $group_arg ) ||
( $initial_arg && ( $host_arg && $group_arg ) ) ||
( $initial_arg && ( $host_arg && $group_arg && ( $prefix_arg || $dryrun_arg ))) ||
( $croncreate || $croncheck || $failed_arg || $showgroups || $tidy_arg || $dump_arg ) ||
( $host_arg && $group_arg && $setprop ) || (defined $get_oob_snapshots_arg)
( $host_arg && $group_arg && $setprop ) ||
( $showrep_recentbackups_arg && $host_arg ) ||
( $showrep_recenttasks_arg ) ||
( $showrep_jobstatus_arg ) ||
(defined $get_oob_snapshots_arg)
);
usage('The option --get_oob_snapshots can only take the values 1 for oob-only or 2 to include failed backups.') if ($get_oob_snapshots_arg && ($get_oob_snapshots_arg >=3 || $get_oob_snapshots_arg <0));

Expand Down Expand Up @@ -371,6 +512,18 @@ sub usage {
$command --showgroups # show available groups
$command --help # show this help message
$command --version # show version number and help

$command --showrep-recentbackups # show recent backups report (requires --host=...)
$command --showrep-recenttasks # show recent tasks report
$command --showrep-jobstatus # show job status report
# if no filter args are specified only --lastxhours is set to 24.
# the following filters are supported:
# --taskid=<TASKID>
# --host=<HOSTNAME>
# --group=<BaNG-GROUP>
# --jobid=<JOBID>
# --jobstatus=(QUEUED|STARTED|PROCESSED|FINISHED)
# --lastxhours=<hours>

Optional Arguments:

Expand Down
14 changes: 6 additions & 8 deletions docs/Configuration.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,15 @@ BaNG Configuration
Config files
------------

All configuration files are stored in the local `etc/` folder.
All configuration files are stored in the local `etc/` folder. The settings are designed in a sort of hierarchy, where more specific settings can be used to override more general ones. The default settings apply to all, but can be overridden by group- and host-specific settings.

* `defaults_servers.yaml` : common default settings for servers
* `defaults_hosts.yaml` : common default settings for hosts
* `servers/<servername>_defaults.yaml` : server-specific settings
* `groups/<groupname>.yaml` : group-specific settings
* `hosts/<hostname>_<groupname>.yaml` : host-specific settings
* `defaults_hosts.yaml` : common default settings for all hosts backed up by BaNG
* `defaults_servers.yaml` : common default settings for all BaNG backup servers
* `servers/<servername>_defaults.yaml` : settings specific to a given BaNG backup server
* `groups/<groupname>.yaml` : common settings for a given group of hosts backed up by BaNG
* `hosts/<hostname>_<groupname>.yaml` : settings specific to a given host and group pair
* `excludes/excludelist_<name>` : rsync exclude list for given group or host

The default settings apply to all, but can be overridden by server-, group- and host-specific settings.


Add a new host
--------------
Expand Down
2 changes: 1 addition & 1 deletion docs/Installation.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ mv etc/servers/bangserver_defaults.yaml etc/servers/`hostname -s`_defaults.yaml

Fields you typically want to change in the config files:

* `defaults_servers.yaml`: `report_to`
* `defaults_servers.yaml`: `report_from` and `report_to`
* `defaults_hosts.yaml`: `BKP_TARGET_HOST` and `BKP_TARGET_PATH`

Make sure the `BKP_TARGET_PATH` folder where backups should be stored exists
Expand Down
2 changes: 2 additions & 0 deletions etc.example/defaults_servers.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ path_taskset:
path_timeout:
timeout_duration:
#
report_from: root@example.com
report_to: admin@example.com
xymon_server:
#
Expand Down Expand Up @@ -85,6 +86,7 @@ HELP:
path_rsync:
path_dar:

report_from: email-addresses used as report sender
report_to: email-addresses for reporting (comma separated)
xymon_server: hostname of xymon server

Expand Down
24 changes: 12 additions & 12 deletions lib/BaNG/BackupServer.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ sub check_client_connection {

my $state = 0;
my $msg = 'Host offline';
my $p = Net::Ping->new( 'tcp', 2 );
my $p = Net::Ping->new( 'icmp', 2 );

eval {
if ( $p->ping($host) ) {
Expand All @@ -157,7 +157,7 @@ sub check_client_connection {
$msg = 'Host not pingable because behind a Gateway-Host';
}
$p->close();
1;
1;
} or do {
my $theerror = $@;
$msg = "Exception caught in Net::Ping - $theerror";
Expand Down Expand Up @@ -209,10 +209,10 @@ sub get_backup_folders {
$REGEX = ".*_failed" if $folder_type == 1; # show all *_failed folders
$REGEX = "\\([0-9\./_]*\\|.*_failed\$\\)" if $folder_type == 2; # show all folders, except "current" folder
$REGEX = "[0-9]+\.[0-9]+\.[0-9]+_[0-9]+[-_.]+.*" if ($folder_type == 3 || $folder_type == 4); # get all oob

if ( $server eq $servername ) {
@backup_folders = `find $bkpdir -mindepth 1 -maxdepth 1 -type d -regex '${bkpdir}/$REGEX' 2>/dev/null`;
@backup_folders = grep(!/.*_failed$/,@backup_folders) if ($folder_type == 3); # filter out failed if we don't want them
@backup_folders = grep(!/.*_failed$/,@backup_folders) if ($folder_type == 3); # filter out failed if we don't want them
} else {
@backup_folders = remote_command( $server, "$servers{$server}{serverconfig}{remote_app_folder}/bang_getBackupFolders", $bkpdir );
}
Expand Down Expand Up @@ -269,8 +269,8 @@ sub pre_queue_checks {
$endstamp = $startstamp;
$jobid = create_timeid( $taskid, $host, $group );
logit( $taskid, $host, $group, "Error: host $host is offline" );
bangstat_start_backupjob( $taskid, $jobid, $host, $group, $startstamp, $endstamp, $hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER},
$hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER}, targetpath( $host, $group ), '0', '-1', '' ) unless $noreport;
bangstat_report_pre_queue_error( $taskid, $jobid, $host, $group, $startstamp, $endstamp, $hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER},
$hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER}, targetpath( $host, $group ), ERRSTATUS_NOERR, JOBSTATUS_FAILED_OFFLINE, '' ) unless $noreport;
return 0;
}

Expand All @@ -283,8 +283,8 @@ sub pre_queue_checks {
$endstamp = $startstamp;
$jobid = create_timeid( $taskid, $host, $group );
logit( $taskid, $host, $group, "Error: ". $hosts{"$host-$group"}->{hostconfig}->{BKP_RSYNC_RSHELL} ." on host $host not working" );
bangstat_start_backupjob( $taskid, $jobid, $host, $group, $startstamp, $endstamp, $hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER},
$hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER}, targetpath( $host, $group ), '0', '-2', '' ) unless $noreport;
bangstat_report_pre_queue_error( $taskid, $jobid, $host, $group, $startstamp, $endstamp, $hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER},
$hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER}, targetpath( $host, $group ), ERRSTATUS_NOERR, JOBSTATUS_FAILED_RSHELL, '' ) unless $noreport;
mail_report( $taskid, $host, $group, () ) if $serverconfig{report_to};
return 0;
}
Expand All @@ -294,8 +294,8 @@ sub pre_queue_checks {
$endstamp = $startstamp;
$jobid = create_timeid( $taskid, $host, $group );
logit( $taskid, $host, $group, "RSYNC command " . ($serverconfig{path_rsync} || "") . " not found!" );
bangstat_start_backupjob( $taskid, $jobid, $host, $group, $startstamp, $endstamp, $hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER},
$hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER}, targetpath( $host, $group ), '0', '-5', '' ) unless $noreport;
bangstat_report_pre_queue_error( $taskid, $jobid, $host, $group, $startstamp, $endstamp, $hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER},
$hosts{"$host-$group"}->{hostconfig}->{BKP_SOURCE_FOLDER}, targetpath( $host, $group ), ERRSTATUS_NOERR, JOBSTATUS_FAILED_RSYNCAPP, '' ) unless $noreport;
return 0;
}

Expand Down Expand Up @@ -704,7 +704,7 @@ sub get_oob_snapshot_dirs {
elsif ($get_oob_snapshots_mode == 2) {
$find_mode = 4;
}

foreach my $hostgroup (sort keys %$hosts ) {
unless ( $hosts{$hostgroup}->{hostconfig}->{BKP_TARGET_HOST} ne $servername ) {
$oob_dirs{$hostgroup}= [get_backup_folders($hosts{$hostgroup}->{hostname},$hosts{$hostgroup}->{group}, $find_mode)];
Expand All @@ -714,7 +714,7 @@ sub get_oob_snapshot_dirs {
#}
}
return %oob_dirs;

}

1;
Loading