Repository navigation
Expand file tree
/
Copy pathdyode_logs.py
More file actions
279 lines (240 loc) · 10.8 KB
/
Copy pathdyode_logs.py
File metadata and controls
279 lines (240 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Weekly archiving and inspection for the DYODE transfer logs.
Rotation itself is logrotate's job (packaging/logrotate/dyode-transfer).
This tool runs once a week, from a systemd timer, and packs the daily files
logrotate left behind into one tarball:
/var/log/dyode-transfer/dyode.log-20260921 rotated by logrotate
/var/log/dyode-transfer/transfer.jsonl-20260921
-> /var/log/dyode-transfer/archive/dyode-logs-2026-W38.tar.gz
It only ever touches ROTATED files -- the live 'dyode.log' and
'transfer.jsonl' that the daemons hold open are never read, moved or
deleted, so it does not matter whether this runs before or after logrotate.
Packing a week together rather than letting logrotate gzip each day
compresses considerably better: consecutive days of the same log are highly
redundant and gzip sees that redundancy inside one tar stream.
dyode_logs.py --archive pack last week, prune old archives
dyode_logs.py --archive --dry-run say what it would do
dyode_logs.py --stats summarize transfer.jsonl
"""
import argparse
import collections
import datetime
import json
import os
import re
import sys
import tarfile
DEFAULT_DIR = "/var/log/dyode-transfer"
ARCHIVE_SUBDIR = "archive"
DEFAULT_KEEP_WEEKS = 26
# logrotate 'dateext' with 'dateformat -%Y%m%d' produces 'dyode.log-20260921'.
# The plain numeric suffixes logrotate uses without dateext ('dyode.log.1')
# are matched too, so a hand-edited logrotate config still archives.
ROTATED_RE = re.compile(r"^(?P<base>.+?)[.-](?P<stamp>\d{8}|\d+)(?P<gz>\.gz)?$")
ARCHIVE_RE = re.compile(r"^dyode-logs-(?P<year>\d{4})-W(?P<week>\d{2})\.tar\.gz$")
def week_tag(when):
"""ISO year and week, e.g. '2026-W38'. ISO weeks start on Monday."""
iso = when.isocalendar()
return "%04d-W%02d" % (iso[0], iso[1])
def rotated_files(directory, live_names):
"""Rotated log files in `directory`, oldest suffix first.
`live_names` are the files the daemons still have open; they are skipped
however they sort, which is what makes this safe to run at any time.
"""
found = []
for name in sorted(os.listdir(directory)):
path = os.path.join(directory, name)
if not os.path.isfile(path) or name in live_names:
continue
match = ROTATED_RE.match(name)
if match and match.group("base") in live_names:
found.append(path)
return found
def archive(directory=DEFAULT_DIR, live_names=("dyode.log", "transfer.jsonl"),
keep_weeks=DEFAULT_KEEP_WEEKS, when=None, dry_run=False,
out=sys.stdout):
"""Pack rotated logs into one tarball and prune old ones.
Returns (tarball path or None, list of files packed, list of archives
pruned).
"""
live_names = set(live_names)
when = when or datetime.date.today()
files = rotated_files(directory, live_names)
if not files:
print("nothing to archive in %s" % directory, file=out)
return None, [], prune(directory, keep_weeks, when, dry_run, out)
archive_dir = os.path.join(directory, ARCHIVE_SUBDIR)
tarball = os.path.join(archive_dir,
"dyode-logs-%s.tar.gz" % week_tag(when))
if dry_run:
print("would create %s with %d file(s):" % (tarball, len(files)),
file=out)
for path in files:
print(" %s" % os.path.basename(path), file=out)
return tarball, files, prune(directory, keep_weeks, when, True, out)
os.makedirs(archive_dir, mode=0o750, exist_ok=True)
# Write beside the target, then rename: a crash or a full disk leaves a
# .partial behind rather than a truncated archive that looks complete.
# gzip streams cannot be appended to, so a second run in the same ISO
# week rewrites the tarball with the old members plus the new files
# rather than overwriting a week of logs.
partial = tarball + ".partial"
try:
with tarfile.open(partial, "w:gz") as tar:
carried = _copy_members(tarball, tar) if os.path.exists(tarball) else 0
for path in files:
tar.add(path, arcname=os.path.basename(path))
except OSError:
if os.path.exists(partial):
os.remove(partial)
raise
os.replace(partial, tarball)
for path in files:
os.remove(path)
print("archived %d file(s) into %s%s"
% (len(files), tarball,
" (kept %d already there)" % carried if carried else ""),
file=out)
return tarball, files, prune(directory, keep_weeks, when, False, out)
def _copy_members(source, tar):
"""Copy every member of the `source` tarball into open tarfile `tar`."""
copied = 0
with tarfile.open(source, "r:gz") as old:
for member in old.getmembers():
if not member.isfile():
continue
extracted = old.extractfile(member)
if extracted is None:
continue
tar.addfile(member, extracted)
copied += 1
return copied
def prune(directory, keep_weeks, when=None, dry_run=False, out=sys.stdout):
"""Delete archives older than `keep_weeks`. Returns the paths removed."""
archive_dir = os.path.join(directory, ARCHIVE_SUBDIR)
if keep_weeks <= 0 or not os.path.isdir(archive_dir):
return []
when = when or datetime.date.today()
cutoff = when - datetime.timedelta(weeks=keep_weeks)
removed = []
for name in sorted(os.listdir(archive_dir)):
match = ARCHIVE_RE.match(name)
if not match:
continue
year, week = int(match.group("year")), int(match.group("week"))
try:
monday = datetime.date.fromisocalendar(year, week, 1)
except ValueError:
continue
if monday >= cutoff:
continue
path = os.path.join(archive_dir, name)
removed.append(path)
if dry_run:
print("would remove %s" % path, file=out)
else:
os.remove(path)
print("removed %s" % path, file=out)
return removed
# --------------------------------------------------------------------------
# Reading the JSON stream back
# --------------------------------------------------------------------------
def read_events(path):
"""Yield the events in a .jsonl file, skipping anything unparsable.
A partial last line is normal if the file is read while a transfer is
running, so a bad line is skipped rather than raising.
"""
with open(path, "r", encoding="utf-8", errors="replace") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
event = json.loads(line)
except ValueError:
continue
if isinstance(event, dict):
yield event
def summarize(events):
"""Totals per event type, files and bytes moved, and log lines."""
totals = collections.Counter()
files_sent = files_stored = files_rejected = 0
bytes_sent = bytes_stored = 0
log_lines = collections.Counter()
for event in events:
name = event.get("event", "?")
totals[name] += 1
if name == "batch_sent":
files_sent += event.get("files", 0)
bytes_sent += event.get("bytes", 0)
elif name == "batch_received":
files_stored += event.get("files_stored", 0)
files_rejected += event.get("files_rejected", 0)
bytes_stored += event.get("bytes", 0)
elif name == "logs_stats":
# Periodic counters from 'logs' modules; each record covers the
# minute since the previous one, so they simply add up.
if event.get("side") == "in":
log_lines["sent"] += event.get("lines_sent", 0)
log_lines["dropped_at_source"] += event.get("dropped", 0)
else:
log_lines["delivered"] += event.get("lines", 0)
log_lines["batches_lost"] += event.get("lost_batches", 0)
return {"events": dict(totals), "files_sent": files_sent,
"files_stored": files_stored, "files_rejected": files_rejected,
"bytes_sent": bytes_sent, "bytes_stored": bytes_stored,
"log_lines": dict(log_lines)}
def human_bytes(count):
for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
if count < 1024 or unit == "TiB":
return "%.1f %s" % (count, unit) if unit != "B" else "%d B" % count
count /= 1024.0
def main(argv=None):
parser = argparse.ArgumentParser(
description="Archive and inspect the DYODE transfer logs.")
parser.add_argument("--dir", default=DEFAULT_DIR,
help="log directory (default: %s)" % DEFAULT_DIR)
parser.add_argument("--archive", action="store_true",
help="pack rotated logs into this week's tarball")
parser.add_argument("--stats", action="store_true",
help="summarize the JSON event log")
parser.add_argument("--keep-weeks", type=int, default=DEFAULT_KEEP_WEEKS,
help="archives to keep (default: %d, 0 = forever)"
% DEFAULT_KEEP_WEEKS)
parser.add_argument("--json", dest="json_name", default="transfer.jsonl",
help="event log file name, for --stats")
parser.add_argument("--dry-run", action="store_true",
help="say what would happen, change nothing")
args = parser.parse_args(argv)
if not args.archive and not args.stats:
parser.error("choose --archive or --stats")
if not os.path.isdir(args.dir):
print("no such directory: %s" % args.dir, file=sys.stderr)
return 1
if args.archive:
archive(args.dir, keep_weeks=args.keep_weeks, dry_run=args.dry_run)
if args.stats:
path = os.path.join(args.dir, args.json_name)
if not os.path.isfile(path):
print("no event log at %s" % path, file=sys.stderr)
return 1
stats = summarize(read_events(path))
print("%s" % path)
for name, count in sorted(stats["events"].items()):
print(" %-20s %d" % (name, count))
print(" files sent %d (%s)"
% (stats["files_sent"], human_bytes(stats["bytes_sent"])))
print(" files stored %d (%s)"
% (stats["files_stored"], human_bytes(stats["bytes_stored"])))
print(" files rejected %d" % stats["files_rejected"])
logs = stats["log_lines"]
if logs:
print(" log lines sent %d" % logs.get("sent", 0))
print(" log lines delivered %d" % logs.get("delivered", 0))
print(" log batches lost %d" % logs.get("batches_lost", 0))
print(" log lines dropped %d (input buffer full)"
% logs.get("dropped_at_source", 0))
return 0
if __name__ == "__main__":
sys.exit(main())