Filter entries by level and pattern

CHANGELOG.md

@@ -4,4 +4,5 @@
 
 - Added the `relay-logs` command, which prints files or standard input colored by level.
 - Added `-f` to follow files across rotation and truncation, and `-n` to start with the last lines of each file.
+- Added `-l` to filter by level and `-g` and `-x` to include or exclude lines by pattern, keeping stack frames with their entry.
 - Added `parseLine`, which recognizes text, JSON, and logfmt lines, and `sanitizeLine`, which removes terminal control sequences.

README.md

@@ -23,6 +23,17 @@ relay-logs -f -n 50 api.log worker.log
 
 When several files are followed, their lines are interleaved in the order they arrive.
 
+## Filter
+
+```sh
+relay-logs -l warn app.log
+relay-logs -f -g timeout -g refused -x /health api.log
+```
+
+`-l` shows only entries at a level or above: `trace`, `debug`, `info`, `warn`, `error`, or `fatal`. Lines without a recognizable level are hidden by `-l`, except indented lines such as stack frames, which stay with the entry above them.
+
+`-g` shows only lines matching a regular expression, and `-x` hides lines matching one. Both may be repeated. Patterns ignore case unless they contain an uppercase letter.
+
 ## Recognized lines
 
 - Text lines with an optional leading timestamp and a level such as `ERROR`, `[warn]`, `<info>`, or `FATAL:`

docs/architecture.md

@@ -6,5 +6,4 @@ Relay Logs turns lines from files or standard input into log entries and display
 2. `openSources` opens each file, or standard input, as a labeled stream of lines. `followFile` polls a file for appended lines and notices truncation and rotation by its size and identity. `mergeSources` interleaves followed files as their lines arrive.
 3. `parseLine` sanitizes a line and recognizes its level, timestamp, and message. The original text is kept in `raw`, so nothing is lost when a line is not recognized.
 4. `formatPlain` prints an entry for pipes and plain terminals.
-
-Filtering and the interactive viewer build on these modules.
+5. `createFilter` decides which entries are shown by level and pattern, carrying the decision for an entry over to its indented continuation lines.

src/cli.tsx

@@ -2,6 +2,7 @@
 
 import {helpText, parseCliOptions} from './cli/options.js';
 import {parseLine} from './core/entry.js';
+import {createFilter} from './core/filter.js';
 import {concatSources, type LogSource, mergeSources, openSources} from './core/sources.js';
 import {formatPlain} from './format/plain.js';
 import {relayLogsVersion} from './version.js';
@@ -46,10 +47,16 @@ const main = async (): Promise<void> => {
     process.exitCode = 1;
   };
   const lines = options.follow ? mergeSources(sources, onError) : concatSources(sources, onError);
+  const shown = createFilter({
+    ...(options.minLevel === undefined ? {} : {minLevel: options.minLevel}),
+    include: options.include,
+    exclude: options.exclude,
+  });
   let seq = 0;
   for await (const {label, line} of lines) {
     seq += 1;
     const entry = parseLine(line, {seq, ...(label === undefined ? {} : {source: label})});
+    if (!shown(entry)) continue;
     process.stdout.write(`${formatPlain(entry, format)}\n`);
   }
 };

src/cli/options.ts

@@ -1,3 +1,6 @@
+import {compilePattern} from '../core/filter.js';
+import {type Level, levels, parseLevelName} from '../core/levels.js';
+
 export interface CliOptions {
   help: boolean;
   version: boolean;
@@ -7,6 +10,9 @@ export interface CliOptions {
   follow: boolean;
   /** Start with only the last lines of each file. */
   lines: number | undefined;
+  minLevel: Level | undefined;
+  include: RegExp[];
+  exclude: RegExp[];
   files: string[];
 }
 
@@ -22,6 +28,10 @@ level.
 Options:
   -f, --follow       Keep printing lines as files grow, across rotation and truncation
   -n, --lines <n>    Start with the last n lines of each file (10 when following)
+  -l, --level <lvl>  Show only entries at this level or above (trace, debug, info, warn,
+                     error, fatal)
+  -g, --grep <re>    Show only lines matching this pattern; repeat to match any of several
+  -x, --exclude <re> Hide lines matching this pattern; may be repeated
   --color            Always color output
   --no-color         Never color output
   -h, --help         Show this help
@@ -35,6 +45,9 @@ export const parseCliOptions = (arguments_: readonly string[]): CliOptions => {
     color: undefined,
     follow: false,
     lines: undefined,
+    minLevel: undefined,
+    include: [],
+    exclude: [],
     files: [],
   };
   let onlyFiles = false;
@@ -54,6 +67,9 @@ export const parseCliOptions = (arguments_: readonly string[]): CliOptions => {
     else if (argument === '-v' || argument === '--version') options.version = true;
     else if (argument === '-f' || argument === '--follow') options.follow = true;
     else if (name === '-n' || name === '--lines') options.lines = parseCount(name, value());
+    else if (name === '-l' || name === '--level') options.minLevel = parseLevel(value());
+    else if (name === '-g' || name === '--grep') options.include.push(compilePattern(value()));
+    else if (name === '-x' || name === '--exclude') options.exclude.push(compilePattern(value()));
     else if (argument === '--color') options.color = true;
     else if (argument === '--no-color') options.color = false;
     else throw new Error(`Unknown option: ${argument}`);
@@ -75,3 +91,11 @@ const parseCount = (name: string, text: string): number => {
   }
   return count;
 };
+
+const parseLevel = (text: string): Level => {
+  const level = parseLevelName(text);
+  if (level === undefined) {
+    throw new Error(`Unknown level: ${text}. Use one of ${levels.join(', ')}`);
+  }
+  return level;
+};

src/core/filter.ts

@@ -0,0 +1,52 @@
+import type {LogEntry} from './entry.js';
+import {type Level, levelRank} from './levels.js';
+
+export interface FilterOptions {
+  /** Hide entries below this level. */
+  minLevel?: Level;
+  /** Show only entries matching at least one of these patterns. */
+  include?: readonly RegExp[];
+  /** Hide entries matching any of these patterns. */
+  exclude?: readonly RegExp[];
+}
+
+/**
+ * Compile a search pattern. Patterns are regular expressions and are case-insensitive unless
+ * they contain an uppercase letter, so `timeout` finds `Timeout` but `Timeout` does not find
+ * `timeout`.
+ */
+export const compilePattern = (pattern: string): RegExp => {
+  try {
+    return new RegExp(pattern, /\p{Lu}/u.test(pattern) ? 'u' : 'iu');
+  } catch {
+    throw new Error(`Not a valid pattern: ${pattern}`);
+  }
+};
+
+/**
+ * Create a predicate that decides whether an entry is shown.
+ *
+ * Indented lines without a level, such as stack frames, continue the entry above them from the
+ * same source and are shown or hidden with it. Other lines without a level are hidden by
+ * `minLevel`, since their level is not known.
+ */
+export const createFilter = (options: FilterOptions): ((entry: LogEntry) => boolean) => {
+  const {minLevel, include = [], exclude = []} = options;
+  const minimum = minLevel === undefined ? undefined : levelRank(minLevel);
+  const lastShown = new Map<string | undefined, boolean>();
+
+  return (entry) => {
+    const continuation = entry.level === 'unknown' && /^\s/.test(entry.raw);
+    const previous = lastShown.get(entry.source);
+    if (continuation && previous !== undefined) return previous;
+
+    let shown = true;
+    if (minimum !== undefined) {
+      shown = entry.level !== 'unknown' && levelRank(entry.level) >= minimum;
+    }
+    if (shown && include.length > 0) shown = include.some((pattern) => pattern.test(entry.raw));
+    if (shown && exclude.length > 0) shown = !exclude.some((pattern) => pattern.test(entry.raw));
+    lastShown.set(entry.source, shown);
+    return shown;
+  };
+};

src/index.ts

@@ -1,6 +1,8 @@
 export * from './core/entry.js';
+export * from './core/filter.js';
 export * from './core/levels.js';
 export * from './core/lines.js';
+export * from './core/follow.js';
 export * from './core/sanitize.js';
 export * from './core/sources.js';
 export * from './format/plain.js';

tests/filter.test.ts

@@ -0,0 +1,57 @@
+import {describe, expect, it} from 'vitest';
+import {parseLine} from '../src/core/entry.js';
+import {compilePattern, createFilter, type FilterOptions} from '../src/core/filter.js';
+
+const shownLines = (options: FilterOptions, lines: string[], source?: string): string[] => {
+  const shown = createFilter(options);
+  return lines
+    .map((line, index) =>
+      parseLine(line, {seq: index + 1, ...(source === undefined ? {} : {source})}),
+    )
+    .filter(shown)
+    .map((entry) => entry.raw);
+};
+
+describe('createFilter', () => {
+  it('shows everything without options', () => {
+    expect(shownLines({}, ['DEBUG a', 'plain'])).toEqual(['DEBUG a', 'plain']);
+  });
+
+  it('hides entries below the minimum level and lines without a level', () => {
+    expect(
+      shownLines({minLevel: 'warn'}, ['DEBUG a', 'INFO b', 'WARN c', 'ERROR d', 'plain']),
+    ).toEqual(['WARN c', 'ERROR d']);
+  });
+
+  it('keeps indented continuation lines with the entry above them', () => {
+    const lines = [
+      'INFO request done',
+      '    detail of info',
+      'ERROR TypeError: x is undefined',
+      '    at handler (app.js:3:5)',
+      '    at main (app.js:9:1)',
+    ];
+    expect(shownLines({minLevel: 'error'}, lines)).toEqual(lines.slice(2));
+  });
+
+  it('matches any include pattern and drops excluded lines', () => {
+    const options = {
+      include: [compilePattern('timeout'), compilePattern('refused')],
+      exclude: [compilePattern('health')],
+    };
+    expect(
+      shownLines(options, ['Timeout on /users', 'timeout on /health', 'connection refused', 'ok']),
+    ).toEqual(['Timeout on /users', 'connection refused']);
+  });
+});
+
+describe('compilePattern', () => {
+  it('ignores case unless the pattern has uppercase letters', () => {
+    expect(compilePattern('error').test('ERROR')).toBe(true);
+    expect(compilePattern('Error').test('ERROR')).toBe(false);
+  });
+
+  it('explains invalid patterns', () => {
+    expect(() => compilePattern('(')).toThrow('Not a valid pattern: (');
+  });
+});