From 58cca69faed0d1dec573ab8ec18441cd52b77ea4 Mon Sep 17 00:00:00 2001 From: podhmo Date: Fri, 17 Nov 2017 23:30:14 +0900 Subject: [PATCH 1/2] --run-force (only ltsv) --- jqfpy/__main__.py | 6 +++++- jqfpy/loading/_json.py | 2 +- jqfpy/loading/_ltsv.py | 9 +++++++-- jqfpy/loading/_yaml.py | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/jqfpy/__main__.py b/jqfpy/__main__.py index 38d20ad..27566be 100644 --- a/jqfpy/__main__.py +++ b/jqfpy/__main__.py @@ -46,6 +46,7 @@ def main(): parser.add_argument("--squash", action="store_true") parser.add_argument("--show-code", action="store_true") parser.add_argument("--additionals") + parser.add_argument("--run-force", action="store_true") args = parser.parse_args() @@ -74,9 +75,12 @@ def main(): transform_fn = jqfpy.exec_pycode(fnname, pycode) def _load(streams): + errport = None + if args.run_force: + errport = sys.stderr for stream in streams: m = loading.get_module(stream, default_format=args.input_format) - for d in m.load(stream, buffered=args.buffered): + for d in m.load(stream, buffered=args.buffered, errport=errport): yield d def _dump(d, *, i): diff --git a/jqfpy/loading/_json.py b/jqfpy/loading/_json.py index 53c3eb3..5201b84 100644 --- a/jqfpy/loading/_json.py +++ b/jqfpy/loading/_json.py @@ -7,7 +7,7 @@ JSONDecodeError = ValueError # for 3.4 -def load(stream, *, buffered=False): +def load(stream, *, buffered=False, errport=None): if buffered: return _load_buffered(stream) else: diff --git a/jqfpy/loading/_ltsv.py b/jqfpy/loading/_ltsv.py index 4b71a3f..c3fedb5 100644 --- a/jqfpy/loading/_ltsv.py +++ b/jqfpy/loading/_ltsv.py @@ -1,9 +1,14 @@ from collections import OrderedDict -def load(stream, *, buffered=False): +def load(stream, *, buffered=False, errport=None): for line in stream: - yield OrderedDict(pair.split(":", 1) for pair in line.rstrip("\n\r").split("\t")) + try: + yield OrderedDict(pair.split(":", 1) for pair in line.rstrip("\n\r").split("\t")) + except Exception: + if errport is None: + raise + errport.write(line) def dump(d, fp, *, squash=False, raw=False, extra_kwargs=None): diff --git a/jqfpy/loading/_yaml.py b/jqfpy/loading/_yaml.py index 024b02b..9b35f6e 100644 --- a/jqfpy/loading/_yaml.py +++ b/jqfpy/loading/_yaml.py @@ -26,7 +26,7 @@ def _represent_str(dumper, instance): return dumper.represent_scalar('tag:yaml.org,2002:str', instance) -def load(stream, *, buffered=False): +def load(stream, *, buffered=False, errport=None): return yaml.load_all(stream, Loader=Loader) From 8284e62ed80e99f9bdabbe96eba5ac61bdec2686 Mon Sep 17 00:00:00 2001 From: podhmo Date: Fri, 17 Nov 2017 23:31:55 +0900 Subject: [PATCH 2/2] examples --- examples/ltsv/04broken.ltsv | 7 +++++ examples/ltsv/04expected.output | 51 +++++++++++++++++++++++++++++++++ examples/ltsv/Makefile | 7 ++++- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 examples/ltsv/04broken.ltsv create mode 100644 examples/ltsv/04expected.output diff --git a/examples/ltsv/04broken.ltsv b/examples/ltsv/04broken.ltsv new file mode 100644 index 0000000..ffc3444 --- /dev/null +++ b/examples/ltsv/04broken.ltsv @@ -0,0 +1,7 @@ +host:127.0.0.1 ident:- user:frank time:[10/Oct/2000:13:55:36 -0700] req:GET /apache_pb.gif HTTP/1.0 status:200 size:2326 referer:http://www.example.com/start.html ua:Mozilla/4.08 [en] (Win98; I ;Nav) +host:127.0.0.1 ident:- user:frank time:[10/Oct/2000:13:55:37 -0700] req:GET /apache_pb.gif HTTP/1.0 status:200 size:2326 referer:http://www.example.com/start.html ua:Mozilla/4.08 [en] (Win98; I ;Nav) +INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): httpbin.org +DEBUG:requests.packages.urllib3.connectionpool:"GET /get?foo=bar&baz=python HTTP/1.1" 200 353 +@ +host:127.0.0.1 ident:- user:frank time:[10/Oct/2000:13:55:38 -0700] req:GET /apache_pb.gif HTTP/1.0 status:200 size:2326 referer:http://www.example.com/start.html ua:Mozilla/4.08 [en] (Win98; I ;Nav) +host:127.0.0.1 ident:- user:frank time:[10/Oct/2000:13:55:39 -0700] req:GET /apache_pb.gif HTTP/1.0 status:200 size:2326 referer:http://www.example.com/start.html ua:Mozilla/4.08 [en] (Win98; I ;Nav) \ No newline at end of file diff --git a/examples/ltsv/04expected.output b/examples/ltsv/04expected.output new file mode 100644 index 0000000..28be084 --- /dev/null +++ b/examples/ltsv/04expected.output @@ -0,0 +1,51 @@ +{ + "host": "127.0.0.1", + "ident": "-", + "referer": "http://www.example.com/start.html", + "req": "GET /apache_pb.gif HTTP/1.0", + "size": "2326", + "status": "200", + "time": "[10/Oct/2000:13:55:36 -0700]", + "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)", + "user": "frank" +} +{ + "host": "127.0.0.1", + "ident": "-", + "referer": "http://www.example.com/start.html", + "req": "GET /apache_pb.gif HTTP/1.0", + "size": "2326", + "status": "200", + "time": "[10/Oct/2000:13:55:37 -0700]", + "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)", + "user": "frank" +} +{ + "INFO": "requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): httpbin.org" +} +{ + "DEBUG": "requests.packages.urllib3.connectionpool:\"GET /get?foo=bar&baz=python HTTP/1.1\" 200 353" +} +{ + "host": "127.0.0.1", + "ident": "-", + "referer": "http://www.example.com/start.html", + "req": "GET /apache_pb.gif HTTP/1.0", + "size": "2326", + "status": "200", + "time": "[10/Oct/2000:13:55:38 -0700]", + "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)", + "user": "frank" +} +{ + "host": "127.0.0.1", + "ident": "-", + "referer": "http://www.example.com/start.html", + "req": "GET /apache_pb.gif HTTP/1.0", + "size": "2326", + "status": "200", + "time": "[10/Oct/2000:13:55:39 -0700]", + "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)", + "user": "frank" +} +@ diff --git a/examples/ltsv/Makefile b/examples/ltsv/Makefile index feaf073..3dcf431 100644 --- a/examples/ltsv/Makefile +++ b/examples/ltsv/Makefile @@ -1,6 +1,6 @@ QJ = python -m jqfpy -S -default: 00 01 02 03 +default: 00 01 02 03 04 00: ${QJ} 00log.ltsv > .qj.output0 @@ -17,3 +17,8 @@ default: 00 01 02 03 03: ${QJ} 'h.omit("time", "req", "referer")' 00log.ltsv > .qj.output3 diff -u 03expected.json .qj.output3 + +04: + ${QJ} --run-force 04broken.ltsv > .qj.output4 2>&1 + diff -u 04expected.output .qj.output4 +