Force option

examples/ltsv/04broken.ltsv

@@ -0,0 +1,7 @@
+host:127.0.0.1	ident:-	user:frank	time:[10/Oct/2000:13:55:36 -0700]	req:GET /apache_pb.gif HTTP/1.0	status:200	size:2326	referer:http://www.example.com/start.html	ua:Mozilla/4.08 [en] (Win98; I ;Nav)
+host:127.0.0.1	ident:-	user:frank	time:[10/Oct/2000:13:55:37 -0700]	req:GET /apache_pb.gif HTTP/1.0	status:200	size:2326	referer:http://www.example.com/start.html	ua:Mozilla/4.08 [en] (Win98; I ;Nav)
+INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): httpbin.org
+DEBUG:requests.packages.urllib3.connectionpool:"GET /get?foo=bar&baz=python HTTP/1.1" 200 353
+@
+host:127.0.0.1	ident:-	user:frank	time:[10/Oct/2000:13:55:38 -0700]	req:GET /apache_pb.gif HTTP/1.0	status:200	size:2326	referer:http://www.example.com/start.html	ua:Mozilla/4.08 [en] (Win98; I ;Nav)
+host:127.0.0.1	ident:-	user:frank	time:[10/Oct/2000:13:55:39 -0700]	req:GET /apache_pb.gif HTTP/1.0	status:200	size:2326	referer:http://www.example.com/start.html	ua:Mozilla/4.08 [en] (Win98; I ;Nav)

examples/ltsv/04expected.output

@@ -0,0 +1,51 @@
+{
+  "host": "127.0.0.1",
+  "ident": "-",
+  "referer": "http://www.example.com/start.html",
+  "req": "GET /apache_pb.gif HTTP/1.0",
+  "size": "2326",
+  "status": "200",
+  "time": "[10/Oct/2000:13:55:36 -0700]",
+  "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)",
+  "user": "frank"
+}
+{
+  "host": "127.0.0.1",
+  "ident": "-",
+  "referer": "http://www.example.com/start.html",
+  "req": "GET /apache_pb.gif HTTP/1.0",
+  "size": "2326",
+  "status": "200",
+  "time": "[10/Oct/2000:13:55:37 -0700]",
+  "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)",
+  "user": "frank"
+}
+{
+  "INFO": "requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): httpbin.org"
+}
+{
+  "DEBUG": "requests.packages.urllib3.connectionpool:\"GET /get?foo=bar&baz=python HTTP/1.1\" 200 353"
+}
+{
+  "host": "127.0.0.1",
+  "ident": "-",
+  "referer": "http://www.example.com/start.html",
+  "req": "GET /apache_pb.gif HTTP/1.0",
+  "size": "2326",
+  "status": "200",
+  "time": "[10/Oct/2000:13:55:38 -0700]",
+  "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)",
+  "user": "frank"
+}
+{
+  "host": "127.0.0.1",
+  "ident": "-",
+  "referer": "http://www.example.com/start.html",
+  "req": "GET /apache_pb.gif HTTP/1.0",
+  "size": "2326",
+  "status": "200",
+  "time": "[10/Oct/2000:13:55:39 -0700]",
+  "ua": "Mozilla/4.08 [en] (Win98; I ;Nav)",
+  "user": "frank"
+}
+@

examples/ltsv/Makefile

@@ -1,6 +1,6 @@
 QJ = python -m jqfpy -S
 
-default: 00 01 02 03
+default: 00 01 02 03 04
 
 00:
 	${QJ} 00log.ltsv > .qj.output0
@@ -17,3 +17,8 @@ default: 00 01 02 03
 03:
 	${QJ} 'h.omit("time", "req", "referer")' 00log.ltsv > .qj.output3
 	diff -u 03expected.json .qj.output3
+
+04:
+	${QJ} --run-force 04broken.ltsv > .qj.output4 2>&1
+	diff -u 04expected.output .qj.output4
+

jqfpy/__main__.py

@@ -46,6 +46,7 @@ def main():
     parser.add_argument("--squash", action="store_true")
     parser.add_argument("--show-code", action="store_true")
     parser.add_argument("--additionals")
+    parser.add_argument("--run-force", action="store_true")
 
     args = parser.parse_args()
 
@@ -74,9 +75,12 @@ def main():
         transform_fn = jqfpy.exec_pycode(fnname, pycode)
 
     def _load(streams):
+        errport = None
+        if args.run_force:
+            errport = sys.stderr
         for stream in streams:
             m = loading.get_module(stream, default_format=args.input_format)
-            for d in m.load(stream, buffered=args.buffered):
+            for d in m.load(stream, buffered=args.buffered, errport=errport):
                 yield d
 
     def _dump(d, *, i):

jqfpy/loading/_json.py

@@ -7,7 +7,7 @@
     JSONDecodeError = ValueError  # for 3.4
 
 
-def load(stream, *, buffered=False):
+def load(stream, *, buffered=False, errport=None):
     if buffered:
         return _load_buffered(stream)
     else:

jqfpy/loading/_ltsv.py

@@ -1,9 +1,14 @@
 from collections import OrderedDict
 
 
-def load(stream, *, buffered=False):
+def load(stream, *, buffered=False, errport=None):
     for line in stream:
-        yield OrderedDict(pair.split(":", 1) for pair in line.rstrip("\n\r").split("\t"))
+        try:
+            yield OrderedDict(pair.split(":", 1) for pair in line.rstrip("\n\r").split("\t"))
+        except Exception:
+            if errport is None:
+                raise
+            errport.write(line)
 
 
 def dump(d, fp, *, squash=False, raw=False, extra_kwargs=None):

jqfpy/loading/_yaml.py

@@ -26,7 +26,7 @@ def _represent_str(dumper, instance):
         return dumper.represent_scalar('tag:yaml.org,2002:str', instance)
 
 
-def load(stream, *, buffered=False):
+def load(stream, *, buffered=False, errport=None):
     return yaml.load_all(stream, Loader=Loader)