From 568409d7806f5c2ba8991626035be3e276ea6a59 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Tue, 23 Jul 2024 16:46:11 -0400 Subject: [PATCH 01/31] commit common --- tickplant/common/schema.q | 17 ++ tickplant/common/sp500.txt | 500 +++++++++++++++++++++++++++++++++++++ 2 files changed, 517 insertions(+) create mode 100644 tickplant/common/schema.q create mode 100644 tickplant/common/sp500.txt diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q new file mode 100644 index 0000000..d5562ec --- /dev/null +++ b/tickplant/common/schema.q @@ -0,0 +1,17 @@ +trade:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$(); tradeid:`guid$()); + +quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsize:`float$();asize:`float$();aprice:`float$()); + +order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();orderid:`guid$()); + +syms:`$'read0`:sp500.txt; + +exchs:`NYSE`NASDAQ`AMEX; + +sides:`B`S; + +//generate random data +genTrades:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; + +// //quote +genQuotes:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?10.;n?10.)}; diff --git a/tickplant/common/sp500.txt b/tickplant/common/sp500.txt new file mode 100644 index 0000000..f66cc11 --- /dev/null +++ b/tickplant/common/sp500.txt @@ -0,0 +1,500 @@ +A.N +AA.N +AAPL.O +ABC.N +ABI.N +ABK.N +ABS.N +ABT.N +ACE.N +ACS.N +ACV.N +ADBE.O +ADCT.O +ADI.N +ADM.N +ADP.N +ADSK.O +AEE.N +AEP.N +AES.N +AET.N +AFL.N +AGN.N +AHC.N +AIG.N +AIV.N +ALL.N +ALTR.O +AMAT.O +AMCC.O +AMD.N +AMGN.O +AMP.N +AMZN.O +AN.N +ANDW.O +AOC.N +APA.N +APC.N +APCC.O +APD.N +APOL.O +ASD.N +ASH.N +ASN.N +ASO.N +AT.N +ATI.N +AV.N +AVP.N +AVY.N +AW.N +AXP.N +AYE.N +AZO.N +BA.N +BAC.N +BAX.N +BBBY.O +BBT.N +BBY.N +BC.N +BCR.N +BDK.N +BDX.N +BEN.N +BFb.N +BHI.N +BIIB.O +BJS.N +BK.N +BLI.N +BLL.N +BLS.N +BMC.N +BMET.O +BMS.N +BMY.N +BNI.N +BOL.N +BR.N +BRCM.O +BSC.N +BSX.N +BUD.N +C.N +CA.N +CAG.N +CAH.N +CAT.N +CB.N +CBE.N +CBSS.O +CC.N +CCE.N +CCL.N +CCU.N +CD.N +CEG.N +CFC.N +CHIR.O +CI.N +CIEN.O +CIN.N +CINF.O +CIT.N +CL.N +CLX.N +CMA.N +CMCSA.O +CMI.N +CMS.N +CMVT.O +CMX.N +CNP.N +COF.N +COH.N +COL.N +COP.N +COST.O +CPB.N +CPN.N +CPWR.O +CSC.N +CSCO.O +CSX.N +CTAS.O +CTB.N +CTL.N +CTX.N +CTXS.O +CVG.N +CVH.N +CVS.N +CVX.N +CZN.N +D.N +DCN.N +DD.N +DDS.N +DE.N +DELL.O +DG.N +DGX.N +DHI.N +DHR.N +DIS.N +DJ.N +DOV.N +DOW.N +DRI.N +DTE.N +DUK.N +DVN.N +DYN.N +EBAY.O +EC.N +ECL.N +ED.N +EDS.N +EFX.N +EIX.N +EK.N +EMC.N +EMN.N +EMR.N +EOG.N +EOP.N +EP.N +EQR.N +ERTS.O +ESRX.O +ET.N +ETN.N +ETR.N +EXC.N +F.N +FCX.N +FD.N +FDC.N +FDO.N +FDX.N +FE.N +FHN.N +FII.N +FISV.O +FITB.O +FLR.N +FNM.N +FO.N +FPL.N +FRE.N +FRX.N +FSH.N +FSLb.N +GAS.N +GCI.N +GD.N +GDT.N +GDW.N +GE.N +GENZ.O +GILD.O +GIS.N +GLW.N +GM.N +GP.N +GPC.N +GPS.N +GR.N +GS.N +GT.N +GTW.N +GWW.N +HAL.N +HAS.N +HBAN.O +HCA.N +HCR.N +HD.N +HDI.N +HET.N +HIG.N +HLT.N +HMA.N +HNZ.N +HON.N +HOT.N +HPC.N +HPQ.N +HRB.N +HSP.N +HSY.N +HUM.N +IBM.N +IFF.N +IGT.N +INTC.O +INTU.O +IP.N +IPG.N +IR.N +ITT.N +ITW.N +JBL.N +JCI.N +JCP.N +JDSU.O +JNJ.N +JNS.N +JNY.N +JP.N +JPM.N +JWN.N +K.N +KBH.N +KEY.N +KG.N +KLAC.O +KMB.N +KMG.N +KMI.N +KO.N +KR.N +KRB.N +KRI.N +KSE.N +KSS.N +LEG.N +LEH.N +LEN.N +LH.N +LIZ.N +LLL.N +LLTC.O +LLY.N +LMT.N +LNC.N +LOW.N +LPX.N +LSI.N +LTD.N +LTR.N +LU.N +LUV.N +LXK.N +MAR.N +MAS.N +MAT.N +MBI.N +MCD.N +MCK.N +MCO.N +MDP.N +MDT.N +MEDI.O +MEL.N +MER.N +MERQE.O +MET.N +MHP.N +MHS.N +MI.N +MIL.N +MKC.N +MMC.N +MMM.N +MNST.O +MO.N +MOLX.O +MON.N +MOT.N +MRK.N +MRO.N +MSFT.O +MTB.N +MTG.N +MU.N +MUR.N +MWD.N +MWV.N +MXIM.O +MYG.N +MYL.N +NAV.N +NBR.N +NCC.N +NCR.N +NE.N +NEM.N +NFB.N +NI.N +NKE.N +NOC.N +NOV.N +NOVL.O +NSC.N +NSM.N +NTAP.O +NTRS.O +NUE.N +NVDA.O +NVLS.O +NWL.N +NWSa.N +NYT.N +ODP.N +OMC.N +OMX.N +ORCL.O +OXY.N +PAYX.O +PBG.N +PBI.N +PCAR.O +PCG.N +PCL.N +PD.N +PDCO.O +PEG.N +PEP.N +PFE.N +PFG.N +PG.N +PGL.N +PGN.N +PGR.N +PH.N +PHM.N +PKI.N +PLD.N +PLL.N +PMCS.O +PMTC.O +PNC.N +PNW.N +PPG.N +PPL.N +PRU.N +PSA.N +PTV.N +PX.N +Q.N +QCOM.O +QLGC.O +R.N +RAI.N +RBK.N +RDC.N +RF.N +RHI.N +RIG.N +ROH.N +ROK.N +RRD.N +RSH.N +RTN.N +RX.N +S.N +SAFC.O +SANM.O +SBL.N +SBUX.O +SCH.N +SEBL.O +SEE.N +SFA.N +SGP.N +SHLD.O +SHW.N +SIAL.O +SLB.N +SLE.N +SLM.N +SLR.N +SNA.N +SNV.N +SO.N +SOV.N +SPG.N +SPLS.O +SRE.N +STA.N +STI.N +STJ.N +STT.N +STZ.N +SUN.N +SUNW.O +SVU.N +SWK.N +SWY.N +SYK.N +SYMC.O +SYY.N +T.N +TAP.N +TE.N +TEK.N +TER.N +TGT.N +THC.N +TIF.N +TIN.N +TJX.N +TLAB.O +TMK.N +TMO.N +TRB.N +TROW.O +TSG.N +TSN.N +TWX.N +TXN.N +TXT.N +TXU.N +TYC.N +UIS.N +UNH.N +UNM.N +UNP.N +UPS.N +USB.N +UST.N +UTX.N +UVN.N +VC.N +VFC.N +VIAb.N +VLO.N +VMC.N +VNO.N +VZ.N +WAG.N +WAT.N +WB.N +WEN.N +WFC.N +WFT.N +WHR.N +WLP.N +WM.N +WMB.N +WMI.N +WMT.N +WPI.N +WWY.N +WY.N +WYE.N +X.N +XEL.N +XL.N +XLNX.O +XOM.N +XRX.N +XTO.N +YHOO.O +YUM.N +ZION.O +ZMH.N From 28133f98d4000b91d122983d4a4026631c82a1bf Mon Sep 17 00:00:00 2001 From: quercussuber Date: Tue, 23 Jul 2024 17:00:01 -0400 Subject: [PATCH 02/31] gitignore and tp --- .gitignore | 2 + tickplant/common/schema.q | 2 +- tickplant/tick.q | 82 +++++++++++++++++++++++++++++++++++++++ tickplant/tick/r.q | 41 ++++++++++++++++++++ tickplant/tick/u.q | 21 ++++++++++ 5 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 tickplant/tick.q create mode 100644 tickplant/tick/r.q create mode 100644 tickplant/tick/u.q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..89fbaff --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +tplog_* diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index d5562ec..5e735bb 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -4,7 +4,7 @@ quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsiz order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();orderid:`guid$()); -syms:`$'read0`:sp500.txt; +syms:`$'read0`:common/sp500.txt; exchs:`NYSE`NASDAQ`AMEX; diff --git a/tickplant/tick.q b/tickplant/tick.q new file mode 100644 index 0000000..4e64853 --- /dev/null +++ b/tickplant/tick.q @@ -0,0 +1,82 @@ +/ q tick.q sym . -p 5001 foo 2>&1 & +/2014.03.12 remove license check +/2013.09.05 warn on corrupt log +/2013.08.14 allow when -u is set +/2012.11.09 use timestamp type rather than time. -19h/"t"/.z.Z -> -16h/"n"/.z.P +/2011.02.10 i->i,j to avoid duplicate data if subscription whilst data in buffer +/2009.07.30 ts day (and "d"$a instead of floor a) +/2008.09.09 .k -> .q, 2.4 +/2008.02.03 tick/r.k allow no log +/2007.09.03 check one day flip +/2006.10.18 check type? +/2006.07.24 pub then log +/2006.02.09 fix(2005.11.28) .z.ts end-of-day +/2006.01.05 @[;`sym;`g#] in tick.k load +/2005.12.21 tick/r.k reset `g#sym +/2005.12.11 feed can send .u.endofday +/2005.11.28 zero-end-of-day +/2005.10.28 allow`time on incoming +/2005.10.10 zero latency +/2005.09.30 .u.i->.u.j + +"kdb+tick date:",string[.z.P]," version:",string[.z.K],"_",string .z.k + +/q tick.q SRC [DST] [-p 5010] [-o h] +system"l common/schema.q" + +if[not system"p";system"p 5010"] + +\l tick/u.q +\d .u +INSTANCE:"one"; +LOGSUFFIX:"tplog"; + +ld:{if[not type key L::`$(-10_string L),string x;.[L;();:;()]];i::j::-11!(-2;L);if[0<=type i;-2 (string L)," is a corrupt log. Truncate to length ",(string last i)," and restart";exit 1];hopen L}; +tick:{ + init[]; + if[not min(`time`exch`sym~3#key flip value@)each t;'`timeexchym]; + @[;`sym;`g#]each t; + d::.z.D; + if[ l::count x; + L::`$":",x,"/log",10#"."; + l::ld d + ] + }; + +endofday:{end d;d+:1;if[l;hclose l;l::0(`.u.ld;d)]}; +ts:{if[dtype first x;a,x;(enlist(count first x)#a),x]]; + t insert x;if[l;l enlist (`upd;t;x);j+:1];}]; + +if[not system"t";system"t 1000"; + .z.ts:{ts .z.D}; + upd:{[t;x]ts"d"$a:.z.P; + if[not -12=type first first x;a:"n"$a;x:$[0>type first x;a,x;(enlist(count first x)#a),x]]; + f:key flip value t;pub[t;$[0>type first x;enlist f!x;flip f!x]];if[l;l enlist (`upd;t;x);i+:1];}]; + +\d . +.u.tick[.u.LOGSUFFIX,"_",.u.INSTANCE]; + +\ + globals used + .u.w - dictionary of tables->(handle;syms) + .u.i - msg count in log file + .u.j - total msg count (log file plus those held in buffer) + .u.t - table names + .u.L - tp log filename, e.g. `:./sym2008.09.11 + .u.l - handle to tp log file + .u.d - date + +/test +>q tick.q +>q tick/ssl.q + +/run +>q tick.q sym . -p 5010 /tick +>q tick/r.q :5010 -p 5011 /rdb +>q sym -p 5012 /hdb +>q tick/ssl.q sym :5010 /feed diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q new file mode 100644 index 0000000..23113a3 --- /dev/null +++ b/tickplant/tick/r.q @@ -0,0 +1,41 @@ +/q tick/r.q [host]:port[:usr:pwd] [host]:port[:usr:pwd] +/2008.09.09 .k ->.q + +if[not "w"=first string .z.o;system "sleep 1"]; + +upd:insert; + + +/ get the ticker plant and history ports, defaults are 5010,5012 +.u.x:`tick`hdb!.z.x,(count .z.x)_(":5010";":5012"); +.u.conn:{hopen `$":",x}each .u.x; + +/ end of day: save, clear, hdb reload +.u.end:{t:tables`.;t@:where `g=attr each t@\:`sym;.Q.hdpf[.u.x `hdb;`:.;x;`sym];@[;`sym;`g#] each t;}; + +/ init schema and sync up from log file;cd to hdb(so client save can run) +// TODO TP SHOULD pass logfile path +.u.rep:{(.[;();:;].)each x;if[null first y;:()];-11!y;}; //system "cd ",1_first "/" vs string last y;}; +/ HARDCODE \cd if other than logdir/db + +/ connect to ticker plant for (schema;(logcount;log)) +// All syms, all tables +// .u.x 0 -> ticker +.u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"; + +.z.pc:{show "Process at handle ",string[p:.u.conn ? x]," disconnected"; .u.pc p} + +.u.pc:{.u.conn[x]:0Ni; show .u.conn} + +.u.ts:{ + if[ null .u.conn`tick; + show "Retrying tickerplant connection ..."; + h:@[hopen;`$":",.u.x`tick;{show "Cannot establish tickerplant connection with ",x; 0Ni}]; + .u.conn[`tick]: h; + if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"]; + ]; + } + +.z.ts:{.u.ts[]} + +\t 5000 \ No newline at end of file diff --git a/tickplant/tick/u.q b/tickplant/tick/u.q new file mode 100644 index 0000000..e827405 --- /dev/null +++ b/tickplant/tick/u.q @@ -0,0 +1,21 @@ +/2019.06.17 ensure sym has g attr for schema returned to new subscriber +/2008.09.09 .k -> .q +/2006.05.08 add +// +\d .u + +init:{w::t!(count t::tables`.)#()} + +del:{w[x]_:w[x;;0]?y};.z.pc:{del[;x]each t}; + +sel:{$[`~y;x;select from x where sym in y]} + +pub:{[t;x]{[t;x;w]if[count x:sel[x]w 1;(neg first w)(`upd;t;x)]}[t;x]each w t} + +add:{$[(count w x)>i:w[x;;0]?.z.w;.[`.u.w;(x;i;1);union;y];w[x],:enlist(.z.w;y)];(x;$[99=type v:value x;sel[v]y;@[0#v;`sym;`g#]])} + +// Sub should return information about the subscription +sub:{if[x~`;:sub[;y]each t];if[not x in t;'x];del[x].z.w;add[x;y]} + +end:{(neg union/[w[;;0]])@\:(`.u.end;x)} + From bd1e223999d96de4520e78c78e0f39db53129894 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Tue, 23 Jul 2024 23:21:39 -0400 Subject: [PATCH 03/31] fix tplog path and ignore --- .gitignore | 3 +-- tickplant/tick.q | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 89fbaff..c316d35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ - -tplog_* +tplog_* \ No newline at end of file diff --git a/tickplant/tick.q b/tickplant/tick.q index 4e64853..c00c705 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -30,6 +30,7 @@ if[not system"p";system"p 5010"] \d .u INSTANCE:"one"; LOGSUFFIX:"tplog"; +ROOT:"../" ld:{if[not type key L::`$(-10_string L),string x;.[L;();:;()]];i::j::-11!(-2;L);if[0<=type i;-2 (string L)," is a corrupt log. Truncate to length ",(string last i)," and restart";exit 1];hopen L}; tick:{ @@ -38,7 +39,7 @@ tick:{ @[;`sym;`g#]each t; d::.z.D; if[ l::count x; - L::`$":",x,"/log",10#"."; + L::`$":",ROOT,x,"/log",10#"."; l::ld d ] }; @@ -60,6 +61,9 @@ if[not system"t";system"t 1000"; \d . .u.tick[.u.LOGSUFFIX,"_",.u.INSTANCE]; +// Chain TS + +.z.ts:{`order insert genOrders[10;x]; y x}[;oldts:.z.ts] \ globals used From 1f3bbce8c8c08da8be458dfcd5a25c92b793c51b Mon Sep 17 00:00:00 2001 From: quercussuber Date: Tue, 23 Jul 2024 23:29:21 -0400 Subject: [PATCH 04/31] lightweight trading simulator nocontext --- tickplant/common/schema.q | 19 ++++++++++++++++++- tickplant/common/sim.q | 7 +++++++ tickplant/tick/r.q | 2 ++ tickplant/tick/u.q | 2 ++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tickplant/common/sim.q diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index 5e735bb..2f3e86a 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -2,7 +2,7 @@ trade:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size: quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsize:`float$();asize:`float$();aprice:`float$()); -order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();orderid:`guid$()); +order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); syms:`$'read0`:common/sp500.txt; @@ -15,3 +15,20 @@ genTrades:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?si // //quote genQuotes:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?10.;n?10.)}; + +// order +genOrders:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; + +// +fillOrders:{`trade insert x} + + + + +// tradeOrders:{ +// if[not `order in tables[]; :(::)]; +// `tradeable upsert order; +// tradeable: update traded:0b from tradeable; +// tofill:neg[x]?tradeable +// update traded:1b from `tradeable where orderid in tofill`orderid; +// } \ No newline at end of file diff --git a/tickplant/common/sim.q b/tickplant/common/sim.q new file mode 100644 index 0000000..9bfab33 --- /dev/null +++ b/tickplant/common/sim.q @@ -0,0 +1,7 @@ +// load the simulation context +// not switching namespace to avoid finangling with tables + +// random X trades, doenst flip side or anything, just a way to +// generate some trades that can be joined with order +.sim.aggressOrder:{.u.conn[`tick](`.u.ins;`trade;neg[x]?order)} + diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q index 23113a3..9873dc3 100644 --- a/tickplant/tick/r.q +++ b/tickplant/tick/r.q @@ -1,5 +1,7 @@ /q tick/r.q [host]:port[:usr:pwd] [host]:port[:usr:pwd] /2008.09.09 .k ->.q +system"l common/schema.q" +system"l common/sim.q" if[not "w"=first string .z.o;system "sleep 1"]; diff --git a/tickplant/tick/u.q b/tickplant/tick/u.q index e827405..acfcbbf 100644 --- a/tickplant/tick/u.q +++ b/tickplant/tick/u.q @@ -19,3 +19,5 @@ sub:{if[x~`;:sub[;y]each t];if[not x in t;'x];del[x].z.w;add[x;y]} end:{(neg union/[w[;;0]])@\:(`.u.end;x)} +// simulate some trades happening +ins:{[t;d] if[not t in tables`.;'`unknown_table];t insert select time:.z.p, exch, sym, price, size, side, tradeid:neg[count d]?0Ng from d} From 8196b784281bc7f4e4ea489717e3a8f8df7e77e7 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Wed, 24 Jul 2024 16:37:41 -0400 Subject: [PATCH 05/31] Lightweight simulation engine --- tickplant/cep/cep.q | 11 +++++++++++ tickplant/common/sim.q | 3 ++- tickplant/tick.q | 4 ++-- tickplant/tick/r.q | 6 +----- tickplant/tick/u.q | 2 +- todo | 3 +++ 6 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 tickplant/cep/cep.q create mode 100644 todo diff --git a/tickplant/cep/cep.q b/tickplant/cep/cep.q new file mode 100644 index 0000000..8e62946 --- /dev/null +++ b/tickplant/cep/cep.q @@ -0,0 +1,11 @@ +/q cep/cep.q SRC :5010 + +system"l tick/r.q" +system"l common/sim.q" +system"l cep/lib.q" + +"kdb+tick CEP SIM date:",string[.z.P]," version:",string[.z.K],"_",string .z.k +.z.ts:{ + .u.ts[]; + .sim.ts mod[;10]`int$`second$x; + } \ No newline at end of file diff --git a/tickplant/common/sim.q b/tickplant/common/sim.q index 9bfab33..11efb67 100644 --- a/tickplant/common/sim.q +++ b/tickplant/common/sim.q @@ -1,7 +1,8 @@ // load the simulation context // not switching namespace to avoid finangling with tables - +if[not `conn in key`.u; 'udotqmissing] // random X trades, doenst flip side or anything, just a way to // generate some trades that can be joined with order .sim.aggressOrder:{.u.conn[`tick](`.u.ins;`trade;neg[x]?order)} +.sim.ts:{[n].sim.aggressOrder n}; \ No newline at end of file diff --git a/tickplant/tick.q b/tickplant/tick.q index c00c705..a51e807 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -19,7 +19,7 @@ /2005.10.10 zero latency /2005.09.30 .u.i->.u.j -"kdb+tick date:",string[.z.P]," version:",string[.z.K],"_",string .z.k +"kdb+tick TICK date:",string[.z.P]," version:",string[.z.K],"_",string .z.k /q tick.q SRC [DST] [-p 5010] [-o h] system"l common/schema.q" @@ -63,7 +63,7 @@ if[not system"t";system"t 1000"; .u.tick[.u.LOGSUFFIX,"_",.u.INSTANCE]; // Chain TS -.z.ts:{`order insert genOrders[10;x]; y x}[;oldts:.z.ts] +.z.ts:{.u.upd[`order;genOrders[10;x]]; y x}[;oldts:.z.ts] \ globals used diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q index 9873dc3..a194c84 100644 --- a/tickplant/tick/r.q +++ b/tickplant/tick/r.q @@ -1,7 +1,6 @@ /q tick/r.q [host]:port[:usr:pwd] [host]:port[:usr:pwd] /2008.09.09 .k ->.q system"l common/schema.q" -system"l common/sim.q" if[not "w"=first string .z.o;system "sleep 1"]; @@ -37,7 +36,4 @@ upd:insert; if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"]; ]; } - -.z.ts:{.u.ts[]} - -\t 5000 \ No newline at end of file + \ No newline at end of file diff --git a/tickplant/tick/u.q b/tickplant/tick/u.q index acfcbbf..a682aa6 100644 --- a/tickplant/tick/u.q +++ b/tickplant/tick/u.q @@ -20,4 +20,4 @@ sub:{if[x~`;:sub[;y]each t];if[not x in t;'x];del[x].z.w;add[x;y]} end:{(neg union/[w[;;0]])@\:(`.u.end;x)} // simulate some trades happening -ins:{[t;d] if[not t in tables`.;'`unknown_table];t insert select time:.z.p, exch, sym, price, size, side, tradeid:neg[count d]?0Ng from d} +ins:{[t;d] if[not t in tables`.;'`unknown_table]; upd[t]select time:.z.p, exch, sym, price, size, side, tradeid:neg[count d]?0Ng from d} diff --git a/todo b/todo new file mode 100644 index 0000000..c4aba0c --- /dev/null +++ b/todo @@ -0,0 +1,3 @@ +optimal attributes +add log lines for connections established/dropped +fix hdb root \ No newline at end of file From 4aebccf1043d07d68542e4fda9da6b31d6f9834f Mon Sep 17 00:00:00 2001 From: quercussuber Date: Wed, 24 Jul 2024 17:12:54 -0400 Subject: [PATCH 06/31] cleanup dead code --- tickplant/common/schema.q | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index 2f3e86a..cd4021f 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -17,18 +17,4 @@ genTrades:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?si genQuotes:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?10.;n?10.)}; // order -genOrders:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; - -// -fillOrders:{`trade insert x} - - - - -// tradeOrders:{ -// if[not `order in tables[]; :(::)]; -// `tradeable upsert order; -// tradeable: update traded:0b from tradeable; -// tofill:neg[x]?tradeable -// update traded:1b from `tradeable where orderid in tofill`orderid; -// } \ No newline at end of file +genOrders:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; \ No newline at end of file From 7e394cac8de9ccbf6c07274793121d944d771a38 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Wed, 24 Jul 2024 17:13:01 -0400 Subject: [PATCH 07/31] ignore db --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c316d35..8352ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -tplog_* \ No newline at end of file +tplog_* +db/ \ No newline at end of file From 4c9f162fa06366a69b504f9f9b58805f8eaf34de Mon Sep 17 00:00:00 2001 From: quercussuber Date: Wed, 24 Jul 2024 17:13:15 -0400 Subject: [PATCH 08/31] move db config setting --- tickplant/tick/r.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q index a194c84..5c0f9cc 100644 --- a/tickplant/tick/r.q +++ b/tickplant/tick/r.q @@ -12,7 +12,8 @@ upd:insert; .u.conn:{hopen `$":",x}each .u.x; / end of day: save, clear, hdb reload -.u.end:{t:tables`.;t@:where `g=attr each t@\:`sym;.Q.hdpf[.u.x `hdb;`:.;x;`sym];@[;`sym;`g#] each t;}; +HDBROOT:`$":../db" +.u.end:{t:tables`.;t@:where `g=attr each t@\:`sym;.Q.hdpf[.u.x `hdb;HDBROOT;x;`sym];@[;`sym;`g#] each t;}; / init schema and sync up from log file;cd to hdb(so client save can run) // TODO TP SHOULD pass logfile path @@ -36,4 +37,3 @@ upd:insert; if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"]; ]; } - \ No newline at end of file From 816a08b2ab30555eb808fb57837b2b863fd1ee57 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Wed, 24 Jul 2024 17:13:38 -0400 Subject: [PATCH 09/31] lightweight ohlc calcs in rt and eod --- tickplant/cep/cep.q | 8 ++++++++ tickplant/cep/lib.q | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tickplant/cep/lib.q diff --git a/tickplant/cep/cep.q b/tickplant/cep/cep.q index 8e62946..97f46f3 100644 --- a/tickplant/cep/cep.q +++ b/tickplant/cep/cep.q @@ -4,8 +4,16 @@ system"l tick/r.q" system"l common/sim.q" system"l cep/lib.q" +uend:.u.end; + +.u.end:{ + .cep.end x; + uend x; + } + "kdb+tick CEP SIM date:",string[.z.P]," version:",string[.z.K],"_",string .z.k .z.ts:{ .u.ts[]; .sim.ts mod[;10]`int$`second$x; + .cep.ts[]; } \ No newline at end of file diff --git a/tickplant/cep/lib.q b/tickplant/cep/lib.q new file mode 100644 index 0000000..52b1a08 --- /dev/null +++ b/tickplant/cep/lib.q @@ -0,0 +1,33 @@ +calcOHLCVV_1m:{ + update `g#sym from + 0!select + open: first price, + high: max price, + low: min price, + close: last price, + volume: sum size, + vwap: size wavg price + by sym, 1 xbar time.minute + from trade + } + +calcOHLCVV_1d:{ + update `g#sym from + 0!select + open: first price, + high: max price, + low: min price, + close: last price, + volume: sum size, + vwap: size wavg price + by sym + from trade + } + +last_ts:.z.P; + +.cep.ts:{ + if[.z.P-last_ts>=00:01:00.000; ohlcvv_1m:: calcOHLCVV_1m[]]; + } + +.cep.end:{ohlcvv_1m::calcOHLCVV_1m[]; ohlcvv_1d::calcOHLCVV_1d[];} \ No newline at end of file From 1cc2fc7a492301df8ea41f9aef1b9069bb590cd3 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Wed, 24 Jul 2024 18:01:25 -0400 Subject: [PATCH 10/31] console log framework --- tickplant/cep/cep.q | 3 ++- tickplant/common/log.q | 24 ++++++++++++++++++++++++ tickplant/tick.q | 7 +++++-- tickplant/tick/r.q | 1 + 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tickplant/common/log.q diff --git a/tickplant/cep/cep.q b/tickplant/cep/cep.q index 97f46f3..3ea698c 100644 --- a/tickplant/cep/cep.q +++ b/tickplant/cep/cep.q @@ -1,7 +1,7 @@ /q cep/cep.q SRC :5010 - system"l tick/r.q" system"l common/sim.q" +system"l common/log.q" system"l cep/lib.q" uend:.u.end; @@ -13,6 +13,7 @@ uend:.u.end; "kdb+tick CEP SIM date:",string[.z.P]," version:",string[.z.K],"_",string .z.k .z.ts:{ + .log.logInfo"Running CEP ts"; .u.ts[]; .sim.ts mod[;10]`int$`second$x; .cep.ts[]; diff --git a/tickplant/common/log.q b/tickplant/common/log.q new file mode 100644 index 0000000..e7954ff --- /dev/null +++ b/tickplant/common/log.q @@ -0,0 +1,24 @@ +if[`log in key`.;:(`log_already_loaded)]; +\d .log +LEVELS:`info`warning`error`debug + +// Use +// info when you want to log high level state and flow information +// warning when you want to log messages that indicates non-ideal state or potential problems +// error when you want to log meesages that indicate a non-recoverable error +// debug when you want to log a debug message +level: `info + +logmsg:{[level;msg] + if[11h = type msg; msg:string msg]; + 0N!string[.z.P]," ",upper[string level]," ",msg; + } + +logInfo:logmsg[`info] +logWarning:logmsg[`warning] +logError:logmsg[`error] +logDebug:logmsg[`debug] + +// TODO be nice to have both stdout/stderr and file logging +// and also only print to stdout if the level is lower than set up +\d . \ No newline at end of file diff --git a/tickplant/tick.q b/tickplant/tick.q index a51e807..32175e9 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -23,6 +23,7 @@ /q tick.q SRC [DST] [-p 5010] [-o h] system"l common/schema.q" +system"l common/log.q" if[not system"p";system"p 5010"] @@ -44,16 +45,18 @@ tick:{ ] }; -endofday:{end d;d+:1;if[l;hclose l;l::0(`.u.ld;d)]}; +endofday:{.log.logInfo"End of day started"; end d;d+:1;if[l;hclose l;l::0(`.u.ld;d)]}; ts:{if[dtype first x;a,x;(enlist(count first x)#a),x]]; t insert x;if[l;l enlist (`upd;t;x);j+:1];}]; if[not system"t";system"t 1000"; + .log.logInfo"Ticker started in stream mode"; .z.ts:{ts .z.D}; upd:{[t;x]ts"d"$a:.z.P; if[not -12=type first first x;a:"n"$a;x:$[0>type first x;a,x;(enlist(count first x)#a),x]]; @@ -63,7 +66,7 @@ if[not system"t";system"t 1000"; .u.tick[.u.LOGSUFFIX,"_",.u.INSTANCE]; // Chain TS -.z.ts:{.u.upd[`order;genOrders[10;x]]; y x}[;oldts:.z.ts] +.z.ts:{.log.logInfo"Running tick ts";.u.upd[`order;genOrders[10;x]]; y x}[;oldts:.z.ts] \ globals used diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q index 5c0f9cc..980f15f 100644 --- a/tickplant/tick/r.q +++ b/tickplant/tick/r.q @@ -1,6 +1,7 @@ /q tick/r.q [host]:port[:usr:pwd] [host]:port[:usr:pwd] /2008.09.09 .k ->.q system"l common/schema.q" +system"l common/log.q" if[not "w"=first string .z.o;system "sleep 1"]; From dd1dfd384a9ee9a3004b99b4298b2ef47314a1e9 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Thu, 25 Jul 2024 11:25:31 -0400 Subject: [PATCH 11/31] lasttrade and lastquote --- tickplant/cep/cep.q | 3 ++- tickplant/cep/lib.q | 12 +++++++++++- tickplant/common/schema.q | 4 ++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tickplant/cep/cep.q b/tickplant/cep/cep.q index 3ea698c..a159d2a 100644 --- a/tickplant/cep/cep.q +++ b/tickplant/cep/cep.q @@ -11,10 +11,11 @@ uend:.u.end; uend x; } +SIM_TRADE:10; "kdb+tick CEP SIM date:",string[.z.P]," version:",string[.z.K],"_",string .z.k .z.ts:{ .log.logInfo"Running CEP ts"; .u.ts[]; - .sim.ts mod[;10]`int$`second$x; + .sim.ts mod[;SIM_TRADE]`int$`second$x; .cep.ts[]; } \ No newline at end of file diff --git a/tickplant/cep/lib.q b/tickplant/cep/lib.q index 52b1a08..9eed8e5 100644 --- a/tickplant/cep/lib.q +++ b/tickplant/cep/lib.q @@ -24,10 +24,20 @@ calcOHLCVV_1d:{ from trade } +// For sampling purposes only - not a realistic calculation +calcMID_1m:{select mid_price:avg price by exch, sym, minute: 1 xbar time.minute from order} + last_ts:.z.P; .cep.ts:{ if[.z.P-last_ts>=00:01:00.000; ohlcvv_1m:: calcOHLCVV_1m[]]; + mid_1m::calcMID_1m[]; } -.cep.end:{ohlcvv_1m::calcOHLCVV_1m[]; ohlcvv_1d::calcOHLCVV_1d[];} \ No newline at end of file +.cep.end:{.cep.ts[]; ohlcvv_1d::calcOHLCVV_1d[];} + +upd:{ + .[insert;(x;y);{.log.logError"Insert failed for table=",string[y]," with error=",x}[;x]]; + if[`trade=x; lastTrade,:y]; + if[`order=x; lastOrder,:y] + }; diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index cd4021f..3975141 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -4,6 +4,10 @@ quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsiz order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); +lastTrade:([sym:`symbol$();exch:`symbol$()] time:`timestamp$();price:`float$();size:`float$();side:`symbol$();tradeid:`guid$()); + +lastOrder:([sym:`symbol$();exch:`symbol$()] time:`timestamp$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); + syms:`$'read0`:common/sp500.txt; exchs:`NYSE`NASDAQ`AMEX; From f11b6433aa78d3692e7d4188e9448d6ceac6cbe3 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sat, 27 Jul 2024 11:57:01 -0400 Subject: [PATCH 12/31] tick log improve --- tickplant/common/log.q | 11 +++++++---- tickplant/tick.q | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tickplant/common/log.q b/tickplant/common/log.q index e7954ff..03a4ff9 100644 --- a/tickplant/common/log.q +++ b/tickplant/common/log.q @@ -1,17 +1,20 @@ if[`log in key`.;:(`log_already_loaded)]; \d .log LEVELS:`info`warning`error`debug - +// .log.test[`something]: {(levels ? `info) >= levels ? `warning} // Use // info when you want to log high level state and flow information // warning when you want to log messages that indicates non-ideal state or potential problems // error when you want to log meesages that indicate a non-recoverable error // debug when you want to log a debug message -level: `info +level: LEVELS ? `debug +setLogLevel:{[l] .log.level::LEVELS ? l}; +canLog:{.log.level <= .log.LEVELS ?x} -logmsg:{[level;msg] +logmsg:{[l;msg] + if[not canLog l; :(::)]; if[11h = type msg; msg:string msg]; - 0N!string[.z.P]," ",upper[string level]," ",msg; + 0N!string[.z.P]," ",upper[string l]," ",msg; } logInfo:logmsg[`info] diff --git a/tickplant/tick.q b/tickplant/tick.q index 32175e9..31f9bda 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -32,6 +32,7 @@ if[not system"p";system"p 5010"] INSTANCE:"one"; LOGSUFFIX:"tplog"; ROOT:"../" +.log.level:1 ld:{if[not type key L::`$(-10_string L),string x;.[L;();:;()]];i::j::-11!(-2;L);if[0<=type i;-2 (string L)," is a corrupt log. Truncate to length ",(string last i)," and restart";exit 1];hopen L}; tick:{ @@ -65,8 +66,9 @@ if[not system"t";system"t 1000"; \d . .u.tick[.u.LOGSUFFIX,"_",.u.INSTANCE]; // Chain TS +SIM_ORDER:1000 -.z.ts:{.log.logInfo"Running tick ts";.u.upd[`order;genOrders[10;x]]; y x}[;oldts:.z.ts] +.z.ts:{.log.logInfo"Running tick ts";.u.upd[`order;genOrders[SIM_ORDER;x]]; y x}[;oldts:.z.ts] \ globals used From 7c858523ea51f0d787774a68c15f9f0a2f022528 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sat, 27 Jul 2024 11:57:24 -0400 Subject: [PATCH 13/31] additional calc simulations --- tickplant/cep/lib.q | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tickplant/cep/lib.q b/tickplant/cep/lib.q index 9eed8e5..e26f203 100644 --- a/tickplant/cep/lib.q +++ b/tickplant/cep/lib.q @@ -1,6 +1,5 @@ calcOHLCVV_1m:{ - update `g#sym from - 0!select + select open: first price, high: max price, low: min price, @@ -12,8 +11,7 @@ calcOHLCVV_1m:{ } calcOHLCVV_1d:{ - update `g#sym from - 0!select + select open: first price, high: max price, low: min price, @@ -24,9 +22,15 @@ calcOHLCVV_1d:{ from trade } + // For sampling purposes only - not a realistic calculation calcMID_1m:{select mid_price:avg price by exch, sym, minute: 1 xbar time.minute from order} +calcVWAP:{[s;e;r] + $[r=1; select sym, exch, second, vwap,res:r from vwap_1s where sym=s,exch=e; + select vwap: sum[wsp]%sum ws, sum num_trade by sym,exch,r xbar second from vwap_1s where sym=s,exch=e] + }; + last_ts:.z.P; .cep.ts:{ @@ -38,6 +42,26 @@ last_ts:.z.P; upd:{ .[insert;(x;y);{.log.logError"Insert failed for table=",string[y]," with error=",x}[;x]]; - if[`trade=x; lastTrade,:y]; - if[`order=x; lastOrder,:y] + if[`trade=x; + if[not .cep.replay; lastTrade,:y]; + // having this separate table is questionable - native wavg performs well compared to running sum x*y%sum y + // but it feels like there is a scale point at which it will be faster to use the running sum when calculating + // multiple vwap horizons instead of recalculating the wavg for each horizon + // We shall see... + // leaving vwap here to compare results with the native wavg + vwap_1s+: select wsp:size wsum price, ws:sum size, vwap:size wavg price, num_trade:count i by exch,sym,time.second from trade + where ([]time.second;sym;exch) in select time.second,sym,exch from y + ]; + if[`order=x; if[not .cep.replay; lastOrder,:y]]; }; + + +// MACD +ema12:ema[2%13;] +ema26:ema[2%17;] +macd: {ema12[x] - ema26 x} +signal:{ema[2%10;macd x]} + +calcMACD:{[s;e] + select minute, exch, ema12 close, ema26 close, macd close, signal macd close by sym from calcOHLCVV_1m[] + } \ No newline at end of file From be9a53c59a181842428bb394bf3b10eafb291f71 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sat, 27 Jul 2024 11:58:16 -0400 Subject: [PATCH 14/31] custom eod logic --- tickplant/cep/cep.q | 22 ++++++++++++++++++++-- tickplant/cep/schema.q | 3 +++ tickplant/common/schema.q | 4 ---- tickplant/tick/r.q | 15 ++++++--------- 4 files changed, 29 insertions(+), 15 deletions(-) create mode 100644 tickplant/cep/schema.q diff --git a/tickplant/cep/cep.q b/tickplant/cep/cep.q index a159d2a..972adfc 100644 --- a/tickplant/cep/cep.q +++ b/tickplant/cep/cep.q @@ -4,9 +4,26 @@ system"l common/sim.q" system"l common/log.q" system"l cep/lib.q" -uend:.u.end; +uend:{ + if[null x;'daycantbenull]; + // t:tables`.; + t:`trade`order; + t@:where `g=attr each t@\:`sym; + .Q.hdpf[.u.x `hdb;HDBROOT;x;`sym]; + @[;`sym;`g#] each t; + }; +/ connect to ticker plant for (schema;(logcount;log)) +// All syms, all tables +// .u.x 0 -> ticker +.cep.replay:1b; +.u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"; +.cep.replay:0b; + + +k).Q.hdpf:{[h;d;p;f](@[`.;;0#]dpft[d;p;f]@)'t@>(#.:)'t:`trade`order;if[h:@[hopen;h;0];h"\\l .";>h]} .u.end:{ + if[null x;'daycantbenull]; .cep.end x; uend x; } @@ -18,4 +35,5 @@ SIM_TRADE:10; .u.ts[]; .sim.ts mod[;SIM_TRADE]`int$`second$x; .cep.ts[]; - } \ No newline at end of file + } + diff --git a/tickplant/cep/schema.q b/tickplant/cep/schema.q new file mode 100644 index 0000000..3a801dd --- /dev/null +++ b/tickplant/cep/schema.q @@ -0,0 +1,3 @@ +HDB_PERSIST:`trade`order`vwap_1s; + +vwap_1s:update `g#sym from ([exch:`$();sym:`$();second:`second$()];wsp:`float$();ws:`float$();vwap:`float$();num_trade:`long$()); diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index 3975141..cd4021f 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -4,10 +4,6 @@ quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsiz order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); -lastTrade:([sym:`symbol$();exch:`symbol$()] time:`timestamp$();price:`float$();size:`float$();side:`symbol$();tradeid:`guid$()); - -lastOrder:([sym:`symbol$();exch:`symbol$()] time:`timestamp$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); - syms:`$'read0`:common/sp500.txt; exchs:`NYSE`NASDAQ`AMEX; diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q index 980f15f..5a2345c 100644 --- a/tickplant/tick/r.q +++ b/tickplant/tick/r.q @@ -7,7 +7,6 @@ if[not "w"=first string .z.o;system "sleep 1"]; upd:insert; - / get the ticker plant and history ports, defaults are 5010,5012 .u.x:`tick`hdb!.z.x,(count .z.x)_(":5010";":5012"); .u.conn:{hopen `$":",x}each .u.x; @@ -17,14 +16,8 @@ HDBROOT:`$":../db" .u.end:{t:tables`.;t@:where `g=attr each t@\:`sym;.Q.hdpf[.u.x `hdb;HDBROOT;x;`sym];@[;`sym;`g#] each t;}; / init schema and sync up from log file;cd to hdb(so client save can run) -// TODO TP SHOULD pass logfile path -.u.rep:{(.[;();:;].)each x;if[null first y;:()];-11!y;}; //system "cd ",1_first "/" vs string last y;}; -/ HARDCODE \cd if other than logdir/db - -/ connect to ticker plant for (schema;(logcount;log)) -// All syms, all tables -// .u.x 0 -> ticker -.u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"; +// log file path comes from .u.L in the ticker +.u.rep:{(.[;();:;].)each x;if[null first y;:()];-11!y;}; .z.pc:{show "Process at handle ",string[p:.u.conn ? x]," disconnected"; .u.pc p} @@ -38,3 +31,7 @@ HDBROOT:`$":../db" if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"]; ]; } + +// remember to call .u.rep in the client +// this is so clients can choose to override upd before +// calling .u.rep \ No newline at end of file From 4d3f0d27b9cc6dc7f15477fc1d8468f502eb3194 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sat, 27 Jul 2024 18:11:54 -0400 Subject: [PATCH 15/31] sample orderbook w/match logic --- tickplant/common/schema.q | 10 ++-- tickplant/orderbook/orderbook.q | 91 +++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 tickplant/orderbook/orderbook.q diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index cd4021f..081337f 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -4,11 +4,11 @@ quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsiz order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); -syms:`$'read0`:common/sp500.txt; +syms:10#`$'read0`:common/sp500.txt; -exchs:`NYSE`NASDAQ`AMEX; +exchs:1#`NYSE; // `NASDAQ`AMEX; -sides:`B`S; +sides:`u?`B`S; //generate random data genTrades:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; @@ -16,5 +16,7 @@ genTrades:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?si // //quote genQuotes:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?10.;n?10.)}; + // order -genOrders:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; \ No newline at end of file +// fixed floor price to avoid market crosses for now +genOrders:{[n;start] sds:n?sides; (asc start+0D00:00:01*til n;n?exchs;n?syms;(0 9)[sds]+n?10.;n?10.;sds;neg[n]?0Ng)}; \ No newline at end of file diff --git a/tickplant/orderbook/orderbook.q b/tickplant/orderbook/orderbook.q new file mode 100644 index 0000000..e96d343 --- /dev/null +++ b/tickplant/orderbook/orderbook.q @@ -0,0 +1,91 @@ +system"l tick/r.q" + +.u.end:{'implementme} + +// side bid - aggressing, means selling +order_bid:update `g#sym from ([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); price:`float$(); size:`float$(); orderid:`guid$(); rcvtime:`timestamp$(); oid:`guid$()); +// side offer - aggressing means buying +order_ask: order_bid + + +upd:{ + if[not x=`order; `nop]; + // stamp + y:update oid:neg[count y]?0Ng,rcvtime:.z.p from y; + `order_ask insert delete side from (y where y[`side]=`S); + `order_bid insert delete side from (y where y[`side]=`B); + } + +.u.conn[`tick]"(.u.sub[`order;`];`.u `i`L)"; + +// TODO nit be functional +.orderbook.buildl2:{`sym`exch`btime`bsize`bprice`aprice`asize`atime xcols `sym`exch xasc 0!(`sym`exch xkey `bprice xdesc select btime:max time, bsize:sum size, btot_orders:count i by sym, exch, bprice:price from order_bid) uj (`sym`exch xkey `aprice xasc select atot_orders:count i, asize:sum size, atime:max time by sym, exch, aprice:price from order_ask)} + + +// find matchs for order x passed as arg +// match `sym`exch`time`price`size`side`orderid!(`A.N;`NYSE;.z.P;10;1;`S;"G"$"5ae7962d-49f2-404d-5aec-f7c8abbae288") +.orderbook.match:{[x] + // find all orders that can fill x + as:`S; + x[`side]:as; + // this can be refactored + rside:$[as=`S;`order_bid;`order_ask]; + aside:$[as=`S;`order_ask;`order_bid]; + // TODO nit make this functional + + o:`price xdesc $[`S=as; select from rside where sym=x`sym, exch=x`exch, price>=x`price; select from rside where sym=x`sym, exch=x`exch, price<=x`price]; + if[0=count o; :(0b)]; + // HANDLE INCOMING ORDER ID + ro:first o; + qty:ro[`size] - x`size; + x[`price]:ro`price; + + // NIT - not a fan of using cascade ifs, and rather use $ then else, but this is more readable since there is no inherent return value and only used for side effects + // if both aggressor and resting are filled, remove and send Execution report + if[qty=0; + delete from rside where orderid=ro`orderid; + delete from aside where orderid=x`orderid; + // .orderbook.reportExecution each (x;ro); + .orderbook.reportTrade x; + :(1b) + ]; + + if[qty<0; + // If resting fully filled, and aggressor partially filled, send Execution report to both, remove the resting order and update the aggressor + delete from rside where orderid=ro`orderid; + x[`size]:neg qty; + ro[`side]:as; + // should this be an upsert? + update size:x`size from aside where orderid=x`orderid; + // .orderbook.reportExecution each (x;ro); + .orderbook.reportTrade ro; + :(1b) + ]; + + if[qty>0; + // If aggressor fully filled, and resting order partially filled, send Execution report to both, remove the aggressor and update the resting order + delete from aside where orderid=x`orderid; + ro[`size]-:x[`size]; + // .orderbook.reportExecution each (x;ro); + .orderbook.reportTrade x; + :(1b) + ]; + } + +// whatever approach used here, an amend at, will do best if used properly! +.orderbook.reportTrade:{ + // send execution report + .log.logInfo"Sending Execution Report for orderid=",.Q.s1 x; + x[`time]:.z.p; + x[`tradeid]:last -1?0Ng; + .u.conn[`tick](`.u.upd;`trade;`time`sym`exch`price`size`side`tradeid#x); + } + +.z.ts:{ + .log.logInfo"Running OrderBook ts"; + .u.ts[]; + .orderbook.buildl2[]; // send this to subs? + .orderbook.match each 0!select by sym,exch from order_ask where price=(min;price)fby([]sym;exch); + } + +.log.setLogLevel`info; \ No newline at end of file From d9bd5334fdbf129a2cb21356df846b3fb7f92b85 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 13:59:04 -0400 Subject: [PATCH 16/31] dont rng messages in ticker --- tickplant/tick.q | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tickplant/tick.q b/tickplant/tick.q index 31f9bda..83092fd 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -68,7 +68,8 @@ if[not system"t";system"t 1000"; // Chain TS SIM_ORDER:1000 -.z.ts:{.log.logInfo"Running tick ts";.u.upd[`order;genOrders[SIM_ORDER;x]]; y x}[;oldts:.z.ts] +.tick.ts:{.u.upd[`order;genOrders[SIM_ORDER;x]]; } +.z.ts:{.log.logInfo"Running tick ts";y x}[;oldts:.z.ts] \ globals used From b87c23bc2978b11d069d4c71289b3dc948c06772 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 14:06:02 -0400 Subject: [PATCH 17/31] defensive protection against singletons --- tickplant/cep/lib.q | 1 + 1 file changed, 1 insertion(+) diff --git a/tickplant/cep/lib.q b/tickplant/cep/lib.q index e26f203..54a682d 100644 --- a/tickplant/cep/lib.q +++ b/tickplant/cep/lib.q @@ -49,6 +49,7 @@ upd:{ // multiple vwap horizons instead of recalculating the wavg for each horizon // We shall see... // leaving vwap here to compare results with the native wavg + if[99h=type y; y:enlist y]; vwap_1s+: select wsp:size wsum price, ws:sum size, vwap:size wavg price, num_trade:count i by exch,sym,time.second from trade where ([]time.second;sym;exch) in select time.second,sym,exch from y ]; From 0adce0d453cbd0ee6237a5f1a9c1a26f7b2a5845 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 14:06:16 -0400 Subject: [PATCH 18/31] remove whitespaces --- tickplant/cep/lib.q | 2 -- 1 file changed, 2 deletions(-) diff --git a/tickplant/cep/lib.q b/tickplant/cep/lib.q index 54a682d..b37e495 100644 --- a/tickplant/cep/lib.q +++ b/tickplant/cep/lib.q @@ -22,7 +22,6 @@ calcOHLCVV_1d:{ from trade } - // For sampling purposes only - not a realistic calculation calcMID_1m:{select mid_price:avg price by exch, sym, minute: 1 xbar time.minute from order} @@ -56,7 +55,6 @@ upd:{ if[`order=x; if[not .cep.replay; lastOrder,:y]]; }; - // MACD ema12:ema[2%13;] ema26:ema[2%17;] From e5dbc70838c5f3e17d636a1c0734eb758a48c80b Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 14:07:10 -0400 Subject: [PATCH 19/31] generate order prices as pattern --- tickplant/common/schema.q | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index 081337f..3ca7c7b 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -6,6 +6,8 @@ order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size: syms:10#`$'read0`:common/sp500.txt; +sym_px:`sym`px!(syms;count[syms]?100.); + exchs:1#`NYSE; // `NASDAQ`AMEX; sides:`u?`B`S; @@ -19,4 +21,7 @@ genQuotes:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?10 // order // fixed floor price to avoid market crosses for now -genOrders:{[n;start] sds:n?sides; (asc start+0D00:00:01*til n;n?exchs;n?syms;(0 9)[sds]+n?10.;n?10.;sds;neg[n]?0Ng)}; \ No newline at end of file +genOrders:{[n;start] + sims:n?flip sym_px; + sds:n?sides; + (asc start+0D00:00:01*til n;n?exchs;sims`sym;sims[`px]+(-1 1)[sds]*(n?1+1000)%100;n?10.;sds;neg[n]?0Ng)}; \ No newline at end of file From df62aab7d4455a0caa7236e682ab4a097283c576 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 14:08:09 -0400 Subject: [PATCH 20/31] Move order generation to feedhandler.q --- tickplant/feedhandler/feedhandler.q | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tickplant/feedhandler/feedhandler.q diff --git a/tickplant/feedhandler/feedhandler.q b/tickplant/feedhandler/feedhandler.q new file mode 100644 index 0000000..88f22f0 --- /dev/null +++ b/tickplant/feedhandler/feedhandler.q @@ -0,0 +1,30 @@ +// q feedhandler/feedhandler.q :5010 -c 13 317 -t 100 +system"l tick/r.q" + +.u.end:{'implementme} + +upd:{ + if[not x=`order; `nop]; + // stamp + y:update oid:neg[count y]?0Ng,rcvtime:.z.p from y; + `order_ask insert delete side from (y where y[`side]=`S); + `order_bid insert delete side from (y where y[`side]=`B); + } + +SIM_ORDERS:1000 + +.fh.i:0; + +.fh.ts:{ + batch:genOrders[SIM_ORDERS;.z.p]; + .fh.i+:SIM_ORDERS; + // TODO trap errors like ticker being down + .u.conn[`tick](`.u.upd;`order;batch); + sym_px[`px]+:@[;(last 1?count sym_px[`px])? count sym_px[`px];neg] %[;1000]count[sym_px`px]?100; + }; + +.z.ts:{ + .log.logInfo"Running Fedhandler ts"; + .u.ts[]; + .fh.ts[]; + } \ No newline at end of file From 9c1634f39b8cf3c0d9069c652ccd24eeee30be27 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 14:08:49 -0400 Subject: [PATCH 21/31] refactor order match engine --- tickplant/orderbook/orderbook.q | 35 +++++++++++++++++++-------------- tickplant/orderbook/schema.q | 6 ++++++ tickplant/orderbook/test.q | 4 ++++ 3 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 tickplant/orderbook/schema.q create mode 100644 tickplant/orderbook/test.q diff --git a/tickplant/orderbook/orderbook.q b/tickplant/orderbook/orderbook.q index e96d343..f69c06b 100644 --- a/tickplant/orderbook/orderbook.q +++ b/tickplant/orderbook/orderbook.q @@ -1,13 +1,7 @@ system"l tick/r.q" - +system"l orderbook/schema.q" .u.end:{'implementme} -// side bid - aggressing, means selling -order_bid:update `g#sym from ([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); price:`float$(); size:`float$(); orderid:`guid$(); rcvtime:`timestamp$(); oid:`guid$()); -// side offer - aggressing means buying -order_ask: order_bid - - upd:{ if[not x=`order; `nop]; // stamp @@ -18,15 +12,17 @@ upd:{ .u.conn[`tick]"(.u.sub[`order;`];`.u `i`L)"; -// TODO nit be functional -.orderbook.buildl2:{`sym`exch`btime`bsize`bprice`aprice`asize`atime xcols `sym`exch xasc 0!(`sym`exch xkey `bprice xdesc select btime:max time, bsize:sum size, btot_orders:count i by sym, exch, bprice:price from order_bid) uj (`sym`exch xkey `aprice xasc select atot_orders:count i, asize:sum size, atime:max time by sym, exch, aprice:price from order_ask)} +.orderbook.buildl2:{r:0!(`sym`exch`bpx xdesc `bypx_norm "b") uj (`sym`exch`apx xasc bypx_norm "a");.orderbook.l2cols xcols r} +bypx:{`sym`exch xkey select max time,sum size, tot_orders:count i by sym,exch,price from x} +bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x;`order_ask;order_bid]} + // find matchs for order x passed as arg // match `sym`exch`time`price`size`side`orderid!(`A.N;`NYSE;.z.P;10;1;`S;"G"$"5ae7962d-49f2-404d-5aec-f7c8abbae288") .orderbook.match:{[x] // find all orders that can fill x - as:`S; + as:last 1?`S`B; x[`side]:as; // this can be refactored rside:$[as=`S;`order_bid;`order_ask]; @@ -39,7 +35,7 @@ upd:{ ro:first o; qty:ro[`size] - x`size; x[`price]:ro`price; - + // borrowing some GOLANG verbosity.... // NIT - not a fan of using cascade ifs, and rather use $ then else, but this is more readable since there is no inherent return value and only used for side effects // if both aggressor and resting are filled, remove and send Execution report if[qty=0; @@ -72,20 +68,29 @@ upd:{ ]; } -// whatever approach used here, an amend at, will do best if used properly! +// whatever approach used here, will an amendat do best if used properly? .orderbook.reportTrade:{ // send execution report - .log.logInfo"Sending Execution Report for orderid=",.Q.s1 x; + .log.logInfo"Sending Trade Report for trade=",.Q.s1 x; x[`time]:.z.p; x[`tradeid]:last -1?0Ng; .u.conn[`tick](`.u.upd;`trade;`time`sym`exch`price`size`side`tradeid#x); } +// Tells the trader that there is an execution +.orderbook.reportExecution:{'implement_me} + .z.ts:{ .log.logInfo"Running OrderBook ts"; .u.ts[]; - .orderbook.buildl2[]; // send this to subs? - .orderbook.match each 0!select by sym,exch from order_ask where price=(min;price)fby([]sym;exch); + // .orderbook.buildl2[]; // send this to subs? + // nit - find a better way to do this + offers: 0!select by sym,exch from order_ask where price=(min;price)fby([]sym;exch); + bids: 0!select by sym,exch from order_bid where price=(max;price)fby([]sym;exch); + // TODO consider different approach where the two top of books are matched against + // each other instead of doing ToB for each side vs rest + // It should save on one query to order_asks and order_bids + .orderbook.match each offers,bids; } .log.setLogLevel`info; \ No newline at end of file diff --git a/tickplant/orderbook/schema.q b/tickplant/orderbook/schema.q new file mode 100644 index 0000000..ec91dc6 --- /dev/null +++ b/tickplant/orderbook/schema.q @@ -0,0 +1,6 @@ +// side bid - aggressing, means selling +order_bid:update `g#sym from ([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); price:`float$(); size:`float$(); orderid:`guid$(); rcvtime:`timestamp$(); oid:`guid$()); +// side offer - aggressing means buying +order_ask: order_bid + +.orderbook.l2cols: `sym`exch`btime`btot_orders`bsz`bpx`apx`asz`atot_orders`atime; \ No newline at end of file diff --git a/tickplant/orderbook/test.q b/tickplant/orderbook/test.q new file mode 100644 index 0000000..a57e1e6 --- /dev/null +++ b/tickplant/orderbook/test.q @@ -0,0 +1,4 @@ +// unit test order book matching logic +// generate sample orders table +// cross the spread +// verify fill From 7a00f6a756cf1bf82b330c25fba6e089be9d32fd Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 15:59:53 -0400 Subject: [PATCH 22/31] refactoring tick connection handling --- tickplant/common/conn.q | 15 +++++++++++++++ tickplant/tick/r.q | 16 ---------------- 2 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 tickplant/common/conn.q diff --git a/tickplant/common/conn.q b/tickplant/common/conn.q new file mode 100644 index 0000000..7d147d9 --- /dev/null +++ b/tickplant/common/conn.q @@ -0,0 +1,15 @@ +/ get the ticker plant and history ports, defaults are 5010,5012 +.u.x:`tick`hdb!.z.x,(count .z.x)_(":5010";":5012"); + +.z.pc:{show "Process at handle ",string[p:.u.conn ? x]," disconnected"; .u.pc p} + +.u.pc:{.u.conn[x]:0Ni; show .u.conn} + +.u.ts:{ + if[ null .u.conn`tick; + show "Retrying tickerplant connection ..."; + h:@[hopen;`$":",.u.x`tick;{show "Cannot establish tickerplant connection with ",x; 0Ni}]; + .u.conn[`tick]: h; + if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"]; + ]; + } \ No newline at end of file diff --git a/tickplant/tick/r.q b/tickplant/tick/r.q index 5a2345c..a06dcf7 100644 --- a/tickplant/tick/r.q +++ b/tickplant/tick/r.q @@ -7,10 +7,6 @@ if[not "w"=first string .z.o;system "sleep 1"]; upd:insert; -/ get the ticker plant and history ports, defaults are 5010,5012 -.u.x:`tick`hdb!.z.x,(count .z.x)_(":5010";":5012"); -.u.conn:{hopen `$":",x}each .u.x; - / end of day: save, clear, hdb reload HDBROOT:`$":../db" .u.end:{t:tables`.;t@:where `g=attr each t@\:`sym;.Q.hdpf[.u.x `hdb;HDBROOT;x;`sym];@[;`sym;`g#] each t;}; @@ -19,18 +15,6 @@ HDBROOT:`$":../db" // log file path comes from .u.L in the ticker .u.rep:{(.[;();:;].)each x;if[null first y;:()];-11!y;}; -.z.pc:{show "Process at handle ",string[p:.u.conn ? x]," disconnected"; .u.pc p} - -.u.pc:{.u.conn[x]:0Ni; show .u.conn} - -.u.ts:{ - if[ null .u.conn`tick; - show "Retrying tickerplant connection ..."; - h:@[hopen;`$":",.u.x`tick;{show "Cannot establish tickerplant connection with ",x; 0Ni}]; - .u.conn[`tick]: h; - if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"]; - ]; - } // remember to call .u.rep in the client // this is so clients can choose to override upd before From 7b47a66d8a05d8a8c073d9a60f8622a59f75fe7a Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 16:00:42 -0400 Subject: [PATCH 23/31] Moving genOrder to feedhandler lib --- tickplant/common/schema.q | 24 +----------------------- tickplant/feedhandler/feedhandler.q | 17 ++++++++++------- tickplant/feedhandler/lib.q | 17 +++++++++++++++++ tickplant/tick.q | 3 --- 4 files changed, 28 insertions(+), 33 deletions(-) create mode 100644 tickplant/feedhandler/lib.q diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index 3ca7c7b..54c0ecb 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -2,26 +2,4 @@ trade:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size: quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsize:`float$();asize:`float$();aprice:`float$()); -order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); - -syms:10#`$'read0`:common/sp500.txt; - -sym_px:`sym`px!(syms;count[syms]?100.); - -exchs:1#`NYSE; // `NASDAQ`AMEX; - -sides:`u?`B`S; - -//generate random data -genTrades:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?sides;neg[n]?0Ng)}; - -// //quote -genQuotes:{[n;start] (asc start+0D00:00:01*til n;n?exchs;n?syms;n?10.;n?10.;n?10.;n?10.)}; - - -// order -// fixed floor price to avoid market crosses for now -genOrders:{[n;start] - sims:n?flip sym_px; - sds:n?sides; - (asc start+0D00:00:01*til n;n?exchs;sims`sym;sims[`px]+(-1 1)[sds]*(n?1+1000)%100;n?10.;sds;neg[n]?0Ng)}; \ No newline at end of file +order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); \ No newline at end of file diff --git a/tickplant/feedhandler/feedhandler.q b/tickplant/feedhandler/feedhandler.q index 88f22f0..e200ccb 100644 --- a/tickplant/feedhandler/feedhandler.q +++ b/tickplant/feedhandler/feedhandler.q @@ -1,15 +1,15 @@ // q feedhandler/feedhandler.q :5010 -c 13 317 -t 100 +// This process simulates a number of orders coming in from the market +// and sends them to the TICKER process system"l tick/r.q" +system"l common/conn.q" +system"l feedhandler/lib.q" .u.end:{'implementme} -upd:{ - if[not x=`order; `nop]; - // stamp - y:update oid:neg[count y]?0Ng,rcvtime:.z.p from y; - `order_ask insert delete side from (y where y[`side]=`S); - `order_bid insert delete side from (y where y[`side]=`B); - } +.u.conn:{hopen `$":",x}each enlist[`tick]#.u.x; + +upd:{'implementme} SIM_ORDERS:1000 @@ -20,6 +20,9 @@ SIM_ORDERS:1000 .fh.i+:SIM_ORDERS; // TODO trap errors like ticker being down .u.conn[`tick](`.u.upd;`order;batch); + // Moving the market by symbol either up/down by a random amount so prices look like they are moving + // NIT could have used the prices already generated for batch[`price] but that doenst generate + // the same price movement for all symbols sym_px[`px]+:@[;(last 1?count sym_px[`px])? count sym_px[`px];neg] %[;1000]count[sym_px`px]?100; }; diff --git a/tickplant/feedhandler/lib.q b/tickplant/feedhandler/lib.q new file mode 100644 index 0000000..2e36d3a --- /dev/null +++ b/tickplant/feedhandler/lib.q @@ -0,0 +1,17 @@ +exchs:`NYSE`NASDAQ`AMEX; + +sides:`u?`B`S; + +syms:10#`$'read0`:common/sp500.txt; + +// Carry price movements across restarts +sym_px:@[get;`:sym_px;{`sym`px!(syms;count[syms]?100.)}]; +`:sym_px set sym_px; + +// order +genOrders:{[n;start] + sims:n?flip sym_px; + sds:n?sides; + // Generating integers up to 1000 and then dividing by 100 to get floats of set precision + // also forcing BUY prices to be generally smaller than SELL prices (can still be crossed) + (asc start+0D00:00:01*til n;n?exchs;sims`sym;sims[`px]+(-1 1)[sds]*(n?1+1000)%100;n?10.;sds;neg[n]?0Ng)}; \ No newline at end of file diff --git a/tickplant/tick.q b/tickplant/tick.q index 83092fd..8f77508 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -65,10 +65,7 @@ if[not system"t";system"t 1000"; \d . .u.tick[.u.LOGSUFFIX,"_",.u.INSTANCE]; -// Chain TS -SIM_ORDER:1000 -.tick.ts:{.u.upd[`order;genOrders[SIM_ORDER;x]]; } .z.ts:{.log.logInfo"Running tick ts";y x}[;oldts:.z.ts] \ From 992202d980ca07a493d3eca6b95872a94797707c Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 28 Jul 2024 16:11:43 -0400 Subject: [PATCH 24/31] check for stale matching in book --- .gitignore | 5 ++- tickplant/orderbook/orderbook.q | 57 +++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 8352ff1..2095605 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ tplog_* -db/ \ No newline at end of file +db/ +logs/ +data/ +sym_px \ No newline at end of file diff --git a/tickplant/orderbook/orderbook.q b/tickplant/orderbook/orderbook.q index f69c06b..4bb9b60 100644 --- a/tickplant/orderbook/orderbook.q +++ b/tickplant/orderbook/orderbook.q @@ -1,23 +1,33 @@ +// q orderbook/orderbook.q :5010 -c 13 317 -t 100 +// This process simulates a bookbuilder/matching engine that +// receives orders from the TICKER process and matches them against the order book +// sending execution reports back to the TICKER process +// It is meant to be run in conjunction with the TICKER process, and has ZERO +// state maintenance across restarts, nor recovery from crashes.. YET system"l tick/r.q" +system"l common/conn.q" system"l orderbook/schema.q" + +.u.conn:{hopen `$":",x}each enlist[`tick]#.u.x; + +.log.setLogLevel`info; + .u.end:{'implementme} +.u.rep:{'implementme} upd:{ - if[not x=`order; `nop]; - // stamp + if[not x=`order; :(`nop)]; y:update oid:neg[count y]?0Ng,rcvtime:.z.p from y; `order_ask insert delete side from (y where y[`side]=`S); `order_bid insert delete side from (y where y[`side]=`B); + if[00:02:00 < .z.p-.orderbook.lastMatch; .log.logWarning "Nothing matched in over 2 minutes, but data incoming. Is timer on?"]; } -.u.conn[`tick]"(.u.sub[`order;`];`.u `i`L)"; - .orderbook.buildl2:{r:0!(`sym`exch`bpx xdesc `bypx_norm "b") uj (`sym`exch`apx xasc bypx_norm "a");.orderbook.l2cols xcols r} bypx:{`sym`exch xkey select max time,sum size, tot_orders:count i by sym,exch,price from x} bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x;`order_ask;order_bid]} - // find matchs for order x passed as arg // match `sym`exch`time`price`size`side`orderid!(`A.N;`NYSE;.z.P;10;1;`S;"G"$"5ae7962d-49f2-404d-5aec-f7c8abbae288") .orderbook.match:{[x] @@ -30,7 +40,7 @@ bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x; // TODO nit make this functional o:`price xdesc $[`S=as; select from rside where sym=x`sym, exch=x`exch, price>=x`price; select from rside where sym=x`sym, exch=x`exch, price<=x`price]; - if[0=count o; :(0b)]; + if[0=count o; :(0Np)]; // HANDLE INCOMING ORDER ID ro:first o; qty:ro[`size] - x`size; @@ -43,7 +53,7 @@ bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x; delete from aside where orderid=x`orderid; // .orderbook.reportExecution each (x;ro); .orderbook.reportTrade x; - :(1b) + :(.z.p) ]; if[qty<0; @@ -55,7 +65,7 @@ bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x; update size:x`size from aside where orderid=x`orderid; // .orderbook.reportExecution each (x;ro); .orderbook.reportTrade ro; - :(1b) + :(.z.p) ]; if[qty>0; @@ -64,13 +74,12 @@ bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x; ro[`size]-:x[`size]; // .orderbook.reportExecution each (x;ro); .orderbook.reportTrade x; - :(1b) + :(.z.p) ]; } // whatever approach used here, will an amendat do best if used properly? .orderbook.reportTrade:{ - // send execution report .log.logInfo"Sending Trade Report for trade=",.Q.s1 x; x[`time]:.z.p; x[`tradeid]:last -1?0Ng; @@ -80,17 +89,39 @@ bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x; // Tells the trader that there is an execution .orderbook.reportExecution:{'implement_me} + +// Stateful resumes from last known state +.orderbook.toSnapshot:{system"x .z.zd"; (`$":data/orderbook/order_ask") set order_ask; (`$":data/orderbook/order_bid") set order_bid;} +.orderbook.fromSnapshot:{ + `order_ask set @[get;`$":data/orderbook/order_ask";{order_ask}]; + `order_bid set @[get;`$":data/orderbook/order_bid";{order_bid}]; + } +// Should there be logic to snapshot trades that have been reported already? +// Maybe can change tradeid to be an hash of the trade itself to use as a key +// and keep state for it +.orderbook.start:{ + .log.logInfo "kdb+tick ORDERBOOK date:",string[.z.P]," version:",string[.z.K],"_",string .z.k; + .orderbook.fromSnapshot[]; + .u.conn[`tick]"(.u.sub[`order;`];`.u `i`L)"; + } + +.orderbook.lastMatch:0Np; + .z.ts:{ .log.logInfo"Running OrderBook ts"; .u.ts[]; - // .orderbook.buildl2[]; // send this to subs? + .orderbook.toSnapshot[]; // nit - find a better way to do this offers: 0!select by sym,exch from order_ask where price=(min;price)fby([]sym;exch); bids: 0!select by sym,exch from order_bid where price=(max;price)fby([]sym;exch); // TODO consider different approach where the two top of books are matched against // each other instead of doing ToB for each side vs rest // It should save on one query to order_asks and order_bids - .orderbook.match each offers,bids; + .orderbook.lastMatch: max .orderbook.match each offers,bids; + // Build after matches, otherwise it'll look like crossed books always + // .orderbook.buildl2[]; // send this to subs? + + if[00:02:00 < .z.p-.orderbook.lastMatch; .log.logWarning "Nothing matched in over 2 minutes, is everything okay?" ]; } -.log.setLogLevel`info; \ No newline at end of file +.orderbook.start[] \ No newline at end of file From e47314c74a737ee415408bec800d31783e080128 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:35:41 +0200 Subject: [PATCH 25/31] moving log path to std dir --- tickplant/tick.q | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tickplant/tick.q b/tickplant/tick.q index 8f77508..727e110 100644 --- a/tickplant/tick.q +++ b/tickplant/tick.q @@ -31,8 +31,8 @@ if[not system"p";system"p 5010"] \d .u INSTANCE:"one"; LOGSUFFIX:"tplog"; -ROOT:"../" -.log.level:1 +ROOT:"data/" +.log.setLogLevel`INFO; ld:{if[not type key L::`$(-10_string L),string x;.[L;();:;()]];i::j::-11!(-2;L);if[0<=type i;-2 (string L)," is a corrupt log. Truncate to length ",(string last i)," and restart";exit 1];hopen L}; tick:{ From d52ac6c92cbf02d3cc2428f8b15cdd901d2edd02 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:36:01 +0200 Subject: [PATCH 26/31] refactor to common pattern --- tickplant/cep/cep.q | 60 +++++++++++++++++++++++++-------------------- tickplant/cep/lib.q | 15 +++++------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/tickplant/cep/cep.q b/tickplant/cep/cep.q index 972adfc..45d4d22 100644 --- a/tickplant/cep/cep.q +++ b/tickplant/cep/cep.q @@ -1,39 +1,45 @@ /q cep/cep.q SRC :5010 + system"l tick/r.q" -system"l common/sim.q" -system"l common/log.q" +system"l common/conn.q" system"l cep/lib.q" -uend:{ - if[null x;'daycantbenull]; - // t:tables`.; - t:`trade`order; - t@:where `g=attr each t@\:`sym; - .Q.hdpf[.u.x `hdb;HDBROOT;x;`sym]; - @[;`sym;`g#] each t; - }; - -/ connect to ticker plant for (schema;(logcount;log)) -// All syms, all tables -// .u.x 0 -> ticker -.cep.replay:1b; -.u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"; -.cep.replay:0b; - - -k).Q.hdpf:{[h;d;p;f](@[`.;;0#]dpft[d;p;f]@)'t@>(#.:)'t:`trade`order;if[h:@[hopen;h;0];h"\\l .";>h]} -.u.end:{ - if[null x;'daycantbenull]; - .cep.end x; - uend x; +.u.conn:{hopen `$":",x}each enlist[`tick]#.u.x; + +// uend:{ +// if[null x;'daycantbenull]; +// // t:tables`.; +// t:`trade`order; +// t@:where `g=attr each t@\:`sym; +// .Q.hdpf[.u.x `hdb;HDBROOT;x;`sym]; +// @[;`sym;`g#] each t; +// }; + +// k).Q.hdpf:{[h;d;p;f](@[`.;;0#]dpft[d;p;f]@)'t@>(#.:)'t:`trade`order;if[h:@[hopen;h;0];h"\\l .";>h]} +// .u.end:{ +// if[null x;'daycantbenull]; +// .cep.end x; +// uend x; +// } + +.cep.start:{ + .log.logInfo "kdb+tick CEP date:",string[.z.P]," version:",string[.z.K],"_",string .z.k; + .cep.replay::1b; + .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"; + .cep.replay::0b; + } + +.cep.lastTs:.z.P; + +.cep.ts:{ + // if[.z.P-.cep.lastTs>=00:01:00.000; ohlcvv_1m:: calcOHLCVV_1m[]]; + // mid_1m::calcMID_1m[]; } -SIM_TRADE:10; -"kdb+tick CEP SIM date:",string[.z.P]," version:",string[.z.K],"_",string .z.k .z.ts:{ .log.logInfo"Running CEP ts"; .u.ts[]; - .sim.ts mod[;SIM_TRADE]`int$`second$x; .cep.ts[]; } +.cep.start[] \ No newline at end of file diff --git a/tickplant/cep/lib.q b/tickplant/cep/lib.q index b37e495..0745433 100644 --- a/tickplant/cep/lib.q +++ b/tickplant/cep/lib.q @@ -23,7 +23,7 @@ calcOHLCVV_1d:{ } // For sampling purposes only - not a realistic calculation -calcMID_1m:{select mid_price:avg price by exch, sym, minute: 1 xbar time.minute from order} +calcTickMID:{select mid_price:avg price by exch, sym, minute: 1 xbar time.minute from order} calcVWAP:{[s;e;r] $[r=1; select sym, exch, second, vwap,res:r from vwap_1s where sym=s,exch=e; @@ -32,12 +32,9 @@ calcVWAP:{[s;e;r] last_ts:.z.P; -.cep.ts:{ - if[.z.P-last_ts>=00:01:00.000; ohlcvv_1m:: calcOHLCVV_1m[]]; - mid_1m::calcMID_1m[]; - } -.cep.end:{.cep.ts[]; ohlcvv_1d::calcOHLCVV_1d[];} + +// .cep.end:{.cep.ts[]; ohlcvv_1d::calcOHLCVV_1d[];} upd:{ .[insert;(x;y);{.log.logError"Insert failed for table=",string[y]," with error=",x}[;x]]; @@ -48,9 +45,9 @@ upd:{ // multiple vwap horizons instead of recalculating the wavg for each horizon // We shall see... // leaving vwap here to compare results with the native wavg - if[99h=type y; y:enlist y]; - vwap_1s+: select wsp:size wsum price, ws:sum size, vwap:size wavg price, num_trade:count i by exch,sym,time.second from trade - where ([]time.second;sym;exch) in select time.second,sym,exch from y + // if[99h=type y; y:enlist y]; + // vwap_1s+: select wsp:size wsum price, ws:sum size, vwap:size wavg price, num_trade:count i by exch,sym,time.second from trade + // where ([]time.second;sym;exch) in select time.second,sym,exch from y ]; if[`order=x; if[not .cep.replay; lastOrder,:y]]; }; From 8dfc0b625805fadb343130258859a6a8760fb778 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:36:19 +0200 Subject: [PATCH 27/31] unused sim --- tickplant/common/sim.q | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 tickplant/common/sim.q diff --git a/tickplant/common/sim.q b/tickplant/common/sim.q deleted file mode 100644 index 11efb67..0000000 --- a/tickplant/common/sim.q +++ /dev/null @@ -1,8 +0,0 @@ -// load the simulation context -// not switching namespace to avoid finangling with tables -if[not `conn in key`.u; 'udotqmissing] -// random X trades, doenst flip side or anything, just a way to -// generate some trades that can be joined with order -.sim.aggressOrder:{.u.conn[`tick](`.u.ins;`trade;neg[x]?order)} - -.sim.ts:{[n].sim.aggressOrder n}; \ No newline at end of file From 03e9e1ea9bcc3f80296dc7362b3e6b789b6c8ae7 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:36:37 +0200 Subject: [PATCH 28/31] refactor orderbook and add matching --- tickplant/orderbook/lib.q | 91 ++++++++++++++++++++++ tickplant/orderbook/orderbook.q | 131 ++++++++++---------------------- tickplant/orderbook/schema.q | 8 +- 3 files changed, 137 insertions(+), 93 deletions(-) create mode 100644 tickplant/orderbook/lib.q diff --git a/tickplant/orderbook/lib.q b/tickplant/orderbook/lib.q new file mode 100644 index 0000000..885b857 --- /dev/null +++ b/tickplant/orderbook/lib.q @@ -0,0 +1,91 @@ +// Stateful resumes from last known state +.orderbook.INSTANCE_PATH:`$":data/orderbook_",.orderbook.INSTANCE,"/" +// NIT - this should be refactored to minimize race conditions between the book timestamp and when it snapshots +// possibly do not snapshot a separate variable +.orderbook.writeSnapshot:{(` sv .orderbook.INSTANCE_PATH,x)set get x;`.orderbook.lastBook set max order_ask[`time],order_bid`time} +.orderbook.readSnapshot:{x set @[get;.orderbook.INSTANCE_PATH,x;{.orderbook[x]}[x]]} + +.orderbook.toSnapshot:{system"x .z.zd";.orderbook.writeSnapshot each `order_ask`order_bid} +.orderbook.fromSnapshot:{.orderbook.readSnapshot each -2?`order_ask`order_bid; `.orderbook.lastBook set max order_ask[`time],order_bid`time} + + +.orderbook.upd:{[t;d] + if[not t=`order; :(`nop)]; + // NIT - could skip messages if for some reason in the same batch there is message with the time before the last book + // but never received from the book. TBC if this would happen in practice + d:update oid:neg[count d]?0Ng,rcvtime:.z.p from d; + `order_ask insert (d where d[`side]=`S); + `order_bid insert (d where d[`side]=`B); + if[00:02:00 < .z.p-.orderbook.lastMatch; .log.logWarning "Nothing matched in over 2 minutes, but data incoming. Is timer on?"]; + } + +// match `sym`exch`time`price`size`side`orderid!(`A.N;`NYSE;.z.P;10;1;`S;"G"$"5ae7962d-49f2-404d-5aec-f7c8abbae288") +// match tries to match an incoming order against the order book +// it will return a trade if the order is fully or partially filled +// it will return a null if there are no fills +// it can either allow for sweeping or not +.orderbook.match:{[x] + // find all orders that can fill x + as:x[`side]; + // this can be refactored + rside:$[as=`S;`order_bid;`order_ask]; + aside:$[as=`S;`order_ask;`order_bid]; + o:`price xdesc $[`S=as; + select from rside where sym=x`sym, exch=x`exch, price>=x`price; + select from rside where sym=x`sym, exch=x`exch, price<=x`price]; + + if[0=count o; :(0Np)]; + + // if[10; + // If aggressor fully filled, and resting order partially filled, send Execution report to both, remove the aggressor and update the resting order + delete from aside where orderid=x`orderid; + trade[`size]:x`size; + update size:qty from rside where orderid=ro`orderid; + ]; + `executions insert trade; + .z.p + } + + +// `time`sym`exch`price`size`side`tradeid`roid`aoid!(.z.p;x`sym;x`exch;x`price;x`size;x`side;last -1?0Ng;x`orderid;last -1?0Ng) +// whatever approach used here, will an amendat do best if used properly? +.orderbook.reportTrade:{ + .log.logInfo"Sending Trade Report for trade=",.Q.s1 x; + .u.conn[`tick](`.u.upd;`trade;x); + } + + +bypx:{select max time,sum size, tot_orders:count i by sym,exch,price from x} +bypx_norm:{ + f:idesc; s:"b"; if[x~`order_ask;f:iasc;s:"a"]; + r:![bypx x;();{x!x}`sym`exch;enlist[`ind]!enlist(f;`price)]; + r:(`sym`exch`ind,`$'s,/:("px";"time";"sz";"tot_orders")) xcol `sym`exch`ind xkey r + } + + +.orderbook.buildl2:{ + r:`sym`exch`bpx xdesc bypx_norm[`order_bid] uj bypx_norm`order_ask; + `time`exch`sym xcols update time:.z.p from .orderbook.l2cols xcols 0!r + } diff --git a/tickplant/orderbook/orderbook.q b/tickplant/orderbook/orderbook.q index 4bb9b60..34bcc7e 100644 --- a/tickplant/orderbook/orderbook.q +++ b/tickplant/orderbook/orderbook.q @@ -1,127 +1,78 @@ -// q orderbook/orderbook.q :5010 -c 13 317 -t 100 +// q orderbook/orderbook.q :5010 -c 13 317 -t 100 -instance one // This process simulates a bookbuilder/matching engine that // receives orders from the TICKER process and matches them against the order book // sending execution reports back to the TICKER process // It is meant to be run in conjunction with the TICKER process, and has ZERO // state maintenance across restarts, nor recovery from crashes.. YET + system"l tick/r.q" -system"l common/conn.q" +.log.setLogLevel`warning; + + +.orderbook.INSTANCE: "one" +system"l orderbook/lib.q" system"l orderbook/schema.q" +system"l common/conn.q" .u.conn:{hopen `$":",x}each enlist[`tick]#.u.x; -.log.setLogLevel`info; .u.end:{'implementme} -.u.rep:{'implementme} +.u.rep:{(.[;();:;].)x;if[null first y;:()];-11!y;}; -upd:{ +// TODO could refactor this? +upd:{ if[not x=`order; :(`nop)]; - y:update oid:neg[count y]?0Ng,rcvtime:.z.p from y; - `order_ask insert delete side from (y where y[`side]=`S); - `order_bid insert delete side from (y where y[`side]=`B); - if[00:02:00 < .z.p-.orderbook.lastMatch; .log.logWarning "Nothing matched in over 2 minutes, but data incoming. Is timer on?"]; - } - -.orderbook.buildl2:{r:0!(`sym`exch`bpx xdesc `bypx_norm "b") uj (`sym`exch`apx xasc bypx_norm "a");.orderbook.l2cols xcols r} -bypx:{`sym`exch xkey select max time,sum size, tot_orders:count i by sym,exch,price from x} -bypx_norm:{(`sym`exch,`$'x,/:("px";"time";"sz";"tot_orders")) xcol bypx $["a"~x;`order_ask;order_bid]} - - -// find matchs for order x passed as arg -// match `sym`exch`time`price`size`side`orderid!(`A.N;`NYSE;.z.P;10;1;`S;"G"$"5ae7962d-49f2-404d-5aec-f7c8abbae288") -.orderbook.match:{[x] - // find all orders that can fill x - as:last 1?`S`B; - x[`side]:as; - // this can be refactored - rside:$[as=`S;`order_bid;`order_ask]; - aside:$[as=`S;`order_ask;`order_bid]; - // TODO nit make this functional - - o:`price xdesc $[`S=as; select from rside where sym=x`sym, exch=x`exch, price>=x`price; select from rside where sym=x`sym, exch=x`exch, price<=x`price]; - if[0=count o; :(0Np)]; - // HANDLE INCOMING ORDER ID - ro:first o; - qty:ro[`size] - x`size; - x[`price]:ro`price; - // borrowing some GOLANG verbosity.... - // NIT - not a fan of using cascade ifs, and rather use $ then else, but this is more readable since there is no inherent return value and only used for side effects - // if both aggressor and resting are filled, remove and send Execution report - if[qty=0; - delete from rside where orderid=ro`orderid; - delete from aside where orderid=x`orderid; - // .orderbook.reportExecution each (x;ro); - .orderbook.reportTrade x; - :(.z.p) + d:y; + if[.orderbook.replay; + d:$[1=count y;enlist;flip] cols[x]!y; + if[.orderbook.lastBook>max d`time; :(`nop)] ]; - - if[qty<0; - // If resting fully filled, and aggressor partially filled, send Execution report to both, remove the resting order and update the aggressor - delete from rside where orderid=ro`orderid; - x[`size]:neg qty; - ro[`side]:as; - // should this be an upsert? - update size:x`size from aside where orderid=x`orderid; - // .orderbook.reportExecution each (x;ro); - .orderbook.reportTrade ro; - :(.z.p) - ]; - - if[qty>0; - // If aggressor fully filled, and resting order partially filled, send Execution report to both, remove the aggressor and update the resting order - delete from aside where orderid=x`orderid; - ro[`size]-:x[`size]; - // .orderbook.reportExecution each (x;ro); - .orderbook.reportTrade x; - :(.z.p) - ]; - } - -// whatever approach used here, will an amendat do best if used properly? -.orderbook.reportTrade:{ - .log.logInfo"Sending Trade Report for trade=",.Q.s1 x; - x[`time]:.z.p; - x[`tradeid]:last -1?0Ng; - .u.conn[`tick](`.u.upd;`trade;`time`sym`exch`price`size`side`tradeid#x); + .orderbook.upd[x;d]; + if[not .orderbook.replay; .orderbook.toSnapshot[]]; } - -// Tells the trader that there is an execution -.orderbook.reportExecution:{'implement_me} +// When on, an aggressor order will sweep the book until it's fully filled or the book is empty +// When off, an aggressor order will only match against the top of the book +.orderbook.allowSweep:1b; +// Last time there was an execution +.orderbook.lastMatch:0Np; +// Last time .z.ts ran +.orderbook.lastTs:0Np; -// Stateful resumes from last known state -.orderbook.toSnapshot:{system"x .z.zd"; (`$":data/orderbook/order_ask") set order_ask; (`$":data/orderbook/order_bid") set order_bid;} -.orderbook.fromSnapshot:{ - `order_ask set @[get;`$":data/orderbook/order_ask";{order_ask}]; - `order_bid set @[get;`$":data/orderbook/order_bid";{order_bid}]; - } // Should there be logic to snapshot trades that have been reported already? // Maybe can change tradeid to be an hash of the trade itself to use as a key // and keep state for it .orderbook.start:{ .log.logInfo "kdb+tick ORDERBOOK date:",string[.z.P]," version:",string[.z.K],"_",string .z.k; .orderbook.fromSnapshot[]; - .u.conn[`tick]"(.u.sub[`order;`];`.u `i`L)"; + .orderbook.replay:1b; + .log.logDebug "Replaying from tpLogs"; + r:(.u.conn`tick)"(.u.sub[`order;`];`.u `i`L)"; + .u.rep . r; + .orderbook.replay:0b; + .log.logDebug "Replaying Done"; } -.orderbook.lastMatch:0Np; +.z.exit:{.log.logInfo"Running OrderBook zExit";.z.ts[];} +// should add a snapshot and replay from snapshot onward .z.ts:{ .log.logInfo"Running OrderBook ts"; .u.ts[]; - .orderbook.toSnapshot[]; + .orderbook.lastTs:.z.p; // nit - find a better way to do this offers: 0!select by sym,exch from order_ask where price=(min;price)fby([]sym;exch); bids: 0!select by sym,exch from order_bid where price=(max;price)fby([]sym;exch); - // TODO consider different approach where the two top of books are matched against - // each other instead of doing ToB for each side vs rest - // It should save on one query to order_asks and order_bids - .orderbook.lastMatch: max .orderbook.match each offers,bids; + // This simulations assumes that a crossed book will uncross the next tick by way of trading the top of the book + tomatch: offers,bids; + if[0=count tomatch;:(`nop)]; + .orderbook.lastMatch:max .orderbook.match each tomatch; // Build after matches, otherwise it'll look like crossed books always - // .orderbook.buildl2[]; // send this to subs? - - if[00:02:00 < .z.p-.orderbook.lastMatch; .log.logWarning "Nothing matched in over 2 minutes, is everything okay?" ]; + .u.conn[`tick](`.u.upd; `orderbookl2; .orderbook.buildl2[]); // send this to subs? + toReport:select from executions where time>=.orderbook.lastTs; + if[count toReport; .orderbook.reportTrade delete roid,aoid from toReport]; + if[00:02:00 < .z.p-.orderbook.lastMatch; .log.logWarning "Nothing matched in over 2 minutes, is everything okay?"]; } .orderbook.start[] \ No newline at end of file diff --git a/tickplant/orderbook/schema.q b/tickplant/orderbook/schema.q index ec91dc6..3d55df5 100644 --- a/tickplant/orderbook/schema.q +++ b/tickplant/orderbook/schema.q @@ -1,6 +1,8 @@ // side bid - aggressing, means selling -order_bid:update `g#sym from ([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); price:`float$(); size:`float$(); orderid:`guid$(); rcvtime:`timestamp$(); oid:`guid$()); +.orderbook.order_bid:update `g#sym from ([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); price:`float$(); size:`float$();side:`symbol$(); orderid:`guid$(); rcvtime:`timestamp$(); oid:`guid$()); // side offer - aggressing means buying -order_ask: order_bid +.orderbook.order_ask: .orderbook.order_bid -.orderbook.l2cols: `sym`exch`btime`btot_orders`bsz`bpx`apx`asz`atot_orders`atime; \ No newline at end of file +.orderbook.l2cols: `sym`exch`ind`btime`btot_orders`bsz`bpx`apx`asz`atot_orders`atime; + +executions:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$(); tradeid:`guid$(); roid:`guid$(); aoid:`guid$()); From 39f666e83bdc34e025e601c1e5ba8d677812e493 Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:36:59 +0200 Subject: [PATCH 29/31] adding l2 schema --- tickplant/common/schema.q | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tickplant/common/schema.q b/tickplant/common/schema.q index 54c0ecb..9f5229d 100644 --- a/tickplant/common/schema.q +++ b/tickplant/common/schema.q @@ -2,4 +2,6 @@ trade:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size: quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsize:`float$();asize:`float$();aprice:`float$()); -order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); \ No newline at end of file +order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$()); + +orderbookl2:([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); ind:`int$(); btime:`timestamp$();btot_orders:`int$();bsz:`float$();bpx:`float$();apx:`float$();asz:`float$();atot_orders:`int$();atime:`timestamp$()); \ No newline at end of file From db448d8817fcb84357bfbcfbf57debc7ea4776fd Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:37:30 +0200 Subject: [PATCH 30/31] bootstrap script --- tickplant/bootstrap.zsh | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 tickplant/bootstrap.zsh diff --git a/tickplant/bootstrap.zsh b/tickplant/bootstrap.zsh new file mode 100755 index 0000000..29a7424 --- /dev/null +++ b/tickplant/bootstrap.zsh @@ -0,0 +1,50 @@ +#!/bin/zsh +# $ ./bootstrap.zsh [run|stop] +# This script is used to bootstrap the tickplant service. It is called by the +# first start the ticker process, then the feedhandler process, then the hdb etc +# The script also stops the processes if they are running. + +#find if another process is running on 5010 and kill it if it is +echo "Stopping ticker|feed|hdb processes" +lsof -i :5010 | grep q | awk '{print $2}' | xargs kill -SIGTERM +lsof -i :5011 | grep q | awk '{print $2}' | xargs kill -SIGTERM +lsof -i :5012 | grep q | awk '{print $2}' | xargs kill -SIGTERM + + + +# stop the processes if running in mode stop +if [[ $1 == "stop" ]]; then + echo "Stopped tickplant processes" + exit 0 +fi + +if [[ $1 == "clean" ]]; then + echo "Cleaned logs" + rm -v 5010 + rm -v 5012 + rm -rfv data/* + rm sym_px + exit 0 +fi +set -e # exit on error +alias q="QHOME=~/shadow/q rlwrap -H /var/tmp/hist${PPID} /Users/emelis/shadow/q/m64/q" +TICK_LOG=logs/tick.log +q tick.q -t 1000 -c 31 317 2>&1 > ${TICK_LOG} & +TICK="${!}" +echo "TICKER PID: ${TICK} " + +#find if another instance of feedhandler is running and kill it if it is +ps -ef | grep feedhandler | grep -v grep | awk '{print $2}' | xargs kill -SIGTERM +q feedhandler/feedhandler.q :5010 -t 1000 -c 31 317 2>&1 > logs/feedhandler.log & +FH="${!}" +echo "FH PID: ${FH}" + +q -p 5012 -c 31 317 > logs/hdb.log & +HDB="${!}" +echo "HDB PID: ${HDB}" + +ps -ef | grep orderbook | grep -v grep | awk '{print $2}' | xargs kill -SIGTERM +q orderbook/orderbook.q :5010 -t 1 -c 31 317 2>&1 > logs/orderbook.log & +ORDERBOOK="${!}" +echo "ORDERBOOK PID: ${ORDERBOOK}" + From 2f7fa52195749eaa3f90027d8c5a86772dee3f6e Mon Sep 17 00:00:00 2001 From: quercussuber Date: Sun, 18 Aug 2024 18:38:39 +0200 Subject: [PATCH 31/31] feedhandler refactor better simulations --- tickplant/feedhandler/feedhandler.q | 24 ++++++++++++++++++------ tickplant/feedhandler/lib.q | 16 ++++------------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tickplant/feedhandler/feedhandler.q b/tickplant/feedhandler/feedhandler.q index e200ccb..153940e 100644 --- a/tickplant/feedhandler/feedhandler.q +++ b/tickplant/feedhandler/feedhandler.q @@ -11,21 +11,33 @@ system"l feedhandler/lib.q" upd:{'implementme} -SIM_ORDERS:1000 +SIM_ORDERS:100 .fh.i:0; +.fh.genOrders:{[n;start] + sims:n?0!sym_px; + sds:n?sides; + // Generating integers then dividing to get floats of set precision + // also forcing BUY prices to be generally smaller than SELL prices (can still be crossed) + px:abs sims[`px]+(-1 1)[sds]*floor[sims`px]%100; + sz:0.01|(n?100)%sims`px; + batch:(n#start;n?exchs;sims`sym;px;sz;sds;neg[n]?0Ng); + batch + } + .fh.ts:{ - batch:genOrders[SIM_ORDERS;.z.p]; + n:SIM_ORDERS; + batch:.fh.genOrders[n;.z.p]; .fh.i+:SIM_ORDERS; // TODO trap errors like ticker being down .u.conn[`tick](`.u.upd;`order;batch); - // Moving the market by symbol either up/down by a random amount so prices look like they are moving - // NIT could have used the prices already generated for batch[`price] but that doenst generate - // the same price movement for all symbols - sym_px[`px]+:@[;(last 1?count sym_px[`px])? count sym_px[`px];neg] %[;1000]count[sym_px`px]?100; + // Simulating market movements + `sym_px set select px:med price,sz:med size by sym from flip cols[order]!batch; }; +.z.exit:{`:sym_px set sym_px;} + .z.ts:{ .log.logInfo"Running Fedhandler ts"; .u.ts[]; diff --git a/tickplant/feedhandler/lib.q b/tickplant/feedhandler/lib.q index 2e36d3a..9624070 100644 --- a/tickplant/feedhandler/lib.q +++ b/tickplant/feedhandler/lib.q @@ -1,17 +1,9 @@ -exchs:`NYSE`NASDAQ`AMEX; +exchs:enlist`MYX; sides:`u?`B`S; -syms:10#`$'read0`:common/sp500.txt; +syms:1#`$'read0`:common/sp500.txt; // Carry price movements across restarts -sym_px:@[get;`:sym_px;{`sym`px!(syms;count[syms]?100.)}]; -`:sym_px set sym_px; - -// order -genOrders:{[n;start] - sims:n?flip sym_px; - sds:n?sides; - // Generating integers up to 1000 and then dividing by 100 to get floats of set precision - // also forcing BUY prices to be generally smaller than SELL prices (can still be crossed) - (asc start+0D00:00:01*til n;n?exchs;sims`sym;sims[`px]+(-1 1)[sds]*(n?1+1000)%100;n?10.;sds;neg[n]?0Ng)}; \ No newline at end of file +sym_px:@[get;`:sym_px;{px:(count[syms]?1+100);([sym:syms]px:px;sz:px%10)}]; +`:sym_px set sym_px; \ No newline at end of file