Tp first

.gitignore

@@ -0,0 +1,5 @@
+tplog_*
+db/
+logs/
+data/
+sym_px

tickplant/bootstrap.zsh

@@ -0,0 +1,50 @@
+#!/bin/zsh
+# $ ./bootstrap.zsh [run|stop]
+# This script is used to bootstrap the tickplant service. It is called by the
+# first start the ticker process, then the feedhandler process, then the hdb etc
+# The script also stops the processes if they are running.
+
+#find if another process is running on 5010 and kill it if it is
+echo "Stopping ticker|feed|hdb processes"
+lsof -i :5010 | grep q | awk '{print $2}' | xargs kill -SIGTERM
+lsof -i :5011 | grep q | awk '{print $2}' | xargs kill -SIGTERM
+lsof -i :5012 | grep q | awk '{print $2}' | xargs kill -SIGTERM
+
+
+
+# stop the processes if running in mode stop
+if [[ $1 == "stop" ]]; then
+    echo "Stopped tickplant processes"
+    exit 0
+fi
+
+if [[ $1 == "clean" ]]; then
+    echo "Cleaned logs"
+    rm -v 5010
+    rm -v 5012
+    rm -rfv data/*
+    rm sym_px
+    exit 0
+fi
+set -e # exit on error
+alias q="QHOME=~/shadow/q rlwrap -H /var/tmp/hist${PPID} /Users/emelis/shadow/q/m64/q"
+TICK_LOG=logs/tick.log
+q tick.q -t 1000 -c 31 317 2>&1 > ${TICK_LOG} &
+TICK="${!}"
+echo "TICKER PID: ${TICK} "
+
+#find if another instance of feedhandler is running and kill it if it is
+ps -ef | grep feedhandler | grep -v grep | awk '{print $2}' | xargs kill -SIGTERM
+q feedhandler/feedhandler.q :5010 -t 1000 -c 31 317 2>&1 > logs/feedhandler.log &
+FH="${!}"
+echo "FH PID: ${FH}"
+
+q -p 5012 -c 31 317 > logs/hdb.log &
+HDB="${!}"
+echo "HDB PID: ${HDB}"
+
+ps -ef | grep orderbook | grep -v grep | awk '{print $2}' | xargs kill -SIGTERM
+q orderbook/orderbook.q :5010 -t 1 -c 31 317 2>&1 > logs/orderbook.log &
+ORDERBOOK="${!}"
+echo "ORDERBOOK PID: ${ORDERBOOK}"
+

tickplant/cep/cep.q

@@ -0,0 +1,45 @@
+/q cep/cep.q SRC :5010
+
+system"l tick/r.q"
+system"l common/conn.q"
+system"l cep/lib.q"
+
+.u.conn:{hopen `$":",x}each enlist[`tick]#.u.x;
+
+// uend:{
+//     if[null x;'daycantbenull];
+//     // t:tables`.;
+//     t:`trade`order;
+//     t@:where `g=attr each t@\:`sym;
+//     .Q.hdpf[.u.x `hdb;HDBROOT;x;`sym];
+//     @[;`sym;`g#] each t;
+//     };
+
+// k).Q.hdpf:{[h;d;p;f](@[`.;;0#]dpft[d;p;f]@)'t@>(#.:)'t:`trade`order;if[h:@[hopen;h;0];h"\\l .";>h]}
+// .u.end:{
+//     if[null x;'daycantbenull];
+//     .cep.end x;
+//     uend x;
+//     }
+
+.cep.start:{
+    .log.logInfo "kdb+tick CEP date:",string[.z.P]," version:",string[.z.K],"_",string .z.k;
+    .cep.replay::1b;
+    .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)";
+    .cep.replay::0b;
+    }
+
+.cep.lastTs:.z.P;
+
+.cep.ts:{
+    // if[.z.P-.cep.lastTs>=00:01:00.000; ohlcvv_1m:: calcOHLCVV_1m[]];
+    // mid_1m::calcMID_1m[];
+    }
+
+.z.ts:{
+    .log.logInfo"Running CEP ts";
+    .u.ts[];
+    .cep.ts[];
+  }
+
+.cep.start[]

tickplant/cep/lib.q

@@ -0,0 +1,63 @@
+calcOHLCVV_1m:{
+    select
+    open: first price,
+    high: max price,
+    low: min price,
+    close: last price,
+    volume: sum size,
+    vwap: size wavg price
+    by sym, 1 xbar time.minute
+    from trade
+    }
+
+calcOHLCVV_1d:{
+    select
+    open: first price,
+    high: max price,
+    low: min price,
+    close: last price,
+    volume: sum size,
+    vwap: size wavg price
+    by sym
+    from trade
+    }
+
+// For sampling purposes only - not a realistic calculation
+calcTickMID:{select mid_price:avg price by exch, sym, minute: 1 xbar time.minute from order}
+
+calcVWAP:{[s;e;r]
+    $[r=1; select sym, exch, second, vwap,res:r from vwap_1s where sym=s,exch=e;
+           select vwap: sum[wsp]%sum ws, sum num_trade by sym,exch,r xbar second from vwap_1s where sym=s,exch=e]
+    };
+
+last_ts:.z.P;
+
+
+
+// .cep.end:{.cep.ts[]; ohlcvv_1d::calcOHLCVV_1d[];}
+
+upd:{
+    .[insert;(x;y);{.log.logError"Insert failed for table=",string[y]," with error=",x}[;x]];
+    if[`trade=x; 
+        if[not .cep.replay; lastTrade,:y];
+        // having this separate table is questionable - native wavg performs well compared to running sum x*y%sum y
+        // but it feels like there is a scale point at which it will be faster to use the running sum when calculating
+        // multiple vwap horizons instead of recalculating the wavg for each horizon
+        // We shall see...
+        // leaving vwap here to compare results with the native wavg
+        // if[99h=type y; y:enlist y];
+       // vwap_1s+: select wsp:size wsum price, ws:sum size, vwap:size wavg price, num_trade:count i by exch,sym,time.second from trade
+         //   where ([]time.second;sym;exch) in select time.second,sym,exch from y
+        ];
+    if[`order=x;  if[not .cep.replay; lastOrder,:y]];
+    };
+
+// MACD
+ema12:ema[2%13;]
+ema26:ema[2%17;]
+macd: {ema12[x] - ema26 x}
+signal:{ema[2%10;macd x]}
+
+calcMACD:{[s;e]
+    select minute, exch, ema12 close, ema26 close, macd close, signal macd close by sym from calcOHLCVV_1m[]
+    }

tickplant/cep/schema.q

@@ -0,0 +1,3 @@
+HDB_PERSIST:`trade`order`vwap_1s;
+
+vwap_1s:update `g#sym from ([exch:`$();sym:`$();second:`second$()];wsp:`float$();ws:`float$();vwap:`float$();num_trade:`long$());

tickplant/common/conn.q

@@ -0,0 +1,15 @@
+/ get the ticker plant and history ports, defaults are 5010,5012
+.u.x:`tick`hdb!.z.x,(count .z.x)_(":5010";":5012");
+
+.z.pc:{show "Process at handle ",string[p:.u.conn ? x]," disconnected"; .u.pc p}
+
+.u.pc:{.u.conn[x]:0Ni; show .u.conn}
+
+.u.ts:{
+    if[ null .u.conn`tick;
+        show "Retrying tickerplant connection ...";
+        h:@[hopen;`$":",.u.x`tick;{show "Cannot establish tickerplant connection with ",x; 0Ni}];
+        .u.conn[`tick]: h;
+        if[ not null .u.conn`tick; .u.rep .(.u.conn`tick)"(.u.sub[`;`];`.u `i`L)"];
+    ];
+  }

tickplant/common/log.q

@@ -0,0 +1,27 @@
+if[`log in key`.;:(`log_already_loaded)];
+\d .log
+LEVELS:`info`warning`error`debug
+// .log.test[`something]: {(levels ? `info) >= levels ? `warning}
+// Use 
+// info when you want to log high level state and flow information
+// warning when you want to log messages that indicates non-ideal state or potential problems
+// error when you want to log meesages that indicate a non-recoverable error
+// debug when you want to log a debug message
+level: LEVELS ? `debug
+setLogLevel:{[l] .log.level::LEVELS ? l};
+canLog:{.log.level <= .log.LEVELS ?x}
+
+logmsg:{[l;msg]
+    if[not canLog l; :(::)];
+    if[11h = type msg; msg:string msg];
+    0N!string[.z.P]," ",upper[string l]," ",msg;
+  }
+
+logInfo:logmsg[`info]
+logWarning:logmsg[`warning]
+logError:logmsg[`error]
+logDebug:logmsg[`debug]
+
+// TODO be nice to have both stdout/stderr and file logging
+// and also only print to stdout if the level is lower than set up
+\d .

tickplant/common/schema.q

@@ -0,0 +1,7 @@
+trade:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$(); tradeid:`guid$());
+
+quote:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();bprice:`float$();bsize:`float$();asize:`float$();aprice:`float$());
+
+order:([]time:`timestamp$();exch:`symbol$();sym:`symbol$();price:`float$();size:`float$();side:`symbol$();orderid:`guid$());
+
+orderbookl2:([]time:`timestamp$(); exch:`symbol$(); sym:`symbol$(); ind:`int$(); btime:`timestamp$();btot_orders:`int$();bsz:`float$();bpx:`float$();apx:`float$();asz:`float$();atot_orders:`int$();atime:`timestamp$());