From 7c49559e76ed20c433db5b3757aade15ae69b978 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 8 Nov 2016 11:43:42 +0800 Subject: [PATCH 1/6] add docker file --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb9cec8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM hseeberger/scala-sbt +ADD . /burrower/ +RUN ls /burrower +WORKDIR /burrower +RUN sbt assembly +CMD java -cp "target/scala-2.11/burrower-0.2-SNAPSHOT.jar:conf/" com.github.splee.burrower.OffsetMonitor From 69c839ed5905c76ece525120e8c34d2524478998 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 8 Nov 2016 14:13:07 +0800 Subject: [PATCH 2/6] allow to config by environment --- Dockerfile | 1 - conf/application.conf.example | 4 ++- src/main/resources/reference.conf | 6 +++-- .../github/splee/burrower/OffsetMonitor.scala | 27 +++++++++---------- .../splee/burrower/write/InfluxWriter.scala | 14 +++++----- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb9cec8..cfa5069 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM hseeberger/scala-sbt ADD . /burrower/ -RUN ls /burrower WORKDIR /burrower RUN sbt assembly CMD java -cp "target/scala-2.11/burrower-0.2-SNAPSHOT.jar:conf/" com.github.splee.burrower.OffsetMonitor diff --git a/conf/application.conf.example b/conf/application.conf.example index f0a1c35..4df49c5 100644 --- a/conf/application.conf.example +++ b/conf/application.conf.example @@ -12,6 +12,8 @@ burrower { host = 127.0.0.1 port = 8086 database = "metrics" - series = "kafka.consumer_lag" + series = "kafka_consumer_lag" + username = "" + password = "" } } diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index 4c719da..c5f0f84 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -7,7 +7,9 @@ burrower { influx = { host = "localhost" port = 8086 - database = "metrics" - series = "kafka.consumer_lag" + database = "burrower" + series = "kafka_consumer_lag" + userName = "" + password = "" } } \ No newline at end of file diff --git a/src/main/scala/com/github/splee/burrower/OffsetMonitor.scala b/src/main/scala/com/github/splee/burrower/OffsetMonitor.scala index a6a923f..8c5ec82 100644 --- a/src/main/scala/com/github/splee/burrower/OffsetMonitor.scala +++ b/src/main/scala/com/github/splee/burrower/OffsetMonitor.scala @@ -2,8 +2,9 @@ package com.github.splee.burrower import com.github.splee.burrower.lag.{BurrowConsumerStatus, BurrowPartitionLag, Lag, LagGroup} import com.github.splee.burrower.write.{ConsoleWriter, InfluxWriter, Writer} -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions} import com.typesafe.scalalogging.LazyLogging + import scalaj.http._ import play.api.libs.json._ @@ -11,10 +12,8 @@ object OffsetMonitor extends LazyLogging { def main(args: Array[String]): Unit = { val conf = ConfigFactory.load() - - val bHost = conf.getString("burrower.burrow.host") - val bPort = conf.getInt("burrower.burrow.port") - + var bHost:String = sys.env.getOrElse("BURROW_HOST",conf.getString("burrower.burrow.host")).toString + val bPort = sys.env.getOrElse("BURROW_PORT",conf.getInt("burrower.burrow.port")).toString.toInt val writer = buildWriter(conf) logger.info("Creating monitor...") @@ -30,7 +29,7 @@ object OffsetMonitor extends LazyLogging { } def buildWriter(conf: Config): Writer = { - val writerType = conf.getString("burrower.writer") + val writerType = sys.env.getOrElse("BURROWER_WRITER",conf.getString("burrower.writer")) logger.info(f"Creating $writerType writer...") if (writerType == "console") @@ -46,18 +45,18 @@ object OffsetMonitor extends LazyLogging { def buildInfluxWriter(conf: Config): InfluxWriter = new InfluxWriter( - conf.getString("burrower.influx.host"), - conf.getInt("burrower.influx.port"), - conf.getString("burrower.influx.database"), - conf.getString("burrower.influx.series"), - conf.getString("burrower.influx.userName"), - conf.getString("burrower.influx.password") + sys.env.getOrElse("INFLUXDB_HOST",conf.getString("burrower.influx.host")), + sys.env.getOrElse("INFLUXDB_PORT",conf.getInt("burrower.influx.port")).toString.toInt, + sys.env.getOrElse("INFLUXDB_DATABASE",conf.getString("burrower.influx.database")), + sys.env.getOrElse("INFLUXDB_SERIES",conf.getString("burrower.influx.series")), + sys.env.getOrElse("INFLUXDB_USER",conf.getString("burrower.influx.userName")), + sys.env.getOrElse("INFLUXDB_PASSWORD",conf.getString("burrower.influx.password")) ) } class OffsetMonitor ( - burrowHost: String, - burrowPort: Number, + burrowHost: String="localhost", + burrowPort: Number=8000, writer: Writer ) extends Runnable with LazyLogging { diff --git a/src/main/scala/com/github/splee/burrower/write/InfluxWriter.scala b/src/main/scala/com/github/splee/burrower/write/InfluxWriter.scala index 8a97b59..6b73872 100644 --- a/src/main/scala/com/github/splee/burrower/write/InfluxWriter.scala +++ b/src/main/scala/com/github/splee/burrower/write/InfluxWriter.scala @@ -10,12 +10,12 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.util.{Failure, Success} class InfluxWriter( - influxHost: String, - influxPort: Int, - influxDatabase: String, - influxSeries: String, - userName: String, - password: String + influxHost: String="localhost", + influxPort: Int=8086, + influxDatabase: String="burrower", + influxSeries: String="kafka_consumer_lag", + userName: String="", + password: String="" ) extends Writer with LazyLogging { val influxdb = InfluxDB.connect(influxHost, influxPort, userName, password) @@ -36,7 +36,7 @@ class InfluxWriter( case Success(v) => logger.debug("Metrics sent to InfluxDB") case Failure(e) => - logger.debug(f"Sending metrics to InfluxDB failed: ${e.getMessage}") + logger.error(f"Sending metrics to InfluxDB failed: ${e.getMessage}") }) } } From 09c05007f6ebfdefbc9e83133d7d3861d04b3eea Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 8 Nov 2016 14:20:49 +0800 Subject: [PATCH 3/6] modify read me --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 98b3c34..b0d27dc 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ Metrics are sent to InfluxDB with the following values and tags: 1. Update configuration, saving it as `conf/application.conf` (example config file: `burrow/conf/application.conf.example`) 1. `java -cp target/scala-2.11/burrower-0.2-SNAPSHOT.jar:conf/ com.github.splee.burrower.OffsetMonitor` +## Docker build & Run +* cd burrower +* docker build . +* docker run -it --env BURROW_HOST=127.0.0.1 --env BURROWER_WRITER=influxdb --env INFLUXDB_HOST=127.0.0.1 ## Planned Features * Ability to specify any class implmenting `com.github.splee.burrower.write.Writer` to write metrics to custom back ends. From b123123307335617a155ca40852ef53c11533b62 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 8 Nov 2016 14:21:56 +0800 Subject: [PATCH 4/6] format --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b0d27dc..25e725e 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,11 @@ Metrics are sent to InfluxDB with the following values and tags: 1. `java -cp target/scala-2.11/burrower-0.2-SNAPSHOT.jar:conf/ com.github.splee.burrower.OffsetMonitor` ## Docker build & Run + * cd burrower * docker build . * docker run -it --env BURROW_HOST=127.0.0.1 --env BURROWER_WRITER=influxdb --env INFLUXDB_HOST=127.0.0.1 + ## Planned Features * Ability to specify any class implmenting `com.github.splee.burrower.write.Writer` to write metrics to custom back ends. From f4f3b486519ad961de7e22ab34a7b1f55b827244 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 8 Nov 2016 14:25:01 +0800 Subject: [PATCH 5/6] no message --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 25e725e..13d951f 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Metrics are sent to InfluxDB with the following values and tags: * cd burrower * docker build . -* docker run -it --env BURROW_HOST=127.0.0.1 --env BURROWER_WRITER=influxdb --env INFLUXDB_HOST=127.0.0.1 +* docker run -it --env BURROW_HOST=127.0.0.1 --env BURROWER_WRITER=influxdb --env INFLUXDB_HOST=127.0.0.1 imageName ## Planned Features From 745f8439990c5865b34f5e8fb60e15e281967e26 Mon Sep 17 00:00:00 2001 From: ben Date: Mon, 14 Nov 2016 14:51:47 +0800 Subject: [PATCH 6/6] fix parse bug --- .../com/github/splee/burrower/lag/BurrowConsumerStatus.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/github/splee/burrower/lag/BurrowConsumerStatus.scala b/src/main/scala/com/github/splee/burrower/lag/BurrowConsumerStatus.scala index db6d424..8425f84 100644 --- a/src/main/scala/com/github/splee/burrower/lag/BurrowConsumerStatus.scala +++ b/src/main/scala/com/github/splee/burrower/lag/BurrowConsumerStatus.scala @@ -14,6 +14,6 @@ case class BurrowConsumerStatus( partitions: List[BurrowPartitionLag], partition_count: Int, maxlag: Option[BurrowPartitionLag], - totallag: Int + totallag: Long )