diff --git a/README.md b/README.md index 556df64..9cb8551 100644 --- a/README.md +++ b/README.md @@ -11,20 +11,20 @@ The easiest way to run the scraper with stand-alone mode is to use the neat CLI sbt assembly # Run it with -java -jar target/*/scraper.jar --categories prodaja --pages 10 +java -jar scraper/*/*/scraper.jar --categories prodaja --pages 10 ``` By default, the scraper spits out [JSON]. ```bash -java -jar target/*/scraper.jar --categories prodaja --pages 2 | jq -R 'fromjson?' +java -jar scraper/*/*/scraper.jar --categories prodaja --pages 2 | jq -R 'fromjson?' ``` So to make things bit easier for your eyes your can use [jq] to format or restructure output further for example to CSV. ```bash -java -jar target/*/scraper.jar --categories prodaja --pages 10 \ +java -jar scraper/*/*/scraper.jar --categories prodaja --pages 10 \ | jq -R 'fromjson?' \ | jq -r "([.refNumber, .title, .price, .location.latitude, .location.longitude]) | @csv" \ > prodaja.csv @@ -33,7 +33,7 @@ java -jar target/*/scraper.jar --categories prodaja --pages 10 \ Adjusting parallelism and other [fine `application.conf` switches][configuration] can be easily done via loading of different configuration. ```bash -java -Dconfig.resource=quick.conf -jar target/*/scraper.jar --categories najem +java -Dconfig.resource=quick.conf -jar scraper/*/*/scraper.jar --categories najem ``` Some [configuration options][configuration] can also be adjusted via environment variables i.e. @@ -43,10 +43,15 @@ INITIAL_CATEGORIES=prodaja,najem CATEGORY_PAGES_LIMIT=3 ``` +## Resources + +- https://blog.softwaremill.com/running-akka-cluster-on-kubernetes-e4cd2913e951 + + ## Authors - [Oto Brglez](https://github.com/otobrglez) -[configuration]: src/main/resources/application.conf +[configuration]: scraper/src/main/resources/application.conf [jq]: https://stedolan.github.io/jq/ [JSON]: https://www.json.org/json-en.html diff --git a/build.sbt b/build.sbt index 9ab08a9..d1a19f5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,60 +1,33 @@ -name := "pinkstack-realestate" - -version := "0.0.1" - -scalaVersion := "2.13.2" - -libraryDependencies ++= Seq( - // Akka and Akka Streams - "com.typesafe.akka" %% "akka-stream" % "2.6.6", - "com.typesafe.akka" %% "akka-http" % "10.1.12", - - // FP - "org.typelevel" %% "cats-core" % "2.0.0", - - // Lenses - "com.github.julien-truffaut" %% "monocle-core" % "2.0.3", - "com.github.julien-truffaut" %% "monocle-macro" % "2.0.3", - "com.github.julien-truffaut" %% "monocle-law" % "2.0.3" % "test", - - // Configuration - "com.typesafe" % "config" % "1.4.0", - "com.github.pureconfig" %% "pureconfig" % "0.12.3", - - // URL Parsing - "io.lemonlabs" %% "scala-uri" % "2.2.2", - - // CLI - "com.monovore" %% "decline" % "1.2.0", - - // Parsing - "org.jsoup" % "jsoup" % "1.13.1", - - // JSON - "io.circe" %% "circe-core" % "0.12.3", - "io.circe" %% "circe-generic" % "0.12.3", - "io.circe" %% "circe-parser" % "0.12.3", - - // Logging - "ch.qos.logback" % "logback-classic" % "1.2.3", - "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2", - "com.typesafe.akka" %% "akka-slf4j" % "2.6.6", - - // Testing - "org.scalatest" %% "scalatest" % "3.2.0" % "test", - "org.scalatest" %% "scalatest-flatspec" % "3.2.0" % "test", - "org.scalatest" %% "scalatest-shouldmatchers" % "3.2.0" % "test" -) - -scalacOptions ++= Seq( - "-deprecation", - "-encoding", "UTF-8", - "-feature", - "-target:jvm-1.11", - "-unchecked" -) - -enablePlugins(JavaAppPackaging) - -mainClass in assembly := Some("com.pinkstack.realestate.apps.Scraper") -assemblyJarName in assembly := "scraper.jar" \ No newline at end of file +import Dependencies._ +import Settings._ +import scala.sys.process._ + +lazy val scraper = (project in file("scraper")) + .settings(sharedSettings: _*) + .settings( + name := "scraper", + Compile / mainClass := Some(""), + libraryDependencies ++= + akka ++ akkaHttp ++ cats + ++ monocle ++ configurationLibs ++ scalaUri + ++ decline ++ jsoup ++ circe + ++ akkaHttpCirce ++ logging ++ scalaTest, + Compile / mainClass := Some("com.pinkstack.realestate.scraper.apps.Scraper"), + ) + .enablePlugins(JavaAppPackaging) + .settings(assemblyJarName in assembly := "scraper.jar") + +lazy val cluster = (project in file("cluster")) + .settings(sharedSettings: _*) + .settings( + name := "cluster", + Compile / mainClass := Some(""), + libraryDependencies ++= + akka ++ akkaHttp ++ cats + ++ monocle ++ configurationLibs ++ scalaUri + ++ decline ++ jsoup ++ circe + ++ akkaHttpCirce ++ logging ++ scalaTest + ) + .enablePlugins(JavaAppPackaging) + .settings(assemblyJarName in assembly := "cluster.jar") + .dependsOn(scraper) diff --git a/cluster/src/main/scala/com/pinkstack/realestate/cluster/NodeApp.scala b/cluster/src/main/scala/com/pinkstack/realestate/cluster/NodeApp.scala new file mode 100644 index 0000000..6e02dee --- /dev/null +++ b/cluster/src/main/scala/com/pinkstack/realestate/cluster/NodeApp.scala @@ -0,0 +1,53 @@ +/** + * Author: Oto Brglez - + */ +package com.pinkstack.realestate.cluster + +import akka.actor.typed._ +import akka.actor.typed.scaladsl._ +import akka.cluster.ClusterEvent.MemberEvent +import akka.cluster.typed._ +import com.monovore.decline._ +import com.typesafe.config.{Config, ConfigFactory} + +object ClusterListener { + + sealed trait Event + + case class ClusterChange(event: MemberEvent) extends Event + + def apply(): Behavior[Event] = Behaviors.setup[Event] { context => + val clusterEvents: ActorRef[MemberEvent] = context.messageAdapter(ClusterChange) + Cluster(context.system).subscriptions ! Subscribe(clusterEvents, classOf[MemberEvent]) + + Behaviors.receiveMessage { + case ClusterChange(event: MemberEvent) => + context.log.info("🐝 {}", event) + Behaviors.same + } + } +} + +object Node { + def apply(): Behavior[Nothing] = Behaviors.setup[Nothing] { context => + context.spawn(ClusterListener(), "ClusterListener") + Behaviors.empty + } +} + +object System { + val configuration: Int => Config = port => + ConfigFactory.parseString( + s"""akka.remote.artery.canonical.port=$port""".stripMargin) + .withFallback(ConfigFactory.load("clusteringV4.conf")) + + def apply(port: Int): ActorSystem[Nothing] = + ActorSystem[Nothing](Node(), "AppV4", configuration(port)) +} + +object NodeApp extends CommandApp( + name = "node", header = "Boots up a single (seed) cluster node", + main = for { + port <- Opts.option[Int]("port", "port number").withDefault(0) + } yield System(port) +) \ No newline at end of file diff --git a/project/Dependencies.scala b/project/Dependencies.scala new file mode 100644 index 0000000..861fd9d --- /dev/null +++ b/project/Dependencies.scala @@ -0,0 +1,98 @@ +import sbt._ + +object Dependencies { + + object V { + val akka = "2.6.6" + val akkaHttp = "10.1.12" + + val cats = "2.0.0" + + val monocle = "2.0.3" + + val typesafeConfig = "1.4.0" + val pureConfig = "0.12.3" + + val scalaUri = "2.2.2" + + val decline = "1.2.0" + + val jsoup = "1.13.1" + val circe = "0.12.3" + + val akkaHttpCirce = "1.31.0" + val logbackClassic = "1.2.3" + val scalaLogging = "3.9.2" + val akkaSlf4j = "2.6.6" + + val scalaTest = "3.2.0" + } + + lazy val akka: Seq[ModuleID] = Seq( + "com.typesafe.akka" %% "akka-actor-typed" % V.akka, + "com.typesafe.akka" %% "akka-stream" % V.akka, + + "com.typesafe.akka" %% "akka-cluster" % V.akka, + "com.typesafe.akka" %% "akka-cluster-typed" % V.akka, + + "com.typesafe.akka" %% "akka-discovery" % V.akka, + "com.typesafe.akka" %% "akka-cluster-tools" % V.akka + ) + + lazy val akkaHttp: Seq[ModuleID] = Seq( + "com.typesafe.akka" %% "akka-http" % V.akkaHttp + ) + + lazy val cats: Seq[ModuleID] = Seq( + "org.typelevel" %% "cats-core" % V.cats + ) + + lazy val monocle: Seq[ModuleID] = Seq( + "com.github.julien-truffaut" %% "monocle-core" % V.monocle, + "com.github.julien-truffaut" %% "monocle-macro" % V.monocle, + "com.github.julien-truffaut" %% "monocle-law" % V.monocle % "test" + ) + + lazy val configurationLibs: Seq[ModuleID] = Seq( + "com.typesafe" % "config" % V.typesafeConfig, + "com.github.pureconfig" %% "pureconfig" % V.pureConfig + ) + + lazy val scalaUri: Seq[ModuleID] = Seq( + "io.lemonlabs" %% "scala-uri" % V.scalaUri + ) + + lazy val decline: Seq[ModuleID] = Seq( + "com.monovore" %% "decline" % V.decline + ) + + lazy val jsoup: Seq[ModuleID] = Seq( + "org.jsoup" % "jsoup" % V.jsoup + ) + + lazy val circe: Seq[ModuleID] = Seq( + "io.circe" %% "circe-core" % V.circe, + "io.circe" %% "circe-generic" % V.circe, + "io.circe" %% "circe-parser" % V.circe + ) + + lazy val akkaHttpCirce: Seq[ModuleID] = Seq( + "de.heikoseeberger" %% "akka-http-circe" % V.akkaHttpCirce + ) + + lazy val logging: Seq[ModuleID] = Seq( + "ch.qos.logback" % "logback-classic" % V.logbackClassic, + "com.typesafe.scala-logging" %% "scala-logging" % V.scalaLogging, + "com.typesafe.akka" %% "akka-slf4j" % V.akkaSlf4j + ) + + lazy val scalaTest: Seq[ModuleID] = Seq( + "org.scalatest" %% "scalatest" % V.scalaTest % "test", + "org.scalatest" %% "scalatest-flatspec" % V.scalaTest % "test", + "org.scalatest" %% "scalatest-shouldmatchers" % V.scalaTest % "test" + ) + + lazy val javaAgentsLibs: Seq[ModuleID] = Seq( + "io.kamon" % "kanela-agent" % "1.0.5" + ) +} diff --git a/project/Settings.scala b/project/Settings.scala new file mode 100644 index 0000000..757358b --- /dev/null +++ b/project/Settings.scala @@ -0,0 +1,35 @@ +import sbt._ +import sbt.Keys._ +import sbt.nio.Keys._ + +object Settings { + lazy val sharedSettings = Seq( + Compile / packageDoc / mappings := Seq(), + Global / onChangedBuildSource := ReloadOnSourceChanges, + + scalaVersion := "2.13.2", + organization := "com.pinkstack.realestate", + + Test / fork := true, + Test / javaOptions ++= Seq("-Xmx2g"), + + scalacOptions ++= Seq( + "-deprecation", + "-encoding", "UTF-8", + "-feature", + "-explaintypes", + "-unchecked", + "-Xlint:-unused,_", + "-language:implicitConversions", + "-language:higherKinds", + "-language:postfixOps", + "-Yrangepos", + "-target:jvm-1.11" + ), + resolvers ++= Seq( + Resolver.bintrayRepo("ovotech", "maven"), + "Confluent Maven Repository" at "https://packages.confluent.io/maven/", + "jitpack" at "https://jitpack.io" + ) + ) +} \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index bff0726..3eca7d1 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,2 +1,15 @@ +logLevel := Level.Warn + +// addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") + addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.3") + addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") + +addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent" % "0.1.5") + +addSbtPlugin("io.kamon" % "sbt-kanela-runner" % "2.0.6") + +resolvers += Resolver.sonatypeRepo("releases") + + diff --git a/src/main/resources/akka.conf b/scraper/src/main/resources/akka.conf similarity index 100% rename from src/main/resources/akka.conf rename to scraper/src/main/resources/akka.conf diff --git a/src/main/resources/application.conf b/scraper/src/main/resources/application.conf similarity index 81% rename from src/main/resources/application.conf rename to scraper/src/main/resources/application.conf index 0227d84..3febf6a 100644 --- a/src/main/resources/application.conf +++ b/scraper/src/main/resources/application.conf @@ -11,17 +11,17 @@ pagination { } fetching { - categories-parallelism = 4 - estates-parallelism = 4 + categories-parallelism = 3 + estates-parallelism = 3 } throttling-estates = { # Number of elements "per" - elements = 20 + elements = 10 # Duration "per second" per = 1 - maximum-burst = 30 + maximum-burst = 20 } diff --git a/scraper/src/main/resources/clusteringV4.conf b/scraper/src/main/resources/clusteringV4.conf new file mode 100644 index 0000000..f02205e --- /dev/null +++ b/scraper/src/main/resources/clusteringV4.conf @@ -0,0 +1,56 @@ +akka { + # Log the complete configuration at INFO level when the actor system is started. + # This is useful when you are uncertain of what configuration is used. + ## log-config-on-start = on + + # Loggers to register at boot time (akka.event.Logging$DefaultLogger logs + # to STDOUT) + loggers = ["akka.event.slf4j.Slf4jLogger"] + + # Log level used by the configured loggers (see "loggers") as soon + # as they have been started; before that, see "stdout-loglevel" + # Options: OFF, ERROR, WARNING, INFO, DEBUG + loglevel = "INFO" # "DEBUG" + + # Log level for the very basic logger activated during ActorSystem startup. + # This logger prints the log messages to stdout (System.out). + # Options: OFF, ERROR, WARNING, INFO, DEBUG + stdout-loglevel = "DEBUG" + + # Filter of log events that is used by the LoggingAdapter before + # publishing log events to the eventStream. + logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" + + http { + client { + idle-timeout = 120s + } + host-connection-pool { + idle-timeout = 200s + } + } + + actor { + provider = "cluster" + } + + remote { + log-remote-lifecycle-events = off + artery.enabled = on + artery.canonical.hostname = "127.0.0.1" + artery.canonical.port = 0 + } + + cluster { + seed-nodes = [ + "akka://AppV4@127.0.0.1:4000", + "akka://AppV4@127.0.0.1:4001", + ] + auto-down-unreachable-after = 10s + downing-provider-class = "akka.cluster.sbr.SplitBrainResolverProvider" + } + + coordinated-shutdown.exit-jvm = on + coordinated-shutdown.terminate-actor-system = on + cluster.shutdown-after-unsuccessful-join-seed-nodes = 20s +} \ No newline at end of file diff --git a/scraper/src/main/resources/clusteringv3.conf b/scraper/src/main/resources/clusteringv3.conf new file mode 100644 index 0000000..922820b --- /dev/null +++ b/scraper/src/main/resources/clusteringv3.conf @@ -0,0 +1,86 @@ +akka { + # Log the complete configuration at INFO level when the actor system is started. + # This is useful when you are uncertain of what configuration is used. + ## log-config-on-start = on + + # Loggers to register at boot time (akka.event.Logging$DefaultLogger logs + # to STDOUT) + loggers = ["akka.event.slf4j.Slf4jLogger"] + + # Log level used by the configured loggers (see "loggers") as soon + # as they have been started; before that, see "stdout-loglevel" + # Options: OFF, ERROR, WARNING, INFO, DEBUG + loglevel = "DEBUG" # "DEBUG" + + # Log level for the very basic logger activated during ActorSystem startup. + # This logger prints the log messages to stdout (System.out). + # Options: OFF, ERROR, WARNING, INFO, DEBUG + stdout-loglevel = "DEBUG" + + # Filter of log events that is used by the LoggingAdapter before + # publishing log events to the eventStream. + logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" + + http { + client { + idle-timeout = 120s + } + host-connection-pool { + idle-timeout = 200s + } + } + + actor { + provider = "cluster" + } + + remote { + log-remote-lifecycle-events = off + artery.enabled = on + + artery { + canonical.hostname = "localhost" + canonical.port = 0 + canonical.port = ${?C_PORT} + } + } + + cluster { + seed-nodes = [ + "akka://"${clustering.cluster.name}"@"${clustering.seed-ip}":"${clustering.seed-port}, + ] + auto-down-unreachable-after = 10s + downing-provider-class = "akka.cluster.sbr.SplitBrainResolverProvider" + } + + coordinated-shutdown.exit-jvm = on + coordinated-shutdown.terminate-actor-system = on + + cluster.shutdown-after-unsuccessful-join-seed-nodes = 20s +} + +http { + ip = 127.0.0.1 + ip = ${?SERVER_IP} + port = 7000 + port = ${?SERVER_PORT} +} + +clustering { + ip = "127.0.0.1" + ip = ${?CLUSTER_IP} + + port = 4000 + port = ${?CLUSTER_PORT} + + seed-ip = "localhost" + seed-ip = ${?CLUSTER_SEED_IP} + + seed-port = 4000 + seed-port = ${?CLUSTER_SEED_PORT} + + seed-port_two = 4001 + seed-port_two = ${?CLUSTER_SEED_PORT_TWO} + + cluster.name = "mySystem" +} \ No newline at end of file diff --git a/src/main/resources/logback.xml b/scraper/src/main/resources/logback.xml similarity index 93% rename from src/main/resources/logback.xml rename to scraper/src/main/resources/logback.xml index dd16d27..22dbfbc 100644 --- a/src/main/resources/logback.xml +++ b/scraper/src/main/resources/logback.xml @@ -18,7 +18,7 @@ - + diff --git a/src/main/resources/quick.conf b/scraper/src/main/resources/quick.conf similarity index 100% rename from src/main/resources/quick.conf rename to scraper/src/main/resources/quick.conf diff --git a/src/main/scala/com/pinkstack/realestate/Configuration.scala b/scraper/src/main/scala/com/pinkstack/realestate/scraper/Configuration.scala similarity index 93% rename from src/main/scala/com/pinkstack/realestate/Configuration.scala rename to scraper/src/main/scala/com/pinkstack/realestate/scraper/Configuration.scala index 5a27c96..0417add 100644 --- a/src/main/scala/com/pinkstack/realestate/Configuration.scala +++ b/scraper/src/main/scala/com/pinkstack/realestate/scraper/Configuration.scala @@ -1,4 +1,4 @@ -package com.pinkstack.realestate +package com.pinkstack.realestate.scraper import pureconfig._ import pureconfig.generic.auto._ diff --git a/src/main/scala/com/pinkstack/realestate/Domain.scala b/scraper/src/main/scala/com/pinkstack/realestate/scraper/Domain.scala similarity index 94% rename from src/main/scala/com/pinkstack/realestate/Domain.scala rename to scraper/src/main/scala/com/pinkstack/realestate/scraper/Domain.scala index 442ebac..880f146 100644 --- a/src/main/scala/com/pinkstack/realestate/Domain.scala +++ b/scraper/src/main/scala/com/pinkstack/realestate/scraper/Domain.scala @@ -1,4 +1,4 @@ -package com.pinkstack.realestate +package com.pinkstack.realestate.scraper import akka.http.scaladsl.model.{HttpRequest, Uri} import io.lemonlabs.uri.Url @@ -16,8 +16,6 @@ object Domain { s"EstateRequest ${request.uri}" } - implicit val lemonLabsUrlToAkkaUrl: Url => Uri = url => Uri(url.toString()) - case class Location(latitude: Double, longitude: Double) case class Estate(title: String, @@ -29,3 +27,7 @@ object Domain { categories: Map[String, String] = Map.empty[String, String], location: Option[Location] = None) } + +object Implicits { + implicit val lemonLabsUrlToAkkaUrl: Url => Uri = url => Uri(url.toString()) +} \ No newline at end of file diff --git a/src/main/scala/com/pinkstack/realestate/EstateParser.scala b/scraper/src/main/scala/com/pinkstack/realestate/scraper/EstateParser.scala similarity index 95% rename from src/main/scala/com/pinkstack/realestate/EstateParser.scala rename to scraper/src/main/scala/com/pinkstack/realestate/scraper/EstateParser.scala index 45ca9f2..24f4ce1 100644 --- a/src/main/scala/com/pinkstack/realestate/EstateParser.scala +++ b/scraper/src/main/scala/com/pinkstack/realestate/scraper/EstateParser.scala @@ -1,9 +1,9 @@ -package com.pinkstack.realestate +package com.pinkstack.realestate.scraper import cats._ import cats.data.{Validated, ValidatedNec} import cats.implicits._ -import com.pinkstack.realestate.Domain._ +import Domain._ import io.lemonlabs.uri.Url import org.jsoup.nodes.Document @@ -65,17 +65,15 @@ object NepEstateParser { categoriesOpt.fold[ValidationResult[Map[String, String]]](NoCategories.invalidNec)(_.validNec) } - val validatedPrice: Transformer[Double] = { document => + val validatedPrice: Transformer[Double] = document => Option(document.selectFirst("meta[itemprop='price']")) .flatMap(_.attr("content").toDoubleOption) .fold[ValidationResult[Double]](NoPrice.invalidNec)(_.validNec) - } - val validatedRefNumber: Transformer[String] = { document => + val validatedRefNumber: Transformer[String] = document => Option(document.selectFirst("span.ikona-sh")) .map(_.attr("data-id")) .fold[ValidationResult[String]](NoRefNumber.invalidNec)(_.validNec) - } val validatedLocationDetails: Transformer[Map[String, Vector[String]]] = { document => import scala.jdk.CollectionConverters._ diff --git a/src/main/scala/com/pinkstack/realestate/NepClient.scala b/scraper/src/main/scala/com/pinkstack/realestate/scraper/NepClient.scala similarity index 94% rename from src/main/scala/com/pinkstack/realestate/NepClient.scala rename to scraper/src/main/scala/com/pinkstack/realestate/scraper/NepClient.scala index 59596d4..5ec82a3 100644 --- a/src/main/scala/com/pinkstack/realestate/NepClient.scala +++ b/scraper/src/main/scala/com/pinkstack/realestate/scraper/NepClient.scala @@ -1,12 +1,14 @@ -package com.pinkstack.realestate +package com.pinkstack.realestate.scraper +import cats._ +import cats.implicits._ +import cats.data.NonEmptyList import akka.NotUsed import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.model.HttpRequest import akka.stream.ThrottleMode import akka.stream.scaladsl.{Flow, Merge, Source} -import cats.data.NonEmptyChain import cats.data.Validated.{Invalid, Valid} import com.typesafe.scalalogging.LazyLogging import io.lemonlabs.uri.typesafe.dsl._ @@ -23,6 +25,7 @@ final case class NepClient(timeout: FiniteDuration = 4.seconds)( val configuration: Configuration) extends LazyLogging { import Domain._ + import Implicits._ final val ROOT_URL = Url.parse("https://www.nepremicnine.net") @@ -73,12 +76,15 @@ final case class NepClient(timeout: FiniteDuration = 4.seconds)( .map(doc => (categoryRequests(doc), estateRequests(doc))) } + val fetchEstatePageViaUrl: Url => Future[Option[Estate]] = url => + fetchEstatePage(EstateRequest(HttpRequest(uri = url), "")) + val fetchEstatePage: EstateRequest => Future[Option[Estate]] = { estateRequest => val parse: Document => Option[Estate] = { document => NepEstateParser.parse(document) match { case Valid(value: Estate) => Some(value.copy(scrapedFromUrl = estateRequest.request.uri.toString)) - case Invalid(errors: NonEmptyChain[ParserValidation]) => + case Invalid(errors) => logger.error(s"${errors} at ${estateRequest.request.uri}") None } diff --git a/src/main/scala/com/pinkstack/realestate/apps/Scraper.scala b/scraper/src/main/scala/com/pinkstack/realestate/scraper/apps/Scraper.scala similarity index 85% rename from src/main/scala/com/pinkstack/realestate/apps/Scraper.scala rename to scraper/src/main/scala/com/pinkstack/realestate/scraper/apps/Scraper.scala index 8c9d07f..855657a 100644 --- a/src/main/scala/com/pinkstack/realestate/apps/Scraper.scala +++ b/scraper/src/main/scala/com/pinkstack/realestate/scraper/apps/Scraper.scala @@ -1,12 +1,12 @@ -package com.pinkstack.realestate.apps +package com.pinkstack.realestate.scraper.apps import akka.actor.ActorSystem import akka.stream.scaladsl.Sink import cats.data._ import cats.implicits._ import com.monovore.decline._ -import com.pinkstack.realestate.Domain.Estate -import com.pinkstack.realestate.{Configuration, NepClient} +import com.pinkstack.realestate.scraper._ +import com.pinkstack.realestate.scraper.Domain._ import io.circe.generic.auto._ import io.circe.syntax._ import monocle.macros.GenLens @@ -30,10 +30,18 @@ object Implicits { } } +import Implicits._ + object Main { import Implicits._ + val estatePrinter: Option[Estate] => Unit = { + case Some(estate: Estate) => + println(estate.asJson.noSpaces) + case None => + } + def withConfiguration(categories: CategoryList, numberOfPages: Int): Unit = { val configurationLens: Configuration => Configuration = GenLens[Configuration](_.seed.initialCategories).set(categories) compose @@ -45,14 +53,11 @@ object Main { NepClient() .pipeline - .collect { case Some(estate: Estate) => estate.asJson.noSpaces } - .runWith(Sink.foreach(println)) + .runWith(Sink.foreach(estatePrinter)) .onComplete(_ => system.terminate()) } } -import com.pinkstack.realestate.apps.Implicits._ - object Scraper extends CommandApp(name = "scraper", header = "Scraper is used for fetching real estate listings directly in CLI.", main = { diff --git a/src/test/resources/404-error-page.html b/scraper/src/test/resources/404-error-page.html similarity index 100% rename from src/test/resources/404-error-page.html rename to scraper/src/test/resources/404-error-page.html diff --git a/src/test/resources/dravograd-stanovanje.html b/scraper/src/test/resources/dravograd-stanovanje.html similarity index 100% rename from src/test/resources/dravograd-stanovanje.html rename to scraper/src/test/resources/dravograd-stanovanje.html diff --git a/src/test/resources/ljubljana-skladisce-no-price.html b/scraper/src/test/resources/ljubljana-skladisce-no-price.html similarity index 100% rename from src/test/resources/ljubljana-skladisce-no-price.html rename to scraper/src/test/resources/ljubljana-skladisce-no-price.html diff --git a/src/test/resources/novigrad-hisa.html b/scraper/src/test/resources/novigrad-hisa.html similarity index 100% rename from src/test/resources/novigrad-hisa.html rename to scraper/src/test/resources/novigrad-hisa.html diff --git a/src/test/resources/sela-posest.html b/scraper/src/test/resources/sela-posest.html similarity index 100% rename from src/test/resources/sela-posest.html rename to scraper/src/test/resources/sela-posest.html diff --git a/src/test/scala/NepEstateParserSpec.scala b/scraper/src/test/scala/NepEstateParserSpec.scala similarity index 88% rename from src/test/scala/NepEstateParserSpec.scala rename to scraper/src/test/scala/NepEstateParserSpec.scala index 363102c..2b772a4 100644 --- a/src/test/scala/NepEstateParserSpec.scala +++ b/scraper/src/test/scala/NepEstateParserSpec.scala @@ -4,8 +4,9 @@ import java.nio.file.Path import cats._ import cats.data.Validated.{Invalid, Valid} import cats.implicits._ -import com.pinkstack.realestate.Domain.Estate -import com.pinkstack.realestate.{NepEstateParser, NoTitleFound, NotActive} +import com.pinkstack.realestate.scraper.Domain.Estate +import com.pinkstack.realestate.scraper.NepEstateParser +import com.pinkstack.realestate.scraper.NepEstateParser._ import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.scalatest.{EitherValues, PartialFunctionValues} @@ -28,7 +29,7 @@ class NepEstateParserSpec extends AnyFlatSpec "An NepEstateParser" should "throw an error" in { read("404-error-page.html").map { doc => - NepEstateParser.parse(doc) match { + parse(doc) match { case Invalid(e) => assert(true) case Valid(a) => fail("should fail not be valid") } @@ -37,7 +38,7 @@ class NepEstateParserSpec extends AnyFlatSpec it should "regular listing" in { read("dravograd-stanovanje.html").map { doc => - NepEstateParser.parse(doc).toEither match { + parse(doc).toEither match { case Right(Estate(title, refNumber, price, visibility, scrapedFromUrl, locationDetails, categories, location)) => assert(title == "Prodaja, Stanovanje, 4-sobno: DRAVOGRAD, 146.2 m2") assert(categories("Posredovanje") == "Prodaja") @@ -55,7 +56,7 @@ class NepEstateParserSpec extends AnyFlatSpec estates.map(e => (e._1, e._2, read(e._1))).foreach { case (name, expectedPrice, Some(document)) => - NepEstateParser.parse(document).toEither match { + parse(document).toEither match { case Right(estate: Estate) => assert(estate.price == expectedPrice) case Left(value) => diff --git a/src/main/scala/com/pinkstack/realestate/apps/MasterApp.scala b/src/main/scala/com/pinkstack/realestate/apps/MasterApp.scala deleted file mode 100644 index e4d18ce..0000000 --- a/src/main/scala/com/pinkstack/realestate/apps/MasterApp.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.pinkstack.realestate.apps - -object MasterApp extends App { - -} diff --git a/src/main/scala/com/pinkstack/realestate/apps/ScraperStream.scala b/src/main/scala/com/pinkstack/realestate/apps/ScraperStream.scala deleted file mode 100644 index 90d7c56..0000000 --- a/src/main/scala/com/pinkstack/realestate/apps/ScraperStream.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.pinkstack.realestate.apps - -import akka.actor.ActorSystem -import akka.stream.scaladsl._ -import com.pinkstack.realestate.Domain._ -import com.pinkstack.realestate.{Configuration, NepClient} -import com.typesafe.scalalogging.LazyLogging -import io.circe.generic.auto._ -import io.circe.syntax._ - -import scala.concurrent.ExecutionContextExecutor - -object ScraperStream extends App with LazyLogging { - logger.info("ScraperApp starting.") - - implicit val system: ActorSystem = ActorSystem("scraperAppTwo") - implicit val context: ExecutionContextExecutor = system.dispatcher - implicit val configuration: Configuration = Configuration.loadOrThrow - - val estatePrinter: Option[Estate] => Unit = { - case Some(estate: Estate) => - println(estate.asJson.noSpaces) - case None => - } - - NepClient() - .pipeline - .runWith(Sink.foreach(estatePrinter)) - .onComplete(_ => system.terminate()) -} diff --git a/worksheets/coordinates.sc b/worksheets/coordinates.sc deleted file mode 100644 index 0bc50db..0000000 --- a/worksheets/coordinates.sc +++ /dev/null @@ -1,11 +0,0 @@ -val coords = "(45.9818666,14.2267596)" - -val c = - """(-?\d+(\.\d+)?),\s*(-?\d+(\.\d+)?)""".r.findFirstIn(coords) - .map(_.split(",", 2).map(_.toDouble).toList) - -c match { - case Some(v) => v - case _ => "nopu" -} - diff --git a/worksheets/parsing.sc b/worksheets/parsing.sc deleted file mode 100644 index b8acb45..0000000 --- a/worksheets/parsing.sc +++ /dev/null @@ -1,16 +0,0 @@ -import org.jsoup.Jsoup - -val documents: Seq[String] = - "https://www.nepremicnine.net/oglasi-prodaja/medulin-stanovanje_6347567/" :: - "https://www.nepremicnine.net/oglasi-prodaja/sezana-garaza_6347637/" :: - "https://www.nepremicnine.net/oglasi-prodaja/sestdobe-posest_6347670/" :: - Nil - - - -// Jsoup.connect(documents.head).get() - -"this".split(",") match { - case Array(xs) => "x" - case _ => "y" -} \ No newline at end of file diff --git a/worksheets/ranges.sc b/worksheets/ranges.sc deleted file mode 100644 index e7f4b12..0000000 --- a/worksheets/ranges.sc +++ /dev/null @@ -1 +0,0 @@ -Range(1, 10).toList \ No newline at end of file