diff --git a/.env b/.env index 1ec90e0f..c41d626c 100644 --- a/.env +++ b/.env @@ -4,3 +4,7 @@ CUTTLY_API_KEY="" KEYSTORE_PASSWORD="" KEYMANAGER_PASSWORD="" KEYSTORE_PATH="" +# Credentials to access rdfshape DB on mongo db atlas +MONGO_DATABASE="" +MONGO_USER="" +MONGO_PASSWORD="" diff --git a/build.sbt b/build.sbt index 8bf64c2b..3bc4a6f1 100644 --- a/build.sbt +++ b/build.sbt @@ -56,6 +56,7 @@ lazy val scallopVersion = "3.3.1" lazy val seleniumVersion = "2.45.0" lazy val silencerVersion = "1.4.2" lazy val typesafeConfigVersion = "1.3.4" +lazy val mongodbVersion = "4.1.1" // WebJars lazy val jqueryVersion = "3.4.1" @@ -105,11 +106,12 @@ lazy val any23_scraper = "org.apache.any23.plugins" % "apache-any23-html-s lazy val rdf4j_runtime = "org.eclipse.rdf4j" % "rdf4j-runtime" % rdf4jVersion lazy val scalaj ="org.scalaj" %% "scalaj-http" % scalajVersion -lazy val play = "com.typesafe.play" %% "play-json" % playVersion +lazy val play = "com.typesafe.play" %% "play-json" % playVersion lazy val jquery = "org.webjars" % "jquery" % jqueryVersion lazy val bootstrap = "org.webjars" % "bootstrap" % bootstrapVersion +lazy val mongodb = "org.mongodb.scala" %% "mongo-scala-driver" % mongodbVersion // Compiler plugin modules lazy val scalaMacrosParadise = "org.scalamacros" % "paradise" % scalaMacrosVersion cross CrossVersion.full @@ -120,8 +122,8 @@ lazy val rdfshape = project .in(file(".")) .enablePlugins( ScalaUnidocPlugin, - SiteScaladocPlugin, - AsciidoctorPlugin, + SiteScaladocPlugin, + AsciidoctorPlugin, SbtNativePackager, WindowsPlugin, JavaAppPackaging, @@ -143,7 +145,7 @@ lazy val rdfshape = project scalaLogging, scallop, compilerPlugin("com.github.ghik" %% "silencer-plugin" % silencerVersion), - "com.github.ghik" %% "silencer-lib" % silencerVersion % Provided + "com.github.ghik" %% "silencer-lib" % silencerVersion % Provided ), cancelable in Global := true, fork := true, @@ -158,7 +160,7 @@ lazy val server = project .settings(commonSettings, publishSettings) .settings( buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion), - buildInfoPackage := "es.weso.rdfshape.buildinfo" + buildInfoPackage := "es.weso.rdfshape.buildinfo" ) .settings( libraryDependencies ++= Seq( @@ -178,9 +180,11 @@ lazy val server = project scalaj, play, utilsTest % Test, + mongodb, + // webJars jquery, - bootstrap + bootstrap, ), crossScalaVersions := supportedScalaVersions, ) @@ -192,8 +196,8 @@ lazy val server = project lazy val noDocProjects = Seq[ProjectReference]() lazy val noPublishSettings = Seq( -// publish := (), -// publishLocal := (), + // publish := (), + // publishLocal := (), publishArtifact := false ) @@ -224,28 +228,28 @@ lazy val compilationSettings = Seq( "-feature", // Emit warning and location for usages of features that should be imported explicitly. "-encoding", "UTF-8", "-language:_", "-unchecked", // Enable additional warnings where generated code depends on assumptions. -// "-Xfuture", // Turn on future language features. -// "-Xlint", + // "-Xfuture", // Turn on future language features. + // "-Xlint", "-Yrangepos", -// "-Ylog-classpath", -// "-Yno-adapted-args", // Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver -// "-Ywarn-dead-code", // Warn when dead code is identified. -// "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. -// "-Ywarn-inaccessible", // Warn about inaccessible types in method signatures. -// "-Ywarn-infer-any", // Warn when a type argument is inferred to be `Any`. -// "-Ywarn-nullary-override", // Warn when non-nullary `def f()' overrides nullary `def f'. -// "-Ywarn-nullary-unit", // Warn when nullary methods return Unit. -// "-Ywarn-numeric-widen", // Warn when numerics are widened. -// "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. -// "-Ywarn-unused:imports", // Warn if an import selector is not referenced. -// "-Ywarn-unused:locals", // Warn if a local definition is unused. -// "-Ywarn-unused:params", // Warn if a value parameter is unused. -// "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. -// "-Ywarn-unused:privates", // Warn if a private member is unused. -// "-Ywarn-value-discard", // Warn when non-Unit expression results are unused. -// "-Xfatal-warnings", // Fail the compilation if there are any warnings. -// "-Ypartial-unification", -) + // "-Ylog-classpath", + // "-Yno-adapted-args", // Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver + // "-Ywarn-dead-code", // Warn when dead code is identified. + // "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. + // "-Ywarn-inaccessible", // Warn about inaccessible types in method signatures. + // "-Ywarn-infer-any", // Warn when a type argument is inferred to be `Any`. + // "-Ywarn-nullary-override", // Warn when non-nullary `def f()' overrides nullary `def f'. + // "-Ywarn-nullary-unit", // Warn when nullary methods return Unit. + // "-Ywarn-numeric-widen", // Warn when numerics are widened. + // "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. + // "-Ywarn-unused:imports", // Warn if an import selector is not referenced. + // "-Ywarn-unused:locals", // Warn if a local definition is unused. + // "-Ywarn-unused:params", // Warn if a value parameter is unused. + // "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. + // "-Ywarn-unused:privates", // Warn if a private member is unused. + // "-Ywarn-value-discard", // Warn when non-Unit expression results are unused. + // "-Xfatal-warnings", // Fail the compilation if there are any warnings. + // "-Ypartial-unification", + ) ) @@ -253,9 +257,9 @@ lazy val compilationSettings = Seq( lazy val wixSettings = Seq( wixProductId := "39b564d5-d381-4282-ada9-87244c76e14b", wixProductUpgradeId := "6a710435-9af4-4adb-a597-98d3dd0bade1" -// The same numbers as in the docs? -// wixProductId := "ce07be71-510d-414a-92d4-dff47631848a", -// wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424" + // The same numbers as in the docs? + // wixProductId := "ce07be71-510d-414a-92d4-dff47631848a", + // wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424" ) lazy val ghPagesSettings = Seq( @@ -279,12 +283,12 @@ lazy val publishSettings = Seq( autoAPIMappings := true, apiURL := Some(url("http://labra.github.io/rdfshape/latest/api/")), pomExtra := - - labra - Jose Emilio Labra Gayo - https://github.com/labra/ - - , + + labra + Jose Emilio Labra Gayo + https://github.com/labra/ + + , scalacOptions in doc ++= Seq( "-diagrams-debug", "-doc-source-url", @@ -299,6 +303,6 @@ lazy val publishSettings = Seq( ) // silence all warnings on autogenerated files -scalacOptions += "-P:silencer:pathFilters=target/.*" +scalacOptions += "-P:silencer:pathFilters=target/.*" // Make sure you only exclude warnings for the project directories, i.e. make builds reproducible scalacOptions += s"-P:silencer:sourceRoots=${baseDirectory.value.getCanonicalPath}" diff --git a/modules/server/src/main/scala/es/weso/server/PermalinkService.scala b/modules/server/src/main/scala/es/weso/server/PermalinkService.scala index 9b5ece41..814baab6 100644 --- a/modules/server/src/main/scala/es/weso/server/PermalinkService.scala +++ b/modules/server/src/main/scala/es/weso/server/PermalinkService.scala @@ -1,54 +1,193 @@ package es.weso.server +import java.time.Instant +import java.util.Calendar +import java.util.concurrent.TimeUnit + import cats.effect._ import es.weso.server.APIDefinitions._ -import es.weso.server.QueryParams.UrlParam +import es.weso.server.QueryParams.{UrlCodeParam, UrlParam, urlCode} import org.http4s._ import org.http4s.client.Client import org.http4s.dsl.Http4sDsl -import play.api.libs.json.{Json => PJson} -import scalaj.http.Http -import java.net.URL +import org.mongodb.scala._ +import org.mongodb.scala.model.Filters.equal +import org.mongodb.scala.result.{InsertOneResult, UpdateResult} -class PermalinkService[F[_]]()(implicit F: Effect[F], cs: ContextShift[F]) - extends Http4sDsl[F] { +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Promise} +import scala.util.Random +import java.net.{MalformedURLException, URL} - val urlShortenerEndpoint = "https://cutt.ly/api/api.php" - val urlShortenerAcceptCode = 7 +import org.mongodb.scala.model.Updates.set - case class RequestData(domain: String, url: String) +class PermalinkService[F[_]](blocker: Blocker, client: Client[F])(implicit F: Effect[F], cs: ContextShift[F]) + extends Http4sDsl[F] { + lazy val mongoClient: MongoClient = MongoClient(mongoConnectionString) + lazy val db: MongoDatabase = mongoClient.getDatabase(mongoDatabase) + lazy val collection: MongoCollection[Document] = db.getCollection(collectionName) + + // Utils for url generation + val urlPrefix = "http://rdfshape.weso.es/link/" + val random: Random.type = Random val routes: HttpRoutes[F] = HttpRoutes.of[F] { - // Query URL shortener API + // Insert a reference to the permalink in DB case GET -> Root / `api` / "permalink" / "generate" :? UrlParam(url) => - // Request the shortened URL - try { - val res = Http(urlShortenerEndpoint) - .param("key", sys.env.getOrElse("CUTTLY_API_KEY", "")) - .param("short", url) - .asString - - val responseJson = PJson.parse(res.body) - // Check for a valid response from code - if (res.code == 200 && (responseJson \ "url" \ "status").as[Int] == urlShortenerAcceptCode) { - val url = new URL(s"${(responseJson \ "url" \ "shortLink").as[String]}").toURI - Ok(url.toString) + + val existingUrl = retrieveUrl(url) + if (existingUrl.isDefined){ + Ok(existingUrl.get) + } + else { + try { + val longUrl = new URL(url) + val urlCode = Instant.now.getEpochSecond.toString + random.nextInt(10) + val shortUrl: String = urlPrefix + urlCode + + // Create doc + val doc: Document = Document( + // "_id" -> Autogenerated, + "longUrl" -> url, + "shortUrl" -> shortUrl, + "urlCode" -> urlCode.toLong, + "date" -> Calendar.getInstance().getTime + ) + + // Insert doc + val observable: Observable[InsertOneResult] = collection.insertOne(doc) + observable.subscribe(new Observer[InsertOneResult] { + override def onSubscribe(subscription: Subscription): Unit = subscription.request(1) + override def onNext(result: InsertOneResult): Unit = println(s"Created permalink: $url => $shortUrl") + override def onError(e: Throwable): Unit = println(s"Permalink creation failed: ${e.getMessage}") + override def onComplete(): Unit = println("Permalink processing completed.") + }) + + Created(shortUrl) + + } catch { + case _: MalformedURLException => + BadRequest(s"Invalid URL provided for shortening: $url") + case _: Exception => + InternalServerError(s"Could not execute generate the permalink for url: $url") } - else - InternalServerError(url) + } - catch { + + // Retrieve a URL given the link + case GET -> Root / `api` / "permalink" / "get" :? + UrlCodeParam(urlCode) => + try { + val code = urlCode.toLong + val promise = Promise[F[Response[F]]] + + // Fetch document in database + val observable: SingleObservable[Document] = collection.find(equal("urlCode", code)).first() + observable.subscribe(new Observer[Document] { + + override def onSubscribe(subscription: Subscription): Unit = subscription.request(1) + override def onNext(result: Document): Unit = { + val longUrl = result.getString("longUrl") + val urlCode = result.getLong("urlCode") + + println(s"Retrieved original url: $urlCode => $longUrl") + promise.success(Ok(longUrl)) + + // Refresh use date of the link + updateUrl(urlCode) + } + override def onError(e: Throwable): Unit = { + println(s"Original url recovery failed: ${e.getMessage}") + promise.success(BadGateway(s"Original url recovery failed for code: $urlCode")) + } + override def onComplete(): Unit = { + if (!promise.isCompleted) { + println(s"Could not find the original url for code: $urlCode") + promise.success(NotFound(s"Could not find the original url for code: $urlCode")) + } + println("Permalink processing completed.") + } + }) + + val result = Await.result(promise.future, Duration(8, TimeUnit.SECONDS)) + result + + + } catch { + case _: NumberFormatException => + BadRequest(s"Invalid permalink code: $urlCode") case _: Exception => - InternalServerError(url) + InternalServerError(s"Could not execute the request for the permalink with code: $urlCode") + } + } + + private def retrieveUrl (url: String): Option[String] = + { + + val promise = Promise[Option[String]] + + // Fetch document in database + val observable: SingleObservable[Document] = collection.find(equal("longUrl", url)).first() + observable.subscribe(new Observer[Document] { + override def onSubscribe(subscription: Subscription): Unit = subscription.request(1) + override def onNext(result: Document): Unit = { + val shortUrl = result.getString("shortUrl") + val urlCode = result.getLong("urlCode") + + println(s"Retrieved permalink: $url => $shortUrl") + promise.success(Option(shortUrl)) + + // Refresh use date of the link + updateUrl(urlCode) } - } + override def onError(e: Throwable): Unit = { + println(s"Permalink recovery failed: ${e.getMessage}") + promise.success(None) + } + override def onComplete(): Unit = { + if (!promise.isCompleted) { + println(s"Could not find the permalink for url: $url") + promise.success(None) + } + println("Permalink processing completed.") + } + }) + + val result = Await.result(promise.future, Duration(8, TimeUnit.SECONDS)) + result + } + + + private def updateUrl (code: Long): Unit = + { + println(s"URL code to update: $code") + // Update date of document in database + val observable: SingleObservable[UpdateResult] = collection.updateOne(equal("urlCode", code), + set("date", Calendar.getInstance().getTime)) + + observable.subscribe(new Observer[UpdateResult] { + override def onSubscribe(subscription: Subscription): Unit = subscription.request(1) + override def onNext(result: UpdateResult): Unit = { + println(s"Refreshed date of permalink: $code") + } + override def onError(e: Throwable): Unit = Unit + override def onComplete(): Unit = Unit + }) + } + + // DB credentials + private val mongoUser = sys.env.getOrElse("MONGO_USER", "") + private val mongoPassword = sys.env.getOrElse("MONGO_PASSWORD", "") + private val mongoDatabase = sys.env.getOrElse("MONGO_DATABASE", "") + private val collectionName = "permalinks" + private val mongoConnectionString = + s"mongodb+srv://$mongoUser:$mongoPassword@cluster0.pnja6.mongodb.net/$mongoDatabase" + + "?retryWrites=true&w=majority" } object PermalinkService { def apply[F[_]: Effect: ContextShift](blocker: Blocker, client: Client[F]): PermalinkService[F] = - new PermalinkService[F]() + new PermalinkService[F](blocker, client) } - - diff --git a/modules/server/src/main/scala/es/weso/server/QueryParams.scala b/modules/server/src/main/scala/es/weso/server/QueryParams.scala index 455a8969..8bbd43f1 100644 --- a/modules/server/src/main/scala/es/weso/server/QueryParams.scala +++ b/modules/server/src/main/scala/es/weso/server/QueryParams.scala @@ -21,6 +21,7 @@ object QueryParams { lazy val schemaFormat = "schemaFormat" lazy val shape = "shape" lazy val url = "url" + lazy val urlCode = "urlCode" object DataParameter extends OptionalQueryParamDecoderMatcher[String](data) object OptDataParam extends OptionalQueryParamDecoderMatcher[String](data) object OptEndpointParam extends OptionalQueryParamDecoderMatcher[String](endpoint) @@ -63,7 +64,8 @@ object QueryParams { object LanguageParam extends QueryParamDecoderMatcher[String]("language") object LabelParam extends QueryParamDecoderMatcher[String]("label") object UrlParam extends QueryParamDecoderMatcher[String](url) + object UrlCodeParam extends QueryParamDecoderMatcher[String](urlCode) object LimitParam extends OptionalQueryParamDecoderMatcher[String]("limit") object ContinueParam extends OptionalQueryParamDecoderMatcher[String]("continue") -} \ No newline at end of file +} diff --git a/modules/server/src/test/scala/es/weso/server/compoundData/CompoundDataTest.scala b/modules/server/src/test/scala/es/weso/server/compoundData/CompoundDataTest.scala index b32affdf..355f3bb7 100644 --- a/modules/server/src/test/scala/es/weso/server/compoundData/CompoundDataTest.scala +++ b/modules/server/src/test/scala/es/weso/server/compoundData/CompoundDataTest.scala @@ -1,18 +1,13 @@ -package es.weso.server +package es.weso.server.compoundData -import cats._ -import cats.effect._ import cats.implicits._ +import es.weso.server.format._ +import es.weso.server.merged.CompoundData._ +import es.weso.server.merged.{CompoundData, DataElement, DataTextArea} import io.circe.parser._ +import io.circe.syntax._ import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should._ -import es.weso.server.merged.CompoundData._ -import io.circe._ -import io.circe.syntax._ -import es.weso.server.merged.CompoundData -import es.weso.server.merged.DataElement -import es.weso.server.merged.DataTextArea -import es.weso.server.format._ class CompoundDataTest extends AnyFunSpec with Matchers { describe(s"Compound data") { @@ -39,7 +34,17 @@ class CompoundDataTest extends AnyFunSpec with Matchers { |]""".stripMargin, CompoundData(List(DataElement.empty.copy(data = Some("p"), dataFormat = JsonLd, activeDataTab = DataTextArea))) ) - shouldNotParse("""|[{"data": "p", + + // Wrong value in "activeDataTab" should default to "#dataTextArea" + shouldParse("""|[{"data": "p", + | "dataFormat": "Json", + | "activeDataTab": "#asdf" + | } + |]""".stripMargin, + CompoundData(List(DataElement(Some("p"), None, None, None, dataFormat = JsonDataFormat, DataTextArea))) + ) + + shouldNotParse("""|[{"dataBadParam": "p", | "dataFormat": "Json", | "activeDataTab": "#asdf" | }