Moved URL shortener to our custom database

.env

@@ -4,3 +4,7 @@ CUTTLY_API_KEY=""
 KEYSTORE_PASSWORD=""
 KEYMANAGER_PASSWORD=""
 KEYSTORE_PATH=""
+# Credentials to access rdfshape DB on mongo db atlas
+MONGO_DATABASE=""
+MONGO_USER=""
+MONGO_PASSWORD=""

build.sbt

@@ -56,6 +56,7 @@ lazy val scallopVersion        = "3.3.1"
 lazy val seleniumVersion       = "2.45.0"
 lazy val silencerVersion       = "1.4.2"
 lazy val typesafeConfigVersion = "1.3.4"
+lazy val mongodbVersion        = "4.1.1"
 
 // WebJars
 lazy val jqueryVersion         = "3.4.1"
@@ -105,11 +106,12 @@ lazy val any23_scraper     = "org.apache.any23.plugins"   % "apache-any23-html-s
 lazy val rdf4j_runtime     = "org.eclipse.rdf4j"          % "rdf4j-runtime"        % rdf4jVersion
 
 lazy val scalaj            ="org.scalaj"                  %% "scalaj-http"         % scalajVersion
-lazy val play             = "com.typesafe.play"           %% "play-json"           % playVersion
+lazy val play              = "com.typesafe.play"          %% "play-json"           % playVersion
 
 lazy val jquery            = "org.webjars"                % "jquery"               % jqueryVersion
 lazy val bootstrap         = "org.webjars"                % "bootstrap"            % bootstrapVersion
 
+lazy val mongodb           = "org.mongodb.scala"          %% "mongo-scala-driver"  % mongodbVersion
 
 // Compiler plugin modules
 lazy val scalaMacrosParadise = "org.scalamacros"      % "paradise"        % scalaMacrosVersion cross CrossVersion.full
@@ -120,8 +122,8 @@ lazy val rdfshape = project
   .in(file("."))
   .enablePlugins(
     ScalaUnidocPlugin,
-    SiteScaladocPlugin, 
-    AsciidoctorPlugin, 
+    SiteScaladocPlugin,
+    AsciidoctorPlugin,
     SbtNativePackager,
     WindowsPlugin,
     JavaAppPackaging,
@@ -143,7 +145,7 @@ lazy val rdfshape = project
       scalaLogging,
       scallop,
       compilerPlugin("com.github.ghik" %% "silencer-plugin" % silencerVersion),
-      "com.github.ghik" %% "silencer-lib" % silencerVersion % Provided      
+      "com.github.ghik" %% "silencer-lib" % silencerVersion % Provided
     ),
     cancelable in Global      := true,
     fork                      := true,
@@ -158,7 +160,7 @@ lazy val server = project
   .settings(commonSettings, publishSettings)
   .settings(
     buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion),
-    buildInfoPackage := "es.weso.rdfshape.buildinfo" 
+    buildInfoPackage := "es.weso.rdfshape.buildinfo"
   )
   .settings(
     libraryDependencies ++= Seq(
@@ -178,9 +180,11 @@ lazy val server = project
       scalaj,
       play,
       utilsTest % Test,
+      mongodb,
+
       // webJars
       jquery,
-      bootstrap
+      bootstrap,
     ),
     crossScalaVersions := supportedScalaVersions,
   )
@@ -192,8 +196,8 @@ lazy val server = project
 lazy val noDocProjects = Seq[ProjectReference]()
 
 lazy val noPublishSettings = Seq(
-//  publish := (),
-//  publishLocal := (),
+  //  publish := (),
+  //  publishLocal := (),
   publishArtifact := false
 )
 
@@ -224,38 +228,38 @@ lazy val compilationSettings = Seq(
     "-feature",                          // Emit warning and location for usages of features that should be imported explicitly.  "-encoding", "UTF-8",
     "-language:_",
     "-unchecked",                        // Enable additional warnings where generated code depends on assumptions.
-//    "-Xfuture",                          // Turn on future language features.
-//    "-Xlint",
+    //    "-Xfuture",                          // Turn on future language features.
+    //    "-Xlint",
     "-Yrangepos",
-//    "-Ylog-classpath",
-//    "-Yno-adapted-args",                 // Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver
-//    "-Ywarn-dead-code",                  // Warn when dead code is identified.
-//    "-Ywarn-extra-implicit",             // Warn when more than one implicit parameter section is defined.
-//    "-Ywarn-inaccessible",               // Warn about inaccessible types in method signatures.
-//    "-Ywarn-infer-any",                  // Warn when a type argument is inferred to be `Any`.
-//    "-Ywarn-nullary-override",           // Warn when non-nullary `def f()' overrides nullary `def f'.
-//    "-Ywarn-nullary-unit",               // Warn when nullary methods return Unit.
-//    "-Ywarn-numeric-widen",              // Warn when numerics are widened.
-//    "-Ywarn-unused:implicits",           // Warn if an implicit parameter is unused.
-//    "-Ywarn-unused:imports",             // Warn if an import selector is not referenced.
-//    "-Ywarn-unused:locals",              // Warn if a local definition is unused.
-//    "-Ywarn-unused:params",              // Warn if a value parameter is unused.
-//    "-Ywarn-unused:patvars",             // Warn if a variable bound in a pattern is unused.
-//    "-Ywarn-unused:privates",            // Warn if a private member is unused.
-//    "-Ywarn-value-discard",              // Warn when non-Unit expression results are unused.
-//    "-Xfatal-warnings",                  // Fail the compilation if there are any warnings.
-//    "-Ypartial-unification",
-)
+    //    "-Ylog-classpath",
+    //    "-Yno-adapted-args",                 // Do not adapt an argument list (either by inserting () or creating a tuple) to match the receiver
+    //    "-Ywarn-dead-code",                  // Warn when dead code is identified.
+    //    "-Ywarn-extra-implicit",             // Warn when more than one implicit parameter section is defined.
+    //    "-Ywarn-inaccessible",               // Warn about inaccessible types in method signatures.
+    //    "-Ywarn-infer-any",                  // Warn when a type argument is inferred to be `Any`.
+    //    "-Ywarn-nullary-override",           // Warn when non-nullary `def f()' overrides nullary `def f'.
+    //    "-Ywarn-nullary-unit",               // Warn when nullary methods return Unit.
+    //    "-Ywarn-numeric-widen",              // Warn when numerics are widened.
+    //    "-Ywarn-unused:implicits",           // Warn if an implicit parameter is unused.
+    //    "-Ywarn-unused:imports",             // Warn if an import selector is not referenced.
+    //    "-Ywarn-unused:locals",              // Warn if a local definition is unused.
+    //    "-Ywarn-unused:params",              // Warn if a value parameter is unused.
+    //    "-Ywarn-unused:patvars",             // Warn if a variable bound in a pattern is unused.
+    //    "-Ywarn-unused:privates",            // Warn if a private member is unused.
+    //    "-Ywarn-value-discard",              // Warn when non-Unit expression results are unused.
+    //    "-Xfatal-warnings",                  // Fail the compilation if there are any warnings.
+    //    "-Ypartial-unification",
+  )
 )
 
 
 
 lazy val wixSettings = Seq(
   wixProductId        := "39b564d5-d381-4282-ada9-87244c76e14b",
   wixProductUpgradeId := "6a710435-9af4-4adb-a597-98d3dd0bade1"
-// The same numbers as in the docs?
-// wixProductId := "ce07be71-510d-414a-92d4-dff47631848a",
-// wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424"
+  // The same numbers as in the docs?
+  // wixProductId := "ce07be71-510d-414a-92d4-dff47631848a",
+  // wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424"
 )
 
 lazy val ghPagesSettings = Seq(
@@ -279,12 +283,12 @@ lazy val publishSettings = Seq(
   autoAPIMappings := true,
   apiURL          := Some(url("http://labra.github.io/rdfshape/latest/api/")),
   pomExtra        := <developers>
-                       <developer>
-                         <id>labra</id>
-                         <name>Jose Emilio Labra Gayo</name>
-                         <url>https://github.com/labra/</url>
-                       </developer>
-                     </developers>,
+    <developer>
+      <id>labra</id>
+      <name>Jose Emilio Labra Gayo</name>
+      <url>https://github.com/labra/</url>
+    </developer>
+  </developers>,
   scalacOptions in doc ++= Seq(
     "-diagrams-debug",
     "-doc-source-url",
@@ -299,6 +303,6 @@ lazy val publishSettings = Seq(
 )
 
 // silence all warnings on autogenerated files
-scalacOptions += "-P:silencer:pathFilters=target/.*" 
+scalacOptions += "-P:silencer:pathFilters=target/.*"
 // Make sure you only exclude warnings for the project directories, i.e. make builds reproducible
 scalacOptions += s"-P:silencer:sourceRoots=${baseDirectory.value.getCanonicalPath}"

modules/server/src/main/scala/es/weso/server/PermalinkService.scala

@@ -1,54 +1,193 @@
 package es.weso.server
 
+import java.time.Instant
+import java.util.Calendar
+import java.util.concurrent.TimeUnit
+
 import cats.effect._
 import es.weso.server.APIDefinitions._
-import es.weso.server.QueryParams.UrlParam
+import es.weso.server.QueryParams.{UrlCodeParam, UrlParam, urlCode}
 import org.http4s._
 import org.http4s.client.Client
 import org.http4s.dsl.Http4sDsl
-import play.api.libs.json.{Json => PJson}
-import scalaj.http.Http
-import java.net.URL
+import org.mongodb.scala._
+import org.mongodb.scala.model.Filters.equal
+import org.mongodb.scala.result.{InsertOneResult, UpdateResult}
 
-class PermalinkService[F[_]]()(implicit F: Effect[F], cs: ContextShift[F])
-  extends Http4sDsl[F] {
+import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, Promise}
+import scala.util.Random
+import java.net.{MalformedURLException, URL}
 
-  val urlShortenerEndpoint = "https://cutt.ly/api/api.php"
-  val urlShortenerAcceptCode = 7
+import org.mongodb.scala.model.Updates.set
 
-  case class RequestData(domain: String, url: String)
+class PermalinkService[F[_]](blocker: Blocker, client: Client[F])(implicit F: Effect[F], cs: ContextShift[F])
+  extends Http4sDsl[F] {
 
+  lazy val mongoClient: MongoClient = MongoClient(mongoConnectionString)
+  lazy val db: MongoDatabase                     = mongoClient.getDatabase(mongoDatabase)
+  lazy val collection: MongoCollection[Document] = db.getCollection(collectionName)
+
+  // Utils for url generation
+  val urlPrefix           = "http://rdfshape.weso.es/link/"
+  val random: Random.type = Random
   val routes: HttpRoutes[F] = HttpRoutes.of[F] {
 
-    // Query URL shortener API
+    // Insert a reference to the permalink in DB
     case GET -> Root / `api` / "permalink" / "generate" :?
       UrlParam(url) =>
-      // Request the shortened URL
-      try {
-        val res = Http(urlShortenerEndpoint)
-          .param("key", sys.env.getOrElse("CUTTLY_API_KEY", ""))
-          .param("short", url)
-          .asString
-
-        val responseJson = PJson.parse(res.body)
-        // Check for a valid response from code
-        if (res.code == 200 && (responseJson \ "url" \ "status").as[Int] == urlShortenerAcceptCode) {
-          val url = new URL(s"${(responseJson \ "url" \ "shortLink").as[String]}").toURI
-          Ok(url.toString)
+
+      val existingUrl = retrieveUrl(url)
+      if (existingUrl.isDefined){
+        Ok(existingUrl.get)
+      }
+      else {
+        try {
+          val longUrl = new URL(url)
+          val urlCode          = Instant.now.getEpochSecond.toString + random.nextInt(10)
+          val shortUrl: String = urlPrefix + urlCode
+
+          // Create doc
+          val doc: Document = Document(
+            // "_id" -> Autogenerated,
+            "longUrl"  -> url,
+            "shortUrl" -> shortUrl,
+            "urlCode"  -> urlCode.toLong,
+            "date"     -> Calendar.getInstance().getTime
+          )
+
+          // Insert doc
+          val observable: Observable[InsertOneResult] = collection.insertOne(doc)
+          observable.subscribe(new Observer[InsertOneResult] {
+            override def onSubscribe(subscription: Subscription): Unit = subscription.request(1)
+            override def onNext(result: InsertOneResult): Unit         = println(s"Created permalink: $url => $shortUrl")
+            override def onError(e: Throwable): Unit                   = println(s"Permalink creation failed: ${e.getMessage}")
+            override def onComplete(): Unit                            = println("Permalink processing completed.")
+          })
+
+          Created(shortUrl)
+
+        } catch {
+          case _: MalformedURLException =>
+            BadRequest(s"Invalid URL provided for shortening: $url")
+          case _: Exception =>
+            InternalServerError(s"Could not execute generate the permalink for url: $url")
         }
-        else
-          InternalServerError(url)
+
       }
-      catch {
+
+    // Retrieve a URL given the link
+    case GET -> Root / `api` / "permalink" / "get" :?
+      UrlCodeParam(urlCode) =>
+      try {
+        val code    = urlCode.toLong
+        val promise = Promise[F[Response[F]]]
+
+        // Fetch document in database
+        val observable: SingleObservable[Document] = collection.find(equal("urlCode", code)).first()
+        observable.subscribe(new Observer[Document] {
+
+          override def onSubscribe(subscription: Subscription): Unit = subscription.request(1)
+          override def onNext(result: Document): Unit = {
+            val longUrl = result.getString("longUrl")
+            val urlCode = result.getLong("urlCode")
+
+            println(s"Retrieved original url: $urlCode => $longUrl")
+            promise.success(Ok(longUrl))
+
+            // Refresh use date of the link
+            updateUrl(urlCode)
+          }
+          override def onError(e: Throwable): Unit = {
+            println(s"Original url recovery failed: ${e.getMessage}")
+            promise.success(BadGateway(s"Original url recovery failed for code: $urlCode"))
+          }
+          override def onComplete(): Unit = {
+            if (!promise.isCompleted) {
+              println(s"Could not find the original url for code: $urlCode")
+              promise.success(NotFound(s"Could not find the original url for code: $urlCode"))
+            }
+            println("Permalink processing completed.")
+          }
+        })
+
+        val result = Await.result(promise.future, Duration(8, TimeUnit.SECONDS))
+        result
+
+
+      } catch {
+        case _: NumberFormatException =>
+          BadRequest(s"Invalid permalink code: $urlCode")
         case _: Exception =>
-          InternalServerError(url)
+          InternalServerError(s"Could not execute the request for the permalink with code: $urlCode")
+      }
+  }
+
+  private def retrieveUrl (url: String): Option[String] =
+  {
+
+    val promise = Promise[Option[String]]
+
+    // Fetch document in database
+    val observable: SingleObservable[Document] = collection.find(equal("longUrl", url)).first()
+    observable.subscribe(new Observer[Document] {
+      override def onSubscribe(subscription: Subscription): Unit = subscription.request(1)
+      override def onNext(result: Document): Unit = {
+        val shortUrl = result.getString("shortUrl")
+        val urlCode = result.getLong("urlCode")
+
+        println(s"Retrieved permalink: $url => $shortUrl")
+        promise.success(Option(shortUrl))
+
+        // Refresh use date of the link
+        updateUrl(urlCode)
       }
-    }
+      override def onError(e: Throwable): Unit = {
+        println(s"Permalink recovery failed: ${e.getMessage}")
+        promise.success(None)
+      }
+      override def onComplete(): Unit = {
+        if (!promise.isCompleted) {
+          println(s"Could not find the permalink for url: $url")
+          promise.success(None)
+        }
+        println("Permalink processing completed.")
+      }
+    })
+
+    val result = Await.result(promise.future, Duration(8, TimeUnit.SECONDS))
+    result
+  }
+
+
+  private def updateUrl (code: Long): Unit =
+  {
+    println(s"URL code to update: $code")
+    // Update date of document in database
+    val observable: SingleObservable[UpdateResult] = collection.updateOne(equal("urlCode", code),
+      set("date", Calendar.getInstance().getTime))
+
+    observable.subscribe(new Observer[UpdateResult] {
+      override def onSubscribe(subscription: Subscription): Unit = subscription.request(1)
+      override def onNext(result: UpdateResult): Unit = {
+        println(s"Refreshed date of permalink: $code")
+      }
+      override def onError(e: Throwable): Unit = Unit
+      override def onComplete(): Unit = Unit
+    })
+  }
+
+  // DB credentials
+  private val mongoUser     = sys.env.getOrElse("MONGO_USER", "")
+  private val mongoPassword = sys.env.getOrElse("MONGO_PASSWORD", "")
+  private val mongoDatabase = sys.env.getOrElse("MONGO_DATABASE", "")
+  private val collectionName                = "permalinks"
+  private val mongoConnectionString =
+    s"mongodb+srv://$mongoUser:$mongoPassword@cluster0.pnja6.mongodb.net/$mongoDatabase" +
+      "?retryWrites=true&w=majority"
 }
 
 object PermalinkService {
   def apply[F[_]: Effect: ContextShift](blocker: Blocker, client: Client[F]): PermalinkService[F] =
-    new PermalinkService[F]()
+    new PermalinkService[F](blocker, client)
 }
-
-