diff --git a/.github/workflows/scala-release.yml b/.github/workflows/scala-release.yml index 66383a9..e44b2b1 100644 --- a/.github/workflows/scala-release.yml +++ b/.github/workflows/scala-release.yml @@ -11,14 +11,16 @@ jobs: runs-on: ubuntu-latest steps: - - - uses: actions/checkout@v2 - - - name: Set up JDK 11 - uses: actions/setup-java@v2 + - uses: actions/checkout@v4 + + - name: Setup JDK + uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: 11 + distribution: temurin + java-version: 17 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@v1 - name: Get the version id: get_version @@ -34,7 +36,7 @@ jobs: run: cp target/universal/*.tgz code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz - name: Upload a Build Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz path: code-examples-manager-${{steps.get_version.outputs.VERSION}}.tgz diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index 92170c7..4ab070f 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -1,27 +1,23 @@ -name: Scala CI - +name: CI on: + pull_request: + branches: [ master ] push: branches: [ master ] paths-ignore: - 'README.md' - pull_request: - branches: [ master ] - jobs: build: - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v2 - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 11 - - - name: Run tests - run: sbt test + - name: Checkout + uses: actions/checkout@v4 + - name: Setup JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + - name: Setup sbt launcher + uses: sbt/setup-sbt@v1 + - name: Build and Test + run: sbt test diff --git a/.gitignore b/.gitignore index aa50240..9876dd4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,28 @@ -.DS_Store/ +# Security purposes +.envrc + +# temporary files *~ -.~* -.idea/ -.bsp/ +*.log nohup.out *.swp -*.iml -target/ -plot*.html -test*.log -*.tmp -null +.attach_pid* +# build related +target/ project/target +metals.sbt + +# IDE related .metals/ +.bloop/ +.bsp/ +.vscode/ +.idea/ tmp-*.gif tmp-*.png -private-application*.conf +# application related +private-application*.conf* +.lmdb/ diff --git a/.scalafmt.conf b/.scalafmt.conf index 1fb6b5b..b350268 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = 3.7.2 +version = 3.9.4 runner.dialect = scala3 align.preset = most maxColumn = 200 diff --git a/README.md b/README.md index fe8aa06..026f620 100644 --- a/README.md +++ b/README.md @@ -3,18 +3,18 @@ Code example manager (CEM) is a software managing your notes, scripts and code examples. It provides publish mechanisms to [github.com][githubcom] (as [gists][gists]) or [gitlab.com][gitlabcom] (as [snippets][snippets]). It also automates execution for -testable examples this is a quite useful when you have to deal with many examples. +testable examples, this is a quite useful to manage a high number of examples. All my notes, scripts and code examples (my programming knowledge base) are now managed using this tool, you can take a look to **[my public gists overview on github][mygists]** to illustrate the publishing work achieved by CEM. -![](images/cloudtags.png) - Current [Code example manager (CEM)][cem] implementation is just a command line tool which compare locally available examples with already published ones in order to find what it should do (add, update, do nothing). +![](images/cloudtags.png) + ## Why ? Code examples are very important, each example is most of the time designed to focus @@ -28,7 +28,10 @@ Managing hundreds of published code example files as gists (github) and/or snipp is really not easy and time-consuming, in particular if you want to keep them up to date. This is the main issue addressed by this software. -![](images/created-examples-trend.png) +- My open-source examples evolution trend : + ![](images/created-examples-trend.png) +- and the execution status trend for executable/testable ones : + ![](images/testable-examples-status.png) As you can see through the previous charts, once you have industrialized your notes and code examples, analytics on your examples become quite easy, and a lot of advanced features become @@ -62,7 +65,7 @@ Instructions example with github.com publishing configuration : ``` - Run the following command from your terminal (`cs` is the [coursier][cs] CLI command): ``` - cs launch fr.janalyse:code-examples-manager_3:2.2.0 + cs launch fr.janalyse:code-examples-manager_3:2.4.0 ``` - you can even use `cs launch fr.janalyse:code-examples-manager_3:latest.release` to always use the latest release - current release is : [![][CodeExamplesManagerImg]][CodeExamplesManagerLnk] @@ -202,7 +205,7 @@ Get an access token from gitlab : ### Github authentication token configuration -Get an access token from gitlab.com : +Get an access token from github.com : - Got to your user **settings** - Select **Developer settings** - Select **Personal access tokens** diff --git a/build.sbt b/build.sbt index 3194c80..df4f1a4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,31 +1,23 @@ -organization := "fr.janalyse" name := "code-examples-manager" -homepage := Some(new URL("https://github.com/dacr/code-examples-manager")) +organization := "fr.janalyse" +description := "Tool to manage set of code examples : synchronize and publish, automated execution, ..." -licenses += "Apache 2" -> url(s"https://www.apache.org/licenses/LICENSE-2.0.txt") +licenses += "NON-AI-APACHE2" -> url(s"https://github.com/non-ai-licenses/non-ai-licenses/blob/main/NON-AI-APACHE2") -scmInfo := Some( - ScmInfo( - url(s"https://github.com/dacr/code-examples-manager.git"), - s"git@github.com:dacr/code-examples-manager.git" - ) -) +scalaVersion := "3.6.4" -scalaVersion := "3.3.0" - -mainClass := Some("fr.janalyse.cem.Main") +scalacOptions += "-Xkind-projector:underscores" lazy val versions = new { - val sttp = "3.8.15" - val zio = "2.0.13" - val zionio = "2.0.1" + val sttp = "3.11.0" + val zio = "2.1.17" + val zionio = "2.0.2" val zioproc = "0.7.2" - val zioconfig = "4.0.0-RC16" - val ziologging = "2.1.13" - val ziolmdb = "1.1.0" - val naturalsort = "1.0.2" - val jgit = "6.5.0.202303070854-r" - val logback = "1.4.7" + val zioconfig = "4.0.4" + val ziologging = "2.5.0" + val ziolmdb = "2.0.1" + val naturalsort = "1.0.7" + val jgit = "7.2.0.202503040940-r" } libraryDependencies ++= Seq( @@ -38,7 +30,7 @@ libraryDependencies ++= Seq( "dev.zio" %% "zio-nio" % versions.zionio, "dev.zio" %% "zio-process" % versions.zioproc, "dev.zio" %% "zio-logging" % versions.ziologging, - "dev.zio" %% "zio-logging-slf4j" % versions.ziologging, + "dev.zio" %% "zio-logging-slf4j-bridge" % versions.ziologging, "dev.zio" %% "zio-config" % versions.zioconfig, "dev.zio" %% "zio-config-typesafe" % versions.zioconfig, "dev.zio" %% "zio-config-magnolia" % versions.zioconfig, @@ -46,24 +38,28 @@ libraryDependencies ++= Seq( "com.softwaremill.sttp.client3" %% "async-http-client-backend-zio" % versions.sttp, "com.softwaremill.sttp.client3" %% "zio-json" % versions.sttp, "fr.janalyse" %% "naturalsort" % versions.naturalsort, - "org.eclipse.jgit" % "org.eclipse.jgit" % versions.jgit, - "ch.qos.logback" % "logback-classic" % versions.logback + "org.eclipse.jgit" % "org.eclipse.jgit" % versions.jgit ) -//excludeDependencies += "org.scala-lang.modules" % "scala-collection-compat_2.13" - testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") -enablePlugins(SbtTwirl) - -// TODO - to remove when twirl will be available for scala3 -libraryDependencies := libraryDependencies.value.map { - case module if module.name == "twirl-api" => module.cross(CrossVersion.for3Use2_13) - case module => module -} - TwirlKeys.templateImports += "fr.janalyse.cem.model._" +mainClass := Some("fr.janalyse.cem.Main") + // ZIO-LMDB requires special authorization at JVM level ThisBuild / fork := true ThisBuild / javaOptions ++= Seq("--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED") + +enablePlugins(SbtTwirl) + +homepage := Some(url("https://github.com/dacr/code-examples-manager")) +scmInfo := Some(ScmInfo(url(s"https://github.com/dacr/code-examples-manager.git"), s"git@github.com:dacr/code-examples-manager.git")) +developers := List( + Developer( + id = "dacr", + name = "David Crosson", + email = "crosson.david@gmail.com", + url = url("https://github.com/dacr") + ) +) diff --git a/images/cloudtags.png b/images/cloudtags.png index e857acc..ca59a56 100644 Binary files a/images/cloudtags.png and b/images/cloudtags.png differ diff --git a/images/created-examples-trend.png b/images/created-examples-trend.png index 74ff797..f03a8b9 100644 Binary files a/images/created-examples-trend.png and b/images/created-examples-trend.png differ diff --git a/images/testable-examples-status.png b/images/testable-examples-status.png new file mode 100644 index 0000000..bc3613e Binary files /dev/null and b/images/testable-examples-status.png differ diff --git a/project/build.properties b/project/build.properties index 6ee5b93..0cedcca 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ # suppress inspection "UnusedProperty" for whole file -sbt.version=1.8.2 +sbt.version=1.10.11 diff --git a/project/plugins.sbt b/project/plugins.sbt index 97ba078..af717e8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,7 @@ -addSbtPlugin("com.github.sbt" % "sbt-release" % "1.1.0") -addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.18") -addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.3") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") -addSbtPlugin("com.typesafe.play" % "sbt-twirl" % "1.6.0-RC2") -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.4") +addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") +addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.2") +addSbtPlugin("org.playframework.twirl" % "sbt-twirl" % "2.0.8") diff --git a/publish.sbt b/publish.sbt index 5af2ee0..a3c164b 100644 --- a/publish.sbt +++ b/publish.sbt @@ -1,40 +1,35 @@ -pomIncludeRepository := { _ => false } - -releaseCrossBuild := true -releasePublishArtifactsAction := PgpKeys.publishSigned.value -publishMavenStyle := true +pomIncludeRepository := { _ => false } +publishMavenStyle := true Test / publishArtifact := false -publishTo := Some(if (isSnapshot.value) Opts.resolver.sonatypeSnapshots else Opts.resolver.sonatypeStaging) +releaseCrossBuild := true +versionScheme := Some("semver-spec") -Global / PgpKeys.useGpg := true // workaround with pgp and sbt 1.2.x -pgpSecretRing := pgpPublicRing.value // workaround with pgp and sbt 1.2.x - -pomExtra in Global := { - - - dacr - David Crosson - https://github.com/dacr - - +publishTo := { + // For accounts created after Feb 2021: + // val nexus = "https://s01.oss.sonatype.org/" + val nexus = "https://oss.sonatype.org/" + if (isSnapshot.value) Some("snapshots" at nexus + "content/repositories/snapshots") + else Some("releases" at nexus + "service/local/staging/deploy/maven2") } -releaseTagComment := s"Releasing ${(ThisBuild / version).value}" -releaseCommitMessage := s"Setting version to ${(ThisBuild / version).value}" +releasePublishArtifactsAction := PgpKeys.publishSigned.value + +releaseTagComment := s"Releasing ${(ThisBuild / version).value}" +releaseCommitMessage := s"Setting version to ${(ThisBuild / version).value}" releaseNextCommitMessage := s"[ci skip] Setting version to ${(ThisBuild / version).value}" -import ReleaseTransformations._ +import ReleaseTransformations.* releaseProcess := Seq[ReleaseStep]( checkSnapshotDependencies, inquireVersions, - //runClean, + runClean, runTest, setReleaseVersion, commitReleaseVersion, tagRelease, publishArtifacts, + releaseStepCommand("sonatypeReleaseAll"), setNextVersion, commitNextVersion, - releaseStepCommand("sonatypeReleaseAll"), pushChanges ) diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml deleted file mode 100644 index b49ba56..0000000 --- a/src/main/resources/logback.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - %msg %mdc%n - - - - - - - - - - diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index bd2e3a8..782c626 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -86,3 +86,8 @@ lmdb { name = code-examples-manager-data sync = false } + +logger { + #format = "%highlight{%timestamp{yyyy-MM-dd'T'HH:mm:ssZ} %fixed{7}{%level} [%fiberId] %name:%line %message %kvs %cause}" + format = "%highlight{%level [%fiberId] \"%message\" %spans %kvs %cause}}" +} diff --git a/src/main/scala/fr/janalyse/cem/Execute.scala b/src/main/scala/fr/janalyse/cem/Execute.scala index 7a46461..84f3e42 100644 --- a/src/main/scala/fr/janalyse/cem/Execute.scala +++ b/src/main/scala/fr/janalyse/cem/Execute.scala @@ -64,7 +64,7 @@ object Execute { .map(_.split("\\s+").toList) ) .orElseFail(RunFailure(s"Example ${example.uuid} as no run-with directive")) - results <- makeCommandProcess(command, workingDir) @@ annotated("example-run-command" -> command.mkString(" ")) + results <- makeCommandProcess(command, workingDir) @@ annotated("/run-command" -> command.mkString(" ")) } yield results } @@ -73,7 +73,7 @@ object Execute { exampleFilePath <- ZIO.fromOption(example.filepath).orElseFail(RunFailure("Example has no path for its content")) workingDir <- ZIO.fromOption(exampleFilePath.parent).orElseFail(RunFailure("Example file path content has no parent directory")) command <- ZIO.succeed(example.testWith.getOrElse(s"sleep ${timeoutDuration.getSeconds()}").trim.split("\\s+").toList) - results <- makeCommandProcess(command, workingDir) @@ annotated("example-test-command" -> command.mkString(" ")) + results <- makeCommandProcess(command, workingDir) @@ annotated("/test-command" -> command.mkString(" ")) } yield results } @@ -122,7 +122,8 @@ object Execute { _ <- upsertRunStatus(runStatus) } yield runStatus - ZIO.logAnnotate("file", example.filename)(result) + // ZIO.logAnnotate("/file", example.filename)(result) + result } def runTestableExamples(runnableExamples: List[CodeExample], parallelism: Int) = { @@ -133,8 +134,10 @@ object Execute { runSessionUUID = UUID.randomUUID() // runStatuses <- ZIO.foreachExec(runnableExamples)(execStrategy)(example => runExample(example, runSessionDate, runSessionUUID)) runStatuses <- ZIO.foreachExec(runnableExamples)(execStrategy) { example => - runExample(example, runSessionDate, runSessionUUID) - @@ annotated("example-uuid" -> example.uuid.toString, "example-filename" -> example.filename) + ZIO.logSpan("/run") { + runExample(example, runSessionDate, runSessionUUID) + @@ annotated("/uuid" -> example.uuid.toString, "/file" -> example.filename) + } } successes = runStatuses.filter(_.success) failures = runStatuses.filterNot(_.success) @@ -178,7 +181,7 @@ object Execute { } yield () } - def executeEffect(keywords: Set[String] = Set.empty): ZIO[FileSystemService & LMDB, Throwable | ExampleIssue, List[RunStatus]] = { + def executeEffect(keywords: Set[String] = Set.empty): ZIO[FileSystemService & LMDB, Throwable | ExampleIssue, List[RunStatus]] = ZIO.logSpan("/runs") { for { _ <- ZIO.log("Searching examples...") examples <- Synchronize.examplesCollect diff --git a/src/main/scala/fr/janalyse/cem/Main.scala b/src/main/scala/fr/janalyse/cem/Main.scala index 6eb9953..81f419c 100644 --- a/src/main/scala/fr/janalyse/cem/Main.scala +++ b/src/main/scala/fr/janalyse/cem/Main.scala @@ -4,10 +4,10 @@ import zio.* import zio.config.* import zio.config.typesafe.* import zio.config.magnolia.* - -import zio.logging.{LogFormat, removeDefaultLoggers} -import zio.logging.backend.SLF4J +import zio.logging.{ConsoleLoggerConfig, LogFormat, consoleLogger} +import zio.logging.slf4j.bridge.Slf4jBridge import sttp.client3.asynchttpclient.zio.AsyncHttpClientZioBackend +import zio.Runtime.removeDefaultLoggers import zio.lmdb.{LMDB, LMDBConfig} object Main extends ZIOAppDefault { @@ -35,8 +35,9 @@ object Main extends ZIOAppDefault { configProvider } - override val bootstrap = - removeDefaultLoggers ++ SLF4J.slf4j(format = LogFormat.colored) ++ ZLayer.fromZIO(configProviderLogic.map(provider => Runtime.setConfigProvider(provider))).flatten + val configLayer = ZLayer.fromZIO(configProviderLogic.map(provider => Runtime.setConfigProvider(provider))).flatten + + override val bootstrap = configLayer ++ ( removeDefaultLoggers >>> configLayer >>> consoleLogger() >>> Slf4jBridge.initialize) val httpClientLayer = AsyncHttpClientZioBackend.layer() diff --git a/src/main/scala/fr/janalyse/cem/RemoteGithubOperations.scala b/src/main/scala/fr/janalyse/cem/RemoteGithubOperations.scala index a4be5b2..d17bb9e 100644 --- a/src/main/scala/fr/janalyse/cem/RemoteGithubOperations.scala +++ b/src/main/scala/fr/janalyse/cem/RemoteGithubOperations.scala @@ -170,7 +170,7 @@ object RemoteGithubOperations { Obj( "description" -> Str(description), "public" -> Bool(publicBool), - "files" -> Obj(files: _*) + "files" -> Obj(files*) ) } @@ -221,7 +221,7 @@ object RemoteGithubOperations { Obj( "description" -> Str(description), - "files" -> Obj(files: _*) + "files" -> Obj(files*) ) } diff --git a/src/main/scala/fr/janalyse/cem/Synchronize.scala b/src/main/scala/fr/janalyse/cem/Synchronize.scala index a1894f0..5d03faa 100644 --- a/src/main/scala/fr/janalyse/cem/Synchronize.scala +++ b/src/main/scala/fr/janalyse/cem/Synchronize.scala @@ -66,7 +66,7 @@ object Synchronize { validExamples = foundExamples.collect { case Right(example) => example } invalidExamples = foundExamples.collect { case Left(example) => example } _ <- ZIO.foreach(invalidExamples)(issue => ZIO.logWarning(issue.toString)) - _ <- examplesCheckCoherency(foundExamples) + _ <- examplesCheckCoherency(foundExamples) } yield validExamples } @@ -99,17 +99,19 @@ object Synchronize { _ <- ZIO.logInfo(s"found ${testable.size} testable and executable examples using run-with directive") _ <- ZIO.logInfo(s"found ${missingRunWith.size} testable examples without run-with instruction") _ <- ZIO.logInfo(s"add runWith to those examples${missingRunWith.mkString("\n -", "\n -", "")}") - _ <- ZIO.cond(duplicatedUUIDs.size == 0, (), RuntimeException(s"Duplicated UUIDs ${duplicatedUUIDs.keys.mkString(",")}")) // TODO enhance error management + _ <- ZIO.cond(duplicatedUUIDs.size == 0, (), RuntimeException(s"Duplicated UUIDs ${duplicatedUUIDs.keys.mkString(",")}")) // TODO enhance error management _ <- ZIO.cond(duplicatedSummaries.size == 0, (), RuntimeException(s"Duplicated summaries ${duplicatedSummaries.keys.mkString(",")}")) // TODO enhance error management } yield () } - val examplesCollect: ZIO[FileSystemService & LMDB, ExampleIssue | Throwable, List[CodeExample]] = for { - searchRootDirectories <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.examples.searchRootDirectories) - searchRoots <- examplesValidSearchRoots(searchRootDirectories) - _ <- ZIO.log(s"Searching examples in ${searchRoots.mkString(",")}") - localExamples <- examplesCollectFor(searchRoots) - } yield localExamples + val examplesCollect: ZIO[FileSystemService & LMDB, ExampleIssue | Throwable, List[CodeExample]] = ZIO.logSpan("/collect") { + for { + searchRootDirectories <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.examples.searchRootDirectories) + searchRoots <- examplesValidSearchRoots(searchRootDirectories) + _ <- ZIO.log(s"Searching examples in ${searchRoots.mkString(",")}") + localExamples <- examplesCollectFor(searchRoots) + } yield localExamples + } def computeWorkToDo(examples: Iterable[CodeExample], states: Iterable[RemoteExampleState]): List[WhatToDo] = { val statesByUUID = states.map(state => state.uuid -> state).toMap @@ -171,7 +173,7 @@ object Synchronize { } } - def examplesPublish(examples: Iterable[CodeExample]): RIO[SttpClient, Unit] = { + def examplesPublish(examples: Iterable[CodeExample]): RIO[SttpClient, Unit] = ZIO.logSpan("/publish") { for { adapters <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.publishAdapters) _ <- ZIO.foreachPar(adapters.toList) { case (adapterName, adapterConfig) => @@ -187,7 +189,7 @@ object Synchronize { .map { case (key, examples) => key -> examples.size } } - def statsEffect(examples: List[CodeExample]) = + def statsEffect(examples: List[CodeExample]) = ZIO.logSpan("stats") { for { metaInfo <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.metaInfo) version = metaInfo.version @@ -208,6 +210,7 @@ object Synchronize { |Defined keywords : ${examples.flatMap(_.keywords).distinct.sorted.mkString(",")} |""".stripMargin } yield message + } val versionEffect = for { @@ -226,18 +229,19 @@ object Synchronize { |""".stripMargin } yield message - def synchronizeEffect: ZIO[SttpClient & FileSystemService & LMDB, ExampleIssue | Throwable, Unit] = for { - startTime <- Clock.nanoTime - metaInfo <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.metaInfo) - appName = metaInfo.name - version <- versionEffect - _ <- ZIO.log(s"\n$version") - examples <- examplesCollect - stats <- statsEffect(examples) - _ <- ZIO.log(s"\n$stats") - _ <- examplesPublish(examples) - endTime <- Clock.nanoTime - _ <- ZIO.log(s"$appName publishing operations took ${(endTime - startTime) / 1000000}ms") - } yield () + def synchronizeEffect: ZIO[SttpClient & FileSystemService & LMDB, ExampleIssue | Throwable, Unit] = { + ZIO.logSpan("/synchronize") { + for { + metaInfo <- ZIO.config(ApplicationConfig.config).map(_.codeExamplesManagerConfig.metaInfo) + appName = metaInfo.name + version <- versionEffect + _ <- ZIO.log(s"\n$version") + examples <- examplesCollect + stats <- statsEffect(examples) + _ <- ZIO.log(s"\n$stats") + _ <- examplesPublish(examples) + } yield () + } + } } diff --git a/src/main/scala/fr/janalyse/cem/model/CodeExample.scala b/src/main/scala/fr/janalyse/cem/model/CodeExample.scala index 4f0812e..8596970 100644 --- a/src/main/scala/fr/janalyse/cem/model/CodeExample.scala +++ b/src/main/scala/fr/janalyse/cem/model/CodeExample.scala @@ -19,7 +19,7 @@ import fr.janalyse.cem.FileSystemService import fr.janalyse.cem.tools.* import fr.janalyse.cem.tools.Hashes.sha1 import zio.* -import zio.lmdb.* +import zio.lmdb.*, zio.lmdb.json.* import zio.nio.charset.Charset import zio.nio.file.* import zio.json.* @@ -63,20 +63,20 @@ case class CodeExample( updatedCount: Option[Int] = None, // computed from GIT history attachments: Map[String, String] = Map.empty, // embedded lastSeen: Option[OffsetDateTime] = None // last seen/used date, useful for database garbage collection purposes -) { +) derives LMDBCodecJson { def fileExtension: String = filename.split("[.]", 2).drop(1).headOption.getOrElse("") def isTestable: Boolean = keywords.contains("@testable") def isExclusive: Boolean = keywords.contains("@exclusive") // exclusive examples are executed sequentially def shouldFail: Boolean = keywords.contains("@fail") - def isPublishable: Boolean = !publish.isEmpty + def isPublishable: Boolean = publish.nonEmpty override def toString: String = s"$category $filename $uuid $summary" } object CodeExample { given JsonEncoder[Path] = JsonEncoder[String].contramap(p => p.toString) given JsonDecoder[Path] = JsonDecoder[String].map(p => Path(p)) - given JsonDecoder[CodeExample] = DeriveJsonDecoder.gen - given JsonEncoder[CodeExample] = DeriveJsonEncoder.gen + //given JsonDecoder[CodeExample] = DeriveJsonDecoder.gen + //given JsonEncoder[CodeExample] = DeriveJsonEncoder.gen def exampleContentExtractValue(from: String, key: String): Option[String] = { val RE = ("""(?m)(?i)^(?:(?:// )|(?:## )|(?:- )|(?:-- )) *""" + key + """ *: *(.*)$""").r diff --git a/src/main/scala/fr/janalyse/cem/model/CodeExampleMetaData.scala b/src/main/scala/fr/janalyse/cem/model/CodeExampleMetaData.scala index 39b0556..33738b1 100644 --- a/src/main/scala/fr/janalyse/cem/model/CodeExampleMetaData.scala +++ b/src/main/scala/fr/janalyse/cem/model/CodeExampleMetaData.scala @@ -2,11 +2,12 @@ package fr.janalyse.cem.model import fr.janalyse.cem.tools.GitMetaData import zio.json.* +import zio.lmdb.json.LMDBCodecJson import java.time.OffsetDateTime case class CodeExampleMetaData( gitMetaData: Option[GitMetaData], metaDataFileContentHash: String, - metaDataLastUsed: OffsetDateTime // last seen/used date, usefull for database garbage collection purposes -) derives JsonCodec + metaDataLastUsed: OffsetDateTime // last seen/used date, useful for database garbage collection purposes +) derives LMDBCodecJson diff --git a/src/main/scala/fr/janalyse/cem/model/RunStatus.scala b/src/main/scala/fr/janalyse/cem/model/RunStatus.scala index e645c0b..606e551 100644 --- a/src/main/scala/fr/janalyse/cem/model/RunStatus.scala +++ b/src/main/scala/fr/janalyse/cem/model/RunStatus.scala @@ -1,6 +1,8 @@ package fr.janalyse.cem.model import zio.json.* +import zio.lmdb.json.LMDBCodecJson + import java.util.UUID import java.time.OffsetDateTime @@ -15,5 +17,5 @@ case class RunStatus( success: Boolean, timeout: Boolean, runState: String -) derives JsonCodec +) derives LMDBCodecJson diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml deleted file mode 100644 index 02956d9..0000000 --- a/src/test/resources/logback-test.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - ${log.application.output:-test.log} - - %date %level [%thread] %logger{10} %msg%n - - - - - - %date %level [%thread] %logger{10} %msg%n - - - - - - - - - diff --git a/src/test/scala/fr/janalyse/cem/RemoteGithubOperationsSpec.scala b/src/test/scala/fr/janalyse/cem/RemoteGithubOperationsSpec.scala index ebeb098..228e239 100644 --- a/src/test/scala/fr/janalyse/cem/RemoteGithubOperationsSpec.scala +++ b/src/test/scala/fr/janalyse/cem/RemoteGithubOperationsSpec.scala @@ -24,6 +24,9 @@ import fr.janalyse.cem.model.WhatToDo.* import fr.janalyse.cem.tools.DescriptionTools.* import org.junit.runner.RunWith import sttp.client3.asynchttpclient.zio.AsyncHttpClientZioBackend +import sttp.capabilities.zio.ZioStreams +import sttp.capabilities.WebSockets +import sttp.client3.testing.SttpBackendStub import zio.nio.file.Path import zio.lmdb.LMDB @@ -85,13 +88,13 @@ class RemoteGithubOperationsSpec extends ZIOSpecDefault { results <- githubRemoteExamplesChangesApply(config, todos) } yield results - val stubbedLayer = ZLayer.succeed( - AsyncHttpClientZioBackend.stub - .whenRequestMatches(_.uri.toString() == "https://api.github.com/gists/6e40f8239fa6828ab45a064b8131fdfc") - .thenRespond("""{"id":"aa-bb", "html_url":"https://truc/aa-bb"}""") - ) + val stub: SttpBackendStub[Task, Any] = AsyncHttpClientZioBackend.stub + .whenRequestMatches(_.uri.toString() == "https://api.github.com/gists/6e40f8239fa6828ab45a064b8131fdfc") + .thenRespond("""{"id":"aa-bb", "html_url":"https://truc/aa-bb"}""") + + val stubLayer = ZLayer.succeed(stub) - logic.provide(stubbedLayer).map(result => assertTrue(true)) + logic.provide(stubLayer).map(result => assertTrue(true)) } // ---------------------------------------------------------------------------------------------- diff --git a/src/test/scala/fr/janalyse/cem/SynchronizeSpec.scala b/src/test/scala/fr/janalyse/cem/SynchronizeSpec.scala index de8fee2..162f8c4 100644 --- a/src/test/scala/fr/janalyse/cem/SynchronizeSpec.scala +++ b/src/test/scala/fr/janalyse/cem/SynchronizeSpec.scala @@ -19,27 +19,29 @@ import zio.* import zio.test.* import zio.test.Assertion.* import org.junit.runner.RunWith -import fr.janalyse.cem.model.CodeExample +import fr.janalyse.cem.model.{CodeExample, ExampleIssue} + import java.util.UUID +import scala.util.Success @RunWith(classOf[zio.test.junit.ZTestJUnitRunner]) class SynchronizeSpec extends ZIOSpecDefault { // ---------------------------------------------------------------------------------------------- val t1 = test("check examples coherency success with valid examples") { - val examplesWithIssues = List( - CodeExample.build(filepath = None, filename = "pi-1.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd")), - CodeExample.build(filepath = None, filename = "pi-2.sc", content = "42", uuid = UUID.fromString("a49b0c53-3ec3-4404-bd7d-c249a4868a2b")) + val examplesWithIssues: List[Either[ExampleIssue, CodeExample]] = List( + Right(CodeExample.build(filepath = None, filename = "pi-1.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd"))), + Right(CodeExample.build(filepath = None, filename = "pi-2.sc", content = "42", uuid = UUID.fromString("a49b0c53-3ec3-4404-bd7d-c249a4868a2b"))) ) assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues))(isUnit) } // ---------------------------------------------------------------------------------------------- val t2 = test("check examples coherency should fail on duplicates UUID") { - val examplesWithIssues = List( - CodeExample.build(filepath = None, filename = "pi-1.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd")), - CodeExample.build(filepath = None, filename = "pi-2.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd")) + val examplesWithIssues: List[Either[ExampleIssue, CodeExample]] = List( + Right(CodeExample.build(filepath = None, filename = "pi-1.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd"))), + Right(CodeExample.build(filepath = None, filename = "pi-2.sc", content = "42", uuid = UUID.fromString("e7f1879c-c893-4b3d-bac1-f11f641e90bd"))) ) //assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues).exit)(fails(isSubtype[Exception](anything))) - assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues).exit)(fails(hasMessage(containsString("duplicated UUIDs")))) + assertZIO(Synchronize.examplesCheckCoherency(examplesWithIssues).exit)(fails(hasMessage(containsString("Duplicated UUIDs")))) } // ---------------------------------------------------------------------------------------------- diff --git a/version.sbt b/version.sbt index 45bdb41..3001a65 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "2.3.4-SNAPSHOT" +ThisBuild / version := "2.4.9-SNAPSHOT"