Skip to content

Add source location URL to CI test failures. #311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zainab-ali wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add source location URL to CI test failures. #311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/features/troubleshooting_failures.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,16 @@ object SourceLocationSuite extends FunSuite {
println(weaver.docs.Output.runSuites(SourceLocationSuite))
```

## Displaying source locations in CI test runs

Source locations in CI runs are displayed as urls. If you use GitHub Actions, the source code URL is determined automatically.

You can display urls in other CI providers using the `WEAVER_SOURCE_URL` environment variable.

```sh
export WEAVER_SOURCE_URL=https://gitlab.com/my-org/my-repo/-/tree/my-branch/
```

## Displaying a trace

If you have a test codebase with many nested helper functions, you may want to display a trace of source locations. You can do this with the `traced(here)` function.
Expand Down
2 changes: 2 additions & 0 deletions modules/core-cats/shared/src/main/scala/weaver/CatsUnsafeRun.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import scala.concurrent.Future

import cats.effect.unsafe.implicits.global
import cats.effect.{ FiberIO, IO }
import cats.effect.std.Env

object CatsUnsafeRun extends CatsUnsafeRun

Expand All @@ -13,6 +14,7 @@ trait CatsUnsafeRun extends UnsafeRun[IO] with CatsUnsafeRunPlatformCompat {

override implicit val parallel = IO.parallelForIO
override implicit val effect = IO.asyncForIO
override def env: Env[IO] = IO.envForIO

def cancel(token: CancelToken): Unit = unsafeRunSync(token.cancel)

Expand Down
2 changes: 1 addition & 1 deletion modules/core/shared/src/main/scala/weaver/Formatter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ object Formatter {
case Success => withPrefix(green("+ "))
case OnlyTagNotAllowedInCI(_) | Failures(_) | Exception(_) =>
withPrefix(red("- "))
case Ignored(_, _) =>
case Ignored(_, _, _) =>
withPrefix(yellow("- ")) + yellow(" !!! IGNORED !!!")
}
}
Expand Down
60 changes: 45 additions & 15 deletions modules/core/shared/src/main/scala/weaver/Result.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,29 @@ private[weaver] sealed trait Result {
private[weaver] object Result {
import Formatter._

def fromAssertion(assertion: Expectations): Result = assertion.run match {
case Valid(_) => Success
case Invalid(failed) =>
Failures(failed.map(ex =>
Failures.Failure(ex.message, ex, ex.locations)))
}
def fromAssertion(
sourceLocationUrl: Option[String],
assertion: Expectations): Result =
assertion.run match {
case Valid(_) => Success
case Invalid(failed) =>
Failures(failed.map(ex =>
Failures.Failure(ex.message, ex, sourceLocationUrl, ex.locations)))
}

case object Success extends Result {
def formatted: Option[String] = None
}

final case class Ignored(reason: String, location: SourceLocation)
final case class Ignored(
reason: String,
sourceLocationUrl: Option[String],
location: SourceLocation)
extends Result {

def formatted: Option[String] = {
Some(formatDescription(reason,
sourceLocationUrl = sourceLocationUrl,
Copy link
Copy Markdown
Contributor

@majk-p majk-p Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nitpick: this single named parameter looks odd, can we align it with the rest of the code?

List(location),
Console.YELLOW,
TAB2.prefix))
Expand All @@ -40,6 +47,7 @@ private[weaver] object Result {
val failure = failures.head
val formattedMessage = formatDescription(
failure.msg,
failures.head.sourceLocationUrl,
failure.locations.toList,
Console.RED,
TAB2.prefix
Expand All @@ -53,6 +61,7 @@ private[weaver] object Result {

formatDescription(
msg,
sourceLocationUrl,
locations.toList,
Console.RED,
s" [$idx] "
Expand All @@ -67,6 +76,7 @@ private[weaver] object Result {
final case class Failure(
msg: String,
source: ExpectationFailed,
sourceLocationUrl: Option[String],
locations: NonEmptyList[SourceLocation])
}

Expand All @@ -77,6 +87,7 @@ private[weaver] object Result {
def formatted: Option[String] = {
val formattedMessage = formatDescription(
"'Only' tag is not allowed when `isCI=true`",
None,
List(location),
Console.RED,
TAB2.prefix
Expand All @@ -102,13 +113,13 @@ private[weaver] object Result {

val success: Result = Success

def from(error: Throwable): Result = {
def from(sourceLocationUrl: Option[String], error: Throwable): Result = {
error match {
case ex: IgnoredException =>
Ignored(ex.reason, ex.location)
Ignored(ex.reason, sourceLocationUrl, ex.location)
case exs: ExpectationsFailed =>
Failures(exs.failures.map { ex =>
Failures.Failure(ex.message, ex, ex.locations)
Failures.Failure(ex.message, ex, sourceLocationUrl, ex.locations)
})
case other =>
Exception(other)
Expand Down Expand Up @@ -137,6 +148,7 @@ private[weaver] object Result {

if (errorOutputLines.nonEmpty) {
formatDescription(errorOutputLines.mkString(EOL),
None,
Nil,
Console.RED,
TAB2.prefix)
Expand All @@ -145,6 +157,7 @@ private[weaver] object Result {

val formattedMessage = formatDescription(
msg,
None,
Nil,
Console.RED,
TAB2.prefix
Expand All @@ -159,20 +172,21 @@ private[weaver] object Result {

private def formatDescription(
message: String,
sourceLocationUrl: Option[String],
location: List[SourceLocation],
color: String,
prefix: String): String = {

val prefixIsWhitespace = prefix.trim.isEmpty
val footer = locationFooter(location)
val footer = locationFooter(sourceLocationUrl, location)
val lines = (message.split("\\r?\\n") ++ footer).zipWithIndex.map {
case (line, index) =>
val linePrefix =
if (prefixIsWhitespace && line.trim.isEmpty) "" else prefix
if (index == 0)
color + linePrefix + line +
location
.map(l => s" (${l.fileRelativePath}:${l.line})")
.map(l => s" (${formatLocationPath(sourceLocationUrl, l)})")
.mkString("\n")
else
color + linePrefix + line
Expand All @@ -181,14 +195,30 @@ private[weaver] object Result {
lines.mkString(EOL) + Console.RESET
}

private def locationFooter(locations: List[SourceLocation]): List[String] = {
private def locationFooter(
sourceLocationUrl: Option[String],
locations: List[SourceLocation]): List[String] = {
val lines = locations.flatMap { l =>
val prefix = s"${l.fileRelativePath}:${l.line}"
l.sourceCode.fold(List.empty[String]) { sourceCode =>
val pointer = " " * (sourceCode.column - 1) + "^"
List(prefix, sourceCode.sourceLine, pointer)
List(formatLocationPath(sourceLocationUrl, l),
sourceCode.sourceLine,
pointer)
}
}
if (lines.nonEmpty) "" :: lines else Nil
}

private def formatLocationPath(
sourceLocationUrl: Option[String],
l: SourceLocation): String =
sourceLocationUrl match {
case Some(url) =>
// Display a URL to a source location on a CI host. Line numbers are typically referenced with #L anchors.
s"${url}${l.fileRelativePath}#L${l.line}"
Copy link
Copy Markdown
Contributor

@majk-p majk-p Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should the user be able to override the anchor with config/env var?

case None =>
// Display a path to a local file.
s"${l.fileRelativePath}:${l.line}"
}

}
26 changes: 15 additions & 11 deletions modules/core/shared/src/main/scala/weaver/Test.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,25 @@ import cats.data.Chain
import cats.effect.Ref
import cats.effect.Clock
import cats.effect.Concurrent
import cats.effect.std.Env
import cats.syntax.all._

import weaver.internals.SourceLocationUrl
object Test {

def apply[F[_]](name: String, f: Log[F] => F[Expectations])(
implicit F: Defer[F],
G: Concurrent[F],
C: Clock[F]): F[TestOutcome] = {
C: Clock[F],
E: Env[F]
Copy link
Copy Markdown
Collaborator

@Baccata Baccata Apr 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure it's necessary for the change to impact these layers of the framework. You could centralise the querying of the environment in the formatter (in an unsafe fashion), and apply the github-specific rendering there.

I appreciate that reading the environment does not fall strictly in the pure-FP paradigm that Typelevel ougth to strive for, but the sbt-test-interface that weaver implements forces some compromise onto us. I'm happy to deviate a little bit out of the CE idioms if it minimises the impact / the amount of things the information needs to be threaded through. Additionally, the set of environment variables is pretty much immutable during the lifetime of a program.

): F[TestOutcome] = {
for {
ref <- Ref[F].of(Chain.empty[Log.Entry])
start <- C.realTime
res <- Defer[F]
ref <- Ref[F].of(Chain.empty[Log.Entry])
start <- C.realTime
sourceUrl <- SourceLocationUrl[F]
res <- Defer[F]
.defer(f(Log.collected[F, Chain](ref, C.realTime.map(_.toMillis))))
.map(Result.fromAssertion)
.handleError(ex => Result.from(ex))
.map(Result.fromAssertion(sourceUrl, _))
.handleError(ex => Result.from(sourceUrl, ex))
end <- C.realTime
logs <- ref.get
} yield TestOutcome(name, end - start, res, logs)
Expand All @@ -33,8 +37,8 @@ object Test {
val (attempt, duration) = Try(ex()) -> (System.currentTimeMillis() - start)

val res = attempt match {
case Success(assertions) => Result.fromAssertion(assertions)
case Failure(e) => Result.from(e)
case Success(assertions) => Result.fromAssertion(None, assertions)
case Failure(e) => Result.from(None, e)
}

TestOutcome(name, duration.millis, res, Chain.empty)
Expand All @@ -43,7 +47,7 @@ object Test {
def apply[F[_]](name: String, f: F[Expectations])(
implicit F: Defer[F],
G: Concurrent[F],
C: Clock[F]
C: Clock[F],
E: Env[F]
): F[TestOutcome] = apply[F](name, (_: Log[F]) => f)

}
5 changes: 3 additions & 2 deletions modules/core/shared/src/main/scala/weaver/TestOutcome.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ object TestOutcome {
extends TestOutcome {

def status: TestStatus = result match {
case Result.Success => TestStatus.Success
case Result.Ignored(_, _) => TestStatus.Ignored
case Result.Success => TestStatus.Success
case Result.Ignored(_, _, _) => TestStatus.Ignored
case Result.OnlyTagNotAllowedInCI(_) | Result.Failures(_) =>
TestStatus.Failure
case Result.Exception(_) => TestStatus.Exception
Expand All @@ -47,6 +47,7 @@ object TestOutcome {
case Result.Failures(failures) =>
Some(new ExpectationsFailed(failures.map(_.source)))
case Result.OnlyTagNotAllowedInCI(_) | Result.Ignored(
_,
_,
_) | Result.Success => None
}
Expand Down
2 changes: 2 additions & 0 deletions modules/core/shared/src/main/scala/weaver/UnsafeRun.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ import scala.concurrent.duration.FiniteDuration

import cats.Parallel
import cats.effect.{ Async, Clock }
import cats.effect.std.Env

trait EffectCompat[F[_]] {
implicit def parallel: Parallel[F]
implicit def effect: Async[F]
def env: Env[F]
protected[weaver] def clock: Clock[F] = effect

private[weaver] final def sleep(duration: FiniteDuration): F[Unit] =
Expand Down
29 changes: 29 additions & 0 deletions modules/core/shared/src/main/scala/weaver/internals/SourceLocationUrl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package weaver.internals

import cats.effect.std.Env
import cats.Monad
import cats.syntax.all._

/**
* Constructs base URL to use for displaying source locations in failure
* messages.
*/
private[weaver] object SourceLocationUrl {
def apply[F[_]](implicit E: Env[F], F: Monad[F]): F[Option[String]] = {
// Users can support non-GitHub CIs by setting WEAVER_SOURCE_URL
val readWeaverUrl = E.get("WEAVER_SOURCE_URL")
// If tests are run on GitHub Actions, construct a URL to the test source code.
// See https://docs.github.com/en/actions/reference/workflows-and-actions/variables#default-environment-variables
val readGitHubUrl = (
E.get("GITHUB_SERVER_URL"),
E.get("GITHUB_REPOSITORY"),
E.get("GITHUB_SHA")
).tupled.map(_.mapN { case (serverUrl, repo, sha) =>
s"$serverUrl/$repo/tree/$sha/"
})
(readWeaverUrl, readGitHubUrl).mapN {
case (weaverUrl, gitHubUrl) => weaverUrl.orElse(gitHubUrl)
}
}

}
30 changes: 24 additions & 6 deletions modules/core/shared/src/main/scala/weaver/suites.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ trait BaseSuiteClass {}
// A version of EffectSuite that has a type member instead of a type parameter.
protected[weaver] trait EffectSuiteAux {
type EffectType[A]
protected def effectCompat: EffectCompat[EffectType]
implicit protected def effect: Async[EffectType]
}

Expand Down Expand Up @@ -158,25 +159,38 @@ private[weaver] object TagAnalysisResult {
abstract class MutableFSuite[F[_]] extends SharedResourceSuite[F] {
def pureTest(name: TestName)(run: => Expectations): Unit =
registerTest(name)(_ =>
Test(name.name, effect.delay(run))(effect, effect, effectCompat.clock))
Test(name.name, effect.delay(run))(effect,
effect,
effectCompat.clock,
effectCompat.env))
def loggedTest(name: TestName)(run: Log[F] => F[Expectations]): Unit =
registerTest(name)(_ =>
Test[F](name.name, log => run(log))(effect, effect, effectCompat.clock))
Test[F](name.name, log => run(log))(effect,
effect,
effectCompat.clock,
effectCompat.env))
def test(name: TestName): PartiallyAppliedTest =
new PartiallyAppliedTest(name)

class PartiallyAppliedTest(name: TestName) {
def apply(run: => F[Expectations]): Unit =
registerTest(name)(_ =>
Test(name.name, run)(effect, effect, effectCompat.clock))
Test(name.name, run)(effect,
effect,
effectCompat.clock,
effectCompat.env))
def apply(run: Res => F[Expectations]): Unit =
registerTest(name)(res =>
Test(name.name, run(res))(effect, effect, effectCompat.clock))
Test(name.name, run(res))(effect,
effect,
effectCompat.clock,
effectCompat.env))
def apply(run: (Res, Log[F]) => F[Expectations]): Unit =
registerTest(name)(res =>
Test[F](name.name, log => run(res, log))(effect,
effect,
effectCompat.clock))
effectCompat.clock,
effectCompat.env))

// this alias helps using pattern matching on `Res`
def usingRes(run: Res => F[Expectations]): Unit = apply(run)
Expand All @@ -189,7 +203,11 @@ abstract class FunSuiteF[F[_]] extends SharedResourceSuite[F] {
override final def maxParallelism: Int = 1

def test(name: TestName)(run: => Expectations): Unit =
registerTest(name)(_ => effect.pure(Test.pure(name.name)(() => run)))
registerTest(name)(_ =>
Test(name.name, effect.delay(run))(effect,
effect,
effectCompat.clock,
effectCompat.env))

}

Expand Down
Loading
Loading