Generate links to external projects in scaladoc.

Review by @VladUreche.

Author: vigdorchik

src/compiler/scala/tools/nsc/doc/Settings.scala

@@ -194,6 +194,12 @@ class Settings(error: String => Unit, val printMsg: String => Unit = println(_))
     "Expand all type aliases and abstract types into full template pages. (locally this can be done with the @template annotation)"
   )
 
+  val docExternalUrls = MultiStringSetting (
+    "-external-urls",
+    "externalUrl(s)",
+    "comma-separated list of package_names=doc_URL for external dependencies, where package names are ':'-separated"
+  )
+
   val docGroups = BooleanSetting (
     "-groups",
     "Group similar functions together (based on the @group annotation)"
@@ -238,6 +244,22 @@ class Settings(error: String => Unit, val printMsg: String => Unit = println(_))
     }
   }
 
+  // TODO: Enable scaladoc to scoop up the package list from another scaladoc site, just as javadoc does
+  //   -external-urls 'http://www.scala-lang.org/archives/downloads/distrib/files/nightly/docs/library'
+  // should trigger scaldoc to fetch the package-list file. The steps necessary:
+  // 1 - list all packages generated in scaladoc in the package-list file, exactly as javadoc:
+  //     see http://docs.oracle.com/javase/6/docs/api/package-list for http://docs.oracle.com/javase/6/docs/api
+  // 2 - download the file and add the packages to the list
+  lazy val extUrlMapping: Map[String, String] = (Map.empty[String, String] /: docExternalUrls.value) {
+    case (map, binding) =>
+      val idx = binding indexOf "="
+      val pkgs = binding substring (0, idx) split ":"
+      var url = binding substring (idx + 1)
+      val index = "/index.html"
+      url = if (url.endsWith(index)) url else url + index
+      map ++ (pkgs map (_ -> url))
+  }
+
   /**
    *  This is the hardcoded area of Scaladoc. This is where "undesirable" stuff gets eliminated. I know it's not pretty,
    *  but ultimately scaladoc has to be useful. :)

src/compiler/scala/tools/nsc/doc/html/HtmlPage.scala

@@ -138,7 +138,8 @@ abstract class HtmlPage extends Page { thisPage =>
         <span class="extmbr" name={ mbr.qualifiedName }>{ inlineToHtml(text) }</span>
     case Tooltip(tooltip) =>
       <span class="extype" name={ tooltip }>{ inlineToHtml(text) }</span>
-    // TODO: add case LinkToExternal here
+    case LinkToExternal(name, url) =>
+      <a href={ url } class="extype" target="_top">{ inlineToHtml(text) }</a>
     case NoLink =>
       inlineToHtml(text)
   }

src/compiler/scala/tools/nsc/doc/html/page/Template.scala

@@ -41,7 +41,7 @@ class Template(universe: doc.Universe, generator: DiagramGenerator, tpl: DocTemp
       <script type="text/javascript" src={ relativeLinkTo{List("jquery-ui.js", "lib")} }></script>
       <script type="text/javascript" src={ relativeLinkTo{List("template.js", "lib")} }></script>
       <script type="text/javascript" src={ relativeLinkTo{List("tools.tooltip.js", "lib")} }></script>
-      { if (universe.settings.docDiagrams.isSetByUser) {
+      { if (universe.settings.docDiagrams.value) {
       <script type="text/javascript" src={ relativeLinkTo{List("modernizr.custom.js", "lib")} }></script>
       <script type="text/javascript" src={ relativeLinkTo{List("diagrams.js", "lib")} } id="diagrams-js"></script>
       } else NodeSeq.Empty }
@@ -289,6 +289,7 @@ class Template(universe: doc.Universe, generator: DiagramGenerator, tpl: DocTemp
       fullComment={ if(memberComment.filter(_.label=="div").isEmpty) "no" else "yes" }
       group={ mbr.group }>
       <a id={ mbr.signature }/>
+      <a id={ mbr.signatureCompat }/>
       { signature(mbr, false) }
       { memberComment }
     </li>
@@ -645,17 +646,28 @@ class Template(universe: doc.Universe, generator: DiagramGenerator, tpl: DocTemp
       case _ => NodeSeq.Empty
     }
 
-    val typeHierarchy = if (s.docDiagrams.isSetByUser) mbr match {
-      case dtpl: DocTemplateEntity if isSelf && !isReduced =>
-        makeDiagramHtml(dtpl, dtpl.inheritanceDiagram, "Type Hierarchy", "inheritance-diagram")
-      case _ => NodeSeq.Empty
-    } else NodeSeq.Empty // diagrams not generated
+    def createDiagram(f: DocTemplateEntity => Option[Diagram], description: String, id: String): NodeSeq =
+      if (s.docDiagrams.value) mbr match {
+        case dtpl: DocTemplateEntity if isSelf && !isReduced =>
+          val diagram = f(dtpl)
+          if (diagram.isDefined) {
+            val s = universe.settings
+            val diagramSvg = generator.generate(diagram.get, tpl, this)
+            if (diagramSvg != NodeSeq.Empty) {
+              <div class="toggleContainer block diagram-container" id={ id + "-container"}>
+                <span class="toggle diagram-link">{ description }</span>
+                <a href="http://docs.scala-lang.org/overviews/scaladoc/usage.html#diagrams" target="_blank" class="diagram-help">Learn more about scaladoc diagrams</a>
+                <div class="diagram" id={ id }>{
+                  diagramSvg
+                }</div>
+              </div>
+            } else NodeSeq.Empty
+          } else NodeSeq.Empty
+        case _ => NodeSeq.Empty
+      } else NodeSeq.Empty // diagrams not generated
 
-    val contentHierarchy = if (s.docDiagrams.isSetByUser) mbr match {
-      case dtpl: DocTemplateEntity if isSelf && !isReduced =>
-        makeDiagramHtml(dtpl, dtpl.contentDiagram, "Content Hierarchy", "content-diagram")
-      case _ => NodeSeq.Empty
-    } else NodeSeq.Empty // diagrams not generated
+    val typeHierarchy = createDiagram(_.inheritanceDiagram, "Type Hierarchy", "inheritance-diagram")
+    val contentHierarchy = createDiagram(_.contentDiagram, "Content Hierarchy", "content-diagram")
 
     memberComment ++ paramComments ++ attributesBlock ++ linearization ++ subclasses ++ typeHierarchy ++ contentHierarchy
   }
@@ -946,22 +958,6 @@ class Template(universe: doc.Universe, generator: DiagramGenerator, tpl: DocTemp
       scala.xml.Text(ub.typeParamName + " is a subclass of " + ub.upperBound.name + " (" + ub.typeParamName + " <: ") ++
         typeToHtml(ub.upperBound, true) ++ scala.xml.Text(")")
   }
-
-  def makeDiagramHtml(tpl: DocTemplateEntity, diagram: Option[Diagram], description: String, id: String) = {
-    if (diagram.isDefined) {
-      val s = universe.settings
-      val diagramSvg = generator.generate(diagram.get, tpl, this)
-      if (diagramSvg != NodeSeq.Empty) {
-        <div class="toggleContainer block diagram-container" id={ id + "-container"}>
-          <span class="toggle diagram-link">{ description }</span>
-          <a href="http://docs.scala-lang.org/overviews/scaladoc/usage.html#diagrams" target="_blank" class="diagram-help">Learn more about scaladoc diagrams</a>
-          <div class="diagram" id={ id }>{
-            diagramSvg
-          }</div>
-        </div>
-      } else NodeSeq.Empty
-    } else NodeSeq.Empty
-  }
 }
 
 object Template {

src/compiler/scala/tools/nsc/doc/model/Entity.scala

@@ -196,6 +196,9 @@ trait MemberEntity extends Entity {
   /** The identity of this member, used for linking */
   def signature: String
 
+  /** Compatibility signature, will be removed from future versions */
+  def signatureCompat: String
+
   /** Indicates whether the member is inherited by implicit conversion */
   def isImplicitlyInherited: Boolean
 
@@ -625,4 +628,4 @@ trait UpperBoundedTypeParamConstraint extends TypeParamConstraint {
   /** toString for debugging */
   override def toString = typeParamName + " is a subclass of " + upperBound.name + " (" + typeParamName + " <: " +
     upperBound.name + ")"
-}
+}

src/compiler/scala/tools/nsc/doc/model/LinkTo.scala

@@ -9,10 +9,10 @@ package model
 import scala.collection._
 
 abstract sealed class LinkTo
-case class LinkToTpl(tpl: DocTemplateEntity) extends LinkTo
-case class LinkToMember(mbr: MemberEntity, inTpl: DocTemplateEntity) extends LinkTo
-case class Tooltip(name: String) extends LinkTo { def this(tpl: TemplateEntity) = this(tpl.qualifiedName) }
-// case class LinkToExternal(name: String, url: String) extends LinkTo // for SI-191, whenever Manohar will have time
+final case class LinkToTpl(tpl: DocTemplateEntity) extends LinkTo
+final case class LinkToMember(mbr: MemberEntity, inTpl: DocTemplateEntity) extends LinkTo
+final case class Tooltip(name: String) extends LinkTo { def this(tpl: TemplateEntity) = this(tpl.qualifiedName) }
+final case class LinkToExternal(name: String, url: String) extends LinkTo
 case object NoLink extends LinkTo // you should use Tooltip if you have a name from the user, this is only in case all fails
 
 object LinkToTpl {

src/compiler/scala/tools/nsc/doc/model/MemberLookup.scala

@@ -11,6 +11,7 @@ trait MemberLookup {
   thisFactory: ModelFactory =>
 
   import global._
+  import rootMirror.RootPackage, rootMirror.EmptyPackage
 
   def makeEntityLink(title: Inline, pos: Position, query: String, inTplOpt: Option[DocTemplateImpl]) =
     new EntityLink(title) { lazy val link = memberLookup(pos, query, inTplOpt) }
@@ -21,23 +22,44 @@ trait MemberLookup {
     var members = breakMembers(query)
     //println(query + " => " + members)
 
-    // (1) Lookup in the root package, as most of the links are qualified
-    var linkTo: List[LinkTo] = lookupInRootPackage(pos, members)
+    // (1) First look in the root package, as most of the links are qualified
+    val fromRoot = lookupInRootPackage(pos, members)
 
-    // (2) Recursively go into each
-    if (inTplOpt.isDefined) {
-      var currentTpl = inTplOpt.get
-      while (currentTpl != null && !currentTpl.isRootPackage && (linkTo.isEmpty)) {
-        linkTo = lookupInTemplate(pos, members, currentTpl)
-        currentTpl = currentTpl.inTemplate
-      }
+    // (2) Or recursively go into each containing template.
+    val fromParents = inTplOpt.fold(Stream.empty[DocTemplateImpl]) { tpl =>
+      Stream.iterate(tpl)(_.inTemplate)
+    }.takeWhile (tpl => tpl != null && !tpl.isRootPackage).map { tpl =>
+      lookupInTemplate(pos, members, tpl.asInstanceOf[EntityImpl].sym)
     }
 
-    // (3) Look at external links
-    if (linkTo.isEmpty) {
-      // TODO: IF THIS IS THE ROOT PACKAGE, LOOK AT EXTERNAL LINKS
+    val syms = (fromRoot +: fromParents) find (!_.isEmpty) getOrElse Nil
+    val linkTo = createLinks(syms) match {
+      case Nil if !syms.isEmpty =>
+        // (3) Look at external links
+        syms.flatMap { case (sym, owner) =>
+
+          // reconstruct the original link
+          def linkName(sym: Symbol) = {
+            def isRoot(s: Symbol) = s.isRootSymbol || s.isEmptyPackage || s.isEmptyPackageClass
+            def nameString(s: Symbol) = s.nameString + (if ((s.isModule || s.isModuleClass) && !s.isPackage) "$" else "")
+            val packageSuffix = if (sym.isPackage) ".package" else ""
+
+            sym.ownerChain.reverse.filterNot(isRoot(_)).map(nameString(_)).mkString(".") + packageSuffix
+          }
+
+          if (sym.isClass || sym.isModule || sym.isTrait || sym.isPackage)
+            findExternalLink(linkName(sym))
+          else if (owner.isClass || owner.isModule || owner.isTrait || owner.isPackage)
+            findExternalLink(linkName(owner) + "@" + externalSignature(sym))
+          else
+            None
+        }
+      case links => links
     }
 
+    //println(createLinks(syms))
+    //println(linkTo)
+
     // (4) if we still haven't found anything, create a tooltip, if we found too many, report
     if (linkTo.isEmpty){
       if (!settings.docNoLinkWarnings.value)
@@ -97,9 +119,23 @@ trait MemberLookup {
   private object OnlyType extends SearchStrategy
   private object OnlyTerm extends SearchStrategy
 
-  private def lookupInRootPackage(pos: Position, members: List[String]) = lookupInTemplate(pos, members, makeRootPackage)
+  private def lookupInRootPackage(pos: Position, members: List[String]) =
+    if (members.length == 1)
+      lookupInTemplate(pos, members, EmptyPackage) ::: lookupInTemplate(pos, members, RootPackage)
+    else
+      lookupInTemplate(pos, members, RootPackage)
 
-  private def lookupInTemplate(pos: Position, members: List[String], inTpl: DocTemplateImpl): List[LinkTo] = {
+  private def createLinks(syms: List[(Symbol, Symbol)]): List[LinkTo] =
+    syms.flatMap { case (sym, owner) =>
+      if (sym.isClass || sym.isModule || sym.isTrait || sym.isPackage)
+        findTemplateMaybe(sym) map (LinkToTpl(_))
+      else
+        findTemplateMaybe(owner) flatMap { inTpl =>
+          inTpl.members find (_.asInstanceOf[EntityImpl].sym == sym) map (LinkToMember(_, inTpl))
+        }
+    }
+
+  private def lookupInTemplate(pos: Position, members: List[String], container: Symbol): List[(Symbol, Symbol)] = {
     // Maintaining compatibility with previous links is a bit tricky here:
     // we have a preference for term names for all terms except for the last, where we prefer a class:
     // How to do this:
@@ -108,53 +144,56 @@ trait MemberLookup {
     //     * we look for terms with the last member's name
     //     * we look for types with the same name, all the way up
     val result = members match {
-      case Nil =>
-        Nil
+      case Nil => Nil
       case mbrName::Nil =>
-        var members = lookupInTemplate(pos, mbrName, inTpl, OnlyType)
-        if (members.isEmpty)
-          members = lookupInTemplate(pos, mbrName, inTpl, OnlyTerm)
-
-        members.map(_ match {
-          case tpl: DocTemplateEntity => LinkToTpl(tpl)
-          case mbr => LinkToMember(mbr, inTpl)
-        })
+        var syms = lookupInTemplate(pos, mbrName, container, OnlyType) map ((_, container))
+        if (syms.isEmpty)
+          syms = lookupInTemplate(pos, mbrName, container, OnlyTerm) map ((_, container))
+        syms
 
       case tplName::rest =>
+        def completeSearch(syms: List[Symbol]) =
+          syms filter {sym => sym.isPackage || sym.isClass || sym.isModule} flatMap (lookupInTemplate(pos, rest, _))
 
-        def completeSearch(mbrs: List[MemberImpl]) =
-          mbrs.collect({case d:DocTemplateImpl => d}).flatMap(tpl => lookupInTemplate(pos, rest, tpl))
-
-        var members = completeSearch(lookupInTemplate(pos, tplName, inTpl, OnlyTerm))
-        if (members.isEmpty)
-          members = completeSearch(lookupInTemplate(pos, tplName, inTpl, OnlyType))
-
-        members
+        completeSearch(lookupInTemplate(pos, tplName, container, OnlyTerm)) match {
+          case Nil => completeSearch(lookupInTemplate(pos, tplName, container, OnlyType))
+          case syms => syms
+      }
     }
-    //println("lookupInTemplate(" + members + ", " + inTpl + ") => " + result)
+    //println("lookupInTemplate(" + members + ", " + container + ") => " + result)
     result
   }
 
-  private def lookupInTemplate(pos: Position, member: String, inTpl: DocTemplateImpl, strategy: SearchStrategy): List[MemberImpl] = {
+  private def lookupInTemplate(pos: Position, member: String, container: Symbol, strategy: SearchStrategy): List[Symbol] = {
     val name = member.stripSuffix("$").stripSuffix("!").stripSuffix("*")
+    def signatureMatch(sym: Symbol): Boolean = externalSignature(sym).startsWith(name)
+
+    // We need to cleanup the bogus classes created by the .class file parser. For example, [[scala.Predef]] resolves
+    // to (bogus) class scala.Predef loaded by the class loader -- which we need to eliminate by looking at the info
+    // and removing NoType classes
+    def cleanupBogusClasses(syms: List[Symbol]) = { syms.filter(_.info != NoType) }
+
+    def syms(name: Name) = container.info.nonPrivateMember(name).alternatives
+    def termSyms = cleanupBogusClasses(syms(newTermName(name)))
+    def typeSyms = cleanupBogusClasses(syms(newTypeName(name)))
+
     val result = if (member.endsWith("$"))
-      inTpl.members.filter(mbr => (mbr.name == name) && (mbr.isTerm))
+      termSyms
     else if (member.endsWith("!"))
-      inTpl.members.filter(mbr => (mbr.name == name) && (mbr.isType))
+      typeSyms
     else if (member.endsWith("*"))
-      inTpl.members.filter(mbr => (mbr.signature.startsWith(name)))
-    else {
+      cleanupBogusClasses(container.info.nonPrivateDecls) filter signatureMatch
+    else
       if (strategy == BothTypeAndTerm)
-        inTpl.members.filter(_.name == name)
+        termSyms ::: typeSyms
       else if (strategy == OnlyType)
-        inTpl.members.filter(mbr => (mbr.name == name) && (mbr.isType))
+        typeSyms
       else if (strategy == OnlyTerm)
-        inTpl.members.filter(mbr => (mbr.name == name) && (mbr.isTerm))
+        termSyms
       else
         Nil
-    }
 
-    //println("lookupInTemplate(" + member + ", " + inTpl + ") => " + result)
+    //println("lookupInTemplate(" + member + ", " + container + ") => " + result)
     result
   }
 
@@ -170,7 +209,11 @@ trait MemberLookup {
       if ((query.charAt(index) == '.' || query.charAt(index) == '#') &&
           ((index == 0) || (query.charAt(index-1) != '\\'))) {
 
-        members ::= query.substring(last_index, index).replaceAll("\\\\([#\\.])", "$1")
+        val member = query.substring(last_index, index).replaceAll("\\\\([#\\.])", "$1")
+        // we want to allow javadoc-style links [[#member]] -- which requires us to remove empty members from the first
+        // elemnt in the list
+        if ((member != "") || (!members.isEmpty))
+          members ::= member
         last_index = index + 1
       }
       index += 1
@@ -184,4 +227,4 @@ trait MemberLookup {
 object MemberLookup {
   private[this] var _showExplanation = true
   def showExplanation: Boolean = if (_showExplanation) { _showExplanation = false; true } else false
-}
+}

src/compiler/scala/tools/nsc/doc/model/ModelFactory.scala

@@ -209,7 +209,8 @@ class ModelFactory(val global: Global, val settings: doc.Settings) {
       ((!sym.isTrait && ((sym hasFlag Flags.ABSTRACT) || (sym hasFlag Flags.DEFERRED)) && (!isImplicitlyInherited)) ||
       sym.isAbstractClass || sym.isAbstractType) && !sym.isSynthetic
     def isTemplate = false
-    lazy val signature = {
+    def signature = externalSignature(sym)
+    lazy val signatureCompat = {
 
       def defParams(mbr: Any): String = mbr match {
         case d: MemberEntity with Def =>
@@ -1082,5 +1083,17 @@ class ModelFactory(val global: Global, val settings: doc.Settings) {
     (settings.docExpandAllTypes.value && (bSym.sourceFile != null)) ||
     { val rawComment = global.expandedDocComment(bSym, inTpl.sym)
       rawComment.contains("@template") || rawComment.contains("@documentable") }
+
+  def findExternalLink(name: String): Option[LinkTo] =
+    settings.extUrlMapping find {
+      case (pkg, _) => name startsWith pkg
+    } map {
+      case (_, url) => LinkToExternal(name, url + "#" + name)
+    }
+
+  def externalSignature(sym: Symbol) = {
+    sym.info // force it, otherwise we see lazy types
+    (sym.nameString + sym.signatureString).replaceAll("\\s", "")
+  }
 }