.10.dp.source-code.url.scala Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of core_2.10 Show documentation
Core drx utilities that have no other dependencies other than the core scala lib
The newest version!
/*
   Copyright 2010 Aaron J. Radke

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
package cc.drx

import scala.collection.JavaConverters._
import scala.concurrent.blocking

object URL{
  implicit object ParsableURL extends Parsable[URL]{ def apply(v:String):URL = URL(v) }
  def apply(url:String,defaultProtocol:Option[String]=Some("https")):URL = {
    val autoURL = defaultProtocol map {proto => if(url contains "://") url else proto+"://"+url} getOrElse url
    new URL(new java.net.URL(autoURL))
  }
  val Href = Pluck("""href\s*="(.+?)"""".r)

  final class URLStringContext(val sc:StringContext) extends AnyVal{
    def url(args:Any*):URL = URL(sc.s(args:_*))
  }

  //old java style class construct to make it work is ugly TODO wrap this as a function which returns a Future of the type
  //
  //

  class Connection(url:URL, timeout:Time=10.s,userpass:Option[(String,String)]=None) extends StringMap{
    private val c = url.url.openConnection // java.net.URLConnection

    //--if password is set
    userpass.foreach{case (user,pass) => 
      c.setRequestProperty ("Authorization", 
        "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary(s"$user:$pass".getBytes)
      )
    }

    c.setConnectTimeout(timeout.ms.toInt)
    c.setReadTimeout(timeout.ms.toInt)
    c.connect() //if not already connected
    // private val c = url.openConnection //
    def in:Input = new Input(c.getInputStream)

    private def safe(k:String) = Option(k).getOrElse("")
    private lazy val header:Map[String,List[String]] = c.getHeaderFields.asScala.map{
      case (k,vs) => safe(k) -> vs.asScala.toList
    }.toMap

    def getString(key:String):Option[String] = normHeader get key.toLowerCase
    lazy val keys = normHeader.keys.toList

    private lazy val normHeader = header.map{case (k,v) => k.toLowerCase -> v.mkString(", ")}
    lazy val content = in.toString

    def niceHeader:String = header.map{case (k,v) => (k+":").fit(30) + v.mkString(", ")}.toList.sorted mkString "\n" 

    //Date TODO add date parser type from the http url date format
    //https://tools.ietf.org/html/rfc2616#page-20
    //http://stackoverflow.com/a/8642463/622016
    //DateTimeFormatter.RFC_1123_DATE_TIME.
    def modified:Date = Date(c.getLastModified)
    def date:Date = Date(c.getDate)
    def expiration:Date = Date(c.getExpiration)
	 def encoding:String = c.getContentEncoding
    private val RelLink = Pluck("""<([^>]+)>\s*;\s*rel\s*="([^"]+)"""".r, 1, 2)
    lazy val relLinks:Map[String,URL] = {
      for(link <- getString("link").toList; (a,b) <- link pluckAll RelLink) yield b -> URL(a)
    }.toMap

    def get(file:File):File = {
      // val localFile = if(local.isDir && url.file.isDefined) local/url.file.get else local
      in copy file.out //side effect
      file
    }
  }

}
//TODO there feels like there should be a super type of URL and File maybe something like Source??
/**TODO similar wrapper kind as File should generate Input and Output*/
class URL(val url:java.net.URL) extends AnyVal{
  //--inter-opt
  def toJava:java.net.URL = url
  override def toString = url.toString

  //--construction
  def /(that:File):URL = this / that.path
  def /(filename:String):URL = copy(path = Some(path.getOrElse("") + "/" + filename))
  def /(filename:Symbol):URL = this / filename.name
  def ++(optParam:Option[(Any,Any)]):URL = this ++ optParam.toList
  def ++(moreParams:Iterable[(Any,Any)]):URL = {
    val ps = params ++ moreParams.map{case (a,b) => (anyToParam(a), anyToParam(b))}
    if(ps.isEmpty) this
    else copy(query = Some(ps.map{case (a,b) => a+"="+b}.mkString("&")))
  }

  //--support
  private def anyToParam(x:Any):String = x match {
    case Symbol(v) => v
    case Some(v:Any) => v.toString
    case v:Iterable[Any] => v map anyToParam mkString ","
    case _ => x.toString
  }

  def connect(implicit ec:ExecutionContext) = Future(blocking{new URL.Connection(this)}) //TODO shift normal get to use this mechanism
  def connectBasicAuth(user:String,pass:String)(implicit ec:ExecutionContext) = Future(blocking{
    new URL.Connection(this,10.s, Some(user->pass) )
  }) //TODO shift normal get to use this mechanism

  //--simple rest
  // def in:Input = new Input(url.openStream) //TODO
  def get(file:File)(implicit ec:ExecutionContext):Future[File] = connect map {_ get file}
  def get(implicit ec:ExecutionContext):Future[String] = connect.map{_.in.toString}
  def getBasicAuth(user:String,pass:String)(implicit ec:ExecutionContext):Future[String] = connect.map{_.in.toString}

  // @deprecated("stop using the blocks","v0.2.15")
  // def get:String = connect(Implicit.ec).map{_.in.toString}.block(10.s) //forced bloc

  // def get(implicit ec:ExecutionContext):Future[String] = {import Implicit.ec; connect map {_.in.toString} block 20.s getOrElse ""}
  // def get:String = scala.io.Source.fromURL(url).mkString //TODO make this more robust with timeouts , futures and stream closing and header parsing, content type detection

  def hrefs(implicit ec:ExecutionContext):Future[List[String]] = for(res <- get) yield { res.pluckAll(URL.Href).toList }

  /* recursive with futures is harder....TODO 
  def hrefs(depth:Int)(implicit ec:ExecutionContext):Future[List[String]] = if(depth < 1 ) hrefs(ec) else 
    for(as <- hrefs; a <- as; bs <- (this / a).hrefs(depth-1); b <- bs) yield b
    hrefs.map{as => 
      as ++ {}
    }
    ; a <- as; bs <- (this / a).hrefs(depth-1); b <- bs) yield b
    // hrefs.map{as =>  as flatMap {a => (this / a) hrefs (depth-1)} }
  */

  //--required params
  def protocol:String = url.getProtocol
  def host:String = url.getHost
  //--optional params
  def user:Option[String] = Option(url.getUserInfo)
  def query:Option[String] = Option(url.getQuery)
  def port:Option[Int] = url.getPort match {case -1 => None; case p => Option(p)}
  def file:Option[File] = path map File.apply
  def path:Option[String] = url.getPath.toOption //getPath returns empty string "" so DrxString toOption sets it to None
  def ref:Option[String] = Option(url.getRef)

  //--convenience
  def params:List[(String,String)] = {
    // for(q:String <- query.toList:List[String]; p:String <- q split '&'; t:(String,String) <- (p split '=').toTuple2.toOption) yield t
    query.map{q => 
      q.split('&').toList.map{p => (p split '=').toTuple2 getOrElse ("parse"->"error")}
    } getOrElse List()
  }

  //--copy portions
  def copy(
      protocol:String=protocol,
      user:Option[String]=user,
      host:String=host,
      port:Option[Int]=port,
      path:Option[String]=path,
      query:Option[String]=query
      //TODO add ref and authority
    ):URL = URL(protocol + "://" +++ user.map{_+"@"} + host +++ port.map{":"+_} +++ path +++ query.map{"?"+_})
}