.13.e3.source-code.url.scala Maven / Gradle / Ivy
/*
Copyright 2010 Aaron J. Radke
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cc.drx
import scala.collection.JavaConverters._
import scala.concurrent.blocking
object Token{
def find(key:String):Option[Token] = {
val fullKey = key.toUpperCase +"_TOKEN"
OS.Env.get(fullKey) map Token.apply
}
def github = find("github")
}
case class Token(value:String)
object Mime{
final class MimeStringContext(val sc:StringContext) extends AnyVal{
def mime(args:Any*):Mime = Mime(sc.s(args:_*))
}
private lazy val fromExt:Map[String,Mime] = {
for( (mime,exts) <- fromMime; ext <- exts) yield ext -> mime
}.toMap
private lazy val fromMime:Map[Mime,List[String]] = {for (
in <- Input.resourceOption(file"mime.types").toList; //Option is safe if mime.types is not available
line <- in.lines; //assumes mime.types exists
args = line.trim split "\\s+";
if args.size > 1;
mime = new Mime(args.head); //the only way to make new Mime is using the new operator
ext <- args //include the map to itself if it already exists
) yield mime -> args.toList}.toMap
val default:Mime = new Mime("application/octet-stream")
def extsOf(mime:Mime):List[String] = fromMime.get(mime) getOrElse Nil
def get(ext:String):Option[Mime] = fromExt.get(ext)
def apply(ext:String):Mime = get(ext) getOrElse default //least specific is default if not found
}
class Mime(val name:String){
override def toString = s"Mime($name)"
def exts:List[String] = Mime.extsOf(this)
def typ:String = name.split('/').head //TODO make this more robust
def subtyp:String = name.split('/').last //TODO make this more robust
}
object URL{
implicit object ParsableURL extends Parsable[URL]{ def apply(v:String):URL = URL(v) }
def apply(url:String):URL = apply(url, Some("https"))
def apply(url:String,defaultProtocol:Option[String]):URL = {
val autoURL = defaultProtocol map {proto => if(url contains "://") url else proto+"://"+url} getOrElse url
new URL(new java.net.URL(autoURL))
}
val Href = Pluck("""href\s*="(.+?)"""".r)
final class URLStringContext(val sc:StringContext) extends AnyVal{
def url(args:Any*):URL = URL(sc.s(args:_*))
def http(args:Any*):URL = URL(sc.s(args:_*)).copy(protocol="http")
def https(args:Any*):URL = URL(sc.s(args:_*)).copy(protocol="https")
}
//old java style class construct to make it work is ugly TODO wrap this as a function which returns a Future of the type
trait Connectable[A]{ def apply(c:Connection):A }
object Connectable{
implicit case object ConnectableString extends Connectable[String]{
def apply(c:Connection):String = c.in.asString
}
implicit case object ConnectableJson extends Connectable[Json]{
def apply(c:Connection):Json = Json(c.in)
}
implicit case object ConnectableFile extends Connectable[File]{
def apply(c:Connection):File = {
val f = File(c.url.file.map{_.name} getOrElse "un-named")
c.in copyTo f.out
f
}
}
}
// Such an excelent guiding SO examples at https://stackoverflow.com/a/2793153/622016
class Connection(val url:URL, val timeout:Time=20.s) extends StringMap{
private val c = url.url.openConnection // get the java.net.URLConnection
//--GET set request properties BEFORE connect
if(url.postData.isEmpty) url.props.foreach{case (k,v) => c.setRequestProperty(k,v) }
c.setConnectTimeout(timeout.ms.toInt)
c.setReadTimeout(timeout.ms.toInt)
// c.setChunkedStreamingMode(7000) //on http
c.setDoInput(true) //true is default
//--set request properties AFTER connect https://stackoverflow.com/a/4206094/622016
for(postInput <- url.postData){
// c setRequestMethod "POST"
// c setInstanceFollowRedirects false
c setDoOutput true
c setUseCaches false
val postProps:Map[String,String] = if(url.props contains "Content-Type") url.props else
url.props + ("Content-Type" -> "application/x-www-form-urlencoded; charset=utf-8")
//TODO set content length if from a known file length
// c.setRequestProperty( "Content-Length", Integer.toString( postDataLength ))
// c.setRequestProperty( "Accept-Charset", "UTF-8") //TODO cleverly set acceptable charset
postProps.foreach{case (k,v) => c.setRequestProperty(k,v) }
val out = new Output(c.getOutputStream)
postInput copyTo out
}
//c.connect() //if not already connected (Note: this breaks a POST if it is called before the post data
// //writing to c.getOutputStream and reading from c.getInputStream does the actual connect call
//
val in:Input = new Input(c.getInputStream) //this is val always kick off reading (and not allow new ins)
lazy val content = in.asString //TODO think if this should be kept around//if used it breaks the inputStream
def responseCode:Option[Int] =
for(resp <- header.get(""); v <- resp.headOption;
i <- Try{(v.trim split " " apply 1).toInt}.toOption) yield i
private def safe(k:String) = Option(k).getOrElse("")
private lazy val header:Map[String,List[String]] = Java.toScala(c.getHeaderFields).map{
case (k,vs) => safe(k) -> Java.toScala(vs).toList
}.toMap
def getString(key:String):Option[String] = normHeader get key.toLowerCase
lazy val keys = normHeader.keys.toList
private lazy val normHeader = header.map{case (k,v) => k.toLowerCase -> v.mkString(", ")}
def niceHeader:String = header.map{case (k,v) => (k+":").fit(30) + v.mkString(", ")}.toList.sorted mkString "\n"
//Date TODO add date parser type from the http url date format
//https://tools.ietf.org/html/rfc2616#page-20
//http://stackoverflow.com/a/8642463/622016
//DateTimeFormatter.RFC_1123_DATE_TIME.
def modified:Date = Date(c.getLastModified)
def date:Date = Date(c.getDate)
def expiration:Date = Date(c.getExpiration)
def encoding:String = c.getContentEncoding
//--common scrape/plucks
private val RelLink = Pluck("""<([^>]+)>\s*;\s*rel\s*="([^"]+)"""".r, 1, 2)
lazy val relLinks:Map[String,URL] = {
for(link <- getString("link").toList; (a,b) <- link pluckAll RelLink) yield b -> URL(a)
}.toMap
def copyTo(file:File):File = {
// val localFile = if(local.isDir && url.file.isDefined) local/url.file.get else local
in copyTo file.out //side effect
file
}
lazy val hrefs:List[URL] = content.pluckAll(URL.Href).toList map {url href _}
}
}
//TODO there feels like there should be a super type of URL and File maybe something like Source??
/**TODO similar wrapper kind as File should generate Input and Output*/
class URL(val url:java.net.URL, val props:Map[String,String]=Map.empty, val postData:Option[Input]=None){
//--inter-opt
def toJava:java.net.URL = url //note: loses request header props
override def toString = {
val header = if(props.nonEmpty) props.map{case (k,v) => s"$k:$v"}.mkString( "{", ", ", "}" ) else ""
val post = if(postData.nonEmpty) " (with Post data)" else ""
url.toString + header + post
}
def ==(that:URL):Boolean = this.toString == that.toString
// def hashCode:Int = ???
//--construction
def /(that:File):URL = this / that.path
def /(filename:String):URL = {
val basePath = path.getOrElse("")
val sep = if(basePath endsWith "/") "" else "/"
copy(path = Some(basePath + sep + filename))
}
def /(filename:Symbol):URL = this / filename.name
def *(param:(Any,Any)):URL = this ** List(param)
def **(optParam:Option[(Any,Any)]):URL = this ** optParam.toList
def **(moreParams:Iterable[(Any,Any)]):URL = {
val newParams = moreParams.map{case (a,b) => (anyToParam(a), anyToParam(b))}
def isNewKey(k:String) = newParams.exists{_._1 == k}
val oldParams = params.filter{case (k,v) => !isNewKey(k)} //remove any params that exist in the new list
val ps = oldParams ++ newParams
if(ps.isEmpty) this
else copy(query = Some(ps.map{case (a,b) => a+"="+b}.mkString("&")))
}
/**expand a full url from ahref from a url*/
def href(ref:String):URL = {
if(ref contains "://") URL(ref)
else if(ref startsWith "/") copy(path = Some(ref))
else if(ref startsWith "#") copy(path = Some(path.getOrElse("") + ref))
else this / ref
}
def addHeader(kvs:Tuple2[String,String]*) = copy(props = props ++ kvs)
/**quickly add tuple headers with an operator instead of [[addHeader]] */
def ^(kv:Tuple2[String,String]) = copy(props = props + kv)
//--composition operator types (do the right thing)
def ~(token:Token):URL = copy(props = props + ("Authorization" -> s"token ${token.value}") ) //github style token auth
//TODO add other auth keys like: "Authorization: Bearer "
def ~(mime:Mime):URL = copy(props = props + ("Content-Type" -> mime.name))
def ~(param:Tuple2[Any,Any]):URL = this ** List(param)
def ~(postData:Input):URL = copy(postData = Some(postData))
def ~(postFile:File):URL = this ~ postFile.in ~ postFile.mime
//--support
private def anyToParam(x:Any):String = x match {
case Symbol(v) => v
case Some(v:Any) => v.toString
case v:Iterable[Any] => v map anyToParam mkString ","
case _ => x.toString
}
// def connect(implicit ec:ExecutionContext):Future[URL.Connection] = Future(blocking{get.start})
//--simple rest
// def fetch(file:File)(implicit ec:ExecutionContext):Future[File] = connect map {_ copyTo file}
// def fetch(implicit ec:ExecutionContext):Future[String] = connect.map{_.in.asString}
// @deprecated("use fetch instead since get has meaning in scala","dq")
// def get(file:File)(implicit ec:ExecutionContext):Future[File] = connect map {_ copyTo file}
// @deprecated("use fetch instead since get has meaning in scala","dq")
// def get(implicit ec:ExecutionContext):Future[String] = connect.map{_.in.toString}
import Implicit.ec //used as a default to make things super simple
/**Ultra simple BUT blocking and many string assumptions*/
// def get:String = connect.map{_.in.asString}.block(20.s) getOrElse ""
// def get[A](implicit ec:ExecutionContext, cb:cc.drx.URL.Connectable[A]):Future[A] = connect map {cb.apply}
//
def async[A](f:URL.Connection => A)(implicit ec:ExecutionContext):Future[A] = Future(blocking(f(get)))
def async[A](implicit cb:cc.drx.URL.Connectable[A],ec:ExecutionContext):Future[A] = Future(blocking(cb(get)))
// def async[A](f:URL.Connection => A):Future[A] = Future(blocking{ f(this.get) })
def get:URL.Connection = new URL.Connection(this)
def in:Input = get.in
///*just use url.in copyTo outputFile **/
// def copyTo(file:File):Future[File] = connect map {_ copyTo file}
def as[A](implicit cb:cc.drx.URL.Connectable[A]):A = cb apply get
// def asFuture[A](implicit ec:ExecutionContext, cb:cc.drx.URL.Connectable[A]):A = connect map cb.apply
/**Ultra simple BUT blocking and many many assumptions (strings return...)*/
// def post(file:File):String = (this ~ file.mime).connect(file.in).map{_.in.toString}.block(20.s) getOrElse ""
// def post:String = ??? //TODO implement the form query encoded params
// @deprecated("stop using the blocks","v0.2.15")
// def get:String = connect(Implicit.ec).map{_.in.toString}.block(10.s) //forced bloc
// def get(implicit ec:ExecutionContext):Future[String] = {import Implicit.ec; connect map {_.in.toString} block 20.s getOrElse ""}
// def get:String = scala.io.Source.fromURL(url).mkString //TODO make this more robust with timeouts , futures and stream closing and header parsing, content type detection
// def hrefs(implicit ec:ExecutionContext):Future[List[String]] = for(res <- connect map {_.in.asString}) yield { res.pluckAll(URL.Href).toList }
/* recursive with futures is harder....TODO
def hrefs(depth:Int)(implicit ec:ExecutionContext):Future[List[String]] = if(depth < 1 ) hrefs(ec) else
for(as <- hrefs; a <- as; bs <- (this / a).hrefs(depth-1); b <- bs) yield b
hrefs.map{as =>
as ++ {}
}
; a <- as; bs <- (this / a).hrefs(depth-1); b <- bs) yield b
// hrefs.map{as => as flatMap {a => (this / a) hrefs (depth-1)} }
*/
//--required params
def protocol:String = url.getProtocol
def host:String = url.getHost
//--optional params
def user:Option[String] = Option(url.getUserInfo)
def query:Option[String] = Option(url.getQuery)
def port:Option[Int] = url.getPort match {case -1 => None; case p => Option(p)}
def file:Option[File] = path map File.apply
def path:Option[String] = url.getPath.toOption //getPath returns empty string "" so DrxString toOption sets it to None
def ref:Option[String] = Option(url.getRef)
//--convenience
lazy val params:List[(String,String)] = {
// for(q:String <- query.toList:List[String]; p:String <- q split '&'; t:(String,String) <- (p split '=').toTuple2.toOption) yield t
//TODO use build in urlencoder/decoder parser instead
query.map{q =>
q.split('&').toList.map{p => (p split '=').toTuple2 getOrElse ("parse"->"error")}
} getOrElse List()
}
private lazy val paramsMap = params.toMap
def getParam(key:String):Option[String] = paramsMap.get(key)
//--copy portions
def copy(
protocol:String=protocol,
user:Option[String]=user,
host:String=host,
port:Option[Int]=port,
path:Option[String]=path,
query:Option[String]=query,
props:Map[String,String]=props,
postData:Option[Input]=postData
):URL = {
val urlString:String = protocol + "://" +++ user.map{_+"@"} + host +++ port.map{":"+_} +++ path +++ query.map{"?"+_}
new URL(new java.net.URL(urlString),props,postData)
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy