.10.dp.source-code.repo.scala Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of core_2.10 Show documentation
Show all versions of core_2.10 Show documentation
Core drx utilities that have no other dependencies other than the core scala lib
The newest version!
/*
Copyright 2010 Aaron J. Radke
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cc.drx
//TODO remove extra toList functions
//TODO add auto detect of file type urls and use file.list instead of webpage grok
//TODO add filter of artifact types
//TODO generalize organization and artifacts for other layouts like ivy..
/** (Repo) can find (Org -> Project -> Release -> Artifact)
* Repo have two major input types:
* 1) user reference using org/project/release
* 2) dir listing of an artifact path (whose parents or filename can determine the org/project/release
*/
object Repo{
/** Org -> Project -> Release -> Artifact */
case class Artifact(release:Release, ext:String, alt:Option[String]=None){
def *(newAlt:String):Artifact = copy(alt=Some(newAlt))
def alt(newAlt:String):Artifact = copy(alt=Some(newAlt))
//TODO def get:Try[File] = ??? is the file already cached?? else download
def path(implicit style:Style):File = style.path(this)
def baseKind:String = alt match {
case Some("javadoc") => "doc"
case Some("sources") => "src"
case _ => ext.split(".").head
}
}
private def clean(n:String):String = {
val a = if(n endsWith "/") n dropRight 1 else n
val b = if(a startsWith ":") a drop 1 else a
return b
}
/** Org -> Project -> Release */
object Release{
def apply(org:String, prj:String, ver:String):Release = Org(org)/prj/ver
}
case class Release(project:Project, tag:Version){
private def baseAndAlt = tag.name.span(_ != '-')
def base:String = Version(baseAndAlt._1).name
def alt:Option[String] = {val a = baseAndAlt._1; if(a == "") None else Some(a)}
def *(altNew:String) = Release(project, Version(base +++ alt.map{"-"+_}))
private val ArtifactAltPat = """\-([^.]+)\.(.+)""".r
private val ArtifactPat = """\.(.+)""".r
def /(ext:String):Artifact = { //TODO use the style decomposers instead of these custom tags
val inits = List(
project.name+"-"+tag.name,
project.name +++ project.alt.map("_"+_)
)
inits find {ext startsWith _} map { init =>
val rest = ext.drop(init.size)
rest match {
case ArtifactAltPat(alt,ext) => Artifact(this,ext,Some(alt))
case ArtifactPat(ext) => Artifact(this,ext)
case _ => Artifact(this,"",Some(rest))
}
} getOrElse Artifact(this,ext)
}
def list(implicit repos:Repos) = listFirst(repos){_ artifacts this}
def path(implicit style:Style):File = style.path(this)
def nice = project.org.name .fit(40,cc.drx.Style.Left) + "# " +
project.name .fit(20,cc.drx.Style.Left) + ": " +
tag.name
}
/** Org -> Project */
case class Project(org:Org, name:String, alt:Option[String]=None){
def *(altNew:String) = copy(alt=Some(altNew))
def /(tag:String) = Release(this, Version(clean(tag)))
def list(implicit repos:Repos):List[Release] = listFirst(repos){_ releases this}.toList sortBy (_.tag)
def latest(implicit repos:Repos):Option[Release] = list(repos).lastOption
def stable(implicit repos:Repos):Option[Release] = list(repos).filter(_.tag.isFinal).lastOption
def path(implicit style:Style):File = style.path(this)
def stats(implicit repos:Repos):Map[String,Int] = repos.headOption.map{_.source.stats(this)} getOrElse Map()
}
private val ProjectAltPat = """([^_]+)_(.+)""".r
/** Org */
object Org{ def apply(org:String):Org = Org(org.trim.split("""[./]+""").toList) }
case class Org(domains:List[String]){
def name = domains mkString "."
def /(prj:String) = clean(prj) match {
case ProjectAltPat(p,alt) => Project(this, p, Some(alt))
case p => Project(this, p, None)
}
def list(implicit repos:Repos) = listFirst(repos){_.projects(this).toList} //TODO add find first rather than find all
def path(implicit style:Style):File = style.path(this)
}
//implementation functions
private def hrefs(url:URL,depth:Int=0):List[String] = {
import Implicit.ec
url.hrefs.map{_ filterNot blackList}.block(60.s).getOrElse(Nil) //TODO note depth is only zero here (no recursive hrefs)
}
private def files(file:File,depth:Int=0):List[String] = {
val names = if(depth == 0) file.list
else file.walk(maxDepth=depth) filter (_.isFile)
names.toList map {_.name} filterNot blackList
}
private val blackStarts = ". # maven-metadata readme" split " "
private def blackList(name:String):Boolean = blackStarts exists (name.toLowerCase startsWith _)
//-----Styles--------------
object Style{
implicit val defaultStyle:Style = Maven
}
trait Style{
def filename(o:Org):String
def filename(p:Project):String
def filename(r:Release):String
def filename(a:Artifact):String
//TODO make this abstract and implement specifically for each repo style
//This provides a mechanism from a listing to a canonical form
def artifact(f:File):Option[Artifact] = for(pf <- f.parent; r <- release(pf) ) yield r / f.name
def release(f:File):Option[Release] = for(pf <- f.parent; p <- project(pf) ) yield p / f.name
def project(f:File):Option[Project] = for(pf <- f.parent; o <- org(pf) ) yield o / f.name
def org(f:File):Option[Org] = Some(Org(f.unixPath))
def artifactDepth:Int
def releaseDepth:Int = 0
def projectDepth:Int = 0
//----
def path(o:Org):File = File(filename(o))
def path(p:Project):File = path(p.org) / filename(p)
def path(r:Release):File = path(r.project) / filename(r)
def path(a:Artifact):File = path(a.release) / filename(a)
}
/** / [_alt] / / [_alt]-. */
object Maven extends Style{
def artifactDepth:Int = 0
def filename(o:Org):String = o.domains mkString "/"
def filename(p:Project):String = p.name +++ p.alt.map("_"+_)
def filename(r:Release):String = r.tag.name
def filename(a:Artifact):String = filename(a.release.project) + ("-"+a.release.tag.name) +++ a.alt.map("-"+_) + ("."+a.ext)
}
/** / [_alt] / / s / [_alt]. */
object Ivy extends Style{
def artifactDepth:Int = 1
def filename(o:Org):String = o.domains mkString "."
def filename(p:Project):String = p.name +++ p.alt.map("_"+_)
def filename(r:Release):String = r.tag.name
def filename(a:Artifact):String = {
val baseDir = a.baseKind+"s/"
if(a.baseKind == "ivy") baseDir + a.ext
else baseDir + filename(a.release.project) + "."+a.ext
}
/** copy (& transform) a cache or repo (useful for maintaining repos on air-gapped networks)
* Example: Repo.Ivy.sync(file"~/.ivy/cache", file"~/ivyrepo", dryRun=false) */
lazy val sync = Sync(src => {
val ivyXml = Glob.ignoreCase("ivy-*.xml")
val ivyData = Glob.ignoreCase("ivydata-*.properties")
val ivyOrg = Glob.ignoreCase("ivy-*.xml.original")
def parent = src.parent.getOrElse(src) //simple way to do parent matches
val name = src.name
if(name == ".sbt.ivy.lock") Sync.Skip(src, 'SbtLock)
else if(ivyOrg matches name) Sync.Skip(src, 'IvyOrg)
else if(ivyData matches name) Sync.Skip(src, 'IvyData)
else if(src.path contains "-SNAPSHOT") Sync.Skip(src, 'Snapshot)
else if( (ivyXml matches name) && (parent.name != "ivys")) {
Sync.Copy( File(src.path.replaceAllLiterally(name, s"ivys/$name")), 'IvyXml) //transform the location
}
else Sync.Copy(src, 'Artifact)
})
}
/** / [_alt] / / s / [_alt]. */
object IvyCache extends Style{
def artifactDepth:Int = 1
def filename(o:Org):String = o.domains mkString "."
def filename(p:Project):String = p.name +++ p.alt.map("_"+_)
def filename(r:Release):String = "" //no specific release folder
def filename(a:Artifact):String =
if(a.baseKind == "ivy") "ivy-"+a.release.tag.name+ a.ext.drop(3) //drop ivy from the ext
else a.baseKind+"s/" + filename(a.release.project) + "-" + a.release.tag.name + "."+a.ext
}
object Github extends Style{
def artifactDepth:Int = 0
def filename(o:Org):String = ???
def filename(p:Project):String = ???
def filename(r:Release):String = ???
def filename(a:Artifact):String = ???
}
/** / / scala_ / sbt_ / / s / . */
case class SbtPlugin(sbtVersion:String="0.13",scalaVersion:String="2.10") extends Style{
//TODO use implicits for defaults of sbtVersion and scalaVersion
//TODO test each subsection
def artifactDepth:Int = 1
def filename(o:Org):String = o.domains mkString "."
def filename(p:Project):String = p.name +++ p.alt.map("_"+_) + s"/scala_$scalaVersion/sbt_$sbtVersion"
def filename(r:Release):String = r.tag.name
def filename(a:Artifact):String = a.ext+"s/" + a.release.project.name + "."+a.ext
}
//-----Sources--------------
/**Generic repo source (required functions)*/
trait Source {
def get(a:Artifact,style:Style):Option[File]
def releases(x:Project,style:Style):List[Release]
def projects(x:Org, style:Style):List[Project]
def artifacts(x:Release, style:Style):List[Artifact]
def stats(p:Project):Map[String,Int] = Map[String,Int]()
}
trait SourcePathList extends Source{
def list(f:File,depth:Int):List[String]
//----
def releases(x:Project,style:Style):List[Release] = list(style path x, style.releaseDepth) map (x / _ )
def projects(x:Org, style:Style):List[Project] = list(style path x, style.projectDepth) map (x / _ )
def artifacts(x:Release, style:Style):List[Artifact] = list(style path x, style.artifactDepth) map (x / _ )
}
class SourceFile(baseDir:File) extends SourcePathList {
def list(f:File,depth:Int):List[String] = files(baseDir / f.unixPath, depth).toList
def get(a:Artifact, style:Style):Option[File] = {val f = style.path(a); if(f.exists) Some(f) else None}
}
class SourceURL(baseURL:URL) extends SourcePathList {
def list(f:File,depth:Int):List[String] = hrefs(baseURL / (f.unixPath + "/"), depth).toList
def get(a:Artifact,style:Style):Option[File] = ??? //TODO add url downloader to a cache repo
}
class SourceGithub(userpass:Option[(String,String)] = None) extends Source {
private val api = URL("https://api.github.com") ++ List("per_page" -> 100) //TODO add auto pagination listings...
// println(api)
//TODO add a per_page=100 parameter for better pagination https://developer.github.com/guides/traversing-with-pagination/
//TODO get next from the rel.next header to list the with per_page=100 parameter for better pagination https://developer.github.com/guides/traversing-with-pagination/
private val Name = Pluck(""""name"\s*:\s*"([^"]+)"""".r)
private val FullName = Pluck(""""full_name"\s*:\s*"([^"]+)"""".r)
//https://api.github.com/repos/scala/scala/tags
private def blockingGet(url:URL):String = {
import Implicit.ec
// println(url)
(userpass match {
case Some( (user, pass) ) => url.getBasicAuth(user,pass)
case None => url.get
}).block(60.s).getOrElse(s"Error downloading $url")
}
def releases(x:Project,style:Style):List[Release] =
blockingGet(api / "repos" / x.org.name / x.name / "tags").pluckAll(Name).toList map {x / _}
def projects(x:Org, style:Style):List[Project] =
blockingGet(api / "users" / x.name / "repos").pluckAll(FullName).toList map {fullName =>
val name = fullName.replaceAllLiterally(x.name+"/", "") //take off the organization name if it exists
x / name
}
def artifacts(x:Release, style:Style):List[Artifact] = List(x / "zip") //TODO make the get actually get this zipball
def get(a:Artifact,style:Style):Option[File] = ??? //TODO add url downloader to a cache repo
override def stats(p:Project):Map[String,Int] = {
val json = blockingGet(api / "repos" / p.org.name / p.name)
def count(k:String):Option[Int] = json pluck Pluck((k.quote + """\s*:\s*(\d+)""").r) map {_.toInt}
(for(k <- "forks watchers stargazers open_issues subscribers" split " "; v <- count(k+"_count")) yield k->v).toMap
// "forks watchers stargazers open_issues subscribers".split(" ") mapIf; v <- count(k+"_count")) yield k->v).toMap //TODO make this cleaner with a mapIf that flatten's options
}
}
private def listFirst[B](repos:Repos)(list:Repo => List[B]):List[B] = repos.foldLeft(List[B]()){
case (Nil, repo) => list(repo)
case (x, _) => x
}
/**Explicit repository Source(file,url) and Style(maven,ivy,sbt)*/
class Repo(val source:Source, val name:Option[String], val style:Style){
def releases(x:Project) :List[Release] = source.releases(x, style)
def projects(x:Org) :List[Project] = source.projects(x, style)
def artifacts(x:Release):List[Artifact] = source.artifacts(x, style)
// def latest(x:Project):Option[Release] = releases(x).sortBy{_.tag}.lastOption
// def latest(org:String,prj:String):Option[Release] = releases(x).sortBy{_.tag}.lastOption
def get(a:Artifact):Option[File] = source.get(a,style)
def as(newStyle:Style):Repo = new Repo(source,name,newStyle)
def /(orgName:String):Org = Org(orgName)
}
def apply(base:File, style:Style):Repo = new Repo(new SourceFile(base), None, style)
def apply(base:URL, style:Style):Repo = new Repo(new SourceURL(base), None, style)
//---repos
//val github = ???
val ivyLocal = Repo(File.home / ".ivy2/local", Ivy)
val ivyCache = Repo(File.home / ".ivy2/cache", IvyCache)
val mavenLocal = Repo(File.home / ".m2/repository", Maven)
val mavenCentral = Repo(URL("http://central.maven.org/maven2"), Maven)
//http://dl.bintray.com/sbt/sbt-plugin-releases/com.eed3si9n/sbt-assembly/scala_2.10/sbt_0.13/
// val mavenCentralGoogle = Repo(url"https://maven-central.storage.googleapis.com",Maven)
val sbtPlugins = Repo(URL("http://dl.bintray.com/sbt/sbt-plugin-releases"), SbtPlugin())
val github = new Repo(new SourceGithub(), None, Github)
def github(user:String,pass:String) = new Repo(new SourceGithub(Some(user -> pass)), None, Github)
type Repos = Iterable[Repo]
implicit val defaultRepos:Repos = List[Repo](ivyLocal, ivyCache, mavenLocal, mavenCentral, sbtPlugins, github)
}