main.name.remal.gradle_plugins.plugins.vcs.VcsOperationsGit.kt Maven / Gradle / Ivy
package name.remal.gradle_plugins.plugins.vcs
import com.google.common.collect.MultimapBuilder
import com.google.common.collect.SetMultimap
import name.remal.asStream
import name.remal.debug
import name.remal.filterNotNull
import name.remal.forSelfAndEachParent
import name.remal.gradle_plugins.api.AutoService
import name.remal.gradle_plugins.dsl.DEFAULT_IO_TIMEOUT
import name.remal.gradle_plugins.dsl.extensions.withPrefix
import name.remal.gradle_plugins.dsl.utils.getGradleLogger
import name.remal.gradle_plugins.dsl.utils.retryIO
import name.remal.gradle_plugins.utils.setSshSessionFactory
import name.remal.gradle_plugins.utils.setTimeout
import name.remal.gradle_plugins.utils.toCredentialsProvider
import name.remal.gradle_plugins.utils.toSshSessionFactory
import name.remal.nullIf
import name.remal.nullIfEmpty
import name.remal.orNull
import name.remal.packageName
import name.remal.retry
import name.remal.stream
import name.remal.toList
import name.remal.toSet
import name.remal.use
import org.eclipse.jgit.api.CreateBranchCommand.SetupUpstreamMode.NOTRACK
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.api.RebaseCommand.Operation.BEGIN
import org.eclipse.jgit.api.TransportCommand
import org.eclipse.jgit.api.errors.EmptyCommitException
import org.eclipse.jgit.api.errors.GitAPIException
import org.eclipse.jgit.api.errors.JGitInternalException
import org.eclipse.jgit.api.errors.TransportException
import org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_URL
import org.eclipse.jgit.lib.ConfigConstants.CONFIG_SUBMODULE_SECTION
import org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME
import org.eclipse.jgit.lib.Constants.DOT_GIT
import org.eclipse.jgit.lib.Constants.HEAD
import org.eclipse.jgit.lib.Constants.MASTER
import org.eclipse.jgit.lib.Constants.R_HEADS
import org.eclipse.jgit.lib.Constants.R_TAGS
import org.eclipse.jgit.lib.Constants.STASH
import org.eclipse.jgit.lib.ObjectId
import org.eclipse.jgit.lib.ObjectId.isId
import org.eclipse.jgit.lib.ProgressMonitor
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.lib.Repository.shortenRefName
import org.eclipse.jgit.merge.MergeStrategy
import org.eclipse.jgit.revwalk.RevCommit
import org.eclipse.jgit.revwalk.RevObject
import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.submodule.SubmoduleWalk
import org.eclipse.jgit.transport.CredentialsProvider
import org.eclipse.jgit.transport.PushResult
import org.eclipse.jgit.transport.RefSpec
import org.eclipse.jgit.transport.RemoteConfig
import org.eclipse.jgit.transport.RemoteRefUpdate.Status.AWAITING_REPORT
import org.eclipse.jgit.transport.RemoteRefUpdate.Status.OK
import org.eclipse.jgit.transport.RemoteRefUpdate.Status.UP_TO_DATE
import org.eclipse.jgit.transport.SshSessionFactory
import org.eclipse.jgit.transport.TagOpt.FETCH_TAGS
import org.eclipse.jgit.transport.TagOpt.NO_TAGS
import org.eclipse.jgit.transport.URIish
import org.gradle.api.logging.Logger
import java.io.Closeable
import java.io.File
import java.io.IOException
import java.lang.Math.floor
import java.lang.Math.max
import java.lang.System.currentTimeMillis
import java.lang.System.nanoTime
import java.nio.file.Path
import java.util.concurrent.TimeUnit.SECONDS
import java.util.stream.Stream
import org.eclipse.jgit.errors.TransportException as CoreTransportException
class VcsOperationsGit(repositoryDir: File) : VcsOperations(), Closeable {
companion object {
private val logger = getGradleLogger(VcsOperationsGit::class.java).withPrefix("GIT: ")
}
private val vcsRootPath: Path = repositoryDir.toPath().toAbsolutePath().normalize()
override val vcsRootDir: File = vcsRootPath.toFile()
private val repository: Repository = wrapJGitExceptions { FileRepositoryBuilder.create(File(vcsRootDir, DOT_GIT)) }
private val git: Git = wrapJGitExceptions { Git(repository) }
private var credentialsProvider: CredentialsProvider? = null
private var sshSessionFactory: SshSessionFactory? = null
override val trueMasterBranch = MASTER
private fun Repository.getCurrentBranch(): String? {
val head = exactRef(HEAD) ?: return null
if (!head.isSymbolic) return null
val branch = head.target?.name?.let(Repository::shortenRefName) ?: return null
if (STASH == branch) return null
return branch
}
override val trueCurrentBranch: String? get() = wrapJGitExceptions { repository.getCurrentBranch() }
override val isCommitted: Boolean get() = wrapJGitExceptions { !git.status().call().hasUncommittedChanges() }
override var commitAuthor: CommitAuthor? = null
private fun getTagsForObjectId(): SetMultimap {
return MultimapBuilder.hashKeys().treeSetValues().build().apply {
repository.refDatabase.getRefsByPrefix(R_TAGS).forEach { ref ->
val peeledRef = repository.refDatabase.peel(ref) ?: ref
val objectId: ObjectId = peeledRef.peeledObjectId ?: peeledRef.objectId
val tagName: String = shortenRefName(ref.name)
put(objectId, tagName)
}
}
}
override fun walkCommits(): Stream = wrapJGitExceptions {
val tags = getTagsForObjectId()
return git.log().call().iterator().asStream()
.map { it.toCommit(tags.get(it.id).toList()) }
}
override fun getCurrentCommit(): Commit? {
val revCommit = git.log().setMaxCount(1).call().firstOrNull() ?: return null
val tags = getTagsForObjectId()
return revCommit.toCommit(tags.get(revCommit.id).toList())
}
private fun Git.createBranchForCommit(commit: RevCommit): String {
while (true) {
val tempBranchName = "commit-${commit.name}-${currentTimeMillis()}"
if (branchList().call().any { it.name == tempBranchName }) {
Thread.sleep(1)
continue
}
logger.lifecycle("Creating temporary branch: {} pointing to commit {}", tempBranchName, commit.name)
checkout()
.setName(tempBranchName)
.setCreateBranch(true)
.setStartPoint(commit)
.setUpstreamMode(NOTRACK)
.setProgressMonitor(LoggerProgressMonitor())
.call()
return tempBranchName
}
}
private fun Git.getRemoteNameToPush(): String? {
val remoteNames = remoteList().call().map(RemoteConfig::getName)
if (DEFAULT_REMOTE_NAME in remoteNames) return DEFAULT_REMOTE_NAME
return remoteNames.firstOrNull()
}
private fun Iterable.report() {
forEach { result ->
result.messages.splitToSequence('\n')
.map(String::trimEnd)
.joinToString("\n")
.trim('\n')
.nullIfEmpty()
?.let(logger::lifecycle)
val remoteUpdates = result.remoteUpdates
if (remoteUpdates.isEmpty()) {
logger.lifecycle("No branches were pushed")
} else {
val failedUpdates = remoteUpdates.filter {
it.status != UP_TO_DATE
&& it.status != AWAITING_REPORT
&& it.status != OK
}
val resultMessage = buildString {
if (failedUpdates.isEmpty()) {
append("Push result:")
} else {
append("Push failed:")
}
remoteUpdates.forEach {
append("\n ").append(it.remoteName).append(": ").append(it.status?.name)
it.message?.trim().nullIfEmpty()?.let { append(": ").append(it) }
}
}
if (failedUpdates.isEmpty()) {
logger.lifecycle(resultMessage)
} else {
throw GitOperationException(resultMessage)
}
}
}
}
@Suppress("ComplexMethod", "LongMethod")
private fun Git.commitImpl(message: String, submodulePath: String? = null, relativePaths: Collection = emptyList()) {
val logger = logger.withSubmodulePrefix(submodulePath)
if (status().apply { relativePaths.forEach { addPath(it) } }.call().isClean) {
logger.lifecycle("Nothing to commit, working tree is clean")
return
}
logger.lifecycle("Committing changes...")
add()
.apply {
if (relativePaths.isNotEmpty()) {
relativePaths.forEach { addFilepattern(it) }
} else {
addFilepattern(".")
}
}
.call()
val commit = try {
commit()
.apply {
if (relativePaths.isNotEmpty()) {
relativePaths.forEach { setOnly(it) }
} else {
setAll(true)
}
}
.apply {
setMessage(message)
commitAuthor?.also {
setAuthor(it.name, it.email)
setCommitter(it.name, it.email)
}
setInsertChangeId(true)
setAllowEmpty(false)
}
.call()
.also { logger.lifecycle("Committed: {}", it.name) }
} catch (e: EmptyCommitException) {
logger.lifecycle("Nothing to commit, working tree clean")
return
}
val remoteName = getRemoteNameToPush()
if (submodulePath != null) {
val currentSubmoduleBranch = repository.getCurrentBranch()
if (currentSubmoduleBranch != null && remoteName != null) {
logger.lifecycle("Fetching branches...")
val fetchResult = retryTransport {
fetch()
.apply {
isCheckFetchedObjects = true
setTagOpt(NO_TAGS)
}
.setProgressMonitor(LoggerProgressMonitor())
.setupTransport()
.call()
.also {
it.messages.nullIfEmpty()?.let(logger::lifecycle)
if (it.advertisedRefs.isEmpty()) {
logger.lifecycle("No branches were fetched")
} else {
it.advertisedRefs.forEach { logger.lifecycle("Fetched branch: {}", it.name) }
}
}
}
try {
logger.debug("Fetching tags...")
retryTransport {
fetch()
.apply {
isCheckFetchedObjects = true
setTagOpt(FETCH_TAGS)
}
.setProgressMonitor(LoggerProgressMonitor())
.setupTransport()
.call()
.also {
it.messages.nullIfEmpty()?.let(logger::lifecycle)
if (it.advertisedRefs.isEmpty()) {
logger.debug("No tags were fetched")
} else {
it.advertisedRefs.forEach { logger.debug("Fetched tag: {}", it.name) }
}
}
}
} catch (e: Exception) {
logger.warn("Tags can't be fetched: {}", e.toString())
}
val advertisedRef = fetchResult.getAdvertisedRef(R_HEADS + currentSubmoduleBranch) ?: fetchResult.getAdvertisedRef(currentSubmoduleBranch)
val commitToMerge = advertisedRef?.objectId
val headId = repository.resolve(HEAD)
if (commitToMerge != null && commitToMerge != headId) {
val mergeStrategy = MergeStrategy.RECURSIVE
val canMerge = mergeStrategy.newMerger(repository, true).merge(headId, commitToMerge)
if (!canMerge) {
throw GitOperationException("Submodule $submodulePath: commit ${headId.name} can't be merged with fetched ${commitToMerge.name}")
}
logger.lifecycle("Starting rebase...")
val rebaseResult = rebase()
.apply {
setUpstream(commitToMerge)
setOperation(BEGIN)
setStrategy(mergeStrategy)
}
.setProgressMonitor(LoggerProgressMonitor())
.call()
rebaseResult.status.let {
if (it.isSuccessful) {
logger.lifecycle("Rebase status: {}", it)
} else {
throw GitOperationException("Submodule $submodulePath: Rebase status: $it")
}
}
}
}
}
if (remoteName != null) {
val destination = currentBranch.nullIf { submodulePath != null }?.let { R_HEADS + it }
val isHeadDetached = isId(repository.fullBranch)
if (isHeadDetached) {
logger.warn("The repository $HEAD currently doesn't point to a branch, but directly refers to a commit (SHA)")
if (destination == null) throw GitOperationException("Current branch is not set, so remote branch can't be defined")
createBranchForCommit(commit)
}
if (destination != null) {
logger.lifecycle("Pushing commit {} to {} (remote {})", commit.name, destination, remoteName)
} else {
logger.lifecycle("Pushing commit {} (remote {})", commit.name, remoteName)
}
retryTransport {
push()
.apply {
if (destination != null) {
setRefSpecs(RefSpec().setDestination(destination))
}
isAtomic = true
}
.setProgressMonitor(LoggerProgressMonitor())
.setupTransport()
.call()
.report()
}
} else {
logger.lifecycle("Skip pushing commit {}, as there are no remotes set", commit.name)
}
}
@Suppress("ComplexMethod", "LongMethod")
override fun commitFiles(message: String, files: Collection) = wrapJGitExceptions {
val relativePaths: Collection = if (files.isEmpty()) {
emptyList()
} else {
files.asSequence()
.map(File::toPath)
.map(Path::toAbsolutePath)
.map(Path::normalize)
.map { path ->
if (!path.startsWith(vcsRootPath)) {
throw IllegalArgumentException("'$path' path doesn't start with '$vcsRootPath'")
}
return@map vcsRootPath.relativize(path)
}
.map(Path::toString)
.map {
if (File.separatorChar != '/') {
it.replace(File.separatorChar, '/')
} else {
it
}
}
.distinct()
.toList()
}
val submodulePaths = mutableListOf()
SubmoduleWalk.forIndex(repository).use { walk ->
while (walk.next()) {
val submoduleGit = Git(walk.repository ?: continue)
val submodulePath = walk.path
submodulePaths.add(submodulePath)
val filteredPaths: List = if (relativePaths.isEmpty()) {
emptyList()
} else {
relativePaths.mapNotNull { path ->
if (submodulePath != null && path.startsWith("$submodulePath/")) {
path.substring(submodulePath.length + 1)
} else {
null
}
}
}
submoduleGit.commitImpl(
message = message,
submodulePath = submodulePath,
relativePaths = filteredPaths
)
}
}
val filteredPaths: List = if (relativePaths.isEmpty()) {
emptyList()
} else {
relativePaths.mapNotNull { path ->
if (submodulePaths.any { path.startsWith("$it/") }) {
null
} else {
path
}
}
}
git.commitImpl(
message = message,
relativePaths = filteredPaths
)
}
override fun getAllTagNames(): Set = wrapJGitExceptions {
return repository.refDatabase.getRefsByPrefix(R_TAGS).mapTo(mutableSetOf(), { shortenRefName(it.name) })
}
override fun createTag(commitId: String, tagName: String, message: String) = wrapJGitExceptions {
if (tagName.isEmpty()) throw IllegalArgumentException("tagName is empty")
logger.lifecycle(
"{} tag '{}' for commit {}",
if (tagName in getAllTagNames()) "Updating" else "Creating",
tagName,
commitId
)
git.tag()
.apply {
if (commitId.isNotEmpty()) objectId = getRevObject(commitId)
name = tagName
isAnnotated = message.isNotEmpty()
if (message.isNotEmpty()) setMessage(message)
isForceUpdate = true
}
.call()
if (git.remoteList().call().isNotEmpty()) {
logger.lifecycle("Pushing tag '{}'", tagName)
retryTransport {
git.push()
.apply {
add(R_TAGS + tagName)
isAtomic = true
}
.setProgressMonitor(LoggerProgressMonitor())
.setupTransport()
.call()
.report()
}
} else {
logger.lifecycle("Skip pushing tag {}, as there are no remotes set", tagName)
}
Unit
}
private tailrec fun findTagWithDepth(
tags: SetMultimap,
walk: RevWalk,
revCommits: List,
predicate: (tagName: String) -> Boolean,
depth: Int = 0,
processedCommitIds: MutableSet = hashSetOf()
): TagsWithDepth? {
revCommits.forEach { revCommit ->
val id = revCommit.id
processedCommitIds.add(id)
val currentTags = tags.get(id) ?: return@forEach
currentTags.filter(predicate).nullIfEmpty()?.let {
return TagsWithDepth(it.toSet(), depth)
}
}
val nextCommits = revCommits.stream()
.peek { if (null == it.parents) walk.parseBody(it) }
.filter { null != it.parents }
.flatMap { it.parents.stream() }
.filter { it.id !in processedCommitIds }
.toList()
if (nextCommits.isEmpty()) return null
return findTagWithDepth(tags, walk, nextCommits, predicate, depth + 1, processedCommitIds)
}
override fun findTagWithDepth(predicate: (tagName: String) -> Boolean): TagsWithDepth? = wrapJGitExceptions {
val tags = getTagsForObjectId()
if (tags.values().none(predicate)) return null
val tagWithDepth = RevWalk(repository).use { walk ->
val headRevCommit = walk.parseCommit(repository.resolve(HEAD) ?: return null)
return@use findTagWithDepth(tags, walk, listOf(headRevCommit), predicate)
}
.also { logger.debug("tagWithDepth = {}", it) }
var order = -1
val tagWithDepthByOrder = git.log().call().iterator().asStream()
.peek { ++order }
.map { revCommit ->
val commitTags = tags.get(revCommit.id) ?: return@map null
commitTags.filter(predicate).nullIfEmpty()?.let {
return@map TagsWithDepth(it.toSet(), order)
}
return@map null
}
.filterNotNull()
.findFirst().orNull
.also { logger.debug("tagWithDepthByOrder = {}", it) }
if (tagWithDepthByOrder != null && tagWithDepth != null) {
if (tagWithDepthByOrder.depth < tagWithDepth.depth) {
return tagWithDepthByOrder
} else {
return tagWithDepth
}
} else if (tagWithDepthByOrder != null) {
return tagWithDepthByOrder
} else if (tagWithDepth != null) {
return tagWithDepth
} else {
return null
}
}
private fun Git.setRemoteUri(uri: String, submodulePath: String? = null) {
val logger = logger.withSubmodulePrefix(submodulePath)
val parsedUri = URIish(uri)
val remotes = remoteList().call()
val remoteUris = remotes.stream()
.flatMap { it.urIs.stream() }
.toSet()
if (parsedUri in remoteUris) {
return
}
if (remotes.isNotEmpty()) {
logger.lifecycle("Clearing all remotes")
remotes.forEach { remote ->
remoteRemove().apply { setRemoteName(remote.name) }.call()
}
}
logger.lifecycle("Adding remote: {}", parsedUri)
remoteAdd()
.apply {
setName(DEFAULT_REMOTE_NAME)
setUri(parsedUri)
}
.call()
}
override fun setUnauthorizedRemoteURI(uri: String) = wrapJGitExceptions {
git.setRemoteUri(uri)
SubmoduleWalk.forIndex(repository).use { walk ->
while (walk.next()) {
val submoduleGit = Git(walk.repository ?: continue)
val remoteUri = walk.remoteUrl
submoduleGit.setRemoteUri(remoteUri, walk.path)
try {
retryIO {
repository.config.setString(
CONFIG_SUBMODULE_SECTION,
walk.moduleName,
CONFIG_KEY_URL,
remoteUri
)
repository.config.save()
}
} catch (e: Throwable) {
logger.debug(e)
}
}
}
credentialsProvider = null
sshSessionFactory = null
}
override fun setUsernamePasswordAuth(username: String, password: CharArray) = wrapJGitExceptions {
logger.lifecycle("Authenticating using username {}", username)
credentialsProvider = UsernamePasswordVcsAuth(username, password).toCredentialsProvider()
sshSessionFactory = null
}
override fun setSSHAuth(privateKeyFile: File, password: CharArray?) = wrapJGitExceptions {
if (logger.isDebugEnabled) {
logger.debug("Authenticating using SSH key file: {}", privateKeyFile.absoluteFile)
} else {
logger.lifecycle("Authenticating using SSH key")
}
credentialsProvider = null
sshSessionFactory = SSHVcsAuth(privateKeyFile.absoluteFile, password).toSshSessionFactory()
}
private fun > T.setupTransport(): T = apply {
setTimeout(DEFAULT_IO_TIMEOUT)
credentialsProvider.let(this::setCredentialsProvider)
sshSessionFactory.let(this::setSshSessionFactory)
}
private fun getObjectId(id: String): ObjectId {
return repository.resolve(if (id.isNotEmpty()) id else HEAD)
}
private fun getRevObject(id: String): RevObject {
val objectId = getObjectId(id)
return RevWalk(repository).use { it.parseAny(objectId) }
}
private class LoggerProgressMonitor : ProgressMonitor {
companion object {
private val MIN_INTERVAL_NANOS = SECONDS.toNanos(5)
}
private var title: String? = null
private var totalWork: Int = 0
private var latestOutputTimestamp: Long = 0
private var lastLoggedMessage: String? = null
override fun beginTask(title: String, totalWork: Int) {
this.title = title
this.totalWork = totalWork
this.latestOutputTimestamp = nanoTime()
this.lastLoggedMessage = null
}
override fun update(completed: Int) {
updateImpl(totalWork)
}
override fun endTask() {
updateImpl(totalWork, true)
}
private fun updateImpl(currentWork: Int, force: Boolean = false) {
val timestamp = nanoTime()
val elapsedTime = timestamp - latestOutputTimestamp
if ((force && lastLoggedMessage != null) || elapsedTime >= MIN_INTERVAL_NANOS) {
val message = buildString {
append(title)
append(": ")
if (totalWork > 0) {
val percentDone = floor(currentWork.toDouble() / totalWork).toInt() * 100
repeat(max(0, 3 - percentDone.toString().length)) { append(' ') }
append(percentDone)
append("% (")
repeat(max(0, totalWork.toString().length - currentWork.toString().length)) { append(' ') }
append(currentWork)
append('/')
append(totalWork)
append(')')
}
}
if (lastLoggedMessage != message) {
logger.lifecycle(message)
latestOutputTimestamp = timestamp
lastLoggedMessage = message
}
}
}
override fun start(totalTasks: Int) {
// do nothing
}
override fun isCancelled() = false
}
@Volatile
private var isClosed: Boolean = false
override fun close() = wrapJGitExceptions {
if (!isClosed) {
synchronized(this) {
if (!isClosed) {
git.close()
repository.close()
isClosed = true
}
}
}
}
private fun Logger.withSubmodulePrefix(submodulePath: String? = null) = if (submodulePath == null) {
this
} else {
this.withPrefix("Submodule '$submodulePath': ", true)
}
}
private inline fun wrapJGitExceptions(action: () -> R): R {
try {
return action()
} catch (e: GitAPIException) {
throw GitOperationException(e)
} catch (e: JGitInternalException) {
throw GitOperationException(e)
} catch (e: Throwable) {
if (e.javaClass.packageName == CoreTransportException::class.java.packageName
|| e.javaClass.packageName == GitAPIException::class.java.packageName
) {
throw GitOperationException(e)
} else {
throw e
}
}
}
private inline fun retryTransport(action: () -> R): R = retry(
5,
listOf(
IOException::class.java,
TransportException::class.java
),
1000,
action
)
@AutoService
class VcsOperationsFactoryGit : VcsOperationsFactory {
override fun get(dir: File): VcsOperations? {
dir.forSelfAndEachParent { currentDir ->
if (File(currentDir, DOT_GIT).isDirectory) {
return VcsOperationsGit(currentDir)
}
}
return null
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy