Merge pull request #681 from team-lab/feature/improve-file-list-performance

Feature/improve file list performance

src/main/scala/gitbucket/core/util/JGitUtil.scala

@@ -196,62 +196,106 @@ object JGitUtil {
    * @return HTML of the file list
    */
   def getFileList(git: Git, revision: String, path: String = "."): List[FileInfo] = {
-    var list = new scala.collection.mutable.ListBuffer[(ObjectId, FileMode, String, String, Option[String])]
-
     using(new RevWalk(git.getRepository)){ revWalk =>
       val objectId  = git.getRepository.resolve(revision)
+      if(objectId==null) return Nil
       val revCommit = revWalk.parseCommit(objectId)
 
-      val treeWalk = if (path == ".") {
+      def useTreeWalk(rev:RevCommit)(f:TreeWalk => Any): Unit = if (path == ".") {
         val treeWalk = new TreeWalk(git.getRepository)
-        treeWalk.addTree(revCommit.getTree)
+        treeWalk.addTree(rev.getTree)
-        treeWalk
+        using(treeWalk)(f)
       } else {
-        val treeWalk = TreeWalk.forPath(git.getRepository, path, revCommit.getTree)
+        val treeWalk = TreeWalk.forPath(git.getRepository, path, rev.getTree)
-        treeWalk.enterSubtree()
+        if(treeWalk != null){
-        treeWalk
+          treeWalk.enterSubtree
+          using(treeWalk)(f)
+        }
+      }
+      @tailrec
+      def simplifyPath(tuple: (ObjectId, FileMode, String, Option[String], RevCommit)): (ObjectId, FileMode, String, Option[String], RevCommit) = tuple match {
+        case (oid, FileMode.TREE, name, _, commit ) =>
+          (using(new TreeWalk(git.getRepository)) { walk =>
+            walk.addTree(oid)
+            // single tree child, or None
+            if(walk.next() && walk.getFileMode(0) == FileMode.TREE){
+              Some((walk.getObjectId(0), walk.getFileMode(0), name + "/" + walk.getNameString, None, commit)).filterNot(_ => walk.next())
+            } else {
+              None
+            }
+          }) match {
+            case Some(child) => simplifyPath(child)
+            case _ => tuple
+          }
+        case _ => tuple
       }
 
-      using(treeWalk) { treeWalk =>
+      def tupleAdd(tuple:(ObjectId, FileMode, String, Option[String]), rev:RevCommit) = tuple match {
-        while (treeWalk.next()) {
+        case (oid, fmode, name, opt) => (oid, fmode, name, opt, rev)
-          // submodule
-          val linkUrl = if(treeWalk.getFileMode(0) == FileMode.GITLINK){
-            getSubmodules(git, revCommit.getTree).find(_.path == treeWalk.getPathString).map(_.url)
-          } else None
-
-          list.append((treeWalk.getObjectId(0), treeWalk.getFileMode(0), treeWalk.getPathString, treeWalk.getNameString, linkUrl))
       }
 
-        list.transform(tuple =>
-          if (tuple._2 != FileMode.TREE)
-            tuple
-          else
-            simplifyPath(tuple)
-        )
-
       @tailrec
-        def simplifyPath(tuple: (ObjectId, FileMode, String, String, Option[String])): (ObjectId, FileMode, String, String, Option[String]) = {
+      def findLastCommits(result:List[(ObjectId, FileMode, String, Option[String], RevCommit)],
-          val list = new scala.collection.mutable.ListBuffer[(ObjectId, FileMode, String, String, Option[String])]
+                          restList:List[((ObjectId, FileMode, String, Option[String]), Map[RevCommit, RevCommit])],
-          using(new TreeWalk(git.getRepository)) { walk =>
+                          revIterator:java.util.Iterator[RevCommit]): List[(ObjectId, FileMode, String, Option[String], RevCommit)] ={
-            walk.addTree(tuple._1)
+        if(restList.isEmpty){
-            while (walk.next() && list.size < 2) {
+          result
-              val linkUrl = if (walk.getFileMode(0) == FileMode.GITLINK) {
+        }else if(!revIterator.hasNext){ // maybe, revCommit has only 1 log. other case, restList be empty
-                getSubmodules(git, revCommit.getTree).find(_.path == walk.getPathString).map(_.url)
+          result ++ restList.map{ case (tuple, map) => tupleAdd(tuple, map.values.headOption.getOrElse(revCommit)) }
-              } else None
+        }else{
-              list.append((walk.getObjectId(0), walk.getFileMode(0), tuple._3 + "/" + walk.getPathString, tuple._4 + "/" + walk.getNameString, linkUrl))
+          val newCommit = revIterator.next
+          val (thisTimeChecks,skips) = restList.partition{ case (tuple, parentsMap) => parentsMap.contains(newCommit) }
+          if(thisTimeChecks.isEmpty){
+            findLastCommits(result, restList, revIterator)
+          }else{
+            var nextRest = skips
+            var nextResult = result
+            // Map[(name, oid), (tuple, parentsMap)]
+            val rest = scala.collection.mutable.Map(thisTimeChecks.map{ t => (t._1._3 -> t._1._1) -> t }:_*)
+            lazy val newParentsMap = newCommit.getParents.map(_ -> newCommit).toMap
+            useTreeWalk(newCommit){ walk =>
+              while(walk.next){
+                rest.remove(walk.getNameString -> walk.getObjectId(0)).map{ case (tuple, _) =>
+                  if(newParentsMap.isEmpty){
+                    nextResult +:= tupleAdd(tuple, newCommit)
+                  }else{
+                    nextRest +:= tuple -> newParentsMap
                   }
                 }
-          if (list.size != 1 || list.exists(_._2 != FileMode.TREE))
+              }
-            tuple
+            }
-          else
+            rest.values.map{ case (tuple, parentsMap) =>
-            simplifyPath(list(0))
+              val restParentsMap = parentsMap - newCommit
+              if(restParentsMap.isEmpty){
+                nextResult +:= tupleAdd(tuple, parentsMap(newCommit))
+              }else{
+                nextRest +:= tuple -> restParentsMap
+              }
+            }
+            findLastCommits(nextResult, nextRest, revIterator)
           }
         }
       }
 
-    val commits = getLatestCommitFromPaths(git, list.toList.map(_._3), revision)
+      var fileList: List[(ObjectId, FileMode, String, Option[String])] = Nil
-    list.map { case (objectId, fileMode, path, name, linkUrl) =>
+      useTreeWalk(revCommit){ treeWalk =>
-      defining(commits(path)){ commit =>
+        while (treeWalk.next()) {
+          val linkUrl =if (treeWalk.getFileMode(0) == FileMode.GITLINK) {
+            getSubmodules(git, revCommit.getTree).find(_.path == treeWalk.getPathString).map(_.url)
+          } else None
+          fileList +:= (treeWalk.getObjectId(0), treeWalk.getFileMode(0), treeWalk.getNameString, linkUrl)
+        }
+      }
+      revWalk.markStart(revCommit)
+      if(path != "."){
+        revWalk.setTreeFilter(PathFilter.create(path))
+      }
+      val it = revWalk.iterator
+      val lastCommit = it.next
+      val nextParentsMap = Option(lastCommit).map(_.getParents.map(_ -> lastCommit).toMap).getOrElse(Map())
+      findLastCommits(List.empty, fileList.map(a => a -> nextParentsMap), it)
+        .map(simplifyPath)
+        .map { case (objectId, fileMode, name, linkUrl, commit) =>
           FileInfo(
             objectId,
             fileMode == FileMode.TREE || fileMode == FileMode.GITLINK,
@@ -262,7 +306,6 @@ object JGitUtil {
             commit.getAuthorIdent.getName,
             commit.getAuthorIdent.getEmailAddress,
             linkUrl)
-      }
         }.sortWith { (file1, file2) =>
           (file1.isDirectory, file2.isDirectory) match {
             case (true , false) => true
@@ -271,6 +314,7 @@ object JGitUtil {
           }
         }.toList
     }
+  }
 
   /**
    * Returns the first line of the commit message.

src/test/scala/gitbucket/core/service/MergeServiceSpec.scala

@@ -5,6 +5,7 @@ import gitbucket.core.util.JGitUtil
 import gitbucket.core.util.Directory._
 import gitbucket.core.util.Implicits._
 import gitbucket.core.util.ControlUtil._
+import gitbucket.core.util.GitSpecUtil._
 
 import org.apache.commons.io.FileUtils
 import org.eclipse.jgit.api.Git
@@ -14,51 +15,22 @@ import org.eclipse.jgit.revwalk._
 import org.eclipse.jgit.treewalk._
 import org.specs2.mutable.Specification
 
+import java.io.File
 import java.nio.file._
 import java.util.Date
 
-
 class MergeServiceSpec extends Specification {
   sequential
   val service = new MergeService{}
   val branch = "master"
   val issueId = 10
-  def initRepository(owner:String, name:String) = {
+  def initRepository(owner:String, name:String): File = {
-      val repo1Dir = getRepositoryDir(owner, name)
+    val dir = createTestRepository(getRepositoryDir(owner, name))
-      RepositoryCache.clear()
+    using(Git.open(dir)){ git =>
-      FileUtils.deleteQuietly(repo1Dir)
-      Files.createDirectories(repo1Dir.toPath())
-      JGitUtil.initRepository(repo1Dir)
-      using(Git.open(repo1Dir)){ git =>
       createFile(git, s"refs/heads/master", "test.txt", "hoge" )
       git.branchCreate().setStartPoint(s"refs/heads/master").setName(s"refs/pull/${issueId}/head").call()
     }
-      repo1Dir
+    dir
-  }
-  def createFile(git:Git, branch:String, name:String, content:String){
-    val builder  = DirCache.newInCore.builder()
-    val inserter = git.getRepository.newObjectInserter()
-    val headId   = git.getRepository.resolve(branch + "^{commit}")
-    builder.add(JGitUtil.createDirCacheEntry(name, FileMode.REGULAR_FILE,
-      inserter.insert(Constants.OBJ_BLOB, content.getBytes("UTF-8"))))
-    builder.finish()
-    JGitUtil.createNewCommit(git, inserter, headId, builder.getDirCache.writeTree(inserter),
-      branch, "dummy", "dummy@example.com", "Initial commit")
-  }
-  def getFile(git:Git, branch:String, path:String) = {
-      val revCommit = JGitUtil.getRevCommitFromId(git, git.getRepository.resolve(branch))
-        val objectId = using(new TreeWalk(git.getRepository)){ walk =>
-          walk.addTree(revCommit.getTree)
-          walk.setRecursive(true)
-          @scala.annotation.tailrec
-          def _getPathObjectId: ObjectId = walk.next match {
-            case true if(walk.getPathString == path) => walk.getObjectId(0)
-            case true  => _getPathObjectId
-            case false => throw new Exception(s"not found ${branch} / ${path}")
-          }
-          _getPathObjectId
-        }
-        JGitUtil.getContentInfo(git, path, objectId)
   }
   def createConfrict(git:Git) = {
     createFile(git, s"refs/heads/${branch}", "test.txt", "hoge2" )

src/test/scala/gitbucket/core/util/GitSpecUtil.scala

@@ -0,0 +1,110 @@
+package gitbucket.core.util
+
+import gitbucket.core.model._
+import gitbucket.core.util.Directory._
+import gitbucket.core.util.ControlUtil._
+
+import org.apache.commons.io.FileUtils
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.dircache.DirCache
+import org.eclipse.jgit.lib._
+import org.eclipse.jgit.revwalk._
+import org.eclipse.jgit.treewalk._
+import org.eclipse.jgit.merge._
+import org.eclipse.jgit.errors._
+
+import java.nio.file._
+import java.util.Date
+import java.io.File
+
+object GitSpecUtil {
+  def withTestFolder[U](f: File => U) {
+    val folder = new File(System.getProperty("java.io.tmpdir"), "test-" + System.nanoTime)
+    if(!folder.mkdirs()){
+      throw new java.io.IOException("can't create folder "+folder.getAbsolutePath)
+    }
+    try {
+      f(folder)
+    } finally {
+      FileUtils.deleteQuietly(folder)
+    }
+  }
+  def withTestRepository[U](f: Git => U) = withTestFolder(folder => using(Git.open(createTestRepository(folder)))(f))
+  def createTestRepository(dir: File): File = {
+    RepositoryCache.clear()
+    FileUtils.deleteQuietly(dir)
+    Files.createDirectories(dir.toPath())
+    JGitUtil.initRepository(dir)
+    dir
+  }
+  def createFile(git: Git, branch: String, name: String, content: String,
+                 autorName: String = "dummy", autorEmail: String = "dummy@example.com",
+                 message: String = "test commit") {
+    val builder = DirCache.newInCore.builder()
+    val inserter = git.getRepository.newObjectInserter()
+    val headId = git.getRepository.resolve(branch + "^{commit}")
+    if(headId!=null){
+      JGitUtil.processTree(git, headId){ (path, tree) =>
+        if(name != path){
+          builder.add(JGitUtil.createDirCacheEntry(path, tree.getEntryFileMode, tree.getEntryObjectId))
+        }
+      }
+    }
+    builder.add(JGitUtil.createDirCacheEntry(name, FileMode.REGULAR_FILE,
+      inserter.insert(Constants.OBJ_BLOB, content.getBytes("UTF-8"))))
+    builder.finish()
+    JGitUtil.createNewCommit(git, inserter, headId, builder.getDirCache.writeTree(inserter),
+      branch, autorName, autorEmail, message)
+    inserter.flush()
+    inserter.release()
+  }
+  def getFile(git: Git, branch: String, path: String) = {
+    val revCommit = JGitUtil.getRevCommitFromId(git, git.getRepository.resolve(branch))
+    val objectId = using(new TreeWalk(git.getRepository)) { walk =>
+      walk.addTree(revCommit.getTree)
+      walk.setRecursive(true)
+      @scala.annotation.tailrec
+      def _getPathObjectId: ObjectId = walk.next match {
+        case true if (walk.getPathString == path) => walk.getObjectId(0)
+        case true                                 => _getPathObjectId
+        case false                                => throw new Exception(s"not found ${branch} / ${path}")
+      }
+      _getPathObjectId
+    }
+    JGitUtil.getContentInfo(git, path, objectId)
+  }
+  def mergeAndCommit(git: Git, into:String, branch:String, message:String = null):Unit = {
+    val repository = git.getRepository
+    val merger = MergeStrategy.RECURSIVE.newMerger(repository, true)
+    val mergeBaseTip = repository.resolve(into)
+    val mergeTip = repository.resolve(branch)
+    val conflicted = try {
+      !merger.merge(mergeBaseTip, mergeTip)
+    } catch {
+      case e: NoMergeBaseException => true
+    }
+    if(conflicted){
+      throw new RuntimeException("conflict!")
+    }
+    val mergeTipCommit = using(new RevWalk( repository ))(_.parseCommit( mergeTip ))
+    val committer = mergeTipCommit.getCommitterIdent;
+    // creates merge commit
+    val mergeCommit = new CommitBuilder()
+    mergeCommit.setTreeId(merger.getResultTreeId)
+    mergeCommit.setParentIds(Array[ObjectId](mergeBaseTip, mergeTip): _*)
+    mergeCommit.setAuthor(committer)
+    mergeCommit.setCommitter(committer)
+    mergeCommit.setMessage(message)
+    // insertObject and got mergeCommit Object Id
+    val inserter = repository.newObjectInserter
+    val mergeCommitId = inserter.insert(mergeCommit)
+    inserter.flush()
+    inserter.release()
+    // update refs
+    val refUpdate = repository.updateRef(into)
+    refUpdate.setNewObjectId(mergeCommitId)
+    refUpdate.setForceUpdate(true)
+    refUpdate.setRefLogIdent(committer)
+    refUpdate.update()
+  }
+}

src/test/scala/gitbucket/core/util/JGitUtilSpec.scala

@@ -0,0 +1,93 @@
+package gitbucket.core.util
+
+import org.specs2.mutable._
+import GitSpecUtil._
+
+class JGitUtilSpec extends Specification {
+
+  "getFileList(git: Git, revision: String, path)" should {
+    withTestRepository { git =>
+      def list(branch: String, path: String) =
+        JGitUtil.getFileList(git, branch, path).map(finfo => (finfo.name, finfo.message, finfo.isDirectory))
+      list("master", ".") mustEqual Nil
+      list("master", "dir/subdir") mustEqual Nil
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      createFile(git, "master", "README.md", "body1", message = "commit1")
+
+      list("master", ".") mustEqual List(("README.md", "commit1", false))
+      list("master", "dir/subdir") mustEqual Nil
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      createFile(git, "master", "README.md", "body2", message = "commit2")
+
+      list("master", ".") mustEqual List(("README.md", "commit2", false))
+      list("master", "dir/subdir") mustEqual Nil
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      createFile(git, "master", "dir/subdir/File3.md", "body3", message = "commit3")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit3", true), ("README.md", "commit2", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false))
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      createFile(git, "master", "dir/subdir/File4.md", "body4", message = "commit4")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("README.md", "commit2", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      createFile(git, "master", "README5.md", "body5", message = "commit5")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("README.md", "commit2", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      createFile(git, "master", "README.md", "body6", message = "commit6")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual Nil
+      list("branch", "dir/subdir") mustEqual Nil
+
+      git.branchCreate().setName("branch").setStartPoint("master").call()
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual List(("dir/subdir", "commit4", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("branch", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+
+      createFile(git, "branch", "dir/subdir/File3.md", "body7", message = "commit7")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual List(("dir/subdir", "commit7", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("branch", "dir/subdir") mustEqual List(("File3.md", "commit7", false), ("File4.md", "commit4", false))
+
+      createFile(git, "master", "dir8/File8.md", "body8", message = "commit8")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("dir8", "commit8", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual List(("dir/subdir", "commit7", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("branch", "dir/subdir") mustEqual List(("File3.md", "commit7", false), ("File4.md", "commit4", false))
+
+      createFile(git, "branch", "dir/subdir9/File9.md", "body9", message = "commit9")
+
+      list("master", ".") mustEqual List(("dir/subdir", "commit4", true), ("dir8", "commit8", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit3", false), ("File4.md", "commit4", false))
+      list("branch", ".") mustEqual List(("dir", "commit9", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("branch", "dir/subdir") mustEqual List(("File3.md", "commit7", false), ("File4.md", "commit4", false))
+
+      mergeAndCommit(git, "master", "branch", message = "merge10")
+
+      list("master", ".") mustEqual List(("dir", "commit9", true), ("dir8", "commit8", true), ("README.md", "commit6", false), ("README5.md", "commit5", false))
+      list("master", "dir/subdir") mustEqual List(("File3.md", "commit7", false), ("File4.md", "commit4", false))
+    }
+  }
+}