Add initial support for libgit2/rugged involving massive changes

Large change set here! Difficult to split this up into modular changes as Grit at the time was the heart of the processing. The initial implementation done here for Rugged works, and all the specs are currently passing. The GPG signing issue is no longer an issue, and Rugged also provides a substantial speed boost and memory reduction than the Grit implementation. The current state also run on Ruby 2.0.x as we are not held back by Grit. Remove: * The use of Grit for git processing * repo.rb as Rugged::Repository provided sufficient repo finding capabilities * Mixin for Grit::Blob to include LanguageSniffer functionality * blob_finder.rb as Rugged::Diff::Patch provides sufficient functionality to acquire the individual SHA's of blobs affected in the diff, which then can be Rugged::Repository.lookup(sha) * FileSummary struct and instead just use the patch information from DiffSummary Add: * Rugged for git processing * Direct use of LanguageSniffer in diff_summary to detect the language of the diff's content (using the blob in question) * Additional functionality to diff_summary.rb as it now is responsible for working with patches (provides #additions, #deletions, #status) * Additional information on LanguageSummary (:added_files, :deleted_files, :modified_files) Refactor: * How we found the commits to process, Grit::Commit.find_all(…) was replaces with Rugged::Walker * spec_helper.rb to now use Rugged::Repository instead of our own class * The terminology of :create, :delete to :added_files, :deleted_files Fix: * The ability to handle time_since and time_until for commits to work with Rugged (check on each commit if it is valid -- #valid_commit?) * Specs to work with new implementation and terminologies * fixtures to use the updated key names

Author: kevinjalbert

git_statistics.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
  gem.required_ruby_version = '>= 1.9.1'
 
  gem.add_dependency('json')
- gem.add_dependency('grit')
+ gem.add_dependency('rugged')
  gem.add_dependency('language_sniffer')
 
  gem.add_development_dependency "rspec", "~> 2.12.0"

lib/git_statistics.rb

@@ -5,7 +5,8 @@ class CLI
  attr_reader :repository, :options
 
  def initialize(dir)
- @repository = dir.nil? ? Repo.new(Dir.pwd) : Repo.new(dir)
+ repository_location = dir.nil? ? Rugged::Repository.discover(Dir.pwd) : Rugged::Repository.discover(dir)
+ @repository = Rugged::Repository.new(repository_location)
  @collected = false
  @collector = nil
  @options = OpenStruct.new(
@@ -35,14 +36,13 @@ def collect_and_only_update
  if options.update
  # Ensure commit directory is present
  @collector = Collector.new(repository, options.limit, false, options.pretty)
- commits_directory = repository.working_dir + ".git_statistics"
+ commits_directory = repository.workdir + ".git_statistics/"
  FileUtils.mkdir_p(commits_directory)
  file_count = Utilities.number_of_matching_files(commits_directory, /\d+\.json/) - 1
 
  if file_count >= 0
- time_since = Utilities.get_modified_time(commits_directory + "#{file_count}.json")
- # Only use --since if there is data present
- @collector.collect(options.branch, {:since => time_since})
+ time_since = Utilities.get_modified_time(commits_directory + "#{file_count}.json").to_s
+ @collector.collect(options.branch, {:time_since => time_since})
  @collected = true
  end
  end

lib/git_statistics/blob_finder.rb

@@ -4,52 +4,65 @@ class Collector
  attr_accessor :repo, :commits_path, :commits
 
  def initialize(repo, limit, fresh, pretty)
- Grit::Git.git_timeout = 0
- Grit::Git.git_max_size = 0
-
  @repo = repo
- @commits_path = repo.working_dir + ".git_statistics"
+ @commits_path = repo.workdir + ".git_statistics"
  @commits = Commits.new(@commits_path, fresh, limit, pretty)
  end
 
  def collect(branch, options = {})
- # Collect branches to use for git log
- branches = branch ? [] : @repo.branches.compact.map(&:name)
- Grit::Commit.find_all(@repo, nil, options).each do |commit|
- extract_commit(commit)
- @commits.flush_commits
+ walker = Rugged::Walker.new(repo)
+ master_head = Rugged::Branch.lookup(repo, "master").tip
+
+ walker.push(master_head)
+ walker.each_with_index do |commit, count|
+ if valid_commit?(commit, options)
+ extract_commit(commit, count + 1)
+ @commits.flush_commits
+ end
  end
+
  @commits.flush_commits(true)
  end
 
+ def valid_commit?(commit, options)
+ if !options[:time_since].nil?
+ return false unless commit.author[:time] > DateTime.parse(options[:time_since].to_s).to_time
+ end
+
+ if !options[:time_until].nil?
+ return false unless commit.author[:time] < DateTime.parse(options[:time_until].to_s).to_time
+ end
+
+ return true
+ end
+
  def acquire_commit_meta(commit_summary)
  # Initialize commit data
- data = (@commits[commit_summary.sha] ||= Hash.new(0))
+ data = (@commits[commit_summary.oid] ||= Hash.new(0))
 
- data[:author] = commit_summary.author.name
- data[:author_email] = commit_summary.author.email
- data[:time] = commit_summary.authored_date.to_s
+ data[:author] = commit_summary.author[:name]
+ data[:author_email] = commit_summary.author[:email]
+ data[:time] = commit_summary.author[:time].to_s
  data[:merge] = commit_summary.merge?
  data[:additions] = commit_summary.additions
  data[:deletions] = commit_summary.deletions
  data[:net] = commit_summary.net
- data[:new_files] = commit_summary.new_files
- data[:removed_files] = commit_summary.removed_files
- data[:files] = commit_summary.files
+ data[:added_files] = commit_summary.added_files
+ data[:deleted_files] = commit_summary.deleted_files
+ data[:modified_files] = commit_summary.modified_files
+ data[:files] = commit_summary.file_stats.map{ |file| file.to_json }
 
  return data
  end
 
- def extract_commit(commit)
- unless commit.nil?
- commit_summary = CommitSummary.new(commit)
- Log.info "Extracting #{commit_summary.sha}"
+ def extract_commit(commit, count)
+ Log.info "Extracting(#{count}) #{commit.oid}"
+ commit_summary = CommitSummary.new(@repo, commit)
 
-  # Acquire meta information about commit
-  commit_data = acquire_commit_meta(commit_summary)
+ # Acquire meta information about commit
+ commit_data = acquire_commit_meta(commit_summary)
 
- return commit_data
- end
+ return commit_data
  end
 
  end

lib/git_statistics/collector.rb

@@ -1,7 +1,10 @@
 module GitStatistics
  class CommitSummary < SimpleDelegator
- def initialize(commit)
+ def initialize(repo, commit)
  super(commit)
+ @repo = repo
+ @diff = diff(commit.parents.first)
+ @patches = @diff.patches
  end
 
  # A Git commit is a merge if it has more than one parent
@@ -10,13 +13,18 @@ def merge?
  end
 
  # How many files were removed in this commit
- def removed_files
- cached_show.select { |diff| diff.deleted_file == true }.count
+ def deleted_files
+ file_stats.select { |file| file.status == :deleted }.count
  end
 
  # How many files were added in this commit
- def new_files
- cached_show.select { |diff| diff.new_file == true }.count
+ def added_files
+ file_stats.select { |file| file.status == :added }.count
+ end
+
+ # How many files were modified (not added/deleted) in this commit
+ def modified_files
+ file_stats.select { |file| file.status == :modified }.count
  end
 
  # How many total additions in this commit?
@@ -35,18 +43,18 @@ def net
  end
 
  def file_stats
- @cached_file_stats ||= diffstats.map { |diff| DiffSummary.new(diff, current_tree) }
+ @cached_file_stats ||= diffstats.map { |diff| DiffSummary.new(@repo, diff) }
  end
 
- LanguageSummary = Struct.new(:name, :additions, :deletions, :net)
+ LanguageSummary = Struct.new(:name, :additions, :deletions, :net, :added_files, :deleted_files, :modified_files)
 
  # Array of LanguageSummary objects (one for each language) for simple calculations
  def languages
  grouped_language_files.collect do |language, stats|
  additions = summarize(stats, :additions)
  deletions = summarize(stats, :deletions)
  net = summarize(stats, :net)
- LanguageSummary.new(language, additions, deletions, net)
+ LanguageSummary.new(language, additions, deletions, net, added_files, deleted_files, modified_files)
  end
  end
 
@@ -55,72 +63,13 @@ def grouped_language_files
  file_stats.group_by(&:language)
  end
 
- FileSummary = Struct.new(:name, :language, :additions, :deletions, :net, :filestatus)
-
- # Array of FileSummary objects (one for each file) for simple calculations
- def files
- file_stats.collect{ |stats| determine_file_summary(stats) }
- end
-
- def cached_show
- @cached_commit_show ||= show
- end
-
  # Files touched in this commit
- def file_names
- diffstats.map(&:filename)
- end
-
- # Fetch the current Grit::Repo tree from this commit
- def current_tree
- @current_tree ||= repo.tree(sha)
+ def filenames
+ file_stats.map(&:filename)
  end
 
  private
 
- def determine_file_summary(stats)
- filestatus = :modified
- language = stats.language
-
- # Determine if this file could be a new or deleted file
- if (stats.additions > 0 && stats.deletions == 0) || (stats.additions == 0 && stats.deletions > 0)
- # Extract file status from commit's diff object
- cached_show.each do |diff|
- if stats.filename == diff.b_path
- filestatus = :create if diff.new_file
- filestatus = :delete if diff.deleted_file
- break
- end
- end
- end
-
- # Determine language of blob
- if stats.tree?
- # Trees have no language (the tree's blobs are still processed via the remainder diffstats)
- language = "Unknown"
- elsif stats.submodule?
- language = "Submodule"
- elsif stats.blob.nil?
- # If blob is nil (i.e., deleted file) grab the previous version of this blob using the parents of the current commit
- blob = BlobFinder.get_blob(self.parents.first, stats.filename)
- blob = BlobFinder.get_blob(self.parents.last, stats.filename) if blob.nil?
-
- # Determine language of newly found blob
- if blob.kind_of? Grit::Tree
- language = "Unknown"
- elsif blob.kind_of? Grit::Submodule
- language = "Submodule"
- elsif blob.nil? || blob.language.nil?
- language = "Unknown"
- else
- language = blob.language.to_s
- end
- end
-
- # TODO Converts file summary into hash to keep json compatibility (for now)
- Hash[FileSummary.new(stats.filename, language, stats.additions, stats.deletions, stats.net, filestatus).each_pair.to_a]
- end
-
  def summarize(stats, what)
  stats.map(&what).inject(0, :+)
  end
@@ -130,19 +79,7 @@ def commit_summary(what)
  end
 
  def diffstats
- if merge?
- merge_diffstats
- else
- stats.to_diffstat
- end
- end
-
- # Hackery coming...
- DIFFSTAT_REGEX = /([-|\d]+)\s+([-|\d]+)\s+(.+)/i
- def merge_diffstats
- native_diff = repo.git.native(:diff, {numstat: true}, parents.join("..."))
- per_file_info = native_diff.scan(DIFFSTAT_REGEX)
- per_file_info.map { |add, del, file| Grit::DiffStat.new(file, add.to_i, del.to_i) }
+ @patches
  end
 
  end

lib/git_statistics/commit_summary.rb

@@ -117,8 +117,9 @@ def add_language_stats(data, file)
  data[:languages][file[:language].to_sym][:additions] += file[:additions]
  data[:languages][file[:language].to_sym][:deletions] += file[:deletions]
 
- if file[:filestatus] != nil
- data[:languages][file[:language].to_sym][file[:filestatus].to_sym] += 1
+ # Keep count of languages status (i.e., added, deleted) and keep keys consistent (i.e., added_files, deleted_files)
+ if file[:status] != nil
+ data[:languages][file[:language].to_sym][(file[:status]+'_files').to_sym] += 1
  end
 
  return data
@@ -130,8 +131,8 @@ def add_commit_stats(data, commit)
  data[:commits] += 1
  data[:additions] += commit[:additions]
  data[:deletions] += commit[:deletions]
- data[:create] += commit[:new_files] if commit[:new_files] > 0
- data[:delete] += commit[:removed_files] if commit[:removed_files] > 0
+ data[:added_files] += commit[:added_files] if !commit[:added_files].nil?
+ data[:deleted_files] += commit[:deleted_files] if !commit[:deleted_files].nil?
  return data
  end